improve test case

* S allius/update python (#430)

* add-on: bump python to version 3.12.10-r1 (#429)

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+- add-on: bump python to version 3.12.10-r1
 - set no of pv modules for MS800 GEN3PLUS inverters
 - fix the paths to copy the config.example.toml file during proxy start
 - add MQTT topic `dcu_power` for setting output power on DCUs

app/requirements-test.txt

@@ -4,5 +4,5 @@
     pytest-cov==6.1.1
     python-dotenv==1.1.0
     mock==5.2.0
-    coverage==7.8.1
+    coverage==7.8.2
     jinja2-cli==0.8.2

app/src/scheduler.py

@@ -12,7 +12,7 @@ class Schedule:
     count = 0
 
     @classmethod
-    def start(cls) -> None:
+    def start(cls) -> None:    # pragma: no cover
         '''Start the scheduler and schedule the tasks (cron jobs)'''
         logging.debug("Scheduler init")
         cls.mqtt = Mqtt(None)
@@ -20,7 +20,7 @@ class Schedule:
         crontab('0 0 * * *', func=cls.atmidnight, start=True)
 
     @classmethod
-    async def atmidnight(cls) -> None:
+    async def atmidnight(cls) -> None:    # pragma: no cover
         '''Clear daily counters at midnight'''
         logging.info("Clear daily counters at midnight")
 

app/tests/test_mqtt.py

@@ -3,7 +3,8 @@ import pytest
 import asyncio
 import aiomqtt
 import logging
-from aiomqtt import MqttError
+from aiomqtt import MqttError, MessagesIterator
+from aiomqtt import Message as AiomqttMessage
 from mock import patch, Mock
 
 from async_stream import AsyncIfcImpl
@@ -34,6 +35,26 @@ def test_hostname():
     # else:
         return 'test.mosquitto.org'
 
+@pytest.fixture(scope="function")
+def aiomqtt_mock(monkeypatch):
+    recv_que = asyncio.Queue()
+        
+    async def my_aenter(self):
+        return self
+    async def my_subscribe(self, *arg):
+        return
+    async def my_anext(self):
+        return await recv_que.get()
+    async def my_receive(self, topic: str, payload: bytes):
+        msg = AiomqttMessage(topic, payload,qos=0, retain=False, mid=0, properties=None)
+        await recv_que.put(msg)
+        await asyncio.sleep(0)  # dispath the msg
+
+    monkeypatch.setattr(aiomqtt.Client, "__aenter__", my_aenter)
+    monkeypatch.setattr(aiomqtt.Client, "subscribe", my_subscribe)
+    monkeypatch.setattr(MessagesIterator, "__anext__", my_anext)
+    monkeypatch.setattr(Mqtt, "receive", my_receive, False)
+
 @pytest.fixture
 def config_mqtt_conn(test_hostname, test_port):
     Config.act_config = {'mqtt':{'host': test_hostname, 'port': test_port, 'user': '', 'passwd': ''},
@@ -161,13 +182,17 @@ async def test_ha_reconnect(config_mqtt_conn):
         await m.close()
 
 @pytest.mark.asyncio
-async def test_mqtt_no_config(config_no_conn):
+async def test_mqtt_no_config(config_no_conn, monkeypatch):
     _ = config_no_conn
     assert asyncio.get_running_loop()
 
     on_connect =  asyncio.Event()
     async def cb():
         on_connect.set()
+    async def my_publish(*args):
+        return
+
+    monkeypatch.setattr(aiomqtt.Client, "publish", my_publish)
 
     try:
         m = Mqtt(cb)
@@ -176,9 +201,9 @@ async def test_mqtt_no_config(config_no_conn):
         assert not on_connect.is_set()
         try:
             await m.publish('homeassistant/status', 'online')
-            assert False
+            assert m.published == 1
         except Exception:
-            pass          
+            assert False          
     except TimeoutError:
         assert False
     finally:
@@ -250,92 +275,119 @@ async def test_mqtt_except_def_config(config_def_conn, monkeypatch, caplog):
     assert 'MQTT is unconfigured; Check your config.toml!' in caplog.text
 
 @pytest.mark.asyncio
-async def test_msg_dispatch(config_mqtt_conn, spy_modbus_cmd):
+async def test_mqtt_dispatch(config_mqtt_conn, aiomqtt_mock, spy_modbus_cmd):
     _ = config_mqtt_conn
+    _ = aiomqtt_mock
     spy = spy_modbus_cmd
     try:
         m = Mqtt(None)
-        msg = aiomqtt.Message(topic= 'homeassistant/status', payload= b'online', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        assert m.ha_restarts == 0
+        await m.receive('homeassistant/status', b'online')  # send the message
         assert m.ha_restarts == 1
 
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/rated_load', payload= b'2', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/rated_load', payload= b'2')
         spy.assert_awaited_once_with(Modbus.WRITE_SINGLE_REG, 0x2008, 2, logging.INFO)
-        
+
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/out_coeff', payload= b'100', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/out_coeff', payload= b'100')
         spy.assert_awaited_once_with(Modbus.WRITE_SINGLE_REG, 0x202c, 1024, logging.INFO)
         
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/out_coeff', payload= b'50', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/out_coeff', payload= b'50')
         spy.assert_awaited_once_with(Modbus.WRITE_SINGLE_REG, 0x202c, 512, logging.INFO)
     
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/modbus_read_regs', payload= b'0x3000, 10', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/modbus_read_regs', payload= b'0x3000, 10')
         spy.assert_awaited_once_with(Modbus.READ_REGS, 0x3000, 10, logging.INFO)
 
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10')
         spy.assert_awaited_once_with(Modbus.READ_INPUTS, 0x3000, 10, logging.INFO)
 
         # test dispatching with empty mapping table
         m.topic_defs.clear()
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10')
         spy.assert_not_called()
 
         # test dispatching with incomplete mapping table - invalid fnc defined
         m.topic_defs.append(
             {'prefix': 'entity_prefix', 'topic': '/+/modbus_read_inputs',
-             'full_topic': 'tsun/+/modbus_read_inputs', 'fnc': 'invalid'}
+             'full_topic': 'tsun/+/modbus_read_inputs', 'fnc': 'addr'}
         )
         spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/modbus_read_inputs', payload= b'0x3000, 10')
         spy.assert_not_called()
 
+    except MqttError:
+        assert False
+    except Exception:
+        assert False          
     finally:
         await m.close()
 
 @pytest.mark.asyncio
-async def test_msg_dispatch_err(config_mqtt_conn, spy_modbus_cmd):
+async def test_mqtt_dispatch_cb(config_mqtt_conn, aiomqtt_mock):
     _ = config_mqtt_conn
+    _ = aiomqtt_mock
+
+    on_connect =  asyncio.Event()
+    async def cb():
+        on_connect.set()
+    try:
+        m = Mqtt(cb)
+        assert m.ha_restarts == 0
+        await m.receive('homeassistant/status', b'online')  # send the message
+        assert on_connect.is_set()
+        assert m.ha_restarts == 1
+
+    except MqttError:
+        assert False
+    except Exception:
+        assert False          
+    finally:
+        await m.close()
+
+@pytest.mark.asyncio
+async def test_mqtt_dispatch_err(config_mqtt_conn, aiomqtt_mock, spy_modbus_cmd, caplog):
+    _ = config_mqtt_conn
+    _ = aiomqtt_mock
     spy = spy_modbus_cmd
+
+    LOGGER = logging.getLogger("mqtt")
+    LOGGER.propagate = True
+    LOGGER.setLevel(logging.INFO)
+
     try:
         m = Mqtt(None)
+
         # test out of range param
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/out_coeff', payload= b'-1', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/out_coeff', payload= b'-1')
         spy.assert_not_called()
 
         # test unknown node_id
-        spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_2/out_coeff', payload= b'2', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_2/out_coeff', payload= b'2')
         spy.assert_not_called()
 
         # test invalid fload param
-        spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/out_coeff', payload= b'2, 3', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/out_coeff', payload= b'2, 3')
+        spy.assert_not_called()
+    
+        await m.receive(topic= 'tsun/inv_1/modbus_read_regs', payload= b'0x3000, 10, 7')
         spy.assert_not_called()
 
-        spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/modbus_read_regs', payload= b'0x3000, 10, 7', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
-        spy.assert_not_called()
-
-        spy.reset_mock()
-        msg = aiomqtt.Message(topic= 'tsun/inv_1/dcu_power', payload= b'100W', qos= 0, retain = False, mid= 0, properties= None)
-        await m.dispatch_msg(msg)
+        await m.receive(topic= 'tsun/inv_1/dcu_power', payload= b'100W')
         spy.assert_not_called()
 
+        with caplog.at_level(logging.INFO):
+            msg = aiomqtt.Message(topic= 'tsun/inv_1/out_coeff', payload= b'2', qos= 0, retain = False, mid= 0, properties= None)
+            for _ in m.each_inverter(msg, "addr"):
+                pass  # do nothing here
+        assert 'Cmd not supported by: inv_1/' in caplog.text
+    except MqttError:
+        assert False
+    except Exception:
+        assert False          
     finally:
         await m.close()
 

app/tests/test_server.py

@@ -4,6 +4,10 @@ import logging
 import os
 from mock import patch
 from server import app, Server, ProxyState, HypercornLogHndl
+from inverter_base import InverterBase
+from gen3.talent import Talent
+
+from test_inverter_base import FakeReader, FakeWriter
 
 pytest_plugins = ('pytest_asyncio',)
 
@@ -108,20 +112,20 @@ class TestServerClass:
         assert logging.getLogger('hypercorn.access').level == logging.INFO
         assert logging.getLogger('hypercorn.error').level == logging.INFO
 
-        os.environ["LOG_LVL"] = "WARN"
-        s.parse_args(['--log_backups', '3'])
-        s.init_logging_system()
-        assert s.log_backups == 3
-        assert s.log_level == logging.WARNING
-        assert logging.handlers.log_backups == 3
-        assert logging.getLogger().level == s.log_level
-        assert logging.getLogger('msg').level == s.log_level
-        assert logging.getLogger('conn').level == s.log_level
-        assert logging.getLogger('data').level == s.log_level
-        assert logging.getLogger('tracer').level == s.log_level
-        assert logging.getLogger('asyncio').level == s.log_level
-        assert logging.getLogger('hypercorn.access').level == logging.INFO
-        assert logging.getLogger('hypercorn.error').level == logging.INFO
+        with patch.dict(os.environ, {'LOG_LVL': 'WARN'}):
+            s.parse_args(['--log_backups', '3'])
+            s.init_logging_system()
+            assert s.log_backups == 3
+            assert s.log_level == logging.WARNING
+            assert logging.handlers.log_backups == 3
+            assert logging.getLogger().level == s.log_level
+            assert logging.getLogger('msg').level == s.log_level
+            assert logging.getLogger('conn').level == s.log_level
+            assert logging.getLogger('data').level == s.log_level
+            assert logging.getLogger('tracer').level == s.log_level
+            assert logging.getLogger('asyncio').level == s.log_level
+            assert logging.getLogger('hypercorn.access').level == logging.INFO
+            assert logging.getLogger('hypercorn.error').level == logging.INFO
 
     def test_build_config_error(self, caplog):
         s = self.FakeServer()
@@ -202,17 +206,81 @@ class TestApp:
     @pytest.mark.asyncio
     async def test_healthy(self):
         """Test the healthy route."""
+        reader = FakeReader()
+        writer =  FakeWriter()
 
-        ProxyState.set_up(False)
-        client = app.test_client()
-        response = await client.get('/-/healthy')
-        assert response.status_code == 200
-        result = await response.get_data()
-        assert result == b"I'm fine"
+        with InverterBase(reader, writer, 'tsun', Talent):
+            ProxyState.set_up(False)
+            client = app.test_client()
+            response = await client.get('/-/healthy')
+            assert response.status_code == 200
+            result = await response.get_data()
+            assert result == b"I'm fine"
 
-        ProxyState.set_up(True)
-        response = await client.get('/-/healthy')
-        assert response.status_code == 200
-        result = await response.get_data()
-        assert result == b"I'm fine"
+            ProxyState.set_up(True)
+            response = await client.get('/-/healthy')
+            assert response.status_code == 200
+            result = await response.get_data()
+            assert result == b"I'm fine"
 
+    @pytest.mark.asyncio
+    async def test_unhealthy(self, monkeypatch, caplog):
+        """Test the healthy route."""
+        def result_false(self):
+            return False
+        
+        LOGGER = logging.getLogger("mqtt")
+        LOGGER.propagate = True
+        LOGGER.setLevel(logging.INFO)
+
+        monkeypatch.setattr(InverterBase, "healthy", result_false)
+        InverterBase._registry.clear()
+        reader = FakeReader()
+        writer = FakeWriter()
+
+        with caplog.at_level(logging.INFO) and InverterBase(reader, writer, 'tsun', Talent):
+            ProxyState.set_up(False)
+            client = app.test_client()
+            response = await client.get('/-/healthy')
+            assert response.status_code == 200
+            result = await response.get_data()
+            assert result == b"I'm fine"
+            assert "" == caplog.text
+
+            ProxyState.set_up(True)
+            response = await client.get('/-/healthy')
+            assert response.status_code == 503
+            result = await response.get_data()
+            assert result == b"I have a problem"
+            assert "" == caplog.text
+
+    @pytest.mark.asyncio
+    async def test_healthy_exception(self, monkeypatch, caplog):
+        """Test the healthy route."""
+        def result_except(self):
+            raise ValueError
+        
+        LOGGER = logging.getLogger("mqtt")
+        LOGGER.propagate = True
+        LOGGER.setLevel(logging.INFO)
+
+        monkeypatch.setattr(InverterBase, "healthy", result_except)
+        InverterBase._registry.clear()
+        reader = FakeReader()
+        writer = FakeWriter()
+
+        with caplog.at_level(logging.INFO) and InverterBase(reader, writer, 'tsun', Talent):
+            ProxyState.set_up(False)
+            client = app.test_client()
+            response = await client.get('/-/healthy')
+            assert response.status_code == 200
+            result = await response.get_data()
+            assert result == b"I'm fine"
+            assert "" == caplog.text
+
+            ProxyState.set_up(True)
+            response = await client.get('/-/healthy')
+            assert response.status_code == 200
+            result = await response.get_data()
+            assert result == b"I'm fine"
+            assert "Exception:" in caplog.text

app/tests/test_solarman.py

@@ -709,6 +709,19 @@ def msg_modbus_rsp():  # 0x1510
     msg += b'\x15'
     return msg
 
+@pytest.fixture
+def msg_modbus_rsp_mb_4():  # 0x1510, MODBUS Type:4
+    msg  = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn()  +b'\x02\x01'
+    msg += total()  
+    msg += hb()
+    msg += b'\x0a\xe2\xfa\x33\x01\x04\x28\x40\x10\x08\xd8'
+    msg += b'\x00\x00\x13\x87\x00\x31\x00\x68\x02\x58\x00\x00\x01\x53\x00\x02'
+    msg += b'\x00\x00\x01\x52\x00\x02\x00\x00\x01\x53\x00\x03\x00\x00\x00\x04'
+    msg += b'\x00\x01\x00\x00\x9e\xa4'
+    msg += correct_checksum(msg)
+    msg += b'\x15'
+    return msg
+
 @pytest.fixture
 def msg_modbus_interim_rsp():  # 0x0510
     msg  = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn()  +b'\x02\x01'
@@ -2241,6 +2254,61 @@ async def test_modbus_scaning(config_tsun_scan, heartbeat_ind_msg, heartbeat_rsp
     assert next(m.mb_timer.exp_count) == 3
     m.close()
 
+@pytest.mark.asyncio
+async def test_modbus_scaning_inv_rsp(config_tsun_scan, heartbeat_ind_msg, heartbeat_rsp_msg, msg_modbus_rsp_mb_4):
+    _ = config_tsun_scan
+    assert asyncio.get_running_loop()
+
+    m = MemoryStream(heartbeat_ind_msg, (0x15,0x56,0))
+    m.append_msg(msg_modbus_rsp_mb_4)
+    assert m.mb_scan == False
+    assert asyncio.get_running_loop() == m.mb_timer.loop
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    assert m.mb_timer.tim == None
+    m.read()         # read complete msg, and dispatch msg
+    assert m.mb_scan == True
+    assert m.mb_start_reg == 0xff80
+    assert m.mb_step == 0x40
+    assert m.mb_bytes == 0x14
+    assert asyncio.get_running_loop() == m.mb_timer.loop
+ 
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.snr == 2070233889
+    assert m.control == 0x4710
+     
+    assert m.msg_recvd[0]['control']==0x4710
+    assert m.msg_recvd[0]['seq']=='84:11'
+    assert m.msg_recvd[0]['data_len']==0x1
+
+    assert m.ifc.tx_fifo.get()==heartbeat_rsp_msg
+    assert m.ifc.fwd_fifo.get()==heartbeat_ind_msg
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+
+    m.ifc.tx_clear() # clear send buffer for next test
+    assert isclose(m.mb_timeout, 0.5)
+    assert next(m.mb_timer.exp_count) == 0
+    
+    await asyncio.sleep(0.5)
+    assert m.sent_pdu==b'\xa5\x17\x00\x10E\x12\x84!Ce{\x02\xb0\x02\x00\x00\x00\x00\x00\x00' \
+                       b'\x00\x00\x00\x00\x00\x00\x01\x03\xff\xc0\x00\x14\x75\xed\x33\x15'
+    assert m.ifc.tx_fifo.get()==b''
+
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 2
+    assert m.msg_recvd[1]['control']==0x1510
+    assert m.msg_recvd[1]['seq']=='03:03'
+    assert m.msg_recvd[1]['data_len']==0x3b
+    assert m.mb.last_addr == 1
+    assert m.mb.last_fcode == 3   
+    assert m.mb.last_reg == 0xffc0   # mb_start_reg + mb_step
+    assert m.mb.last_len == 20
+    assert m.mb.err == 3
+
+    assert next(m.mb_timer.exp_count) == 2
+    m.close()
+
 @pytest.mark.asyncio
 async def test_start_client_mode(my_loop, config_tsun_inv1, str_test_ip):
     _ = config_tsun_inv1

ha_addons/Makefile

@@ -192,7 +192,7 @@ $(repro_all_subdirs) :
 	mkdir -p $@
 
 $(repro_all_templates) : $(INST_BASE)/ha_addon_%/config.yaml: $(TEMPL)/config.jinja $(TEMPL)/%_data.json $(SRC)/.version FORCE
-	$(JINJA) --strict -D AppVersion=$(VERSION)-$* -D BuildID=$(BUILD_ID) $< $(filter %.json,$^) -o $@
+	$(JINJA) --strict -D AppVersion=$(VERSION)-$*$(RC) -D BuildID=$(BUILD_ID) $< $(filter %.json,$^) -o $@
 
 $(repro_all_apparmor) : $(INST_BASE)/ha_addon_%/apparmor.txt: $(TEMPL)/apparmor.jinja $(TEMPL)/%_data.json
 	$(JINJA) --strict $< $(filter %.json,$^) -o $@

ha_addons/ha_addon/Dockerfile

@@ -18,7 +18,7 @@ ARG BUILD_FROM="ghcr.io/hassio-addons/base:17.2.5"
 FROM $BUILD_FROM AS base
 
 # Installiere Python, pip und virtuelle Umgebungstools
-RUN apk add --no-cache python3=3.12.10-r0 py3-pip=24.3.1-r0 && \
+RUN apk add --no-cache python3=3.12.10-r1 py3-pip=24.3.1-r0 && \
     python -m venv /opt/venv && \
     . /opt/venv/bin/activate
 

ha_addons/templates/rc_data.json

@@ -2,7 +2,6 @@
 {
     "name": "TSUN-Proxy (Release Candidate)",
     "description": "MQTT Proxy for TSUN Photovoltaic Inverters",
-    "version": "rc",
     "image": "ghcr.io/s-allius/tsun-gen3-addon",
     "slug": "tsun-proxy-rc",
     "advanced": true,