fix fix_systemtest

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Stefan Allius <stefan.allius@t-online.de>

.env_example

@@ -6,4 +6,9 @@ PRIVAT_CONTAINER_REGISTRY=docker.io/<user>/
 
 # registry for official container (preview, rc, rel)
 PUBLIC_CONTAINER_REGISTRY=ghcr.io/<user>/
-PUBLIC_CR_KEY=
+PUBLIC_CR_KEY=  
+
+# define serial number of GEN3PLUS devices for systemtests
+# the serialnumber are coded as 4-byte hex-strings
+SOLARMAN_INV_SNR='00000000'
+SOLARMAN_DCU_SNR='00000000'

.github/workflows/python-app.yml

@@ -49,7 +49,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82 --ignore=F821 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 --exit-zero --ignore=C901,E121,E123,E126,E133,E226,E241,E242,E704,W503,W504,W505 --format=pylint --output-file=output_flake.txt --exclude=*.pyc app/src/
     - name: Test with pytest

README.md

@@ -8,7 +8,7 @@
 <p align="center">
     <a href="https://opensource.org/licenses/BSD-3-Clause"><img alt="License: BSD-3-Clause" src="https://img.shields.io/badge/License-BSD_3--Clause-green.svg"></a>
     <a href="https://www.python.org/downloads/release/python-3130/"><img alt="Supported Python versions" src="https://img.shields.io/badge/python-3.13-blue.svg"></a>
-    <a href="https://sbtinstruments.github.io/aiomqtt/introduction.html"><img alt="Supported aiomqtt versions" src="https://img.shields.io/badge/aiomqtt-2.3.0-lightblue.svg"></a>
+    <a href="https://aiomqtt.bo3hm.com/introduction.html"><img alt="Supported aiomqtt versions" src="https://img.shields.io/badge/aiomqtt-2.3.1-lightblue.svg"></a>
     <a href="https://libraries.io/pypi/aiocron"><img alt="Supported aiocron versions" src="https://img.shields.io/badge/aiocron-1.8-lightblue.svg"></a>
     <a href="https://toml.io/en/v1.0.0"><img alt="Supported toml versions" src="https://img.shields.io/badge/toml-1.0.0-lightblue.svg"></a>
     <br>
@@ -430,15 +430,15 @@ A combination with a red question mark should work, but I have not checked it in
   <tr><td>GEN3 micro inverters (quad MPPT):<br>MS3000</td><td align="center">✔️</td><td align="center">✔️</td><td align="center">✔️</td><td align="center">➖</td><td align="center">➖</td></tr>
   <tr><td>GEN3 PLUS micro inverters:<br>MS1600, MS1800, MS2000<br>MS2000-D, MS800</td><td align="center">➖</td><td align="center">➖</td><td align="center">➖</td><td align="center">✔️</td><td align="center">✔️</td></tr>
   <tr><td>GEN3 PLUS storage systems:<br>DC1000</td><td align="center">➖</td><td align="center">➖</td><td align="center">➖</td><td align="center">✔️</td><td align="center">✔️</td></tr>
-  <tr><td>GEN3 PLUS smart meter:<br>TSOL-MG3-MS</td><td align="center">➖</td><td align="center">➖</td><td align="center">➖</td><td align="center">❓</td><td align="center">❓</td></tr>
+  <tr><td>GEN3 PLUS smart meter:<br>TSOL-MG3-MS, DDZY422-D2</td><td align="center">➖</td><td align="center">➖</td><td align="center">➖</td><td align="center">❓</td><td align="center">❓</td></tr>
 </<tr><td>TITAN micro inverters:<br>TSOL-MP3000, MP2250</td><td align="center">❓</td><td align="center">❓</td><td align="center">❓</td><td align="center">❓</td><td align="center">❓</td></tr>
 </table>
 
 ```txt
 Legend
 ➖: Firmware not available for this devices
-✔️: proxy support testet
+✔️: Proxy support testet
-❓: proxy support possible but not testet
+❓: Proxy support unknown. There is an open port, but all known protocols do not work.
 🚧: Proxy support in preparation
 ```
 

app/requirements-test.txt

@@ -1,8 +1,8 @@
-    flake8==7.1.2
+    flake8==7.2.0
     pytest==8.3.5
     pytest-asyncio==0.26.0
-    pytest-cov==6.0.0
+    pytest-cov==6.1.0
     python-dotenv==1.1.0
     mock==5.2.0
-    coverage==7.7.1
+    coverage==7.8.0
     jinja2-cli==0.8.2

app/requirements.txt

@@ -1,4 +1,4 @@
-    aiomqtt==2.3.0
+    aiomqtt==2.3.1
     schema==0.7.7
     aiocron==2.1
-    aiohttp==3.11.14
+    aiohttp==3.11.16

app/tests/test_async_stream.py

@@ -84,7 +84,10 @@ async def test_close_cb():
         return 0.1
     def closed():
         nonlocal cnt
-        nonlocal ifc
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.close()  # clears the closed callback
         cnt += 1
     
@@ -113,7 +116,6 @@ async def test_close_cb():
 
 @pytest.mark.asyncio
 async def test_read():
-    global test
     assert asyncio.get_running_loop()
     reader = FakeReader()
     reader.test  = FakeReader.RD_TEST_13_BYTES
@@ -124,11 +126,13 @@ async def test_read():
         return 1
     def closed():
         nonlocal cnt
-        nonlocal ifc
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.close()  # clears the closed callback
         cnt += 1
     def app_read():
-        nonlocal ifc
         ifc.proc_start -= 3
         return 0.01  # async wait of 0.01 
     cnt = 0
@@ -151,7 +155,6 @@ async def test_read():
 
 @pytest.mark.asyncio
 async def test_write():
-    global test
     assert asyncio.get_running_loop()
     reader = FakeReader()
     reader.test  = FakeReader.RD_TEST_13_BYTES
@@ -162,11 +165,13 @@ async def test_write():
         return 1
     def closed():
         nonlocal cnt
-        nonlocal ifc
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.close()  # clears the closed callback
         cnt += 1
     def app_read():
-        nonlocal ifc
         ifc.proc_start -= 3
         return 0.01  # async wait of 0.01 
     
@@ -203,7 +208,6 @@ async def test_publ_mqtt_cb():
         return 0.1
     async def publ_mqtt():
         nonlocal cnt
-        nonlocal ifc
         cnt += 1
     
     cnt = 0
@@ -233,7 +237,10 @@ async def test_create_remote_cb():
         return 0.1
     async def create_remote():
         nonlocal cnt
-        nonlocal ifc
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.close()  # clears the closed callback
         cnt += 1
     
@@ -255,7 +262,6 @@ async def test_create_remote_cb():
 
 @pytest.mark.asyncio
 async def test_sw_exception():
-    global test
     assert asyncio.get_running_loop()
     reader = FakeReader()
     reader.test  = FakeReader.RD_TEST_SW_EXCEPT
@@ -266,7 +272,10 @@ async def test_sw_exception():
         return 1
     def closed():
         nonlocal cnt
-        nonlocal ifc
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.close()  # clears the closed callback
         cnt += 1
     cnt = 0
@@ -285,7 +294,6 @@ async def test_sw_exception():
 
 @pytest.mark.asyncio
 async def test_os_error():
-    global test
     assert asyncio.get_running_loop()
     reader = FakeReader()
     reader.test  = FakeReader.RD_TEST_OS_ERROR
@@ -293,12 +301,11 @@ async def test_os_error():
     reader.on_recv.set()
     writer =  FakeWriter()
     cnt = 0
+
     def timeout():
         return 1
     def closed():
         nonlocal cnt
-        nonlocal ifc
-        ifc.close()  # clears the closed callback
         cnt += 1
     cnt = 0
     ifc =  AsyncStreamClient(reader, writer, None, closed)
@@ -361,10 +368,13 @@ async def test_forward():
     assert asyncio.get_running_loop()
     remote = StreamPtr(None)
     cnt = 0
-
     async def _create_remote():
-        nonlocal cnt, remote, ifc
+        nonlocal cnt
         create_remote(remote, TestType.FWD_NO_EXCPT)
+        # The callback will be called after the AsyncStreamServer
+        # constructer has finished and so ifc must be defined in the
+        # upper scope
+        assert "ifc" in locals()
         ifc.fwd_add(b'test-forward_msg2 ')
         cnt += 1
     
@@ -382,7 +392,7 @@ async def test_forward_with_conn():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote, ifc
+        nonlocal cnt
         cnt += 1
     
     cnt = 0
@@ -417,7 +427,7 @@ async def test_forward_sw_except():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_SW_EXCPT)
         cnt += 1
     
@@ -435,7 +445,7 @@ async def test_forward_os_error():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_OS_ERROR)
         cnt += 1
     
@@ -453,7 +463,7 @@ async def test_forward_os_error2():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_OS_ERROR, True)
         cnt += 1
     
@@ -471,7 +481,7 @@ async def test_forward_os_error3():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_OS_ERROR_NO_STREAM)
         cnt += 1
     
@@ -489,7 +499,7 @@ async def test_forward_runtime_error():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_RUNTIME_ERROR)
         cnt += 1
     
@@ -507,7 +517,7 @@ async def test_forward_runtime_error2():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_RUNTIME_ERROR, True)
         cnt += 1
     
@@ -525,7 +535,7 @@ async def test_forward_runtime_error3():
     cnt = 0
 
     async def _create_remote():
-        nonlocal cnt, remote
+        nonlocal cnt
         create_remote(remote, TestType.FWD_RUNTIME_ERROR_NO_STREAM, True)
         cnt += 1
     
@@ -543,7 +553,7 @@ async def test_forward_resp():
     cnt = 0
 
     def _close_cb():
-        nonlocal cnt, remote, ifc
+        nonlocal cnt
         cnt += 1
     
     cnt = 0
@@ -559,9 +569,8 @@ async def test_forward_resp2():
     assert asyncio.get_running_loop()
     remote = StreamPtr(None)
     cnt = 0
-
     def _close_cb():
-        nonlocal cnt, remote, ifc
+        nonlocal cnt
         cnt += 1
     
     cnt = 0
@@ -571,3 +580,4 @@ async def test_forward_resp2():
     await ifc.client_loop('')
     assert cnt == 1
     del ifc
+ 

app/tests/test_inverter_base.py

@@ -85,7 +85,6 @@ def patch_open_connection():
         return FakeReader(), FakeWriter()
     
     def new_open(host: str, port: int):
-        global test
         if test == MockType.RD_TEST_TIMEOUT:
             raise ConnectionRefusedError
         elif test == MockType.RD_TEST_EXCEPT:
@@ -318,7 +317,7 @@ async def test_remote_conn_to_loopback(config_conn, patch_open_connection):
     assert cnt == 0
 
 @pytest.mark.asyncio
-async def test_remote_conn_to_None(config_conn, patch_open_connection):
+async def test_remote_conn_to_none(config_conn, patch_open_connection):
     '''check if get_extra_info() return None in case of an error'''
     _ = config_conn
     _ = patch_open_connection

app/tests/test_inverter_g3.py

@@ -85,7 +85,6 @@ def patch_open_connection():
         return FakeReader(), FakeWriter()
     
     def new_open(host: str, port: int):
-        global test
         if test == MockType.RD_TEST_TIMEOUT:
             raise ConnectionRefusedError
         elif test == MockType.RD_TEST_EXCEPT:

app/tests/test_inverter_g3p.py

@@ -84,7 +84,6 @@ def patch_open_connection():
         return FakeReader(), FakeWriter()
     
     def new_open(host: str, port: int):
-        global test
         if test == MockType.RD_TEST_TIMEOUT:
             raise ConnectionRefusedError
         elif test == MockType.RD_TEST_EXCEPT:

app/tests/test_mqtt.py

@@ -96,7 +96,6 @@ def test_native_client(test_hostname, test_port):
 
 @pytest.mark.asyncio
 async def test_mqtt_connection(config_mqtt_conn):
-    global NO_MOSQUITTO_TEST
     if NO_MOSQUITTO_TEST:
         pytest.skip('skipping, since Mosquitto is not reliable at the moment')
 
@@ -122,7 +121,6 @@ async def test_mqtt_connection(config_mqtt_conn):
 
 @pytest.mark.asyncio
 async def test_ha_reconnect(config_mqtt_conn):
-    global NO_MOSQUITTO_TEST
     if NO_MOSQUITTO_TEST:
         pytest.skip('skipping, since Mosquitto is not reliable at the moment')
 

ha_addons/ha_addon/Dockerfile

@@ -13,7 +13,7 @@
 # 1 Build Base Image #
 ######################
 
-ARG BUILD_FROM="ghcr.io/hassio-addons/base:17.2.2"
+ARG BUILD_FROM="ghcr.io/hassio-addons/base:17.2.3"
 # hadolint ignore=DL3006
 FROM $BUILD_FROM AS base
 

system_tests/test_tcp_socket_v2.py

@@ -5,13 +5,14 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-SOLARMAN_SNR = os.getenv('SOLARMAN_SNR', '00000080')
+SOLARMAN_INV_SNR = os.getenv('SOLARMAN_INV_SNR', '00000080')
+SOLARMAN_DCU_SNR = os.getenv('SOLARMAN_INV_SNR', '00000080')
 
 def get_sn() -> bytes:
-    return bytes.fromhex(SOLARMAN_SNR)
+    return bytes.fromhex(SOLARMAN_INV_SNR)
 
 def get_dcu_sn() -> bytes:
-    return b'\x20\x43\x65\x7b'
+    return bytes.fromhex(SOLARMAN_DCU_SNR)
 
 def get_dcu_no() -> bytes:
     return b'4100000000000001'
@@ -27,7 +28,7 @@ def correct_checksum(buf):
     return checksum.to_bytes(length=1)
 
 @pytest.fixture
-def MsgContactInfo(): # Contact Info message
+def msg_contact_info(): # Contact Info message
     msg  = b'\xa5\xd4\x00\x10\x41\x00\x01' +get_sn()  +b'\x02\xba\xd2\x00\x00'
     msg += b'\x19\x00\x00\x00\x00\x00\x00\x00\x05\x3c\x78\x01\x64\x01\x4c\x53'
     msg += b'\x57\x35\x42\x4c\x45\x5f\x31\x37\x5f\x30\x32\x42\x30\x5f\x31\x2e'
@@ -46,13 +47,13 @@ def MsgContactInfo(): # Contact Info message
     return msg
 
 @pytest.fixture
-def MsgContactResp(): # Contact Response message
+def msg_contact_resp(): # Contact Response message
     msg  = b'\xa5\x0a\x00\x10\x11\x01\x01' +get_sn()  +b'\x02\x01\x6a\xfd\x8f'
     msg += b'\x65\x3c\x00\x00\x00\x75\x15'
     return msg
 
 @pytest.fixture
-def MsgDataInd(): 
+def msg_data_ind(): 
     msg  = b'\xa5\x99\x01\x10\x42\x59\x84' +get_sn()  +b'\x01\xb0\x02\x2c\x87'
     msg += b'\x22\x32\xb7\x29\x00\x00\xd6\xcf\xe1\x33\x01\x00\x0c\x05\x00\x00'
     msg += b'\x59\x31\x37\x45\x37\x41\x30\x46\x30\x31\x30\x42\x30\x31\x33\x45'
@@ -86,14 +87,14 @@ def MsgDataInd():
     return msg
 
 @pytest.fixture
-def MsgDataResp(): # Contact Response message
+def msg_data_rsp(): # Contact Response message
     msg  = b'\xa5\x0a\x00\x10\x12\x80\x84' +get_sn()  +b'\x01\x01\xd1\x96\x04'
     msg += b'\x66\x3c\x00\x00\x00\xed\x15'
     return msg
 
 
 @pytest.fixture
-def MsgInvalidInfo(): # Contact Info message wrong start byte
+def msg_invalid_info(): # Contact Info message wrong start byte
     msg  = b'\x47\xd4\x00\x10\x41\x00\x01' +get_sn()  +b'\x02\xba\xd2\x00\x00'
     msg += b'\x19\x00\x00\x00\x00\x00\x00\x00\x05\x3c\x78\x01\x64\x01\x4c\x53'
     msg += b'\x57\x35\x42\x4c\x45\x5f\x31\x37\x5f\x30\x32\x42\x30\x5f\x31\x2e'
@@ -169,7 +170,7 @@ def dcu_data_rsp_msg():  # 0x1210
     return msg
 
 @pytest.fixture(scope="session")
-def ClientConnection():
+def client_connection():
     host = 'logger.talent-monitoring.com'
     port = 10000
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
@@ -178,15 +179,15 @@ def ClientConnection():
         yield s
         s.close()
 
-def checkResponse(data, Msg):
+def check_response(data, msg):
     check = bytearray(data)
-    check[5]= Msg[5]            # ignore seq
+    check[5]= msg[5]            # ignore seq
-    check[13:18]= Msg[13:18]    # ignore timestamp + first byte of repeat time
+    check[13:18]= msg[13:18]    # ignore timestamp + first byte of repeat time
-    check[21]= Msg[21]          # ignore crc
+    check[21]= msg[21]          # ignore crc
-    assert check == Msg
+    assert check == msg
 
 
-def tempClientConnection():
+def tempclient_connection():
     host = 'logger.talent-monitoring.com'
     port = 10000
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
@@ -198,53 +199,53 @@ def tempClientConnection():
 
 def test_open_close():
     try:
-        for _ in tempClientConnection():
+        for _ in tempclient_connection():
-            pass  # test generator tempClientConnection()
+            pass  # test generator tempclient_connection()
-    except:
+    except TimeoutError:
         assert False
 
-def test_conn_msg(ClientConnection,MsgContactInfo, MsgContactResp):
+def test_conn_msg(client_connection,msg_contact_info, msg_contact_resp):
-    s = ClientConnection
+    s = client_connection
     try:
-        s.sendall(MsgContactInfo)
+        s.sendall(msg_contact_info)
-        # time.sleep(2.5)
+        time.sleep(2.5)
         data = s.recv(1024)
     except TimeoutError:
         pass
     # time.sleep(2.5)
-    checkResponse(data, MsgContactResp)
+    check_response(data, msg_contact_resp)
 
-def test_data_ind(ClientConnection,MsgDataInd, MsgDataResp):
+def test_data_ind(client_connection,msg_data_ind, msg_data_rsp):
-    s = ClientConnection
+    s = client_connection
     try:
-        s.sendall(MsgDataInd)
+        s.sendall(msg_data_ind)
         # time.sleep(2.5)
         data = s.recv(1024)
     except TimeoutError:
         pass
     # time.sleep(2.5)
-    checkResponse(data, MsgDataResp)
+    check_response(data, msg_data_rsp)
 
-def test_inavlid_msg(ClientConnection,MsgInvalidInfo,MsgContactInfo, MsgContactResp):
+def test_inavlid_msg(client_connection,msg_invalid_info,msg_contact_info, msg_contact_resp):
-    s = ClientConnection
+    s = client_connection
     try:
-        s.sendall(MsgInvalidInfo)
+        s.sendall(msg_invalid_info)
         # time.sleep(2.5)
         data = s.recv(1024)
     except TimeoutError:
         pass
     # time.sleep(2.5)
     try:
-        s.sendall(MsgContactInfo)
+        s.sendall(msg_contact_info)
         # time.sleep(2.5)
         data = s.recv(1024)
     except TimeoutError:
         pass
     # time.sleep(2.5)
-    checkResponse(data, MsgContactResp)
+    check_response(data, msg_contact_resp)
 
-def test_dcu_dev(ClientConnection,dcu_dev_ind_msg, dcu_dev_rsp_msg):
+def test_dcu_dev(client_connection,dcu_dev_ind_msg, dcu_dev_rsp_msg):
-    s = ClientConnection
+    s = client_connection
     try:
         s.sendall(dcu_dev_ind_msg)
         # time.sleep(2.5)
@@ -252,10 +253,10 @@ def test_dcu_dev(ClientConnection,dcu_dev_ind_msg, dcu_dev_rsp_msg):
     except TimeoutError:
         pass
     # time.sleep(2.5)
-    checkResponse(data, dcu_dev_rsp_msg)
+    check_response(data, dcu_dev_rsp_msg)
 
-def test_dcu_ind(ClientConnection,dcu_data_ind_msg, dcu_data_rsp_msg):
+def test_dcu_ind(client_connection,dcu_data_ind_msg, dcu_data_rsp_msg):
-    s = ClientConnection
+    s = client_connection
     try:
         s.sendall(dcu_data_ind_msg)
         # time.sleep(2.5)
@@ -263,4 +264,4 @@ def test_dcu_ind(ClientConnection,dcu_data_ind_msg, dcu_data_rsp_msg):
     except TimeoutError:
         pass
     # time.sleep(2.5)
-    checkResponse(data, dcu_data_rsp_msg)
+    check_response(data, dcu_data_rsp_msg)