From 6eebd0c852b03a2ee3855dbc49dec636e78bccdc Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 29 Apr 2024 22:48:41 +0200 Subject: [PATCH 01/18] make timestamp handling stateless --- app/src/gen3/talent.py | 56 +++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 46302ac..b94c331 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -40,6 +40,7 @@ class Talent(Message): self.id_str = id_str self.contact_name = b'' self.contact_mail = b'' + self.ts_offset = 0 self.db = InfosG3() self.switch = { 0x00: self.msg_contact_info, @@ -156,6 +157,24 @@ class Talent(Message): ts = (datetime.now() - datetime(1970, 1, 1)).total_seconds() return round(ts*1000) + def _update_header(self, _forward_buffer): + '''update header for message before forwarding, + add time offset to timestamp''' + _len = len(_forward_buffer) + result = struct.unpack_from('!lB', _forward_buffer, 0) + id_len = result[1] # len of variable id string + if _len < 2*id_len + 21: + return + + result = struct.unpack_from('!B', _forward_buffer, id_len+6) + msg_code = result[0] + if msg_code == 0x71 or msg_code == 0x04: + result = struct.unpack_from('!q', _forward_buffer, 13+2*id_len) + ts = result[0] + self.ts_offset + logger.debug(f'offset: {self.ts_offset:08x}' + f' proxy-time: {ts:08x}') + struct.pack_into('!q', _forward_buffer, 13+2*id_len, ts) + # check if there is a complete header in the buffer, parse it # and set # self.header_len @@ -256,38 +275,35 @@ class Talent(Message): def msg_get_time(self): tsun = Config.get('tsun') - if tsun['enabled']: - if self.ctrl.is_ind(): + if self.ctrl.is_ind(): + if self.data_len == 0: + ts = self._timestamp() + logger.debug(f'time: {ts:08x}') + self.__build_header(0x91) + self._send_buffer += struct.pack('!q', ts) + self.__finish_send_msg() + if tsun['enabled']: + if self.data_len == 0: + self.forward(self._recv_buffer, self.header_len + + self.data_len) if self.data_len >= 8: ts = self._timestamp() result = struct.unpack_from('!q', self._recv_buffer, self.header_len) + self.ts_offset = result[0]-ts logger.debug(f'tsun-time: {result[0]:08x}' - f' proxy-time: {ts:08x}') - else: - logger.warning('Unknown Ctrl') - self.inc_counter('Unknown_Ctrl') - self.forward(self._recv_buffer, self.header_len+self.data_len) + f' proxy-time: {ts:08x}', + f' offset: {self.ts_offset}') else: - if self.ctrl.is_ind(): - if self.data_len == 0: - ts = self._timestamp() - logger.debug(f'time: {ts:08x}') - - self.__build_header(0x91) - self._send_buffer += struct.pack('!q', ts) - self.__finish_send_msg() - - else: - logger.warning('Unknown Ctrl') - self.inc_counter('Unknown_Ctrl') + logger.warning('Unknown Ctrl') + self.inc_counter('Unknown_Ctrl') def parse_msg_header(self): result = struct.unpack_from('!lB', self._recv_buffer, self.header_len) data_id = result[0] # len of complete message id_len = result[1] # len of variable id string - logger.debug(f'Data_ID: {data_id} id_len: {id_len}') + logger.debug(f'Data_ID: 0x{data_id:08x} id_len: {id_len}') msg_hdr_len = 5+id_len+9 From 63547bb51f038746497c43f639224cb98b312769 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 29 Apr 2024 22:51:31 +0200 Subject: [PATCH 02/18] adapt tests for stateless timestamp handling --- app/tests/test_talent.py | 76 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 89fd420..0b806c7 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -43,7 +43,8 @@ class MemoryStream(Talent): return copied_bytes def _timestamp(self): - return 1700260990000 + # return 1700260990000 + return 1691246944000 def _Talent__flush_recv_msg(self) -> None: super()._Talent__flush_recv_msg() @@ -101,6 +102,18 @@ def MsgControllerInd(): # Data indication from the controller msg += b'\x49\x00\x00\x00\x02\x00\x0d\x04\x08\x49\x00\x00\x00\x00\x00\x07\xa1\x84\x49\x00\x00\x00\x01\x00\x0c\x50\x59\x49\x00\x00\x00\x4c\x00\x0d\x1f\x60\x49\x00\x00\x00\x00' return msg +@pytest.fixture +def MsgControllerIndTsOffs(): # Data indication from the controller - offset 0x1000 + msg = b'\x00\x00\x01\x2f\x10R170000000000001\x91\x71\x0e\x10\x00\x00\x10R170000000000001' + msg += b'\x01\x00\x00\x01\x89\xc6\x63\x45\x50' + msg += b'\x00\x00\x00\x15\x00\x09\x2b\xa8\x54\x10\x52\x53\x57\x5f\x34\x30\x30\x5f\x56\x31\x2e\x30\x30\x2e\x30\x36\x00\x09\x27\xc0\x54\x06\x52\x61\x79\x6d\x6f' + msg += b'\x6e\x00\x09\x2f\x90\x54\x0b\x52\x53\x57\x2d\x31\x2d\x31\x30\x30\x30\x31\x00\x09\x5a\x88\x54\x0f\x74\x2e\x72\x61\x79\x6d\x6f\x6e\x69\x6f\x74\x2e\x63\x6f\x6d\x00\x09\x5a\xec\x54' + msg += b'\x1c\x6c\x6f\x67\x67\x65\x72\x2e\x74\x61\x6c\x65\x6e\x74\x2d\x6d\x6f\x6e\x69\x74\x6f\x72\x69\x6e\x67\x2e\x63\x6f\x6d\x00\x0d\x00\x20\x49\x00\x00\x00\x01\x00\x0c\x35\x00\x49\x00' + msg += b'\x00\x00\x64\x00\x0c\x96\xa8\x49\x00\x00\x00\x1d\x00\x0c\x7f\x38\x49\x00\x00\x00\x01\x00\x0c\xfc\x38\x49\x00\x00\x00\x01\x00\x0c\xf8\x50\x49\x00\x00\x01\x2c\x00\x0c\x63\xe0\x49' + msg += b'\x00\x00\x00\x00\x00\x0c\x67\xc8\x49\x00\x00\x00\x00\x00\x0c\x50\x58\x49\x00\x00\x00\x01\x00\x09\x5e\x70\x49\x00\x00\x13\x8d\x00\x09\x5e\xd4\x49\x00\x00\x13\x8d\x00\x09\x5b\x50' + msg += b'\x49\x00\x00\x00\x02\x00\x0d\x04\x08\x49\x00\x00\x00\x00\x00\x07\xa1\x84\x49\x00\x00\x00\x01\x00\x0c\x50\x59\x49\x00\x00\x00\x4c\x00\x0d\x1f\x60\x49\x00\x00\x00\x00' + return msg + @pytest.fixture def MsgControllerAck(): # Get Time Request message return b'\x00\x00\x00\x14\x10R170000000000001\x99\x71\x01' @@ -117,6 +130,14 @@ def MsgInverterInd(): # Data indication from the controller msg += b'\x54\x10T170000000000001\x00\x00\x00\x32\x54\x0a\x54\x53\x4f\x4c\x2d\x4d\x53\x36\x30\x30\x00\x00\x00\x3c\x54\x05\x41\x2c\x42\x2c\x43' return msg +@pytest.fixture +def MsgInverterIndTsOffs(): # Data indication from the controller + offset 256 + msg = b'\x00\x00\x00\x8b\x10R170000000000001\x91\x04\x01\x90\x00\x01\x10R170000000000001' + msg += b'\x01\x00\x00\x01\x89\xc6\x63\x62\x08' + msg += b'\x00\x00\x00\x06\x00\x00\x00\x0a\x54\x08\x4d\x69\x63\x72\x6f\x69\x6e\x76\x00\x00\x00\x14\x54\x04\x54\x53\x55\x4e\x00\x00\x00\x1E\x54\x07\x56\x35\x2e\x30\x2e\x31\x31\x00\x00\x00\x28' + msg += b'\x54\x10T170000000000001\x00\x00\x00\x32\x54\x0a\x54\x53\x4f\x4c\x2d\x4d\x53\x36\x30\x30\x00\x00\x00\x3c\x54\x05\x41\x2c\x42\x2c\x43' + return msg + @pytest.fixture def MsgInverterAck(): # Get Time Request message return b'\x00\x00\x00\x14\x10R170000000000001\x99\x04\x01' @@ -407,9 +428,10 @@ def test_msg_get_time(ConfigTsunInv1, MsgGetTime): assert int(m.ctrl)==145 assert m.msg_id==34 assert m.header_len==23 + assert m.ts_offset==0 assert m.data_len==0 assert m._forward_buffer==MsgGetTime - assert m._send_buffer==b'' + assert m._send_buffer==b'\x00\x00\x00\x1b\x10R170000000000001\x91"\x00\x00\x01\x89\xc6,_\x00' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 m.close() @@ -425,9 +447,10 @@ def test_msg_get_time_autark(ConfigNoTsunInv1, MsgGetTime): assert int(m.ctrl)==145 assert m.msg_id==34 assert m.header_len==23 + assert m.ts_offset==0 assert m.data_len==0 assert m._forward_buffer==b'' - assert m._send_buffer==b'\x00\x00\x00\x1b\x10R170000000000001\x91"\x00\x00\x01\x8b\xdfs\xcc0' + assert m._send_buffer==bytearray(b'\x00\x00\x00\x1b\x10R170000000000001\x91"\x00\x00\x01\x89\xc6,_\x00') assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 m.close() @@ -443,8 +466,9 @@ def test_msg_time_resp(ConfigTsunInv1, MsgTimeResp): assert int(m.ctrl)==145 assert m.msg_id==34 assert m.header_len==23 + assert m.ts_offset==3600000 assert m.data_len==8 - assert m._forward_buffer==MsgTimeResp + assert m._forward_buffer==b'' assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 m.close() @@ -461,6 +485,7 @@ def test_msg_time_resp_autark(ConfigNoTsunInv1, MsgTimeResp): assert int(m.ctrl)==145 assert m.msg_id==34 assert m.header_len==23 + assert m.ts_offset==0 assert m.data_len==8 assert m._forward_buffer==b'' assert m._send_buffer==b'' @@ -479,8 +504,9 @@ def test_msg_time_invalid(ConfigTsunInv1, MsgTimeInvalid): assert int(m.ctrl)==148 assert m.msg_id==34 assert m.header_len==23 + assert m.ts_offset==0 assert m.data_len==0 - assert m._forward_buffer==MsgTimeInvalid + assert m._forward_buffer==b'' assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 m.close() @@ -496,6 +522,7 @@ def test_msg_time_invalid_autark(ConfigNoTsunInv1, MsgTimeInvalid): assert m.unique_id == 'R170000000000001' assert int(m.ctrl)==148 assert m.msg_id==34 + assert m.ts_offset==0 assert m.header_len==23 assert m.data_len==0 assert m._forward_buffer==b'' @@ -503,7 +530,7 @@ def test_msg_time_invalid_autark(ConfigNoTsunInv1, MsgTimeInvalid): assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 m.close() -def test_msg_cntrl_ind(ConfigTsunInv1, MsgControllerInd, MsgControllerAck): +def test_msg_cntrl_ind(ConfigTsunInv1, MsgControllerInd, MsgControllerIndTsOffs, MsgControllerAck): ConfigTsunInv1 m = MemoryStream(MsgControllerInd, (0,)) m.db.stat['proxy']['Unknown_Ctrl'] = 0 @@ -516,7 +543,12 @@ def test_msg_cntrl_ind(ConfigTsunInv1, MsgControllerInd, MsgControllerAck): assert m.msg_id==113 assert m.header_len==23 assert m.data_len==284 + m.ts_offset = 0 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgControllerInd + m.ts_offset = -4096 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgControllerIndTsOffs assert m._send_buffer==MsgControllerAck assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 m.close() @@ -552,12 +584,17 @@ def test_msg_cntrl_invalid(ConfigTsunInv1, MsgControllerInvalid): assert m.msg_id==113 assert m.header_len==23 assert m.data_len==1 + m.ts_offset = 0 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgControllerInvalid + m.ts_offset = -4096 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgControllerInvalid assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 m.close() -def test_msg_inv_ind(ConfigTsunInv1, MsgInverterInd, MsgInverterAck): +def test_msg_inv_ind(ConfigTsunInv1, MsgInverterInd, MsgInverterIndTsOffs, MsgInverterAck): ConfigTsunInv1 tracer.setLevel(logging.DEBUG) m = MemoryStream(MsgInverterInd, (0,)) @@ -571,7 +608,12 @@ def test_msg_inv_ind(ConfigTsunInv1, MsgInverterInd, MsgInverterAck): assert m.msg_id==4 assert m.header_len==23 assert m.data_len==120 + m.ts_offset = 0 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgInverterInd + m.ts_offset = +256 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgInverterIndTsOffs assert m._send_buffer==MsgInverterAck assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 m.close() @@ -609,6 +651,11 @@ def test_msg_inv_invalid(ConfigTsunInv1, MsgInverterInvalid): assert m.msg_id==4 assert m.header_len==23 assert m.data_len==1 + m.ts_offset = 0 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgInverterInvalid + m.ts_offset = 256 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgInverterInvalid assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 @@ -628,6 +675,11 @@ def test_msg_ota_req(ConfigTsunInv1, MsgOtaReq): assert m.msg_id==19 assert m.header_len==23 assert m.data_len==259 + m.ts_offset = 0 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgOtaReq + m.ts_offset = 4096 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgOtaReq assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 @@ -650,6 +702,11 @@ def test_msg_ota_ack(ConfigTsunInv1, MsgOtaAck): assert m.msg_id==19 assert m.header_len==23 assert m.data_len==1 + m.ts_offset = 0 + m._update_header(m._forward_buffer) + assert m._forward_buffer==MsgOtaAck + m.ts_offset = 256 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgOtaAck assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 @@ -670,7 +727,12 @@ def test_msg_ota_invalid(ConfigTsunInv1, MsgOtaInvalid): assert m.msg_id==19 assert m.header_len==23 assert m.data_len==1 + m.ts_offset = 0 + m._update_header(m._forward_buffer) assert m._forward_buffer==MsgOtaInvalid + m.ts_offset = 4096 + assert m._forward_buffer==MsgOtaInvalid + m._update_header(m._forward_buffer) assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 assert m.db.stat['proxy']['OTA_Start_Msg'] == 0 From 81d551e47fadefea88afc5ea8a19fee7573b821b Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 30 Apr 2024 11:49:59 +0200 Subject: [PATCH 03/18] initial version --- app/tests/timestamp_old.svg | 2 ++ app/tests/timestamp_old.yuml | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 app/tests/timestamp_old.svg create mode 100644 app/tests/timestamp_old.yuml diff --git a/app/tests/timestamp_old.svg b/app/tests/timestamp_old.svg new file mode 100644 index 0000000..739a420 --- /dev/null +++ b/app/tests/timestamp_old.svg @@ -0,0 +1,2 @@ + +InverterInverterProxyProxyCloudCloudMQTT-BrokerMQTT-BrokerContactIndstore Contact Infoin proxyContactRsp (Ok)getTimeReqContactIndContactRsp (Ok)getTimeReqTimeRsp (time)TimeRsp (time)set clock ininverterDataInd (ts:=time)DataRspDataInd (ts)DataIndDataRspDataInd (ts:=time)DataRspDataInd (ts)DataIndDataRsp \ No newline at end of file diff --git a/app/tests/timestamp_old.yuml b/app/tests/timestamp_old.yuml new file mode 100644 index 0000000..8f2e99c --- /dev/null +++ b/app/tests/timestamp_old.yuml @@ -0,0 +1,26 @@ +// {type:sequence} +// {generate:true} + +[Inverter]ContactInd>[Proxy] +[Proxy]-[note: store Contact Info in proxy{bg:cornsilk}] +[Proxy]ContactRsp (Ok).>[Inverter] + +[Inverter]getTimeReq>[Proxy] +[Proxy]ContactInd>[Cloud] +[Cloud]ContactRsp (Ok).>[Proxy] +[Proxy]getTimeReq>[Cloud] +[Cloud]TimeRsp (time).>[Proxy] +[Proxy]TimeRsp (time).>[Inverter] +[Inverter]-[note: set clock in inverter{bg:cornsilk}] + +[Inverter]DataInd (ts:=time)>[Proxy] +[Proxy]DataRsp>[Inverter] +[Proxy]DataInd (ts)>>[Cloud] +[Proxy]DataInd>>[MQTT-Broker] +[Cloud]DataRsp>>[Proxy] + +[Inverter]DataInd (ts:=time)>[Proxy] +[Proxy]DataRsp>[Inverter] +[Proxy]DataInd (ts)>>[Cloud] +[Proxy]DataInd>>[MQTT-Broker] +[Cloud]DataRsp>>[Proxy] From 9985917ad2c18f673b388163ca409d95b80998a3 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 8 Jun 2024 23:15:38 +0200 Subject: [PATCH 04/18] add more type annotations --- app/src/async_stream.py | 4 +++- app/src/gen3/connection_g3.py | 6 ++++-- app/src/gen3/inverter_g3.py | 4 ++-- app/src/gen3plus/connection_g3p.py | 6 ++++-- app/src/gen3plus/inverter_g3p.py | 4 ++-- app/src/server.py | 6 +++--- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 17f5f59..1bcabf3 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -1,5 +1,6 @@ import logging import traceback +from asyncio import StreamReader, StreamWriter from messages import hex_dump_memory logger = logging.getLogger('conn') @@ -7,7 +8,8 @@ logger = logging.getLogger('conn') class AsyncStream(): - def __init__(self, reader, writer, addr) -> None: + def __init__(self, reader: StreamReader, writer: StreamWriter, + addr) -> None: logger.debug('AsyncStream.__init__') self.reader = reader self.writer = writer diff --git a/app/src/gen3/connection_g3.py b/app/src/gen3/connection_g3.py index c93156e..7730069 100644 --- a/app/src/gen3/connection_g3.py +++ b/app/src/gen3/connection_g3.py @@ -1,5 +1,6 @@ import logging # import gc +from asyncio import StreamReader, StreamWriter from async_stream import AsyncStream from gen3.talent import Talent @@ -8,12 +9,13 @@ logger = logging.getLogger('conn') class ConnectionG3(AsyncStream, Talent): - def __init__(self, reader, writer, addr, remote_stream, server_side: bool, + def __init__(self, reader: StreamReader, writer: StreamWriter, + addr, remote_stream: 'ConnectionG3', server_side: bool, id_str=b'') -> None: AsyncStream.__init__(self, reader, writer, addr) Talent.__init__(self, server_side, id_str) - self.remoteStream = remote_stream + self.remoteStream: 'ConnectionG3' = remote_stream ''' Our puplic methods diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index 1930f0e..939d235 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -1,7 +1,7 @@ -import asyncio import logging import traceback import json +from asyncio import StreamReader, StreamWriter, asyncio from config import Config from inverter import Inverter from gen3.connection_g3 import ConnectionG3 @@ -44,7 +44,7 @@ class InverterG3(Inverter, ConnectionG3): destroyed ''' - def __init__(self, reader, writer, addr): + def __init__(self, reader: StreamReader, writer: StreamWriter, addr): super().__init__(reader, writer, addr, None, True) self.__ha_restarts = -1 diff --git a/app/src/gen3plus/connection_g3p.py b/app/src/gen3plus/connection_g3p.py index a9362ce..ecc5625 100644 --- a/app/src/gen3plus/connection_g3p.py +++ b/app/src/gen3plus/connection_g3p.py @@ -1,5 +1,6 @@ import logging # import gc +from asyncio import StreamReader, StreamWriter from async_stream import AsyncStream from gen3plus.solarman_v5 import SolarmanV5 @@ -8,12 +9,13 @@ logger = logging.getLogger('conn') class ConnectionG3P(AsyncStream, SolarmanV5): - def __init__(self, reader, writer, addr, remote_stream, + def __init__(self, reader: StreamReader, writer: StreamWriter, + addr, remote_stream: 'ConnectionG3P', server_side: bool) -> None: AsyncStream.__init__(self, reader, writer, addr) SolarmanV5.__init__(self, server_side) - self.remoteStream = remote_stream + self.remoteStream: 'ConnectionG3P' = remote_stream ''' Our puplic methods diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index 487fe1e..0a942f3 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -1,7 +1,7 @@ -import asyncio import logging import traceback import json +from asyncio import StreamReader, StreamWriter, asyncio from config import Config from inverter import Inverter from gen3plus.connection_g3p import ConnectionG3P @@ -44,7 +44,7 @@ class InverterG3P(Inverter, ConnectionG3P): destroyed ''' - def __init__(self, reader, writer, addr): + def __init__(self, reader: StreamReader, writer: StreamWriter, addr): super().__init__(reader, writer, addr, None, True) self.__ha_restarts = -1 diff --git a/app/src/server.py b/app/src/server.py index 18dc401..65e504c 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -1,5 +1,5 @@ import logging -import asyncio +from asyncio import StreamReader, StreamWriter, asyncio import signal import os from logging import config # noqa F401 @@ -11,14 +11,14 @@ from scheduler import Schedule from config import Config -async def handle_client(reader, writer): +async def handle_client(reader: StreamReader, writer: StreamWriter): '''Handles a new incoming connection and starts an async loop''' addr = writer.get_extra_info('peername') await InverterG3(reader, writer, addr).server_loop(addr) -async def handle_client_v2(reader, writer): +async def handle_client_v2(reader: StreamReader, writer: StreamWriter): '''Handles a new incoming connection and starts an async loop''' addr = writer.get_extra_info('peername') From 0a1891832672f7f57208a648273027feec58c494 Mon Sep 17 00:00:00 2001 From: Stefan Allius <122395479+s-allius@users.noreply.github.com> Date: Sat, 8 Jun 2024 23:23:56 +0200 Subject: [PATCH 05/18] Update python-app.yml run also on pushes to issue branches --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2c7031b..9a5f624 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,7 +5,7 @@ name: Python application on: push: - branches: [ "main", "dev-*" ] + branches: [ "main", "dev-*", "issue*" ] paths-ignore: - '**.md' # Do no build on *.md changes - '**.yml' # Do no build on *.yml changes From a499c5e6b02b7cb3162b8e40d66ea7c6cbc344b5 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 8 Jun 2024 23:33:25 +0200 Subject: [PATCH 06/18] add more type annotations --- app/src/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/messages.py b/app/src/messages.py index 6736b0b..f54b313 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -59,7 +59,7 @@ class Message(metaclass=IterRegistry): STATE_CLOSED = 3 def __init__(self, server_side: bool, send_modbus_cb: - Callable[[bytes, int, str], None], mb_timeout): + Callable[[bytes, int, str], None], mb_timeout: int): self._registry.append(weakref.ref(self)) self.server_side = server_side From d0bd5994205a79f058457b148b62aba9fc7ae3d6 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 8 Jun 2024 23:54:52 +0200 Subject: [PATCH 07/18] fix Generator annotation for ha_proxy_confs --- app/src/infos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/infos.py b/app/src/infos.py index ee5a38a..a32e5b7 100644 --- a/app/src/infos.py +++ b/app/src/infos.py @@ -343,7 +343,7 @@ class Infos: dict[counter] -= 1 def ha_proxy_confs(self, ha_prfx: str, node_id: str, snr: str) \ - -> Generator[tuple[dict, str], None, None]: + -> Generator[tuple[str, str, str, str], None, None]: '''Generator function yields json register struct for home-assistant auto configuration and the unique entity string, for all proxy registers From 37c2246132089a9254273191d802b64c3ea32fa2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 8 Jun 2024 23:57:46 +0200 Subject: [PATCH 08/18] fix names of issue branches --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 9a5f624..f51ae3d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,7 +5,7 @@ name: Python application on: push: - branches: [ "main", "dev-*", "issue*" ] + branches: [ "main", "dev-*", "*/issue*" ] paths-ignore: - '**.md' # Do no build on *.md changes - '**.yml' # Do no build on *.yml changes From 3bc2b262b5cbf83842e8dcac7a41ec492704442f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 8 Jun 2024 23:59:13 +0200 Subject: [PATCH 09/18] add more type annotations --- app/src/messages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/messages.py b/app/src/messages.py index f54b313..dbe24b5 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -1,6 +1,6 @@ import logging import weakref -from typing import Callable +from typing import Callable, Generator if __name__ == "app.src.messages": @@ -45,7 +45,7 @@ def hex_dump_memory(level, info, data, num): class IterRegistry(type): - def __iter__(cls): + def __iter__(cls) -> Generator['Message', None, None]: for ref in cls._registry: obj = ref() if obj is not None: From 7b9550773d5f09c93d86a914cf990b464c155a21 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 01:25:06 +0200 Subject: [PATCH 10/18] don't use depricated varn anymore --- app/src/gen3plus/solarman_v5.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 64e4536..f38673c 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -339,8 +339,8 @@ class SolarmanV5(Message): def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' cause the state is not UP anymore') + logger.warning(f'[{self.node_id}] ignore MODBUS cmd,' + ' cause the state is not UP anymore') return self.__build_header(0x4510) self._send_buffer += struct.pack(' None: if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore AT+ cmd,' - ' as the state is not UP') + logger.warning(f'[{self.node_id}] ignore AT+ cmd,' + ' as the state is not UP') return AT_cmd = AT_cmd.strip() @@ -375,8 +375,7 @@ class SolarmanV5(Message): node_id = self.node_id key = 'at_resp' logger.info(f'{key}: {data_json}') - asyncio.ensure_future( - self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json)) # noqa: E501 + await self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json) # noqa: E501 return self.forward_at_cmd_resp = False From 730229cfb08df83c9cdef7e8610e71a211130ec4 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 01:26:21 +0200 Subject: [PATCH 11/18] don't mark all test as async --- app/tests/test_modbus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index b44cb12..cde756d 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -5,9 +5,9 @@ from app.src.modbus import Modbus from app.src.infos import Infos, Register pytest_plugins = ('pytest_asyncio',) -pytestmark = pytest.mark.asyncio(scope="module") +# pytestmark = pytest.mark.asyncio(scope="module") -class TestHelper(Modbus): +class TestHelper(Modbus): # pragma: no cover def __init__(self): super().__init__(self.send_cb) self.db = Infos() From 1deab4be6a65fefd7a2c5eee43fd4db0dd6dd065 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:01:04 +0200 Subject: [PATCH 12/18] fix imports --- app/src/gen3/inverter_g3.py | 3 ++- app/src/gen3plus/inverter_g3p.py | 3 ++- app/src/server.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index 939d235..6f72ca9 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -1,7 +1,8 @@ import logging import traceback import json -from asyncio import StreamReader, StreamWriter, asyncio +import asyncio +from asyncio import StreamReader, StreamWriter from config import Config from inverter import Inverter from gen3.connection_g3 import ConnectionG3 diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index 0a942f3..d69273c 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -1,7 +1,8 @@ import logging import traceback import json -from asyncio import StreamReader, StreamWriter, asyncio +import asyncio +from asyncio import StreamReader, StreamWriter from config import Config from inverter import Inverter from gen3plus.connection_g3p import ConnectionG3P diff --git a/app/src/server.py b/app/src/server.py index 65e504c..f575194 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -1,7 +1,8 @@ import logging -from asyncio import StreamReader, StreamWriter, asyncio +import asyncio import signal import os +from asyncio import StreamReader, StreamWriter from logging import config # noqa F401 from messages import Message from inverter import Inverter From ff8adb5632012256b64ae31359f0ec7b44d0537e Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:02:43 +0200 Subject: [PATCH 13/18] fix solarman unit tests - fake Mqtt class --- app/src/gen3plus/solarman_v5.py | 10 +++--- app/tests/test_solarman.py | 61 +++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index f38673c..97c7f90 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -375,7 +375,7 @@ class SolarmanV5(Message): node_id = self.node_id key = 'at_resp' logger.info(f'{key}: {data_json}') - await self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json) # noqa: E501 + await self.mqtt.publish(f'{self.entity_prfx}{node_id}{key}', data_json) # noqa: E501 return self.forward_at_cmd_resp = False @@ -502,8 +502,9 @@ class SolarmanV5(Message): self.__forward_msg() - async def publish_mqtt(self, key, data): - await self.mqtt.publish(key, data) # pragma: no cover + def publish_mqtt(self, key, data): + asyncio.ensure_future( + self.mqtt.publish(key, data)) def get_cmd_rsp_log_lvl(self) -> int: ftype = self._recv_buffer[self.header_len] @@ -527,8 +528,7 @@ class SolarmanV5(Message): node_id = self.node_id key = 'at_resp' logger.info(f'{key}: {data_json}') - asyncio.ensure_future( - self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json)) # noqa: E501 + self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json) # noqa: E501 return elif ftype == self.MB_RTU_CMD: valid = data[1] diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 9deae56..ef29998 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -24,12 +24,24 @@ class Writer(): def write(self, pdu: bytearray): self.sent_pdu = pdu + +class Mqtt(): + def __init__(self): + self.key = '' + self.data = '' + + async def publish(self, key, data): + self.key = key + self.data = data + + class MemoryStream(SolarmanV5): def __init__(self, msg, chunks = (0,), server_side: bool = True): super().__init__(server_side) if server_side: self.mb.timeout = 1 # overwrite for faster testing self.writer = Writer() + self.mqtt = Mqtt() self.__msg = msg self.__msg_len = len(msg) self.__chunks = chunks @@ -43,6 +55,8 @@ class MemoryStream(SolarmanV5): self.test_exception_async_write = False self.entity_prfx = '' self.at_acl = {'mqtt': {'allow': ['AT+'], 'block': ['AT+WEBU']}, 'tsun': {'allow': ['AT+Z', 'AT+UPURL', 'AT+SUPDATE', 'AT+TIME'], 'block': ['AT+WEBU']}} + self.key = '' + self.data = '' def _timestamp(self): return timestamp @@ -54,6 +68,10 @@ class MemoryStream(SolarmanV5): self.__msg += msg self.__msg_len += len(msg) + def publish_mqtt(self, key, data): + self.key = key + self.data = data + def _read(self) -> int: copied_bytes = 0 try: @@ -478,9 +496,10 @@ def AtCommandIndMsgBlock(): # 0x4510 @pytest.fixture def AtCommandRspMsg(): # 0x1510 - msg = b'\xa5\x0a\x00\x10\x15\x03\x03' +get_sn() +b'\x01\x01' + msg = b'\xa5\x11\x00\x10\x15\x03\x03' +get_sn() +b'\x01\x01' msg += total() msg += hb() + msg += b'\x00\x00\x00\x00+ok' msg += correct_checksum(msg) msg += b'\x15' return msg @@ -1243,48 +1262,64 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, m.close() @pytest.mark.asyncio -async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, AtCommandIndMsg): +async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, AtCommandIndMsg, AtCommandRspMsg): ConfigTsunAllowAll m = MemoryStream(DeviceIndMsg, (0,), True) m.append_msg(InverterIndMsg) - m.read() + m.append_msg(AtCommandRspMsg) + m.read() # read device ind assert m.control == 0x4110 assert str(m.seq) == '01:01' - assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._recv_buffer==InverterIndMsg + AtCommandRspMsg # unhandled next message assert m._send_buffer==DeviceRspMsg assert m._forward_buffer==DeviceIndMsg m._send_buffer = bytearray(0) # clear send buffer for next test m._forward_buffer = bytearray(0) # clear send buffer for next test await m.send_at_cmd('AT+TIME=214028,1,60,120') - assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._recv_buffer==InverterIndMsg + AtCommandRspMsg # unhandled next message assert m._send_buffer==b'' assert m._forward_buffer==b'' assert str(m.seq) == '01:01' + assert m.mqtt.key == '' + assert m.mqtt.data == "" - m.read() + m.read() # read inverter ind assert m.control == 0x4210 assert str(m.seq) == '02:02' - assert m._recv_buffer==b'' + assert m._recv_buffer==AtCommandRspMsg # unhandled next message assert m._send_buffer==InverterRspMsg assert m._forward_buffer==InverterIndMsg m._send_buffer = bytearray(0) # clear send buffer for next test m._forward_buffer = bytearray(0) # clear send buffer for next test await m.send_at_cmd('AT+TIME=214028,1,60,120') - assert m._recv_buffer==b'' + assert m._recv_buffer==AtCommandRspMsg # unhandled next message assert m._send_buffer==AtCommandIndMsg assert m._forward_buffer==b'' assert str(m.seq) == '02:03' + assert m.mqtt.key == '' + assert m.mqtt.data == "" m._send_buffer = bytearray(0) # clear send buffer for next test + m.read() # read at resp + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m._recv_buffer==b'' + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert m.key == 'at_resp' + assert m.data == "+ok" + m.test_exception_async_write = True await m.send_at_cmd('AT+TIME=214028,1,60,120') assert m._recv_buffer==b'' assert m._send_buffer==b'' assert m._forward_buffer==b'' - assert str(m.seq) == '02:04' + assert str(m.seq) == '03:04' assert m.forward_at_cmd_resp == False + assert m.mqtt.key == '' + assert m.mqtt.data == "" m.close() @pytest.mark.asyncio @@ -1306,6 +1341,8 @@ async def test_AT_cmd_blocked(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, In assert m._send_buffer==b'' assert m._forward_buffer==b'' assert str(m.seq) == '01:01' + assert m.mqtt.key == '' + assert m.mqtt.data == "" m.read() assert m.control == 0x4210 @@ -1322,6 +1359,8 @@ async def test_AT_cmd_blocked(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, In assert m._forward_buffer==b'' assert str(m.seq) == '02:02' assert m.forward_at_cmd_resp == False + assert m.mqtt.key == 'at_resp' + assert m.mqtt.data == "'AT+WEBU' is forbidden" m.close() def test_AT_cmd_ind(ConfigTsunInv1, AtCommandIndMsg): @@ -1386,7 +1425,7 @@ def test_msg_at_command_rsp1(ConfigTsunInv1, AtCommandRspMsg): assert m.control == 0x1510 assert str(m.seq) == '03:03' assert m.header_len==11 - assert m.data_len==10 + assert m.data_len==17 assert m._forward_buffer==AtCommandRspMsg assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 @@ -1405,7 +1444,7 @@ def test_msg_at_command_rsp2(ConfigTsunInv1, AtCommandRspMsg): assert m.control == 0x1510 assert str(m.seq) == '03:03' assert m.header_len==11 - assert m.data_len==10 + assert m.data_len==17 assert m._forward_buffer==b'' assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 From 57d6785f15fdc48e81b250a0289e569e7567a745 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:22:23 +0200 Subject: [PATCH 14/18] print image build time during proxy start --- app/Dockerfile | 2 +- app/entrypoint.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Dockerfile b/app/Dockerfile index 30cdae8..0ef685e 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -63,7 +63,7 @@ RUN python -m pip install --no-cache --no-index /root/wheels/* && \ COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh COPY config . COPY src . - +RUN date > /build-date.txt EXPOSE 5005 # command to run on container start diff --git a/app/entrypoint.sh b/app/entrypoint.sh index 7698d41..b6f3d11 100644 --- a/app/entrypoint.sh +++ b/app/entrypoint.sh @@ -5,6 +5,7 @@ user="$(id -u)" echo "######################################################" echo "# prepare: '$SERVICE_NAME' Version:$VERSION" echo "# for running with UserID:$UID, GroupID:$GID" +echo "# Image built: $(cat /build-date.txt) " echo "#" if [ "$user" = '0' ]; then From c8113e2f609fc538d9bb07a747030be36b0a5f95 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:29:43 +0200 Subject: [PATCH 15/18] update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4eeed..57edfbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- print imgae build time during proxy start +- add type annotations +- improve async unit test and fix pytest warnings +- run github tests even for pulls on issue branches + +## [0.8.0] - 2024-06-07 + - improve logging: add protocol or node_id to connection logs - improve logging: log ignored AT+ or MODBUS commands - improve tracelog: log level depends on message type and source From d6093e6b1181572182ca26692c1cb552a7da38f1 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:40:08 +0200 Subject: [PATCH 16/18] fix pytest collect warning --- app/tests/test_modbus.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index cde756d..0009182 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -7,7 +7,7 @@ from app.src.infos import Infos, Register pytest_plugins = ('pytest_asyncio',) # pytestmark = pytest.mark.asyncio(scope="module") -class TestHelper(Modbus): # pragma: no cover +class ModbusTestHelper(Modbus): def __init__(self): super().__init__(self.send_cb) self.db = Infos() @@ -35,7 +35,7 @@ def test_modbus_crc(): def test_build_modbus_pdu(): '''Check building and sending a MODBUS RTU''' - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,6,0x2000,0x12) assert mb.pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' assert mb._Modbus__check_crc(mb.pdu) @@ -47,7 +47,7 @@ def test_build_modbus_pdu(): def test_recv_req(): '''Receive a valid request, which must transmitted''' - mb = TestHelper() + mb = ModbusTestHelper() assert mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 @@ -56,7 +56,7 @@ def test_recv_req(): def test_recv_req_crc_err(): '''Receive a request with invalid CRC, which must be dropped''' - mb = TestHelper() + mb = ModbusTestHelper() assert not mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') assert mb.pdu == None assert mb.last_fcode == 0 @@ -66,7 +66,7 @@ def test_recv_req_crc_err(): def test_recv_resp_crc_err(): '''Receive a response with invalid CRC, which must be dropped''' - mb = TestHelper() + mb = ModbusTestHelper() # simulate a transmitted request mb.req_pend = True mb.last_addr = 1 @@ -86,7 +86,7 @@ def test_recv_resp_crc_err(): def test_recv_resp_invalid_addr(): '''Receive a response with wrong server addr, which must be dropped''' - mb = TestHelper() + mb = ModbusTestHelper() mb.req_pend = True # simulate a transmitted request mb.last_addr = 1 @@ -109,7 +109,7 @@ def test_recv_resp_invalid_addr(): def test_recv_recv_fcode(): '''Receive a response with wrong function code, which must be dropped''' - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,4,0x300e,2) assert mb.que.qsize() == 0 assert mb.req_pend @@ -130,7 +130,7 @@ def test_recv_recv_fcode(): def test_recv_resp_len(): '''Receive a response with wrong data length, which must be dropped''' - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,3,0x300e,3) assert mb.que.qsize() == 0 assert mb.req_pend @@ -152,7 +152,7 @@ def test_recv_resp_len(): def test_recv_unexpect_resp(): '''Receive a response when we havb't sent a request''' - mb = TestHelper() + mb = ModbusTestHelper() assert not mb.req_pend # check unexpected response, which must be dropped @@ -167,7 +167,7 @@ def test_recv_unexpect_resp(): def test_parse_resp(): '''Receive matching response and parse the values''' - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,3,0x3007,6) assert mb.que.qsize() == 0 assert mb.req_pend @@ -191,7 +191,7 @@ def test_parse_resp(): assert not mb.req_pend def test_queue(): - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,3,0x3022,4) assert mb.que.qsize() == 0 assert mb.req_pend @@ -210,7 +210,7 @@ def test_queue(): def test_queue2(): '''Check queue handling for build_msg() calls''' - mb = TestHelper() + mb = ModbusTestHelper() mb.build_msg(1,3,0x3007,6) mb.build_msg(1,6,0x2008,4) assert mb.que.qsize() == 1 @@ -258,7 +258,7 @@ def test_queue2(): def test_queue3(): '''Check queue handling for recv_req() calls''' - mb = TestHelper() + mb = ModbusTestHelper() assert mb.recv_req(b'\x01\x03\x30\x07\x00\x06{\t', mb.resp_handler) assert mb.recv_req(b'\x01\x06\x20\x08\x00\x04\x02\x0b', mb.resp_handler) assert mb.que.qsize() == 1 @@ -315,7 +315,7 @@ def test_queue3(): async def test_timeout(): '''Test MODBUS response timeout and RTU retransmitting''' assert asyncio.get_running_loop() - mb = TestHelper() + mb = ModbusTestHelper() mb.max_retries = 2 mb.timeout = 0.1 # 100ms timeout for fast testing, expect a time resolution of at least 10ms assert asyncio.get_running_loop() == mb.loop @@ -363,7 +363,7 @@ async def test_timeout(): def test_recv_unknown_data(): '''Receive a response with an unknwon register''' - mb = TestHelper() + mb = ModbusTestHelper() assert 0x9000 not in mb.map mb.map[0x9000] = {'reg': Register.TEST_REG1, 'fmt': '!H', 'ratio': 1} From 53f6a5447d9f22b4b2c516ebb409facfae6dde3f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:41:01 +0200 Subject: [PATCH 17/18] cleanup msg_get_time handler --- app/src/gen3/talent.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index b4c7aa6..57ac794 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -41,7 +41,7 @@ class Talent(Message): self.id_str = id_str self.contact_name = b'' self.contact_mail = b'' - self.ts_offset = 0 + self.ts_offset = 0 # time offset between tsun cloud and local self.db = InfosG3() self.switch = { 0x00: self.msg_contact_info, @@ -142,8 +142,8 @@ class Talent(Message): def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' cause the state is not UP anymore') + logger.warning(f'[{self.node_id}] ignore MODBUS cmd,' + ' cause the state is not UP anymore') return self.__build_header(0x70, 0x77) @@ -324,7 +324,6 @@ class Talent(Message): return True def msg_get_time(self): - tsun = Config.get('tsun') if self.ctrl.is_ind(): if self.data_len == 0: ts = self._timestamp() @@ -332,22 +331,22 @@ class Talent(Message): self.__build_header(0x91) self._send_buffer += struct.pack('!q', ts) self.__finish_send_msg() - if tsun['enabled']: - if self.data_len == 0: - self.forward(self._recv_buffer, self.header_len + - self.data_len) - if self.data_len >= 8: - ts = self._timestamp() - result = struct.unpack_from('!q', self._recv_buffer, - self.header_len) - self.ts_offset = result[0]-ts - logger.debug(f'tsun-time: {result[0]:08x}' - f' proxy-time: {ts:08x}', - f' offset: {self.ts_offset}') + + elif self.data_len >= 8: + ts = self._timestamp() + result = struct.unpack_from('!q', self._recv_buffer, + self.header_len) + self.ts_offset = result[0]-ts + logger.debug(f'tsun-time: {int(result[0]):08x}' + f' proxy-time: {ts:08x}' + f' offset: {self.ts_offset}') + return # ignore received response else: logger.warning('Unknown Ctrl') self.inc_counter('Unknown_Ctrl') + self.forward(self._recv_buffer, self.header_len+self.data_len) + def parse_msg_header(self): result = struct.unpack_from('!lB', self._recv_buffer, self.header_len) From 06b63f554d2d2c080d8fde00748e364845e14b8c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 9 Jun 2024 11:41:29 +0200 Subject: [PATCH 18/18] addapt unit test --- app/tests/test_talent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 126dab9..7632a09 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -549,7 +549,7 @@ def test_msg_time_resp_autark(ConfigNoTsunInv1, MsgTimeResp): assert int(m.ctrl)==145 assert m.msg_id==34 assert m.header_len==23 - assert m.ts_offset==0 + assert m.ts_offset==3600000 assert m.data_len==8 assert m._forward_buffer==b'' assert m._send_buffer==b'' @@ -570,7 +570,7 @@ def test_msg_time_invalid(ConfigTsunInv1, MsgTimeInvalid): assert m.header_len==23 assert m.ts_offset==0 assert m.data_len==0 - assert m._forward_buffer==b'' + assert m._forward_buffer==MsgTimeInvalid assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 m.close()