From 7b4ed406a1a937e9141248b272090d16b113298a Mon Sep 17 00:00:00 2001 From: Stefan Allius <122395479+s-allius@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:26:01 +0200 Subject: [PATCH 001/118] Update README.md Exchange logger fw version with the real inverter fw version in the compatibility table --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a26179..b3ef585 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ In the following table you will find an overview of which inverter model has bee A combination with a red question mark should work, but I have not checked it in detail. - + From 5d0c95d6e66f9a2779eaffaa9eec1ca5387d392f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 1 May 2024 11:57:02 +0200 Subject: [PATCH 002/118] fix typo --- app/src/gen3plus/infos_g3p.py | 2 +- app/src/infos.py | 6 ++++-- app/tests/test_infos_g3p.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/gen3plus/infos_g3p.py b/app/src/gen3plus/infos_g3p.py index 0191d04..b0adc0a 100644 --- a/app/src/gen3plus/infos_g3p.py +++ b/app/src/gen3plus/infos_g3p.py @@ -19,7 +19,7 @@ class RegisterMap: 0x4102001a: {'reg': Register.HEARTBEAT_INTERVAL, 'fmt': ' Date: Wed, 1 May 2024 11:57:32 +0200 Subject: [PATCH 003/118] Add Modbus_Command counter --- app/tests/test_infos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/test_infos.py b/app/tests/test_infos.py index d3b542e..c3e6ddf 100644 --- a/app/tests/test_infos.py +++ b/app/tests/test_infos.py @@ -17,13 +17,13 @@ def test_statistic_counter(): assert val == None or val == 0 i.static_init() # initialize counter - assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 0, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0}}) + assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 0, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "Modbus_Command": 0}}) val = i.dev_value(Register.INVERTER_CNT) # valid and initiliazed addr assert val == 0 i.inc_counter('Inverter_Cnt') - assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 1, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0}}) + assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 1, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "Modbus_Command": 0}}) val = i.dev_value(Register.INVERTER_CNT) assert val == 1 From 58c3333fcc85a1686594306c97e697998015d339 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 2 May 2024 23:55:59 +0200 Subject: [PATCH 004/118] initial checkin --- app/src/modbus.py | 56 ++++++++++++++++++++++++++++++++++++++++ app/src/singleton.py | 9 +++++++ app/tests/test_modbus.py | 21 +++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 app/src/modbus.py create mode 100644 app/src/singleton.py create mode 100644 app/tests/test_modbus.py diff --git a/app/src/modbus.py b/app/src/modbus.py new file mode 100644 index 0000000..9745f23 --- /dev/null +++ b/app/src/modbus.py @@ -0,0 +1,56 @@ +import struct + +if __name__ == "app.src.modbus": + from app.src.singleton import Singleton +else: # pragma: no cover + from singleton import Singleton + +####### +# TSUN uses the Modbus in the RTU transmission mode. +# see: https://modbus.org/docs/Modbus_over_serial_line_V1_02.pdf +# +# A Modbus PDU consists of: 'Function-Code' + 'Data' +# A Modbus RTU message consists of: 'Addr' + 'Modbus-PDU' + 'CRC-16' +# +# The 16-bit CRC is known as CRC-16-ANSI(reverse) +# see: https://en.wikipedia.org/wiki/Computation_of_cyclic_redundancy_checks +####### + +CRC_POLY = 0xA001 # (LSBF/reverse) +CRC_INIT = 0xFFFF + + +class Modbus(metaclass=Singleton): + MB_WRITE_SINGLE_REG = 6 + MB_READ_SINGLE_REG = 3 + __crc_tab = [] + + def __init__(self): + self.__build_crc_tab(CRC_POLY) + + def build_msg(self, addr, func, reg, val): + msg = struct.pack('>BBHH', addr, func, reg, val) + msg += struct.pack(' bool: + return 0 == self.__calc_crc(msg) + + def __calc_crc(self, buffer: bytes) -> int: + crc = CRC_INIT + + for cur in buffer: + crc = (crc >> 8) ^ self.__crc_tab[(crc ^ cur) & 0xFF] + return crc + + def __build_crc_tab(self, poly) -> None: + for index in range(256): + data = index << 1 + crc = 0 + for _ in range(8, 0, -1): + data >>= 1 + if (data ^ crc) & 1: + crc = (crc >> 1) ^ poly + else: + crc >>= 1 + self.__crc_tab.append(crc) diff --git a/app/src/singleton.py b/app/src/singleton.py new file mode 100644 index 0000000..48778b9 --- /dev/null +++ b/app/src/singleton.py @@ -0,0 +1,9 @@ +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + # logger_mqtt.debug('singleton: __call__') + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, + cls).__call__(*args, **kwargs) + return cls._instances[cls] diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py new file mode 100644 index 0000000..0e9cf5b --- /dev/null +++ b/app/tests/test_modbus.py @@ -0,0 +1,21 @@ +# test_with_pytest.py +# import pytest, logging +from app.src.modbus import Modbus + + +def test_modbus_crc(): + mb = Modbus() + assert 0x0b02 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04') + assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') + assert mb.check_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') + + assert 0xc803 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x00') + assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') + assert mb.check_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') + +def test_build_modbus_pdu(): + mb = Modbus() + pdu = mb.build_msg(1,6,0x2000,0x12) + assert pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' + assert mb.check_crc(pdu) + From 1d9cbf314e96d0bd61d383ee3eba8fd03c11e37e Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 2 May 2024 23:56:42 +0200 Subject: [PATCH 005/118] add Modbus tests --- app/tests/test_talent.py | 122 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 89fd420..fc4ed4e 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -3,6 +3,10 @@ import pytest, logging from app.src.gen3.talent import Talent, Control from app.src.config import Config from app.src.infos import Infos +from app.src.modbus import Modbus + + +pytest_plugins = ('pytest_asyncio',) # initialize the proxy statistics Infos.static_init() @@ -19,6 +23,7 @@ class MemoryStream(Talent): self.__chunk_idx = 0 self.msg_count = 0 self.addr = 'Test: SrvSide' + self.send_msg_ofs = 0 def append_msg(self, msg): self.__msg += msg @@ -50,6 +55,10 @@ class MemoryStream(Talent): self.msg_count += 1 return + async def flush_send_msg(self): + pass + + @pytest.fixture def MsgContactInfo(): # Contact Info message @@ -170,6 +179,26 @@ def MsgOtaAck(): # Over the air update rewuest from tsun cloud def MsgOtaInvalid(): # Get Time Request message return b'\x00\x00\x00\x14\x10R170000000000001\x99\x13\x01' +@pytest.fixture +def MsgModbusCmd(): + msg = b'\x00\x00\x00\x20\x10R170000000000001' + msg += b'\x70\x77\x00\x01\xa3\x28\x08\x01\x06\x20\x08' + msg += b'\x00\x00\x03\xc8' + return msg + +@pytest.fixture +def MsgModbusRsp(): + msg = b'\x00\x00\x00\x20\x10R170000000000001' + msg += b'\x91\x77\x17\x18\x19\x1a\x08\x01\x06\x20\x08' + msg += b'\x00\x00\x03\xc8' + return msg + +@pytest.fixture +def MsgModbusInv(): + msg = b'\x00\x00\x00\x20\x10R170000000000001' + msg += b'\x99\x77\x17\x18\x19\x1a\x08\x01\x06\x20\x08' + msg += b'\x00\x00\x03\xc8' + return msg def test_read_message(MsgContactInfo): m = MemoryStream(MsgContactInfo, (0,)) @@ -740,3 +769,96 @@ def test_proxy_counter(): assert Infos.new_stat_data == {'proxy': True} assert 0 == m.db.stat['proxy']['Unknown_Msg'] m.close() + +def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): + ConfigTsunInv1 + m = MemoryStream(MsgModbusCmd, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==112 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==13 + assert m._forward_buffer==MsgModbusCmd + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 1 + m.close() + +def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_rep = False + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==145 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==13 + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_rep = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==145 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==13 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInv): + ConfigTsunInv1 + m = MemoryStream(MsgModbusInv, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==153 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==13 + assert m._forward_buffer==MsgModbusInv + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +@pytest.mark.asyncio +async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): + ConfigTsunInv1 + m = MemoryStream(b'', (0,), False) + m.id_str = b"R170000000000001" + await m.send_modbus_cmd(Modbus.MB_WRITE_SINGLE_REG, 0x2008, 0) + assert 0==m.send_msg_ofs + assert m._forward_buffer==b'' + assert m._send_buffer==MsgModbusCmd + m.close() From dba3b458ba34d787c5afae755da22ea9d542c7d5 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 2 May 2024 23:59:55 +0200 Subject: [PATCH 006/118] add Modbus support --- app/src/gen3/talent.py | 58 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 46302ac..f983c79 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -3,12 +3,15 @@ import logging import time from datetime import datetime + if __name__ == "app.src.gen3.talent": from app.src.messages import hex_dump_memory, Message + from app.src.modbus import Modbus from app.src.config import Config from app.src.gen3.infos_g3 import InfosG3 else: # pragma: no cover from messages import hex_dump_memory, Message + from modbus import Modbus from config import Config from gen3.infos_g3 import InfosG3 @@ -41,13 +44,18 @@ class Talent(Message): self.contact_name = b'' self.contact_mail = b'' self.db = InfosG3() + self.forward_modbus_rep = False self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, 0x22: self.msg_get_time, 0x71: self.msg_collector_data, + # 0x76: + 0x77: self.msg_modbus, + # 0x78: 0x04: self.msg_inverter_data, } + self.mb = Modbus() ''' Our puplic methods @@ -115,6 +123,18 @@ class Talent(Message): f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}') return + async def send_modbus_cmd(self, func, addr, val) -> None: + self.forward_modbus_rep = False + self.__build_header(0x70, 0x77) + self._send_buffer += b'\x00\x01\xa3\x28' + modbus_msg = self.mb.build_msg(1, func, addr, val) + self._send_buffer += struct.pack('!B', len(modbus_msg)) + self._send_buffer += modbus_msg + _len = self.__finish_send_msg() + hex_dump_memory(logging.INFO, 'Send Modbus Command:', + self._send_buffer[self.send_msg_ofs:], _len) + await self.flush_send_msg() + def _init_new_client_conn(self) -> bool: contact_name = self.contact_name contact_mail = self.contact_mail @@ -190,17 +210,20 @@ class Talent(Message): self.header_valid = True return - def __build_header(self, ctrl) -> None: + def __build_header(self, ctrl, msg_id=None) -> None: + if not msg_id: + msg_id = self.msg_id self.send_msg_ofs = len(self._send_buffer) self._send_buffer += struct.pack(f'!l{len(self.id_str)+1}pBB', - 0, self.id_str, ctrl, self.msg_id) - fnc = self.switch.get(self.msg_id, self.msg_unknown) + 0, self.id_str, ctrl, msg_id) + fnc = self.switch.get(msg_id, self.msg_unknown) logger.info(self.__flow_str(self.server_side, 'tx') + f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') - def __finish_send_msg(self) -> None: + def __finish_send_msg(self) -> int: _len = len(self._send_buffer) - self.send_msg_ofs struct.pack_into('!l', self._send_buffer, self.send_msg_ofs, _len-4) + return _len def __dispatch_msg(self) -> None: fnc = self.switch.get(self.msg_id, self.msg_unknown) @@ -348,6 +371,33 @@ class Talent(Message): self.inc_counter('Unknown_Ctrl') self.forward(self._recv_buffer, self.header_len+self.data_len) + def parse_modbus_header(self): + + msg_hdr_len = 5 + + result = struct.unpack_from('!lB', self._recv_buffer, + self.header_len + 4) + modbus_len = result[1] + logger.debug(f'Ref: {result[0]}') + logger.debug(f'Modbus Len: {modbus_len}') + # logger.info(f'time: {datetime.utcfromtimestamp(result[2]).strftime( + # "%Y-%m-%d %H:%M:%S")}') + return msg_hdr_len, modbus_len + + def msg_modbus(self): + hdr_len, modbus_len = self.parse_modbus_header() + + if self.ctrl.is_req(): + self.forward_modbus_rep = True + self.inc_counter('Modbus_Command') + elif self.ctrl.is_ind(): + if not self.forward_modbus_rep: + return + else: + logger.warning('Unknown Ctrl') + self.inc_counter('Unknown_Ctrl') + self.forward(self._recv_buffer, self.header_len+self.data_len) + def msg_unknown(self): logger.warning(f"Unknow Msg: ID:{self.msg_id}") self.inc_counter('Unknown_Msg') From 5fdad484f480860bc96899b54fb078454eee5a83 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 00:03:02 +0200 Subject: [PATCH 007/118] add flush_send_msg() implementation --- app/src/gen3/connection_g3.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/src/gen3/connection_g3.py b/app/src/gen3/connection_g3.py index c93156e..3ac1e7c 100644 --- a/app/src/gen3/connection_g3.py +++ b/app/src/gen3/connection_g3.py @@ -29,6 +29,11 @@ class ConnectionG3(AsyncStream, Talent): async def async_publ_mqtt(self) -> None: pass + async def flush_send_msg(self) -> None: + self.writer.write(self._send_buffer) + await self.writer.drain() + self._send_buffer = bytearray(0) + ''' Our private methods ''' From 30dc802fb23cdae858cf362c8df26cd9f0c14adc Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 00:05:34 +0200 Subject: [PATCH 008/118] Add MQTT subscrition for modbus experiences --- app/src/mqtt.py | 57 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 5b2de02..6a69c95 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -1,22 +1,15 @@ import asyncio import logging import aiomqtt +import traceback +from modbus import Modbus +from messages import Message from config import Config +from singleton import Singleton logger_mqtt = logging.getLogger('mqtt') -class Singleton(type): - _instances = {} - - def __call__(cls, *args, **kwargs): - logger_mqtt.debug('singleton: __call__') - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, - cls).__call__(*args, **kwargs) - return cls._instances[cls] - - class Mqtt(metaclass=Singleton): __client = None __cb_MqttIsUp = None @@ -65,6 +58,9 @@ class Mqtt(metaclass=Singleton): password=mqtt['passwd']) interval = 5 # Seconds + ha_status_topic = f"{ha['auto_conf_prefix']}/status" + inv_cnf_topic = "tsun/+/test" + while True: try: async with self.__client: @@ -74,16 +70,32 @@ class Mqtt(metaclass=Singleton): await self.__cb_MqttIsUp() # async with self.__client.messages() as messages: - await self.__client.subscribe( - f"{ha['auto_conf_prefix']}" - "/status") + await self.__client.subscribe(ha_status_topic) + await self.__client.subscribe(inv_cnf_topic) + async for message in self.__client.messages: - status = message.payload.decode("UTF-8") - logger_mqtt.info('Home-Assistant Status:' - f' {status}') - if status == 'online': - self.ha_restarts += 1 - await self.__cb_MqttIsUp() + if message.topic.matches(ha_status_topic): + status = message.payload.decode("UTF-8") + logger_mqtt.info('Home-Assistant Status:' + f' {status}') + if status == 'online': + self.ha_restarts += 1 + await self.__cb_MqttIsUp() + + if message.topic.matches(inv_cnf_topic): + topic = str(message.topic) + node_id = topic.split('/')[1] + '/' + payload = message.payload.decode("UTF-8") + logger_mqtt.info(f'InvCnf: {node_id}:{payload}') + for m in Message: + if m.server_side and m.node_id == node_id: + logger_mqtt.info(f'Found: {node_id}') + fnc = getattr(m, "send_modbus_cmd", None) + if callable(fnc): + # await fnc(Modbus.MB_WRITE_SINGLE_REG, + # 0x2008, 2) + await fnc(Modbus.MB_READ_SINGLE_REG, + 0x2008, 1) except aiomqtt.MqttError: if Config.is_default('mqtt'): @@ -101,3 +113,8 @@ class Mqtt(metaclass=Singleton): logger_mqtt.debug("MQTT task cancelled") self.__client = None return + except Exception: + # self.inc_counter('SW_Exception') + logger_mqtt.error( + f"Exception:\n" + f"{traceback.format_exc()}") From 494c30e4898b67ffb602a0ca5df947e4159fef16 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:21:15 +0200 Subject: [PATCH 009/118] renme __async_write() into async_write() --- app/proxy.svg | 562 ++++++++++++++++++++-------------------- app/proxy.yuml | 4 +- app/src/async_stream.py | 8 +- 3 files changed, 293 insertions(+), 281 deletions(-) diff --git a/app/proxy.svg b/app/proxy.svg index 588835e..dfbdd46 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,340 +4,352 @@ - + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -Singleton + +Singleton A2 - -Mqtt - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() + +Mqtt + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() A1->A2 - - - - - -A10 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt - - - - -A2->A10 - + + A3 - -IterRegistry - - -__iter__ + +Modbus - - -A4 - -Message - -server_side:bool -header_valid:bool -header_len:unsigned -data_len:unsigned -unique_id -node_id -sug_area -_recv_buffer:bytearray -_send_buffer:bytearray -_forward_buffer:bytearray -db:Infos -new_data:list - -_read():void<abstract> -close():void -inc_counter():void -dec_counter():void - - + -A3->A4 - - - - - -A5 - -Talent - -await_conn_resp_cnt -id_str -contact_name -contact_mail -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -close() - - - -A4->A5 - - - - - -A6 - -SolarmanV5 - -control -serial -snr -switch - -msg_unknown() -close() - - - -A4->A6 - - - - - -A7 - -ConnectionG3 - -remoteStream:ConnectionG3 - -close() - - - -A5->A7 - - - - - -A8 - -ConnectionG3P - -remoteStream:ConnectionG3P - -close() - - - -A6->A8 - - - - - -A7->A7 - - -0..1 -has +A1->A3 + + A11 - -InverterG3 - -__ha_restarts - -async_create_remote() -close() + +Inverter + +cls.db_stat +cls.entity_prfx +cls.discovery_prfx +cls.proxy_node_id +cls.proxy_unique_id +cls.mqtt:Mqtt + - + -A7->A11 - - +A2->A11 + - - -A8->A8 - - -0..1 -has + + +A4 + +IterRegistry + + +__iter__ - - -A12 - -InverterG3P - -__ha_restarts - -async_create_remote() -close() + + +A5 + +Message + +server_side:bool +header_valid:bool +header_len:unsigned +data_len:unsigned +unique_id +node_id +sug_area +_recv_buffer:bytearray +_send_buffer:bytearray +_forward_buffer:bytearray +db:Infos +new_data:list + +_read():void<abstract> +close():void +inc_counter():void +dec_counter():void - - -A8->A12 - - + + +A4->A5 + + + + + +A6 + +Talent + +await_conn_resp_cnt +id_str +contact_name +contact_mail +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +close() + + + +A5->A6 + + + + + +A7 + +SolarmanV5 + +control +serial +snr +switch + +msg_unknown() +close() + + + +A5->A7 + + + + + +A8 + +ConnectionG3 + +remoteStream:ConnectionG3 + +close() + + + +A6->A8 + + A9 - -AsyncStream - -reader -writer -addr -r_addr -l_addr - -<async>server_loop() -<async>client_loop() -<async>loop -disc() -close() -__async_read() -__async_write() -__async_forward() + +ConnectionG3P + +remoteStream:ConnectionG3P + +close() - + -A9->A7 - - +A7->A9 + + - - -A9->A8 - - + + +A8->A8 + + +0..1 +has - - -A10->A11 - - + + +A12 + +InverterG3 + +__ha_restarts + +async_create_remote() +close() - - -A10->A12 - - + + +A8->A12 + + + + + +A9->A9 + + +0..1 +has A13 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -update_db -set_db_def_value -get_db_value -ignore_this_device + +InverterG3P + +__ha_restarts + +async_create_remote() +close() + + + +A9->A13 + + + + + +A10 + +AsyncStream + +reader +writer +addr +r_addr +l_addr + +<async>server_loop() +<async>client_loop() +<async>loop +disc() +close() +__async_read() +async_write() +__async_forward() + + + +A10->A8 + + + + + +A10->A9 + + + + + +A11->A12 + + + + + +A11->A13 + + A14 - -InfosG3 - - -ha_confs() -parse() - - - -A13->A14 - - + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +update_db +set_db_def_value +get_db_value +ignore_this_device A15 - -InfosG3P - - -ha_confs() -parse() + +InfosG3 + + +ha_confs() +parse() - + -A13->A15 - - +A14->A15 + + - - -A14->A5 - - + + +A16 + +InfosG3P + + +ha_confs() +parse() + + + +A14->A16 + + - + A15->A6 - - + + + + + +A16->A7 + + diff --git a/app/proxy.yuml b/app/proxy.yuml index 7f5be21..daf5d3c 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -4,13 +4,13 @@ [note: You can stick notes on diagrams too!{bg:cornsilk}] [Singleton]^[Mqtt|ha_restarts;__client;__cb_MqttIsUp|publish();close()] - +[Singleton]^[Modbus] [IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list|_read():void;close():void;inc_counter():void;dec_counter():void] [Message]^[Talent|await_conn_resp_cnt;id_str;contact_name;contact_mail;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;close()] [Message]^[SolarmanV5|control;serial;snr;switch|msg_unknown();;close()] [Talent]^[ConnectionG3|remoteStream:ConnectionG3|close()] [SolarmanV5]^[ConnectionG3P|remoteStream:ConnectionG3P|close()] -[AsyncStream|reader;writer;addr;r_addr;l_addr|server_loop();client_loop();loop;disc();close();;__async_read();__async_write();__async_forward()]^[ConnectionG3] +[AsyncStream|reader;writer;addr;r_addr;l_addr|server_loop();client_loop();loop;disc();close();;__async_read();async_write();__async_forward()]^[ConnectionG3] [AsyncStream]^[ConnectionG3P] [Inverter|cls.db_stat;cls.entity_prfx;cls.discovery_prfx;cls.proxy_node_id;cls.proxy_unique_id;cls.mqtt:Mqtt|]^[InverterG3|__ha_restarts|async_create_remote();;close()] [Inverter]^[InverterG3P|__ha_restarts|async_create_remote();;close()] diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 6c1136c..28873e8 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -61,7 +61,7 @@ class AsyncStream(): await self.__async_read() if self.unique_id: - await self.__async_write() + await self.async_write() await self.__async_forward() await self.async_publ_mqtt() @@ -100,9 +100,9 @@ class AsyncStream(): else: raise RuntimeError("Peer closed.") - async def __async_write(self) -> None: + async def async_write(self, headline='Transmit to ') -> None: if self._send_buffer: - hex_dump_memory(logging.INFO, f'Transmit to {self.addr}:', + hex_dump_memory(logging.INFO, f'{headline}{self.addr}:', self._send_buffer, len(self._send_buffer)) self.writer.write(self._send_buffer) await self.writer.drain() @@ -114,7 +114,7 @@ class AsyncStream(): await self.async_create_remote() if self.remoteStream: if self.remoteStream._init_new_client_conn(): - await self.remoteStream.__async_write() + await self.remoteStream.async_write() if self.remoteStream: self.remoteStream._update_header(self._forward_buffer) From fdedfcbf8e438f0ff8c897018b4d7438797d63e1 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:21:59 +0200 Subject: [PATCH 010/118] reneme Modbus constants --- app/src/modbus.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 9745f23..ebaa365 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -21,8 +21,12 @@ CRC_INIT = 0xFFFF class Modbus(metaclass=Singleton): - MB_WRITE_SINGLE_REG = 6 - MB_READ_SINGLE_REG = 3 + + READ_REGS = 3 + READ_INPUTS = 4 + WRITE_SINGLE_REG = 6 + '''Modbus function codes''' + __crc_tab = [] def __init__(self): From f78d4ac31075cba554be4c3e8638bde54076e136 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:22:31 +0200 Subject: [PATCH 011/118] remove flush_send_msg() --- app/src/gen3/connection_g3.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/src/gen3/connection_g3.py b/app/src/gen3/connection_g3.py index 3ac1e7c..c93156e 100644 --- a/app/src/gen3/connection_g3.py +++ b/app/src/gen3/connection_g3.py @@ -29,11 +29,6 @@ class ConnectionG3(AsyncStream, Talent): async def async_publ_mqtt(self) -> None: pass - async def flush_send_msg(self) -> None: - self.writer.write(self._send_buffer) - await self.writer.drain() - self._send_buffer = bytearray(0) - ''' Our private methods ''' From a2f67e7d3e58f2c17ac2a1a2eebef49a37bf306f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:23:08 +0200 Subject: [PATCH 012/118] use async_write() instead of flush_send_msg() --- app/src/gen3/talent.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index f983c79..ac63cef 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -3,7 +3,6 @@ import logging import time from datetime import datetime - if __name__ == "app.src.gen3.talent": from app.src.messages import hex_dump_memory, Message from app.src.modbus import Modbus @@ -44,6 +43,7 @@ class Talent(Message): self.contact_name = b'' self.contact_mail = b'' self.db = InfosG3() + self.mb = Modbus() self.forward_modbus_rep = False self.switch = { 0x00: self.msg_contact_info, @@ -55,7 +55,6 @@ class Talent(Message): # 0x78: 0x04: self.msg_inverter_data, } - self.mb = Modbus() ''' Our puplic methods @@ -126,14 +125,12 @@ class Talent(Message): async def send_modbus_cmd(self, func, addr, val) -> None: self.forward_modbus_rep = False self.__build_header(0x70, 0x77) - self._send_buffer += b'\x00\x01\xa3\x28' + self._send_buffer += b'\x00\x01\xa3\x28' # fixme modbus_msg = self.mb.build_msg(1, func, addr, val) self._send_buffer += struct.pack('!B', len(modbus_msg)) self._send_buffer += modbus_msg - _len = self.__finish_send_msg() - hex_dump_memory(logging.INFO, 'Send Modbus Command:', - self._send_buffer[self.send_msg_ofs:], _len) - await self.flush_send_msg() + self.__finish_send_msg() + await self.async_write('Send Modbus Command:') def _init_new_client_conn(self) -> bool: contact_name = self.contact_name @@ -220,10 +217,9 @@ class Talent(Message): logger.info(self.__flow_str(self.server_side, 'tx') + f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') - def __finish_send_msg(self) -> int: + def __finish_send_msg(self) -> None: _len = len(self._send_buffer) - self.send_msg_ofs struct.pack_into('!l', self._send_buffer, self.send_msg_ofs, _len-4) - return _len def __dispatch_msg(self) -> None: fnc = self.switch.get(self.msg_id, self.msg_unknown) @@ -375,11 +371,11 @@ class Talent(Message): msg_hdr_len = 5 - result = struct.unpack_from('!lB', self._recv_buffer, - self.header_len + 4) + result = struct.unpack_from('!lBB', self._recv_buffer, + self.header_len) modbus_len = result[1] logger.debug(f'Ref: {result[0]}') - logger.debug(f'Modbus Len: {modbus_len}') + logger.debug(f'Modbus MsgLen: {modbus_len} Func:{result[2]}') # logger.info(f'time: {datetime.utcfromtimestamp(result[2]).strftime( # "%Y-%m-%d %H:%M:%S")}') return msg_hdr_len, modbus_len From 763af8b4cfcff38441fdff78b89843c44f6d3a56 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:24:06 +0200 Subject: [PATCH 013/118] add send_modbus_cmd() --- app/src/gen3plus/solarman_v5.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index d23efa3..83f4d49 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -6,12 +6,14 @@ from datetime import datetime if __name__ == "app.src.gen3plus.solarman_v5": from app.src.messages import hex_dump_memory, Message + from app.src.modbus import Modbus from app.src.config import Config from app.src.gen3plus.infos_g3p import InfosG3P from app.src.infos import Register else: # pragma: no cover from messages import hex_dump_memory, Message from config import Config + from modbus import Modbus from gen3plus.infos_g3p import InfosG3P from infos import Register # import traceback @@ -56,6 +58,8 @@ class SolarmanV5(Message): self.snr = 0 self.db = InfosG3P() self.time_ofs = 0 + self.mb = Modbus() + self.forward_modbus_rep = False self.switch = { 0x4210: self.msg_data_ind, # real time data @@ -293,6 +297,14 @@ class SolarmanV5(Message): self._heartbeat()) self.__finish_send_msg() + async def send_modbus_cmd(self, func, addr, val) -> None: + self.forward_modbus_rep = False + self.__build_header(0x4510) + self._send_buffer += struct.pack(' None: self.__build_header(0x4510) self._send_buffer += struct.pack(f' Date: Fri, 3 May 2024 18:24:48 +0200 Subject: [PATCH 014/118] use async_write instead of flush_send_msg() --- app/tests/test_talent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index fc4ed4e..7c996ab 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -55,7 +55,7 @@ class MemoryStream(Talent): self.msg_count += 1 return - async def flush_send_msg(self): + async def async_write(self, headline=''): pass @@ -857,7 +857,7 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(b'', (0,), False) m.id_str = b"R170000000000001" - await m.send_modbus_cmd(Modbus.MB_WRITE_SINGLE_REG, 0x2008, 0) + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0==m.send_msg_ofs assert m._forward_buffer==b'' assert m._send_buffer==MsgModbusCmd From 3dbcee63f6907ac08bd6e95fba55d6d1e68198ec Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 3 May 2024 18:25:37 +0200 Subject: [PATCH 015/118] add Modbus topics --- app/src/mqtt.py | 60 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 6a69c95..9d6e83e 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -59,7 +59,9 @@ class Mqtt(metaclass=Singleton): interval = 5 # Seconds ha_status_topic = f"{ha['auto_conf_prefix']}/status" - inv_cnf_topic = "tsun/+/test" + mb_rated_topic = "tsun/+/rated_load" # fixme + mb_reads_topic = "tsun/+/modbus_read_regs" # fixme + mb_inputs_topic = "tsun/+/modbus_read_inputs" # fixme while True: try: @@ -71,7 +73,9 @@ class Mqtt(metaclass=Singleton): # async with self.__client.messages() as messages: await self.__client.subscribe(ha_status_topic) - await self.__client.subscribe(inv_cnf_topic) + await self.__client.subscribe(mb_rated_topic) + await self.__client.subscribe(mb_reads_topic) + await self.__client.subscribe(mb_inputs_topic) async for message in self.__client.messages: if message.topic.matches(ha_status_topic): @@ -82,20 +86,18 @@ class Mqtt(metaclass=Singleton): self.ha_restarts += 1 await self.__cb_MqttIsUp() - if message.topic.matches(inv_cnf_topic): - topic = str(message.topic) - node_id = topic.split('/')[1] + '/' - payload = message.payload.decode("UTF-8") - logger_mqtt.info(f'InvCnf: {node_id}:{payload}') - for m in Message: - if m.server_side and m.node_id == node_id: - logger_mqtt.info(f'Found: {node_id}') - fnc = getattr(m, "send_modbus_cmd", None) - if callable(fnc): - # await fnc(Modbus.MB_WRITE_SINGLE_REG, - # 0x2008, 2) - await fnc(Modbus.MB_READ_SINGLE_REG, - 0x2008, 1) + if message.topic.matches(mb_rated_topic): + await self.modbus_cmd(message, + Modbus.WRITE_SINGLE_REG, + 1, 0x2008) + + if message.topic.matches(mb_reads_topic): + await self.modbus_cmd(message, + Modbus.READ_REGS, 2) + + if message.topic.matches(mb_inputs_topic): + await self.modbus_cmd(message, + Modbus.READ_INPUTS, 2) except aiomqtt.MqttError: if Config.is_default('mqtt'): @@ -114,7 +116,31 @@ class Mqtt(metaclass=Singleton): self.__client = None return except Exception: - # self.inc_counter('SW_Exception') + # self.inc_counter('SW_Exception') # fixme logger_mqtt.error( f"Exception:\n" f"{traceback.format_exc()}") + + async def modbus_cmd(self, message, func, params=0, addr=0, val=0): + topic = str(message.topic) + node_id = topic.split('/')[1] + '/' + # refactor into a loop over a table + payload = message.payload.decode("UTF-8") + logger_mqtt.info(f'InvCnf: {node_id}:{payload}') + for m in Message: + if m.server_side and m.node_id == node_id: + logger_mqtt.info(f'Found: {node_id}') + fnc = getattr(m, "send_modbus_cmd", None) + res = payload.split(',') + if params != len(res): + logger_mqtt.error(f'Parameter expected: {params}, ' + f'got: {len(res)}') + return + + if callable(fnc): + if params == 1: + val = int(payload) + elif params == 2: + addr = int(res[0], base=16) + val = int(res[1]) # lenght + await fnc(func, addr, val) From eda8ef1db634f030c2a3a8ab3d0d730f9e0e14e1 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:13:51 +0200 Subject: [PATCH 016/118] add Modbus and AT command handler --- app/src/gen3plus/solarman_v5.py | 101 +++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 83f4d49..a459e89 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -48,6 +48,8 @@ class Sequence(): class SolarmanV5(Message): + AT_CMD = 1 + MB_RTU_CMD = 2 def __init__(self, server_side: bool): super().__init__(server_side) @@ -59,7 +61,7 @@ class SolarmanV5(Message): self.db = InfosG3P() self.time_ofs = 0 self.mb = Modbus() - self.forward_modbus_rep = False + self.forward_modbus_resp = False self.switch = { 0x4210: self.msg_data_ind, # real time data @@ -88,7 +90,7 @@ class SolarmanV5(Message): # # MODbus or AT cmd 0x4510: self.msg_command_req, # from server - 0x1510: self.msg_response, # from inverter + 0x1510: self.msg_command_rsp, # from inverter } ''' @@ -298,23 +300,48 @@ class SolarmanV5(Message): self.__finish_send_msg() async def send_modbus_cmd(self, func, addr, val) -> None: - self.forward_modbus_rep = False + self.forward_modbus_resp = False self.__build_header(0x4510) - self._send_buffer += struct.pack(' None: + async def send_at_cmd(self, AT_cmd: str) -> None: self.__build_header(0x4510) - self._send_buffer += struct.pack(f'> 8 @@ -325,21 +352,7 @@ class SolarmanV5(Message): self.new_data[key] = True if inv_update: - db = self.db - MaxPow = db.get_db_value(Register.MAX_DESIGNED_POWER, 0) - Rated = db.get_db_value(Register.RATED_POWER, 0) - Model = None - if MaxPow == 2000: - if Rated == 800 or Rated == 600: - Model = f'TSOL-MS{MaxPow}({Rated})' - else: - Model = f'TSOL-MS{MaxPow}' - elif MaxPow == 1800 or MaxPow == 1600: - Model = f'TSOL-MS{MaxPow}' - if Model: - logger.info(f'Model: {Model}') - self.db.set_db_def_value(Register.EQUIPMENT_MODEL, Model) - + self.__build_model_name() ''' Message handler methods ''' @@ -402,11 +415,42 @@ class SolarmanV5(Message): data = self._recv_buffer[self.header_len:] result = struct.unpack_from(' 4: + logger.info(f'first byte modbus:{data[14]}') + inv_update = False + for key, update in self.mb.recv_resp(self.db, data[14:-2]): + if update: + if key == 'inverter': + inv_update = True + self.new_data[key] = True + + if inv_update: + self.__build_model_name() + + if not self.forward_modbus_resp: + return + self.__forward_msg() + def msg_hbeat_ind(self): data = self._recv_buffer[self.header_len:] result = struct.unpack_from(' Date: Sun, 5 May 2024 20:14:51 +0200 Subject: [PATCH 017/118] add modbus resp handler --- app/src/gen3/talent.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index ac63cef..6225aa0 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -44,7 +44,7 @@ class Talent(Message): self.contact_mail = b'' self.db = InfosG3() self.mb = Modbus() - self.forward_modbus_rep = False + self.forward_modbus_resp = False self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, @@ -123,14 +123,17 @@ class Talent(Message): return async def send_modbus_cmd(self, func, addr, val) -> None: - self.forward_modbus_rep = False + self.forward_modbus_resp = False self.__build_header(0x70, 0x77) - self._send_buffer += b'\x00\x01\xa3\x28' # fixme - modbus_msg = self.mb.build_msg(1, func, addr, val) + self._send_buffer += b'\x00\x01\xa3\x28' # fixme + modbus_msg = self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) self._send_buffer += struct.pack('!B', len(modbus_msg)) self._send_buffer += modbus_msg self.__finish_send_msg() - await self.async_write('Send Modbus Command:') + try: + await self.async_write('Send Modbus Command:') + except Exception: + self._send_buffer = bytearray(0) def _init_new_client_conn(self) -> bool: contact_name = self.contact_name @@ -384,10 +387,16 @@ class Talent(Message): hdr_len, modbus_len = self.parse_modbus_header() if self.ctrl.is_req(): - self.forward_modbus_rep = True + self.forward_modbus_resp = True self.inc_counter('Modbus_Command') elif self.ctrl.is_ind(): - if not self.forward_modbus_rep: + logger.debug(f'Modbus Ind MsgLen: {modbus_len}') + for key, update in self.mb.recv_resp(self.db, self._recv_buffer[ + self.header_len + hdr_len:]): + if update: + self.new_data[key] = True + + if not self.forward_modbus_resp: return else: logger.warning('Unknown Ctrl') From 808bf2fe873642149c7d187e07211b0b8381c20a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:15:36 +0200 Subject: [PATCH 018/118] add MQTT topic for AT commands --- app/src/mqtt.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 9d6e83e..7257038 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -62,6 +62,7 @@ class Mqtt(metaclass=Singleton): mb_rated_topic = "tsun/+/rated_load" # fixme mb_reads_topic = "tsun/+/modbus_read_regs" # fixme mb_inputs_topic = "tsun/+/modbus_read_inputs" # fixme + mb_at_cmd_topic = "tsun/+/at_cmd" # fixme while True: try: @@ -76,6 +77,7 @@ class Mqtt(metaclass=Singleton): await self.__client.subscribe(mb_rated_topic) await self.__client.subscribe(mb_reads_topic) await self.__client.subscribe(mb_inputs_topic) + await self.__client.subscribe(mb_at_cmd_topic) async for message in self.__client.messages: if message.topic.matches(ha_status_topic): @@ -99,6 +101,9 @@ class Mqtt(metaclass=Singleton): await self.modbus_cmd(message, Modbus.READ_INPUTS, 2) + if message.topic.matches(mb_at_cmd_topic): + await self.at_cmd(message) + except aiomqtt.MqttError: if Config.is_default('mqtt'): logger_mqtt.info( @@ -116,11 +121,26 @@ class Mqtt(metaclass=Singleton): self.__client = None return except Exception: - # self.inc_counter('SW_Exception') # fixme + # self.inc_counter('SW_Exception') # fixme logger_mqtt.error( f"Exception:\n" f"{traceback.format_exc()}") + def each_inverter(self, message, func_name: str): + topic = str(message.topic) + node_id = topic.split('/')[1] + '/' + for m in Message: + if m.server_side and m.node_id == node_id: + logger_mqtt.debug(f'Found: {node_id}') + fnc = getattr(m, func_name, None) + if callable(fnc): + yield fnc + else: + logger_mqtt.warning(f'Cmd not supported by: {node_id}') + + else: + logger_mqtt.warning(f'Node_id: {node_id} not found') + async def modbus_cmd(self, message, func, params=0, addr=0, val=0): topic = str(message.topic) node_id = topic.split('/')[1] + '/' @@ -144,3 +164,8 @@ class Mqtt(metaclass=Singleton): addr = int(res[0], base=16) val = int(res[1]) # lenght await fnc(func, addr, val) + + async def at_cmd(self, message): + payload = message.payload.decode("UTF-8") + for fnc in self.each_inverter(message, "send_at_cmd"): + await fnc(payload) From 283ae31af2f4e4c55b162ffa5b18041352031fdf Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:16:28 +0200 Subject: [PATCH 019/118] parse modbus message and store values in db --- app/src/modbus.py | 112 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index ebaa365..0f16705 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -1,9 +1,11 @@ import struct +import logging +from typing import Generator if __name__ == "app.src.modbus": - from app.src.singleton import Singleton + from app.src.infos import Register else: # pragma: no cover - from singleton import Singleton + from infos import Register ####### # TSUN uses the Modbus in the RTU transmission mode. @@ -20,23 +22,123 @@ CRC_POLY = 0xA001 # (LSBF/reverse) CRC_INIT = 0xFFFF -class Modbus(metaclass=Singleton): - +class Modbus(): + INV_ADDR = 1 READ_REGS = 3 READ_INPUTS = 4 WRITE_SINGLE_REG = 6 '''Modbus function codes''' __crc_tab = [] + map = { + 0x2007: {'reg': Register.MAX_DESIGNED_POWER, 'fmt': '!H', 'ratio': 1}, # noqa: E501 + 0x420100c0: {'reg': Register.INVERTER_STATUS, 'fmt': '!H'}, # noqa: E501 + 0x3008: {'reg': Register.VERSION, 'fmt': '!H', 'eval': "f'v{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 + 0x3009: {'reg': Register.GRID_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x300a: {'reg': Register.GRID_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x300b: {'reg': Register.GRID_FREQUENCY, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x300c: {'reg': Register.INVERTER_TEMP, 'fmt': '!H', 'eval': 'result-40'}, # noqa: E501 + 0x300e: {'reg': Register.RATED_POWER, 'fmt': '!H', 'ratio': 1}, # noqa: E501 + 0x300f: {'reg': Register.OUTPUT_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3010: {'reg': Register.PV1_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3011: {'reg': Register.PV1_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x3012: {'reg': Register.PV1_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3013: {'reg': Register.PV2_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3014: {'reg': Register.PV2_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x3015: {'reg': Register.PV2_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3016: {'reg': Register.PV3_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3017: {'reg': Register.PV3_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x3018: {'reg': Register.PV3_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x3019: {'reg': Register.PV4_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x301a: {'reg': Register.PV4_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x301b: {'reg': Register.PV4_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 + 0x301c: {'reg': Register.DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x420100fa: {'reg': Register.TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x301f: {'reg': Register.PV1_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x42010100: {'reg': Register.PV1_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3022: {'reg': Register.PV2_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x42010106: {'reg': Register.PV2_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3025: {'reg': Register.PV3_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x4201010c: {'reg': Register.PV3_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3028: {'reg': Register.PV4_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 + 0x42010112: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + } def __init__(self): - self.__build_crc_tab(CRC_POLY) + if not len(self.__crc_tab): + self.__build_crc_tab(CRC_POLY) + self.last_fcode = 0 + self.last_len = 0 + self.last_reg = 0 def build_msg(self, addr, func, reg, val): msg = struct.pack('>BBHH', addr, func, reg, val) msg += struct.pack(' bool: + logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') + if not self.check_crc(buf): + logging.error('Modbus: CRC error') + return False + if buf[0] != self.INV_ADDR: + logging.info(f'Modbus: Wrong addr{buf[0]}') + return False + res = struct.unpack_from('>BHH', buf, 1) + self.last_fcode = res[0] + self.last_reg = res[1] + self.last_len = res[2] + return True + + def recv_resp(self, info_db, buf: bytearray) -> Generator[tuple[str, bool], + None, None]: + logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') + if not self.check_crc(buf): + logging.error('Modbus: CRC error') + return + if buf[0] != self.INV_ADDR: + logging.info(f'Modbus: Wrong addr {buf[0]}') + return + if buf[1] != self.last_fcode: + logging.info(f'Modbus: Wrong fcode {buf[1]} != {self.last_fcode}') + return + elmlen = buf[2] >> 1 + if elmlen != self.last_len: + logging.info(f'Modbus: len error {elmlen} != {self.last_len}') + return + + for i in range(0, elmlen): + val = struct.unpack_from('>H', buf, 3+2*i) + addr = self.last_reg+i + # logging.info(f'Modbus: 0x{addr:04x}: {val[0]}') + if addr in self.map: + row = self.map[addr] + info_id = row['reg'] + result = val[0] + # fmt = row['fmt'] + # res = struct.unpack_from(fmt, buf, addr) + # result = res[0] + + if 'eval' in row: + result = eval(row['eval']) + if 'ratio' in row: + result = round(result * row['ratio'], 2) + + keys, level, unit, must_incr = info_db._key_obj(info_id) + + if keys: + name, update = info_db.update_db(keys, must_incr, result) + yield keys[0], update + else: + name = str(f'info-id.0x{addr:x}') + update = False + + info_db.tracer.log(level, f'GEN3PLUS: {name} : {result}{unit}' + f' update: {update}') + def check_crc(self, msg) -> bool: return 0 == self.__calc_crc(msg) From 5822f5de50feb8aa564b7ca43a8669a8409d6ccc Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:18:19 +0200 Subject: [PATCH 020/118] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ddf9b0..0eede6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- parse Modbus values and store them in the database +- add cron task to request the output power every minute +- GEN3PLUS: add MQTT topics to send AT commands to the inverter +- add MQTT topics to send Modbus commands to the inverter - convert data collect interval to minutes - add postfix for rc and dev versions to the version number - change logging level to DEBUG for some logs From 29ee540a19c3b00814e796a7e7225a71dcff0499 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:18:45 +0200 Subject: [PATCH 021/118] add cron tasks for modbus requests every minute --- app/src/scheduler.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/src/scheduler.py b/app/src/scheduler.py index b5d238d..dc45890 100644 --- a/app/src/scheduler.py +++ b/app/src/scheduler.py @@ -3,6 +3,8 @@ import json from mqtt import Mqtt from aiocron import crontab from infos import ClrAtMidnight +from modbus import Modbus +from messages import Message logger_mqtt = logging.getLogger('mqtt') @@ -17,7 +19,9 @@ class Schedule: cls.mqtt = Mqtt(None) crontab('0 0 * * *', func=cls.atmidnight, start=True) - # crontab('*/5 * * * *', func=cls.atmidnight, start=True) + + # every minute + crontab('* * * * *', func=cls.regular_modbus_cmds, start=True) @classmethod async def atmidnight(cls) -> None: @@ -28,3 +32,12 @@ class Schedule: logger_mqtt.debug(f'{key}: {data}') data_json = json.dumps(data) await cls.mqtt.publish(f"{key}", data_json) + + @classmethod + async def regular_modbus_cmds(cls): + # logging.info("Regular Modbus requests") + for m in Message: + if m.server_side: + fnc = getattr(m, "send_modbus_cmd", None) + if callable(fnc): + await fnc(Modbus.READ_REGS, 0x300e, 2) From bf0f152d5a26ac79e8549fb50d136fd8a0955ce1 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 5 May 2024 20:20:19 +0200 Subject: [PATCH 022/118] add unit tests for modbus --- app/tests/test_modbus.py | 20 ++++++++++++++++++++ app/tests/test_solarman.py | 21 ++++++++++++++------- app/tests/test_talent.py | 4 ++-- system_tests/test_tcp_socket.py | 2 +- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index 0e9cf5b..fcec232 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -1,7 +1,12 @@ # test_with_pytest.py # import pytest, logging from app.src.modbus import Modbus +from app.src.infos import Infos +class TestHelper(Modbus): + def __init__(self): + super().__init__() + self.db = Infos() def test_modbus_crc(): mb = Modbus() @@ -19,3 +24,18 @@ def test_build_modbus_pdu(): assert pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' assert mb.check_crc(pdu) +def test_build_recv(): + mb = TestHelper() + pdu = mb.build_msg(1,3,0x300e,0x2) + assert pdu == b'\x01\x03\x30\x0e\x00\x02\xaa\xc8' + assert mb.check_crc(pdu) + call = 0 + for key, update in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4'): + if key == 'grid': + assert update == True + elif key == 'inverter': + assert update == True + else: + assert False + call += 1 + assert 2 == call diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 48f5509..e699ae7 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -6,6 +6,9 @@ from app.src.gen3plus.solarman_v5 import SolarmanV5 from app.src.config import Config from app.src.infos import Infos, Register + +pytest_plugins = ('pytest_asyncio',) + # initialize the proxy statistics Infos.static_init() @@ -54,6 +57,9 @@ class MemoryStream(SolarmanV5): pass return copied_bytes + async def async_write(self, headline=''): + pass + def _SolarmanV5__flush_recv_msg(self) -> None: super()._SolarmanV5__flush_recv_msg() self.msg_count += 1 @@ -725,7 +731,7 @@ def test_device_rsp(ConfigTsunInv1, DeviceRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==b'' # DeviceRspMsg + assert m._forward_buffer==DeviceRspMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -743,7 +749,7 @@ def test_inverter_rsp(ConfigTsunInv1, InverterRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==b'' # InverterRspMsg + assert m._forward_buffer==InverterRspMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -779,7 +785,7 @@ def test_heartbeat_rsp(ConfigTsunInv1, HeartbeatRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==b'' # HeartbeatRspMsg + assert m._forward_buffer==HeartbeatRspMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -820,7 +826,7 @@ def test_sync_start_rsp(ConfigTsunInv1, SyncStartRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==b'' # HeartbeatRspMsg + assert m._forward_buffer==SyncStartRspMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -856,7 +862,7 @@ def test_sync_end_rsp(ConfigTsunInv1, SyncEndRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==b'' # HeartbeatRspMsg + assert m._forward_buffer==SyncEndRspMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -942,7 +948,8 @@ def test_build_logger_modell(ConfigTsunAllowAll, DeviceIndMsg): assert 'V1.1.00.0B' == m.db.get_db_value(Register.COLLECTOR_FW_VERSION, 0).rstrip('\00') m.close() -def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, AtCommandIndMsg): +@pytest.mark.asyncio +async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, AtCommandIndMsg): ConfigTsunAllowAll m = MemoryStream(DeviceIndMsg, (0,), True) m.read() @@ -954,7 +961,7 @@ def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, AtCommandIndMsg) m._send_buffer = bytearray(0) # clear send buffer for next test m._forward_buffer = bytearray(0) # clear send buffer for next test - m.send_at_cmd('AT+TIME=214028,1,60,120') + await m.send_at_cmd('AT+TIME=214028,1,60,120') assert m._recv_buffer==b'' assert m._send_buffer==AtCommandIndMsg assert m._forward_buffer==b'' diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 7c996ab..4b1de2f 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -795,7 +795,7 @@ def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): m = MemoryStream(MsgModbusRsp, (0,), False) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_rep = False + m.forward_modbus_resp = False m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 @@ -816,7 +816,7 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): m = MemoryStream(MsgModbusRsp, (0,), False) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_rep = True + m.forward_modbus_resp = True m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 diff --git a/system_tests/test_tcp_socket.py b/system_tests/test_tcp_socket.py index 606ea68..f01a0a0 100644 --- a/system_tests/test_tcp_socket.py +++ b/system_tests/test_tcp_socket.py @@ -224,7 +224,7 @@ def test_send_inv_data(ClientConnection, MsgTimeStampReq, MsgTimeStampResp, MsgI data = s.recv(1024) except TimeoutError: pass - # time.sleep(32.5) + time.sleep(32.5) # assert data == MsgTimeStampResp try: s.sendall(MsgInvData) From f804b755a4752732e5a5753002d000707356bfe0 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 6 May 2024 23:18:47 +0200 Subject: [PATCH 023/118] improve modbus trace --- app/src/gen3/talent.py | 2 +- app/src/gen3plus/solarman_v5.py | 5 +++-- app/src/modbus.py | 9 +++++---- app/tests/test_modbus.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 6225aa0..2c70062 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -392,7 +392,7 @@ class Talent(Message): elif self.ctrl.is_ind(): logger.debug(f'Modbus Ind MsgLen: {modbus_len}') for key, update in self.mb.recv_resp(self.db, self._recv_buffer[ - self.header_len + hdr_len:]): + self.header_len + hdr_len:], self.new_data): if update: self.new_data[key] = True diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index a459e89..5805361 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -434,11 +434,12 @@ class SolarmanV5(Message): elif ftype == self.MB_RTU_CMD: valid = data[1] modbus_msg_len = self.data_len - 14 - logger.info(f'modbus_len:{modbus_msg_len} accepted:{valid}') + logger.debug(f'modbus_len:{modbus_msg_len} accepted:{valid}') if valid == 1 and modbus_msg_len > 4: logger.info(f'first byte modbus:{data[14]}') inv_update = False - for key, update in self.mb.recv_resp(self.db, data[14:-2]): + for key, update in self.mb.recv_resp(self.db, data[14:-2], + self.node_id): if update: if key == 'inverter': inv_update = True diff --git a/app/src/modbus.py b/app/src/modbus.py index 0f16705..37eadff 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -93,8 +93,8 @@ class Modbus(): self.last_len = res[2] return True - def recv_resp(self, info_db, buf: bytearray) -> Generator[tuple[str, bool], - None, None]: + def recv_resp(self, info_db, buf: bytearray, node_id: str) -> \ + Generator[tuple[str, bool], None, None]: logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): logging.error('Modbus: CRC error') @@ -136,8 +136,9 @@ class Modbus(): name = str(f'info-id.0x{addr:x}') update = False - info_db.tracer.log(level, f'GEN3PLUS: {name} : {result}{unit}' - f' update: {update}') + info_db.tracer.log(level, + f'MODBUS({node_id}): {name} : {result}' + f'{unit} update: {update}') def check_crc(self, msg) -> bool: return 0 == self.__calc_crc(msg) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index fcec232..b1764e9 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -30,7 +30,7 @@ def test_build_recv(): assert pdu == b'\x01\x03\x30\x0e\x00\x02\xaa\xc8' assert mb.check_crc(pdu) call = 0 - for key, update in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4'): + for key, update in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): if key == 'grid': assert update == True elif key == 'inverter': From 54d2bf4439eb5b9ae9e5df12397cfdd520ae560f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 17:52:51 +0200 Subject: [PATCH 024/118] set err value for unit tests --- app/src/modbus.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 37eadff..e8a8d0e 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -32,12 +32,13 @@ class Modbus(): __crc_tab = [] map = { 0x2007: {'reg': Register.MAX_DESIGNED_POWER, 'fmt': '!H', 'ratio': 1}, # noqa: E501 - 0x420100c0: {'reg': Register.INVERTER_STATUS, 'fmt': '!H'}, # noqa: E501 + # 0x????: {'reg': Register.INVERTER_STATUS, 'fmt': '!H'}, # noqa: E501 0x3008: {'reg': Register.VERSION, 'fmt': '!H', 'eval': "f'v{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 0x3009: {'reg': Register.GRID_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x300a: {'reg': Register.GRID_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x300b: {'reg': Register.GRID_FREQUENCY, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x300c: {'reg': Register.INVERTER_TEMP, 'fmt': '!H', 'eval': 'result-40'}, # noqa: E501 + # 0x300d 0x300e: {'reg': Register.RATED_POWER, 'fmt': '!H', 'ratio': 1}, # noqa: E501 0x300f: {'reg': Register.OUTPUT_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x3010: {'reg': Register.PV1_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 @@ -53,15 +54,15 @@ class Modbus(): 0x301a: {'reg': Register.PV4_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x301b: {'reg': Register.PV4_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x301c: {'reg': Register.DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - 0x420100fa: {'reg': Register.TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + # 0x301d: {'reg': Register.TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x301f: {'reg': Register.PV1_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - 0x42010100: {'reg': Register.PV1_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + # 0x3020: {'reg': Register.PV1_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3022: {'reg': Register.PV2_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - 0x42010106: {'reg': Register.PV2_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + # 0x3023: {'reg': Register.PV2_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3025: {'reg': Register.PV3_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - 0x4201010c: {'reg': Register.PV3_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + # 0x3026: {'reg': Register.PV3_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3028: {'reg': Register.PV4_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - 0x42010112: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + # 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } def __init__(self): @@ -70,6 +71,7 @@ class Modbus(): self.last_fcode = 0 self.last_len = 0 self.last_reg = 0 + self.err = 0 def build_msg(self, addr, func, reg, val): msg = struct.pack('>BBHH', addr, func, reg, val) @@ -77,38 +79,47 @@ class Modbus(): self.last_fcode = func self.last_reg = reg self.last_len = val + self.err = 0 return msg def recv_req(self, buf: bytearray) -> bool: - logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') + # logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): + self.err = 1 logging.error('Modbus: CRC error') return False if buf[0] != self.INV_ADDR: + self.err = 2 logging.info(f'Modbus: Wrong addr{buf[0]}') return False res = struct.unpack_from('>BHH', buf, 1) self.last_fcode = res[0] self.last_reg = res[1] self.last_len = res[2] + self.err = 0 return True def recv_resp(self, info_db, buf: bytearray, node_id: str) -> \ Generator[tuple[str, bool], None, None]: - logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') + # logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): logging.error('Modbus: CRC error') + self.err = 1 return if buf[0] != self.INV_ADDR: logging.info(f'Modbus: Wrong addr {buf[0]}') + self.err = 2 return if buf[1] != self.last_fcode: logging.info(f'Modbus: Wrong fcode {buf[1]} != {self.last_fcode}') + self.err = 3 return elmlen = buf[2] >> 1 if elmlen != self.last_len: logging.info(f'Modbus: len error {elmlen} != {self.last_len}') + self.err = 4 return + self.err = 0 for i in range(0, elmlen): val = struct.unpack_from('>H', buf, 3+2*i) From d5010fe053894f55ed903eaa6e10593fe81124f2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 17:56:54 +0200 Subject: [PATCH 025/118] parse modbus corect if we have received more than one message --- app/src/gen3/talent.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 2c70062..e6d3b67 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -55,6 +55,7 @@ class Talent(Message): # 0x78: 0x04: self.msg_inverter_data, } + self.modbus_elms = 0 # for unit tests ''' Our puplic methods @@ -377,10 +378,8 @@ class Talent(Message): result = struct.unpack_from('!lBB', self._recv_buffer, self.header_len) modbus_len = result[1] - logger.debug(f'Ref: {result[0]}') - logger.debug(f'Modbus MsgLen: {modbus_len} Func:{result[2]}') - # logger.info(f'time: {datetime.utcfromtimestamp(result[2]).strftime( - # "%Y-%m-%d %H:%M:%S")}') + # logger.debug(f'Ref: {result[0]}') + # logger.debug(f'Modbus MsgLen: {modbus_len} Func:{result[2]}') return msg_hdr_len, modbus_len def msg_modbus(self): @@ -390,11 +389,14 @@ class Talent(Message): self.forward_modbus_resp = True self.inc_counter('Modbus_Command') elif self.ctrl.is_ind(): - logger.debug(f'Modbus Ind MsgLen: {modbus_len}') + # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') + self.modbus_elms = 0 for key, update in self.mb.recv_resp(self.db, self._recv_buffer[ - self.header_len + hdr_len:], self.new_data): + self.header_len + hdr_len:self.header_len+self.data_len], + self.new_data): if update: self.new_data[key] = True + self.modbus_elms += 1 if not self.forward_modbus_resp: return From 39beb0cb44136694c37c114cb270eba2a37e5a9d Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 18:02:09 +0200 Subject: [PATCH 026/118] add more modbus tests --- app/tests/test_talent.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 4b1de2f..cc9ab85 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -200,6 +200,15 @@ def MsgModbusInv(): msg += b'\x00\x00\x03\xc8' return msg +@pytest.fixture +def MsgModbusResp20(): + msg = b'\x00\x00\x00\x45\x10R170000000000001' + msg += b'\x91\x77\x17\x18\x19\x1a\x2d\x01\x03\x28\x51' + msg += b'\x09\x08\xd3\x00\x29\x13\x87\x00\x3e\x00\x00\x01\x2c\x03\xb4\x00' + msg += b'\x08\x00\x00\x00\x00\x01\x59\x01\x21\x03\xe6\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\xdb\x6b' + return msg + def test_read_message(MsgContactInfo): m = MemoryStream(MsgContactInfo, (0,)) m.read() # read complete msg, and dispatch msg @@ -852,6 +861,33 @@ def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInv): assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() +def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): + ConfigTsunInv1 + # receive more bytes than expected (7 bytes from the next msg) + m = MemoryStream(MsgModbusResp20+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_resp = True + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==0x91 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==50 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer==b'' + assert m.mb.err == 0 + assert m.modbus_elms == 20-1 # register 0x300d is unknown, so one value can't be mapped + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + @pytest.mark.asyncio async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 From 02d9f01947f4eea7fbfe2054bb3da8ce1bb08f4c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 18:32:56 +0200 Subject: [PATCH 027/118] don't send AT or Modbus cmds on closed connections --- app/src/gen3/talent.py | 2 ++ app/src/gen3plus/solarman_v5.py | 6 ++++-- app/src/mqtt.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index e6d3b67..1534446 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -45,6 +45,7 @@ class Talent(Message): self.db = InfosG3() self.mb = Modbus() self.forward_modbus_resp = False + self.closed = False self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, @@ -66,6 +67,7 @@ class Talent(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() + self.closed = True def __set_serial_no(self, serial_no: str): diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 5805361..126e06e 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -62,6 +62,7 @@ class SolarmanV5(Message): self.time_ofs = 0 self.mb = Modbus() self.forward_modbus_resp = False + self.closed = False self.switch = { 0x4210: self.msg_data_ind, # real time data @@ -102,6 +103,7 @@ class SolarmanV5(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() + self.closed = True def __set_serial_no(self, snr: int): serial_no = str(snr) @@ -434,9 +436,9 @@ class SolarmanV5(Message): elif ftype == self.MB_RTU_CMD: valid = data[1] modbus_msg_len = self.data_len - 14 - logger.debug(f'modbus_len:{modbus_msg_len} accepted:{valid}') + # logger.debug(f'modbus_len:{modbus_msg_len} accepted:{valid}') if valid == 1 and modbus_msg_len > 4: - logger.info(f'first byte modbus:{data[14]}') + # logger.info(f'first byte modbus:{data[14]}') inv_update = False for key, update in self.mb.recv_resp(self.db, data[14:-2], self.node_id): diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 7257038..3469201 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -130,7 +130,7 @@ class Mqtt(metaclass=Singleton): topic = str(message.topic) node_id = topic.split('/')[1] + '/' for m in Message: - if m.server_side and m.node_id == node_id: + if m.server_side and not m.closed and (m.node_id == node_id): logger_mqtt.debug(f'Found: {node_id}') fnc = getattr(m, func_name, None) if callable(fnc): @@ -148,7 +148,7 @@ class Mqtt(metaclass=Singleton): payload = message.payload.decode("UTF-8") logger_mqtt.info(f'InvCnf: {node_id}:{payload}') for m in Message: - if m.server_side and m.node_id == node_id: + if m.server_side and not m.closed and (m.node_id == node_id): logger_mqtt.info(f'Found: {node_id}') fnc = getattr(m, "send_modbus_cmd", None) res = payload.split(',') From e15387b1ff7dfff1afce02532fa7377b2f52ab91 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 19:41:07 +0200 Subject: [PATCH 028/118] fix modbus trace --- app/src/gen3/talent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 1534446..45eb081 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -395,7 +395,7 @@ class Talent(Message): self.modbus_elms = 0 for key, update in self.mb.recv_resp(self.db, self._recv_buffer[ self.header_len + hdr_len:self.header_len+self.data_len], - self.new_data): + self.node_id): if update: self.new_data[key] = True self.modbus_elms += 1 From 3fd528bdbee1fd63b6c80a8524e28b5237c438a7 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 21:20:12 +0200 Subject: [PATCH 029/118] improve logging --- app/src/gen3/infos_g3.py | 4 ++-- app/src/gen3plus/infos_g3p.py | 4 ++-- app/src/infos.py | 2 +- app/src/modbus.py | 8 ++++---- app/src/scheduler.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/src/gen3/infos_g3.py b/app/src/gen3/infos_g3.py index 7e45634..d3fb987 100644 --- a/app/src/gen3/infos_g3.py +++ b/app/src/gen3/infos_g3.py @@ -161,7 +161,7 @@ class InfosG3(Infos): update = False name = str(f'info-id.0x{addr:x}') - self.tracer.log(level, f'GEN3: {name} : {result}{unit}' - f' update: {update}') + if update: + self.tracer.log(level, f'GEN3: {name} : {result}{unit}') i += 1 diff --git a/app/src/gen3plus/infos_g3p.py b/app/src/gen3plus/infos_g3p.py index b0adc0a..ed8d9bd 100644 --- a/app/src/gen3plus/infos_g3p.py +++ b/app/src/gen3plus/infos_g3p.py @@ -122,5 +122,5 @@ class InfosG3P(Infos): name = str(f'info-id.0x{addr:x}') update = False - self.tracer.log(level, f'GEN3PLUS: {name} : {result}{unit}' - f' update: {update}') + if update: + self.tracer.log(level, f'GEN3PLUS: {name} : {result}{unit}') diff --git a/app/src/infos.py b/app/src/infos.py index e9e8ebe..dadacd7 100644 --- a/app/src/infos.py +++ b/app/src/infos.py @@ -193,7 +193,7 @@ class Infos: Register.SERIAL_NUMBER: {'name': ['inverter', 'Serial_Number'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 Register.EQUIPMENT_MODEL: {'name': ['inverter', 'Equipment_Model'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 Register.NO_INPUTS: {'name': ['inverter', 'No_Inputs'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.MAX_DESIGNED_POWER: {'name': ['inverter', 'Max_Designed_Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'inverter', 'dev_cla': None, 'stat_cla': None, 'id': 'designed_power_', 'fmt': '| string + " W"', 'name': 'Max Designed Power', 'icon': 'mdi:lightning-bolt', 'ent_cat': 'diagnostic'}}, # noqa: E501 + Register.MAX_DESIGNED_POWER: {'name': ['inverter', 'Max_Designed_Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'inverter', 'dev_cla': None, 'stat_cla': None, 'id': 'designed_power_', 'fmt': '| string + " W"', 'name': 'Max Designed Power', 'icon': 'mdi:lightning-bolt', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.RATED_POWER: {'name': ['inverter', 'Rated_Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'inverter', 'dev_cla': None, 'stat_cla': None, 'id': 'rated_power_', 'fmt': '| string + " W"', 'name': 'Rated Power', 'icon': 'mdi:lightning-bolt', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV1_MANUFACTURER: {'name': ['inverter', 'PV1_Manufacturer'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 diff --git a/app/src/modbus.py b/app/src/modbus.py index e8a8d0e..25c5734 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -146,10 +146,10 @@ class Modbus(): else: name = str(f'info-id.0x{addr:x}') update = False - - info_db.tracer.log(level, - f'MODBUS({node_id}): {name} : {result}' - f'{unit} update: {update}') + if update: + info_db.tracer.log(level, + f'MODBUS[{node_id}]: {name} : {result}' + f'{unit}') def check_crc(self, msg) -> bool: return 0 == self.__calc_crc(msg) diff --git a/app/src/scheduler.py b/app/src/scheduler.py index dc45890..a1e763b 100644 --- a/app/src/scheduler.py +++ b/app/src/scheduler.py @@ -40,4 +40,4 @@ class Schedule: if m.server_side: fnc = getattr(m, "send_modbus_cmd", None) if callable(fnc): - await fnc(Modbus.READ_REGS, 0x300e, 2) + await fnc(Modbus.READ_REGS, 0x3008, 20) From 2301511242380ea365494d0c351dab760abc6a4e Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 22:11:55 +0200 Subject: [PATCH 030/118] update documentation --- CHANGELOG.md | 1 + README.md | 9 +- app/proxy.svg | 466 ++++++++++++++++++++++++++----------------------- app/proxy.yuml | 8 +- 4 files changed, 255 insertions(+), 229 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eede6e..0ec1d47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- add timeout monitoring for received packets - parse Modbus values and store them in the database - add cron task to request the output power every minute - GEN3PLUS: add MQTT topics to send AT commands to the inverter diff --git a/README.md b/README.md index 1a26179..342a643 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,15 @@ If you use a Pi-hole, you can also store the host entry in the Pi-hole. ## Features -- supports TSUN GEN3 PLUS inverters: TSOL-MS2000, MS1800 and MS1600 -- supports TSUN GEN3 inverters: TSOL-MS800, MS700, MS600, MS400, MS350 and MS300 +- Supports TSUN GEN3 PLUS inverters: TSOL-MS2000, MS1800 and MS1600 +- Supports TSUN GEN3 inverters: TSOL-MS800, MS700, MS600, MS400, MS350 and MS300 - `MQTT` support - `Home-Assistant` auto-discovery support +- `MODBUS` support via MQTT topics +- `AT Command` support via MQTT topics (GEN3PLUS only) +- Faster DataUp interval sends measurement data to the MQTT broker every minute - Self-sufficient island operation without internet -- runs in a non-root Docker Container +- Runs in a non-root Docker Container ## Home Assistant Screenshots diff --git a/app/proxy.svg b/app/proxy.svg index dfbdd46..cef1e69 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,352 +4,372 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -Singleton + +Singleton A2 - -Mqtt - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() + +Mqtt + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() A1->A2 - - - - - -A3 - -Modbus - - - -A1->A3 - - + + A11 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt - + +Inverter + +cls.db_stat +cls.entity_prfx +cls.discovery_prfx +cls.proxy_node_id +cls.proxy_unique_id +cls.mqtt:Mqtt + - + A2->A11 - + + + + +A3 + +Modbus + + +build_msg() +recv_req() +recv_resp() +check_crc() A4 - -IterRegistry - - -__iter__ + +IterRegistry + + +__iter__ A5 - -Message - -server_side:bool -header_valid:bool -header_len:unsigned -data_len:unsigned -unique_id -node_id -sug_area -_recv_buffer:bytearray -_send_buffer:bytearray -_forward_buffer:bytearray -db:Infos -new_data:list - -_read():void<abstract> -close():void -inc_counter():void -dec_counter():void + +Message + +server_side:bool +header_valid:bool +header_len:unsigned +data_len:unsigned +unique_id +node_id +sug_area +_recv_buffer:bytearray +_send_buffer:bytearray +_forward_buffer:bytearray +db:Infos +new_data:list + +_read():void<abstract> +close():void +inc_counter():void +dec_counter():void - + A4->A5 - - + + A6 - -Talent - -await_conn_resp_cnt -id_str -contact_name -contact_mail -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -close() + +Talent + +await_conn_resp_cnt +id_str +contact_name +contact_mail +db:InfosG3 +mb:Modbus +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +close() - + A5->A6 - - + + A7 - -SolarmanV5 - -control -serial -snr -switch - -msg_unknown() -close() + +SolarmanV5 + +control +serial +snr +db:InfosG3P +mb:Modbus +switch + +msg_unknown() +close() - + A5->A7 - - + + + + + +A6->A3 + + +1 +has A8 - -ConnectionG3 - -remoteStream:ConnectionG3 - -close() + +ConnectionG3 + +remoteStream:ConnectionG3 + +close() - + A6->A8 - - + + + + + +A7->A3 + + +1 +has A9 - -ConnectionG3P - -remoteStream:ConnectionG3P - -close() + +ConnectionG3P + +remoteStream:ConnectionG3P + +close() A7->A9 - - + + - + A8->A8 - - -0..1 -has + + +0..1 +has A12 - -InverterG3 - -__ha_restarts - -async_create_remote() -close() + +InverterG3 + +__ha_restarts + +async_create_remote() +close() - + A8->A12 - - + + - + A9->A9 - - -0..1 -has + + +0..1 +has A13 - -InverterG3P - -__ha_restarts - -async_create_remote() -close() + +InverterG3P + +__ha_restarts + +async_create_remote() +close() - + A9->A13 - - + + A10 - -AsyncStream - -reader -writer -addr -r_addr -l_addr - -<async>server_loop() -<async>client_loop() -<async>loop -disc() -close() -__async_read() -async_write() -__async_forward() + +AsyncStream + +reader +writer +addr +r_addr +l_addr + +<async>server_loop() +<async>client_loop() +<async>loop +disc() +close() +__async_read() +async_write() +__async_forward() - + A10->A8 - - + + - + A10->A9 - - + + - + A11->A12 - - + + - + A11->A13 - - + + A14 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -update_db -set_db_def_value -get_db_value -ignore_this_device + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +update_db +set_db_def_value +get_db_value +ignore_this_device A15 - -InfosG3 - - -ha_confs() -parse() + +InfosG3 + + +ha_confs() +parse() - + A14->A15 - - + + A16 - -InfosG3P - - -ha_confs() -parse() + +InfosG3P + + +ha_confs() +parse() - + A14->A16 - - + + - + A15->A6 - - + + - + A16->A7 - - + + diff --git a/app/proxy.yuml b/app/proxy.yuml index daf5d3c..7514a93 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -4,12 +4,14 @@ [note: You can stick notes on diagrams too!{bg:cornsilk}] [Singleton]^[Mqtt|ha_restarts;__client;__cb_MqttIsUp|publish();close()] -[Singleton]^[Modbus] +[Modbus||build_msg();recv_req();recv_resp();check_crc()] [IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list|_read():void;close():void;inc_counter():void;dec_counter():void] -[Message]^[Talent|await_conn_resp_cnt;id_str;contact_name;contact_mail;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;close()] -[Message]^[SolarmanV5|control;serial;snr;switch|msg_unknown();;close()] +[Message]^[Talent|await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;close()] +[Message]^[SolarmanV5|control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;close()] [Talent]^[ConnectionG3|remoteStream:ConnectionG3|close()] +[Talent]has-1>[Modbus] [SolarmanV5]^[ConnectionG3P|remoteStream:ConnectionG3P|close()] +[SolarmanV5]has-1>[Modbus] [AsyncStream|reader;writer;addr;r_addr;l_addr|server_loop();client_loop();loop;disc();close();;__async_read();async_write();__async_forward()]^[ConnectionG3] [AsyncStream]^[ConnectionG3P] [Inverter|cls.db_stat;cls.entity_prfx;cls.discovery_prfx;cls.proxy_node_id;cls.proxy_unique_id;cls.mqtt:Mqtt|]^[InverterG3|__ha_restarts|async_create_remote();;close()] From eab109ddab4d51668266358ef514e1882a0705ad Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 7 May 2024 22:37:17 +0200 Subject: [PATCH 031/118] install pytest-asyncio --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 8f4f09a..518a148 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -37,7 +37,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest + pip install flake8 pytest pytest-asyncio if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | From 5fc1b16627facdb2ff225c4a72ee1f5896e904cb Mon Sep 17 00:00:00 2001 From: Stefan Allius <122395479+s-allius@users.noreply.github.com> Date: Tue, 7 May 2024 22:52:20 +0200 Subject: [PATCH 032/118] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 342a643..27b84c7 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you use a Pi-hole, you can also store the host entry in the Pi-hole. - `MQTT` support - `Home-Assistant` auto-discovery support - `MODBUS` support via MQTT topics -- `AT Command` support via MQTT topics (GEN3PLUS only) +- `AT-Command` support via MQTT topics (GEN3PLUS only) - Faster DataUp interval sends measurement data to the MQTT broker every minute - Self-sufficient island operation without internet - Runs in a non-root Docker Container From 0ae6dffc6bd0c0a99545dcefacc8a5ead3d14000 Mon Sep 17 00:00:00 2001 From: Stefan Allius <122395479+s-allius@users.noreply.github.com> Date: Tue, 7 May 2024 22:54:23 +0200 Subject: [PATCH 033/118] Update test_talent.py --- app/tests/test_talent.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index cc9ab85..a94df4f 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -876,12 +876,12 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): assert m.msg_count == 1 assert m.id_str == b"R170000000000001" assert m.unique_id == 'R170000000000001' - assert int(m.ctrl)==0x91 - assert m.msg_id==119 - assert m.header_len==23 - assert m.data_len==50 + assert int(m.ctrl) == 0x91 + assert m.msg_id == 119 + assert m.header_len == 23 + assert m.data_len == 50 assert m._forward_buffer==MsgModbusResp20 - assert m._send_buffer==b'' + assert m._send_buffer == b'' assert m.mb.err == 0 assert m.modbus_elms == 20-1 # register 0x300d is unknown, so one value can't be mapped assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 @@ -894,7 +894,7 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): m = MemoryStream(b'', (0,), False) m.id_str = b"R170000000000001" await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) - assert 0==m.send_msg_ofs - assert m._forward_buffer==b'' - assert m._send_buffer==MsgModbusCmd + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == MsgModbusCmd m.close() From 2d176894d356bbf3cb428ca34a248b518a324a87 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 8 May 2024 23:46:24 +0200 Subject: [PATCH 034/118] remove unneeded sleep() call --- system_tests/test_tcp_socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/test_tcp_socket.py b/system_tests/test_tcp_socket.py index f01a0a0..e2b64f8 100644 --- a/system_tests/test_tcp_socket.py +++ b/system_tests/test_tcp_socket.py @@ -224,7 +224,7 @@ def test_send_inv_data(ClientConnection, MsgTimeStampReq, MsgTimeStampResp, MsgI data = s.recv(1024) except TimeoutError: pass - time.sleep(32.5) + # time.sleep(32.5) # assert data == MsgTimeStampResp try: s.sendall(MsgInvData) From 2ec0a59cd3f0738cf863cd66fc9cadb1c124e9ff Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 8 May 2024 23:48:41 +0200 Subject: [PATCH 035/118] add modbus long int support --- app/src/gen3/talent.py | 2 +- app/src/gen3plus/solarman_v5.py | 4 ++-- app/src/modbus.py | 30 ++++++++++++------------------ 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 45eb081..33d5d8d 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -393,7 +393,7 @@ class Talent(Message): elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 - for key, update in self.mb.recv_resp(self.db, self._recv_buffer[ + for key, update, _ in self.mb.recv_resp(self.db, self._recv_buffer[ self.header_len + hdr_len:self.header_len+self.data_len], self.node_id): if update: diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 126e06e..b175cb3 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -440,8 +440,8 @@ class SolarmanV5(Message): if valid == 1 and modbus_msg_len > 4: # logger.info(f'first byte modbus:{data[14]}') inv_update = False - for key, update in self.mb.recv_resp(self.db, data[14:-2], - self.node_id): + for key, update, _ in self.mb.recv_resp(self.db, data[14:-2], + self.node_id): if update: if key == 'inverter': inv_update = True diff --git a/app/src/modbus.py b/app/src/modbus.py index 25c5734..c69dbb9 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -54,15 +54,15 @@ class Modbus(): 0x301a: {'reg': Register.PV4_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x301b: {'reg': Register.PV4_POWER, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x301c: {'reg': Register.DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - # 0x301d: {'reg': Register.TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x301d: {'reg': Register.TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x301f: {'reg': Register.PV1_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - # 0x3020: {'reg': Register.PV1_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3020: {'reg': Register.PV1_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3022: {'reg': Register.PV2_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - # 0x3023: {'reg': Register.PV2_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3023: {'reg': Register.PV2_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3025: {'reg': Register.PV3_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - # 0x3026: {'reg': Register.PV3_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3026: {'reg': Register.PV3_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 0x3028: {'reg': Register.PV4_DAILY_GENERATION, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 - # 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 + 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } def __init__(self): @@ -122,16 +122,13 @@ class Modbus(): self.err = 0 for i in range(0, elmlen): - val = struct.unpack_from('>H', buf, 3+2*i) addr = self.last_reg+i - # logging.info(f'Modbus: 0x{addr:04x}: {val[0]}') if addr in self.map: row = self.map[addr] info_id = row['reg'] + fmt = row['fmt'] + val = struct.unpack_from(fmt, buf, 3+2*i) result = val[0] - # fmt = row['fmt'] - # res = struct.unpack_from(fmt, buf, addr) - # result = res[0] if 'eval' in row: result = eval(row['eval']) @@ -142,14 +139,11 @@ class Modbus(): if keys: name, update = info_db.update_db(keys, must_incr, result) - yield keys[0], update - else: - name = str(f'info-id.0x{addr:x}') - update = False - if update: - info_db.tracer.log(level, - f'MODBUS[{node_id}]: {name} : {result}' - f'{unit}') + yield keys[0], update, result + if update: + info_db.tracer.log(level, + f'MODBUS[{node_id}]: {name}' + f' : {result}{unit}') def check_crc(self, msg) -> bool: return 0 == self.__calc_crc(msg) From 0ac4b1f5715980a21a2c210cff2e7f023329e018 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 8 May 2024 23:50:04 +0200 Subject: [PATCH 036/118] add more unit tests --- app/tests/test_modbus.py | 121 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index b1764e9..a3d24ac 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -18,24 +18,137 @@ def test_modbus_crc(): assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') assert mb.check_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') + assert 0x5c75 == mb._Modbus__calc_crc(b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46') def test_build_modbus_pdu(): mb = Modbus() pdu = mb.build_msg(1,6,0x2000,0x12) assert pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' assert mb.check_crc(pdu) +def test_recv_req_crc(): + mb = Modbus() + res = mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') + assert not res + assert mb.last_fcode == 0 + assert mb.last_reg == 0 + assert mb.last_len == 0 + assert mb.err == 1 + +def test_recv_req_addr(): + mb = Modbus() + res = mb.recv_req(b'\x02\x06\x20\x00\x00\x12\x02\x34') + assert not res + assert mb.last_fcode == 0 + assert mb.last_reg == 0 + assert mb.last_len == 0 + assert mb.err == 2 + +def test_recv_req(): + mb = Modbus() + res = mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') + assert res + assert mb.last_fcode == 6 + assert mb.last_reg == 0x2000 + assert mb.last_len == 0x12 + assert mb.err == 0 + +def test_recv_recv_crc(): + mb = TestHelper() + mb.last_fcode = 3 + mb.last_reg == 0x300e + mb.last_len == 2 + + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf3', 'test'): + call += 1 + assert mb.err == 1 + assert 0 == call + +def test_recv_recv_addr(): + mb = TestHelper() + mb.last_fcode = 3 + mb.last_reg == 0x300e + mb.last_len == 2 + + call = 0 + for key, update in mb.recv_resp(mb.db, b'\x02\x03\x04\x01\x2c\x00\x46\x88\xf4', 'test'): + call += 1 + assert mb.err == 2 + assert 0 == call + +def test_recv_recv_fcode(): + mb = TestHelper() + mb.last_fcode = 4 + mb.last_reg == 0x300e + mb.last_len == 2 + + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): + call += 1 + assert mb.err == 3 + assert 0 == call + +def test_recv_recv_len(): + mb = TestHelper() + mb.last_fcode = 3 + mb.last_reg == 0x300e + mb.last_len == 2 + + call = 0 + for key, update, _ in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): + call += 1 + assert mb.err == 4 + assert 0 == call + def test_build_recv(): mb = TestHelper() - pdu = mb.build_msg(1,3,0x300e,0x2) - assert pdu == b'\x01\x03\x30\x0e\x00\x02\xaa\xc8' + pdu = mb.build_msg(1,3,0x3007,6) assert mb.check_crc(pdu) + assert mb.err == 0 call = 0 - for key, update in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): + exp_result = ['v0.0.212', 4.4, 0.7, 0.7, 30] + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): if key == 'grid': assert update == True elif key == 'inverter': assert update == True + elif key == 'env': + assert update == True + else: + assert False + assert exp_result[call] == val + call += 1 + assert 0 == mb.err + assert 5 == call + + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): + if key == 'grid': + assert update == False + elif key == 'inverter': + assert update == False + elif key == 'env': + assert update == False + else: + assert False + assert exp_result[call] == val + call += 1 + assert 0 == mb.err + assert 5 == call + +def test_build_long(): + mb = TestHelper() + pdu = mb.build_msg(1,3,0x3022,4) + assert mb.check_crc(pdu) + assert mb.err == 0 + call = 0 + exp_result = [3.0, 28841.4, 113.34] + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46\x75\x5c', 'test'): + if key == 'input': + assert update == True + assert exp_result[call] == val else: assert False call += 1 - assert 2 == call + assert 0 == mb.err + assert 3 == call From 91873d0c340fa7e70491a36a50788124d4b480ba Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 8 May 2024 23:52:31 +0200 Subject: [PATCH 037/118] await wait_closed() on disconnects --- app/src/async_stream.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 28873e8..ac6c54f 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -1,5 +1,6 @@ import logging import traceback +import asyncio from messages import hex_dump_memory logger = logging.getLogger('conn') @@ -27,7 +28,7 @@ class AsyncStream(): # the connection to te TSUN cloud if self.remoteStream: logging.debug("disconnect client connection") - self.remoteStream.disc() + await self.remoteStream.disc() try: await self._async_publ_mqtt_proxy_stat('proxy') except Exception: @@ -58,6 +59,7 @@ class AsyncStream(): while True: try: + # await asyncio.wait_for(self.__async_read(), 0.3) await self.__async_read() if self.unique_id: @@ -65,25 +67,32 @@ class AsyncStream(): await self.__async_forward() await self.async_publ_mqtt() + except asyncio.TimeoutError: + pass + except (ConnectionResetError, ConnectionAbortedError, - BrokenPipeError, - RuntimeError) as error: - logger.warning(f'In loop for l{self.l_addr} | ' - f'r{self.r_addr}: {error}') - self.close() + BrokenPipeError) as error: + logger.error(f'{error} for l{self.l_addr} | ' + f'r{self.r_addr}') + await self.disc() return self + + except RuntimeError as error: + logger.warning(f"{error} for {self.l_addr}") + await self.disc() + return self + except Exception: self.inc_counter('SW_Exception') logger.error( f"Exception for {self.addr}:\n" f"{traceback.format_exc()}") - self.close() - return self - def disc(self) -> None: + async def disc(self) -> None: logger.debug(f'AsyncStream.disc() l{self.l_addr} | r{self.r_addr}') self.writer.close() + await self.writer.wait_closed() def close(self): logger.debug(f'AsyncStream.close() l{self.l_addr} | r{self.r_addr}') From a869ead89a38598de6181606d108c27e7bcddade Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 14:16:15 +0200 Subject: [PATCH 038/118] add MAX_DESIGNED_POWER (only readable by Modbus) --- app/src/gen3/infos_g3.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/gen3/infos_g3.py b/app/src/gen3/infos_g3.py index d3fb987..1de9bf0 100644 --- a/app/src/gen3/infos_g3.py +++ b/app/src/gen3/infos_g3.py @@ -30,6 +30,7 @@ class RegisterMap: 0xffffff05: Register.UNKNOWN_CTRL, 0xffffff06: Register.OTA_START_MSG, 0xffffff07: Register.SW_EXCEPTION, + 0xffffff08: Register.MAX_DESIGNED_POWER, 0xfffffffe: Register.TEST_REG1, 0xffffffff: Register.TEST_REG2, 0x00000640: Register.OUTPUT_POWER, @@ -104,7 +105,8 @@ class InfosG3(Infos): if res: yield res - def parse(self, buf, ind=0) -> Generator[tuple[str, bool], None, None]: + def parse(self, buf, ind=0, node_id: str = '') -> \ + Generator[tuple[str, bool], None, None]: '''parse a data sequence received from the inverter and stores the values in Infos.db @@ -162,6 +164,7 @@ class InfosG3(Infos): name = str(f'info-id.0x{addr:x}') if update: - self.tracer.log(level, f'GEN3: {name} : {result}{unit}') + self.tracer.log(level, f'[\'{node_id}\']GEN3: {name} :' + f' {result}{unit}') i += 1 From 41d9a2a1ef3040b4805b053db625a1a37007865f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 14:19:37 +0200 Subject: [PATCH 039/118] improve logger --- app/src/gen3plus/infos_g3p.py | 5 +++-- app/src/modbus.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/src/gen3plus/infos_g3p.py b/app/src/gen3plus/infos_g3p.py index ed8d9bd..a436dbf 100644 --- a/app/src/gen3plus/infos_g3p.py +++ b/app/src/gen3plus/infos_g3p.py @@ -88,7 +88,7 @@ class InfosG3P(Infos): if res: yield res - def parse(self, buf, msg_type: int, rcv_ftype: int) \ + def parse(self, buf, msg_type: int, rcv_ftype: int, node_id: str = '') \ -> Generator[tuple[str, bool], None, None]: '''parse a data sequence received from the inverter and stores the values in Infos.db @@ -123,4 +123,5 @@ class InfosG3P(Infos): update = False if update: - self.tracer.log(level, f'GEN3PLUS: {name} : {result}{unit}') + self.tracer.log(level, f'[\'{node_id}\']GEN3PLUS: {name}' + f' : {result}{unit}') diff --git a/app/src/modbus.py b/app/src/modbus.py index c69dbb9..0047afc 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -142,7 +142,7 @@ class Modbus(): yield keys[0], update, result if update: info_db.tracer.log(level, - f'MODBUS[{node_id}]: {name}' + f'[\'{node_id}\']MODBUS: {name}' f' : {result}{unit}') def check_crc(self, msg) -> bool: From 5a0456650f6144f2330335efb7e89e9b22e61793 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 14:20:57 +0200 Subject: [PATCH 040/118] avoid sending modbus cmds in critical states --- app/src/gen3/talent.py | 12 +++++++++--- app/src/gen3plus/solarman_v5.py | 7 ++++++- app/src/mqtt.py | 4 ++-- app/tests/test_talent.py | 9 ++++++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 33d5d8d..daf700f 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -35,6 +35,9 @@ class Control: class Talent(Message): + STATE_INIT = 0 + STATE_UP = 2 + STATE_CLOSED = 3 def __init__(self, server_side: bool, id_str=b''): super().__init__(server_side) @@ -45,7 +48,7 @@ class Talent(Message): self.db = InfosG3() self.mb = Modbus() self.forward_modbus_resp = False - self.closed = False + self.state = self.STATE_INIT self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, @@ -67,7 +70,7 @@ class Talent(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() - self.closed = True + self.state = self.STATE_CLOSED def __set_serial_no(self, serial_no: str): @@ -126,6 +129,8 @@ class Talent(Message): return async def send_modbus_cmd(self, func, addr, val) -> None: + if self.state != self.STATE_UP: + return self.forward_modbus_resp = False self.__build_header(0x70, 0x77) self._send_buffer += b'\x00\x01\xa3\x28' # fixme @@ -331,6 +336,7 @@ class Talent(Message): self._send_buffer += b'\x01' self.__finish_send_msg() self.__process_data() + self.state = self.STATE_UP elif self.ctrl.is_resp(): return # ignore received response @@ -359,7 +365,7 @@ class Talent(Message): msg_hdr_len = self.parse_msg_header() for key, update in self.db.parse(self._recv_buffer, self.header_len - + msg_hdr_len): + + msg_hdr_len, self.node_id): if update: self.new_data[key] = True diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index b175cb3..823e5ab 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -302,6 +302,8 @@ class SolarmanV5(Message): self.__finish_send_msg() async def send_modbus_cmd(self, func, addr, val) -> None: + if self.closed: + return self.forward_modbus_resp = False self.__build_header(0x4510) self._send_buffer += struct.pack(' None: + if self.closed: + return self.__build_header(0x4510) self._send_buffer += struct.pack(f'> 8 - for key, update in self.db.parse(self._recv_buffer, msg_type, ftype): + for key, update in self.db.parse(self._recv_buffer, msg_type, ftype, + self.node_id): if update: if key == 'inverter': inv_update = True diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 3469201..484d7df 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -130,7 +130,7 @@ class Mqtt(metaclass=Singleton): topic = str(message.topic) node_id = topic.split('/')[1] + '/' for m in Message: - if m.server_side and not m.closed and (m.node_id == node_id): + if m.server_side and (m.node_id == node_id): logger_mqtt.debug(f'Found: {node_id}') fnc = getattr(m, func_name, None) if callable(fnc): @@ -148,7 +148,7 @@ class Mqtt(metaclass=Singleton): payload = message.payload.decode("UTF-8") logger_mqtt.info(f'InvCnf: {node_id}:{payload}') for m in Message: - if m.server_side and not m.closed and (m.node_id == node_id): + if m.server_side and (m.node_id == node_id): logger_mqtt.info(f'Found: {node_id}') fnc = getattr(m, "send_modbus_cmd", None) res = payload.split(',') diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index a94df4f..202cd96 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -733,10 +733,16 @@ def test_msg_unknown(ConfigTsunInv1, MsgUnknown): m.close() def test_ctrl_byte(): + c = Control(0x70) + assert not c.is_ind() + assert not c.is_resp() + assert c.is_req() c = Control(0x91) + assert not c.is_req() assert c.is_ind() assert not c.is_resp() c = Control(0x99) + assert not c.is_req() assert not c.is_ind() assert c.is_resp() @@ -891,8 +897,9 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): @pytest.mark.asyncio async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 - m = MemoryStream(b'', (0,), False) + m = MemoryStream(b'', (0,), True) m.id_str = b"R170000000000001" + m.state = m.STATE_UP await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' From 5fe455e42fadeeb93e6ab33fa364634b1e804a8e Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 16:46:59 +0200 Subject: [PATCH 041/118] fix typo --- app/src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/config.py b/app/src/config.py index 73a633c..eef0c86 100644 --- a/app/src/config.py +++ b/app/src/config.py @@ -80,7 +80,7 @@ class Config(): try: # make the default config transparaent by copying it # in the config.example file - logging.debug('Copy Defaul Config to config.example.toml') + logging.debug('Copy Default Config to config.example.toml') shutil.copy2("default_config.toml", "config/config.example.toml") From 537d81fa19a5dbf85f790b9d9208625b9a7ff4f7 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 16:49:59 +0200 Subject: [PATCH 042/118] add graceful shutdown --- CHANGELOG.md | 1 + app/src/async_stream.py | 4 ++++ app/src/server.py | 18 ++++++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ec1d47..e4365c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- add graceful shutdown - add timeout monitoring for received packets - parse Modbus values and store them in the database - add cron task to request the output power every minute diff --git a/app/src/async_stream.py b/app/src/async_stream.py index ac6c54f..7c99373 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -90,11 +90,15 @@ class AsyncStream(): f"{traceback.format_exc()}") async def disc(self) -> None: + if self.writer.is_closing(): + return logger.debug(f'AsyncStream.disc() l{self.l_addr} | r{self.r_addr}') self.writer.close() await self.writer.wait_closed() def close(self): + if self.writer.is_closing(): + return logger.debug(f'AsyncStream.close() l{self.l_addr} | r{self.r_addr}') self.writer.close() diff --git a/app/src/server.py b/app/src/server.py index 7151cf2..c7ee03e 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -1,7 +1,6 @@ import logging import asyncio import signal -import functools import os from logging import config # noqa F401 from messages import Message @@ -26,13 +25,23 @@ async def handle_client_v2(reader, writer): await InverterG3P(reader, writer, addr).server_loop(addr) -def handle_SIGTERM(loop): +async def handle_shutdown(loop): '''Close all TCP connections and stop the event loop''' logging.info('Shutdown due to SIGTERM') # - # first, close all open TCP connections + # first, disc all open TCP connections gracefully + # + for stream in Message: + try: + await asyncio.wait_for(stream.disc(), 2) + except Exception: + pass + logging.info('Disconnecting done') + + # + # second, close all open TCP connections # for stream in Message: stream.close() @@ -91,7 +100,8 @@ if __name__ == "__main__": # for signame in ('SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), - functools.partial(handle_SIGTERM, loop)) + lambda loop=loop: asyncio.create_task( + handle_shutdown(loop))) # # Create taska for our listening servera. These must be tasks! If we call From 93e82a22842a16ebc43fb1844aa8e09b513dac3f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 18:22:08 +0200 Subject: [PATCH 043/118] move state variable to the parent class --- app/src/gen3/talent.py | 5 ----- app/src/messages.py | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index daf700f..44c7d0f 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -35,10 +35,6 @@ class Control: class Talent(Message): - STATE_INIT = 0 - STATE_UP = 2 - STATE_CLOSED = 3 - def __init__(self, server_side: bool, id_str=b''): super().__init__(server_side) self.await_conn_resp_cnt = 0 @@ -48,7 +44,6 @@ class Talent(Message): self.db = InfosG3() self.mb = Modbus() self.forward_modbus_resp = False - self.state = self.STATE_INIT self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, diff --git a/app/src/messages.py b/app/src/messages.py index 5bcf711..615c054 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -50,6 +50,9 @@ class IterRegistry(type): class Message(metaclass=IterRegistry): _registry = [] + STATE_INIT = 0 + STATE_UP = 2 + STATE_CLOSED = 3 def __init__(self, server_side: bool): self._registry.append(weakref.ref(self)) @@ -65,6 +68,7 @@ class Message(metaclass=IterRegistry): self._send_buffer = bytearray(0) self._forward_buffer = bytearray(0) self.new_data = {} + self.state = self.STATE_INIT ''' Empty methods, that have to be implemented in any child class which From b240b74994e8903c04420a61d4674464c956f196 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 18:22:43 +0200 Subject: [PATCH 044/118] avoid sending AT/Modbus commands too early - wait until we have received the first data from the inverter --- app/src/gen3plus/solarman_v5.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 823e5ab..2bc2b87 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -62,7 +62,6 @@ class SolarmanV5(Message): self.time_ofs = 0 self.mb = Modbus() self.forward_modbus_resp = False - self.closed = False self.switch = { 0x4210: self.msg_data_ind, # real time data @@ -103,7 +102,7 @@ class SolarmanV5(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() - self.closed = True + self.state = self.STATE_CLOSED def __set_serial_no(self, snr: int): serial_no = str(snr) @@ -302,7 +301,7 @@ class SolarmanV5(Message): self.__finish_send_msg() async def send_modbus_cmd(self, func, addr, val) -> None: - if self.closed: + if self.state != self.STATE_UP: return self.forward_modbus_resp = False self.__build_header(0x4510) @@ -317,7 +316,7 @@ class SolarmanV5(Message): self._send_buffer = bytearray(0) async def send_at_cmd(self, AT_cmd: str) -> None: - if self.closed: + if self.state != self.STATE_UP: return self.__build_header(0x4510) self._send_buffer += struct.pack(f' Date: Thu, 9 May 2024 18:48:59 +0200 Subject: [PATCH 045/118] fix unit tests --- app/tests/test_solarman.py | 58 +++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index e699ae7..2b9f39f 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -5,6 +5,7 @@ from datetime import datetime from app.src.gen3plus.solarman_v5 import SolarmanV5 from app.src.config import Config from app.src.infos import Infos, Register +from app.src.modbus import Modbus pytest_plugins = ('pytest_asyncio',) @@ -363,7 +364,7 @@ def SyncStartFwdMsg(): # 0x4310 @pytest.fixture def AtCommandIndMsg(): # 0x4510 - msg = b'\xa5\x27\x00\x10\x45\x02\x01' +get_sn() +b'\x01\x02\x00' + msg = b'\xa5\x27\x00\x10\x45\x03\x02' +get_sn() +b'\x01\x02\x00' msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' msg += b'AT+TIME=214028,1,60,120\r' msg += correct_checksum(msg) @@ -372,7 +373,7 @@ def AtCommandIndMsg(): # 0x4510 @pytest.fixture def AtCommandRspMsg(): # 0x1510 - msg = b'\xa5\x0a\x00\x10\x15\x02\x02' +get_sn() +b'\x01\x01' + msg = b'\xa5\x0a\x00\x10\x15\x03\x03' +get_sn() +b'\x01\x01' msg += total() msg += hb() msg += correct_checksum(msg) @@ -416,6 +417,15 @@ def SyncEndRspMsg(): # 0x1810 msg += b'\x15' return msg +@pytest.fixture +def MsgModbusCmd(): + msg = b'\xa5\x17\x00\x10\x45\x03\x02' +get_sn() +b'\x02\xb0\x02' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\x20\x08' + msg += b'\x00\x00\x03\xc8' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def ConfigTsunAllowAll(): Config.config = {'solarman':{'enabled': True}, 'inverters':{'allow_all':True}} @@ -876,7 +886,7 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg, AtCommandRspMsg): assert m.snr == 2070233889 # assert m.unique_id == '2070233889' assert m.control == 0x4510 - assert str(m.seq) == '02:02' + assert str(m.seq) == '03:03' assert m.data_len == 39 assert m._recv_buffer==b'' assert m._send_buffer==AtCommandRspMsg @@ -949,15 +959,49 @@ def test_build_logger_modell(ConfigTsunAllowAll, DeviceIndMsg): m.close() @pytest.mark.asyncio -async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, AtCommandIndMsg): - ConfigTsunAllowAll +async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, MsgModbusCmd): + ConfigTsunInv1 m = MemoryStream(DeviceIndMsg, (0,), True) + m.append_msg(InverterIndMsg) m.read() assert m.control == 0x4110 assert str(m.seq) == '01:01' - assert m._recv_buffer==b'' + assert m._recv_buffer==InverterIndMsg # unhandled next message assert m._send_buffer==DeviceRspMsg assert m._forward_buffer==DeviceIndMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear send buffer for next test + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' # modbus command must be ignore, cause connection is still not up + + m.read() + assert m.control == 0x4210 + assert str(m.seq) == '02:02' + assert m._recv_buffer==b'' + assert m._send_buffer==InverterRspMsg + assert m._forward_buffer==InverterIndMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear send buffer for next test + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == MsgModbusCmd + m.close() + +@pytest.mark.asyncio +async def test_AT_cmd(ConfigTsunAllowAll, InverterIndMsg, InverterRspMsg, AtCommandIndMsg): + ConfigTsunAllowAll + m = MemoryStream(InverterIndMsg, (0,), True) + m.read() + assert m.control == 0x4210 + assert str(m.seq) == '02:02' + assert m._recv_buffer==b'' + assert m._send_buffer==InverterRspMsg + assert m._forward_buffer==InverterIndMsg m._send_buffer = bytearray(0) # clear send buffer for next test m._forward_buffer = bytearray(0) # clear send buffer for next test @@ -965,5 +1009,5 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, AtCommandI assert m._recv_buffer==b'' assert m._send_buffer==AtCommandIndMsg assert m._forward_buffer==b'' - assert str(m.seq) == '01:02' + assert str(m.seq) == '02:03' m.close() From b3f0fc97d79c634655097b1e05dadad6805ab6b2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 23:23:33 +0200 Subject: [PATCH 046/118] add more unit tests --- app/tests/test_solarman.py | 86 ++++++++++++++++++++++++++++++++++++-- app/tests/test_talent.py | 16 ++++++- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 2b9f39f..3092e45 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -28,6 +28,7 @@ class MemoryStream(SolarmanV5): self.addr = 'Test: SrvSide' self.db.stat['proxy']['Invalid_Msg_Format'] = 0 self.db.stat['proxy']['AT_Command'] = 0 + self.test_exception_async_write = False def _timestamp(self): return timestamp @@ -59,7 +60,8 @@ class MemoryStream(SolarmanV5): return copied_bytes async def async_write(self, headline=''): - pass + if self.test_exception_async_write: + raise RuntimeError("Peer closed.") def _SolarmanV5__flush_recv_msg(self) -> None: super()._SolarmanV5__flush_recv_msg() @@ -315,6 +317,39 @@ def InverterIndMsg2000(): # 0x4210 rated Power 2000W msg += b'\x15' return msg +@pytest.fixture +def InverterIndMsg800(): # 0x4210 rated Power 800W + msg = b'\xa5\x99\x01\x10\x42\xe6\x9e' +get_sn() +b'\x01\xb0\x02\xbc\xc8' + msg += b'\x24\x32\x6c\x1f\x00\x00\xa0\x47\xe4\x33\x01\x00\x03\x08\x00\x00' + msg += b'\x59\x31\x37\x45\x37\x41\x30\x46\x30\x31\x30\x42\x30\x31\x33\x45' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x40\x10\x08\xc8\x00\x49\x13\x8d\x00\x36\x00\x00\x03\x20\x06\x7a' + msg += b'\x01\x61\x00\xa8\x02\x54\x01\x5a\x00\x8a\x01\xe4\x01\x5a\x00\xbd' + msg += b'\x02\x8f\x00\x11\x00\x01\x00\x00\x00\x0b\x00\x00\x27\x98\x00\x04' + msg += b'\x00\x00\x0c\x04\x00\x03\x00\x00\x0a\xe7\x00\x05\x00\x00\x0c\x75' + msg += b'\x00\x00\x00\x00\x06\x16\x02\x00\x00\x00\x55\xaa\x00\x01\x00\x00' + msg += b'\x00\x00\x00\x00\xff\xff\x03\x20\x00\x03\x04\x00\x04\x00\x04\x00' + msg += b'\x04\x00\x00\x01\xff\xff\x00\x01\x00\x06\x00\x68\x00\x68\x05\x00' + msg += b'\x09\xcd\x07\xb6\x13\x9c\x13\x24\x00\x01\x07\xae\x04\x0f\x00\x41' + msg += b'\x00\x0f\x0a\x64\x0a\x64\x00\x06\x00\x06\x09\xf6\x12\x8c\x12\x8c' + msg += b'\x00\x10\x00\x10\x14\x52\x14\x52\x00\x10\x00\x10\x01\x51\x00\x05' + msg += b'\x04\x00\x00\x01\x13\x9c\x0f\xa0\x00\x4e\x00\x66\x03\xe8\x04\x00' + msg += b'\x09\xce\x07\xa8\x13\x9c\x13\x26\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x04\x00\x04\x00\x00\x00\x00\x00\xff\xff\x00\x00' + msg += b'\x00\x00\x00\x00' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def InverterRspMsg(): # 0x1210 msg = b'\xa5\x0a\x00\x10\x12\x02\02' +get_sn() +b'\x01\x01' @@ -947,6 +982,18 @@ def test_build_modell_2000(ConfigTsunAllowAll, InverterIndMsg2000): assert 'TSOL-MS2000' == m.db.get_db_value(Register.EQUIPMENT_MODEL, 0) m.close() +def test_build_modell_800(ConfigTsunAllowAll, InverterIndMsg800): + ConfigTsunAllowAll + m = MemoryStream(InverterIndMsg800, (0,)) + assert 0 == m.db.get_db_value(Register.MAX_DESIGNED_POWER, 0) + assert None == m.db.get_db_value(Register.RATED_POWER, None) + assert None == m.db.get_db_value(Register.INVERTER_TEMP, None) + m.read() # read complete msg, and dispatch msg + assert 800 == m.db.get_db_value(Register.MAX_DESIGNED_POWER, 0) + assert 800 == m.db.get_db_value(Register.RATED_POWER, 0) + assert 'TSOL-MSxx00' == m.db.get_db_value(Register.EQUIPMENT_MODEL, 0) + m.close() + def test_build_logger_modell(ConfigTsunAllowAll, DeviceIndMsg): ConfigTsunAllowAll m = MemoryStream(DeviceIndMsg, (0,)) @@ -973,6 +1020,7 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, m._send_buffer = bytearray(0) # clear send buffer for next test m._forward_buffer = bytearray(0) # clear send buffer for next test await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert m._recv_buffer==InverterIndMsg # unhandled next message assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' assert m._send_buffer == b'' # modbus command must be ignore, cause connection is still not up @@ -990,12 +1038,35 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' assert m._send_buffer == MsgModbusCmd + + m._send_buffer = bytearray(0) # clear send buffer for next test + m.test_exception_async_write = True + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' m.close() @pytest.mark.asyncio -async def test_AT_cmd(ConfigTsunAllowAll, InverterIndMsg, InverterRspMsg, AtCommandIndMsg): +async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, AtCommandIndMsg): ConfigTsunAllowAll - m = MemoryStream(InverterIndMsg, (0,), True) + m = MemoryStream(DeviceIndMsg, (0,), True) + m.append_msg(InverterIndMsg) + m.read() + assert m.control == 0x4110 + assert str(m.seq) == '01:01' + assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._send_buffer==DeviceRspMsg + assert m._forward_buffer==DeviceIndMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear send buffer for next test + await m.send_at_cmd('AT+TIME=214028,1,60,120') + assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert str(m.seq) == '01:01' + m.read() assert m.control == 0x4210 assert str(m.seq) == '02:02' @@ -1010,4 +1081,13 @@ async def test_AT_cmd(ConfigTsunAllowAll, InverterIndMsg, InverterRspMsg, AtComm assert m._send_buffer==AtCommandIndMsg assert m._forward_buffer==b'' assert str(m.seq) == '02:03' + + m._send_buffer = bytearray(0) # clear send buffer for next test + m.test_exception_async_write = True + await m.send_at_cmd('AT+TIME=214028,1,60,120') + assert m._recv_buffer==b'' + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert str(m.seq) == '02:04' + m.close() diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 202cd96..9e6e8b5 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -24,6 +24,7 @@ class MemoryStream(Talent): self.msg_count = 0 self.addr = 'Test: SrvSide' self.send_msg_ofs = 0 + self.test_exception_async_write = False def append_msg(self, msg): self.__msg += msg @@ -56,7 +57,8 @@ class MemoryStream(Talent): return async def async_write(self, headline=''): - pass + if self.test_exception_async_write: + raise RuntimeError("Peer closed.") @@ -899,9 +901,21 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(b'', (0,), True) m.id_str = b"R170000000000001" + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' + m.state = m.STATE_UP await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' assert m._send_buffer == MsgModbusCmd + + m._send_buffer = bytearray(0) # clear send buffer for next test + m.test_exception_async_write = True + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' m.close() From def57024157a150343557ed369262d438248d718 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 23:31:22 +0200 Subject: [PATCH 047/118] upgrade version fron v3 to v4 --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 518a148..72ce456 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.12" - name: Install dependencies From 6a644841745e13c903ac92a33721e6e5ee85f280 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 23:34:29 +0200 Subject: [PATCH 048/118] read `Designed Power' with Modbus --- app/src/scheduler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/src/scheduler.py b/app/src/scheduler.py index a1e763b..1342643 100644 --- a/app/src/scheduler.py +++ b/app/src/scheduler.py @@ -11,6 +11,7 @@ logger_mqtt = logging.getLogger('mqtt') class Schedule: mqtt = None + count = 0 @classmethod def start(cls) -> None: @@ -36,8 +37,15 @@ class Schedule: @classmethod async def regular_modbus_cmds(cls): # logging.info("Regular Modbus requests") + if 0 == (cls.count % 30): + # logging.info("Regular Modbus Status request") + addr, len = 0x2007, 2 + else: + addr, len = 0x3008, 20 + cls.count += 1 + for m in Message: if m.server_side: fnc = getattr(m, "send_modbus_cmd", None) if callable(fnc): - await fnc(Modbus.READ_REGS, 0x3008, 20) + await fnc(Modbus.READ_REGS, addr, len) From f48596a512c6cef85445e3c04791716c90997649 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 23:38:02 +0200 Subject: [PATCH 049/118] use actions/setup-python@v5 --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 72ce456..2c7031b 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install dependencies From dd438bf2014d909b12f25e9ff681391d0c477c25 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 9 May 2024 23:38:34 +0200 Subject: [PATCH 050/118] add comment --- app/src/gen3/talent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 44c7d0f..d0cc123 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -399,7 +399,7 @@ class Talent(Message): self.node_id): if update: self.new_data[key] = True - self.modbus_elms += 1 + self.modbus_elms += 1 # count for unit tests if not self.forward_modbus_resp: return From 26f108cc5172df28f83963a2ea6169d945d465d4 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 10 May 2024 20:50:37 +0200 Subject: [PATCH 051/118] build version string in the same format as TSUN --- CHANGELOG.md | 1 + app/src/gen3plus/infos_g3p.py | 2 +- app/src/modbus.py | 2 +- app/tests/test_infos_g3p.py | 2 +- app/tests/test_modbus.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4365c7..425d189 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- build version string in the same format as TSUN for GEN3 invterts - add graceful shutdown - add timeout monitoring for received packets - parse Modbus values and store them in the database diff --git a/app/src/gen3plus/infos_g3p.py b/app/src/gen3plus/infos_g3p.py index a436dbf..213bcf2 100644 --- a/app/src/gen3plus/infos_g3p.py +++ b/app/src/gen3plus/infos_g3p.py @@ -25,7 +25,7 @@ class RegisterMap: 0x4201001c: {'reg': Register.POWER_ON_TIME, 'fmt': '>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 + 0x420100d0: {'reg': Register.VERSION, 'fmt': '!H', 'eval': "f'V{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 0x420100d2: {'reg': Register.GRID_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x420100d4: {'reg': Register.GRID_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x420100d6: {'reg': Register.GRID_FREQUENCY, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 diff --git a/app/src/modbus.py b/app/src/modbus.py index 0047afc..148b3f6 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -33,7 +33,7 @@ class Modbus(): map = { 0x2007: {'reg': Register.MAX_DESIGNED_POWER, 'fmt': '!H', 'ratio': 1}, # noqa: E501 # 0x????: {'reg': Register.INVERTER_STATUS, 'fmt': '!H'}, # noqa: E501 - 0x3008: {'reg': Register.VERSION, 'fmt': '!H', 'eval': "f'v{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 + 0x3008: {'reg': Register.VERSION, 'fmt': '!H', 'eval': "f'V{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"}, # noqa: E501 0x3009: {'reg': Register.GRID_VOLTAGE, 'fmt': '!H', 'ratio': 0.1}, # noqa: E501 0x300a: {'reg': Register.GRID_CURRENT, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 0x300b: {'reg': Register.GRID_FREQUENCY, 'fmt': '!H', 'ratio': 0.01}, # noqa: E501 diff --git a/app/tests/test_infos_g3p.py b/app/tests/test_infos_g3p.py index 4af9cb2..a127d13 100644 --- a/app/tests/test_infos_g3p.py +++ b/app/tests/test_infos_g3p.py @@ -83,7 +83,7 @@ def test_parse_4210(InverterData: bytes): assert json.dumps(i.db) == json.dumps({ "controller": {"Power_On_Time": 2051}, - "inverter": {"Serial_Number": "Y17E00000000000E", "Version": "v4.0.10", "Rated_Power": 600, "Max_Designed_Power": 2000, "No_Inputs": 4}, + "inverter": {"Serial_Number": "Y17E00000000000E", "Version": "V4.0.10", "Rated_Power": 600, "Max_Designed_Power": 2000, "No_Inputs": 4}, "env": {"Inverter_Status": 1, "Inverter_Temp": 14}, "grid": {"Voltage": 224.8, "Current": 0.73, "Frequency": 50.05, "Output_Power": 165.8}, "input": {"pv1": {"Voltage": 35.3, "Current": 1.68, "Power": 59.6, "Daily_Generation": 0.04, "Total_Generation": 30.76}, diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index a3d24ac..ed6a68a 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -106,7 +106,7 @@ def test_build_recv(): assert mb.check_crc(pdu) assert mb.err == 0 call = 0 - exp_result = ['v0.0.212', 4.4, 0.7, 0.7, 30] + exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): if key == 'grid': assert update == True From 0e7fbc7820a1a3e5781cc2bb487b4355881bfa7a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 20:46:36 +0200 Subject: [PATCH 052/118] fix Modbus CRC errors - parse Modbus messages well if another msg follows in the receive buffer --- app/src/gen3plus/solarman_v5.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 2bc2b87..1989225 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -419,22 +419,24 @@ class SolarmanV5(Message): self.__send_ack_rsp(0x1310, ftype) def msg_command_req(self): - data = self._recv_buffer[self.header_len:] + data = self._recv_buffer[self.header_len: + self.header_len+self.data_len] result = struct.unpack_from(' 4: # logger.info(f'first byte modbus:{data[14]}') inv_update = False - for key, update, _ in self.mb.recv_resp(self.db, data[14:-2], + for key, update, _ in self.mb.recv_resp(self.db, data[14:], self.node_id): if update: if key == 'inverter': From 3fda08bd25d756cd36f00626af3d2a3049b4de70 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 20:48:57 +0200 Subject: [PATCH 053/118] add more unit tests --- app/tests/test_solarman.py | 245 ++++++++++++++++++++++++++++++++++++- 1 file changed, 242 insertions(+), 3 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 3092e45..befb159 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -461,6 +461,19 @@ def MsgModbusCmd(): msg += b'\x15' return msg +@pytest.fixture +def MsgModbusRsp(): # 0x1510 + msg = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn() +b'\x02\x01' + msg += total() + msg += hb() + msg += b'\x0a\xe2\xfa\x33\x01\x03\x28\x40\x10\x08\xd8' + msg += b'\x00\x00\x13\x87\x00\x31\x00\x68\x02\x58\x00\x00\x01\x53\x00\x02' + msg += b'\x00\x00\x01\x52\x00\x02\x00\x00\x01\x53\x00\x03\x00\x00\x00\x04' + msg += b'\x00\x01\x00\x00\x6c\x68' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def ConfigTsunAllowAll(): Config.config = {'solarman':{'enabled': True}, 'inverters':{'allow_all':True}} @@ -911,7 +924,7 @@ def test_sync_end_rsp(ConfigTsunInv1, SyncEndRspMsg): assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() -def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg, AtCommandRspMsg): +def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): ConfigTsunInv1 m = MemoryStream(AtCommandIndMsg, (0,), False) m.read() # read complete msg, and dispatch msg @@ -921,10 +934,10 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg, AtCommandRspMsg): assert m.snr == 2070233889 # assert m.unique_id == '2070233889' assert m.control == 0x4510 - assert str(m.seq) == '03:03' + assert str(m.seq) == '03:02' assert m.data_len == 39 assert m._recv_buffer==b'' - assert m._send_buffer==AtCommandRspMsg + assert m._send_buffer==b'' assert m._forward_buffer==AtCommandIndMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 assert m.db.stat['proxy']['AT_Command'] == 1 @@ -1005,6 +1018,44 @@ def test_build_logger_modell(ConfigTsunAllowAll, DeviceIndMsg): assert 'V1.1.00.0B' == m.db.get_db_value(Register.COLLECTOR_FW_VERSION, 0).rstrip('\00') m.close() +def test_msg_iterator(): + m1 = SolarmanV5(server_side=True) + m2 = SolarmanV5(server_side=True) + m3 = SolarmanV5(server_side=True) + m3.close() + del m3 + test1 = 0 + test2 = 0 + for key in SolarmanV5: + if key == m1: + test1+=1 + elif key == m2: + test2+=1 + elif type(key) != SolarmanV5: + continue + else: + assert False + assert test1 == 1 + assert test2 == 1 + +def test_proxy_counter(): + m = SolarmanV5(server_side=True) + assert m.new_data == {} + m.db.stat['proxy']['Unknown_Msg'] = 0 + Infos.new_stat_data['proxy'] = False + + m.inc_counter('Unknown_Msg') + assert m.new_data == {} + assert Infos.new_stat_data == {'proxy': True} + assert 1 == m.db.stat['proxy']['Unknown_Msg'] + + Infos.new_stat_data['proxy'] = False + m.dec_counter('Unknown_Msg') + assert m.new_data == {} + assert Infos.new_stat_data == {'proxy': True} + assert 0 == m.db.stat['proxy']['Unknown_Msg'] + m.close() + @pytest.mark.asyncio async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, MsgModbusCmd): ConfigTsunInv1 @@ -1091,3 +1142,191 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIn assert str(m.seq) == '02:04' m.close() + +def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): + ConfigTsunInv1 + m = MemoryStream(MsgModbusCmd, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x4510 + assert str(m.seq) == '03:02' + assert m.header_len==11 + assert m.data_len==23 + assert m._forward_buffer==MsgModbusCmd + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 1 + m.close() + +def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_resp = False + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m.header_len==11 + assert m.data_len==59 + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_resp = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m.header_len==11 + assert m.data_len==59 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp, (0,), False) + m.append_msg(MsgModbusRsp) + + m.forward_modbus_resp = True + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + # assert m.db.db == {'inverter': {'Manufacturer': 'TSUN', 'Equipment_Model': 'TSOL-MSxx00'}} + m.new_data['inverter'] = False + + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 1 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + # assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V4.0.10' + assert m.new_data['inverter'] == True + m.new_data['inverter'] = False + + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 2 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + # assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V4.0.10' + assert m.new_data['inverter'] == False + + m.close() +''' +def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInv): + ConfigTsunInv1 + m = MemoryStream(MsgModbusInv, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl)==153 + assert m.msg_id==119 + assert m.header_len==23 + assert m.data_len==13 + assert m._forward_buffer==MsgModbusInv + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): + ConfigTsunInv1 + # receive more bytes than expected (7 bytes from the next msg) + m = MemoryStream(MsgModbusResp20+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_resp = True + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.id_str == b"R170000000000001" + assert m.unique_id == 'R170000000000001' + assert int(m.ctrl) == 0x91 + assert m.msg_id == 119 + assert m.header_len == 23 + assert m.data_len == 50 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer == b'' + assert m.mb.err == 0 + assert m.modbus_elms == 20-1 # register 0x300d is unknown, so one value can't be mapped + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +@pytest.mark.asyncio +async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): + ConfigTsunInv1 + m = MemoryStream(b'', (0,), True) + m.id_str = b"R170000000000001" + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' + + m.state = m.STATE_UP + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == MsgModbusCmd + + m._send_buffer = bytearray(0) # clear send buffer for next test + m.test_exception_async_write = True + await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) + assert 0 == m.send_msg_ofs + assert m._forward_buffer == b'' + assert m._send_buffer == b'' + m.close() + +def test_zombie_conn(ConfigTsunInv1, MsgInverterInd): + ConfigTsunInv1 + tracer.setLevel(logging.DEBUG) + m1 = MemoryStream(MsgInverterInd, (0,)) + m2 = MemoryStream(MsgInverterInd, (0,)) + m3 = MemoryStream(MsgInverterInd, (0,)) + assert m1.state == m1.STATE_INIT + assert m2.state == m2.STATE_INIT + assert m3.state == m3.STATE_INIT + m1.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_INIT + assert m2.state == m2.STATE_INIT + assert m3.state == m3.STATE_INIT + m2.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_CLOSED + assert m2.state == m2.STATE_INIT + assert m3.state == m3.STATE_INIT + m3.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_CLOSED + assert m2.state == m2.STATE_CLOSED + assert m3.state == m3.STATE_INIT + m1.close() + m2.close() + m3.close() +''' \ No newline at end of file From 73baffe9e08b71c4fbc0715f271fa9d9706a2519 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 20:50:26 +0200 Subject: [PATCH 054/118] also get the 'Daily Generation' every minute --- app/src/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/scheduler.py b/app/src/scheduler.py index 1342643..610f586 100644 --- a/app/src/scheduler.py +++ b/app/src/scheduler.py @@ -41,7 +41,7 @@ class Schedule: # logging.info("Regular Modbus Status request") addr, len = 0x2007, 2 else: - addr, len = 0x3008, 20 + addr, len = 0x3008, 21 cls.count += 1 for m in Message: From 6fcf4f47c2ae4cacc17f8a13bc700021d124f062 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 20:53:39 +0200 Subject: [PATCH 055/118] add more unit tests --- app/tests/test_talent.py | 71 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 9e6e8b5..96feaf1 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -2,7 +2,7 @@ import pytest, logging from app.src.gen3.talent import Talent, Control from app.src.config import Config -from app.src.infos import Infos +from app.src.infos import Infos, Register from app.src.modbus import Modbus @@ -849,6 +849,41 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() +def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): + ConfigTsunInv1 + m = MemoryStream(MsgModbusResp20, (0,), False) + m.append_msg(MsgModbusResp20) + + m.forward_modbus_resp = True + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + assert m.db.db == {} + m.new_data['inverter'] = False + + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 1 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer==b'' + assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V5.1.09' + assert m.new_data['inverter'] == True + m.new_data['inverter'] = False + + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 2 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer==b'' + assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V5.1.09' + assert m.new_data['inverter'] == False + + m.close() + def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInv): ConfigTsunInv1 m = MemoryStream(MsgModbusInv, (0,), False) @@ -919,3 +954,37 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert m._forward_buffer == b'' assert m._send_buffer == b'' m.close() +''' +def test_zombie_conn(ConfigTsunInv1, MsgInverterInd): + ConfigTsunInv1 + tracer.setLevel(logging.DEBUG) + start_val = MemoryStream._RefNo + + m1 = MemoryStream(MsgInverterInd, (0,)) + assert MemoryStream._RefNo == 1 + start_val + assert m1.RefNo == 1 + start_val + m2 = MemoryStream(MsgInverterInd, (0,)) + assert MemoryStream._RefNo == 2 + start_val + assert m2.RefNo == 2 + start_val + m3 = MemoryStream(MsgInverterInd, (0,)) + assert MemoryStream._RefNo == 3 + start_val + assert m3.RefNo == 3 + start_val + assert m1.state == m1.STATE_INIT + assert m2.state == m2.STATE_INIT + assert m3.state == m3.STATE_INIT + m1.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_UP + assert m2.state == m2.STATE_INIT + assert m3.state == m3.STATE_INIT + m2.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_CLOSED + assert m2.state == m2.STATE_UP + assert m3.state == m3.STATE_INIT + m3.read() # read complete msg, and set unique_id + assert m1.state == m1.STATE_CLOSED + assert m2.state == m2.STATE_CLOSED + assert m3.state == m3.STATE_UP + m1.close() + m2.close() + m3.close() +''' \ No newline at end of file From 4ea70dee64a6f3e4b5476ab9f9b1419d79133d1a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 20:55:31 +0200 Subject: [PATCH 056/118] improve connection handling - insure close() call after graceful disconnect, to release proxy internal resources - timeout handler disconnect inverter connection if no message was received for longer than 2.5 minutes --- app/src/async_stream.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 7c99373..56f475d 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -59,28 +59,35 @@ class AsyncStream(): while True: try: - # await asyncio.wait_for(self.__async_read(), 0.3) - await self.__async_read() + if self.state == self.STATE_UP and self.server_side: + await asyncio.wait_for(self.__async_read(), 150) + else: + await self.__async_read() if self.unique_id: await self.async_write() await self.__async_forward() await self.async_publ_mqtt() - except asyncio.TimeoutError: - pass - except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError) as error: logger.error(f'{error} for l{self.l_addr} | ' f'r{self.r_addr}') await self.disc() + self.close() return self except RuntimeError as error: logger.warning(f"{error} for {self.l_addr}") await self.disc() + self.close() + return self + + except asyncio.TimeoutError: + logger.warning(f"Timeout for {self.l_addr}") + await self.disc() + self.close() return self except Exception: From e43a02c508068e94fe370238d2d5ee8d4ef25c62 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 23:40:46 +0200 Subject: [PATCH 057/118] improve modbus parsing - parse Modbus messages well if another msg follows in the receive buffer --- app/src/gen3/talent.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index d0cc123..f82f47c 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -347,6 +347,7 @@ class Talent(Message): self._send_buffer += b'\x01' self.__finish_send_msg() self.__process_data() + self.state = self.STATE_UP elif self.ctrl.is_resp(): return # ignore received response @@ -387,15 +388,20 @@ class Talent(Message): def msg_modbus(self): hdr_len, modbus_len = self.parse_modbus_header() + data = self._recv_buffer[self.header_len: + self.header_len+self.data_len] if self.ctrl.is_req(): + if not self.mb.recv_req(data[hdr_len:]): + return + self.forward_modbus_resp = True self.inc_counter('Modbus_Command') elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 - for key, update, _ in self.mb.recv_resp(self.db, self._recv_buffer[ - self.header_len + hdr_len:self.header_len+self.data_len], + for key, update, _ in self.mb.recv_resp(self.db, data[ + hdr_len:], self.node_id): if update: self.new_data[key] = True From 1ae7784bee83898d2b370ec98cd3de16d60357b0 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 11 May 2024 23:41:40 +0200 Subject: [PATCH 058/118] add more unit tests --- app/src/gen3plus/solarman_v5.py | 4 + app/tests/test_solarman.py | 149 ++++++++++++++++++-------------- 2 files changed, 88 insertions(+), 65 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 1989225..a766ce8 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -92,6 +92,7 @@ class SolarmanV5(Message): 0x4510: self.msg_command_req, # from server 0x1510: self.msg_command_rsp, # from inverter } + self.modbus_elms = 0 # for unit tests ''' Our puplic methods @@ -447,8 +448,11 @@ class SolarmanV5(Message): if valid == 1 and modbus_msg_len > 4: # logger.info(f'first byte modbus:{data[14]}') inv_update = False + self.modbus_elms = 0 + for key, update, _ in self.mb.recv_resp(self.db, data[14:], self.node_id): + self.modbus_elms += 1 if update: if key == 'inverter': inv_update = True diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index befb159..f7f4a21 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -388,7 +388,7 @@ def SyncStartRspMsg(): # 0x1310 @pytest.fixture def SyncStartFwdMsg(): # 0x4310 - msg = b'\xa5\x2f\x00\x10\x43\x0e\x0d' +get_sn() +b'\x81\x7a\x0b\x2e\x32' + msg = b'\xa5\x2f\x00\x10\x43\x0d\x0e' +get_sn() +b'\x81\x7a\x0b\x2e\x32' msg += b'\x39\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x41\x6c\x6c\x69\x75\x73' msg += b'\x2d\x48\x6f\x6d\x65\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x61\x01' @@ -474,6 +474,28 @@ def MsgModbusRsp(): # 0x1510 msg += b'\x15' return msg +@pytest.fixture +def MsgModbusInvalid(): # 0x1510 + msg = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn() +b'\x02\x00' + msg += total() + msg += hb() + msg += b'\x0a\xe2\xfa\x33\x01\x03\x28\x40\x10\x08\xd8' + msg += b'\x00\x00\x13\x87\x00\x31\x00\x68\x02\x58\x00\x00\x01\x53\x00\x02' + msg += b'\x00\x00\x01\x52\x00\x02\x00\x00\x01\x53\x00\x03\x00\x00\x00\x04' + msg += b'\x00\x01\x00\x00\x6c\x68' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + +@pytest.fixture +def MsgUnknownCmd(): + msg = b'\xa5\x17\x00\x10\x45\x03\x02' +get_sn() +b'\x03\xb0\x02' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\x20\x08' + msg += b'\x00\x00\x03\xc8' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def ConfigTsunAllowAll(): Config.config = {'solarman':{'enabled': True}, 'inverters':{'allow_all':True}} @@ -864,6 +886,7 @@ def test_sync_start_ind(ConfigTsunInv1, SyncStartIndMsg, SyncStartRspMsg, SyncSt assert m._forward_buffer==SyncStartIndMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + m.seq.server_side = False # simulate forawding to TSUN cloud m._update_header(m._forward_buffer) assert str(m.seq) == '0d:0e' # value after forwarding indication assert m._forward_buffer==SyncStartFwdMsg @@ -924,25 +947,6 @@ def test_sync_end_rsp(ConfigTsunInv1, SyncEndRspMsg): assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() -def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): - ConfigTsunInv1 - m = MemoryStream(AtCommandIndMsg, (0,), False) - m.read() # read complete msg, and dispatch msg - assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.msg_count == 1 - assert m.header_len==11 - assert m.snr == 2070233889 - # assert m.unique_id == '2070233889' - assert m.control == 0x4510 - assert str(m.seq) == '03:02' - assert m.data_len == 39 - assert m._recv_buffer==b'' - assert m._send_buffer==b'' - assert m._forward_buffer==AtCommandIndMsg - assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 - assert m.db.stat['proxy']['AT_Command'] == 1 - m.close() - def test_build_modell_600(ConfigTsunAllowAll, InverterIndMsg): ConfigTsunAllowAll m = MemoryStream(InverterIndMsg, (0,)) @@ -1143,10 +1147,37 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIn m.close() +def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): + ConfigTsunInv1 + m = MemoryStream(AtCommandIndMsg, (0,), False) + m.forward_modbus_resp = False + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['AT_Command'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.header_len==11 + assert m.snr == 2070233889 + # assert m.unique_id == '2070233889' + assert m.control == 0x4510 + assert str(m.seq) == '03:02' + assert m.data_len == 39 + assert m._recv_buffer==b'' + assert m._send_buffer==b'' + assert m._forward_buffer==AtCommandIndMsg + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert m.db.stat['proxy']['AT_Command'] == 1 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + assert m.forward_modbus_resp == False + m.close() + def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(MsgModbusCmd, (0,), False) + m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed @@ -1158,7 +1189,31 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert m._forward_buffer==MsgModbusCmd assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 + assert m.forward_modbus_resp == True + m.close() + +def test_msg_unknown_cmd_req(ConfigTsunInv1, MsgUnknownCmd): + ConfigTsunInv1 + m = MemoryStream(MsgUnknownCmd, (0,), False) + m.forward_modbus_resp = False + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['AT_Command'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x4510 + assert str(m.seq) == '03:02' + assert m.header_len==11 + assert m.data_len==23 + assert m._forward_buffer==MsgUnknownCmd + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['AT_Command'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + assert m.forward_modbus_resp == False m.close() def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): @@ -1233,31 +1288,25 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): assert m.new_data['inverter'] == False m.close() -''' -def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInv): + +def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInvalid): ConfigTsunInv1 - m = MemoryStream(MsgModbusInv, (0,), False) + m = MemoryStream(MsgModbusInvalid, (0,), False) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 - assert m.id_str == b"R170000000000001" - assert m.unique_id == 'R170000000000001' - assert int(m.ctrl)==153 - assert m.msg_id==119 - assert m.header_len==23 - assert m.data_len==13 - assert m._forward_buffer==MsgModbusInv + assert m._forward_buffer==MsgModbusInvalid assert m._send_buffer==b'' - assert m.db.stat['proxy']['Unknown_Ctrl'] == 1 + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() -def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): +def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 # receive more bytes than expected (7 bytes from the next msg) - m = MemoryStream(MsgModbusResp20+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) + m = MemoryStream(MsgModbusRsp+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = True @@ -1267,44 +1316,14 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 - assert m.id_str == b"R170000000000001" - assert m.unique_id == 'R170000000000001' - assert int(m.ctrl) == 0x91 - assert m.msg_id == 119 - assert m.header_len == 23 - assert m.data_len == 50 - assert m._forward_buffer==MsgModbusResp20 + assert m._forward_buffer==MsgModbusRsp assert m._send_buffer == b'' assert m.mb.err == 0 assert m.modbus_elms == 20-1 # register 0x300d is unknown, so one value can't be mapped assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() - -@pytest.mark.asyncio -async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): - ConfigTsunInv1 - m = MemoryStream(b'', (0,), True) - m.id_str = b"R170000000000001" - await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) - assert 0 == m.send_msg_ofs - assert m._forward_buffer == b'' - assert m._send_buffer == b'' - - m.state = m.STATE_UP - await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) - assert 0 == m.send_msg_ofs - assert m._forward_buffer == b'' - assert m._send_buffer == MsgModbusCmd - - m._send_buffer = bytearray(0) # clear send buffer for next test - m.test_exception_async_write = True - await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) - assert 0 == m.send_msg_ofs - assert m._forward_buffer == b'' - assert m._send_buffer == b'' - m.close() - +''' def test_zombie_conn(ConfigTsunInv1, MsgInverterInd): ConfigTsunInv1 tracer.setLevel(logging.DEBUG) From 1658036a261781225b3a72e8397fba0a4e538e36 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 12 May 2024 23:09:51 +0200 Subject: [PATCH 059/118] store modbus params always on the server side --- app/src/gen3/talent.py | 10 +++++----- app/src/gen3plus/solarman_v5.py | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index f82f47c..c007295 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -392,11 +392,11 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): - if not self.mb.recv_req(data[hdr_len:]): - return - - self.forward_modbus_resp = True - self.inc_counter('Modbus_Command') + if not self.remoteStream.mb.recv_req(data[hdr_len:]): + self.inc_counter('Invalid_Msg_Format') + else: + self.inc_counter('Modbus_Command') + self.remoteStream.forward_modbus_resp = True elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index a766ce8..8e926ac 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -427,10 +427,11 @@ class SolarmanV5(Message): if ftype == self.AT_CMD: self.inc_counter('AT_Command') elif ftype == self.MB_RTU_CMD: - if not self.mb.recv_req(data[15:]): - return - self.forward_modbus_resp = True - self.inc_counter('Modbus_Command') + if not self.remoteStream.mb.recv_req(data[15:]): + self.inc_counter('Invalid_Msg_Format') + else: + self.inc_counter('Modbus_Command') + self.remoteStream.forward_modbus_resp = True self.__forward_msg() # self.__send_ack_rsp(0x1510, ftype) From 92469456b75817252b24b36428ca0c761e0ecb13 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 12 May 2024 23:11:55 +0200 Subject: [PATCH 060/118] fix unit tests --- app/tests/test_solarman.py | 28 ++++++++++++++++++---------- app/tests/test_talent.py | 32 ++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index f7f4a21..7c02ed1 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -63,6 +63,12 @@ class MemoryStream(SolarmanV5): if self.test_exception_async_write: raise RuntimeError("Peer closed.") + def createClientStream(self, msg, chunks = (0,)): + c = MemoryStream(msg, chunks, False) + self.remoteStream = c + c. remoteStream = self + return c + def _SolarmanV5__flush_recv_msg(self) -> None: super()._SolarmanV5__flush_recv_msg() self.msg_count += 1 @@ -1174,20 +1180,22 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 - m = MemoryStream(MsgModbusCmd, (0,), False) + m = MemoryStream(b'') + c = m.createClientStream(MsgModbusCmd) + m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.read() # read complete msg, and dispatch msg - assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.msg_count == 1 - assert m.control == 0x4510 - assert str(m.seq) == '03:02' - assert m.header_len==11 - assert m.data_len==23 - assert m._forward_buffer==MsgModbusCmd - assert m._send_buffer==b'' + c.read() # read complete msg, and dispatch msg + assert not c.header_valid # must be invalid, since msg was handled and buffer flushed + assert c.msg_count == 1 + assert c.control == 0x4510 + assert str(c.seq) == '03:02' + assert c.header_len==11 + assert c.data_len==23 + assert c._forward_buffer==MsgModbusCmd + assert c._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 96feaf1..4c9ec65 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -51,6 +51,12 @@ class MemoryStream(Talent): def _timestamp(self): return 1700260990000 + def createClientStream(self, msg, chunks = (0,)): + c = MemoryStream(msg, chunks, False) + self.remoteStream = c + c. remoteStream = self + return c + def _Talent__flush_recv_msg(self) -> None: super()._Talent__flush_recv_msg() self.msg_count += 1 @@ -789,20 +795,22 @@ def test_proxy_counter(): def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 - m = MemoryStream(MsgModbusCmd, (0,), False) + m = MemoryStream(b'') + c = m.createClientStream(MsgModbusCmd) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.read() # read complete msg, and dispatch msg - assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.msg_count == 1 - assert m.id_str == b"R170000000000001" - assert m.unique_id == 'R170000000000001' - assert int(m.ctrl)==112 - assert m.msg_id==119 - assert m.header_len==23 - assert m.data_len==13 - assert m._forward_buffer==MsgModbusCmd - assert m._send_buffer==b'' + c.read() # read complete msg, and dispatch msg + assert not c.header_valid # must be invalid, since msg was handled and buffer flushed + assert c.msg_count == 1 + assert c.id_str == b"R170000000000001" + assert c.unique_id == 'R170000000000001' + assert int(c.ctrl)==112 + assert c.msg_id==119 + assert c.header_len==23 + assert c.data_len==13 + assert c._forward_buffer==MsgModbusCmd + assert c._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 m.close() From 036af8e127f8fbfd2743551d7cadf9fead36d3be Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 13 May 2024 19:49:00 +0200 Subject: [PATCH 061/118] move the Modbus instance to the parent class --- app/src/gen3/talent.py | 1 - app/src/gen3plus/solarman_v5.py | 1 - app/src/messages.py | 5 +++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index c007295..976d468 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -42,7 +42,6 @@ class Talent(Message): self.contact_name = b'' self.contact_mail = b'' self.db = InfosG3() - self.mb = Modbus() self.forward_modbus_resp = False self.switch = { 0x00: self.msg_contact_info, diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 8e926ac..cd1e5d7 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -60,7 +60,6 @@ class SolarmanV5(Message): self.snr = 0 self.db = InfosG3P() self.time_ofs = 0 - self.mb = Modbus() self.forward_modbus_resp = False self.switch = { diff --git a/app/src/messages.py b/app/src/messages.py index 615c054..01e3429 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -3,8 +3,10 @@ import weakref if __name__ == "app.src.messages": from app.src.infos import Infos + from app.src.modbus import Modbus else: # pragma: no cover from infos import Infos + from modbus import Modbus logger = logging.getLogger('msg') @@ -58,6 +60,9 @@ class Message(metaclass=IterRegistry): self._registry.append(weakref.ref(self)) self.server_side = server_side + if server_side: + self.mb = Modbus() + self.header_valid = False self.header_len = 0 self.data_len = 0 From 2e214b1e7197f949d90517c4aafe61cfdcf05e60 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 13 May 2024 22:46:23 +0200 Subject: [PATCH 062/118] avoid sending responses to TSUN for local at commands --- app/src/gen3plus/solarman_v5.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index cd1e5d7..91d2bd0 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -61,6 +61,7 @@ class SolarmanV5(Message): self.db = InfosG3P() self.time_ofs = 0 self.forward_modbus_resp = False + self.forward_at_cmd_resp = False self.switch = { 0x4210: self.msg_data_ind, # real time data @@ -318,6 +319,7 @@ class SolarmanV5(Message): async def send_at_cmd(self, AT_cmd: str) -> None: if self.state != self.STATE_UP: return + self.forward_at_cmd_resp = False self.__build_header(0x4510) self._send_buffer += struct.pack(f' Date: Mon, 13 May 2024 22:47:52 +0200 Subject: [PATCH 063/118] add more unit tests --- app/tests/test_solarman.py | 205 ++++++++++++++++++++++++++++++++++++- app/tests/test_talent.py | 39 ++++++- 2 files changed, 237 insertions(+), 7 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 7c02ed1..46df675 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -356,6 +356,39 @@ def InverterIndMsg800(): # 0x4210 rated Power 800W msg += b'\x15' return msg +@pytest.fixture +def InverterIndMsg_81(): # 0x4210 fcode 0x81 + msg = b'\xa5\x99\x01\x10\x42\x02\x03' +get_sn() +b'\x81\xb0\x02\xbc\xc8' + msg += b'\x24\x32\x6c\x1f\x00\x00\xa0\x07\x04\x03\x01\x00\x03\x08\x00\x00' + msg += b'\x59\x31\x37\x45\x37\x41\x30\x46\x30\x31\x30\x42\x30\x31\x33\x45' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x40\x10\x08\xc8\x00\x49\x13\x8d\x00\x36\x00\x00\x02\x58\x06\x7a' + msg += b'\x01\x61\x00\xa8\x02\x54\x01\x5a\x00\x8a\x01\xe4\x01\x5a\x00\xbd' + msg += b'\x02\x8f\x00\x11\x00\x01\x00\x00\x00\x0b\x00\x00\x27\x98\x00\x04' + msg += b'\x00\x00\x0c\x04\x00\x03\x00\x00\x0a\xe7\x00\x05\x00\x00\x0c\x75' + msg += b'\x00\x00\x00\x00\x06\x16\x02\x00\x00\x00\x55\xaa\x00\x01\x00\x00' + msg += b'\x00\x00\x00\x00\xff\xff\x07\xd0\x00\x03\x04\x00\x04\x00\x04\x00' + msg += b'\x04\x00\x00\x01\xff\xff\x00\x01\x00\x06\x00\x68\x00\x68\x05\x00' + msg += b'\x09\xcd\x07\xb6\x13\x9c\x13\x24\x00\x01\x07\xae\x04\x0f\x00\x41' + msg += b'\x00\x0f\x0a\x64\x0a\x64\x00\x06\x00\x06\x09\xf6\x12\x8c\x12\x8c' + msg += b'\x00\x10\x00\x10\x14\x52\x14\x52\x00\x10\x00\x10\x01\x51\x00\x05' + msg += b'\x04\x00\x00\x01\x13\x9c\x0f\xa0\x00\x4e\x00\x66\x03\xe8\x04\x00' + msg += b'\x09\xce\x07\xa8\x13\x9c\x13\x26\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'\x00\x00\x00\x00\x04\x00\x04\x00\x00\x00\x00\x00\xff\xff\x00\x00' + msg += b'\x00\x00\x00\x00' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def InverterRspMsg(): # 0x1210 msg = b'\xa5\x0a\x00\x10\x12\x02\02' +get_sn() +b'\x01\x01' @@ -365,6 +398,15 @@ def InverterRspMsg(): # 0x1210 msg += b'\x15' return msg +@pytest.fixture +def InverterRspMsg_81(): # 0x1210 fcode 0x81 + msg = b'\xa5\x0a\x00\x10\x12\x03\03' +get_sn() +b'\x81\x01' + msg += total() + msg += hb() + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def UnknownMsg(): # 0x5110 msg = b'\xa5\x0a\x00\x10\x51\x10\x84' +get_sn() +b'\x01\x01\x69\x6f\x09' @@ -467,6 +509,15 @@ def MsgModbusCmd(): msg += b'\x15' return msg +@pytest.fixture +def MsgModbusCmdCrcErr(): + msg = b'\xa5\x17\x00\x10\x45\x03\x02' +get_sn() +b'\x02\xb0\x02' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\x20\x08' + msg += b'\x00\x00\x04\xc8' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def MsgModbusRsp(): # 0x1510 msg = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn() +b'\x02\x01' @@ -502,6 +553,19 @@ def MsgUnknownCmd(): msg += b'\x15' return msg +@pytest.fixture +def MsgUnknownCmdRsp(): # 0x1510 + msg = b'\xa5\x3b\x00\x10\x15\x03\x03' +get_sn() +b'\x03\x01' + msg += total() + msg += hb() + msg += b'\x0a\xe2\xfa\x33\x01\x03\x28\x40\x10\x08\xd8' + msg += b'\x00\x00\x13\x87\x00\x31\x00\x68\x02\x58\x00\x00\x01\x53\x00\x02' + msg += b'\x00\x00\x01\x52\x00\x02\x00\x00\x01\x53\x00\x03\x00\x00\x00\x04' + msg += b'\x00\x01\x00\x00\x6c\x68' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def ConfigTsunAllowAll(): Config.config = {'solarman':{'enabled': True}, 'inverters':{'allow_all':True}} @@ -785,6 +849,52 @@ def test_read_two_messages(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, Inver assert m._send_buffer==b'' m.close() +def test_read_two_messages2(ConfigTsunAllowAll, InverterIndMsg, InverterIndMsg_81, InverterRspMsg, InverterRspMsg_81): + ConfigTsunAllowAll + m = MemoryStream(InverterIndMsg, (0,)) + m.append_msg(InverterIndMsg_81) + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.header_len==11 + assert m.snr == 2070233889 + assert m.unique_id == '2070233889' + assert m.control == 0x4210 + assert m.time_ofs == 0x33e447a0 + assert str(m.seq) == '02:02' + assert m.data_len == 0x199 + assert m.msg_count == 1 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert m._forward_buffer==InverterIndMsg + assert m._send_buffer==InverterRspMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._init_new_client_conn() + assert m._send_buffer==b'' + assert m._recv_buffer==InverterIndMsg_81 + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear forward buffer for next test + m.read() # read complete msg, and dispatch msg + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 2 + assert m.header_len==11 + assert m.snr == 2070233889 + assert m.unique_id == '2070233889' + assert m.control == 0x4210 + assert m.time_ofs == 0x33e447a0 + assert str(m.seq) == '03:03' + assert m.data_len == 0x199 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert m._forward_buffer==InverterIndMsg_81 + assert m._send_buffer==InverterRspMsg_81 + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._init_new_client_conn() + assert m._send_buffer==b'' + m.close() + def test_unkown_message(ConfigTsunInv1, UnknownMsg): ConfigTsunInv1 m = MemoryStream(UnknownMsg, (0,)) @@ -1150,7 +1260,7 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIn assert m._send_buffer==b'' assert m._forward_buffer==b'' assert str(m.seq) == '02:04' - + assert m.forward_at_cmd_resp == False m.close() def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): @@ -1178,6 +1288,44 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): assert m.forward_modbus_resp == False m.close() +def test_msg_at_command_rsp1(ConfigTsunInv1, AtCommandRspMsg): + ConfigTsunInv1 + m = MemoryStream(AtCommandRspMsg) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_at_cmd_resp = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m.header_len==11 + assert m.data_len==10 + assert m._forward_buffer==AtCommandRspMsg + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_msg_at_command_rsp2(ConfigTsunInv1, AtCommandRspMsg): + ConfigTsunInv1 + m = MemoryStream(AtCommandRspMsg) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_at_cmd_resp = False + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m.header_len==11 + assert m.data_len==10 + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(b'') @@ -1187,6 +1335,7 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 c.read() # read complete msg, and dispatch msg assert not c.header_valid # must be invalid, since msg was handled and buffer flushed assert c.msg_count == 1 @@ -1199,6 +1348,33 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert m.forward_modbus_resp == True + m.close() + +def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): + ConfigTsunInv1 + m = MemoryStream(b'') + c = m.createClientStream(MsgModbusCmdCrcErr) + + m.forward_modbus_resp = False + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['AT_Command'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 + c.read() # read complete msg, and dispatch msg + assert not c.header_valid # must be invalid, since msg was handled and buffer flushed + assert c.msg_count == 1 + assert c.control == 0x4510 + assert str(c.seq) == '03:02' + assert c.header_len==11 + assert c.data_len==23 + assert c._forward_buffer==MsgModbusCmdCrcErr + assert c._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['AT_Command'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 1 assert m.forward_modbus_resp == True m.close() @@ -1209,6 +1385,7 @@ def test_msg_unknown_cmd_req(ConfigTsunInv1, MsgUnknownCmd): m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 @@ -1221,12 +1398,13 @@ def test_msg_unknown_cmd_req(ConfigTsunInv1, MsgUnknownCmd): assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 assert m.forward_modbus_resp == False m.close() def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp, (0,), False) + m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = False @@ -1245,7 +1423,7 @@ def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp, (0,), False) + m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = True @@ -1264,7 +1442,7 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp, (0,), False) + m = MemoryStream(MsgModbusRsp) m.append_msg(MsgModbusRsp) m.forward_modbus_resp = True @@ -1297,6 +1475,25 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): m.close() +def test_msg_unknown_rsp(ConfigTsunInv1, MsgUnknownCmdRsp): + ConfigTsunInv1 + m = MemoryStream(MsgUnknownCmdRsp) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.forward_modbus_resp = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.control == 0x1510 + assert str(m.seq) == '03:03' + assert m.header_len==11 + assert m.data_len==59 + assert m._forward_buffer==MsgUnknownCmdRsp + assert m._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInvalid): ConfigTsunInv1 m = MemoryStream(MsgModbusInvalid, (0,), False) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 4c9ec65..ed39111 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -194,6 +194,13 @@ def MsgModbusCmd(): msg += b'\x00\x00\x03\xc8' return msg +@pytest.fixture +def MsgModbusCmdCrcErr(): + msg = b'\x00\x00\x00\x20\x10R170000000000001' + msg += b'\x70\x77\x00\x01\xa3\x28\x08\x01\x06\x20\x08' + msg += b'\x00\x00\x04\xc8' + return msg + @pytest.fixture def MsgModbusRsp(): msg = b'\x00\x00\x00\x20\x10R170000000000001' @@ -800,6 +807,7 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 c.read() # read complete msg, and dispatch msg assert not c.header_valid # must be invalid, since msg was handled and buffer flushed assert c.msg_count == 1 @@ -813,11 +821,36 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert c._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + m.close() + +def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): + ConfigTsunInv1 + m = MemoryStream(b'') + c = m.createClientStream(MsgModbusCmdCrcErr) + + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 + c.read() # read complete msg, and dispatch msg + assert not c.header_valid # must be invalid, since msg was handled and buffer flushed + assert c.msg_count == 1 + assert c.id_str == b"R170000000000001" + assert c.unique_id == 'R170000000000001' + assert int(c.ctrl)==112 + assert c.msg_id==119 + assert c.header_len==23 + assert c.data_len==13 + assert c._forward_buffer==MsgModbusCmdCrcErr + assert c._send_buffer==b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 1 m.close() def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp, (0,), False) + m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = False @@ -838,7 +871,7 @@ def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp, (0,), False) + m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = True @@ -859,7 +892,7 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): ConfigTsunInv1 - m = MemoryStream(MsgModbusResp20, (0,), False) + m = MemoryStream(MsgModbusResp20) m.append_msg(MsgModbusResp20) m.forward_modbus_resp = True From 14425da5fae43398a8e614fbf664be5ee4686198 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 13 May 2024 22:48:44 +0200 Subject: [PATCH 064/118] improve Modbus logging --- app/src/modbus.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 148b3f6..3ef0f5f 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -86,11 +86,11 @@ class Modbus(): # logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): self.err = 1 - logging.error('Modbus: CRC error') + logging.error('Modbus recv: CRC error') return False if buf[0] != self.INV_ADDR: self.err = 2 - logging.info(f'Modbus: Wrong addr{buf[0]}') + logging.info(f'Modbus recv: Wrong addr{buf[0]}') return False res = struct.unpack_from('>BHH', buf, 1) self.last_fcode = res[0] @@ -103,11 +103,11 @@ class Modbus(): Generator[tuple[str, bool], None, None]: # logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): - logging.error('Modbus: CRC error') + logging.error('Modbus resp: CRC error') self.err = 1 return if buf[0] != self.INV_ADDR: - logging.info(f'Modbus: Wrong addr {buf[0]}') + logging.info(f'Modbus resp: Wrong addr {buf[0]}') self.err = 2 return if buf[1] != self.last_fcode: From 841877305d46956a860e2c818a459d410fbddda8 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 15 May 2024 23:15:20 +0200 Subject: [PATCH 065/118] timeout handler removed again, as it has no positive effect --- CHANGELOG.md | 1 - app/src/async_stream.py | 12 +----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425d189..ed901bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - build version string in the same format as TSUN for GEN3 invterts - add graceful shutdown -- add timeout monitoring for received packets - parse Modbus values and store them in the database - add cron task to request the output power every minute - GEN3PLUS: add MQTT topics to send AT commands to the inverter diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 56f475d..196a01f 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -1,6 +1,5 @@ import logging import traceback -import asyncio from messages import hex_dump_memory logger = logging.getLogger('conn') @@ -59,10 +58,7 @@ class AsyncStream(): while True: try: - if self.state == self.STATE_UP and self.server_side: - await asyncio.wait_for(self.__async_read(), 150) - else: - await self.__async_read() + await self.__async_read() if self.unique_id: await self.async_write() @@ -84,12 +80,6 @@ class AsyncStream(): self.close() return self - except asyncio.TimeoutError: - logger.warning(f"Timeout for {self.l_addr}") - await self.disc() - self.close() - return self - except Exception: self.inc_counter('SW_Exception') logger.error( From f4da16987f5084042db709c6ce9f33564f8e3f5a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 18 May 2024 20:18:15 +0200 Subject: [PATCH 066/118] add fifo and timeout handler for modbus --- app/src/gen3/talent.py | 25 +++--- app/src/gen3plus/solarman_v5.py | 23 +++--- app/src/messages.py | 9 +- app/src/modbus.py | 141 +++++++++++++++++++++----------- 4 files changed, 129 insertions(+), 69 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 976d468..12fcf6c 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -36,7 +36,7 @@ class Control: class Talent(Message): def __init__(self, server_side: bool, id_str=b''): - super().__init__(server_side) + super().__init__(server_side, self.send_modbus_cb, 15) self.await_conn_resp_cnt = 0 self.id_str = id_str self.contact_name = b'' @@ -65,6 +65,7 @@ class Talent(Message): # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() self.state = self.STATE_CLOSED + super().close() def __set_serial_no(self, serial_no: str): @@ -122,20 +123,22 @@ class Talent(Message): f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}') return - async def send_modbus_cmd(self, func, addr, val) -> None: - if self.state != self.STATE_UP: - return + def send_modbus_cb(self, modbus_pdu: bytearray): self.forward_modbus_resp = False self.__build_header(0x70, 0x77) self._send_buffer += b'\x00\x01\xa3\x28' # fixme - modbus_msg = self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) - self._send_buffer += struct.pack('!B', len(modbus_msg)) - self._send_buffer += modbus_msg + self._send_buffer += struct.pack('!B', len(modbus_pdu)) + self._send_buffer += modbus_pdu self.__finish_send_msg() - try: - await self.async_write('Send Modbus Command:') - except Exception: - self._send_buffer = bytearray(0) + hex_dump_memory(logging.INFO, f'Send Modbus Command:{self.addr}:', + self._send_buffer, len(self._send_buffer)) + self.writer.write(self._send_buffer) + self._send_buffer = bytearray(0) # self._send_buffer[sent:] + + async def send_modbus_cmd(self, func, addr, val) -> None: + if self.state != self.STATE_UP: + return + self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) def _init_new_client_conn(self) -> bool: contact_name = self.contact_name diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 91d2bd0..f744951 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -52,7 +52,7 @@ class SolarmanV5(Message): MB_RTU_CMD = 2 def __init__(self, server_side: bool): - super().__init__(server_side) + super().__init__(server_side, self.send_modbus_cb, 5) self.header_len = 11 # overwrite construcor in class Message self.control = 0 @@ -104,6 +104,7 @@ class SolarmanV5(Message): # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() self.state = self.STATE_CLOSED + super().close() def __set_serial_no(self, snr: int): serial_no = str(snr) @@ -301,20 +302,22 @@ class SolarmanV5(Message): self._heartbeat()) self.__finish_send_msg() - async def send_modbus_cmd(self, func, addr, val) -> None: - if self.state != self.STATE_UP: - return + def send_modbus_cb(self, pdu: bytearray): self.forward_modbus_resp = False self.__build_header(0x4510) self._send_buffer += struct.pack(' None: + if self.state != self.STATE_UP: + return + self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) async def send_at_cmd(self, AT_cmd: str) -> None: if self.state != self.STATE_UP: diff --git a/app/src/messages.py b/app/src/messages.py index 01e3429..4968609 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -56,12 +56,14 @@ class Message(metaclass=IterRegistry): STATE_UP = 2 STATE_CLOSED = 3 - def __init__(self, server_side: bool): + def __init__(self, server_side: bool, send_modbus_cb, mb_timeout): self._registry.append(weakref.ref(self)) self.server_side = server_side if server_side: - self.mb = Modbus() + self.mb = Modbus(send_modbus_cb, mb_timeout) + else: + self.mb = None self.header_valid = False self.header_len = 0 @@ -91,6 +93,9 @@ class Message(metaclass=IterRegistry): Our puplic methods ''' def close(self) -> None: + if self.mb: + del self.mb + self.mb = None pass # pragma: no cover def inc_counter(self, counter: str) -> None: diff --git a/app/src/modbus.py b/app/src/modbus.py index 3ef0f5f..9302876 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -1,5 +1,6 @@ import struct import logging +import asyncio from typing import Generator if __name__ == "app.src.modbus": @@ -65,85 +66,133 @@ class Modbus(): 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } - def __init__(self): + def __init__(self, snd_handler, timeout: int = 1): if not len(self.__crc_tab): self.__build_crc_tab(CRC_POLY) + self.que = asyncio.Queue(100) + self.snd_handler = snd_handler + self.timeout = timeout + self.last_addr = 0 self.last_fcode = 0 self.last_len = 0 self.last_reg = 0 self.err = 0 + self.loop = asyncio.get_event_loop() + self.req_pend = False + self.tim = None - def build_msg(self, addr, func, reg, val): + def start_timer(self): + if self.req_pend: + return + self.req_pend = True + self.tim = self.loop.call_later(self.timeout, self.timeout_cb) + # logging.debug(f'Modbus start timer {self}') + + def stop_timer(self): + self.req_pend = False + # logging.debug(f'Modbus stop timer {self}') + if self.tim: + self.tim.cancel() + self.get_next_req() + + def timeout_cb(self): + self.req_pend = False + logging.info(f'Modbus timeout {self}') + self.get_next_req() + + def get_next_req(self) -> None: + if self.req_pend: + return + try: + item = self.que.get_nowait() + req = item['req'] + self.rsp_handler = item['rsp_hdl'] + self.last_addr = req[0] + self.last_fcode = req[1] + + res = struct.unpack_from('>HH', req, 2) + self.last_reg = res[0] + self.last_len = res[1] + self.start_timer() + self.snd_handler(req) + except asyncio.QueueEmpty: + pass + + def build_msg(self, addr, func, reg, val) -> None: msg = struct.pack('>BBHH', addr, func, reg, val) msg += struct.pack(' bool: + def recv_req(self, buf: bytearray, rsp_handler=None) -> bool: # logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') if not self.check_crc(buf): self.err = 1 logging.error('Modbus recv: CRC error') return False - if buf[0] != self.INV_ADDR: - self.err = 2 - logging.info(f'Modbus recv: Wrong addr{buf[0]}') - return False - res = struct.unpack_from('>BHH', buf, 1) - self.last_fcode = res[0] - self.last_reg = res[1] - self.last_len = res[2] - self.err = 0 + self.que.put_nowait({'req': buf, + 'rsp_hdl': rsp_handler}) + if self.que.qsize() == 1: + self.get_next_req() + return True def recv_resp(self, info_db, buf: bytearray, node_id: str) -> \ Generator[tuple[str, bool], None, None]: # logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') + if not self.req_pend: + self.err = 5 + return if not self.check_crc(buf): logging.error('Modbus resp: CRC error') self.err = 1 return - if buf[0] != self.INV_ADDR: + if buf[0] != self.last_addr: logging.info(f'Modbus resp: Wrong addr {buf[0]}') self.err = 2 return - if buf[1] != self.last_fcode: - logging.info(f'Modbus: Wrong fcode {buf[1]} != {self.last_fcode}') + fcode = buf[1] + if fcode != self.last_fcode: + logging.info(f'Modbus: Wrong fcode {fcode} != {self.last_fcode}') self.err = 3 return - elmlen = buf[2] >> 1 - if elmlen != self.last_len: - logging.info(f'Modbus: len error {elmlen} != {self.last_len}') - self.err = 4 - return - self.err = 0 + if self.last_addr == self.INV_ADDR and \ + (fcode == 3 or fcode == 4): + elmlen = buf[2] >> 1 + if elmlen != self.last_len: + logging.info(f'Modbus: len error {elmlen} != {self.last_len}') + self.err = 4 + return + self.stop_timer() - for i in range(0, elmlen): - addr = self.last_reg+i - if addr in self.map: - row = self.map[addr] - info_id = row['reg'] - fmt = row['fmt'] - val = struct.unpack_from(fmt, buf, 3+2*i) - result = val[0] + for i in range(0, elmlen): + addr = self.last_reg+i + if addr in self.map: + row = self.map[addr] + info_id = row['reg'] + fmt = row['fmt'] + val = struct.unpack_from(fmt, buf, 3+2*i) + result = val[0] - if 'eval' in row: - result = eval(row['eval']) - if 'ratio' in row: - result = round(result * row['ratio'], 2) + if 'eval' in row: + result = eval(row['eval']) + if 'ratio' in row: + result = round(result * row['ratio'], 2) - keys, level, unit, must_incr = info_db._key_obj(info_id) + keys, level, unit, must_incr = info_db._key_obj(info_id) - if keys: - name, update = info_db.update_db(keys, must_incr, result) - yield keys[0], update, result - if update: - info_db.tracer.log(level, - f'[\'{node_id}\']MODBUS: {name}' - f' : {result}{unit}') + if keys: + name, update = info_db.update_db(keys, must_incr, + result) + yield keys[0], update, result + if update: + info_db.tracer.log(level, + f'[\'{node_id}\']MODBUS: {name}' + f' : {result}{unit}') + else: + self.stop_timer() def check_crc(self, msg) -> bool: return 0 == self.__calc_crc(msg) From 766774224bbff23c0e7e36c9f252af6a66de4b1c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 18 May 2024 21:46:28 +0200 Subject: [PATCH 067/118] adapt unit tests --- app/tests/test_modbus.py | 177 ++++++++++++++++++++++++++++++------- app/tests/test_solarman.py | 17 +++- app/tests/test_talent.py | 20 ++++- 3 files changed, 177 insertions(+), 37 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index ed6a68a..8ea4637 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -1,15 +1,24 @@ # test_with_pytest.py -# import pytest, logging +import pytest +import asyncio from app.src.modbus import Modbus from app.src.infos import Infos +pytest_plugins = ('pytest_asyncio',) +pytestmark = pytest.mark.asyncio(scope="module") + class TestHelper(Modbus): def __init__(self): - super().__init__() + super().__init__(self.send_cb) self.db = Infos() + self.pdu = None + self.send_calls = 0 + def send_cb(self, pdu: bytearray): + self.pdu = pdu + self.send_calls += 1 def test_modbus_crc(): - mb = Modbus() + mb = Modbus(None) assert 0x0b02 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04') assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') assert mb.check_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') @@ -20,33 +29,34 @@ def test_modbus_crc(): assert 0x5c75 == mb._Modbus__calc_crc(b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46') def test_build_modbus_pdu(): - mb = Modbus() - pdu = mb.build_msg(1,6,0x2000,0x12) - assert pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' - assert mb.check_crc(pdu) + mb = TestHelper() + mb.build_msg(1,6,0x2000,0x12) + mb.get_next_req() + assert mb.pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' + assert mb.check_crc(mb.pdu) def test_recv_req_crc(): - mb = Modbus() - res = mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') - assert not res + mb = TestHelper() + mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') + mb.get_next_req() assert mb.last_fcode == 0 assert mb.last_reg == 0 assert mb.last_len == 0 assert mb.err == 1 def test_recv_req_addr(): - mb = Modbus() - res = mb.recv_req(b'\x02\x06\x20\x00\x00\x12\x02\x34') - assert not res - assert mb.last_fcode == 0 - assert mb.last_reg == 0 - assert mb.last_len == 0 - assert mb.err == 2 + mb = TestHelper() + mb.recv_req(b'\x02\x06\x20\x00\x00\x12\x02\x34') + mb.get_next_req() + assert mb.last_addr == 2 + assert mb.last_fcode == 6 + assert mb.last_reg == 0x2000 + assert mb.last_len == 18 def test_recv_req(): - mb = Modbus() - res = mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') - assert res + mb = TestHelper() + mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') + mb.get_next_req() assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 assert mb.last_len == 0x12 @@ -54,6 +64,7 @@ def test_recv_req(): def test_recv_recv_crc(): mb = TestHelper() + mb.req_pend = True mb.last_fcode = 3 mb.last_reg == 0x300e mb.last_len == 2 @@ -66,6 +77,7 @@ def test_recv_recv_crc(): def test_recv_recv_addr(): mb = TestHelper() + mb.req_pend = True mb.last_fcode = 3 mb.last_reg == 0x300e mb.last_len == 2 @@ -75,36 +87,48 @@ def test_recv_recv_addr(): call += 1 assert mb.err == 2 assert 0 == call + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend def test_recv_recv_fcode(): mb = TestHelper() - mb.last_fcode = 4 - mb.last_reg == 0x300e - mb.last_len == 2 - + mb.build_msg(1,4,0x300e,2) + assert mb.que.qsize() == 0 + assert mb.req_pend + call = 0 for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): call += 1 + assert mb.err == 3 assert 0 == call + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend def test_recv_recv_len(): mb = TestHelper() - mb.last_fcode = 3 - mb.last_reg == 0x300e - mb.last_len == 2 - + mb.build_msg(1,3,0x300e,3) + assert mb.que.qsize() == 0 + assert mb.req_pend + assert mb.last_len == 3 call = 0 for key, update, _ in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): call += 1 + assert mb.err == 4 assert 0 == call + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend def test_build_recv(): mb = TestHelper() - pdu = mb.build_msg(1,3,0x3007,6) - assert mb.check_crc(pdu) - assert mb.err == 0 + mb.build_msg(1,3,0x3007,6) + assert mb.que.qsize() == 0 + assert mb.req_pend + call = 0 exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): @@ -121,6 +145,7 @@ def test_build_recv(): assert 0 == mb.err assert 5 == call + mb.req_pend = True call = 0 for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): if key == 'grid': @@ -133,13 +158,20 @@ def test_build_recv(): assert False assert exp_result[call] == val call += 1 + assert 0 == mb.err assert 5 == call + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend def test_build_long(): mb = TestHelper() - pdu = mb.build_msg(1,3,0x3022,4) - assert mb.check_crc(pdu) + mb.build_msg(1,3,0x3022,4) + assert mb.que.qsize() == 0 + assert mb.req_pend + assert mb.last_addr == 1 + assert mb.last_fcode == 3 assert mb.err == 0 call = 0 exp_result = [3.0, 28841.4, 113.34] @@ -150,5 +182,84 @@ def test_build_long(): else: assert False call += 1 + assert 0 == mb.err assert 3 == call + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend + +def test_queue(): + mb = TestHelper() + mb.build_msg(1,3,0x3022,4) + assert mb.que.qsize() == 0 + assert mb.req_pend + + assert mb.send_calls == 1 + assert mb.pdu == b'\x01\x030"\x00\x04\xeb\x03' + mb.pdu = None + mb.get_next_req() + assert mb.send_calls == 1 + assert mb.pdu == None + + assert mb.que.qsize() == 0 + mb.stop_timer() + assert not mb.req_pend + +def test_queue2(): + mb = TestHelper() + mb.build_msg(1,3,0x3007,6) + mb.build_msg(1,6,0x2008,4) + assert mb.que.qsize() == 1 + assert mb.req_pend + + assert mb.send_calls == 1 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + mb.get_next_req() + assert mb.send_calls == 1 + call = 0 + exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): + if key == 'grid': + assert update == True + elif key == 'inverter': + assert update == True + elif key == 'env': + assert update == True + else: + assert False + assert exp_result[call] == val + call += 1 + assert 0 == mb.err + assert 5 == call + + assert mb.send_calls == 2 + assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' + + for key, update, val in mb.recv_resp(mb.db, b'\x01\x06\x20\x08\x00\x04\x02\x0b', 'test'): + pass + + assert mb.que.qsize() == 0 + assert not mb.req_pend + + +@pytest.mark.asyncio +async def test_timeout(): + assert asyncio.get_running_loop() + mb = TestHelper() + assert asyncio.get_running_loop() == mb.loop + mb.build_msg(1,3,0x3007,6) + mb.build_msg(1,6,0x2008,4) + assert mb.que.qsize() == 1 + assert mb.req_pend + + assert mb.send_calls == 1 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + await asyncio.sleep(1.1) # wait for first timeout and next pdu + assert mb.req_pend + assert mb.send_calls == 2 + assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' + await asyncio.sleep(1.1) # wait for second timout + + assert not mb.req_pend + assert mb.que.qsize() == 0 diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 46df675..19132d7 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -16,9 +16,16 @@ Infos.static_init() timestamp = int(time.time()) # 1712861197 heartbeat = 60 +class Writer(): + def write(self, pdu: bytearray): + pass + class MemoryStream(SolarmanV5): def __init__(self, msg, chunks = (0,), server_side: bool = True): super().__init__(server_side) + if server_side: + self.mb.timeout = 1 # overwrite for faster testing + self.writer = Writer() self.__msg = msg self.__msg_len = len(msg) self.__chunks = chunks @@ -35,7 +42,6 @@ class MemoryStream(SolarmanV5): def _heartbeat(self) -> int: return heartbeat - def append_msg(self, msg): self.__msg += msg @@ -1446,9 +1452,12 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): m.append_msg(MsgModbusRsp) m.forward_modbus_resp = True + m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 # assert m.db.db == {'inverter': {'Manufacturer': 'TSUN', 'Equipment_Model': 'TSOL-MSxx00'}} m.new_data['inverter'] = False @@ -1465,7 +1474,7 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.mb.err == 0 + assert m.mb.err == 5 assert m.msg_count == 2 assert m._forward_buffer==MsgModbusRsp assert m._send_buffer==b'' @@ -1515,9 +1524,13 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusRsp): m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = True + m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 + m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index ed39111..c0b8e1e 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -12,10 +12,18 @@ pytest_plugins = ('pytest_asyncio',) Infos.static_init() tracer = logging.getLogger('tracer') - + + +class Writer(): + def write(self, pdu: bytearray): + pass + class MemoryStream(Talent): def __init__(self, msg, chunks = (0,), server_side: bool = True): super().__init__(server_side) + if server_side: + self.mb.timeout = 1 # overwrite for faster testing + self.writer = Writer() self.__msg = msg self.__msg_len = len(msg) self.__chunks = chunks @@ -896,9 +904,13 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): m.append_msg(MsgModbusResp20) m.forward_modbus_resp = True + m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 + assert m.db.db == {} m.new_data['inverter'] = False @@ -915,7 +927,7 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.mb.err == 0 + assert m.mb.err == 5 assert m.msg_count == 2 assert m._forward_buffer==MsgModbusResp20 assert m._send_buffer==b'' @@ -952,9 +964,13 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.forward_modbus_resp = True + m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 + m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 From 9c39ea27f7153ababdf7e48f86b773cc0e0799a3 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 18 May 2024 23:10:47 +0200 Subject: [PATCH 068/118] fix unit tests --- app/tests/test_solarman.py | 11 ++++++++--- app/tests/test_talent.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 19132d7..b1d0a04 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -17,8 +17,11 @@ timestamp = int(time.time()) # 1712861197 heartbeat = 60 class Writer(): + def __init__(self): + self.sent_pdu = b'' + def write(self, pdu: bytearray): - pass + self.sent_pdu = pdu class MemoryStream(SolarmanV5): def __init__(self, msg, chunks = (0,), server_side: bool = True): @@ -1200,7 +1203,8 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, assert m._recv_buffer==InverterIndMsg # unhandled next message assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' - assert m._send_buffer == b'' # modbus command must be ignore, cause connection is still not up + assert m.writer.sent_pdu == b'' # modbus command must be ignore, cause connection is still not up + assert m._send_buffer == b'' # modbus command must be ignore, cause connection is still not up m.read() assert m.control == 0x4210 @@ -1214,7 +1218,8 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, DeviceIndMsg, DeviceRspMsg, await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' - assert m._send_buffer == MsgModbusCmd + assert m.writer.sent_pdu == MsgModbusCmd + assert m._send_buffer == b'' m._send_buffer = bytearray(0) # clear send buffer for next test m.test_exception_async_write = True diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index c0b8e1e..01149f9 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -15,8 +15,11 @@ tracer = logging.getLogger('tracer') class Writer(): + def __init__(self): + self.sent_pdu = b'' + def write(self, pdu: bytearray): - pass + self.sent_pdu = pdu class MemoryStream(Talent): def __init__(self, msg, chunks = (0,), server_side: bool = True): @@ -997,19 +1000,22 @@ async def test_msg_build_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' assert m._send_buffer == b'' + assert m.writer.sent_pdu == b'' m.state = m.STATE_UP await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' - assert m._send_buffer == MsgModbusCmd + assert m._send_buffer == b'' + assert m.writer.sent_pdu == MsgModbusCmd - m._send_buffer = bytearray(0) # clear send buffer for next test + m.writer.sent_pdu = bytearray(0) # clear send buffer for next test m.test_exception_async_write = True await m.send_modbus_cmd(Modbus.WRITE_SINGLE_REG, 0x2008, 0) assert 0 == m.send_msg_ofs assert m._forward_buffer == b'' assert m._send_buffer == b'' + assert m.writer.sent_pdu == b'' m.close() ''' def test_zombie_conn(ConfigTsunInv1, MsgInverterInd): From d25173e591c8357783df53ff9579b3ee2e949a13 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 18 May 2024 23:11:49 +0200 Subject: [PATCH 069/118] fix sending next pdu before we have parsed the last response --- app/src/modbus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 9302876..930df86 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -165,10 +165,10 @@ class Modbus(): logging.info(f'Modbus: len error {elmlen} != {self.last_len}') self.err = 4 return - self.stop_timer() - + first_reg = self.last_reg # save last_reg before sending next pdu + self.stop_timer() # stop timer and send next pdu for i in range(0, elmlen): - addr = self.last_reg+i + addr = first_reg+i if addr in self.map: row = self.map[addr] info_id = row['reg'] From 282a459ef0b2d9f9019fb462058e824ce04fa104 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 12:23:58 +0200 Subject: [PATCH 070/118] add Modbus response forwarding --- app/src/gen3/talent.py | 13 ++++++------- app/src/gen3plus/solarman_v5.py | 12 ++++-------- app/src/modbus.py | 7 ++++++- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 12fcf6c..c76c775 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -42,7 +42,6 @@ class Talent(Message): self.contact_name = b'' self.contact_mail = b'' self.db = InfosG3() - self.forward_modbus_resp = False self.switch = { 0x00: self.msg_contact_info, 0x13: self.msg_ota_update, @@ -124,7 +123,6 @@ class Talent(Message): return def send_modbus_cb(self, modbus_pdu: bytearray): - self.forward_modbus_resp = False self.__build_header(0x70, 0x77) self._send_buffer += b'\x00\x01\xa3\x28' # fixme self._send_buffer += struct.pack('!B', len(modbus_pdu)) @@ -394,11 +392,11 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): - if not self.remoteStream.mb.recv_req(data[hdr_len:]): + if not self.remoteStream.mb.recv_req(data[hdr_len:], + self.msg_forward): self.inc_counter('Invalid_Msg_Format') else: self.inc_counter('Modbus_Command') - self.remoteStream.forward_modbus_resp = True elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 @@ -408,14 +406,15 @@ class Talent(Message): if update: self.new_data[key] = True self.modbus_elms += 1 # count for unit tests - - if not self.forward_modbus_resp: - return + return else: logger.warning('Unknown Ctrl') self.inc_counter('Unknown_Ctrl') self.forward(self._recv_buffer, self.header_len+self.data_len) + def msg_forward(self): + self.forward(self._recv_buffer, self.header_len+self.data_len) + def msg_unknown(self): logger.warning(f"Unknow Msg: ID:{self.msg_id}") self.inc_counter('Unknown_Msg') diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index f744951..2d61b68 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -60,7 +60,6 @@ class SolarmanV5(Message): self.snr = 0 self.db = InfosG3P() self.time_ofs = 0 - self.forward_modbus_resp = False self.forward_at_cmd_resp = False self.switch = { @@ -303,7 +302,6 @@ class SolarmanV5(Message): self.__finish_send_msg() def send_modbus_cb(self, pdu: bytearray): - self.forward_modbus_resp = False self.__build_header(0x4510) self._send_buffer += struct.pack(' bool: return 0 == self.__calc_crc(msg) From 476c5f000673a5fcac3f9fbfe2e4f88c655909ff Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 12:24:35 +0200 Subject: [PATCH 071/118] adapt unit tests --- app/tests/test_solarman.py | 35 ++++------------------------------- app/tests/test_talent.py | 30 +++++------------------------- 2 files changed, 9 insertions(+), 56 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index b1d0a04..e339412 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -1277,7 +1277,6 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIn def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): ConfigTsunInv1 m = MemoryStream(AtCommandIndMsg, (0,), False) - m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 @@ -1296,7 +1295,6 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 assert m.db.stat['proxy']['AT_Command'] == 1 assert m.db.stat['proxy']['Modbus_Command'] == 0 - assert m.forward_modbus_resp == False m.close() def test_msg_at_command_rsp1(ConfigTsunInv1, AtCommandRspMsg): @@ -1342,7 +1340,6 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): m = MemoryStream(b'') c = m.createClientStream(MsgModbusCmd) - m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 @@ -1360,7 +1357,6 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 - assert m.forward_modbus_resp == True m.close() def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): @@ -1368,7 +1364,6 @@ def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): m = MemoryStream(b'') c = m.createClientStream(MsgModbusCmdCrcErr) - m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 @@ -1386,13 +1381,11 @@ def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 assert m.db.stat['proxy']['Invalid_Msg_Format'] == 1 - assert m.forward_modbus_resp == True m.close() def test_msg_unknown_cmd_req(ConfigTsunInv1, MsgUnknownCmd): ConfigTsunInv1 m = MemoryStream(MsgUnknownCmd, (0,), False) - m.forward_modbus_resp = False m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 @@ -1410,15 +1403,14 @@ def test_msg_unknown_cmd_req(ConfigTsunInv1, MsgUnknownCmd): assert m.db.stat['proxy']['AT_Command'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 - assert m.forward_modbus_resp == False m.close() def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): + '''Modbus response without a valid Modbus request must be dropped''' ConfigTsunInv1 m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = False m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 @@ -1433,30 +1425,12 @@ def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): m.close() def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): - ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp) - m.db.stat['proxy']['Unknown_Ctrl'] = 0 - m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = True - m.read() # read complete msg, and dispatch msg - assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.msg_count == 1 - assert m.control == 0x1510 - assert str(m.seq) == '03:03' - assert m.header_len==11 - assert m.data_len==59 - assert m._forward_buffer==MsgModbusRsp - assert m._send_buffer==b'' - assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 - assert m.db.stat['proxy']['Modbus_Command'] == 0 - m.close() - -def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): + '''Modbus response with a valid Modbus request must be forwarded''' ConfigTsunInv1 m = MemoryStream(MsgModbusRsp) m.append_msg(MsgModbusRsp) - m.forward_modbus_resp = True + m.mb.rsp_handler = m._SolarmanV5__forward_msg m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 @@ -1494,7 +1468,6 @@ def test_msg_unknown_rsp(ConfigTsunInv1, MsgUnknownCmdRsp): m = MemoryStream(MsgUnknownCmdRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = True m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 @@ -1528,7 +1501,7 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusRsp): m = MemoryStream(MsgModbusRsp+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = True + m.mb.rsp_handler = m._SolarmanV5__forward_msg m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 01149f9..1f53c8c 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -860,11 +860,11 @@ def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): m.close() def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): + '''Modbus response without a valid Modbus request must be dropped''' ConfigTsunInv1 m = MemoryStream(MsgModbusRsp) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = False m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 @@ -880,33 +880,13 @@ def test_msg_modbus_rsp1(ConfigTsunInv1, MsgModbusRsp): assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() -def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): - ConfigTsunInv1 - m = MemoryStream(MsgModbusRsp) - m.db.stat['proxy']['Unknown_Ctrl'] = 0 - m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = True - m.read() # read complete msg, and dispatch msg - assert not m.header_valid # must be invalid, since msg was handled and buffer flushed - assert m.msg_count == 1 - assert m.id_str == b"R170000000000001" - assert m.unique_id == 'R170000000000001' - assert int(m.ctrl)==145 - assert m.msg_id==119 - assert m.header_len==23 - assert m.data_len==13 - assert m._forward_buffer==MsgModbusRsp - assert m._send_buffer==b'' - assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 - assert m.db.stat['proxy']['Modbus_Command'] == 0 - m.close() - -def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): +def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusResp20): + '''Modbus response with a valid Modbus request must be forwarded''' ConfigTsunInv1 m = MemoryStream(MsgModbusResp20) m.append_msg(MsgModbusResp20) - m.forward_modbus_resp = True + m.mb.rsp_handler = m.msg_forward m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 @@ -966,7 +946,7 @@ def test_msg_modbus_fragment(ConfigTsunInv1, MsgModbusResp20): m = MemoryStream(MsgModbusResp20+b'\x00\x00\x00\x45\x10\x52\x31', (0,)) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 - m.forward_modbus_resp = True + m.mb.rsp_handler = m.msg_forward m.mb.last_addr = 1 m.mb.last_fcode = 3 m.mb.last_len = 20 From f30aa07431dfa65696de718df78d858ca0ea141e Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 13:42:29 +0200 Subject: [PATCH 072/118] don't frwd received modbus req directly - use always the fifoto sent valid req to the inverter - code cleanup --- app/src/gen3/talent.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index c76c775..c9cf89d 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -392,11 +392,11 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): - if not self.remoteStream.mb.recv_req(data[hdr_len:], - self.msg_forward): - self.inc_counter('Invalid_Msg_Format') + if self.remoteStream.mb.recv_req(data[hdr_len:], + self.msg_forward): + self.remoteStream.inc_counter('Modbus_Command') else: - self.inc_counter('Modbus_Command') + self.remoteStream.inc_counter('Invalid_Msg_Format') elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 @@ -406,11 +406,10 @@ class Talent(Message): if update: self.new_data[key] = True self.modbus_elms += 1 # count for unit tests - return else: logger.warning('Unknown Ctrl') self.inc_counter('Unknown_Ctrl') - self.forward(self._recv_buffer, self.header_len+self.data_len) + self.forward(self._recv_buffer, self.header_len+self.data_len) def msg_forward(self): self.forward(self._recv_buffer, self.header_len+self.data_len) From c761446c11084e2848be5c940cc5916bf4453f86 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 13:43:51 +0200 Subject: [PATCH 073/118] code cleanup --- app/src/gen3plus/solarman_v5.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 2d61b68..80bcc8a 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -430,11 +430,11 @@ class SolarmanV5(Message): self.inc_counter('AT_Command') self.forward_at_cmd_resp = True elif ftype == self.MB_RTU_CMD: - if not self.remoteStream.mb.recv_req(data[15:], - self.__forward_msg()): - self.inc_counter('Invalid_Msg_Format') - else: + if self.remoteStream.mb.recv_req(data[15:], + self.__forward_msg()): self.inc_counter('Modbus_Command') + else: + self.inc_counter('Invalid_Msg_Format') return self.__forward_msg() From 23ff2bb05cc2ee040f7303d882ae39edb6b90dac Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 13:44:16 +0200 Subject: [PATCH 074/118] fix unit tests --- app/tests/test_talent.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 1f53c8c..ad7b6cb 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -814,6 +814,7 @@ def test_proxy_counter(): def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(b'') + m.id_str = b"R170000000000001" c = m.createClientStream(MsgModbusCmd) m.db.stat['proxy']['Unknown_Ctrl'] = 0 @@ -828,8 +829,12 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert c.msg_id==119 assert c.header_len==23 assert c.data_len==13 - assert c._forward_buffer==MsgModbusCmd + assert c._forward_buffer==b'' assert c._send_buffer==b'' + assert m.id_str == b"R170000000000001" + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.writer.sent_pdu == MsgModbusCmd assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 1 assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 @@ -838,6 +843,7 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): ConfigTsunInv1 m = MemoryStream(b'') + m.id_str = b"R170000000000001" c = m.createClientStream(MsgModbusCmdCrcErr) m.db.stat['proxy']['Unknown_Ctrl'] = 0 @@ -852,8 +858,11 @@ def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): assert c.msg_id==119 assert c.header_len==23 assert c.data_len==13 - assert c._forward_buffer==MsgModbusCmdCrcErr + assert c._forward_buffer==b'' assert c._send_buffer==b'' + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.writer.sent_pdu ==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 assert m.db.stat['proxy']['Invalid_Msg_Format'] == 1 From 3cc5f3ec530c602b0d75e0de1715a2806b115cf2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 13:45:52 +0200 Subject: [PATCH 075/118] - add Modbus fifo and timeout handler --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed901bc..8412be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- add Modbus fifo and timeout handler - build version string in the same format as TSUN for GEN3 invterts - add graceful shutdown - parse Modbus values and store them in the database From 9ac1f6f46d5ddefa526e3a39dc363c4a2a51a729 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 21:17:16 +0200 Subject: [PATCH 076/118] add Modbus retrasmissions --- app/src/gen3/talent.py | 21 ++++++++++++++++----- app/src/gen3plus/solarman_v5.py | 10 ++++++++-- app/src/modbus.py | 32 +++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index c9cf89d..94eec89 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -36,7 +36,7 @@ class Control: class Talent(Message): def __init__(self, server_side: bool, id_str=b''): - super().__init__(server_side, self.send_modbus_cb, 15) + super().__init__(server_side, self.send_modbus_cb, 11) self.await_conn_resp_cnt = 0 self.id_str = id_str self.contact_name = b'' @@ -122,13 +122,21 @@ class Talent(Message): f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}') return - def send_modbus_cb(self, modbus_pdu: bytearray): + def send_modbus_cb(self, modbus_pdu: bytearray, retrans: bool): + if self.state != self.STATE_UP: + return + self.__build_header(0x70, 0x77) self._send_buffer += b'\x00\x01\xa3\x28' # fixme self._send_buffer += struct.pack('!B', len(modbus_pdu)) self._send_buffer += modbus_pdu self.__finish_send_msg() - hex_dump_memory(logging.INFO, f'Send Modbus Command:{self.addr}:', + if retrans: + cmd = 'Retrans' + else: + cmd = 'Command' + + hex_dump_memory(logging.INFO, f'Send Modbus {cmd}:{self.addr}:', self._send_buffer, len(self._send_buffer)) self.writer.write(self._send_buffer) self._send_buffer = bytearray(0) # self._send_buffer[sent:] @@ -392,11 +400,14 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): + # if (self.remoteStream.state != self.STATE_UP): + # logger.info('ignore Modbus Request in wrong state') + # return if self.remoteStream.mb.recv_req(data[hdr_len:], self.msg_forward): - self.remoteStream.inc_counter('Modbus_Command') + self.inc_counter('Modbus_Command') else: - self.remoteStream.inc_counter('Invalid_Msg_Format') + self.inc_counter('Invalid_Msg_Format') elif self.ctrl.is_ind(): # logger.debug(f'Modbus Ind MsgLen: {modbus_len}') self.modbus_elms = 0 diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 80bcc8a..7ac8310 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -301,13 +301,19 @@ class SolarmanV5(Message): self._heartbeat()) self.__finish_send_msg() - def send_modbus_cb(self, pdu: bytearray): + def send_modbus_cb(self, pdu: bytearray, retrans: bool): + if self.state != self.STATE_UP: + return self.__build_header(0x4510) self._send_buffer += struct.pack(' None: if self.req_pend: @@ -106,6 +126,7 @@ class Modbus(): try: item = self.que.get_nowait() req = item['req'] + self.last_req = req self.rsp_handler = item['rsp_hdl'] self.last_addr = req[0] self.last_fcode = req[1] @@ -113,8 +134,9 @@ class Modbus(): res = struct.unpack_from('>HH', req, 2) self.last_reg = res[0] self.last_len = res[1] + self.retry_cnt = 0 self.start_timer() - self.snd_handler(req) + self.snd_handler(self.last_req, retrans=False) except asyncio.QueueEmpty: pass @@ -140,7 +162,7 @@ class Modbus(): return True def recv_resp(self, info_db, buf: bytearray, node_id: str) -> \ - Generator[tuple[str, bool], None, None]: + Generator[tuple[str, bool, any], None, None]: # logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') if not self.req_pend: self.err = 5 @@ -194,7 +216,7 @@ class Modbus(): f' : {result}{unit}') else: self.stop_timer() - + self.counter['retries'][self.retry_cnt] += 1 if self.rsp_handler: self.rsp_handler() self.get_next_req() From 177706c3e6fb0905ecaa0074e4697ed1a75c47af Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 19 May 2024 21:17:56 +0200 Subject: [PATCH 077/118] test Modbus retries --- app/tests/test_modbus.py | 43 ++++++++++++++++++++++++++++++++-------- app/tests/test_talent.py | 2 ++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index 8ea4637..fed9c09 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -13,7 +13,7 @@ class TestHelper(Modbus): self.db = Infos() self.pdu = None self.send_calls = 0 - def send_cb(self, pdu: bytearray): + def send_cb(self, pdu: bytearray, retrans: bool): self.pdu = pdu self.send_calls += 1 @@ -247,19 +247,46 @@ def test_queue2(): async def test_timeout(): assert asyncio.get_running_loop() mb = TestHelper() + mb.max_retries = 2 assert asyncio.get_running_loop() == mb.loop mb.build_msg(1,3,0x3007,6) mb.build_msg(1,6,0x2008,4) + assert mb.que.qsize() == 1 assert mb.req_pend - + assert mb.retry_cnt == 0 assert mb.send_calls == 1 assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' - await asyncio.sleep(1.1) # wait for first timeout and next pdu - assert mb.req_pend - assert mb.send_calls == 2 - assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' - await asyncio.sleep(1.1) # wait for second timout - assert not mb.req_pend + mb.pdu = None + await asyncio.sleep(1.1) # wait for first timeout and retransmittion + assert mb.que.qsize() == 1 + assert mb.req_pend + assert mb.retry_cnt == 1 + assert mb.send_calls == 2 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + + mb.pdu = None + await asyncio.sleep(1.1) # wait for second timeout and retransmittion + assert mb.que.qsize() == 1 + assert mb.req_pend + assert mb.retry_cnt == 2 + assert mb.send_calls == 3 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + + mb.pdu = None + await asyncio.sleep(1.1) # wait for third timeout and next pdu assert mb.que.qsize() == 0 + assert mb.req_pend + assert mb.retry_cnt == 0 + assert mb.send_calls == 4 + assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' + + mb.max_retries = 0 # next pdu without retranmsission + await asyncio.sleep(1.1) # wait for fourth timout + assert mb.que.qsize() == 0 + assert not mb.req_pend + assert mb.retry_cnt == 0 + assert mb.send_calls == 4 + + # assert mb.counter == {} diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index ad7b6cb..3387d99 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -815,6 +815,8 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): ConfigTsunInv1 m = MemoryStream(b'') m.id_str = b"R170000000000001" + m.state = m.STATE_UP + c = m.createClientStream(MsgModbusCmd) m.db.stat['proxy']['Unknown_Ctrl'] = 0 From 6ef6f4cd3439a3ab3c9e28671bcc08023e414145 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 20 May 2024 00:48:23 +0200 Subject: [PATCH 078/118] cleanup --- app/src/modbus.py | 181 +++++++++++++++++++++++++-------------- app/tests/test_modbus.py | 31 +++---- 2 files changed, 133 insertions(+), 79 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 717909c..82c2f01 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -1,7 +1,7 @@ import struct import logging import asyncio -from typing import Generator +from typing import Generator, Callable if __name__ == "app.src.modbus": from app.src.infos import Register @@ -25,10 +25,13 @@ CRC_INIT = 0xFFFF class Modbus(): INV_ADDR = 1 + '''MODBUS slave address of the TSUN inverter''' READ_REGS = 3 + '''MODBUS function code: Read Holding Register''' READ_INPUTS = 4 + '''MODBUS function code: Read Input Register''' WRITE_SINGLE_REG = 6 - '''Modbus function codes''' + '''Modbus function code: Write Single Register''' __crc_tab = [] map = { @@ -66,21 +69,26 @@ class Modbus(): 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } - def __init__(self, snd_handler, timeout: int = 1): + def __init__(self, snd_handler: Callable[[bool], None], timeout: int = 1): if not len(self.__crc_tab): self.__build_crc_tab(CRC_POLY) self.que = asyncio.Queue(100) self.snd_handler = snd_handler + '''Send handler to transmit a MODBUS request''' self.rsp_handler = None + '''Response handler to forward the response''' self.timeout = timeout + '''MODBUS response timeout in seconds''' self.max_retries = 3 + '''Max retransmit for MODBUS requests''' self.retry_cnt = 0 self.last_req = b'' self.counter = {} + '''Dictenary with statistic counter''' self.counter['timeouts'] = 0 self.counter['retries'] = {} - for i in range(0, self.max_retries): - self.counter['retries'][i] = 0 + for i in range(0, self.max_retries+1): + self.counter['retries'][f'{i}'] = 0 self.last_addr = 0 self.last_fcode = 0 self.last_len = 0 @@ -94,80 +102,67 @@ class Modbus(): if type(self.counter) is not None: logging.info(f'Modbus __del__:\n {self.counter}') - def start_timer(self): - if self.req_pend: - return - self.req_pend = True - self.tim = self.loop.call_later(self.timeout, self.timeout_cb) - # logging.debug(f'Modbus start timer {self}') + def build_msg(self, addr: int, func: int, reg: int, val: int) -> None: + """Build MODBUS RTU message frame and add it to the tx queue - def stop_timer(self): - self.req_pend = False - # logging.debug(f'Modbus stop timer {self}') - if self.tim: - self.tim.cancel() - - def timeout_cb(self): - self.req_pend = False - - if self.retry_cnt < self.max_retries: - logging.debug(f'Modbus retrans {self}') - self.retry_cnt += 1 - self.start_timer() - self.snd_handler(self.last_req, retrans=True) - else: - logging.info(f'Modbus timeout {self}') - self.counter['timeouts'] += 1 - self.get_next_req() - - def get_next_req(self) -> None: - if self.req_pend: - return - try: - item = self.que.get_nowait() - req = item['req'] - self.last_req = req - self.rsp_handler = item['rsp_hdl'] - self.last_addr = req[0] - self.last_fcode = req[1] - - res = struct.unpack_from('>HH', req, 2) - self.last_reg = res[0] - self.last_len = res[1] - self.retry_cnt = 0 - self.start_timer() - self.snd_handler(self.last_req, retrans=False) - except asyncio.QueueEmpty: - pass - - def build_msg(self, addr, func, reg, val) -> None: + Keyword arguments: + addr: RTU slave address + func: MODBUS function code + reg: 16-bit register number + val: 16 bit value + """ msg = struct.pack('>BBHH', addr, func, reg, val) msg += struct.pack(' bool: + def recv_req(self, buf: bytearray, + rsp_handler: Callable[[None], None] = None) -> bool: + """Add the received Modbus request to the tx queue + + Keyword arguments: + buf: Modbus RTU pdu incl ADDR byte and trailing CRC + rsp_handler: Callback, if the received pdu is valid + + Returns: + True: PDU was added to the queue + False: PDU was ignored, due to an error + """ # logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') - if not self.check_crc(buf): + if not self.__check_crc(buf): self.err = 1 logging.error('Modbus recv: CRC error') return False self.que.put_nowait({'req': buf, 'rsp_hdl': rsp_handler}) if self.que.qsize() == 1: - self.get_next_req() + self.__send_next_from_que() return True def recv_resp(self, info_db, buf: bytearray, node_id: str) -> \ - Generator[tuple[str, bool, any], None, None]: + Generator[tuple[str, bool, int | float | str], None, None]: + """Generator which check and parse a received MODBUS response. + + Keyword arguments: + info_db: database for info lockups + buf: received Modbus RTU response frame + node_id: string for logging which identifies the slave + + Returns on error and set Self.err to: + 1: CRC error + 2: Wrong server address + 3: Unexpected function code + 4: Unexpected data length + 5: No MODBUS request pending + """ # logging.info(f'recv_resp: first byte modbus:{buf[0]} len:{len(buf)}') if not self.req_pend: self.err = 5 return - if not self.check_crc(buf): + if not self.__check_crc(buf): logging.error('Modbus resp: CRC error') self.err = 1 return @@ -188,7 +183,7 @@ class Modbus(): self.err = 4 return first_reg = self.last_reg # save last_reg before sending next pdu - self.stop_timer() # stop timer and send next pdu + self.__stop_timer() # stop timer and send next pdu for i in range(0, elmlen): addr = first_reg+i @@ -215,23 +210,81 @@ class Modbus(): f'[\'{node_id}\']MODBUS: {name}' f' : {result}{unit}') else: - self.stop_timer() - self.counter['retries'][self.retry_cnt] += 1 + self.__stop_timer() + + self.counter['retries'][f'{self.retry_cnt}'] += 1 if self.rsp_handler: self.rsp_handler() - self.get_next_req() + self.__send_next_from_que() - def check_crc(self, msg) -> bool: + ''' + MODBUS response timer + ''' + def __start_timer(self) -> None: + '''Start response timer and set `req_pend` to True''' + self.req_pend = True + self.tim = self.loop.call_later(self.timeout, self.__timeout_cb) + # logging.debug(f'Modbus start timer {self}') + + def __stop_timer(self) -> None: + '''Stop response timer and set `req_pend` to False''' + self.req_pend = False + # logging.debug(f'Modbus stop timer {self}') + if self.tim: + self.tim.cancel() + + def __timeout_cb(self) -> None: + '''Rsponse timeout handler retransmit pdu or send next pdu''' + self.req_pend = False + + if self.retry_cnt < self.max_retries: + logging.debug(f'Modbus retrans {self}') + self.retry_cnt += 1 + self.__start_timer() + self.snd_handler(self.last_req, retrans=True) + else: + logging.info(f'Modbus timeout {self}') + self.counter['timeouts'] += 1 + self.__send_next_from_que() + + def __send_next_from_que(self) -> None: + '''Get next MODBUS pdu from queue and transmit it''' + if self.req_pend: + return + try: + item = self.que.get_nowait() + req = item['req'] + self.last_req = req + self.rsp_handler = item['rsp_hdl'] + self.last_addr = req[0] + self.last_fcode = req[1] + + res = struct.unpack_from('>HH', req, 2) + self.last_reg = res[0] + self.last_len = res[1] + self.retry_cnt = 0 + self.__start_timer() + self.snd_handler(self.last_req, retrans=False) + except asyncio.QueueEmpty: + pass + + ''' + Helper function for CRC-16 handling + ''' + def __check_crc(self, msg: bytearray) -> bool: + '''Check CRC-16 and returns True if valid''' return 0 == self.__calc_crc(msg) - def __calc_crc(self, buffer: bytes) -> int: + def __calc_crc(self, buffer: bytearray) -> int: + '''Build CRC-16 for buffer and returns it''' crc = CRC_INIT for cur in buffer: crc = (crc >> 8) ^ self.__crc_tab[(crc ^ cur) & 0xFF] return crc - def __build_crc_tab(self, poly) -> None: + def __build_crc_tab(self, poly: int) -> None: + '''Build CRC-16 helper table, must be called exactly one time''' for index in range(256): data = index << 1 crc = 0 diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index fed9c09..1985ee4 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -21,24 +21,25 @@ def test_modbus_crc(): mb = Modbus(None) assert 0x0b02 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04') assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') - assert mb.check_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') + assert mb._Modbus__check_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') assert 0xc803 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x00') assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') - assert mb.check_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') + assert mb._Modbus__check_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8') assert 0x5c75 == mb._Modbus__calc_crc(b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46') + def test_build_modbus_pdu(): mb = TestHelper() mb.build_msg(1,6,0x2000,0x12) - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' - assert mb.check_crc(mb.pdu) + assert mb._Modbus__check_crc(mb.pdu) def test_recv_req_crc(): mb = TestHelper() mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.last_fcode == 0 assert mb.last_reg == 0 assert mb.last_len == 0 @@ -47,7 +48,7 @@ def test_recv_req_crc(): def test_recv_req_addr(): mb = TestHelper() mb.recv_req(b'\x02\x06\x20\x00\x00\x12\x02\x34') - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.last_addr == 2 assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 @@ -56,7 +57,7 @@ def test_recv_req_addr(): def test_recv_req(): mb = TestHelper() mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 assert mb.last_len == 0x12 @@ -88,7 +89,7 @@ def test_recv_recv_addr(): assert mb.err == 2 assert 0 == call assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_recv_recv_fcode(): @@ -104,7 +105,7 @@ def test_recv_recv_fcode(): assert mb.err == 3 assert 0 == call assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_recv_recv_len(): @@ -120,7 +121,7 @@ def test_recv_recv_len(): assert mb.err == 4 assert 0 == call assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_build_recv(): @@ -162,7 +163,7 @@ def test_build_recv(): assert 0 == mb.err assert 5 == call assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_build_long(): @@ -186,7 +187,7 @@ def test_build_long(): assert 0 == mb.err assert 3 == call assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_queue(): @@ -198,12 +199,12 @@ def test_queue(): assert mb.send_calls == 1 assert mb.pdu == b'\x01\x030"\x00\x04\xeb\x03' mb.pdu = None - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.send_calls == 1 assert mb.pdu == None assert mb.que.qsize() == 0 - mb.stop_timer() + mb._Modbus__stop_timer() assert not mb.req_pend def test_queue2(): @@ -215,7 +216,7 @@ def test_queue2(): assert mb.send_calls == 1 assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' - mb.get_next_req() + mb._Modbus__send_next_from_que() assert mb.send_calls == 1 call = 0 exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] From eff3e7558b8ae6bf42f016b8a5948c7d9b897d70 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 20 May 2024 16:53:26 +0200 Subject: [PATCH 079/118] increase test coverage --- app/src/gen3/talent.py | 8 +- app/src/gen3plus/solarman_v5.py | 8 +- app/src/modbus.py | 45 +++--- app/tests/test_modbus.py | 237 ++++++++++++++++++++++---------- 4 files changed, 190 insertions(+), 108 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 94eec89..c1ba2dd 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -122,7 +122,7 @@ class Talent(Message): f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}') return - def send_modbus_cb(self, modbus_pdu: bytearray, retrans: bool): + def send_modbus_cb(self, modbus_pdu: bytearray, state: str): if self.state != self.STATE_UP: return @@ -131,12 +131,8 @@ class Talent(Message): self._send_buffer += struct.pack('!B', len(modbus_pdu)) self._send_buffer += modbus_pdu self.__finish_send_msg() - if retrans: - cmd = 'Retrans' - else: - cmd = 'Command' - hex_dump_memory(logging.INFO, f'Send Modbus {cmd}:{self.addr}:', + hex_dump_memory(logging.INFO, f'Send Modbus {state}:{self.addr}:', self._send_buffer, len(self._send_buffer)) self.writer.write(self._send_buffer) self._send_buffer = bytearray(0) # self._send_buffer[sent:] diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 7ac8310..a53769a 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -301,7 +301,7 @@ class SolarmanV5(Message): self._heartbeat()) self.__finish_send_msg() - def send_modbus_cb(self, pdu: bytearray, retrans: bool): + def send_modbus_cb(self, pdu: bytearray, state: str): if self.state != self.STATE_UP: return self.__build_header(0x4510) @@ -309,11 +309,7 @@ class SolarmanV5(Message): 0x2b0, 0, 0, 0) self._send_buffer += pdu self.__finish_send_msg() - if retrans: - cmd = 'Retrans' - else: - cmd = 'Command' - hex_dump_memory(logging.INFO, f'Send Modbus {cmd}:{self.addr}:', + hex_dump_memory(logging.INFO, f'Send Modbus {state}:{self.addr}:', self._send_buffer, len(self._send_buffer)) self.writer.write(self._send_buffer) self._send_buffer = bytearray(0) # self._send_buffer[sent:] diff --git a/app/src/modbus.py b/app/src/modbus.py index 82c2f01..9cfeacd 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -1,3 +1,16 @@ +'''MODBUS module for TSUN inverter support + +TSUN uses the MODBUS in the RTU transmission mode over serial line. +see: https://modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf +see: https://modbus.org/docs/Modbus_over_serial_line_V1_02.pdf + +A Modbus PDU consists of: 'Function-Code' + 'Data' +A Modbus RTU message consists of: 'Addr' + 'Modbus-PDU' + 'CRC-16' +The inverter is a MODBUS server and the proxy the MODBUS client. + +The 16-bit CRC is known as CRC-16-ANSI(reverse) +see: https://en.wikipedia.org/wiki/Computation_of_cyclic_redundancy_checks +''' import struct import logging import asyncio @@ -8,30 +21,21 @@ if __name__ == "app.src.modbus": else: # pragma: no cover from infos import Register -####### -# TSUN uses the Modbus in the RTU transmission mode. -# see: https://modbus.org/docs/Modbus_over_serial_line_V1_02.pdf -# -# A Modbus PDU consists of: 'Function-Code' + 'Data' -# A Modbus RTU message consists of: 'Addr' + 'Modbus-PDU' + 'CRC-16' -# -# The 16-bit CRC is known as CRC-16-ANSI(reverse) -# see: https://en.wikipedia.org/wiki/Computation_of_cyclic_redundancy_checks -####### CRC_POLY = 0xA001 # (LSBF/reverse) CRC_INIT = 0xFFFF class Modbus(): + '''Simple MODBUS implementation with TX queue and retransmit timer''' INV_ADDR = 1 - '''MODBUS slave address of the TSUN inverter''' + '''MODBUS server address of the TSUN inverter''' READ_REGS = 3 '''MODBUS function code: Read Holding Register''' READ_INPUTS = 4 '''MODBUS function code: Read Input Register''' WRITE_SINGLE_REG = 6 - '''Modbus function code: Write Single Register''' + '''Modbus function code: Write Single Register''' __crc_tab = [] map = { @@ -69,12 +73,12 @@ class Modbus(): 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } - def __init__(self, snd_handler: Callable[[bool], None], timeout: int = 1): + def __init__(self, snd_handler: Callable[[str], None], timeout: int = 1): if not len(self.__crc_tab): self.__build_crc_tab(CRC_POLY) self.que = asyncio.Queue(100) self.snd_handler = snd_handler - '''Send handler to transmit a MODBUS request''' + '''Send handler to transmit a MODBUS RTU request''' self.rsp_handler = None '''Response handler to forward the response''' self.timeout = timeout @@ -99,14 +103,13 @@ class Modbus(): self.tim = None def __del__(self): - if type(self.counter) is not None: - logging.info(f'Modbus __del__:\n {self.counter}') + logging.info(f'Modbus __del__:\n {self.counter}') def build_msg(self, addr: int, func: int, reg: int, val: int) -> None: - """Build MODBUS RTU message frame and add it to the tx queue + """Build MODBUS RTU request frame and add it to the tx queue Keyword arguments: - addr: RTU slave address + addr: RTU server address (inverter) func: MODBUS function code reg: 16-bit register number val: 16 bit value @@ -120,7 +123,7 @@ class Modbus(): def recv_req(self, buf: bytearray, rsp_handler: Callable[[None], None] = None) -> bool: - """Add the received Modbus request to the tx queue + """Add the received Modbus RTU request to the tx queue Keyword arguments: buf: Modbus RTU pdu incl ADDR byte and trailing CRC @@ -241,7 +244,7 @@ class Modbus(): logging.debug(f'Modbus retrans {self}') self.retry_cnt += 1 self.__start_timer() - self.snd_handler(self.last_req, retrans=True) + self.snd_handler(self.last_req, state='Retrans') else: logging.info(f'Modbus timeout {self}') self.counter['timeouts'] += 1 @@ -264,7 +267,7 @@ class Modbus(): self.last_len = res[1] self.retry_cnt = 0 self.__start_timer() - self.snd_handler(self.last_req, retrans=False) + self.snd_handler(self.last_req, state='Command') except asyncio.QueueEmpty: pass diff --git a/app/tests/test_modbus.py b/app/tests/test_modbus.py index 1985ee4..051401f 100644 --- a/app/tests/test_modbus.py +++ b/app/tests/test_modbus.py @@ -2,7 +2,7 @@ import pytest import asyncio from app.src.modbus import Modbus -from app.src.infos import Infos +from app.src.infos import Infos, Register pytest_plugins = ('pytest_asyncio',) pytestmark = pytest.mark.asyncio(scope="module") @@ -13,11 +13,15 @@ class TestHelper(Modbus): self.db = Infos() self.pdu = None self.send_calls = 0 - def send_cb(self, pdu: bytearray, retrans: bool): + self.recv_responses = 0 + def send_cb(self, pdu: bytearray, state: str): self.pdu = pdu self.send_calls += 1 + def resp_handler(self): + self.recv_responses += 1 def test_modbus_crc(): + '''Check CRC-16 calculation''' mb = Modbus(None) assert 0x0b02 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04') assert 0 == mb._Modbus__calc_crc(b'\x01\x06\x20\x08\x00\x04\x02\x0b') @@ -30,101 +34,139 @@ def test_modbus_crc(): assert 0x5c75 == mb._Modbus__calc_crc(b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46') def test_build_modbus_pdu(): + '''Check building and sending a MODBUS RTU''' mb = TestHelper() mb.build_msg(1,6,0x2000,0x12) - mb._Modbus__send_next_from_que() assert mb.pdu == b'\x01\x06\x20\x00\x00\x12\x02\x07' assert mb._Modbus__check_crc(mb.pdu) - -def test_recv_req_crc(): - mb = TestHelper() - mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') - mb._Modbus__send_next_from_que() - assert mb.last_fcode == 0 - assert mb.last_reg == 0 - assert mb.last_len == 0 - assert mb.err == 1 - -def test_recv_req_addr(): - mb = TestHelper() - mb.recv_req(b'\x02\x06\x20\x00\x00\x12\x02\x34') - mb._Modbus__send_next_from_que() - assert mb.last_addr == 2 - assert mb.last_fcode == 6 + assert mb.last_addr == 1 + assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 assert mb.last_len == 18 + assert mb.err == 0 def test_recv_req(): + '''Receive a valid request, which must transmitted''' mb = TestHelper() - mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') - mb._Modbus__send_next_from_que() + assert mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x07') assert mb.last_fcode == 6 assert mb.last_reg == 0x2000 assert mb.last_len == 0x12 assert mb.err == 0 -def test_recv_recv_crc(): +def test_recv_req_crc_err(): + '''Receive a request with invalid CRC, which must be dropped''' mb = TestHelper() + assert not mb.recv_req(b'\x01\x06\x20\x00\x00\x12\x02\x08') + assert mb.pdu == None + assert mb.last_fcode == 0 + assert mb.last_reg == 0 + assert mb.last_len == 0 + assert mb.err == 1 + +def test_recv_resp_crc_err(): + '''Receive a response with invalid CRC, which must be dropped''' + mb = TestHelper() + # simulate a transmitted request mb.req_pend = True + mb.last_addr = 1 mb.last_fcode = 3 mb.last_reg == 0x300e mb.last_len == 2 - + # check matching response, but with CRC error call = 0 for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf3', 'test'): call += 1 assert mb.err == 1 assert 0 == call + assert mb.req_pend == True + # cleanup queue + mb._Modbus__stop_timer() + assert not mb.req_pend -def test_recv_recv_addr(): +def test_recv_resp_invalid_addr(): + '''Receive a response with wrong server addr, which must be dropped''' mb = TestHelper() mb.req_pend = True + # simulate a transmitted request + mb.last_addr = 1 mb.last_fcode = 3 mb.last_reg == 0x300e mb.last_len == 2 + # check not matching response, with wrong server addr call = 0 for key, update in mb.recv_resp(mb.db, b'\x02\x03\x04\x01\x2c\x00\x46\x88\xf4', 'test'): call += 1 assert mb.err == 2 assert 0 == call + assert mb.req_pend == True assert mb.que.qsize() == 0 + + # cleanup queue mb._Modbus__stop_timer() assert not mb.req_pend def test_recv_recv_fcode(): + '''Receive a response with wrong function code, which must be dropped''' mb = TestHelper() mb.build_msg(1,4,0x300e,2) assert mb.que.qsize() == 0 assert mb.req_pend + # check not matching response, with wrong function code call = 0 for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): call += 1 assert mb.err == 3 assert 0 == call + assert mb.req_pend == True assert mb.que.qsize() == 0 + + # cleanup queue mb._Modbus__stop_timer() assert not mb.req_pend -def test_recv_recv_len(): +def test_recv_resp_len(): + '''Receive a response with wrong data length, which must be dropped''' mb = TestHelper() mb.build_msg(1,3,0x300e,3) assert mb.que.qsize() == 0 assert mb.req_pend assert mb.last_len == 3 + + # check not matching response, with wrong data length call = 0 for key, update, _ in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): call += 1 assert mb.err == 4 assert 0 == call + assert mb.req_pend == True assert mb.que.qsize() == 0 + + # cleanup queue mb._Modbus__stop_timer() assert not mb.req_pend -def test_build_recv(): +def test_recv_unexpect_resp(): + '''Receive a response when we havb't sent a request''' + mb = TestHelper() + assert not mb.req_pend + + # check unexpected response, which must be dropped + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): + call += 1 + + assert mb.err == 5 + assert 0 == call + assert mb.req_pend == False + assert mb.que.qsize() == 0 + +def test_parse_resp(): + '''Receive matching response and parse the values''' mb = TestHelper() mb.build_msg(1,3,0x3007,6) assert mb.que.qsize() == 0 @@ -145,49 +187,7 @@ def test_build_recv(): call += 1 assert 0 == mb.err assert 5 == call - - mb.req_pend = True - call = 0 - for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): - if key == 'grid': - assert update == False - elif key == 'inverter': - assert update == False - elif key == 'env': - assert update == False - else: - assert False - assert exp_result[call] == val - call += 1 - - assert 0 == mb.err - assert 5 == call assert mb.que.qsize() == 0 - mb._Modbus__stop_timer() - assert not mb.req_pend - -def test_build_long(): - mb = TestHelper() - mb.build_msg(1,3,0x3022,4) - assert mb.que.qsize() == 0 - assert mb.req_pend - assert mb.last_addr == 1 - assert mb.last_fcode == 3 - assert mb.err == 0 - call = 0 - exp_result = [3.0, 28841.4, 113.34] - for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46\x75\x5c', 'test'): - if key == 'input': - assert update == True - assert exp_result[call] == val - else: - assert False - call += 1 - - assert 0 == mb.err - assert 3 == call - assert mb.que.qsize() == 0 - mb._Modbus__stop_timer() assert not mb.req_pend def test_queue(): @@ -199,25 +199,28 @@ def test_queue(): assert mb.send_calls == 1 assert mb.pdu == b'\x01\x030"\x00\x04\xeb\x03' mb.pdu = None - mb._Modbus__send_next_from_que() assert mb.send_calls == 1 assert mb.pdu == None assert mb.que.qsize() == 0 + + # cleanup queue mb._Modbus__stop_timer() assert not mb.req_pend def test_queue2(): + '''Check queue handling for build_msg() calls''' mb = TestHelper() mb.build_msg(1,3,0x3007,6) mb.build_msg(1,6,0x2008,4) assert mb.que.qsize() == 1 assert mb.req_pend + mb.build_msg(1,3,0x3007,6) + assert mb.que.qsize() == 2 + assert mb.req_pend assert mb.send_calls == 1 assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' - mb._Modbus__send_next_from_que() - assert mb.send_calls == 1 call = 0 exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): @@ -234,21 +237,87 @@ def test_queue2(): assert 0 == mb.err assert 5 == call + assert mb.que.qsize() == 1 assert mb.send_calls == 2 assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' for key, update, val in mb.recv_resp(mb.db, b'\x01\x06\x20\x08\x00\x04\x02\x0b', 'test'): pass + assert mb.que.qsize() == 0 + assert mb.send_calls == 3 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): + call += 1 + assert 0 == mb.err + assert 5 == call + assert mb.que.qsize() == 0 assert not mb.req_pend +def test_queue3(): + '''Check queue handling for recv_req() calls''' + mb = TestHelper() + assert mb.recv_req(b'\x01\x03\x30\x07\x00\x06{\t', mb.resp_handler) + assert mb.recv_req(b'\x01\x06\x20\x08\x00\x04\x02\x0b', mb.resp_handler) + assert mb.que.qsize() == 1 + assert mb.req_pend + assert mb.recv_req(b'\x01\x03\x30\x07\x00\x06{\t') + assert mb.que.qsize() == 2 + assert mb.req_pend + + assert mb.send_calls == 1 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + assert mb.recv_responses == 0 + + call = 0 + exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30] + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): + if key == 'grid': + assert update == True + elif key == 'inverter': + assert update == True + elif key == 'env': + assert update == True + else: + assert False + assert exp_result[call] == val + call += 1 + assert 0 == mb.err + assert 5 == call + assert mb.recv_responses == 1 + + assert mb.que.qsize() == 1 + assert mb.send_calls == 2 + assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' + + for key, update, val in mb.recv_resp(mb.db, b'\x01\x06\x20\x08\x00\x04\x02\x0b', 'test'): + pass + assert 0 == mb.err + assert mb.recv_responses == 2 + + assert mb.que.qsize() == 0 + assert mb.send_calls == 3 + assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'): + call += 1 + assert 0 == mb.err + assert mb.recv_responses == 2 + assert 5 == call + + + assert mb.que.qsize() == 0 + assert not mb.req_pend @pytest.mark.asyncio async def test_timeout(): + '''Test MODBUS response timeout and RTU retransmitting''' assert asyncio.get_running_loop() mb = TestHelper() mb.max_retries = 2 + mb.timeout = 0.1 # 100ms timeout for fast testing, expect a time resolution of at least 10ms assert asyncio.get_running_loop() == mb.loop mb.build_msg(1,3,0x3007,6) mb.build_msg(1,6,0x2008,4) @@ -260,7 +329,7 @@ async def test_timeout(): assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' mb.pdu = None - await asyncio.sleep(1.1) # wait for first timeout and retransmittion + await asyncio.sleep(0.11) # wait for first timeout and retransmittion assert mb.que.qsize() == 1 assert mb.req_pend assert mb.retry_cnt == 1 @@ -268,7 +337,7 @@ async def test_timeout(): assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' mb.pdu = None - await asyncio.sleep(1.1) # wait for second timeout and retransmittion + await asyncio.sleep(0.11) # wait for second timeout and retransmittion assert mb.que.qsize() == 1 assert mb.req_pend assert mb.retry_cnt == 2 @@ -276,7 +345,7 @@ async def test_timeout(): assert mb.pdu == b'\x01\x030\x07\x00\x06{\t' mb.pdu = None - await asyncio.sleep(1.1) # wait for third timeout and next pdu + await asyncio.sleep(0.11) # wait for third timeout and next pdu assert mb.que.qsize() == 0 assert mb.req_pend assert mb.retry_cnt == 0 @@ -284,10 +353,28 @@ async def test_timeout(): assert mb.pdu == b'\x01\x06\x20\x08\x00\x04\x02\x0b' mb.max_retries = 0 # next pdu without retranmsission - await asyncio.sleep(1.1) # wait for fourth timout + await asyncio.sleep(0.11) # wait for fourth timout assert mb.que.qsize() == 0 assert not mb.req_pend assert mb.retry_cnt == 0 assert mb.send_calls == 4 # assert mb.counter == {} + +def test_recv_unknown_data(): + '''Receive a response with an unknwon register''' + mb = TestHelper() + assert 0x9000 not in mb.map + mb.map[0x9000] = {'reg': Register.TEST_REG1, 'fmt': '!H', 'ratio': 1} + + mb.build_msg(1,3,0x9000,2) + + # check matching response, but with CRC error + call = 0 + for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x04\x01\x2c\x00\x46\xbb\xf4', 'test'): + call += 1 + assert mb.err == 0 + assert 0 == call + assert not mb.req_pend + + del mb.map[0x9000] From 3ac48dad1f877e5ab1821218d6a2f25ac783e817 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 20 May 2024 18:33:01 +0200 Subject: [PATCH 080/118] cleanup --- app/proxy.svg | 23 ++++++++++++----------- app/proxy.yuml | 2 +- app/src/gen3/talent.py | 3 --- app/src/infos.py | 32 ++++++++++++++++---------------- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/app/proxy.svg b/app/proxy.svg index cef1e69..9ee1aba 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -65,14 +65,15 @@ A3 - -Modbus - - -build_msg() -recv_req() -recv_resp() -check_crc() + +Modbus + +err +retry_cnt + +build_msg() +recv_req() +recv_resp() @@ -166,9 +167,9 @@ A6->A3 - - -1 + + +1 has diff --git a/app/proxy.yuml b/app/proxy.yuml index 7514a93..e2ce12b 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -4,7 +4,7 @@ [note: You can stick notes on diagrams too!{bg:cornsilk}] [Singleton]^[Mqtt|ha_restarts;__client;__cb_MqttIsUp|publish();close()] -[Modbus||build_msg();recv_req();recv_resp();check_crc()] +[Modbus|err;retry_cnt|build_msg();recv_req();recv_resp()] [IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list|_read():void;close():void;inc_counter():void;dec_counter():void] [Message]^[Talent|await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;close()] [Message]^[SolarmanV5|control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;close()] diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index c1ba2dd..def26fc 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -396,9 +396,6 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): - # if (self.remoteStream.state != self.STATE_UP): - # logger.info('ignore Modbus Request in wrong state') - # return if self.remoteStream.mb.recv_req(data[hdr_len:], self.msg_forward): self.inc_counter('Modbus_Command') diff --git a/app/src/infos.py b/app/src/infos.py index dadacd7..eaf7062 100644 --- a/app/src/infos.py +++ b/app/src/infos.py @@ -224,22 +224,22 @@ class Infos: # 0xffffff03: {'name':['proxy', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha':{'dev':'proxy', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'proxy_volt_', 'fmt':'| float','name': 'Grid Voltage'}}, # noqa: E501 # events - Register.EVENT_401: {'name': ['events', '401_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_402: {'name': ['events', '402_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_403: {'name': ['events', '403_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_404: {'name': ['events', '404_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_405: {'name': ['events', '405_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_406: {'name': ['events', '406_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_407: {'name': ['events', '407_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_408: {'name': ['events', '408_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_409: {'name': ['events', '409_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_410: {'name': ['events', '410_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_411: {'name': ['events', '411_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_412: {'name': ['events', '412_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_413: {'name': ['events', '413_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_414: {'name': ['events', '414_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_415: {'name': ['events', '415_GridFreqOverRating'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 - Register.EVENT_416: {'name': ['events', '416_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_401: {'name': ['events', '401_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_402: {'name': ['events', '402_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_403: {'name': ['events', '403_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_404: {'name': ['events', '404_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_405: {'name': ['events', '405_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_406: {'name': ['events', '406_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_407: {'name': ['events', '407_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_408: {'name': ['events', '408_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_409: {'name': ['events', '409_No_Utility'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_410: {'name': ['events', '410_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_411: {'name': ['events', '411_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_412: {'name': ['events', '412_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_413: {'name': ['events', '413_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_414: {'name': ['events', '414_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_415: {'name': ['events', '415_GridFreqOverRating'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_416: {'name': ['events', '416_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 # grid measures: Register.GRID_VOLTAGE: {'name': ['grid', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'inverter', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'out_volt_', 'fmt': '| float', 'name': 'Grid Voltage', 'ent_cat': 'diagnostic'}}, # noqa: E501 From 98ef252bb0167c7eec4db6e83194d7371bf52e2b Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 20 May 2024 18:51:55 +0200 Subject: [PATCH 081/118] don't forward invalid MODBUS responses --- app/src/gen3plus/solarman_v5.py | 2 +- app/tests/test_solarman.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index a53769a..f9231eb 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -467,7 +467,7 @@ class SolarmanV5(Message): if inv_update: self.__build_model_name() - return + return self.__forward_msg() def msg_hbeat_ind(self): diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index e339412..1092520 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -1489,7 +1489,7 @@ def test_msg_modbus_invalid(ConfigTsunInv1, MsgModbusInvalid): m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.msg_count == 1 - assert m._forward_buffer==MsgModbusInvalid + assert m._forward_buffer==b'' assert m._send_buffer==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 assert m.db.stat['proxy']['Modbus_Command'] == 0 From e43244113434065b7c7732cc1aacb2b0e9fb7fd1 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 21 May 2024 18:56:52 +0200 Subject: [PATCH 082/118] don't log Events as Infos --- app/src/infos.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/app/src/infos.py b/app/src/infos.py index eaf7062..4eadce0 100644 --- a/app/src/infos.py +++ b/app/src/infos.py @@ -224,22 +224,22 @@ class Infos: # 0xffffff03: {'name':['proxy', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha':{'dev':'proxy', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'proxy_volt_', 'fmt':'| float','name': 'Grid Voltage'}}, # noqa: E501 # events - Register.EVENT_401: {'name': ['events', '401_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_402: {'name': ['events', '402_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_403: {'name': ['events', '403_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_404: {'name': ['events', '404_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_405: {'name': ['events', '405_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_406: {'name': ['events', '406_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_407: {'name': ['events', '407_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_408: {'name': ['events', '408_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_409: {'name': ['events', '409_No_Utility'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_410: {'name': ['events', '410_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_411: {'name': ['events', '411_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_412: {'name': ['events', '412_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_413: {'name': ['events', '413_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_414: {'name': ['events', '414_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_415: {'name': ['events', '415_GridFreqOverRating'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 - Register.EVENT_416: {'name': ['events', '416_'], 'level': logging.WARNING, 'unit': ''}, # noqa: E501 + Register.EVENT_401: {'name': ['events', '401_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_402: {'name': ['events', '402_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_403: {'name': ['events', '403_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_404: {'name': ['events', '404_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_405: {'name': ['events', '405_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_406: {'name': ['events', '406_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_407: {'name': ['events', '407_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_408: {'name': ['events', '408_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_409: {'name': ['events', '409_No_Utility'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_410: {'name': ['events', '410_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_411: {'name': ['events', '411_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_412: {'name': ['events', '412_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_413: {'name': ['events', '413_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_414: {'name': ['events', '414_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_415: {'name': ['events', '415_GridFreqOverRating'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 + Register.EVENT_416: {'name': ['events', '416_'], 'level': logging.DEBUG, 'unit': ''}, # noqa: E501 # grid measures: Register.GRID_VOLTAGE: {'name': ['grid', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'inverter', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'out_volt_', 'fmt': '| float', 'name': 'Grid Voltage', 'ent_cat': 'diagnostic'}}, # noqa: E501 From de1c48fa62a512d99d3a992e8f9c06e5538ea34c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 21 May 2024 18:58:10 +0200 Subject: [PATCH 083/118] add keyword for timeout to argument list --- app/src/gen3/talent.py | 2 +- app/src/gen3plus/solarman_v5.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index def26fc..7e54d92 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -36,7 +36,7 @@ class Control: class Talent(Message): def __init__(self, server_side: bool, id_str=b''): - super().__init__(server_side, self.send_modbus_cb, 11) + super().__init__(server_side, self.send_modbus_cb, mb_timeout=11) self.await_conn_resp_cnt = 0 self.id_str = id_str self.contact_name = b'' diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index f9231eb..fb5f968 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -52,7 +52,7 @@ class SolarmanV5(Message): MB_RTU_CMD = 2 def __init__(self, server_side: bool): - super().__init__(server_side, self.send_modbus_cb, 5) + super().__init__(server_side, self.send_modbus_cb, mb_timeout=5) self.header_len = 11 # overwrite construcor in class Message self.control = 0 From 9e38cb93eaf2dcfba19d62fd117d8f54b92f82aa Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 21 May 2024 18:59:30 +0200 Subject: [PATCH 084/118] send StatusReq additionally every 30 minutes --- app/src/scheduler.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/app/src/scheduler.py b/app/src/scheduler.py index 610f586..a076faa 100644 --- a/app/src/scheduler.py +++ b/app/src/scheduler.py @@ -36,16 +36,12 @@ class Schedule: @classmethod async def regular_modbus_cmds(cls): - # logging.info("Regular Modbus requests") - if 0 == (cls.count % 30): - # logging.info("Regular Modbus Status request") - addr, len = 0x2007, 2 - else: - addr, len = 0x3008, 21 - cls.count += 1 - for m in Message: if m.server_side: fnc = getattr(m, "send_modbus_cmd", None) if callable(fnc): - await fnc(Modbus.READ_REGS, addr, len) + await fnc(Modbus.READ_REGS, 0x3008, 21) + if 0 == (cls.count % 30): + # logging.info("Regular Modbus Status request") + await fnc(Modbus.READ_REGS, 0x2007, 2) + cls.count += 1 From da2388941e156953d117b18f047a31c1ed825cd2 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 21 May 2024 19:37:55 +0200 Subject: [PATCH 085/118] allow only one MODBUS retry - More than one retry usually makes no sense, as random errors are usually corrected. If the first retry also fails, the chance that a second or third retry will be successful is very small --- app/src/modbus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index 9cfeacd..fc262bd 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -83,7 +83,7 @@ class Modbus(): '''Response handler to forward the response''' self.timeout = timeout '''MODBUS response timeout in seconds''' - self.max_retries = 3 + self.max_retries = 1 '''Max retransmit for MODBUS requests''' self.retry_cnt = 0 self.last_req = b'' @@ -103,7 +103,7 @@ class Modbus(): self.tim = None def __del__(self): - logging.info(f'Modbus __del__:\n {self.counter}') + logging.debug(f'Modbus __del__:\n {self.counter}') def build_msg(self, addr: int, func: int, reg: int, val: int) -> None: """Build MODBUS RTU request frame and add it to the tx queue From 55fc834a1edc9b0cd21f87bf7f11e8e4bce6f50b Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 22 May 2024 22:52:02 +0200 Subject: [PATCH 086/118] reduce default loggings --- app/src/infos.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/infos.py b/app/src/infos.py index 4eadce0..b408118 100644 --- a/app/src/infos.py +++ b/app/src/infos.py @@ -252,22 +252,22 @@ class Infos: # input measures: Register.PV1_VOLTAGE: {'name': ['input', 'pv1', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv1', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv1_', 'val_tpl': "{{ (value_json['pv1']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV1_CURRENT: {'name': ['input', 'pv1', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv1', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv1_', 'val_tpl': "{{ (value_json['pv1']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV1_POWER: {'name': ['input', 'pv1', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv1', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv1_', 'val_tpl': "{{ (value_json['pv1']['Power'] | float)}}"}}, # noqa: E501 + Register.PV1_POWER: {'name': ['input', 'pv1', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv1', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv1_', 'val_tpl': "{{ (value_json['pv1']['Power'] | float)}}"}}, # noqa: E501 Register.PV2_VOLTAGE: {'name': ['input', 'pv2', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv2', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv2_', 'val_tpl': "{{ (value_json['pv2']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV2_CURRENT: {'name': ['input', 'pv2', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv2', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv2_', 'val_tpl': "{{ (value_json['pv2']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV2_POWER: {'name': ['input', 'pv2', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv2', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv2_', 'val_tpl': "{{ (value_json['pv2']['Power'] | float)}}"}}, # noqa: E501 + Register.PV2_POWER: {'name': ['input', 'pv2', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv2', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv2_', 'val_tpl': "{{ (value_json['pv2']['Power'] | float)}}"}}, # noqa: E501 Register.PV3_VOLTAGE: {'name': ['input', 'pv3', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv3', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv3_', 'val_tpl': "{{ (value_json['pv3']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV3_CURRENT: {'name': ['input', 'pv3', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv3', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv3_', 'val_tpl': "{{ (value_json['pv3']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV3_POWER: {'name': ['input', 'pv3', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv3', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv3_', 'val_tpl': "{{ (value_json['pv3']['Power'] | float)}}"}}, # noqa: E501 + Register.PV3_POWER: {'name': ['input', 'pv3', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv3', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv3_', 'val_tpl': "{{ (value_json['pv3']['Power'] | float)}}"}}, # noqa: E501 Register.PV4_VOLTAGE: {'name': ['input', 'pv4', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv4', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv4_', 'val_tpl': "{{ (value_json['pv4']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV4_CURRENT: {'name': ['input', 'pv4', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv4', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv4_', 'val_tpl': "{{ (value_json['pv4']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV4_POWER: {'name': ['input', 'pv4', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv4', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv4_', 'val_tpl': "{{ (value_json['pv4']['Power'] | float)}}"}}, # noqa: E501 + Register.PV4_POWER: {'name': ['input', 'pv4', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv4', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv4_', 'val_tpl': "{{ (value_json['pv4']['Power'] | float)}}"}}, # noqa: E501 Register.PV5_VOLTAGE: {'name': ['input', 'pv5', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv5', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv5_', 'val_tpl': "{{ (value_json['pv5']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV5_CURRENT: {'name': ['input', 'pv5', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv5', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv5_', 'val_tpl': "{{ (value_json['pv5']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV5_POWER: {'name': ['input', 'pv5', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv5', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv5_', 'val_tpl': "{{ (value_json['pv5']['Power'] | float)}}"}}, # noqa: E501 + Register.PV5_POWER: {'name': ['input', 'pv5', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv5', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv5_', 'val_tpl': "{{ (value_json['pv5']['Power'] | float)}}"}}, # noqa: E501 Register.PV6_VOLTAGE: {'name': ['input', 'pv6', 'Voltage'], 'level': logging.DEBUG, 'unit': 'V', 'ha': {'dev': 'input_pv6', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id': 'volt_pv6_', 'val_tpl': "{{ (value_json['pv6']['Voltage'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 Register.PV6_CURRENT: {'name': ['input', 'pv6', 'Current'], 'level': logging.DEBUG, 'unit': 'A', 'ha': {'dev': 'input_pv6', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id': 'cur_pv6_', 'val_tpl': "{{ (value_json['pv6']['Current'] | float)}}", 'icon': 'mdi:gauge', 'ent_cat': 'diagnostic'}}, # noqa: E501 - Register.PV6_POWER: {'name': ['input', 'pv6', 'Power'], 'level': logging.INFO, 'unit': 'W', 'ha': {'dev': 'input_pv6', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv6_', 'val_tpl': "{{ (value_json['pv6']['Power'] | float)}}"}}, # noqa: E501 + Register.PV6_POWER: {'name': ['input', 'pv6', 'Power'], 'level': logging.DEBUG, 'unit': 'W', 'ha': {'dev': 'input_pv6', 'dev_cla': 'power', 'stat_cla': 'measurement', 'id': 'power_pv6_', 'val_tpl': "{{ (value_json['pv6']['Power'] | float)}}"}}, # noqa: E501 Register.PV1_DAILY_GENERATION: {'name': ['input', 'pv1', 'Daily_Generation'], 'level': logging.DEBUG, 'unit': 'kWh', 'ha': {'dev': 'input_pv1', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id': 'daily_gen_pv1_', 'name': 'Daily Generation', 'val_tpl': "{{ (value_json['pv1']['Daily_Generation'] | float)}}", 'icon': 'mdi:solar-power-variant', 'must_incr': True}}, # noqa: E501 Register.PV1_TOTAL_GENERATION: {'name': ['input', 'pv1', 'Total_Generation'], 'level': logging.DEBUG, 'unit': 'kWh', 'ha': {'dev': 'input_pv1', 'dev_cla': 'energy', 'stat_cla': 'total', 'id': 'total_gen_pv1_', 'name': 'Total Generation', 'val_tpl': "{{ (value_json['pv1']['Total_Generation'] | float)}}", 'icon': 'mdi:solar-power', 'must_incr': True}}, # noqa: E501 Register.PV2_DAILY_GENERATION: {'name': ['input', 'pv2', 'Daily_Generation'], 'level': logging.DEBUG, 'unit': 'kWh', 'ha': {'dev': 'input_pv2', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id': 'daily_gen_pv2_', 'name': 'Daily Generation', 'val_tpl': "{{ (value_json['pv2']['Daily_Generation'] | float)}}", 'icon': 'mdi:solar-power-variant', 'must_incr': True}}, # noqa: E501 From 8fc5eb3670e75ae72ee305e6fe241046f06076ee Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 22 May 2024 22:53:04 +0200 Subject: [PATCH 087/118] log MQTT to data topic --- app/src/modbus.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/app/src/modbus.py b/app/src/modbus.py index fc262bd..8f83c9c 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -21,6 +21,7 @@ if __name__ == "app.src.modbus": else: # pragma: no cover from infos import Register +logger = logging.getLogger('data') CRC_POLY = 0xA001 # (LSBF/reverse) CRC_INIT = 0xFFFF @@ -136,7 +137,7 @@ class Modbus(): # logging.info(f'recv_req: first byte modbus:{buf[0]} len:{len(buf)}') if not self.__check_crc(buf): self.err = 1 - logging.error('Modbus recv: CRC error') + logger.error('Modbus recv: CRC error') return False self.que.put_nowait({'req': buf, 'rsp_hdl': rsp_handler}) @@ -166,23 +167,23 @@ class Modbus(): self.err = 5 return if not self.__check_crc(buf): - logging.error('Modbus resp: CRC error') + logger.error('Modbus resp: CRC error') self.err = 1 return if buf[0] != self.last_addr: - logging.info(f'Modbus resp: Wrong addr {buf[0]}') + logger.info(f'Modbus resp: Wrong addr {buf[0]}') self.err = 2 return fcode = buf[1] if fcode != self.last_fcode: - logging.info(f'Modbus: Wrong fcode {fcode} != {self.last_fcode}') + logger.info(f'Modbus: Wrong fcode {fcode} != {self.last_fcode}') self.err = 3 return if self.last_addr == self.INV_ADDR and \ (fcode == 3 or fcode == 4): elmlen = buf[2] >> 1 if elmlen != self.last_len: - logging.info(f'Modbus: len error {elmlen} != {self.last_len}') + logger.info(f'Modbus: len error {elmlen} != {self.last_len}') self.err = 4 return first_reg = self.last_reg # save last_reg before sending next pdu @@ -241,12 +242,12 @@ class Modbus(): self.req_pend = False if self.retry_cnt < self.max_retries: - logging.debug(f'Modbus retrans {self}') + logger.debug(f'Modbus retrans {self}') self.retry_cnt += 1 self.__start_timer() self.snd_handler(self.last_req, state='Retrans') else: - logging.info(f'Modbus timeout {self}') + logger.info(f'Modbus timeout {self}') self.counter['timeouts'] += 1 self.__send_next_from_que() From 87cc3fb2050afcca3ea2e70cf2f21d57afe56534 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 22 May 2024 22:53:52 +0200 Subject: [PATCH 088/118] fix frong MQTT not found logs --- app/src/mqtt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/mqtt.py b/app/src/mqtt.py index 484d7df..2ce6333 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -129,8 +129,10 @@ class Mqtt(metaclass=Singleton): def each_inverter(self, message, func_name: str): topic = str(message.topic) node_id = topic.split('/')[1] + '/' + found = False for m in Message: if m.server_side and (m.node_id == node_id): + found = True logger_mqtt.debug(f'Found: {node_id}') fnc = getattr(m, func_name, None) if callable(fnc): @@ -138,7 +140,7 @@ class Mqtt(metaclass=Singleton): else: logger_mqtt.warning(f'Cmd not supported by: {node_id}') - else: + if not found: logger_mqtt.warning(f'Node_id: {node_id} not found') async def modbus_cmd(self, message, func, params=0, addr=0, val=0): From 0fc74b0d19b94627323d5ba7818ddcaf1c39e5fc Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 22 May 2024 22:54:23 +0200 Subject: [PATCH 089/118] improve unit test --- app/tests/test_talent.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 3387d99..31bbbef 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -794,21 +794,53 @@ def test_msg_iterator(): assert test2 == 1 def test_proxy_counter(): - m = Talent(server_side=True) + # m = MemoryStream(b'') + # m.close() + Infos.stat['proxy']['Modbus_Command'] = 1 + + m = MemoryStream(b'') + m.id_str = b"R170000000000001" + c = m.createClientStream(b'') + assert m.new_data == {} m.db.stat['proxy']['Unknown_Msg'] = 0 + c.db.stat['proxy']['Unknown_Msg'] = 0 Infos.new_stat_data['proxy'] = False m.inc_counter('Unknown_Msg') + m.close() + m = MemoryStream(b'') + assert m.new_data == {} assert Infos.new_stat_data == {'proxy': True} + assert m.db.new_stat_data == {'proxy': True} + assert c.db.new_stat_data == {'proxy': True} assert 1 == m.db.stat['proxy']['Unknown_Msg'] + assert 1 == c.db.stat['proxy']['Unknown_Msg'] + Infos.new_stat_data['proxy'] = False + + c.inc_counter('Unknown_Msg') + assert m.new_data == {} + assert Infos.new_stat_data == {'proxy': True} + assert m.db.new_stat_data == {'proxy': True} + assert c.db.new_stat_data == {'proxy': True} + assert 2 == m.db.stat['proxy']['Unknown_Msg'] + assert 2 == c.db.stat['proxy']['Unknown_Msg'] + Infos.new_stat_data['proxy'] = False + + c.inc_counter('Modbus_Command') + assert m.new_data == {} + assert Infos.new_stat_data == {'proxy': True} + assert m.db.new_stat_data == {'proxy': True} + assert c.db.new_stat_data == {'proxy': True} + assert 2 == m.db.stat['proxy']['Modbus_Command'] + assert 2 == c.db.stat['proxy']['Modbus_Command'] Infos.new_stat_data['proxy'] = False m.dec_counter('Unknown_Msg') assert m.new_data == {} assert Infos.new_stat_data == {'proxy': True} - assert 0 == m.db.stat['proxy']['Unknown_Msg'] + assert 1 == m.db.stat['proxy']['Unknown_Msg'] m.close() def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): From 5c6f9e74142e188dd84ef1b6fb87dd359e4a531a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 23 May 2024 19:52:55 +0200 Subject: [PATCH 090/118] increase test coverage to 100% --- app/tests/test_solarman.py | 40 +++++++++++++++++++++ app/tests/test_talent.py | 74 +++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 1092520..8d997b4 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -1451,6 +1451,46 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusRsp): assert m.new_data['inverter'] == True m.new_data['inverter'] = False + m.mb.req_pend = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 2 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + # assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V4.0.10' + assert m.new_data['inverter'] == False + + m.close() + +def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusRsp): + '''Modbus response with a valid Modbus request must be forwarded''' + ConfigTsunInv1 + m = MemoryStream(MsgModbusRsp) + m.append_msg(MsgModbusRsp) + + m.mb.rsp_handler = m._SolarmanV5__forward_msg + m.mb.last_addr = 1 + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 + # assert m.db.db == {'inverter': {'Manufacturer': 'TSUN', 'Equipment_Model': 'TSOL-MSxx00'}} + m.new_data['inverter'] = False + + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 1 + assert m._forward_buffer==MsgModbusRsp + assert m._send_buffer==b'' + # assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V4.0.10' + assert m.new_data['inverter'] == True + m.new_data['inverter'] = False + m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.mb.err == 5 diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 31bbbef..c24548c 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -874,7 +874,37 @@ def test_msg_modbus_req(ConfigTsunInv1, MsgModbusCmd): assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() -def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmdCrcErr): +def test_msg_modbus_req2(ConfigTsunInv1, MsgModbusCmd): + ConfigTsunInv1 + m = MemoryStream(b'') + m.id_str = b"R170000000000001" + + c = m.createClientStream(MsgModbusCmd) + + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.db.stat['proxy']['Invalid_Msg_Format'] = 0 + c.read() # read complete msg, and dispatch msg + assert not c.header_valid # must be invalid, since msg was handled and buffer flushed + assert c.msg_count == 1 + assert c.id_str == b"R170000000000001" + assert c.unique_id == 'R170000000000001' + assert int(c.ctrl)==112 + assert c.msg_id==119 + assert c.header_len==23 + assert c.data_len==13 + assert c._forward_buffer==b'' + assert c._send_buffer==b'' + assert m.id_str == b"R170000000000001" + assert m._forward_buffer==b'' + assert m._send_buffer==b'' + assert m.writer.sent_pdu == b'' + assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 1 + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + m.close() + +def test_msg_modbus_req3(ConfigTsunInv1, MsgModbusCmdCrcErr): ConfigTsunInv1 m = MemoryStream(b'') m.id_str = b"R170000000000001" @@ -949,8 +979,50 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusResp20): assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} assert m.db.get_db_value(Register.VERSION) == 'V5.1.09' assert m.new_data['inverter'] == True + + m.new_data['inverter'] = False + m.mb.req_pend = True + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 2 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer==b'' + assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V5.1.09' + assert m.new_data['inverter'] == False + + m.close() + +def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20): + '''Modbus response with a valid Modbus request must be forwarded''' + ConfigTsunInv1 + m = MemoryStream(MsgModbusResp20) + m.append_msg(MsgModbusResp20) + + m.mb.rsp_handler = m.msg_forward + m.mb.last_addr = 1 + m.mb.last_fcode = 3 + m.mb.last_len = 20 + m.mb.last_reg = 0x3008 + m.mb.req_pend = True + m.mb.err = 0 + + assert m.db.db == {} m.new_data['inverter'] = False + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.mb.err == 0 + assert m.msg_count == 1 + assert m._forward_buffer==MsgModbusResp20 + assert m._send_buffer==b'' + assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}} + assert m.db.get_db_value(Register.VERSION) == 'V5.1.09' + assert m.new_data['inverter'] == True + m.new_data['inverter'] = False + assert m.mb.req_pend == False + m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed assert m.mb.err == 5 From edc2c12b5b2de11feeead036698003bcc8a6c5dd Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 27 May 2024 20:52:06 +0200 Subject: [PATCH 091/118] Send MQTT topic for responses to AT+ commands --- app/src/gen3plus/solarman_v5.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index fb5f968..423123a 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -2,6 +2,7 @@ import struct # import json import logging import time +import asyncio from datetime import datetime if __name__ == "app.src.gen3plus.solarman_v5": @@ -441,12 +442,21 @@ class SolarmanV5(Message): self.__forward_msg() + async def publish_mqtt(self, key, data): + await self.mqtt.publish(key, data) # pragma: no cover + def msg_command_rsp(self): data = self._recv_buffer[self.header_len: self.header_len+self.data_len] ftype = data[0] if ftype == self.AT_CMD: if not self.forward_at_cmd_resp: + data_json = data[14:].decode("utf-8") + node_id = self.node_id + key = 'at_resp' + logger.info(f'{key}: {data_json}') + asyncio.ensure_future( + self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json)) # noqa: E501 return elif ftype == self.MB_RTU_CMD: valid = data[1] From fdf3475909eb8431186e16b9ccb089927711e452 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 27 May 2024 20:56:03 +0200 Subject: [PATCH 092/118] fix unit test --- app/tests/test_solarman.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 8d997b4..beb83d6 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -39,6 +39,7 @@ class MemoryStream(SolarmanV5): self.db.stat['proxy']['Invalid_Msg_Format'] = 0 self.db.stat['proxy']['AT_Command'] = 0 self.test_exception_async_write = False + self.entity_prfx = '' def _timestamp(self): return timestamp From ab9e798152dbdbfdd8a28e7d7ad0b866714be63c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 28 May 2024 19:30:58 +0200 Subject: [PATCH 093/118] add typing --- app/src/messages.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/src/messages.py b/app/src/messages.py index 4968609..16e6d1b 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -1,5 +1,7 @@ import logging import weakref +from typing import Callable + if __name__ == "app.src.messages": from app.src.infos import Infos @@ -56,7 +58,8 @@ class Message(metaclass=IterRegistry): STATE_UP = 2 STATE_CLOSED = 3 - def __init__(self, server_side: bool, send_modbus_cb, mb_timeout): + def __init__(self, server_side: bool, send_modbus_cb: + Callable[[bytes, int, str], None], mb_timeout): self._registry.append(weakref.ref(self)) self.server_side = server_side From 66657888ddfced280e6cdab8b822ef7d52501050 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 28 May 2024 19:32:20 +0200 Subject: [PATCH 094/118] add log_level support for modbus commands --- app/src/gen3/talent.py | 17 +++++++++-------- app/src/gen3plus/solarman_v5.py | 16 ++++++++-------- app/src/modbus.py | 18 ++++++++++++------ app/src/mqtt.py | 2 +- app/src/scheduler.py | 4 ++-- app/tests/test_modbus.py | 2 +- app/tests/test_solarman.py | 7 ++++--- app/tests/test_talent.py | 6 +++--- system_tests/test_tcp_socket_v2.py | 16 ++++++++++++---- 9 files changed, 52 insertions(+), 36 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 7e54d92..caf327d 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -122,25 +122,25 @@ class Talent(Message): f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}') return - def send_modbus_cb(self, modbus_pdu: bytearray, state: str): + def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: return - self.__build_header(0x70, 0x77) + self.__build_header(0x70, 0x77, log_lvl) self._send_buffer += b'\x00\x01\xa3\x28' # fixme self._send_buffer += struct.pack('!B', len(modbus_pdu)) self._send_buffer += modbus_pdu self.__finish_send_msg() - hex_dump_memory(logging.INFO, f'Send Modbus {state}:{self.addr}:', + hex_dump_memory(log_lvl, f'Send Modbus {state}:{self.addr}:', self._send_buffer, len(self._send_buffer)) self.writer.write(self._send_buffer) self._send_buffer = bytearray(0) # self._send_buffer[sent:] - async def send_modbus_cmd(self, func, addr, val) -> None: + async def send_modbus_cmd(self, func, addr, val, log_lvl) -> None: if self.state != self.STATE_UP: return - self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) + self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) def _init_new_client_conn(self) -> bool: contact_name = self.contact_name @@ -217,15 +217,16 @@ class Talent(Message): self.header_valid = True return - def __build_header(self, ctrl, msg_id=None) -> None: + def __build_header(self, ctrl, msg_id=None, + log_lvl: int = logging.INFO) -> None: if not msg_id: msg_id = self.msg_id self.send_msg_ofs = len(self._send_buffer) self._send_buffer += struct.pack(f'!l{len(self.id_str)+1}pBB', 0, self.id_str, ctrl, msg_id) fnc = self.switch.get(msg_id, self.msg_unknown) - logger.info(self.__flow_str(self.server_side, 'tx') + - f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') + logger.log(log_lvl, self.__flow_str(self.server_side, 'tx') + + f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') def __finish_send_msg(self) -> None: _len = len(self._send_buffer) - self.send_msg_ofs diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 423123a..1529057 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -248,15 +248,15 @@ class SolarmanV5(Message): return True - def __build_header(self, ctrl) -> None: + def __build_header(self, ctrl, log_lvl: int = logging.INFO) -> None: '''build header for new transmit message''' self.send_msg_ofs = len(self._send_buffer) self._send_buffer += struct.pack( ' None: '''finish the transmit message, set lenght and checksum''' @@ -302,23 +302,23 @@ class SolarmanV5(Message): self._heartbeat()) self.__finish_send_msg() - def send_modbus_cb(self, pdu: bytearray, state: str): + def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: return - self.__build_header(0x4510) + self.__build_header(0x4510, log_lvl) self._send_buffer += struct.pack(' None: + async def send_modbus_cmd(self, func, addr, val, log_lvl) -> None: if self.state != self.STATE_UP: return - self.mb.build_msg(Modbus.INV_ADDR, func, addr, val) + self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) async def send_at_cmd(self, AT_cmd: str) -> None: if self.state != self.STATE_UP: diff --git a/app/src/modbus.py b/app/src/modbus.py index 8f83c9c..0727650 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -74,7 +74,8 @@ class Modbus(): 0x3029: {'reg': Register.PV4_TOTAL_GENERATION, 'fmt': '!L', 'ratio': 0.01}, # noqa: E501 } - def __init__(self, snd_handler: Callable[[str], None], timeout: int = 1): + def __init__(self, snd_handler: Callable[[bytes, int, str], None], + timeout: int = 1): if not len(self.__crc_tab): self.__build_crc_tab(CRC_POLY) self.que = asyncio.Queue(100) @@ -94,6 +95,7 @@ class Modbus(): self.counter['retries'] = {} for i in range(0, self.max_retries+1): self.counter['retries'][f'{i}'] = 0 + self.last_log_lvl = logging.DEBUG self.last_addr = 0 self.last_fcode = 0 self.last_len = 0 @@ -106,7 +108,8 @@ class Modbus(): def __del__(self): logging.debug(f'Modbus __del__:\n {self.counter}') - def build_msg(self, addr: int, func: int, reg: int, val: int) -> None: + def build_msg(self, addr: int, func: int, reg: int, val: int, + log_lvl=logging.DEBUG) -> None: """Build MODBUS RTU request frame and add it to the tx queue Keyword arguments: @@ -118,7 +121,8 @@ class Modbus(): msg = struct.pack('>BBHH', addr, func, reg, val) msg += struct.pack(' bytes: def get_invalid_sn(): return b'R170000000000002' +def correct_checksum(buf): + checksum = sum(buf[1:]) & 0xff + return checksum.to_bytes(length=1) @pytest.fixture def MsgContactInfo(): # Contact Info message @@ -61,10 +64,11 @@ def MsgDataInd(): msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' msg += b'\x00\x01\x12\x02\x12\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' msg += b'\x40\x10\x08\xd8\x00\x09\x13\x84\x00\x35\x00\x00\x02\x58\x00\xd8' - msg += b'\x01\x3f\x00\x17\x00\x4d\x01\x44\x00\x14\x00\x43\x01\x45\x00\x18' + msg += b'\x01\x3f\x00\x17\x00\x4d\x01\x44\x00\x14\x00\x43\x01\x45\x00\x18' msg += b'\x00\x52\x00\x12\x00\x01\x00\x00\x00\x7c\x00\x00\x24\xed\x00\x2c' msg += b'\x00\x00\x0b\x10\x00\x26\x00\x00\x0a\x0f\x00\x30\x00\x00\x0b\x76' - msg += b'\x00\x00\x00\x00\x06\x16\x00\x00\x00\x00\x55\xaa\x00\x01\x00\x00' + + msg += b'\x00\x00\x00\x00\x06\x16\x00\x00\x00\x01\x55\xaa\x00\x01\x00\x00' msg += b'\x00\x00\x00\x00\xff\xff\x07\xd0\x00\x03\x04\x00\x04\x00\x04\x00' msg += b'\x04\x00\x00\x01\xff\xff\x00\x01\x00\x06\x00\x68\x00\x68\x05\x00' msg += b'\x09\xcd\x07\xb6\x13\x9c\x13\x24\x00\x01\x07\xae\x04\x0f\x00\x41' @@ -73,7 +77,9 @@ def MsgDataInd(): msg += b'\x04\x00\x00\x01\x13\x9c\x0f\xa0\x00\x4e\x00\x66\x03\xe8\x04\x00' msg += b'\x09\xce\x07\xa8\x13\x9c\x13\x26\x00\x00\x00\x00\x00\x00\x00\x00' msg += b'\x00\x00\x00\x00\x04\x00\x04\x00\x00\x00\x00\x00\xff\xff\x00\x00' - msg += b'\x00\x00\x00\x00\x24\x15' + msg += b'\x00\x00\x00\x00' + msg += correct_checksum(msg) + msg += b'\x15' return msg @pytest.fixture @@ -147,4 +153,6 @@ def test_data_ind(ClientConnection,MsgDataInd, MsgDataResp): except TimeoutError: pass # time.sleep(2.5) - checkResponse(data, MsgDataResp) \ No newline at end of file + checkResponse(data, MsgDataResp) + + \ No newline at end of file From 3980ac013bf38101b2f3b5581543a69d82fcf791 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 28 May 2024 21:55:42 +0200 Subject: [PATCH 095/118] catch all OSError errors in the read loop --- CHANGELOG.md | 2 ++ app/src/async_stream.py | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8412be2..830c501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- catch all OSError errors in the read loop +- log Modbus traces with different log levels - add Modbus fifo and timeout handler - build version string in the same format as TSUN for GEN3 invterts - add graceful shutdown diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 196a01f..563b948 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -65,9 +65,7 @@ class AsyncStream(): await self.__async_forward() await self.async_publ_mqtt() - except (ConnectionResetError, - ConnectionAbortedError, - BrokenPipeError) as error: + except OSError as error: logger.error(f'{error} for l{self.l_addr} | ' f'r{self.r_addr}') await self.disc() From 063850c7fb148df981a651c36631ac0597cc983f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 30 May 2024 18:38:05 +0200 Subject: [PATCH 096/118] add allow and block filter for AT+ commands --- app/config/default_config.toml | 5 +++++ app/src/config.py | 13 +++++++++++-- app/src/gen3plus/solarman_v5.py | 24 ++++++++++++++++++++++++ app/tests/test_config.py | 18 +++++++++++------- app/tests/test_solarman.py | 1 + 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/app/config/default_config.toml b/app/config/default_config.toml index cd95d75..fbe2651 100644 --- a/app/config/default_config.toml +++ b/app/config/default_config.toml @@ -49,3 +49,8 @@ monitor_sn = 2000000000 # The "Monitoring SN:" can be found on a sticker e #pv3 = {type = 'RSM40-8-410M', manufacturer = 'Risen'} # Optional, PV module descr #pv4 = {type = 'RSM40-8-410M', manufacturer = 'Risen'} # Optional, PV module descr +[gen3plus.at_acl] +tsun.allow = ['AT+Z', 'AT+UPURL', 'AT+SUPDATE'] +tsun.block = [] +mqtt.allow = ['AT+'] +mqtt.block = [] diff --git a/app/src/config.py b/app/src/config.py index eef0c86..e1ef749 100644 --- a/app/src/config.py +++ b/app/src/config.py @@ -3,7 +3,7 @@ import shutil import tomllib import logging -from schema import Schema, And, Use, Optional +from schema import Schema, And, Or, Use, Optional class Config(): @@ -38,6 +38,14 @@ class Config(): 'proxy_node_id': Use(str), 'proxy_unique_id': Use(str) }, + 'gen3plus': { + 'at_acl': { + Or('mqtt', 'tsun'): { + 'allow': [str], + Optional('block', default=[]): [str] + } + } + }, 'inverters': { 'allow_all': Use(bool), And(Use(str), lambda s: len(s) == 16): { Optional('monitor_sn', default=0): Use(int), @@ -125,7 +133,8 @@ class Config(): # merge the default and the user config config = def_config.copy() - for key in ['tsun', 'solarman', 'mqtt', 'ha', 'inverters']: + for key in ['tsun', 'solarman', 'mqtt', 'ha', 'inverters', + 'gen3plus']: if key in usr_config: config[key] |= usr_config[key] diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 1529057..e3ed438 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -91,8 +91,13 @@ class SolarmanV5(Message): # MODbus or AT cmd 0x4510: self.msg_command_req, # from server 0x1510: self.msg_command_rsp, # from inverter + # 0x0510: self.msg_command_rsp, # from inverter } self.modbus_elms = 0 # for unit tests + g3p_cnf = Config.get('gen3plus') + + if 'at_acl' in g3p_cnf: + self.at_acl = g3p_cnf['at_acl'] ''' Our puplic methods @@ -320,9 +325,24 @@ class SolarmanV5(Message): return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) + def at_cmd_forbidden(self, cmd: str, connection: str) -> bool: + return not cmd.startswith(tuple(self.at_acl[connection]['allow'])) or \ + cmd.startswith(tuple(self.at_acl[connection]['block'])) + async def send_at_cmd(self, AT_cmd: str) -> None: if self.state != self.STATE_UP: return + AT_cmd = AT_cmd.strip() + + if self.at_cmd_forbidden(cmd=AT_cmd, connection='mqtt'): + data_json = f'\'{AT_cmd}\' is forbidden' + node_id = self.node_id + key = 'at_resp' + logger.info(f'{key}: {data_json}') + asyncio.ensure_future( + self.publish_mqtt(f'{self.entity_prfx}{node_id}{key}', data_json)) # noqa: E501 + return + self.forward_at_cmd_resp = False self.__build_header(0x4510) self._send_buffer += struct.pack(f' Date: Thu, 30 May 2024 19:32:14 +0200 Subject: [PATCH 097/118] add AT_COMMAND_BLOCKED counter --- app/src/gen3plus/solarman_v5.py | 7 ++++--- app/src/infos.py | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index e3ed438..91ee30b 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -96,7 +96,7 @@ class SolarmanV5(Message): self.modbus_elms = 0 # for unit tests g3p_cnf = Config.get('gen3plus') - if 'at_acl' in g3p_cnf: + if 'at_acl' in g3p_cnf: # pragma: no cover self.at_acl = g3p_cnf['at_acl'] ''' @@ -450,11 +450,12 @@ class SolarmanV5(Message): result = struct.unpack_from(' Date: Thu, 30 May 2024 19:32:53 +0200 Subject: [PATCH 098/118] add missing testcases --- app/tests/test_infos.py | 4 +- app/tests/test_solarman.py | 78 +++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/app/tests/test_infos.py b/app/tests/test_infos.py index c3e6ddf..4b23bfb 100644 --- a/app/tests/test_infos.py +++ b/app/tests/test_infos.py @@ -17,13 +17,13 @@ def test_statistic_counter(): assert val == None or val == 0 i.static_init() # initialize counter - assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 0, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "Modbus_Command": 0}}) + assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 0, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "AT_Command_Blocked": 0, "Modbus_Command": 0}}) val = i.dev_value(Register.INVERTER_CNT) # valid and initiliazed addr assert val == 0 i.inc_counter('Inverter_Cnt') - assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 1, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "Modbus_Command": 0}}) + assert json.dumps(i.stat) == json.dumps({"proxy": {"Inverter_Cnt": 1, "Unknown_SNR": 0, "Unknown_Msg": 0, "Invalid_Data_Type": 0, "Internal_Error": 0,"Unknown_Ctrl": 0, "OTA_Start_Msg": 0, "SW_Exception": 0, "Invalid_Msg_Format": 0, "AT_Command": 0, "AT_Command_Blocked": 0, "Modbus_Command": 0}}) val = i.dev_value(Register.INVERTER_CNT) assert val == 1 diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 52a66a8..06a57f0 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -39,9 +39,10 @@ class MemoryStream(SolarmanV5): self.addr = 'Test: SrvSide' self.db.stat['proxy']['Invalid_Msg_Format'] = 0 self.db.stat['proxy']['AT_Command'] = 0 + self.db.stat['proxy']['AT_Command_Blocked'] = 0 self.test_exception_async_write = False self.entity_prfx = '' - self.at_acl = {'mqtt': {'allow': ['AT+'], 'block': []}, 'tsun': {'allow': ['AT+Z', 'AT+UPURL', 'AT+SUPDATE', 'AT+TIME'], 'block': []}} + self.at_acl = {'mqtt': {'allow': ['AT+'], 'block': ['AT+WEBU']}, 'tsun': {'allow': ['AT+Z', 'AT+UPURL', 'AT+SUPDATE', 'AT+TIME'], 'block': ['AT+WEBU']}} def _timestamp(self): return timestamp @@ -466,6 +467,15 @@ def AtCommandIndMsg(): # 0x4510 msg += b'\x15' return msg +@pytest.fixture +def AtCommandIndMsgBlock(): # 0x4510 + msg = b'\xa5\x17\x00\x10\x45\x03\x02' +get_sn() +b'\x01\x02\x00' + msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg += b'AT+WEBU\r' + msg += correct_checksum(msg) + msg += b'\x15' + return msg + @pytest.fixture def AtCommandRspMsg(): # 0x1510 msg = b'\xa5\x0a\x00\x10\x15\x03\x03' +get_sn() +b'\x01\x01' @@ -1277,11 +1287,49 @@ async def test_AT_cmd(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIn assert m.forward_at_cmd_resp == False m.close() -def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): +@pytest.mark.asyncio +async def test_AT_cmd_blocked(ConfigTsunAllowAll, DeviceIndMsg, DeviceRspMsg, InverterIndMsg, InverterRspMsg, AtCommandIndMsg): + ConfigTsunAllowAll + m = MemoryStream(DeviceIndMsg, (0,), True) + m.append_msg(InverterIndMsg) + m.read() + assert m.control == 0x4110 + assert str(m.seq) == '01:01' + assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._send_buffer==DeviceRspMsg + assert m._forward_buffer==DeviceIndMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear send buffer for next test + await m.send_at_cmd('AT+WEBU') + assert m._recv_buffer==InverterIndMsg # unhandled next message + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert str(m.seq) == '01:01' + + m.read() + assert m.control == 0x4210 + assert str(m.seq) == '02:02' + assert m._recv_buffer==b'' + assert m._send_buffer==InverterRspMsg + assert m._forward_buffer==InverterIndMsg + + m._send_buffer = bytearray(0) # clear send buffer for next test + m._forward_buffer = bytearray(0) # clear send buffer for next test + await m.send_at_cmd('AT+WEBU') + assert m._recv_buffer==b'' + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert str(m.seq) == '02:02' + assert m.forward_at_cmd_resp == False + m.close() + +def test_AT_cmd_ind(ConfigTsunInv1, AtCommandIndMsg): ConfigTsunInv1 m = MemoryStream(AtCommandIndMsg, (0,), False) m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.db.stat['proxy']['AT_Command'] = 0 + m.db.stat['proxy']['AT_Command_Blocked'] = 0 m.db.stat['proxy']['Modbus_Command'] = 0 m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed @@ -1297,6 +1345,32 @@ def test_at_command_ind(ConfigTsunInv1, AtCommandIndMsg): assert m._forward_buffer==AtCommandIndMsg assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 assert m.db.stat['proxy']['AT_Command'] == 1 + assert m.db.stat['proxy']['AT_Command_Blocked'] == 0 + assert m.db.stat['proxy']['Modbus_Command'] == 0 + m.close() + +def test_AT_cmd_ind_block(ConfigTsunInv1, AtCommandIndMsgBlock): + ConfigTsunInv1 + m = MemoryStream(AtCommandIndMsgBlock, (0,), False) + m.db.stat['proxy']['Unknown_Ctrl'] = 0 + m.db.stat['proxy']['AT_Command'] = 0 + m.db.stat['proxy']['AT_Command_Blocked'] = 0 + m.db.stat['proxy']['Modbus_Command'] = 0 + m.read() # read complete msg, and dispatch msg + assert not m.header_valid # must be invalid, since msg was handled and buffer flushed + assert m.msg_count == 1 + assert m.header_len==11 + assert m.snr == 2070233889 + # assert m.unique_id == '2070233889' + assert m.control == 0x4510 + assert str(m.seq) == '03:02' + assert m.data_len == 23 + assert m._recv_buffer==b'' + assert m._send_buffer==b'' + assert m._forward_buffer==b'' + assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 + assert m.db.stat['proxy']['AT_Command'] == 0 + assert m.db.stat['proxy']['AT_Command_Blocked'] == 1 assert m.db.stat['proxy']['Modbus_Command'] == 0 m.close() From 407c1ceb2b59c354a89ca86bb78c2da6eaca8c44 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 30 May 2024 19:40:25 +0200 Subject: [PATCH 099/118] control access via AT commands --- CHANGELOG.md | 1 + README.md | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 830c501..64be905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- add allow and block filter for AT+ commands - catch all OSError errors in the read loop - log Modbus traces with different log levels - add Modbus fifo and timeout handler diff --git a/README.md b/README.md index 27b84c7..89eaba3 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,9 @@ If you use a Pi-hole, you can also store the host entry in the Pi-hole. - `AT-Command` support via MQTT topics (GEN3PLUS only) - Faster DataUp interval sends measurement data to the MQTT broker every minute - Self-sufficient island operation without internet -- Runs in a non-root Docker Container +- Security-Features: + - control access via `AT commands` + - Runs in a non-root Docker Container ## Home Assistant Screenshots From 20f4fd647ca88ed5a5ec6b778069a1e51b698382 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 30 May 2024 19:44:54 +0200 Subject: [PATCH 100/118] update config example --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 89eaba3..ffe9f6d 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,12 @@ pv2 = {type = 'RSM40-8-410M', manufacturer = 'Risen'} # Optional, PV module de pv3 = {type = 'RSM40-8-410M', manufacturer = 'Risen'} # Optional, PV module descr pv4 = {type = 'RSM40-8-410M', manufacturer = 'Risen'} # Optional, PV module descr +[gen3plus.at_acl] +tsun.allow = ['AT+Z', 'AT+UPURL', 'AT+SUPDATE'] # allow this for TSUN access +tsun.block = [] +mqtt.allow = ['AT+'] # allow all via mqtt +mqtt.block = [] + ``` ## Inverter Configuration From 33f215def2843b11170e6475c0abdaa0d1673be1 Mon Sep 17 00:00:00 2001 From: Stefan Allius <122395479+s-allius@users.noreply.github.com> Date: Thu, 30 May 2024 20:30:48 +0200 Subject: [PATCH 101/118] Update README.md fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffe9f6d..a9c5cf4 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ If you use a Pi-hole, you can also store the host entry in the Pi-hole. - Faster DataUp interval sends measurement data to the MQTT broker every minute - Self-sufficient island operation without internet - Security-Features: - - control access via `AT commands` + - control access via `AT-commands` - Runs in a non-root Docker Container ## Home Assistant Screenshots From e850a8c534f86b48e6d9f3cb1bd471e8e2088982 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 20:02:21 +0200 Subject: [PATCH 102/118] set tracer log level by environment value --- app/src/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/server.py b/app/src/server.py index c7ee03e..18dc401 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -83,6 +83,7 @@ if __name__ == "__main__": logging.getLogger('msg').setLevel(log_level) logging.getLogger('conn').setLevel(log_level) logging.getLogger('data').setLevel(log_level) + logging.getLogger('tracer').setLevel(log_level) # logging.getLogger('mqtt').setLevel(log_level) # read config file From d27fe09006184de7fe12e5da248941c84382c347 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 20:03:21 +0200 Subject: [PATCH 103/118] reduce size of trace file - trace heartbeat and regular modbus pakets only with log level DBEUG - don't forwar akc pakets from tsun to inverter since we answered in before --- app/src/gen3/talent.py | 34 ++++++++++++++++++---- app/src/gen3plus/solarman_v5.py | 51 +++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index caf327d..25d15bc 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -52,6 +52,16 @@ class Talent(Message): # 0x78: 0x04: self.msg_inverter_data, } + self.log_lvl = { + 0x00: logging.INFO, + 0x13: logging.INFO, + 0x22: logging.INFO, + 0x71: logging.INFO, + # 0x76: + 0x77: self.get_modbus_log_lvl, + # 0x78: + 0x04: logging.INFO, + } self.modbus_elms = 0 # for unit tests ''' @@ -63,6 +73,7 @@ class Talent(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() + self.log_lvl.clear() self.state = self.STATE_CLOSED super().close() @@ -100,7 +111,11 @@ class Talent(Message): if self.header_valid and len(self._recv_buffer) >= (self.header_len + self.data_len): - hex_dump_memory(logging.INFO, f'Received from {self.addr}:', + log_lvl = self.log_lvl.get(self.msg_id, logging.WARNING) + if callable(log_lvl): + log_lvl = log_lvl() + + hex_dump_memory(log_lvl, f'Received from {self.addr}:', self._recv_buffer, self.header_len+self.data_len) self.__set_serial_no(self.id_str.decode("utf-8")) @@ -126,7 +141,7 @@ class Talent(Message): if self.state != self.STATE_UP: return - self.__build_header(0x70, 0x77, log_lvl) + self.__build_header(0x70, 0x77) self._send_buffer += b'\x00\x01\xa3\x28' # fixme self._send_buffer += struct.pack('!B', len(modbus_pdu)) self._send_buffer += modbus_pdu @@ -217,16 +232,15 @@ class Talent(Message): self.header_valid = True return - def __build_header(self, ctrl, msg_id=None, - log_lvl: int = logging.INFO) -> None: + def __build_header(self, ctrl, msg_id=None) -> None: if not msg_id: msg_id = self.msg_id self.send_msg_ofs = len(self._send_buffer) self._send_buffer += struct.pack(f'!l{len(self.id_str)+1}pBB', 0, self.id_str, ctrl, msg_id) fnc = self.switch.get(msg_id, self.msg_unknown) - logger.log(log_lvl, self.__flow_str(self.server_side, 'tx') + - f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') + logger.info(self.__flow_str(self.server_side, 'tx') + + f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}') def __finish_send_msg(self) -> None: _len = len(self._send_buffer) - self.send_msg_ofs @@ -391,6 +405,14 @@ class Talent(Message): # logger.debug(f'Modbus MsgLen: {modbus_len} Func:{result[2]}') return msg_hdr_len, modbus_len + def get_modbus_log_lvl(self) -> int: + if self.ctrl.is_req(): + return logging.INFO + elif self.ctrl.is_ind(): + if self.server_side: + return self.mb.last_log_lvl + return logging.WARNING + def msg_modbus(self): hdr_len, modbus_len = self.parse_modbus_header() data = self._recv_buffer[self.header_len: diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 1529057..2db9bcf 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -92,6 +92,29 @@ class SolarmanV5(Message): 0x4510: self.msg_command_req, # from server 0x1510: self.msg_command_rsp, # from inverter } + + self.log_lvl = { + + 0x4210: logging.INFO, # real time data + 0x1210: logging.INFO, # at least every 5 minutes + + 0x4710: logging.DEBUG, # heatbeat + 0x1710: logging.DEBUG, # every 2 minutes + + 0x4110: logging.INFO, # device data, sync start + 0x1110: logging.INFO, # every 3 hours + + 0x4310: logging.INFO, # regulary after 3-6 hours + 0x1310: logging.INFO, + + 0x4810: logging.INFO, # sync end + 0x1810: logging.INFO, + + # + # MODbus or AT cmd + 0x4510: logging.INFO, # from server + 0x1510: self.get_cmd_rsp_log_lvl, + } self.modbus_elms = 0 # for unit tests ''' @@ -103,6 +126,7 @@ class SolarmanV5(Message): # so we have to erase self.switch, otherwise this instance can't be # deallocated by the garbage collector ==> we get a memory leak self.switch.clear() + self.log_lvl.clear() self.state = self.STATE_CLOSED super().close() @@ -145,7 +169,10 @@ class SolarmanV5(Message): if self.header_valid and len(self._recv_buffer) >= (self.header_len + self.data_len+2): - hex_dump_memory(logging.INFO, f'Received from {self.addr}:', + log_lvl = self.log_lvl.get(self.control, logging.WARNING) + if callable(log_lvl): + log_lvl = log_lvl() + hex_dump_memory(log_lvl, f'Received from {self.addr}:', self._recv_buffer, self.header_len+self.data_len+2) if self.__trailer_is_ok(self._recv_buffer, self.header_len + self.data_len + 2): @@ -248,15 +275,15 @@ class SolarmanV5(Message): return True - def __build_header(self, ctrl, log_lvl: int = logging.INFO) -> None: + def __build_header(self, ctrl) -> None: '''build header for new transmit message''' self.send_msg_ofs = len(self._send_buffer) self._send_buffer += struct.pack( ' None: '''finish the transmit message, set lenght and checksum''' @@ -305,7 +332,7 @@ class SolarmanV5(Message): def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: return - self.__build_header(0x4510, log_lvl) + self.__build_header(0x4510) self._send_buffer += struct.pack(' int: + ftype = self._recv_buffer[self.header_len] + if ftype == self.AT_CMD: + if self.forward_at_cmd_resp: + return logging.INFO + return logging.DEBUG + elif ftype == self.MB_RTU_CMD: + if self.server_side: + return self.mb.last_log_lvl + + return logging.WARNING + def msg_command_rsp(self): data = self._recv_buffer[self.header_len: self.header_len+self.data_len] @@ -514,4 +553,4 @@ class SolarmanV5(Message): dt = datetime.fromtimestamp(ts) logger.debug(f'ts: {dt.strftime("%Y-%m-%d %H:%M:%S")}') - self.__forward_msg() + # self.__forward_msg() From 685c2dc07b811eedbac4c651a5f7b1721c9b98de Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 20:10:22 +0200 Subject: [PATCH 104/118] fix unit tests --- app/tests/test_solarman.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 6734015..afa5f3c 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -938,7 +938,7 @@ def test_device_rsp(ConfigTsunInv1, DeviceRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==DeviceRspMsg + assert m._forward_buffer==b'' assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -956,7 +956,7 @@ def test_inverter_rsp(ConfigTsunInv1, InverterRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==InverterRspMsg + assert m._forward_buffer==b'' assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -992,7 +992,7 @@ def test_heartbeat_rsp(ConfigTsunInv1, HeartbeatRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==HeartbeatRspMsg + assert m._forward_buffer==b'' assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -1034,7 +1034,7 @@ def test_sync_start_rsp(ConfigTsunInv1, SyncStartRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==SyncStartRspMsg + assert m._forward_buffer==b'' assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() @@ -1070,7 +1070,7 @@ def test_sync_end_rsp(ConfigTsunInv1, SyncEndRspMsg): assert m.data_len == 0x0a assert m._recv_buffer==b'' assert m._send_buffer==b'' - assert m._forward_buffer==SyncEndRspMsg + assert m._forward_buffer==b'' assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0 m.close() From be57d11214e1f2a72bb6d9553cfa1020c039bcfc Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 20:13:45 +0200 Subject: [PATCH 105/118] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 830c501..6afad84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- trace heartbeat and regular modbus pakets witl log level DEBUG +- GEN3PLUS: don't forward ack paket from tsun to the inverter - catch all OSError errors in the read loop - log Modbus traces with different log levels - add Modbus fifo and timeout handler From 5b60d5dae15ce2d61a5ae37481d1ccea920e8fda Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 23:09:14 +0200 Subject: [PATCH 106/118] cleanup --- app/src/gen3plus/solarman_v5.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 978ef78..9cd424b 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -578,4 +578,3 @@ class SolarmanV5(Message): dt = datetime.fromtimestamp(ts) logger.debug(f'ts: {dt.strftime("%Y-%m-%d %H:%M:%S")}') - # self.__forward_msg() From 56f36e9f3ffb600d4174c2845350b8a78502fea7 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 31 May 2024 23:09:33 +0200 Subject: [PATCH 107/118] build release candidate as paket --- app/build.sh | 20 ++++++++++++++------ docker-compose.yaml | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/app/build.sh b/app/build.sh index 07eefed..b87479a 100755 --- a/app/build.sh +++ b/app/build.sh @@ -18,22 +18,30 @@ arr=(${VERSION//./ }) MAJOR=${arr[0]} IMAGE=tsun-gen3-proxy -if [[ $1 == dev ]] || [[ $1 == rc ]] ;then +if [[ $1 == debug ]] || [[ $1 == dev ]] ;then IMAGE=docker.io/sallius/${IMAGE} VERSION=${VERSION}-$1 -elif [[ $1 == rel ]];then +elif [[ $1 == rc ]] || [[ $1 == rel ]];then IMAGE=ghcr.io/s-allius/${IMAGE} else echo argument missing! -echo try: $0 '[dev|rc|rel]' +echo try: $0 '[debug|dev|rc|rel]' exit 1 fi echo version: $VERSION build-date: $BUILD_DATE image: $IMAGE -if [[ $1 == dev ]];then -docker build --build-arg "VERSION=${VERSION}" --build-arg environment=dev --build-arg "LOG_LVL=DEBUG" --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:latest app +if [[ $1 == debug ]];then +docker build --build-arg "VERSION=${VERSION}" --build-arg environment=dev --build-arg "LOG_LVL=DEBUG" --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:dev app +elif [[ $1 == dev ]];then +docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:dev app + elif [[ $1 == rc ]];then -docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:latest app +docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:rc -t ${IMAGE}:${VERSION} app +echo 'login to ghcr.io' +echo $GHCR_TOKEN | docker login ghcr.io -u s-allius --password-stdin +docker push ghcr.io/s-allius/tsun-gen3-proxy:rc +docker push ghcr.io/s-allius/tsun-gen3-proxy:${VERSION} + elif [[ $1 == rel ]];then docker build --no-cache --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:latest -t ${IMAGE}:${MAJOR} -t ${IMAGE}:${VERSION} app echo 'login to ghcr.io' diff --git a/docker-compose.yaml b/docker-compose.yaml index de3e5e5..5b74fef 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -68,6 +68,7 @@ services: tsun-proxy: container_name: tsun-proxy image: ghcr.io/s-allius/tsun-gen3-proxy:latest + # image: ghcr.io/s-allius/tsun-gen3-proxy:rc restart: unless-stopped depends_on: - mqtt From 8baa68e615994bdd2612df3daa1bd2cba8e040db Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 2 Jun 2024 14:08:06 +0200 Subject: [PATCH 108/118] fix typo (wrong bracket) --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5b74fef..fad48fb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - GID=${GID:-1000} dns: - ${DNS1:-8.8.8.8} - - $(DNS2:-4.4.4.4} + - ${DNS2:-4.4.4.4} ports: - 5005:5005 - 10000:10000 From 8204cae2b1cacff3cf1c0d7703af706cd55c4533 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 3 Jun 2024 19:52:37 +0200 Subject: [PATCH 109/118] improve logging output --- CHANGELOG.md | 3 +++ app/src/async_stream.py | 12 +++++++----- app/src/gen3/inverter_g3.py | 2 +- app/src/gen3/talent.py | 6 ++++++ app/src/gen3plus/inverter_g3p.py | 2 +- app/src/gen3plus/solarman_v5.py | 9 +++++++++ app/src/messages.py | 2 +- 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 820aaef..8ce0f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- improve logging: add protocol or node_id to connection logs +- improve logging: log ignored AT+ or MODBUS commands +- fix typo in docker-compose.yaml - trace heartbeat and regular modbus pakets witl log level DEBUG - GEN3PLUS: don't forward ack paket from tsun to the inverter - add allow and block filter for AT+ commands diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 563b948..17f5f59 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -17,11 +17,12 @@ class AsyncStream(): async def server_loop(self, addr): '''Loop for receiving messages from the inverter (server-side)''' - logging.info(f'Accept connection from {addr}') + logging.info(f'[{self.node_id}] Accept connection from {addr}') self.inc_counter('Inverter_Cnt') await self.loop() self.dec_counter('Inverter_Cnt') - logging.info(f'Server loop stopped for r{self.r_addr}') + logging.info(f'[{self.node_id}] Server loop stopped for' + f' r{self.r_addr}') # if the server connection closes, we also have to disconnect # the connection to te TSUN cloud @@ -36,7 +37,8 @@ class AsyncStream(): async def client_loop(self, addr): '''Loop for receiving messages from the TSUN cloud (client-side)''' clientStream = await self.remoteStream.loop() - logging.info(f'Client loop stopped for l{clientStream.l_addr}') + logging.info(f'[{self.node_id}] Client loop stopped for' + f' l{clientStream.l_addr}') # if the client connection closes, we don't touch the server # connection. Instead we erase the client connection stream, @@ -66,14 +68,14 @@ class AsyncStream(): await self.async_publ_mqtt() except OSError as error: - logger.error(f'{error} for l{self.l_addr} | ' + logger.error(f'[{self.node_id}] {error} for l{self.l_addr} | ' f'r{self.r_addr}') await self.disc() self.close() return self except RuntimeError as error: - logger.warning(f"{error} for {self.l_addr}") + logger.info(f"[{self.node_id}] {error} for {self.l_addr}") await self.disc() self.close() return self diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index 5fccbc2..1930f0e 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -56,7 +56,7 @@ class InverterG3(Inverter, ConnectionG3): addr = (host, port) try: - logging.info(f'Connected to {addr}') + logging.info(f'[{self.node_id}] Connected to {addr}') connect = asyncio.open_connection(host, port) reader, writer = await connect self.remoteStream = ConnectionG3(reader, writer, addr, self, diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 25d15bc..be03df7 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -63,6 +63,8 @@ class Talent(Message): 0x04: logging.INFO, } self.modbus_elms = 0 # for unit tests + self.node_id = 'G3' # will be overwritten in __set_serial_no + # self.forwarding = Config.get('tsun')['enabled'] ''' Our puplic methods @@ -139,6 +141,8 @@ class Talent(Message): def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.__build_header(0x70, 0x77) @@ -154,6 +158,8 @@ class Talent(Message): async def send_modbus_cmd(self, func, addr, val, log_lvl) -> None: if self.state != self.STATE_UP: + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index b7b9800..487fe1e 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -56,7 +56,7 @@ class InverterG3P(Inverter, ConnectionG3P): addr = (host, port) try: - logging.info(f'Connected to {addr}') + logging.info(f'[{self.node_id}] Connected to {addr}') connect = asyncio.open_connection(host, port) reader, writer = await connect self.remoteStream = ConnectionG3P(reader, writer, addr, self, diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index 9cd424b..e823f57 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -122,6 +122,9 @@ class SolarmanV5(Message): if 'at_acl' in g3p_cnf: # pragma: no cover self.at_acl = g3p_cnf['at_acl'] + self.node_id = 'G3P' # will be overwritten in __set_serial_no + # self.forwarding = Config.get('solarman')['enabled'] + ''' Our puplic methods ''' @@ -336,6 +339,8 @@ class SolarmanV5(Message): def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.__build_header(0x4510) self._send_buffer += struct.pack(' None: if self.state != self.STATE_UP: + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) @@ -358,6 +365,8 @@ class SolarmanV5(Message): async def send_at_cmd(self, AT_cmd: str) -> None: if self.state != self.STATE_UP: + logger.warn(f'[{self.node_id}] ignore AT+ cmd,' + ' as the state is not UP') return AT_cmd = AT_cmd.strip() diff --git a/app/src/messages.py b/app/src/messages.py index 16e6d1b..6736b0b 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -72,7 +72,7 @@ class Message(metaclass=IterRegistry): self.header_len = 0 self.data_len = 0 self.unique_id = 0 - self.node_id = '' + self.node_id = '' # will be overwritten in the child class's __init__ self.sug_area = '' self._recv_buffer = bytearray(0) self._send_buffer = bytearray(0) From 8f81ceda98c171bf3fe2d524fb6fcfbf6f06f551 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 3 Jun 2024 20:28:14 +0200 Subject: [PATCH 110/118] fix warnings and remove obsolete version --- docker-compose.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index fad48fb..84e2b9f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,3 @@ - -version: '3.0' - services: ####### H O M E - A S S I S T A N T ##### home-assistant: @@ -34,7 +31,7 @@ services: ports: - 8123:8123 volumes: - - ${PROJECT_DIR}./homeassistant/config:/config + - ${PROJECT_DIR:-./}homeassistant/config:/config - /etc/localtime:/etc/localtime:ro healthcheck: test: curl --fail http://0.0.0.0:8123/auth/providers || exit 1 @@ -56,8 +53,8 @@ services: expose: - 1883 volumes: - - ${PROJECT_DIR}./mosquitto/config:/mosquitto/config - - ${PROJECT_DIR}./mosquitto/data:/mosquitto/data + - ${PROJECT_DIR:-./}mosquitto/config:/mosquitto/config + - ${PROJECT_DIR:-./}mosquitto/data:/mosquitto/data networks: outside: ipv4_address: 172.28.1.5 # static IP required to receive mDNS traffic @@ -83,8 +80,8 @@ services: - 5005:5005 - 10000:10000 volumes: - - ${PROJECT_DIR}./tsun-proxy/log:/home/tsun-proxy/log - - ${PROJECT_DIR}./tsun-proxy/config:/home/tsun-proxy/config + - ${PROJECT_DIR:-./}tsun-proxy/log:/home/tsun-proxy/log + - ${PROJECT_DIR:-./}tsun-proxy/config:/home/tsun-proxy/config networks: - outside From ad885e9644c17f66807d2022048a22c4f47ad0ac Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 3 Jun 2024 20:40:35 +0200 Subject: [PATCH 111/118] add Y47 serial numbers --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a9c5cf4..de53248 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ suggested_area = 'balcony' # Optional, suggested installation area for home-a pv1 = {type = 'RSM40-8-405M', manufacturer = 'Risen'} # Optional, PV module descr pv2 = {type = 'RSM40-8-405M', manufacturer = 'Risen'} # Optional, PV module descr -[inverters."Y17xxxxxxxxxxxx1"] +[inverters."Y17xxxxxxxxxxxx1"] # This block is also for inverters with a Y47 serial no monitor_sn = 2000000000 # The "Monitoring SN:" can be found on a sticker enclosed with the inverter node_id = 'inv_3' # MQTT replacement for inverters serial number suggested_area = 'garage' # suggested installation place for home-assistant @@ -241,7 +241,7 @@ Legend 🚧: Proxy support in preparation ``` -❗The new inverters of the GEN3 Plus generation (e.g. MS-2000) use a completely different protocol for data transmission to the TSUN server. These inverters are supported from proxy version 0.6. The serial numbers of these inverters start with `Y17E` instead of `R17E` +❗The new inverters of the GEN3 Plus generation (e.g. MS-2000) use a completely different protocol for data transmission to the TSUN server. These inverters are supported from proxy version 0.6. The serial numbers of these inverters start with `Y17E` or `Y47E` instead of `R17E` If you have one of these combinations with a red question mark, it would be very nice if you could send me a proxy trace so that I can carry out the detailed checks and adjust the device and system tests. [Ask here how to send a trace](https://github.com/s-allius/tsun-gen3-proxy/discussions/categories/traces-for-compatibility-check) From 6e1ed5d1e76bbebd595927becefc22f877a50eab Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 3 Jun 2024 20:59:21 +0200 Subject: [PATCH 112/118] check the docker-compose.yaml file as last step --- app/build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/build.sh b/app/build.sh index b87479a..c654c1c 100755 --- a/app/build.sh +++ b/app/build.sh @@ -49,4 +49,7 @@ echo $GHCR_TOKEN | docker login ghcr.io -u s-allius --password-stdin docker push ghcr.io/s-allius/tsun-gen3-proxy:latest docker push ghcr.io/s-allius/tsun-gen3-proxy:${MAJOR} docker push ghcr.io/s-allius/tsun-gen3-proxy:${VERSION} -fi \ No newline at end of file +fi + +echo 'check docker-compose.yaml file' +docker-compose config -q \ No newline at end of file From e6ecf5911b3cac8ec3b404662ffe3c7183640a8f Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 4 Jun 2024 20:00:39 +0200 Subject: [PATCH 113/118] remove the external network expectation --- docker-compose.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 84e2b9f..4566a80 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -56,10 +56,9 @@ services: - ${PROJECT_DIR:-./}mosquitto/config:/mosquitto/config - ${PROJECT_DIR:-./}mosquitto/data:/mosquitto/data networks: - outside: - ipv4_address: 172.28.1.5 # static IP required to receive mDNS traffic - + - outside + ####### T S U N - P R O X Y ###### tsun-proxy: @@ -91,11 +90,4 @@ services: networks: outside: name: home-assistant - external: true - ipam: - driver: default - config: - - subnet: 172.28.1.0/26 - ip_range: 172.28.1.32/27 - gateway: 172.28.1.62 \ No newline at end of file From 49e2dfbd86270eced7deae1dce62e3e717cbe82a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 4 Jun 2024 20:27:15 +0200 Subject: [PATCH 114/118] optimize docker-compose.yaml file --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce0f47..63c3f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - improve logging: add protocol or node_id to connection logs - improve logging: log ignored AT+ or MODBUS commands -- fix typo in docker-compose.yaml +- fix typo in docker-compose.yaml and remove the external network definition - trace heartbeat and regular modbus pakets witl log level DEBUG - GEN3PLUS: don't forward ack paket from tsun to the inverter - add allow and block filter for AT+ commands From 039a021cdac8560b299084e825e72c510d08bd92 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 4 Jun 2024 21:55:57 +0200 Subject: [PATCH 115/118] cleanup trace output --- app/src/gen3/infos_g3.py | 2 +- app/src/gen3plus/infos_g3p.py | 2 +- app/src/modbus.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/gen3/infos_g3.py b/app/src/gen3/infos_g3.py index 1de9bf0..0dc6a35 100644 --- a/app/src/gen3/infos_g3.py +++ b/app/src/gen3/infos_g3.py @@ -164,7 +164,7 @@ class InfosG3(Infos): name = str(f'info-id.0x{addr:x}') if update: - self.tracer.log(level, f'[\'{node_id}\']GEN3: {name} :' + self.tracer.log(level, f'[{node_id}] GEN3: {name} :' f' {result}{unit}') i += 1 diff --git a/app/src/gen3plus/infos_g3p.py b/app/src/gen3plus/infos_g3p.py index 213bcf2..bf0aed8 100644 --- a/app/src/gen3plus/infos_g3p.py +++ b/app/src/gen3plus/infos_g3p.py @@ -123,5 +123,5 @@ class InfosG3P(Infos): update = False if update: - self.tracer.log(level, f'[\'{node_id}\']GEN3PLUS: {name}' + self.tracer.log(level, f'[{node_id}] GEN3PLUS: {name}' f' : {result}{unit}') diff --git a/app/src/modbus.py b/app/src/modbus.py index 0727650..b428d05 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -216,7 +216,7 @@ class Modbus(): yield keys[0], update, result if update: info_db.tracer.log(level, - f'[\'{node_id}\']MODBUS: {name}' + f'[{node_id}] MODBUS: {name}' f' : {result}{unit}') else: self.__stop_timer() From c59bd1666403499c7f6703516b04a7ff5acc9aaa Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 5 Jun 2024 22:01:48 +0200 Subject: [PATCH 116/118] change log level for some traces --- app/src/gen3/talent.py | 8 ++++---- app/src/gen3plus/solarman_v5.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index be03df7..5c6a9be 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -141,8 +141,8 @@ class Talent(Message): def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.__build_header(0x70, 0x77) @@ -158,8 +158,8 @@ class Talent(Message): async def send_modbus_cmd(self, func, addr, val, log_lvl) -> None: if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index e823f57..c8e3aaa 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -339,8 +339,8 @@ class SolarmanV5(Message): def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.__build_header(0x4510) self._send_buffer += struct.pack(' None: if self.state != self.STATE_UP: - logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) From 0b2631c16218a736be1af574358636cacda69cac Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 7 Jun 2024 19:27:36 +0200 Subject: [PATCH 117/118] beautify some traces --- app/src/gen3/talent.py | 8 ++++---- app/src/gen3plus/solarman_v5.py | 8 ++++---- app/src/modbus.py | 10 ++++++---- app/src/mqtt.py | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 5c6a9be..ad37337 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -141,8 +141,8 @@ class Talent(Message): def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' cause the state is not UP anymore') return self.__build_header(0x70, 0x77) @@ -158,8 +158,8 @@ class Talent(Message): async def send_modbus_cmd(self, func, addr, val, log_lvl) -> None: if self.state != self.STATE_UP: - logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.log(log_lvl, f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index c8e3aaa..64e4536 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -339,8 +339,8 @@ class SolarmanV5(Message): def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str): if self.state != self.STATE_UP: - logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.warn(f'[{self.node_id}] ignore MODBUS cmd,' + ' cause the state is not UP anymore') return self.__build_header(0x4510) self._send_buffer += struct.pack(' None: if self.state != self.STATE_UP: - logger.debug(f'[{self.node_id}] ignore MODBUS cmd,' - ' as the state is not UP') + logger.log(log_lvl, f'[{self.node_id}] ignore MODBUS cmd,' + ' as the state is not UP') return self.mb.build_msg(Modbus.INV_ADDR, func, addr, val, log_lvl) diff --git a/app/src/modbus.py b/app/src/modbus.py index b428d05..8f3778b 100644 --- a/app/src/modbus.py +++ b/app/src/modbus.py @@ -172,23 +172,25 @@ class Modbus(): self.err = 5 return if not self.__check_crc(buf): - logger.error('Modbus resp: CRC error') + logger.error(f'[{node_id}] Modbus resp: CRC error') self.err = 1 return if buf[0] != self.last_addr: - logger.info(f'Modbus resp: Wrong addr {buf[0]}') + logger.info(f'[{node_id}] Modbus resp: Wrong addr {buf[0]}') self.err = 2 return fcode = buf[1] if fcode != self.last_fcode: - logger.info(f'Modbus: Wrong fcode {fcode} != {self.last_fcode}') + logger.info(f'[{node_id}] Modbus: Wrong fcode {fcode}' + f' != {self.last_fcode}') self.err = 3 return if self.last_addr == self.INV_ADDR and \ (fcode == 3 or fcode == 4): elmlen = buf[2] >> 1 if elmlen != self.last_len: - logger.info(f'Modbus: len error {elmlen} != {self.last_len}') + logger.info(f'[{node_id}] Modbus: len error {elmlen}' + f' != {self.last_len}') self.err = 4 return first_reg = self.last_reg # save last_reg before sending next pdu diff --git a/app/src/mqtt.py b/app/src/mqtt.py index a36943d..2f55660 100644 --- a/app/src/mqtt.py +++ b/app/src/mqtt.py @@ -148,10 +148,10 @@ class Mqtt(metaclass=Singleton): node_id = topic.split('/')[1] + '/' # refactor into a loop over a table payload = message.payload.decode("UTF-8") - logger_mqtt.info(f'InvCnf: {node_id}:{payload}') + logger_mqtt.info(f'MODBUS via MQTT: {topic} = {payload}') for m in Message: if m.server_side and (m.node_id == node_id): - logger_mqtt.info(f'Found: {node_id}') + logger_mqtt.debug(f'Found: {node_id}') fnc = getattr(m, "send_modbus_cmd", None) res = payload.split(',') if params != len(res): From a62864218d6ebb8a490850f7c6f3f53db6459691 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 7 Jun 2024 19:48:41 +0200 Subject: [PATCH 118/118] update for version 0.8.0 --- CHANGELOG.md | 5 +- app/proxy.svg | 424 +++++++++++++++++++++++++------------------------ app/proxy.yuml | 4 +- 3 files changed, 221 insertions(+), 212 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c3f31..3d4eeed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - improve logging: add protocol or node_id to connection logs - improve logging: log ignored AT+ or MODBUS commands +- improve tracelog: log level depends on message type and source - fix typo in docker-compose.yaml and remove the external network definition - trace heartbeat and regular modbus pakets witl log level DEBUG - GEN3PLUS: don't forward ack paket from tsun to the inverter -- add allow and block filter for AT+ commands +- GEN3PLUS: add allow and block filter for AT+ commands - catch all OSError errors in the read loop - log Modbus traces with different log levels - add Modbus fifo and timeout handler @@ -27,7 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - change logging level to DEBUG for some logs - remove experimental value Register.VALUE_1 - format Register.POWER_ON_TIME as integer -- ignore non realtime values for now +- ignore catch-up values from the inverters for now ## [0.7.0] - 2024-04-20 diff --git a/app/proxy.svg b/app/proxy.svg index 9ee1aba..b111fd4 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,246 +4,254 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -Singleton + +Singleton A2 - -Mqtt - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() + +Mqtt + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() A1->A2 - - + + A11 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt - + +Inverter + +cls.db_stat +cls.entity_prfx +cls.discovery_prfx +cls.proxy_node_id +cls.proxy_unique_id +cls.mqtt:Mqtt + A2->A11 - + A3 - -Modbus - -err -retry_cnt - -build_msg() -recv_req() -recv_resp() + +Modbus + +que +snd_handler +rsp_handler +timeout:max_retires +last_xxx +err +retry_cnt +req_pend +tim + +build_msg() +recv_req() +recv_resp() A4 - -IterRegistry - - -__iter__ + +IterRegistry + + +__iter__ A5 - -Message - -server_side:bool -header_valid:bool -header_len:unsigned -data_len:unsigned -unique_id -node_id -sug_area -_recv_buffer:bytearray -_send_buffer:bytearray -_forward_buffer:bytearray -db:Infos -new_data:list - -_read():void<abstract> -close():void -inc_counter():void -dec_counter():void + +Message + +server_side:bool +header_valid:bool +header_len:unsigned +data_len:unsigned +unique_id +node_id +sug_area +_recv_buffer:bytearray +_send_buffer:bytearray +_forward_buffer:bytearray +db:Infos +new_data:list +state + +_read():void<abstract> +close():void +inc_counter():void +dec_counter():void A4->A5 - - + + A6 - -Talent - -await_conn_resp_cnt -id_str -contact_name -contact_mail -db:InfosG3 -mb:Modbus -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -close() + +Talent + +await_conn_resp_cnt +id_str +contact_name +contact_mail +db:InfosG3 +mb:Modbus +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +close() A5->A6 - - + + A7 - -SolarmanV5 - -control -serial -snr -db:InfosG3P -mb:Modbus -switch - -msg_unknown() -close() + +SolarmanV5 + +control +serial +snr +db:InfosG3P +mb:Modbus +switch + +msg_unknown() +close() A5->A7 - - + + A6->A3 - - -1 -has + + +1 +has A8 - -ConnectionG3 - -remoteStream:ConnectionG3 - -close() + +ConnectionG3 + +remoteStream:ConnectionG3 + +close() A6->A8 - - + + A7->A3 - - -1 -has + + +1 +has A9 - -ConnectionG3P - -remoteStream:ConnectionG3P - -close() + +ConnectionG3P + +remoteStream:ConnectionG3P + +close() A7->A9 - - + + A8->A8 - - -0..1 -has + + +0..1 +has A12 - -InverterG3 - -__ha_restarts - -async_create_remote() -close() + +InverterG3 + +__ha_restarts + +async_create_remote() +close() A8->A12 - - + + A9->A9 - - -0..1 -has + + +0..1 +has @@ -259,118 +267,118 @@ A9->A13 - - + + A10 - -AsyncStream - -reader -writer -addr -r_addr -l_addr - -<async>server_loop() -<async>client_loop() -<async>loop -disc() -close() -__async_read() -async_write() -__async_forward() + +AsyncStream + +reader +writer +addr +r_addr +l_addr + +<async>server_loop() +<async>client_loop() +<async>loop +disc() +close() +__async_read() +async_write() +__async_forward() A10->A8 - - + + A10->A9 - - + + A11->A12 - - + + A11->A13 - - + + A14 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -update_db -set_db_def_value -get_db_value -ignore_this_device + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +update_db +set_db_def_value +get_db_value +ignore_this_device A15 - -InfosG3 - - -ha_confs() -parse() + +InfosG3 + + +ha_confs() +parse() A14->A15 - - + + A16 - -InfosG3P - - -ha_confs() -parse() + +InfosG3P + + +ha_confs() +parse() A14->A16 - - + + A15->A6 - - + + A16->A7 - - + + diff --git a/app/proxy.yuml b/app/proxy.yuml index e2ce12b..60b506e 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -4,8 +4,8 @@ [note: You can stick notes on diagrams too!{bg:cornsilk}] [Singleton]^[Mqtt|ha_restarts;__client;__cb_MqttIsUp|publish();close()] -[Modbus|err;retry_cnt|build_msg();recv_req();recv_resp()] -[IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list|_read():void;close():void;inc_counter():void;dec_counter():void] +[Modbus|que;;snd_handler;rsp_handler;timeout:max_retires;last_xxx;err;retry_cnt;req_pend;tim|build_msg();recv_req();recv_resp()] +[IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list;state|_read():void;close():void;inc_counter():void;dec_counter():void] [Message]^[Talent|await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;close()] [Message]^[SolarmanV5|control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;close()] [Talent]^[ConnectionG3|remoteStream:ConnectionG3|close()]
Micro Inverter ModelFw. 1.00.06Fw. 1.00.17Fw. 1.00.20Fw. 1.1.00.0B
Micro Inverter ModelFw. 1.00.06Fw. 1.00.17Fw. 1.00.20Fw. 4.0.10
GEN3 micro inverters (single MPPT):
MS300, MS350, MS400
MS400-D		❓❓❓➖
GEN3 micro inverters (dual MPPT):
MS600, MS700, MS800
MS600-D, MS800-D		✔️✔️✔️➖
GEN3 PLUS micro inverters:
MS1600, MS1800, MS2000
MS2000-D		➖➖➖✔️