fix target preview

* GEN3: After inverter firmware update the 'Unknown Msg Type' increases continuously
Fixes #182

* add support for Controller serial no and MAC

* test hardening

* GEN3: add support for new messages of version 3 firmwares

* bump libraries to latest versions

- bump aiomqtt to version 2.3.0
- bump aiohttp to version 3.10.5

* improve test coverage

* reduce cognective complexity

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+- GEN3: add support for new messages of version 3 firmwares
+- add support for controller MAC and serial number
 - GEN3: don't crash on overwritten msg in the receive buffer
 - Reading the version string from the image updates it even if the image is re-pulled without re-deployment
   

app/docker-bake.hcl

@@ -78,7 +78,7 @@ target "dev" {
 
 target "preview" {
   inherits = ["_common", "_prod"]
-  tags = ["${IMAGE}:dev", "${IMAGE}:${VERSION}"]
+  tags = ["${IMAGE}:preview", "${IMAGE}:${VERSION}"]
 }
 
 target "rc" {

app/requirements.txt

@@ -1,4 +1,4 @@
-    aiomqtt==2.2.0
+    aiomqtt==2.3.0
     schema==0.7.7
     aiocron==1.8
-    aiohttp==3.10.2
+    aiohttp==3.10.5

app/src/gen3/infos_g3.py

@@ -14,6 +14,7 @@ class RegisterMap:
         0x00092ba8: Register.COLLECTOR_FW_VERSION,
         0x000927c0: Register.CHIP_TYPE,
         0x00092f90: Register.CHIP_MODEL,
+        0x00094ae8: Register.MAC_ADDR,
         0x00095a88: Register.TRACE_URL,
         0x00095aec: Register.LOGGER_URL,
         0x0000000a: Register.PRODUCT_NAME,

app/src/gen3/talent.py

@@ -56,20 +56,24 @@ class Talent(Message):
             0x00: self.msg_contact_info,
             0x13: self.msg_ota_update,
             0x22: self.msg_get_time,
+            0x99: self.msg_act_time,
             0x71: self.msg_collector_data,
             # 0x76:
             0x77: self.msg_modbus,
             # 0x78:
+            0x87: self.msg_modbus2,
             0x04: self.msg_inverter_data,
         }
         self.log_lvl = {
             0x00: logging.INFO,
             0x13: logging.INFO,
             0x22: logging.INFO,
+            0x99: logging.INFO,
             0x71: logging.INFO,
             # 0x76:
             0x77: self.get_modbus_log_lvl,
             # 0x78:
+            0x87: self.get_modbus_log_lvl,
             0x04: logging.INFO,
         }
         self.modbus_elms = 0    # for unit tests
@@ -127,6 +131,7 @@ class Talent(Message):
                 logger.debug(f'SerialNo {serial_no} not known but accepted!')
 
             self.unique_id = serial_no
+            self.db.set_db_def_value(Register.COLLECTOR_SNR, serial_no)
 
     def read(self) -> float:
         '''process all received messages in the _recv_buffer'''
@@ -170,6 +175,25 @@ class Talent(Message):
             logger.info(self.__flow_str(self.server_side, 'forwrd') +
                         f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}')
 
+    def forward_snd(self) -> None:
+        '''add the actual receive msg to the forwarding queue'''
+        tsun = Config.get('tsun')
+        if tsun['enabled']:
+            _len = len(self._send_buffer) - self.send_msg_ofs
+            struct.pack_into('!l', self._send_buffer, self.send_msg_ofs,
+                             _len-4)
+
+            buffer = self._send_buffer[self.send_msg_ofs:]
+            buflen = _len
+            self._forward_buffer += buffer[:buflen]
+            hex_dump_memory(logging.INFO, 'Store for forwarding:',
+                            buffer, buflen)
+
+            fnc = self.switch.get(self.msg_id, self.msg_unknown)
+            logger.info(self.__flow_str(self.server_side, 'forwrd') +
+                        f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}')
+        self._send_buffer = self._send_buffer[:self.send_msg_ofs]
+
     def send_modbus_cb(self, modbus_pdu: bytearray, log_lvl: int, state: str):
         if self.state != State.up:
             logger.warning(f'[{self.node_id}] ignore MODBUS cmd,'
@@ -400,6 +424,8 @@ class Talent(Message):
                 result = struct.unpack_from('!q', self._recv_buffer,
                                             self.header_len)
                 self.ts_offset = result[0]-ts
+                if self.remote_stream:
+                    self.remote_stream.ts_offset = self.ts_offset
                 logger.debug(f'tsun-time: {int(result[0]):08x}'
                              f'  proxy-time: {ts:08x}'
                              f'  offset: {self.ts_offset}')
@@ -410,6 +436,41 @@ class Talent(Message):
 
         self.forward()
 
+    def msg_act_time(self):
+        if self.ctrl.is_ind():
+            if self.data_len == 9:
+                self.state = State.up  # allow MODBUS cmds
+                if (self.modbus_polling):
+                    self.mb_timer.start(self.mb_first_timeout)
+                    self.db.set_db_def_value(Register.POLLING_INTERVAL,
+                                             self.mb_timeout)
+                self.__build_header(0x99)
+                self._send_buffer += b'\x02'
+                self.__finish_send_msg()
+
+                result = struct.unpack_from('!Bq', self._recv_buffer,
+                                            self.header_len)
+                resp_code = result[0]
+                ts = result[1]+self.ts_offset
+                logger.debug(f'inv-time: {int(result[1]):08x}'
+                             f'  tsun-time: {ts:08x}'
+                             f'  offset: {self.ts_offset}')
+                self.__build_header(0x91)
+                self._send_buffer += struct.pack('!Bq', resp_code, ts)
+                self.forward_snd()
+                return
+        elif self.ctrl.is_resp():
+            result = struct.unpack_from('!B', self._recv_buffer,
+                                        self.header_len)
+            resp_code = result[0]
+            logging.debug(f'TimeActRespCode: {resp_code}')
+            return
+        else:
+            logger.warning(self.TXT_UNKNOWN_CTRL)
+            self.inc_counter('Unknown_Ctrl')
+
+        self.forward()
+
     def parse_msg_header(self):
         result = struct.unpack_from('!lB', self._recv_buffer, self.header_len)
 
@@ -492,6 +553,15 @@ class Talent(Message):
         modbus_len = result[1]
         return msg_hdr_len, modbus_len
 
+    def parse_modbus_header2(self):
+
+        msg_hdr_len = 6
+
+        result = struct.unpack_from('!lBBB', self._recv_buffer,
+                                    self.header_len)
+        modbus_len = result[2]
+        return msg_hdr_len, modbus_len
+
     def get_modbus_log_lvl(self) -> int:
         if self.ctrl.is_req():
             return logging.INFO
@@ -501,6 +571,13 @@ class Talent(Message):
 
     def msg_modbus(self):
         hdr_len, _ = self.parse_modbus_header()
+        self.__msg_modbus(hdr_len)
+
+    def msg_modbus2(self):
+        hdr_len, _ = self.parse_modbus_header2()
+        self.__msg_modbus(hdr_len)
+
+    def __msg_modbus(self, hdr_len):
         data = self._recv_buffer[self.header_len:
                                  self.header_len+self.data_len]
 

app/src/gen3plus/solarman_v5.py

@@ -203,13 +203,15 @@ class SolarmanV5(Message):
             inverters = Config.get('inverters')
             # logger.debug(f'Inverters: {inverters}')
 
-            for inv in inverters.values():
+            for key, inv in inverters.items():
                 # logger.debug(f'key: {key} -> {inv}')
                 if (type(inv) is dict and 'monitor_sn' in inv
                    and inv['monitor_sn'] == snr):
                     self.__set_config_parms(inv)
                     self.db.set_pv_module_details(inv)
                     logger.debug(f'SerialNo {serial_no} allowed! area:{self.sug_area}')  # noqa: E501
+
+                    self.db.set_db_def_value(Register.COLLECTOR_SNR, key)
                     break
             else:
                 self.node_id = ''

app/src/infos.py

@@ -16,6 +16,8 @@ class Register(Enum):
     CHIP_MODEL = 3
     TRACE_URL = 4
     LOGGER_URL = 5
+    MAC_ADDR = 6
+    COLLECTOR_SNR = 7
     PRODUCT_NAME = 20
     MANUFACTURER = 21
     VERSION = 22
@@ -188,8 +190,8 @@ class Infos:
 
     __info_devs = {
         'proxy':      {'singleton': True,   'name': 'Proxy', 'mf': 'Stefan Allius'},  # noqa: E501
-        'controller': {'via': 'proxy',      'name': 'Controller',     'mdl': Register.CHIP_MODEL, 'mf': Register.CHIP_TYPE, 'sw': Register.COLLECTOR_FW_VERSION},  # noqa: E501
+        'controller': {'via': 'proxy',      'name': 'Controller',     'mdl': Register.CHIP_MODEL, 'mf': Register.CHIP_TYPE, 'sw': Register.COLLECTOR_FW_VERSION, 'mac': Register.MAC_ADDR, 'sn': Register.COLLECTOR_SNR},  # noqa: E501
-        'inverter':   {'via': 'controller', 'name': 'Micro Inverter', 'mdl': Register.EQUIPMENT_MODEL, 'mf': Register.MANUFACTURER, 'sw': Register.VERSION},  # noqa: E501
+        'inverter':   {'via': 'controller', 'name': 'Micro Inverter', 'mdl': Register.EQUIPMENT_MODEL, 'mf': Register.MANUFACTURER, 'sw': Register.VERSION, 'sn': Register.SERIAL_NUMBER},  # noqa: E501
         'input_pv1':  {'via': 'inverter',   'name': 'Module PV1', 'mdl': Register.PV1_MODEL, 'mf': Register.PV1_MANUFACTURER},  # noqa: E501
         'input_pv2':  {'via': 'inverter',   'name': 'Module PV2', 'mdl': Register.PV2_MODEL, 'mf': Register.PV2_MANUFACTURER, 'dep': {'reg': Register.NO_INPUTS, 'gte': 2}},  # noqa: E501
         'input_pv3':  {'via': 'inverter',   'name': 'Module PV3', 'mdl': Register.PV3_MODEL, 'mf': Register.PV3_MANUFACTURER, 'dep': {'reg': Register.NO_INPUTS, 'gte': 3}},  # noqa: E501
@@ -222,6 +224,9 @@ class Infos:
         Register.CHIP_MODEL: {'name': ['collector', 'Chip_Model'],       'singleton': False,  'level': logging.DEBUG, 'unit': ''},  # noqa: E501
         Register.TRACE_URL:  {'name': ['collector', 'Trace_URL'],        'singleton': False,  'level': logging.DEBUG, 'unit': ''},  # noqa: E501
         Register.LOGGER_URL: {'name': ['collector', 'Logger_URL'],       'singleton': False,  'level': logging.DEBUG, 'unit': ''},  # noqa: E501
+        Register.MAC_ADDR:   {'name': ['collector', 'MAC-Addr'],         'singleton': False,  'level': logging.INFO,  'unit': ''},  # noqa: E501
+        Register.COLLECTOR_SNR: {'name': ['collector', 'Serial_Number'], 'singleton': False,  'level': logging.INFO,  'unit': ''},  # noqa: E501
+
 
         # inverter values used for device registration:
         Register.PRODUCT_NAME:    {'name': ['inverter', 'Product_Name'],           'level': logging.DEBUG, 'unit': ''},  # noqa: E501
@@ -507,7 +512,7 @@ class Infos:
             dev['name'] = device['name']+' - '+sug_area
             dev['sa'] = device['name']+' - '+sug_area
         self.__add_via_dev(dev, device, key, snr)
-        for key in ('mdl', 'mf', 'sw', 'hw'):      # add optional
+        for key in ('mdl', 'mf', 'sw', 'hw', 'sn'):      # add optional
             # values fpr 'modell', 'manufacturer', 'sw version' and
             # 'hw version'
             if key in device:
@@ -518,8 +523,17 @@ class Infos:
             dev['ids'] = [f"{ha['dev']}"]
         else:
             dev['ids'] = [f"{ha['dev']}_{snr}"]
+        self.__add_connection(dev, device)
         return dev
 
+    def __add_connection(self, dev, device):
+        if 'mac' in device:
+            mac_str = self.dev_value(device['mac'])
+            if mac_str is not None:
+                if 12 == len(mac_str):
+                    mac_str = ':'.join(mac_str[i:i+2] for i in range(0, 12, 2))
+                dev['cns'] = [["mac", f"{mac_str}"]]
+
     def __add_via_dev(self, dev, device, key, snr):
         if 'via' in device:  # add the link to the parent device
             via = device['via']

app/tests/test_infos_g3.py

@@ -364,12 +364,12 @@ def test_build_ha_conf3(contr_data_seq, inv_data_seq, inv_data_seq2):
 
         if id == 'out_power_123':
             assert comp == 'sensor'
-            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Micro Inverter - roof", "sa": "Micro Inverter - roof", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Micro Inverter - roof", "sa": "Micro Inverter - roof", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "sn": "T170000000000001", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
 
         if id == 'daily_gen_123':
             assert comp == 'sensor'
-            assert  d_json == json.dumps({"name": "Daily Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total_increasing", "uniq_id": "daily_gen_123", "val_tpl": "{{value_json['Daily_Generation'] | float}}", "unit_of_meas": "kWh", "ic": "mdi:solar-power-variant", "dev": {"name": "Micro Inverter - roof", "sa": "Micro Inverter - roof", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            assert  d_json == json.dumps({"name": "Daily Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total_increasing", "uniq_id": "daily_gen_123", "val_tpl": "{{value_json['Daily_Generation'] | float}}", "unit_of_meas": "kWh", "ic": "mdi:solar-power-variant", "dev": {"name": "Micro Inverter - roof", "sa": "Micro Inverter - roof", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "sn": "T170000000000001", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
 
         elif id == 'power_pv1_123':
@@ -388,6 +388,32 @@ def test_build_ha_conf3(contr_data_seq, inv_data_seq, inv_data_seq2):
             tests +=1
     assert tests==5
 
+def test_build_ha_conf4(contr_data_seq, inv_data_seq):
+    i = InfosG3()
+    for key, result in i.parse (contr_data_seq):
+        pass  # side effect in calling i.parse()
+    for key, result in i.parse (inv_data_seq):
+        pass  # side effect in calling i.parse()
+    i.set_db_def_value(Register.MAC_ADDR, "00a057123456")
+
+    tests = 0
+    for d_json, comp, node_id, id in i.ha_confs(ha_prfx="tsun/", node_id="garagendach/", snr='123', sug_area = 'roof'):
+        if id == 'signal_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Signal Strength", "stat_t": "tsun/garagendach/controller", "dev_cla": None, "stat_cla": "measurement", "uniq_id": "signal_123", "val_tpl": "{{value_json[\'Signal_Strength\'] | int}}", "unit_of_meas": "%", "ic": "mdi:wifi", "dev": {"name": "Controller - roof", "sa": "Controller - roof", "via_device": "proxy", "mdl": "RSW-1-10001", "mf": "Raymon", "sw": "RSW_400_V1.00.06", "ids": ["controller_123"], "cns": [["mac", "00:a0:57:12:34:56"]]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+    assert tests==1
+
+    i.set_db_def_value(Register.MAC_ADDR, "00:a0:57:12:34:57")
+
+    tests = 0
+    for d_json, comp, node_id, id in i.ha_confs(ha_prfx="tsun/", node_id="garagendach/", snr='123', sug_area = 'roof'):
+        if id == 'signal_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Signal Strength", "stat_t": "tsun/garagendach/controller", "dev_cla": None, "stat_cla": "measurement", "uniq_id": "signal_123", "val_tpl": "{{value_json[\'Signal_Strength\'] | int}}", "unit_of_meas": "%", "ic": "mdi:wifi", "dev": {"name": "Controller - roof", "sa": "Controller - roof", "via_device": "proxy", "mdl": "RSW-1-10001", "mf": "Raymon", "sw": "RSW_400_V1.00.06", "ids": ["controller_123"], "cns": [["mac", "00:a0:57:12:34:57"]]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+    assert tests==1
+
 def test_must_incr_total(inv_data_seq2, inv_data_seq2_zero):
     i = InfosG3()
     tests = 0

app/tests/test_modbus_tcp.py

@@ -4,6 +4,7 @@ import asyncio
 
 from mock import patch
 from enum import Enum
+from enum import Enum
 from app.src.singleton import Singleton
 from app.src.config import Config
 from app.src.infos import Infos
@@ -11,6 +12,10 @@ from app.src.mqtt import Mqtt
 from app.src.messages import Message, State
 from app.src.inverter import Inverter
 from app.src.modbus_tcp import ModbusConn, ModbusTcp
+from app.src.mqtt import Mqtt
+from app.src.messages import Message, State
+from app.src.inverter import Inverter
+from app.src.modbus_tcp import ModbusConn, ModbusTcp
 
 
 pytest_plugins = ('pytest_asyncio',)
@@ -77,6 +82,7 @@ class TestType(Enum):
 
 test  = TestType.RD_TEST_0_BYTES
 
+
 class FakeReader():
     def __init__(self):
         self.on_recv =  asyncio.Event()
@@ -127,6 +133,11 @@ def patch_no_mqtt():
     with patch.object(Mqtt, 'publish') as conn:
         yield conn
 
+@pytest.fixture
+def patch_no_mqtt():
+    with patch.object(Mqtt, 'publish') as conn:
+        yield conn
+
 
 @pytest.mark.asyncio
 async def test_modbus_conn(patch_open):
@@ -194,11 +205,6 @@ async def test_modbus_cnf2(config_conn, patch_no_mqtt, patch_open):
     assert 1 == test
     await asyncio.sleep(0.01)
     assert Infos.stat['proxy']['Inverter_Cnt'] == 0
-    # check that the connection is released
-    for m in Message:
-        if (m.node_id == 'inv_2'):
-            assert False
-
 
 @pytest.mark.asyncio
 async def test_modbus_cnf3(config_conn, patch_no_mqtt, patch_open):

app/tests/test_solarman.py

@@ -184,6 +184,35 @@ def device_rsp_msg():  # 0x1110
     msg += b'\x15'
     return msg
 
+@pytest.fixture
+def device_ind_msg2(): # 0x4110
+    msg  = b'\xa5\xd4\x00\x10\x41\x02\x03' +get_sn()  +b'\x02\xba\xd2\x00\x00'
+    msg += b'\x19\x00\x00\x00\x00\x00\x00\x00\x05\x3c\x78\x01\x64\x01\x4c\x53'
+    msg += b'\x57\x35\x42\x4c\x45\x5f\x31\x37\x5f\x30\x32\x42\x30\x5f\x31\x2e'
+    msg += b'\x30\x35\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x40\x2a\x8f\x4f\x51\x54\x31\x39\x32\x2e'
+    msg += b'\x31\x36\x38\x2e\x38\x30\x2e\x34\x39\x00\x00\x00\x0f\x00\x01\xb0'
+    msg += b'\x02\x0f\x00\xff\x56\x31\x2e\x31\x2e\x30\x30\x2e\x30\x42\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfe\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x41\x6c\x6c\x69\x75\x73\x2d\x48\x6f'
+    msg += b'\x6d\x65\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 
+    msg += correct_checksum(msg)
+    msg += b'\x15'
+    return msg
+
+@pytest.fixture
+def device_rsp_msg2():  # 0x1110
+    msg  = b'\xa5\x0a\x00\x10\x11\x03\x03' +get_sn()  +b'\x02\x01'
+    msg += total()  
+    msg += hb()
+    msg += correct_checksum(msg)
+    msg += b'\x15'
+    return msg
+
 @pytest.fixture
 def invalid_start_byte(): # 0x4110
     msg  = b'\xa4\xd4\x00\x10\x41\x00\x01' +get_sn()  +b'\x02\xba\xd2\x00\x00'
@@ -901,6 +930,54 @@ def test_read_two_messages2(config_tsun_allow_all, inverter_ind_msg, inverter_in
     assert m._send_buffer==b''
     m.close()
 
+def test_read_two_messages3(config_tsun_allow_all, device_ind_msg2, device_rsp_msg2, inverter_ind_msg, inverter_rsp_msg):
+    # test device message received after the inverter masg
+    _ = config_tsun_allow_all
+    m = MemoryStream(inverter_ind_msg, (0,))
+    m.append_msg(device_ind_msg2)
+    assert 0 == m.sensor_list
+    m._init_new_client_conn()
+    m.read()         # read complete msg, and dispatch msg
+    assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 2
+    assert m.header_len==11
+    assert m.snr == 2070233889
+    assert m.unique_id == '2070233889'
+    assert m.msg_recvd[0]['control']==0x4210
+    assert m.msg_recvd[0]['seq']=='02:02'
+    assert m.msg_recvd[0]['data_len']==0x199
+    assert m.msg_recvd[1]['control']==0x4110
+    assert m.msg_recvd[1]['seq']=='03:03'
+    assert m.msg_recvd[1]['data_len']==0xd4
+    assert '02b0' == m.db.get_db_value(Register.SENSOR_LIST, None)
+    assert 0x02b0 == m.sensor_list
+    assert m._forward_buffer==inverter_ind_msg+device_ind_msg2
+    assert m._send_buffer==inverter_rsp_msg+device_rsp_msg2
+
+    m._send_buffer = bytearray(0) # clear send buffer for next test    
+    m._init_new_client_conn()
+    assert m._send_buffer==b''
+    m.close()
+
+def test_unkown_frame_code(config_tsun_inv1, inverter_ind_msg_81, inverter_rsp_msg_81):
+    _ = config_tsun_inv1
+    m = MemoryStream(inverter_ind_msg_81, (0,))
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.header_len==11
+    assert m.snr == 2070233889
+    assert m.unique_id == '2070233889'
+    assert m.control == 0x4210
+    assert str(m.seq) == '03:03'
+    assert m.data_len == 0x199
+    assert m._recv_buffer==b''
+    assert m._send_buffer==inverter_rsp_msg_81
+    assert m._forward_buffer==inverter_ind_msg_81
+    assert m.db.stat['proxy']['Invalid_Msg_Format'] == 0
+    m.close()
+
 def test_unkown_message(config_tsun_inv1, unknown_msg):
     _ = config_tsun_inv1
     m = MemoryStream(unknown_msg, (0,))

app/tests/test_talent.py

@@ -41,6 +41,7 @@ class MemoryStream(Talent):
         self.send_msg_ofs = 0
         self.test_exception_async_write = False
         self.msg_recvd = []
+        self.remote_stream = None
 
     def append_msg(self, msg):
         self.__msg += msg
@@ -138,6 +139,26 @@ def msg_time_rsp_inv(): # Get Time Resonse message
 def msg_time_invalid(): # Get Time Request message
     return b'\x00\x00\x00\x13\x10R170000000000001\x94\x22'           
 
+@pytest.fixture
+def msg_act_time(): # Act Time Indication message
+    return b'\x00\x00\x00\x1c\x10R170000000000001\x91\x99\x01\x00\x00\x01\x89\xc6\x53\x4d\x80'           
+
+@pytest.fixture
+def msg_act_time_ofs(): # Act Time Indication message withoffset 3600
+    return b'\x00\x00\x00\x1c\x10R170000000000001\x91\x99\x01\x00\x00\x01\x89\xc6\x53\x5b\x90'           
+
+@pytest.fixture
+def msg_act_time_ack(): # Act Time Response message
+    return b'\x00\x00\x00\x14\x10R170000000000001\x99\x99\x02'
+
+@pytest.fixture
+def msg_act_time_cmd(): # Act Time Response message
+    return b'\x00\x00\x00\x14\x10R170000000000001\x70\x99\x02'
+
+@pytest.fixture
+def msg_act_time_inv(): # Act Time Indication message withoffset 3600
+    return b'\x00\x00\x00\x1b\x10R170000000000001\x91\x99\x00\x00\x01\x89\xc6\x53\x5b\x90'           
+
 @pytest.fixture
 def msg_controller_ind(): # Data indication from the controller
     msg  =  b'\x00\x00\x01\x2f\x10R170000000000001\x91\x71\x0e\x10\x00\x00\x10R170000000000001'
@@ -442,6 +463,26 @@ def msg_modbus_rsp21():
     msg += b'\x00\x00\x00\x00\x00\x00\x00\xe6\xef'
     return msg
 
+@pytest.fixture
+def msg_modbus_cmd_new():
+    msg  = b'\x00\x00\x00\x20\x10R170000000000001'
+    msg += b'\x70\x77\x00\x01\xa3\x28\x08\x01\x03\x30\x00'
+    msg += b'\x00\x30\x4a\xde'
+    return msg
+
+@pytest.fixture
+def msg_modbus_rsp20_new():
+    msg  = b'\x00\x00\x00\x7e\x10R170000000000001'
+    msg += b'\x91\x87\x00\x01\xa3\x28\x00\x65\x01\x03\x60'
+    msg += b'\x00\x01\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\x51\x09\x09\x17\x00\x17\x13\x88\x00\x40\x00\x00\x02\x58\x02\x23'
+    msg += b'\x00\x07\x00\x00\x00\x00\x01\x4f\x00\xab\x02\x40\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\xc0\x93\x00\x00'
+    msg += b'\x00\x00\x33\xad\x00\x09\x00\x00\x98\x1c\x00\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    msg += b'\xa7\xab'
+    return msg
+
 @pytest.fixture
 def broken_recv_buf(): # There are two message in the buffer, but the second has overwritten the first partly
     msg  = b'\x00\x00\x05\x02\x10R170000000000001\x91\x04\x01\x90\x00\x01\x10R170000000000001'
@@ -892,6 +933,7 @@ def test_msg_contact_invalid(config_tsun_inv1, msg_contact_invalid):
 def test_msg_get_time(config_tsun_inv1, msg_get_time):
     _ = config_tsun_inv1
     m = MemoryStream(msg_get_time, (0,))
+    m.state = State.up
     m.db.stat['proxy']['Unknown_Ctrl'] = 0
     m.read()         # read complete msg, and dispatch msg
     assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
@@ -903,6 +945,7 @@ def test_msg_get_time(config_tsun_inv1, msg_get_time):
     assert m.header_len==23
     assert m.ts_offset==0
     assert m.data_len==0
+    assert m.state==State.pend
     assert m._forward_buffer==msg_get_time
     assert m._send_buffer==b'\x00\x00\x00\x1b\x10R170000000000001\x91"\x00\x00\x01\x89\xc6,_\x00'
     assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
@@ -911,6 +954,7 @@ def test_msg_get_time(config_tsun_inv1, msg_get_time):
 def test_msg_get_time_autark(config_no_tsun_inv1, msg_get_time):
     _ = config_no_tsun_inv1
     m = MemoryStream(msg_get_time, (0,))
+    m.state = State.received
     m.db.stat['proxy']['Unknown_Ctrl'] = 0
     m.read()         # read complete msg, and dispatch msg
     assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
@@ -922,14 +966,19 @@ def test_msg_get_time_autark(config_no_tsun_inv1, msg_get_time):
     assert m.header_len==23
     assert m.ts_offset==0
     assert m.data_len==0
+    assert m.state==State.received
     assert m._forward_buffer==b''
     assert m._send_buffer==bytearray(b'\x00\x00\x00\x1b\x10R170000000000001\x91"\x00\x00\x01\x89\xc6,_\x00')
     assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
     m.close()
 
 def test_msg_time_resp(config_tsun_inv1, msg_time_rsp):
+    # test if ts_offset will be set on client and server side
     _ = config_tsun_inv1
     m = MemoryStream(msg_time_rsp, (0,), False)
+    s = MemoryStream(b'', (0,), True)
+    assert s.ts_offset==0
+    m.remote_stream = s
     m.db.stat['proxy']['Unknown_Ctrl'] = 0
     m.read()         # read complete msg, and dispatch msg
     assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
@@ -940,10 +989,13 @@ def test_msg_time_resp(config_tsun_inv1, msg_time_rsp):
     assert m.msg_id==34
     assert m.header_len==23
     assert m.ts_offset==3600000
+    assert s.ts_offset==3600000
     assert m.data_len==8
     assert m._forward_buffer==b''
     assert m._send_buffer==b''
     assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.remote_stream = None
+    s.close()
     m.close()
 
 def test_msg_time_resp_autark(config_no_tsun_inv1, msg_time_rsp):
@@ -1022,6 +1074,169 @@ def test_msg_time_invalid_autark(config_no_tsun_inv1, msg_time_invalid):
     assert m.db.stat['proxy']['Unknown_Ctrl'] == 1
     m.close()
 
+def test_msg_act_time(config_no_modbus_poll, msg_act_time, msg_act_time_ack):
+    _ = config_no_modbus_poll
+    m = MemoryStream(msg_act_time, (0,))
+    m.ts_offset=0
+    m.mb_timeout = 124
+    m.db.set_db_def_value(Register.POLLING_INTERVAL, 125)
+    m.state = State.received
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.ts_offset==0
+    assert m.header_len==23
+    assert m.data_len==9
+    assert m.state == State.up
+    assert m._forward_buffer==msg_act_time
+    assert m._send_buffer==msg_act_time_ack
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    assert 125 == m.db.get_db_value(Register.POLLING_INTERVAL, 0)
+    m.close()
+
+def test_msg_act_time2(config_tsun_inv1, msg_act_time, msg_act_time_ack):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time, (0,))
+    m.ts_offset=0
+    m.modbus_polling = True
+    m.mb_timeout = 123
+    m.db.set_db_def_value(Register.POLLING_INTERVAL, 125)
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.ts_offset==0
+    assert m.header_len==23
+    assert m.data_len==9
+    assert m._forward_buffer==msg_act_time
+    assert m._send_buffer==msg_act_time_ack
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    assert 123 == m.db.get_db_value(Register.POLLING_INTERVAL, 0)
+    m.close()
+
+def test_msg_act_time_ofs(config_tsun_inv1, msg_act_time, msg_act_time_ofs, msg_act_time_ack):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time, (0,))
+    m.ts_offset=3600
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.ts_offset==3600
+    assert m.header_len==23
+    assert m.data_len==9
+    assert m._forward_buffer==msg_act_time_ofs
+    assert m._send_buffer==msg_act_time_ack
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.close()
+
+def test_msg_act_time_ofs2(config_tsun_inv1, msg_act_time, msg_act_time_ofs, msg_act_time_ack):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time_ofs, (0,))
+    m.ts_offset=-3600
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.ts_offset==-3600
+    assert m.header_len==23
+    assert m.data_len==9
+    assert m._forward_buffer==msg_act_time
+    assert m._send_buffer==msg_act_time_ack
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.close()
+
+def test_msg_act_time_autark(config_no_tsun_inv1, msg_act_time, msg_act_time_ack):
+    _ = config_no_tsun_inv1
+    m = MemoryStream(msg_act_time, (0,))
+    m.ts_offset=0
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.ts_offset==0
+    assert m.header_len==23
+    assert m.data_len==9
+    assert m._forward_buffer==b''
+    assert m._send_buffer==msg_act_time_ack
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.close()
+
+def test_msg_act_time_ack(config_tsun_inv1, msg_act_time_ack):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time_ack, (0,))
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==153
+    assert m.msg_id==153
+    assert m.header_len==23
+    assert m.data_len==1
+    assert m._forward_buffer==b''
+    assert m._send_buffer==b''
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.close()
+
+def test_msg_act_time_cmd(config_tsun_inv1, msg_act_time_cmd):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time_cmd, (0,))
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==112
+    assert m.msg_id==153
+    assert m.header_len==23
+    assert m.data_len==1
+    assert m._forward_buffer==msg_act_time_cmd
+    assert m._send_buffer==b''
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 1
+    m.close()
+
+def test_msg_act_time_inv(config_tsun_inv1, msg_act_time_inv):
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_act_time_inv, (0,))
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==153
+    assert m.header_len==23
+    assert m.data_len==8
+    assert m._forward_buffer==msg_act_time_inv
+    assert m._send_buffer==b''
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    m.close()
+
 def test_msg_cntrl_ind(config_tsun_inv1, msg_controller_ind, msg_controller_ind_ts_offs, msg_controller_ack):
     _ = config_tsun_inv1
     m = MemoryStream(msg_controller_ind, (0,))
@@ -1583,7 +1798,7 @@ def test_msg_modbus_rsp2(config_tsun_inv1, msg_modbus_rsp20):
     assert m.msg_count == 2
     assert m._forward_buffer==msg_modbus_rsp20
     assert m._send_buffer==b''
-    assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Timestamp': m._utc(), 'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'Timestamp': m._utc(), 'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
+    assert m.db.db == {'collector': {'Serial_Number': 'R170000000000001'}, 'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Timestamp': m._utc(), 'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'Timestamp': m._utc(), 'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
     assert m.db.get_db_value(Register.VERSION) == 'V5.1.09'
     assert m.db.get_db_value(Register.TS_GRID) == m._utc()
     assert m.new_data['inverter'] == True
@@ -1613,13 +1828,64 @@ def test_msg_modbus_rsp3(config_tsun_inv1, msg_modbus_rsp21):
     assert m.msg_count == 2
     assert m._forward_buffer==msg_modbus_rsp21
     assert m._send_buffer==b''
-    assert m.db.db == {'inverter': {'Version': 'V5.1.0E', 'Rated_Power': 300}, 'grid': {'Timestamp': m._utc(), 'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'Timestamp': m._utc(), 'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
+    assert m.db.db == {'collector': {'Serial_Number': 'R170000000000001'}, 'inverter': {'Version': 'V5.1.0E', 'Rated_Power': 300}, 'grid': {'Timestamp': m._utc(), 'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'Timestamp': m._utc(), 'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
     assert m.db.get_db_value(Register.VERSION) == 'V5.1.0E'
     assert m.db.get_db_value(Register.TS_GRID) == m._utc()
     assert m.new_data['inverter'] == True
 
     m.close()
 
+def test_msg_modbus_rsp4(config_tsun_inv1, msg_modbus_rsp21):
+    '''Modbus response with a valid Modbus but no new values request must be forwarded'''
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_modbus_rsp21)
+ 
+    m.mb.rsp_handler = m.msg_forward
+    m.mb.last_addr = 1
+    m.mb.last_fcode = 3
+    m.mb.last_len = 20
+    m.mb.last_reg = 0x3008
+    m.mb.req_pend = True
+    m.mb.err = 0
+    db_values = {'collector': {'Serial_Number': 'R170000000000001'}, 'inverter': {'Version': 'V5.1.0E', 'Rated_Power': 300}, 'grid': {'Timestamp': m._utc(), 'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'Timestamp': m._utc(), 'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
+    m.db.db  = db_values
+    m.new_data['inverter'] = False
+
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.mb.err == 0
+    assert m.msg_count == 1
+    assert m._forward_buffer==msg_modbus_rsp21
+    assert m.modbus_elms == 19
+    assert m._send_buffer==b''
+    assert m.db.db == db_values
+    assert m.db.get_db_value(Register.VERSION) == 'V5.1.0E'
+    assert m.db.get_db_value(Register.TS_GRID) == m._utc()
+    assert m.new_data['inverter'] == False
+
+    m.close()
+
+def test_msg_modbus_rsp_new(config_tsun_inv1, msg_modbus_rsp20_new):
+    '''Modbus response in new format with a valid Modbus request must be forwarded'''
+    _ = config_tsun_inv1
+    m = MemoryStream(msg_modbus_rsp20_new)
+    m.db.stat['proxy']['Unknown_Ctrl'] = 0
+    m.db.stat['proxy']['Modbus_Command'] = 0
+    m.read()         # read complete msg, and dispatch msg
+    assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
+    assert m.msg_count == 1
+    assert m.id_str == b"R170000000000001" 
+    assert m.unique_id == 'R170000000000001'
+    assert int(m.ctrl)==145
+    assert m.msg_id==135
+    assert m.header_len==23
+    assert m.data_len==107
+    assert m._forward_buffer==b''
+    assert m._send_buffer==b''
+    assert m.db.stat['proxy']['Unknown_Ctrl'] == 0
+    assert m.db.stat['proxy']['Modbus_Command'] == 0
+    m.close()
+
 def test_msg_modbus_invalid(config_tsun_inv1, msg_modbus_inv):
     _ = config_tsun_inv1
     m = MemoryStream(msg_modbus_inv, (0,), False)