S allius/issue117 (#122)

* add shutdown flag

* add more register definitions

* add start commando for client side connections

* add first support for port 8899

* fix shutdown

* add client_mode configuration

* read client_mode config to setup inverter connections

* add client_mode connections over port 8899

* add preview build

* add documentation for client_mode

* catch os error and log thme with DEBUG level

* update changelog

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+- cleanup shutdown
+- add preview build
+- MODBUS: the last digit of the inverter version is a hexadecimal number [#119](https://github.com/s-allius/tsun-gen3-proxy/issues/119)
+- GEN3PLUS: add client_mode connection on port 8899 [#117](https://github.com/s-allius/tsun-gen3-proxy/issues/117)
+
 ## [0.9.0] - 2024-07-01
 
 - fix exception in MODBUS timeout callback

README.md

@@ -165,6 +165,9 @@ pv2 = {type = 'RSM40-8-405M', manufacturer = 'Risen'}   # Optional, PV module de
 monitor_sn = 2000000000       # The "Monitoring SN:" can be found on a sticker enclosed with the inverter
 node_id = 'inv_3'             # MQTT replacement for inverters serial number  
 suggested_area = 'garage'     # suggested installation place for home-assistant
+# if your inverter supports SSL connections you must use the client_mode. Pls, uncomment
+# the next line and configure the fixed IP of your inverter
+#client_mode = {host = '192.168.0.1', port = 8899}       
 pv1 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 pv2 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 pv3 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
@@ -188,8 +191,19 @@ The standard web interface of the inverter can be accessed at `http://<ip-adress
 
 For our purpose, the hidden URL `http://<ip-adress>/config_hide.html` should be called. There you can see and modify the parameters for accessing the cloud. Here we enter the IP address of our proxy and the IP port `10000` for the `Server A Setting` and for `Optional Server Setting`. The second entry is used as a backup in the event of connection problems.
 
+❗If the IP port is set to 10443 in the inverter configuration, you probably have a firmware with SSL support. In this case, you must use the client-mode configuration.
+
 If access to the web interface does not work, it can also be redirected via DNS redirection, as is necessary for the GEN3 inverters.
 
+## Client Mode (GEN3PLUS only)
+
+Newer GEN3PLUS inverters support SSL encrypted connections over port 10443 to the TSUN cloud. In this case you can't loop the proxy into this connection, since the certicate verification of the inverter don't allow this. You can configure the proxy in client-mode to establish an unencrypted connection to the inverter. For this porpuse the inverter listen on port `8899`.
+
+There are some requirements to be met:
+- the inverter should have a fixed IP
+- the proxy must be able to reach the inverter. You must configure a corresponding route in your router if the inverter and the proxy are in different IP networks
+- add a 'client_mode' line to your config.toml file, to specify the inverter's ip address
+
 ## DNS Settings
 
 ### Loop the proxy into the connection

app/build.sh

@@ -21,11 +21,11 @@ IMAGE=tsun-gen3-proxy
 if [[ $1 == debug ]] || [[ $1 == dev ]] ;then
 IMAGE=docker.io/sallius/${IMAGE}
 VERSION=${VERSION}-$1
-elif [[ $1 == rc ]] || [[ $1 == rel ]];then
+elif [[ $1 == rc ]] || [[ $1 == rel ]] || [[ $1 == preview ]] ;then
 IMAGE=ghcr.io/s-allius/${IMAGE}
 else
 echo argument missing!
-echo try: $0 '[debug|dev|rc|rel]'
+echo try: $0 '[debug|dev|preview|rc|rel]'
 exit 1
 fi
 
@@ -35,6 +35,13 @@ docker build --build-arg "VERSION=${VERSION}" --build-arg environment=dev --buil
 elif [[ $1 == dev ]];then
 docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:dev app
 
+elif [[ $1 == preview ]];then
+docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:preview -t ${IMAGE}:${VERSION} app
+echo 'login to ghcr.io'    
+echo $GHCR_TOKEN | docker login ghcr.io -u s-allius --password-stdin
+docker push -q ghcr.io/s-allius/tsun-gen3-proxy:preview
+docker push -q ghcr.io/s-allius/tsun-gen3-proxy:${VERSION}
+
 elif [[ $1 == rc ]];then
 docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:rc -t ${IMAGE}:${VERSION} app
 echo 'login to ghcr.io'    

app/config/default_config.toml

@@ -44,6 +44,11 @@ inverters.allow_all = true   # allow inverters, even if we have no inverter mapp
 monitor_sn = 2000000000       # The "Monitoring SN:" can be found on a sticker enclosed with the inverter
 #node_id = ''                 # Optional, MQTT replacement for inverters serial number  
 #suggested_area = ''          # Optional, suggested installation place for home-assistant
+
+# if your inverter supports SSL connections you must use the client_mode. Pls, uncomment
+# the next line and configure the fixed IP of your inverter
+#client_mode = {host = '192.168.0.1', port = 8899}  
+
 #pv1 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 #pv2 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 #pv3 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr

app/src/config.py

@@ -53,7 +53,11 @@ class Config():
                                                      Use(lambda s: s + '/'
                                                          if len(s) > 0 and
                                                          s[-1] != '/' else s)),
-
+                Optional('client_mode'): {
+                    'host': Use(str),
+                    Optional('port', default=8899):
+                        And(Use(int), lambda n: 1024 <= n <= 65535)
+                    },
                 Optional('suggested_area',  default=""): Use(str),
                 Optional('pv1'): {
                     Optional('type'): Use(str),

app/src/gen3plus/solarman_v5.py

@@ -66,6 +66,8 @@ class SolarmanV5(Message):
         self.db = InfosG3P()
         self.time_ofs = 0
         self.forward_at_cmd_resp = False
+        self.no_forwarding = False
+        '''not allowed to connect to TSUN cloud by connection type'''
         self.switch = {
 
             0x4210: self.msg_data_ind,   # real time data
@@ -143,6 +145,19 @@ class SolarmanV5(Message):
         self.mb_timer.close()
         super().close()
 
+    async def send_start_cmd(self, snr: int):
+        self.no_forwarding = True
+        self.snr = snr
+        self.__set_serial_no(snr)
+
+        self.__send_ack_rsp(0x1710, ftype=0)
+        await self.async_write('Send Start Command:')
+        self._send_buffer = bytearray(0)
+
+        self.state = State.up
+        self._send_modbus_cmd(Modbus.READ_REGS, 0x2000, 64, logging.INFO)
+        self.mb_timer.start(self.MB_START_TIMEOUT)
+
     def __set_serial_no(self, snr: int):
         serial_no = str(snr)
         if self.unique_id == serial_no:
@@ -198,6 +213,8 @@ class SolarmanV5(Message):
         return 0  # wait 0s before sending a response
 
     def forward(self, buffer, buflen) -> None:
+        if self.no_forwarding:
+            return
         tsun = Config.get('solarman')
         if tsun['enabled']:
             self._forward_buffer = buffer[:buflen]
@@ -380,11 +397,11 @@ class SolarmanV5(Message):
     def mb_timout_cb(self, exp_cnt):
         self.mb_timer.start(self.MB_REGULAR_TIMEOUT)
 
-        self._send_modbus_cmd(Modbus.READ_REGS, 0x3008, 21, logging.DEBUG)
+        self._send_modbus_cmd(Modbus.READ_REGS, 0x3000, 48, logging.DEBUG)
 
         if 0 == (exp_cnt % 30):
             # logging.info("Regular Modbus Status request")
-            self._send_modbus_cmd(Modbus.READ_REGS, 0x2007, 2, logging.DEBUG)
+            self._send_modbus_cmd(Modbus.READ_REGS, 0x2000, 64, logging.DEBUG)
 
     def at_cmd_forbidden(self, cmd: str, connection: str) -> bool:
         return not cmd.startswith(tuple(self.at_acl[connection]['allow'])) or \

app/src/messages.py

@@ -91,6 +91,7 @@ class Message(metaclass=IterRegistry):
         self._forward_buffer = bytearray(0)
         self.new_data = {}
         self.state = State.init
+        self.shutdown_started = False
 
     '''
     Empty methods, that have to be implemented in any child class which

app/src/modbus.py

@@ -41,8 +41,10 @@ class Modbus():
     __crc_tab = []
     map = {
         0x2007: {'reg': Register.MAX_DESIGNED_POWER,   'fmt': '!H', 'ratio':  1},  # noqa: E501
-        # 0x????: {'reg': Register.INVERTER_STATUS,      'fmt': '!H'},                 # noqa: E501
+        0x203e: {'reg': Register.NO_INPUTS,   'fmt': '!H', 'ratio':  1/256},  # noqa: E501
-        0x3008: {'reg': Register.VERSION,              'fmt': '!H', 'eval': "f'V{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"},  # noqa: E501
+
+        0x3000: {'reg': Register.INVERTER_STATUS,      'fmt': '!H'},                 # noqa: E501
+        0x3008: {'reg': Register.VERSION,              'fmt': '!H', 'eval': "f'V{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf:1X}'"},  # noqa: E501
         0x3009: {'reg': Register.GRID_VOLTAGE,         'fmt': '!H', 'ratio':  0.1},  # noqa: E501
         0x300a: {'reg': Register.GRID_CURRENT,         'fmt': '!H', 'ratio': 0.01},  # noqa: E501
         0x300b: {'reg': Register.GRID_FREQUENCY,       'fmt': '!H', 'ratio': 0.01},  # noqa: E501

app/src/modbus_tcp.py

@@ -0,0 +1,73 @@
+import logging
+import traceback
+import asyncio
+from config import Config
+
+# import gc
+from gen3plus.inverter_g3p import InverterG3P
+
+logger = logging.getLogger('conn')
+
+
+class ModbusConn():
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+        self.addr = (host, port)
+        self.stream = None
+
+    async def __aenter__(self) -> 'InverterG3P':
+        '''Establish a client connection to the TSUN cloud'''
+        connection = asyncio.open_connection(self.host, self.port)
+        reader, writer = await connection
+        self.stream = InverterG3P(reader, writer, self.addr)
+        logging.info(f'[{self.stream.node_id}:{self.stream.conn_no}] '
+                     f'Connected to {self.addr}')
+        self.stream.inc_counter('Inverter_Cnt')
+        return self.stream
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.stream.dec_counter('Inverter_Cnt')
+
+
+class ModbusTcp():
+
+    def __init__(self, loop) -> None:
+        inverters = Config.get('inverters')
+        # logging.info(f'Inverters: {inverters}')
+
+        for inv in inverters.values():
+            if (type(inv) is dict
+               and 'monitor_sn' in inv
+               and 'client_mode' in inv):
+                client = inv['client_mode']
+                # logging.info(f"SerialNo:{inv['monitor_sn']} host:{client['host']} port:{client['port']}")  # noqa: E501
+                loop.create_task(self.modbus_loop(client['host'],
+                                                  client['port'],
+                                                  inv['monitor_sn']))
+
+    async def modbus_loop(self, host, port, snr: int) -> None:
+        '''Loop for receiving messages from the TSUN cloud (client-side)'''
+        while True:
+            try:
+                async with ModbusConn(host, port) as stream:
+                    await stream.send_start_cmd(snr)
+                    await stream.loop()
+                    logger.info(f'[{stream.node_id}:{stream.conn_no}] '
+                                f'Connection closed - Shutdown: '
+                                f'{stream.shutdown_started}')
+                    if stream.shutdown_started:
+                        return
+
+            except (ConnectionRefusedError, TimeoutError) as error:
+                logging.debug(f'Inv-conn:{error}')
+
+            except OSError as error:
+                logging.info(f'os-error: {error}')
+
+            except Exception:
+                logging.error(
+                    f"ModbusTcpCreate: Exception for {(host,port)}:\n"
+                    f"{traceback.format_exc()}")
+
+            await asyncio.sleep(10)

app/src/server.py

@@ -11,6 +11,7 @@ from gen3.inverter_g3 import InverterG3
 from gen3plus.inverter_g3p import InverterG3P
 from scheduler import Schedule
 from config import Config
+from modbus_tcp import ModbusTcp
 
 routes = web.RouteTableDef()
 proxy_is_up = False
@@ -94,6 +95,7 @@ async def handle_shutdown(web_task):
     # first, disc all open TCP connections gracefully
     #
     for stream in Message:
+        stream.shutdown_started = True
         try:
             await asyncio.wait_for(stream.disc(), 2)
         except Exception:
@@ -115,6 +117,13 @@ async def handle_shutdown(web_task):
     web_task.cancel()
     await web_task
 
+    #
+    # now cancel all remaining (pending) tasks
+    #
+    pending = asyncio.all_tasks()
+    for task in pending:
+        task.cancel()
+
     #
     # at last, start a coro for stopping the loop
     #
@@ -164,6 +173,7 @@ if __name__ == "__main__":
         logging.info(f'ConfigErr: {ConfigErr}')
     Inverter.class_init()
     Schedule.start()
+    mb_tcp = ModbusTcp(loop)
 
     #
     # Create tasks for our listening servers. These must be tasks! If we call

app/tests/test_modbus.py

@@ -32,7 +32,12 @@ def test_modbus_crc():
     assert mb._Modbus__check_crc(b'\x01\x06\x20\x08\x00\x00\x03\xc8')
 
     assert 0x5c75 == mb._Modbus__calc_crc(b'\x01\x03\x08\x01\x2c\x00\x2c\x02\x2c\x2c\x46')
-    
+    msg = b'\x01\x03\x28\x51'
+    msg += b'\x0e\x08\xd3\x00\x29\x13\x87\x00\x3e\x00\x00\x01\x2c\x03\xb4\x00'
+    msg += b'\x08\x00\x00\x00\x00\x01\x59\x01\x21\x03\xe6\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\xe6\xef'
+    assert 0 == mb._Modbus__calc_crc(msg)
+
 def test_build_modbus_pdu():
     '''Check building and sending a MODBUS RTU'''
     mb = ModbusTestHelper()
@@ -173,7 +178,7 @@ def test_parse_resp():
     assert mb.req_pend
 
     call = 0
-    exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30]
+    exp_result = ['V0.0.2C', 4.4, 0.7, 0.7, 30]
     for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'):
         if key == 'grid':
             assert update == True
@@ -222,7 +227,7 @@ def test_queue2():
     assert mb.send_calls == 1
     assert mb.pdu == b'\x01\x030\x07\x00\x06{\t'
     call = 0
-    exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30]
+    exp_result = ['V0.0.2C', 4.4, 0.7, 0.7, 30]
     for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'):
         if key == 'grid':
             assert update == True
@@ -272,7 +277,7 @@ def test_queue3():
     assert mb.recv_responses == 0
 
     call = 0
-    exp_result = ['V0.0.212', 4.4, 0.7, 0.7, 30]
+    exp_result = ['V0.0.2C', 4.4, 0.7, 0.7, 30]
     for key, update, val in mb.recv_resp(mb.db, b'\x01\x03\x0c\x01\x2c\x00\x2c\x00\x2c\x00\x46\x00\x46\x00\x46\x32\xc8', 'test'):
         if key == 'grid':
             assert update == True

app/tests/test_talent.py

@@ -339,6 +339,15 @@ def MsgModbusResp20():
     msg += b'\x00\x00\x00\x00\x00\x00\x00\xdb\x6b'
     return msg
 
+@pytest.fixture
+def MsgModbusResp21():
+    msg  = b'\x00\x00\x00\x45\x10R170000000000001'
+    msg += b'\x91\x77\x17\x18\x19\x1a\x2d\x01\x03\x28\x51'
+    msg += b'\x0e\x08\xd3\x00\x29\x13\x87\x00\x3e\x00\x00\x01\x2c\x03\xb4\x00'
+    msg += b'\x08\x00\x00\x00\x00\x01\x59\x01\x21\x03\xe6\x00\x00\x00\x00\x00'
+    msg += b'\x00\x00\x00\x00\x00\x00\x00\xe6\xef'
+    return msg
+
 def test_read_message(MsgContactInfo):
     m = MemoryStream(MsgContactInfo, (0,))
     m.read()         # read complete msg, and dispatch msg
@@ -1226,11 +1235,11 @@ def test_msg_modbus_rsp2(ConfigTsunInv1, MsgModbusResp20):
 
     m.close()
 
-def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20):
+def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp21):
     '''Modbus response with a valid Modbus request must be forwarded'''
     ConfigTsunInv1
-    m = MemoryStream(MsgModbusResp20)
+    m = MemoryStream(MsgModbusResp21)
-    m.append_msg(MsgModbusResp20)
+    m.append_msg(MsgModbusResp21)
 
     m.mb.rsp_handler = m.msg_forward
     m.mb.last_addr = 1
@@ -1247,10 +1256,10 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20):
     assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
     assert m.mb.err == 0
     assert m.msg_count == 1
-    assert m._forward_buffer==MsgModbusResp20
+    assert m._forward_buffer==MsgModbusResp21
     assert m._send_buffer==b''
-    assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
+    assert m.db.db == {'inverter': {'Version': 'V5.1.0E', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
-    assert m.db.get_db_value(Register.VERSION) == 'V5.1.09'
+    assert m.db.get_db_value(Register.VERSION) == 'V5.1.0E'
     assert m.new_data['inverter'] == True
     m.new_data['inverter'] = False
     assert m.mb.req_pend == False
@@ -1259,10 +1268,10 @@ def test_msg_modbus_rsp3(ConfigTsunInv1, MsgModbusResp20):
     assert not m.header_valid  # must be invalid, since msg was handled and buffer flushed
     assert m.mb.err == 5
     assert m.msg_count == 2
-    assert m._forward_buffer==MsgModbusResp20
+    assert m._forward_buffer==MsgModbusResp21
     assert m._send_buffer==b''
-    assert m.db.db == {'inverter': {'Version': 'V5.1.09', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
+    assert m.db.db == {'inverter': {'Version': 'V5.1.0E', 'Rated_Power': 300}, 'grid': {'Voltage': 225.9, 'Current': 0.41, 'Frequency': 49.99, 'Output_Power': 94.8}, 'env': {'Inverter_Temp': 22}, 'input': {'pv1': {'Voltage': 0.8, 'Current': 0.0, 'Power': 0.0}, 'pv2': {'Voltage': 34.5, 'Current': 2.89, 'Power': 99.8}, 'pv3': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}, 'pv4': {'Voltage': 0.0, 'Current': 0.0, 'Power': 0.0}}}
-    assert m.db.get_db_value(Register.VERSION) == 'V5.1.09'
+    assert m.db.get_db_value(Register.VERSION) == 'V5.1.0E'
     assert m.new_data['inverter'] == False
 
     m.close()