Merge branch 'main' of https://github.com/s-allius/tsun-gen3-proxy into ssl-connection

- return error string or None
- release Schema structure after building thr config

CHANGELOG.md

@@ -7,8 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
-- GEN3PLUS: add client_mode connection on port 8899 [#117](https://github.com/s-allius/tsun-gen3-proxy/issues/117)
-
 ## [0.9.0] - 2024-07-01
 
 - fix exception in MODBUS timeout callback

app/Dockerfile

@@ -45,7 +45,7 @@ ENV HOME=/home/$SERVICE_NAME
 # set the working directory in the container
 WORKDIR /home/$SERVICE_NAME
 
-VOLUME ["/home/$SERVICE_NAME/log", "/home/$SERVICE_NAME/config"]
+VOLUME ["/home/$SERVICE_NAME/log", "/home/$SERVICE_NAME/config", "/home/$SERVICE_NAME/cert"]
 
 # install the requirements from the wheels packages from the builder stage 
 # and unistall python packages and alpine package manger to reduce attack surface
@@ -64,7 +64,7 @@ COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh
 COPY config .
 COPY src .
 RUN date > /build-date.txt
-EXPOSE 5005 8127 10000
+EXPOSE 5005 8127 10000 10443
 
 # command to run on container start
 ENTRYPOINT ["/root/entrypoint.sh"]

app/build.sh

@@ -21,11 +21,11 @@ IMAGE=tsun-gen3-proxy
 if [[ $1 == debug ]] || [[ $1 == dev ]] ;then
 IMAGE=docker.io/sallius/${IMAGE}
 VERSION=${VERSION}-$1
-elif [[ $1 == rc ]] || [[ $1 == rel ]] || [[ $1 == preview ]] ;then
+elif [[ $1 == rc ]] || [[ $1 == rel ]];then
 IMAGE=ghcr.io/s-allius/${IMAGE}
 else
 echo argument missing!
-echo try: $0 '[debug|dev|preview|rc|rel]'
+echo try: $0 '[debug|dev|rc|rel]'
 exit 1
 fi
 
@@ -35,13 +35,6 @@ docker build --build-arg "VERSION=${VERSION}" --build-arg environment=dev --buil
 elif [[ $1 == dev ]];then
 docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:dev app
 
-elif [[ $1 == preview ]];then
-docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:preview -t ${IMAGE}:${VERSION} app
-echo 'login to ghcr.io'    
-echo $GHCR_TOKEN | docker login ghcr.io -u s-allius --password-stdin
-docker push -q ghcr.io/s-allius/tsun-gen3-proxy:preview
-docker push -q ghcr.io/s-allius/tsun-gen3-proxy:${VERSION}
-
 elif [[ $1 == rc ]];then
 docker build --build-arg "VERSION=${VERSION}" --build-arg environment=production --label "org.opencontainers.image.created=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" --label "org.opencontainers.image.revision=${BRANCH}" -t ${IMAGE}:rc -t ${IMAGE}:${VERSION} app
 echo 'login to ghcr.io'    

app/config/default_config.toml

@@ -44,7 +44,6 @@ inverters.allow_all = true   # allow inverters, even if we have no inverter mapp
 monitor_sn = 2000000000       # The "Monitoring SN:" can be found on a sticker enclosed with the inverter
 #node_id = ''                 # Optional, MQTT replacement for inverters serial number  
 #suggested_area = ''          # Optional, suggested installation place for home-assistant
-#client_mode = {host = '192.168.0.1', port = 8899}
 #pv1 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 #pv2 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr
 #pv3 = {type = 'RSM40-8-410M', manufacturer = 'Risen'}   # Optional, PV module descr

app/src/config.py

@@ -53,11 +53,7 @@ class Config():
                                                      Use(lambda s: s + '/'
                                                          if len(s) > 0 and
                                                          s[-1] != '/' else s)),
-                Optional('client_mode'): {
-                    'host': Use(str),
-                    Optional('port', default=8899):
-                        And(Use(int), lambda n: 1024 <= n <= 65535)
-                    },
+
                 Optional('suggested_area',  default=""): Use(str),
                 Optional('pv1'): {
                     Optional('type'): Use(str),

app/src/gen3plus/solarman_v5.py

@@ -66,8 +66,6 @@ class SolarmanV5(Message):
         self.db = InfosG3P()
         self.time_ofs = 0
         self.forward_at_cmd_resp = False
-        self.no_forwarding = False
-        '''not allowed to connect to TSUN cloud by connection type'''
         self.switch = {
 
             0x4210: self.msg_data_ind,   # real time data
@@ -145,19 +143,6 @@ class SolarmanV5(Message):
         self.mb_timer.close()
         super().close()
 
-    async def send_start_cmd(self, snr: int):
-        self.no_forwarding = True
-        self.snr = snr
-        self.__set_serial_no(snr)
-
-        self.__send_ack_rsp(0x1710, ftype=0)
-        await self.async_write('Send Start Command:')
-        self._send_buffer = bytearray(0)
-
-        self.state = State.up
-        self._send_modbus_cmd(Modbus.READ_REGS, 0x2000, 64, logging.INFO)
-        self.mb_timer.start(self.MB_START_TIMEOUT)
-
     def __set_serial_no(self, snr: int):
         serial_no = str(snr)
         if self.unique_id == serial_no:
@@ -213,8 +198,6 @@ class SolarmanV5(Message):
         return 0  # wait 0s before sending a response
 
     def forward(self, buffer, buflen) -> None:
-        if self.no_forwarding:
-            return
         tsun = Config.get('solarman')
         if tsun['enabled']:
             self._forward_buffer = buffer[:buflen]
@@ -397,11 +380,11 @@ class SolarmanV5(Message):
     def mb_timout_cb(self, exp_cnt):
         self.mb_timer.start(self.MB_REGULAR_TIMEOUT)
 
-        self._send_modbus_cmd(Modbus.READ_REGS, 0x3000, 48, logging.DEBUG)
+        self._send_modbus_cmd(Modbus.READ_REGS, 0x3008, 21, logging.DEBUG)
 
         if 0 == (exp_cnt % 30):
             # logging.info("Regular Modbus Status request")
-            self._send_modbus_cmd(Modbus.READ_REGS, 0x2000, 64, logging.DEBUG)
+            self._send_modbus_cmd(Modbus.READ_REGS, 0x2007, 2, logging.DEBUG)
 
     def at_cmd_forbidden(self, cmd: str, connection: str) -> bool:
         return not cmd.startswith(tuple(self.at_acl[connection]['allow'])) or \

app/src/messages.py

@@ -91,7 +91,6 @@ class Message(metaclass=IterRegistry):
         self._forward_buffer = bytearray(0)
         self.new_data = {}
         self.state = State.init
-        self.shutdown_started = False
 
     '''
     Empty methods, that have to be implemented in any child class which

app/src/modbus.py

@@ -41,9 +41,7 @@ class Modbus():
     __crc_tab = []
     map = {
         0x2007: {'reg': Register.MAX_DESIGNED_POWER,   'fmt': '!H', 'ratio':  1},  # noqa: E501
-        0x203e: {'reg': Register.NO_INPUTS,   'fmt': '!H', 'ratio':  1/256},  # noqa: E501
-
-        0x3000: {'reg': Register.INVERTER_STATUS,      'fmt': '!H'},                 # noqa: E501
+        # 0x????: {'reg': Register.INVERTER_STATUS,      'fmt': '!H'},                 # noqa: E501
         0x3008: {'reg': Register.VERSION,              'fmt': '!H', 'eval': "f'V{(result>>12)}.{(result>>8)&0xf}.{(result>>4)&0xf}{result&0xf}'"},  # noqa: E501
         0x3009: {'reg': Register.GRID_VOLTAGE,         'fmt': '!H', 'ratio':  0.1},  # noqa: E501
         0x300a: {'reg': Register.GRID_CURRENT,         'fmt': '!H', 'ratio': 0.01},  # noqa: E501

app/src/modbus_tcp.py

@@ -1,70 +0,0 @@
-import logging
-import traceback
-import asyncio
-from config import Config
-
-# import gc
-from gen3plus.inverter_g3p import InverterG3P
-
-logger = logging.getLogger('conn')
-
-
-class ModbusConn():
-    def __init__(self, host, port):
-        self.host = host
-        self.port = port
-        self.addr = (host, port)
-        self.stream = None
-
-    async def __aenter__(self) -> 'InverterG3P':
-        '''Establish a client connection to the TSUN cloud'''
-        connection = asyncio.open_connection(self.host, self.port)
-        reader, writer = await connection
-        self.stream = InverterG3P(reader, writer, self.addr)
-        logging.info(f'[{self.stream.node_id}:{self.stream.conn_no}] '
-                     f'Connected to {self.addr}')
-        self.stream.inc_counter('Inverter_Cnt')
-        return self.stream
-
-    async def __aexit__(self, exc_type, exc, tb):
-        self.stream.dec_counter('Inverter_Cnt')
-
-
-class ModbusTcp():
-
-    def __init__(self, loop) -> None:
-        inverters = Config.get('inverters')
-        # logging.info(f'Inverters: {inverters}')
-
-        for inv in inverters.values():
-            if (type(inv) is dict
-               and 'monitor_sn' in inv
-               and 'client_mode' in inv):
-                client = inv['client_mode']
-                # logging.info(f"SerialNo:{inv['monitor_sn']} host:{client['host']} port:{client['port']}")  # noqa: E501
-                loop.create_task(self.modbus_loop(client['host'],
-                                                  client['port'],
-                                                  inv['monitor_sn']))
-
-    async def modbus_loop(self, host, port, snr: int) -> None:
-        '''Loop for receiving messages from the TSUN cloud (client-side)'''
-        while True:
-            try:
-                async with ModbusConn(host, port) as stream:
-                    await stream.send_start_cmd(snr)
-                    await stream.loop()
-                    logger.info(f'[{stream.node_id}:{stream.conn_no}] '
-                                f'Connection closed - Shutdown: '
-                                f'{stream.shutdown_started}')
-                    if stream.shutdown_started:
-                        return
-
-            except (ConnectionRefusedError, TimeoutError) as error:
-                logging.info(f'{error}')
-
-            except Exception:
-                logging.error(
-                    f"ModbusTcpCreate: Exception for {(host,port)}:\n"
-                    f"{traceback.format_exc()}")
-
-            await asyncio.sleep(10)

app/src/server.py

@@ -1,5 +1,6 @@
 import logging
 import asyncio
+import ssl
 import signal
 import os
 from asyncio import StreamReader, StreamWriter
@@ -11,7 +12,6 @@ from gen3.inverter_g3 import InverterG3
 from gen3plus.inverter_g3p import InverterG3P
 from scheduler import Schedule
 from config import Config
-from modbus_tcp import ModbusTcp
 
 routes = web.RouteTableDef()
 proxy_is_up = False
@@ -84,18 +84,22 @@ async def handle_client_v2(reader: StreamReader, writer: StreamWriter):
     await InverterG3P(reader, writer, addr).server_loop(addr)
 
 
-async def handle_shutdown(web_task):
+async def handle_client_v3(reader: StreamReader, writer: StreamWriter):
+    '''Handles a new incoming connection and starts an async loop'''
+    logging.info('Accept on port 10443')
+    addr = writer.get_extra_info('peername')
+    await InverterG3P(reader, writer, addr).server_loop(addr)
+
+
+async def handle_shutdown(loop, runner):
     '''Close all TCP connections and stop the event loop'''
 
     logging.info('Shutdown due to SIGTERM')
-    global proxy_is_up
-    proxy_is_up = False
 
     #
     # first, disc all open TCP connections gracefully
     #
     for stream in Message:
-        stream.shutdown_started = True
         try:
             await asyncio.wait_for(stream.disc(), 2)
         except Exception:
@@ -118,14 +122,7 @@ async def handle_shutdown(web_task):
     await web_task
 
     #
-    # now cancel all remaining (pending) tasks
-    #
-    pending = asyncio.all_tasks()
-    for task in pending:
-        task.cancel()
-
-    #
-    # at last, start a coro for stopping the loop
+    # at last, we stop the loop
     #
     logging.debug("Stop event loop")
     loop.stop()
@@ -173,7 +170,6 @@ if __name__ == "__main__":
         logging.info(f'ConfigErr: {ConfigErr}')
     Inverter.class_init()
     Schedule.start()
-    mb_tcp = ModbusTcp(loop)
 
     #
     # Create tasks for our listening servers. These must be tasks! If we call
@@ -182,6 +178,40 @@ if __name__ == "__main__":
     #
     loop.create_task(asyncio.start_server(handle_client, '0.0.0.0', 5005))
     loop.create_task(asyncio.start_server(handle_client_v2, '0.0.0.0', 10000))
+
+    # https://crypto.stackexchange.com/questions/26591/tls-encryption-with-a-self-signed-pki-and-python-s-asyncio-module
+    '''
+    openssl genrsa -out -des3 ca.key.pem 2048
+    openssl genrsa -out server.key.pem 2048
+    openssl genrsa -out client.key.pem 2048
+
+    openssl req -x509 -new -nodes -key ca.key.pem -sha256 -days 365
+      -out ca.cert.pem -subj /C=US/ST=CA/L=Somewhere/O=Someone/CN=FoobarCA
+
+    openssl req -new -sha256 -key server.key.pem
+     -subj /C=US/ST=CA/L=Somewhere/O=Someone/CN=Foobar -out server.csr
+    openssl x509 -req -in server.csr -CA ca.cert.pem -CAkey ca.key.pem
+      -CAcreateserial -out server.cert.pem -days 365 -sha256
+
+    openssl req -new -sha256 -key client.key.pem
+      -subj /C=US/ST=CA/L=Somewhere/O=Someone/CN=Foobar -out client.csr
+    openssl x509 -req -in client.csr -CA ca.cert.pem -CAkey ca.key.pem
+      -CAcreateserial -out client.cert.pem -days 365 -sha256
+    '''
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+    server_ctx.maximum_version = ssl.TLSVersion.TLSv1_3
+    server_ctx.verify_mode = ssl.CERT_REQUIRED
+    server_ctx.options |= ssl.OP_SINGLE_ECDH_USE
+    server_ctx.options |= ssl.OP_NO_COMPRESSION
+    server_ctx.load_cert_chain(certfile='cert/server.pem',
+                               keyfile='cert/server.key')
+    server_ctx.load_verify_locations(cafile='cert/ca.pem')
+    server_ctx.set_ciphers('ECDH+AESGCM')
+
+    loop.create_task(asyncio.start_server(handle_client_v3, '0.0.0.0', 10443,
+                                          ssl=server_ctx))
     web_task = loop.create_task(webserver('0.0.0.0', 8127))
 
     #
@@ -193,7 +223,7 @@ if __name__ == "__main__":
                                 lambda loop=loop: asyncio.create_task(
                                     handle_shutdown(web_task)))
 
-    loop.set_debug(log_level == logging.DEBUG)
+    loop.set_debug(True)
     try:
         if ConfigErr is None:
             proxy_is_up = True