Merge pull request #14 from s-allius/reduze-size

Reduze size

.gitignore

@@ -5,3 +5,4 @@ homeassistant/**
 tsun_proxy/**
 system_tests/**
 Doku/**
+.DS_Store

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
     "python.testing.pytestArgs": [
+        "-vv",
         "app","system_tests"
     ],
     "python.testing.unittestEnabled": false,

CHANGELOG.md

@@ -7,15 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.0] - 2023-10-07
 
-### Removed
+This version halves the size of the Docker image and reduces the attack surface for security vulnerabilities, by omitting unneeded code. The feature set is exactly the same as the previous release version 0.1.0.
 
-- 
+### Changes
+
+- move from slim-bookworm to an alpine base image
+- install python requirements with pip wheel
+- disable DEBUG log for releases
+- support building of release candidates
+
+## [0.1.0] - 2023-10-06
+
+- refactoring of the connection classes
+- change user id on startup
+- register MQTT topics to home assistant, even if we have multiple inverters
+
+## [0.0.6] - 2023-10-03
+
+- Bump aiomqtt to version 1.2.1
+- Force MQTT registration when the home assistant has set the status to online again
+- fix control byte output in tx trace
+- dealloc async_stream instances in connection termination
+
+## [0.0.5] - 2023-10-01
+
+- Entity icons updated
+- Prints version on start
+- Prepare for MQTT component != sensor
+- Add MQTT origin
+  
+## [0.0.4] - 2023-09-30
+
+- With this patch we ignore the setting 'suggested_area' in config.toml, because it makes no sense with multiple devices. We are looking for a better solution without combining all values into one area again in a later version.
+  
+❗Due to the change from one device to multiple devices in the Home Assistant, the previous MQTT device should be deleted in the Home Assistant after the update to pre-release '0.0.4'. Afterwards, the proxy must be restarted again to ensure that the sub-devices are created completely.
 
 ### Added
 
-- 
+- Register multiple devices at home-assistant instead of one for all measurements.
-## [0.0.3] - 2023-09-27
+  Now we register: a Controller, the inverter and up to 4 input devices to home-assistant.
+  
+## [0.0.3] - 2023-09-28
 
 ### Added
 

app/.dockerignore

@@ -1,3 +1,4 @@
 tests/
 **/__pycache__
 *.pyc
+.DS_Store

app/Dockerfile

@@ -2,67 +2,58 @@ ARG SERVICE_NAME="tsun-proxy"
 ARG UID=1000
 ARG GID=1000
 
-# set base image (host OS)
+#
-FROM python:3.11-slim-bookworm AS builder
+# first stage for our base image
-
+FROM python:3.11-alpine AS base
 USER root
 
-# install gosu for a better su+exec command
+RUN apk update && \
-RUN set -eux; \
+    apk upgrade
-	apt-get update; \
+RUN apk add --no-cache su-exec	
-	apt-get install -y gosu; \
-	rm -rf /var/lib/apt/lists/*; \
-# verify that the binary works
-	gosu nobody true
 
-RUN pip install --upgrade pip
+#
+# second stage for building wheels packages
+FROM base as builder
 
+RUN apk add --no-cache build-base && \
+    python -m pip install --no-cache-dir -U pip wheel
 
-# copy the dependencies file to the working directory
+# copy the dependencies file to the root dir and install requirements
-COPY ./requirements.txt .
+COPY ./requirements.txt /root/
-
+RUN python -OO -m pip wheel --no-cache-dir --wheel-dir=/root/wheels -r /root/requirements.txt
-# install dependencies
-RUN pip install --user -r requirements.txt
 
 
 #
-# second unnamed stage
+# third stage for our runtime image
-FROM python:3.11-slim-bookworm
+FROM base as runtime
 ARG SERVICE_NAME
+ARG VERSION
 ARG UID
 ARG GID
+ARG LOG_LVL
+
+ENV VERSION=$VERSION
 ENV SERVICE_NAME=$SERVICE_NAME
 ENV UID=$UID
 ENV GID=$GID 
+ENV LOG_LVL=$LOG_LVL
 
     
-
-RUN addgroup --gid $GID $SERVICE_NAME && \
-    adduser --ingroup $SERVICE_NAME --shell /bin/false --disabled-password --uid $UID $SERVICE_NAME && \
-    mkdir -p /home/$SERVICE_NAME/log /home/$SERVICE_NAME/config && \
-    chown -R $SERVICE_NAME:$SERVICE_NAME /home/$SERVICE_NAME
-#addgroup -S -g 1883 mosquitto 2>/dev/null && \
- #   adduser -S -u 1883 -D -H -h /var/empty -s /sbin/nologin -G mosquitto -g mosquitto mosquitto 2>/dev/null && \
-  #  mkdir -p /mosquitto/config /mosquitto/data /mosquitto/log && \
-  #  chown -R mosquitto:mosquitto /mosquitto && \
-    
 # set the working directory in the container
 WORKDIR /home/$SERVICE_NAME
 
 # update PATH environment variable
 ENV HOME=/home/$SERVICE_NAME
-ENV PATH=/home/$SERVICE_NAME/.local:$PATH
 
 VOLUME ["/home/$SERVICE_NAME/log", "/home/$SERVICE_NAME/config"]
 
-# copy only the dependencies installation from the 1st stage image
+# install the requirements from the wheels packages from the builder stage
-COPY --from=builder --chown=$SERVICE_NAME:$SERVICE_NAME /root/.local /home/$SERVICE_NAME/.local
+COPY --from=builder /root/wheels /root/wheels
-COPY --from=builder /usr/sbin/gosu /usr/sbin/gosu
+RUN python -m pip install --no-cache --no-index /root/wheels/* && \
-
+    rm -rf /root/wheels
-COPY entrypoint.sh /root/entrypoint.sh
-RUN chmod +x /root/entrypoint.sh
 
 # copy the content of the local src and config directory to the working directory
+COPY --chmod=0700 entrypoint.sh /root/entrypoint.sh
 COPY config .
 COPY src .
 
@@ -72,9 +63,9 @@ EXPOSE 5005
 ENTRYPOINT ["/root/entrypoint.sh"]
 CMD [ "python3", "./server.py" ]
 
-LABEL org.label-schema.build-date=$BUILD_DATE 
+
-LABEL org.opencontainers.image.authors="Stefan Allius <stefan.allius@t-online.de>"
+LABEL org.opencontainers.image.authors="Stefan Allius"
 LABEL org.opencontainers.image.source https://github.com/s-allius/tsun-gen3-proxy
 LABEL org.opencontainers.image.description 'The "TSUN Gen3 Micro-Inverter" proxy enables a reliable connection between TSUN third generation inverters and an MQTT broker to integrate the inverter into typical home automations'
 LABEL org.opencontainers.image.licenses="BSD-3-Clause"
-LABEL org.opencontainers.image.vendor="Stefan Allius>"
+LABEL org.opencontainers.image.vendor="Stefan Allius"

app/build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+BUILD_DATE=$(date -Iminutes)
+VERSION=$(git describe --tags --abbrev=0)
+VERSION="${VERSION:1}"
+arr=(${VERSION//./ })
+MAJOR=${arr[0]}
+IMAGE=tsun-gen3-proxy
+
+if [[ $1 == dev ]] || [[ $1 == rc ]] ;then
+IMAGE=docker.io/sallius/${IMAGE}
+VERSION=${VERSION}-$1
+elif [[ $1 == rel ]];then
+IMAGE=ghcr.io/s-allius/${IMAGE}
+else
+echo argument missing!
+echo try: $0 '[dev|rc|rel]'
+exit 1
+fi
+
+echo version: $VERSION  build-date: $BUILD_DATE   image: $IMAGE
+if [[ $1 == dev ]];then
+docker build --build-arg "VERSION=${VERSION}" --build-arg "LOG_LVL=DEBUG" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest app
+elif [[ $1 == rc ]];then
+docker build --no-cache --build-arg "VERSION=${VERSION}" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest app
+elif [[ $1 == rel ]];then
+docker build --no-cache --build-arg "VERSION=${VERSION}" --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.version=${VERSION}" -t ${IMAGE}:latest -t ${IMAGE}:${MAJOR} -t ${IMAGE}:${VERSION} app
+docker push ghcr.io/s-allius/tsun-gen3-proxy:latest
+docker push ghcr.io/s-allius/tsun-gen3-proxy:${MAJOR}
+docker push ghcr.io/s-allius/tsun-gen3-proxy:${VERSION}
+fi

app/entrypoint.sh

@@ -2,14 +2,23 @@
 set -e
 
 user="$(id -u)"
-echo "#############################################"
+echo "######################################################"
-echo "# start: '$SERVICE_NAME'"
+echo "# prepare: '$SERVICE_NAME' Version:$VERSION"
-echo "# with UserID:$UID, GroupID:$GID"
+echo "# for running with UserID:$UID, GroupID:$GID"
-echo "#############################################"
+echo "#"
 
 if [ "$user" = '0' ]; then
-	[ -d "/home/$SERVICE_NAME" ] && chown -R $SERVICE_NAME:$SERVICE_NAME /home/$SERVICE_NAME || true
+    mkdir -p /home/$SERVICE_NAME/log /home/$SERVICE_NAME/config
-    exec gosu $SERVICE_NAME "$@"
+    
+    if ! id $SERVICE_NAME &> /dev/null; then
+        addgroup --gid $GID $SERVICE_NAME 2> /dev/null
+        adduser -G $SERVICE_NAME -s /bin/false -D -H -g "" -u $UID $SERVICE_NAME
+    fi    
+	chown -R $SERVICE_NAME:$SERVICE_NAME /home/$SERVICE_NAME || true
+    echo "######################################################"
+    echo "#"
+
+    exec su-exec $SERVICE_NAME "$@"
 else
     exec "$@"
 fi

app/requirements.txt

@@ -1,2 +1,2 @@
-    aiomqtt==1.2.0
+    aiomqtt==1.2.1
-    schema
+    schema==0.7.5

app/src/async_stream.py

@@ -1,67 +1,52 @@
-import logging, traceback, aiomqtt, json
+import logging, traceback
 from config import Config
+#import gc
 from messages import Message, hex_dump_memory
-from mqtt import Mqtt
 
 logger = logging.getLogger('conn')
-logger_mqtt = logging.getLogger('mqtt')
 
 class AsyncStream(Message):
 
-    def __init__(self, proxy, reader, writer, addr, stream=None, server_side=True):
+    def __init__(self, reader, writer, addr, remote_stream, server_side: bool) -> None:
         super().__init__()
-        self.proxy  = proxy
         self.reader = reader
         self.writer = writer
-        self.remoteStream = stream
+        self.remoteStream = remote_stream
-        self.addr = addr
         self.server_side = server_side
-        self.mqtt = Mqtt()
+        self.addr = addr
         self.unique_id = 0
         self.node_id = ''
         
     '''
     Our puplic methods
     '''
-    async def set_serial_no(self, serial_no : str):
+    def set_serial_no(self, serial_no : str):
-        logger_mqtt.info(f'SerialNo: {serial_no}')
+        logger.debug(f'SerialNo: {serial_no}')
        
         if self.unique_id != serial_no: 
        
             inverters = Config.get('inverters')
-            #logger_mqtt.debug(f'Inverters: {inverters}')
+            #logger.debug(f'Inverters: {inverters}')
                 
             if serial_no in inverters:
-                logger_mqtt.debug(f'SerialNo {serial_no} allowed!')
+                logger.debug(f'SerialNo {serial_no} allowed!')
                 inv = inverters[serial_no]
                 self.node_id = inv['node_id']
-                sug_area    = inv['suggested_area']
+                self.sug_area    = inv['suggested_area']
             else:    
-                logger_mqtt.debug(f'SerialNo {serial_no} not known!')
+                logger.debug(f'SerialNo {serial_no} not known!')
                 self.node_id = ''
-                sug_area    = ''
+                self.sug_area    = ''
                 if not inverters['allow_all']:
                     self.unique_id = None
             
-                    logger_mqtt.error('ignore message from unknow inverter!')
+                    logger.warning(f'ignore message from unknow inverter! (SerialNo: {serial_no})')
                     return
 
             self.unique_id = serial_no
             
-            ha = Config.get('ha')
-            self.entitiy_prfx = ha['entity_prefix'] + '/'
-            discovery_prfx = ha['discovery_prefix'] + '/'
             
-            if self.server_side:
-                try:
-                    for data_json, id in self.db.ha_confs(self.entitiy_prfx + self.node_id, self.unique_id, sug_area):
-                            logger_mqtt.debug(f'Register: {data_json}')                                
-                            await self.mqtt.publish(f"{discovery_prfx}sensor/{self.node_id}{id}/config", data_json)
             
-                except Exception:
-                    logging.error(
-                        f"Proxy: Exception:\n"
-                        f"{traceback.format_exc()}")
 
     
     async def loop(self) -> None:
@@ -71,18 +56,18 @@ class AsyncStream(Message):
                 await self.__async_read()     
                 
                 if self.id_str:
-                    await self.set_serial_no(self.id_str.decode("utf-8"))
+                    self.set_serial_no(self.id_str.decode("utf-8"))
             
                 if self.unique_id: 
                     await self.__async_write()     
                     await self.__async_forward()
-                    await self.__async_publ_mqtt()
+                    await self.async_publ_mqtt()
                 
       
             except (ConnectionResetError,
                     ConnectionAbortedError,
                     RuntimeError) as error:
-                logger.error(f'In loop for {self.addr}: {error}')
+                logger.warning(f'In loop for {self.addr}: {error}')
                 self.close()
                 return
             except Exception:
@@ -92,11 +77,17 @@ class AsyncStream(Message):
                 self.close()
                 return
             
-    def close(self):
+    def disc(self) -> None:
-        logger.info(f'in async_stream.close() {self.addr}')
+        logger.debug(f'in AsyncStream.disc() {self.addr}')
         self.writer.close()
-        self.proxy = None
+            
-        self.remoteStream = None
+        
+    def close(self):
+        logger.debug(f'in AsyncStream.close() {self.addr}')
+        self.writer.close()
+        super().close()         # call close handler in the parent class
+
+#        logger.info (f'AsyncStream refs: {gc.get_referrers(self)}')
 
     
     '''
@@ -120,8 +111,7 @@ class AsyncStream(Message):
     async def __async_forward(self) -> None:
         if self._forward_buffer:
             if not self.remoteStream:
-                tsun = Config.get('tsun')
+                await self.async_create_remote()  # only implmeneted for server side => syncServerStream
-                self.remoteStream = await self.proxy.CreateClientStream (self, tsun['host'], tsun['port'])
                 
             if self.remoteStream:
                 hex_dump_memory(logging.DEBUG, f'Forward to {self.remoteStream.addr}:', self._forward_buffer, len(self._forward_buffer))
@@ -129,17 +119,14 @@ class AsyncStream(Message):
                 await self.remoteStream.writer.drain()                    
                 self._forward_buffer = bytearray(0)
 
-    async def __async_publ_mqtt(self) -> None:
+    async def async_create_remote(self) -> None:
-        if self.server_side:
+        pass
-            db = self.db.db
+
-            for key in self.new_data:
+    async def async_publ_mqtt(self) -> None:
-                if self.new_data[key] and key in db:
+        pass       
-                    data_json = json.dumps(db[key])
+
-                    logger_mqtt.info(f'{key}: {data_json}')
-                    await self.mqtt.publish(f"{self.entitiy_prfx}{self.node_id}{key}", data_json)
-                    self.new_data[key] = False
 
     def __del__ (self):
-        logger.debug ("AsyncStream __del__")       
+        logging.debug (f"AsyncStream.__del__  {self.addr}")     
 
 

app/src/config.py

@@ -60,7 +60,7 @@ class Config():
             config['inverters'] = def_config['inverters'] | usr_config['inverters']
 
             cls.config = cls.conf_schema.validate(config)
-            logging.debug(f'Readed config: "{cls.config}" ')
+            #logging.debug(f'Readed config: "{cls.config}" ')
  
         except Exception as error:
             logger.error(f'Config.read: {error}')

app/src/infos.py

@@ -1,32 +1,39 @@
-import struct, json, logging
+import struct, json, logging, os
 
 
 
 class Infos:    
     def __init__(self):
         self.db = {}
+        self.app_name = os.getenv('SERVICE_NAME', 'proxy')
+        self.version  = os.getenv('VERSION', 'unknown')
         self.tracer = logging.getLogger('data')
 
+    __info_devs={
+        'controller':{                    'name':'Controller',     'mdl':0x00092f90, 'mf':0x000927c0, 'sw':0x00092ba8},
+        'inverter':  {'via':'controller', 'name':'Micro Inverter', 'mdl':0x00000032, 'mf':0x00000014, 'sw':0x0000001e},
+        'input_pv1': {'via':'inverter',   'name':'Module PV1'},
+        'input_pv2': {'via':'inverter',   'name':'Module PV2'},
+        'input_pv3': {'via':'inverter',   'name':'Module PV3'},
+        'input_pv4': {'via':'inverter',   'name':'Module PV4'},
+    }
+
     __info_defs={
-        # collector values:
+        # collector values used for device registration:
             0x00092ba8:  {'name':['collector', 'Collector_Fw_Version'],       'level': logging.INFO,  'unit': ''},
             0x000927c0:  {'name':['collector', 'Chip_Type'],                  'level': logging.DEBUG, 'unit': ''},
             0x00092f90:  {'name':['collector', 'Chip_Model'],                 'level': logging.DEBUG, 'unit': ''},
             0x00095a88:  {'name':['collector', 'Trace_URL'],                  'level': logging.DEBUG, 'unit': ''},
             0x00095aec:  {'name':['collector', 'Logger_URL'],                 'level': logging.DEBUG, 'unit': ''},
-            0x000cf850:  {'name':['collector', 'Data_Up_Interval'],           'level': logging.DEBUG, 'unit': 's'},
+       
-            0x000005dc:  {'name':['collector', 'Rated_Power'],                'level': logging.DEBUG, 'unit': 'W'},
+        # inverter values used for device registration:
-        # inverter values:
             0x0000000a:  {'name':['inverter', 'Product_Name'],                'level': logging.DEBUG, 'unit': ''},
             0x00000014:  {'name':['inverter', 'Manufacturer'],                'level': logging.DEBUG, 'unit': ''},
             0x0000001e:  {'name':['inverter', 'Version'],                     'level': logging.INFO,  'unit': ''},
             0x00000028:  {'name':['inverter', 'Serial_Number'],               'level': logging.DEBUG, 'unit': ''},
             0x00000032:  {'name':['inverter', 'Equipment_Model'],             'level': logging.DEBUG, 'unit': ''},
-        # env:
-            0x00000514:  {'name':['env', 'Inverter_Temp'],                    'level': logging.DEBUG, 'unit': '°C',  'ha':{'dev_cla': 'temperature', 'stat_cla': 'measurement', 'id':'temp_',   'fmt':'| float','name': 'Inverter Temperature'}},
-            0x000c3500:  {'name':['env', 'Signal_Strength'],                  'level': logging.DEBUG, 'unit': '%' ,  'ha':{'dev_cla': None,          'stat_cla': 'measurement', 'id':'signal_', 'fmt':'| float','name': 'Signal Strength', 'icon':'mdi:wifi'}},
        
-        # events:
+        # events
             0x00000191:  {'name':['events', '401_'],                          'level': logging.DEBUG, 'unit': ''},
             0x00000192:  {'name':['events', '402_'],                          'level': logging.DEBUG, 'unit': ''},
             0x00000193:  {'name':['events', '403_'],                          'level': logging.DEBUG, 'unit': ''},
@@ -43,85 +50,142 @@ class Infos:
             0x0000019e:  {'name':['events', '414_'],                          'level': logging.DEBUG, 'unit': ''},
             0x0000019f:  {'name':['events', '415_GridFreqOverRating'],        'level': logging.DEBUG, 'unit': ''},
             0x000001a0:  {'name':['events', '416_'],                          'level': logging.DEBUG, 'unit': ''},
+        
         # grid measures:
-            0x000003e8:  {'name':['grid', 'Voltage'],                         'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev_cla': 'voltage',     'stat_cla': 'measurement', 'id':'out_volt_',  'fmt':'| float','name': 'Grid Voltage'}},
+            0x000003e8:  {'name':['grid', 'Voltage'],                         'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev':'inverter', 'dev_cla': 'voltage',     'stat_cla': 'measurement', 'id':'out_volt_',  'fmt':'| float','name': 'Grid Voltage'}},
-            0x0000044c:  {'name':['grid', 'Current'],                         'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev_cla': 'current',     'stat_cla': 'measurement', 'id':'out_cur_',   'fmt':'| float','name': 'Grid Current'}},
+            0x0000044c:  {'name':['grid', 'Current'],                         'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev':'inverter', 'dev_cla': 'current',     'stat_cla': 'measurement', 'id':'out_cur_',   'fmt':'| float','name': 'Grid Current'}},
-            0x000004b0:  {'name':['grid', 'Frequency'],                       'level': logging.DEBUG, 'unit': 'Hz',   'ha':{'dev_cla': 'frequency',   'stat_cla': 'measurement', 'id':'out_freq_',  'fmt':'| float','name': 'Grid Frequency'}},
+            0x000004b0:  {'name':['grid', 'Frequency'],                       'level': logging.DEBUG, 'unit': 'Hz',   'ha':{'dev':'inverter', 'dev_cla': 'frequency',   'stat_cla': 'measurement', 'id':'out_freq_',  'fmt':'| float','name': 'Grid Frequency'}},
-            0x00000640:  {'name':['grid', 'Output_Power'],                    'level': logging.INFO,  'unit': 'W',    'ha':{'dev_cla': 'power',       'stat_cla': 'measurement', 'id':'out_power_', 'fmt':'| float','name': 'Actual Power'}},
+            0x00000640:  {'name':['grid', 'Output_Power'],                    'level': logging.INFO,  'unit': 'W',    'ha':{'dev':'inverter', 'dev_cla': 'power',       'stat_cla': 'measurement', 'id':'out_power_', 'fmt':'| float','name': 'Power'}},
+            0x000005dc:  {'name':['env',  'Rated_Power'],                     'level': logging.DEBUG, 'unit': 'W',    'ha':{'dev':'inverter', 'dev_cla': 'power',       'stat_cla': 'measurement', 'id':'rated_power_','fmt':'| int', 'name': 'Rated Power'}},
+            0x00000514:  {'name':['env',  'Inverter_Temp'],                   'level': logging.DEBUG, 'unit': '°C',   'ha':{'dev':'inverter', 'dev_cla': 'temperature', 'stat_cla': 'measurement', 'id':'temp_',   'fmt':'| int','name': 'Temperature'}},
+
         # input measures:           
-            0x000006a4:  {'name':['input', 'pv1', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv1_', 'name': 'Voltage PV1', 'val_tpl' :"{{ (value_json['pv1']['Voltage'] | float)}}", 'unvisible':1}},
+            0x000006a4:  {'name':['input', 'pv1', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev':'input_pv1', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv1_', 'name': 'Voltage', 'val_tpl' :"{{ (value_json['pv1']['Voltage'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000708:  {'name':['input', 'pv1', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv1_',  'name': 'Current PV1', 'val_tpl' :"{{ (value_json['pv1']['Current'] | float)}}", 'unvisible':1}},
+            0x00000708:  {'name':['input', 'pv1', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev':'input_pv1', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv1_',  'name': 'Current', 'val_tpl' :"{{ (value_json['pv1']['Current'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x0000076c:  {'name':['input', 'pv1', 'Power'],                   'level': logging.INFO,  'unit': 'W',    'ha':{'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv1_','name': 'Power PV1',   'val_tpl' :"{{ (value_json['pv1']['Power']   | float)}}"}},
+            0x0000076c:  {'name':['input', 'pv1', 'Power'],                   'level': logging.INFO,  'unit': 'W',    'ha':{'dev':'input_pv1', 'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv1_','name': 'Power',   'val_tpl' :"{{ (value_json['pv1']['Power'] | float)}}", 'icon':'mdi:gauge'}},
-            0x000007d0:  {'name':['input', 'pv2', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv2_', 'name': 'Voltage PV2', 'val_tpl' :"{{ (value_json['pv2']['Voltage'] | float)}}", 'unvisible':1}},
+            0x000007d0:  {'name':['input', 'pv2', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev':'input_pv2', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv2_', 'name': 'Voltage', 'val_tpl' :"{{ (value_json['pv2']['Voltage'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000834:  {'name':['input', 'pv2', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv2_',  'name': 'Current PV2', 'val_tpl' :"{{ (value_json['pv2']['Current'] | float)}}", 'unvisible':1}},
+            0x00000834:  {'name':['input', 'pv2', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev':'input_pv2', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv2_',  'name': 'Current', 'val_tpl' :"{{ (value_json['pv2']['Current'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000898:  {'name':['input', 'pv2', 'Power'],                   'level': logging.INFO,  'unit': 'W',    'ha':{'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv2_','name': 'Power PV2',   'val_tpl' :"{{ (value_json['pv2']['Power']   | float)}}"}},
+            0x00000898:  {'name':['input', 'pv2', 'Power'],                   'level': logging.INFO,  'unit': 'W',    'ha':{'dev':'input_pv2', 'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv2_','name': 'Power',   'val_tpl' :"{{ (value_json['pv2']['Power'] | float)}}", 'icon':'mdi:gauge'}},
-            0x000008fc:  {'name':['input', 'pv3', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv3_', 'name': 'Voltage PV3', 'val_tpl' :"{{ (value_json['pv3']['Voltage'] | float)}}", 'unvisible':1}},
+            0x000008fc:  {'name':['input', 'pv3', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev':'input_pv3', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv3_', 'name': 'Voltage', 'val_tpl' :"{{ (value_json['pv3']['Voltage'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000960:  {'name':['input', 'pv3', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv3_',  'name': 'Current PV3', 'val_tpl' :"{{ (value_json['pv3']['Current'] | float)}}", 'unvisible':1}},
+            0x00000960:  {'name':['input', 'pv3', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev':'input_pv3', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv3_',  'name': 'Current', 'val_tpl' :"{{ (value_json['pv3']['Current'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x000009c4:  {'name':['input', 'pv3', 'Power'],                   'level': logging.DEBUG, 'unit': 'W',    'ha':{'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv3_','name': 'Power PV3',   'val_tpl' :"{{ (value_json['pv3']['Power']   | float)}}"}},
+            0x000009c4:  {'name':['input', 'pv3', 'Power'],                   'level': logging.DEBUG, 'unit': 'W',    'ha':{'dev':'input_pv3', 'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv3_','name': 'Power',   'val_tpl' :"{{ (value_json['pv3']['Power'] | float)}}", 'icon':'mdi:gauge'}},
-            0x00000a28:  {'name':['input', 'pv4', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv4_', 'name': 'Voltage PV4', 'val_tpl' :"{{ (value_json['pv4']['Voltage'] | float)}}", 'unvisible':1}},
+            0x00000a28:  {'name':['input', 'pv4', 'Voltage'],                 'level': logging.DEBUG, 'unit': 'V',    'ha':{'dev':'input_pv4', 'dev_cla': 'voltage', 'stat_cla': 'measurement', 'id':'volt_pv4_', 'name': 'Voltage', 'val_tpl' :"{{ (value_json['pv4']['Voltage'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000a8c:  {'name':['input', 'pv4', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv4_',  'name': 'Current PV4', 'val_tpl' :"{{ (value_json['pv4']['Current'] | float)}}", 'unvisible':1}},
+            0x00000a8c:  {'name':['input', 'pv4', 'Current'],                 'level': logging.DEBUG, 'unit': 'A',    'ha':{'dev':'input_pv4', 'dev_cla': 'current', 'stat_cla': 'measurement', 'id':'cur_pv4_',  'name': 'Current', 'val_tpl' :"{{ (value_json['pv4']['Current'] | float)}}", 'unvisible':1, 'icon':'mdi:gauge'}},
-            0x00000af0:  {'name':['input', 'pv4', 'Power'],                   'level': logging.DEBUG, 'unit': 'W',    'ha':{'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv4_','name': 'Power PV4',   'val_tpl' :"{{ (value_json['pv4']['Power']   | float)}}"}},
+            0x00000af0:  {'name':['input', 'pv4', 'Power'],                   'level': logging.DEBUG, 'unit': 'W',    'ha':{'dev':'input_pv4', 'dev_cla': 'power',   'stat_cla': 'measurement', 'id':'power_pv4_','name': 'Power',   'val_tpl' :"{{ (value_json['pv4']['Power'] | float)}}", 'icon':'mdi:gauge'}},
-            0x00000c1c:  {'name':['input', 'pv1', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv1_','name': 'Daily Generation PV1', 'val_tpl' :"{{ (value_json['pv1']['Daily_Generation'] | float)}}"}},          
+            0x00000c1c:  {'name':['input', 'pv1', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv1', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv1_','name': 'Daily Generation', 'val_tpl' :"{{ (value_json['pv1']['Daily_Generation'] | float)}}", 'icon':'mdi:solar-power-variant'}},          
-            0x00000c80:  {'name':['input', 'pv1', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv1_','name': 'Total Generation PV1', 'val_tpl' :"{{ (value_json['pv1']['Total_Generation'] | float)}}"}},                    
+            0x00000c80:  {'name':['input', 'pv1', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv1', 'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv1_','name': 'Total Generation', 'val_tpl' :"{{ (value_json['pv1']['Total_Generation'] | float)}}", 'icon':'mdi:solar-power'}},                    
-            0x00000ce4:  {'name':['input', 'pv2', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv2_','name': 'Daily Generation PV2', 'val_tpl' :"{{ (value_json['pv2']['Daily_Generation'] | float)}}"}},          
+            0x00000ce4:  {'name':['input', 'pv2', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv2', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv2_','name': 'Daily Generation', 'val_tpl' :"{{ (value_json['pv2']['Daily_Generation'] | float)}}", 'icon':'mdi:solar-power-variant'}},          
-            0x00000d48:  {'name':['input', 'pv2', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv2_','name': 'Total Generation PV2', 'val_tpl' :"{{ (value_json['pv2']['Total_Generation'] | float)}}"}},          
+            0x00000d48:  {'name':['input', 'pv2', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv2', 'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv2_','name': 'Total Generation', 'val_tpl' :"{{ (value_json['pv2']['Total_Generation'] | float)}}", 'icon':'mdi:solar-power'}},          
-            0x00000dac:  {'name':['input', 'pv3', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv3_','name': 'Daily Generation PV3', 'val_tpl' :"{{ (value_json['pv3']['Daily_Generation'] | float)}}"}},          
+            0x00000dac:  {'name':['input', 'pv3', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv3', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv3_','name': 'Daily Generation', 'val_tpl' :"{{ (value_json['pv3']['Daily_Generation'] | float)}}", 'icon':'mdi:solar-power-variant'}},          
-            0x00000e10:  {'name':['input', 'pv3', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv3_','name': 'Total Generation PV3', 'val_tpl' :"{{ (value_json['pv3']['Total_Generation'] | float)}}"}},                    
+            0x00000e10:  {'name':['input', 'pv3', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv3', 'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv3_','name': 'Total Generation', 'val_tpl' :"{{ (value_json['pv3']['Total_Generation'] | float)}}", 'icon':'mdi:solar-power'}},                    
-            0x00000e74:  {'name':['input', 'pv4', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv4_','name': 'Daily Generation PV4', 'val_tpl' :"{{ (value_json['pv4']['Daily_Generation'] | float)}}"}},          
+            0x00000e74:  {'name':['input', 'pv4', 'Daily_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv4', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_pv4_','name': 'Daily Generation', 'val_tpl' :"{{ (value_json['pv4']['Daily_Generation'] | float)}}", 'icon':'mdi:solar-power-variant'}},          
-            0x00000ed8:  {'name':['input', 'pv4', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv4_','name': 'Total Generation PV4', 'val_tpl' :"{{ (value_json['pv4']['Total_Generation'] | float)}}"}},          
+            0x00000ed8:  {'name':['input', 'pv4', 'Total_Generation'],        'level': logging.DEBUG, 'unit': 'kWh',  'ha':{'dev':'input_pv4', 'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_pv4_','name': 'Total Generation', 'val_tpl' :"{{ (value_json['pv4']['Total_Generation'] | float)}}", 'icon':'mdi:solar-power'}},          
         # total:    
-            0x00000b54:  {'name':['total', 'Daily_Generation'],               'level': logging.INFO,  'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_', 'fmt':'| float','name': 'Daily Generation'}},
+            0x00000b54:  {'name':['total', 'Daily_Generation'],               'level': logging.INFO,  'unit': 'kWh',  'ha':{'dev':'inverter', 'dev_cla': 'energy', 'stat_cla': 'total_increasing', 'id':'daily_gen_', 'fmt':'| float','name': 'Daily Generation', 'icon':'mdi:solar-power-variant'}},
-            0x00000bb8:  {'name':['total', 'Total_Generation'],               'level': logging.INFO,  'unit': 'kWh',  'ha':{'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_', 'fmt':'| float','name': 'Total Generation', 'icon':'mdi:solar-power'}},
+            0x00000bb8:  {'name':['total', 'Total_Generation'],               'level': logging.INFO,  'unit': 'kWh',  'ha':{'dev':'inverter', 'dev_cla': 'energy', 'stat_cla': 'total',            'id':'total_gen_', 'fmt':'| float','name': 'Total Generation', 'icon':'mdi:solar-power'}},
-            0x000c96a8:  {'name':['total', 'Power_On_Time'],                  'level': logging.DEBUG, 'unit': 's',    'ha':{'dev_cla': 'duration',  'stat_cla': 'measurement',      'id':'power_on_time_',             'name': 'Power on Time', 'val_tpl':"{{ (value_json['Power_On_Time'] | float)}}", 'nat_prc':'3'}},
+
+        # controller:
+            0x000c3500:  {'name':['controller', 'Signal_Strength'],           'level': logging.DEBUG, 'unit': '%' ,  'ha':{'dev':'controller', 'dev_cla': None,       'stat_cla': 'measurement', 'id':'signal_',         'fmt':'| int', 'name': 'Signal Strength', 'icon':'mdi:wifi'}},
+            0x000c96a8:  {'name':['controller', 'Power_On_Time'],             'level': logging.DEBUG, 'unit': 's',   'ha':{'dev':'controller', 'dev_cla': 'duration', 'stat_cla': 'measurement', 'id':'power_on_time_',                 'name': 'Power on Time',   'val_tpl':"{{ (value_json['Power_On_Time'] | float)}}", 'nat_prc':'3'}},
+            0x000cf850:  {'name':['controller', 'Data_Up_Interval'],          'level': logging.DEBUG, 'unit': 's',   'ha':{'dev':'controller', 'dev_cla': None,       'stat_cla': 'measurement', 'id':'data_up_intval_', 'fmt':'| int', 'name': 'Data Up Interval', 'icon':'mdi:update'}},
 
     }          
                                     
+    def dev_value(self, idx:str|int) -> str|int|float|None:
+        '''returns the stored device value from our database
+
+        idx:int ==> lookup the value in the database and return it as str, int or flout. If the value is not available return 'None'
+        idx:str ==> returns the string as a fixed value without a database loopup  
+        '''
+        if type (idx) is str:
+            return idx               # return idx as a fixed value 
+        elif idx in self.__info_defs:
+            dict = self.db
+            row = self.__info_defs[idx]
+            keys = row['name']
+
+            for key in keys:
+                if key not in dict:
+                    return None      # value not found in the database
+                dict = dict[key]
+            return dict              # value of the reqeusted entry   
+            
+        return None                  # unknwon idx, not in __info_defs
+
 
     def ha_confs(self, prfx="tsun/garagendach/", snr='123', sug_area =''):
+        '''Generator function yields a json register struct for home-assistant auto configuration and a unique entity string
+
+        arguments:
+        prfx:str     ==> MQTT prefix for the home assistant 'stat_t string
+        snr:str      ==> serial number of the inverter, used to build unique entity strings
+        sug_area:str ==> suggested area string from the config file'''
         tab = self.__info_defs
         for key in tab:
             row = tab[key]
+
+            #check if we have details for home assistant
             if 'ha' in row:
                 ha = row['ha']
-                attr = {}
+                if 'comp' in ha:
-                if 'name' in ha:
+                    component = ha['comp']
-                    attr['name']   = ha['name']               # eg.  'name': "Actual Power"
                 else:
-                    attr['name']   = row['name'][-1]          # eg.  'name': "Actual Power"
+                    component = 'sensor'
+                attr = {}                                     # dict to collect all the sensor entity details 
+                if 'name' in ha:
+                    attr['name']   = ha['name']               # take the entity name from the ha dict
+                else:                 
+                    attr['name']   = row['name'][-1]          # otherwise take a name from the name array 
 
                 attr['stat_t'] = prfx +row['name'][0]         # eg.  'stat_t': "tsun/garagendach/grid"
                 attr['dev_cla']  = ha['dev_cla']              # eg.  'dev_cla': 'power'
                 attr['stat_cla'] = ha['stat_cla']             # eg.  'stat_cla': "measurement"
-                attr['uniq_id']  = ha['id']+snr               # eg.  'uniq_id':'out_power_123'
+                attr['uniq_id']  = ha['id']+snr               # build the 'uniq_id' from the id str + the serial no of the inverter
                 if 'val_tpl' in ha:
-                    attr['val_tpl']  = ha['val_tpl']       # eg.   'val_tpl': "{{ value_json['Output_Power']|float }}"
+                    attr['val_tpl']  = ha['val_tpl']          # get value template for complexe data structures
                 elif 'fmt' in ha:
                     attr['val_tpl']  = '{{value_json' + f"['{row['name'][-1]}'] {ha['fmt']}" + '}}'       # eg.   'val_tpl': "{{ value_json['Output_Power']|float }}"
 
                 if 'unit' in row:
-                    attr['unit_of_meas'] = row['unit']        # eg.  'unit_of_meas': 'W'
+                    attr['unit_of_meas'] = row['unit']        # optional add a 'unit_of_meas' e.g. 'W'
                 if 'icon' in ha:
-                    attr['icon'] = ha['icon']                 # eg. 'icon':'mdi:solar-power'
+                    attr['icon'] = ha['icon']                 # optional add an icon for the entity
                 if 'nat_prc' in ha:
-                    attr['suggested_display_precision'] = ha['nat_prc']    
+                    attr['sug_dsp_prc'] = ha['nat_prc']     # optional add the precison of floats
-                #if 'unvisible' in ha:
-                #    attr['entity_registry_visible_default'] = 'False'  
                     
                 # eg. 'dev':{'name':'Microinverter','mdl':'MS-600','ids':["inverter_123"],'mf':'TSUN','sa': 'auf Garagendach'}
                 # attr['dev'] = {'name':'Microinverter','mdl':'MS-600','ids':[f'inverter_{snr}'],'mf':'TSUN','sa': 'auf Garagendach'}   
-                dev = {}
+                if 'dev' in ha:
-                dev['name'] = 'Microinverter' #fixme
+                    device = self.__info_devs[ha['dev']]
-                dev['mdl']  = 'MS-600'        #fixme
+                    dev = {}
-                dev['ids']  = [f'inverter_{snr}']
+
-                dev['mf']   = 'TSUN'          #fixme
+                    # the same name fpr 'name' and 'suggested area', so we get dedicated devices in home assistant with short value name and headline 
-                dev['sa']   = sug_area
+                    if 'name' in device:
-                dev['sw']   = '0.01'          #fixme
+                        dev['name'] = device['name']
-                dev['hw']   = 'Hw0.01'        #fixme
+                        dev['sa']   = device['name']
-                #dev['via_device'] =          #fixme
+                    # fixme: we ignore the suggested area, since one area make no sense for multiple devices    
-                attr['dev'] = dev
+                    #else:    
+                    #    dev['name'] = sug_area  
+                    #    dev['sa']   = sug_area
+
+                    if 'via' in device:                       # add the link to the parent device
+                        dev['via_device'] = f"{device['via']}_{snr}"
+
+                    for key in ('mdl','mf', 'sw', 'hw'):      # add optional values fpr 'modell', 'manufaturer', 'sw version' and 'hw version'
+                        if key in device:
+                            data = self.dev_value(device[key])
+                            if data is not None: dev[key]  = data
+
+                    dev['ids']  = [f"{ha['dev']}_{snr}"]
+                    attr['dev'] = dev
+
+                    origin = {}
+                    origin['name'] = self.app_name
+                    origin['sw'] = self.version
+                    attr['o'] = origin
 
 
-                yield json.dumps (attr), attr['uniq_id']
+                yield json.dumps (attr), component, attr['uniq_id']
    
 
 
@@ -133,7 +197,10 @@ class Infos:
         return d['name'], d['level'], d['unit']
     
     
-    def parse(self, buf):
+    def parse(self, buf) -> None:
+        '''parse a data sequence received from the inverter and stores the values in Infos.db
+        
+        buf: buffer of the sequence to parse'''
         result = struct.unpack_from('!l', buf, 0)
         elms = result[0]
         i = 0

app/src/inverter.py

@@ -0,0 +1,103 @@
+import asyncio, logging, traceback, json
+from config import Config
+from async_stream import AsyncStream
+from mqtt import Mqtt
+#import gc
+
+logger = logging.getLogger('conn')
+
+
+
+class Inverter(AsyncStream):
+
+    def __init__ (self, reader, writer, addr):
+        super().__init__(reader, writer, addr, None, True)
+        self.mqtt = Mqtt()
+        self.ha_restarts = 0
+        ha = Config.get('ha')
+        self.entitiy_prfx = ha['entity_prefix'] + '/'
+        self.discovery_prfx = ha['discovery_prefix'] + '/'
+
+
+    async def server_loop(self, addr):
+        '''Loop for receiving messages from the inverter (server-side)'''
+        logger.info(f'Accept connection from  {addr}')        
+        await self.loop()
+        logging.info(f'Server loop stopped for {addr}')
+        
+        # if the server connection closes, we also have to disconnect the connection to te TSUN cloud
+        if self.remoteStream:
+            logging.debug ("disconnect client connection")
+            self.remoteStream.disc()
+        
+    async def client_loop(self, addr):
+        '''Loop for receiving messages from the TSUN cloud (client-side)'''
+        await self.remoteStream.loop()    
+        logging.info(f'Client loop stopped for {addr}')
+
+        # if the client connection closes, we don't touch the server connection. Instead we erase the client
+        # connection stream, thus on the next received packet from the inverter, we can establish a new connection 
+        # to the TSUN cloud
+        self.remoteStream.remoteStream = None    # erase backlink to inverter instance
+        self.remoteStream = None                 # than erase client connection 
+        
+    async def async_create_remote(self) -> None:
+        '''Establish a client connection to the TSUN cloud'''
+        tsun = Config.get('tsun')
+        host = tsun['host']
+        port = tsun['port']     
+        addr = (host, port)
+            
+        try:
+            logging.info(f'Connected to {addr}')
+            connect = asyncio.open_connection(host, port)
+            reader, writer = await connect    
+            self.remoteStream = AsyncStream(reader, writer, addr, self, False)
+            asyncio.create_task(self.client_loop(addr))
+            
+        except ConnectionRefusedError as error:
+            logging.info(f'{error}')
+        except Exception:
+            logging.error(
+                f"Inverter: Exception for {addr}:\n"
+                f"{traceback.format_exc()}")
+        
+    
+
+    async def async_publ_mqtt(self) -> None:
+        '''puplish data to MQTT broker'''
+        db = self.db.db
+        # check if new inverter or collector infos are available or when the home assistant has changed the status back to online
+        if (('inverter' in self.new_data and self.new_data['inverter']) or 
+             ('collector' in self.new_data and self.new_data['collector']) or
+               self.mqtt.ha_restarts != self.ha_restarts):
+            await self.__register_home_assistant()
+            self.ha_restarts = self.mqtt.ha_restarts
+
+        for key in self.new_data:
+            if self.new_data[key] and key in db:
+                data_json = json.dumps(db[key])
+                logger.info(f'{key}: {data_json}')
+                await self.mqtt.publish(f"{self.entitiy_prfx}{self.node_id}{key}", data_json)
+                self.new_data[key] = False
+
+    async def __register_home_assistant(self) -> None:
+        '''register all our topics at home assistant'''
+        try:
+            for data_json, component, id in self.db.ha_confs(self.entitiy_prfx + self.node_id, self.unique_id, self.sug_area):
+                    #logger.debug(f'MQTT Register: {data_json}')                                
+                    await self.mqtt.publish(f"{self.discovery_prfx}{component}/{self.node_id}{id}/config", data_json)
+        except Exception:
+            logging.error(
+                f"Inverter: Exception:\n"
+                f"{traceback.format_exc()}")
+            
+    def close(self) -> None:
+        logging.debug(f'Inverter.close() {self.addr}')
+        super().close()         # call close handler in the parent class
+#        logger.debug (f'Inverter refs: {gc.get_referrers(self)}')
+
+
+    def __del__ (self):
+        logging.debug ("Inverter.__del__")
+        super().__del__()  

app/src/logging.ini

@@ -25,13 +25,13 @@ qualname=conn
 
 [logger_data]
 level=DEBUG
-handlers=console_handler,file_handler_name1,file_handler_name2
+handlers=file_handler_name1,file_handler_name2
 propagate=0
 qualname=data
 
 [logger_mqtt]
-level=DEBUG
+level=INFO
-handlers=console_handler,file_handler_name1,file_handler_name2
+handlers=console_handler,file_handler_name1
 propagate=0
 qualname=mqtt
 
@@ -43,12 +43,12 @@ qualname=tracer
 
 [handler_console_handler]
 class=StreamHandler
-level=INFO
+level=DEBUG
 formatter=console_formatter
 
 [handler_file_handler_name1]
 class=handlers.TimedRotatingFileHandler
-level=NOTSET
+level=INFO
 formatter=file_formatter
 args=('log/proxy.log', when:='midnight')
 

app/src/messages.py

@@ -18,6 +18,7 @@ def hex_dump_memory(level, info, data, num):
     lines = []
     lines.append(info)
     tracer = logging.getLogger('tracer')
+    if not tracer.isEnabledFor(level): return
 
 
     #data = list((num * ctypes.c_byte).from_address(ptr))
@@ -100,6 +101,13 @@ class Message(metaclass=IterRegistry):
     '''
     Our puplic methods
     '''
+    def close(self) -> None:
+        # we have refernces to methods of this class in self.switch
+        # so we have to erase self.switch, otherwise this instance can't be 
+        # deallocated by the garbage collector ==> we get a memory leak
+        del self.switch 
+        
+
     def read(self) -> None:
         self._read()
         
@@ -186,7 +194,7 @@ class Message(metaclass=IterRegistry):
         self.send_msg_ofs = len (self._send_buffer)
         self._send_buffer += struct.pack(f'!l{len(self.id_str)+1}pBB', 0, self.id_str, ctrl, self.msg_id)
         fnc = self.switch.get(self.msg_id, self.msg_unknown)
-        logger.info(self.__flow_str(self.server_side, 'tx') + f' Ctl: {int(self.ctrl):#02x} Msg: {fnc.__name__!r}' )
+        logger.info(self.__flow_str(self.server_side, 'tx') + f' Ctl: {int(ctrl):#02x} Msg: {fnc.__name__!r}' )
         
     def __finish_send_msg(self) -> None:
         _len = len(self._send_buffer) - self.send_msg_ofs
@@ -287,11 +295,9 @@ class Message(metaclass=IterRegistry):
 
 
     def msg_unknown(self):
+        logger.warning (f"Unknow Msg: ID:{self.msg_id}")       
         self.forward(self._recv_buffer, self.header_len+self.data_len)
 
         
-    def __del__ (self):
-        logger.debug ("Messages __del__")       
-        
 
 

app/src/mqtt.py

@@ -21,8 +21,17 @@ class Mqtt(metaclass=Singleton):
         logger_mqtt.debug(f'MQTT: __init__') 
         loop = asyncio.get_event_loop()
         self.task = loop.create_task(self.__loop())
+        self.ha_restarts = 0        
 
     
+    @property
+    def ha_restarts(self):
+        return self._ha_restarts
+        
+    @ha_restarts.setter
+    def ha_restarts(self, value):
+        self._ha_restarts = value
+            
     def __del__(self):
         logger_mqtt.debug(f'MQTT: __del__')
     
@@ -55,7 +64,11 @@ class Mqtt(metaclass=Singleton):
                     async with self.client.messages() as messages:
                         await self.client.subscribe(f"{ha['auto_conf_prefix']}/status")
                         async for message in messages:
-                            logger_mqtt.info(f'Home-Assistant Status: {message.payload.decode("UTF-8")}')
+                            status = message.payload.decode("UTF-8")
+                            logger_mqtt.info(f'Home-Assistant Status: {status}')
+                            if status == 'online':
+                                self.ha_restarts += 1
+
             except aiomqtt.MqttError:
                 logger_mqtt.info(f"Connection lost; Reconnecting in {interval} seconds ...")
                 await asyncio.sleep(interval)

app/src/proxy.py

@@ -1,43 +0,0 @@
-import asyncio, logging, traceback
-from async_stream import AsyncStream
-
-class Proxy:
-    def __init__ (proxy, reader, writer, addr):
-        proxy.ServerStream = AsyncStream(proxy, reader, writer, addr)
-        proxy.ClientStream = None
-
-    async def server_loop(proxy, addr):
-        logging.info(f'Accept connection from {addr}')        
-        await proxy.ServerStream.loop()
-        logging.info(f'Close server connection {addr}')
-        
-        if proxy.ClientStream:
-            logging.debug ("close client connection")
-            proxy.ClientStream.close()
-        
-    async def client_loop(proxy, addr):
-        await proxy.ClientStream.loop()    
-        logging.info(f'Close client connection {addr}')
-        proxy.ServerStream.remoteStream = None
-        proxy.ClientStream = None
-        
-    async def CreateClientStream (proxy, stream, host, port):
-        addr = (host, port)
-            
-        try:
-            logging.info(f'Connected to {addr}')
-            connect = asyncio.open_connection(host, port)
-            reader, writer = await connect    
-            proxy.ClientStream = AsyncStream(proxy, reader, writer, addr, stream, server_side=False)
-            asyncio.create_task(proxy.client_loop(addr))
-            
-        except ConnectionRefusedError as error:
-            logging.info(f'{error}')
-        except Exception:
-            logging.error(
-                f"Proxy: Exception for {addr}:\n"
-                f"{traceback.format_exc()}")
-        return proxy.ClientStream
-        
-    def __del__ (proxy):
-        logging.debug ("Proxy __del__")

app/src/server.py

@@ -1,8 +1,7 @@
 import logging, asyncio, signal, functools, os
-#from logging.handlers import TimedRotatingFileHandler
 from logging import config
 from async_stream import AsyncStream
-from proxy import Proxy
+from inverter import Inverter
 from config import Config
 from mqtt import Mqtt
 
@@ -11,7 +10,7 @@ async def handle_client(reader, writer):
     '''Handles a new incoming connection and starts an async loop'''
 
     addr = writer.get_extra_info('peername')
-    await Proxy(reader, writer, addr).server_loop(addr) 
+    await Inverter(reader, writer, addr).server_loop(addr) 
 
 
 def handle_SIGTERM(loop):
@@ -33,8 +32,16 @@ def handle_SIGTERM(loop):
     logging.info('Shutdown complete')
     
 
-    
+def get_log_level() -> int:
-
+    '''checks if LOG_LVL is set in the environment and returns the corresponding logging.LOG_LEVEL'''
+    log_level = os.getenv('LOG_LVL', 'INFO')
+    if log_level== 'DEBUG':
+        log_level = logging.DEBUG
+    elif log_level== 'WARN':
+        log_level = logging.WARNING
+    else:
+        log_level = logging.INFO
+    return log_level
 
 
 if __name__ == "__main__":
@@ -42,9 +49,17 @@ if __name__ == "__main__":
     # Setup our daily, rotating logger
     #
     serv_name = os.getenv('SERVICE_NAME', 'proxy')
+    version   = os.getenv('VERSION', 'unknown')
     
     logging.config.fileConfig('logging.ini')
-    logging.info(f'Server "{serv_name}" will be started')
+    logging.info(f'Server "{serv_name} - {version}" will be started')
+    
+    # set lowest-severity for 'root', 'msg', 'conn' and 'data' logger
+    log_level = get_log_level()
+    logging.getLogger().setLevel(log_level)
+    logging.getLogger('msg').setLevel(log_level)
+    logging.getLogger('conn').setLevel(log_level)
+    logging.getLogger('data').setLevel(log_level)
     
     # read config file
     Config.read()    

app/tests/test_infos.py

@@ -12,6 +12,12 @@ def ContrDataSeq(): # Get Time Request message
     msg +=  b'\x49\x00\x00\x00\x02\x00\x0d\x04\x08\x49\x00\x00\x00\x00\x00\x07\xa1\x84\x49\x00\x00\x00\x01\x00\x0c\x50\x59\x49\x00\x00\x00\x4c\x00\x0d\x1f\x60\x49\x00\x00\x00\x00'
     return msg
 
+@pytest.fixture
+def InvDataSeq(): # Data indication from the controller
+    msg =   b'\x00\x00\x00\x06\x00\x00\x00\x0a\x54\x08\x4d\x69\x63\x72\x6f\x69\x6e\x76\x00\x00\x00\x14\x54\x04\x54\x53\x55\x4e\x00\x00\x00\x1E\x54\x07\x56\x35\x2e\x30\x2e\x31\x31\x00\x00\x00\x28'
+    msg +=  b'\x54\x10\x54\x31\x37\x45\x37\x33\x30\x37\x30\x32\x31\x44\x30\x30\x36\x41\x00\x00\x00\x32\x54\x0a\x54\x53\x4f\x4c\x2d\x4d\x53\x36\x30\x30\x00\x00\x00\x3c\x54\x05\x41\x2c\x42\x2c\x43'
+    return msg
+
 
 def test_parse_control(ContrDataSeq):
     i = Infos()
@@ -19,37 +25,94 @@ def test_parse_control(ContrDataSeq):
         pass
 
     assert json.dumps(i.db) == json.dumps(
-{"collector": {"Collector_Fw_Version": "RSW_400_V1.00.06", "Chip_Type": "Raymon", "Chip_Model": "RSW-1-10001", "Trace_URL": "t.raymoniot.com", "Logger_URL": "logger.talent-monitoring.com", "Data_Up_Interval": 300}, "env": {"Signal_Strength": 100}, "total": {"Power_On_Time": 29}})        
+{"collector": {"Collector_Fw_Version": "RSW_400_V1.00.06", "Chip_Type": "Raymon", "Chip_Model": "RSW-1-10001", "Trace_URL": "t.raymoniot.com", "Logger_URL": "logger.talent-monitoring.com"}, "controller": {"Signal_Strength": 100, "Power_On_Time": 29, "Data_Up_Interval": 300}})        
 
-def test_build_ha_conf():
+def test_parse_inverter(InvDataSeq):
     i = Infos()
-    d_json, id = next (i.ha_confs(prfx="tsun/garagendach/", snr='123'))
+    for key, result in i.parse (InvDataSeq):
-    assert id == 'out_power_123'
+        pass
-    assert  d_json == json.dumps({"name": "Actual Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Microinverter", "mdl": "MS-600", "ids": ["inverter_123"], "mf": "TSUN", "sa": "", "sw": "0.01", "hw": "Hw0.01"}})
 
-def test_build_ha_conf2():
+    assert json.dumps(i.db) == json.dumps(
+{"inverter": {"Product_Name": "Microinv", "Manufacturer": "TSUN", "Version": "V5.0.11", "Serial_Number": "T17E7307021D006A", "Equipment_Model": "TSOL-MS600"}})
+
+def test_parse_cont_and_invert(ContrDataSeq, InvDataSeq):
+    i = Infos()
+    for key, result in i.parse (ContrDataSeq):
+        pass
+
+    for key, result in i.parse (InvDataSeq):
+        pass
+
+    assert json.dumps(i.db) == json.dumps(
+    {
+"collector": {"Collector_Fw_Version": "RSW_400_V1.00.06", "Chip_Type": "Raymon", "Chip_Model": "RSW-1-10001", "Trace_URL": "t.raymoniot.com", "Logger_URL": "logger.talent-monitoring.com"}, "controller": {"Signal_Strength": 100, "Power_On_Time": 29, "Data_Up_Interval": 300},
+"inverter": {"Product_Name": "Microinv", "Manufacturer": "TSUN", "Version": "V5.0.11", "Serial_Number": "T17E7307021D006A", "Equipment_Model": "TSOL-MS600"}})
+
+
+def test_build_ha_conf1(ContrDataSeq):
     i = Infos()
     tests = 0
-    for d_json, id in i.ha_confs(prfx="tsun/garagendach/", snr='123'):
+    for d_json, comp, id in i.ha_confs(prfx="tsun/garagendach/", snr='123'):
 
         if id == 'out_power_123':
-            assert  d_json == json.dumps({"name": "Actual Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Microinverter", "mdl": "MS-600", "ids": ["inverter_123"], "mf": "TSUN", "sa": "", "sw": "0.01", "hw": "Hw0.01"}})
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Micro Inverter", "sa": "Micro Inverter", "via_device": "controller_123", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
 
         elif id == 'daily_gen_123':
-            assert  d_json == json.dumps({"name": "Daily Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total_increasing", "uniq_id": "daily_gen_123", "val_tpl": "{{value_json['Daily_Generation'] | float}}", "unit_of_meas": "kWh", "dev": {"name": "Microinverter", "mdl": "MS-600", "ids": ["inverter_123"], "mf": "TSUN", "sa": "", "sw": "0.01", "hw": "Hw0.01"}})
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Daily Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total_increasing", "uniq_id": "daily_gen_123", "val_tpl": "{{value_json['Daily_Generation'] | float}}", "unit_of_meas": "kWh", "icon": "mdi:solar-power-variant", "dev": {"name": "Micro Inverter", "sa": "Micro Inverter", "via_device": "controller_123", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
 
         elif id == 'power_pv1_123':
-            assert  d_json == json.dumps({"name": "Power PV1", "stat_t": "tsun/garagendach/input", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "power_pv1_123", "val_tpl": "{{ (value_json['pv1']['Power'] | float)}}", "unit_of_meas": "W", "dev": {"name": "Microinverter", "mdl": "MS-600", "ids": ["inverter_123"], "mf": "TSUN", "sa": "", "sw": "0.01", "hw": "Hw0.01"}})
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/input", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "power_pv1_123", "val_tpl": "{{ (value_json['pv1']['Power'] | float)}}", "unit_of_meas": "W", "icon": "mdi:gauge", "dev": {"name": "Module PV1", "sa": "Module PV1", "via_device": "inverter_123", "ids": ["input_pv1_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
 
-        elif id == 'total_gen_123':
+        elif id == 'power_pv2_123':
-            assert  d_json == json.dumps({"name": "Total Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total", "uniq_id": "total_gen_123", "val_tpl": "{{value_json['Total_Generation'] | float}}", "unit_of_meas": "kWh", "icon": "mdi:solar-power", "dev": {"name": "Microinverter", "mdl": "MS-600", "ids": ["inverter_123"], "mf": "TSUN", "sa": "", "sw": "0.01", "hw": "Hw0.01"}})
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/input", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "power_pv2_123", "val_tpl": "{{ (value_json['pv2']['Power'] | float)}}", "unit_of_meas": "W", "icon": "mdi:gauge", "dev": {"name": "Module PV2", "sa": "Module PV2", "via_device": "inverter_123", "ids": ["input_pv2_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
             tests +=1
-    assert tests==4
 
-def test_build_ha_conf3():
+        elif id == 'signal_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Signal Strength", "stat_t": "tsun/garagendach/controller", "dev_cla": None, "stat_cla": "measurement", "uniq_id": "signal_123", "val_tpl": "{{value_json[\'Signal_Strength\'] | int}}", "unit_of_meas": "%", "icon": "mdi:wifi", "dev": {"name": "Controller", "sa": "Controller", "ids": ["controller_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+    assert tests==5
+
+def test_build_ha_conf2(ContrDataSeq, InvDataSeq):
     i = Infos()
-    for d_json, id in i.ha_confs(prfx="tsun/garagendach/", snr='123'):
+    for key, result in i.parse (ContrDataSeq):
         pass
+
+    for key, result in i.parse (InvDataSeq):
+        pass
+
+    tests = 0
+    for d_json, comp, id in i.ha_confs(prfx="tsun/garagendach/", snr='123'):
+
+        if id == 'out_power_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/grid", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "out_power_123", "val_tpl": "{{value_json['Output_Power'] | float}}", "unit_of_meas": "W", "dev": {"name": "Micro Inverter", "sa": "Micro Inverter", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+
+        elif id == 'daily_gen_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Daily Generation", "stat_t": "tsun/garagendach/total", "dev_cla": "energy", "stat_cla": "total_increasing", "uniq_id": "daily_gen_123", "val_tpl": "{{value_json['Daily_Generation'] | float}}", "unit_of_meas": "kWh", "icon": "mdi:solar-power-variant", "dev": {"name": "Micro Inverter", "sa": "Micro Inverter", "via_device": "controller_123", "mdl": "TSOL-MS600", "mf": "TSUN", "sw": "V5.0.11", "ids": ["inverter_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+
+        elif id == 'power_pv1_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/input", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "power_pv1_123", "val_tpl": "{{ (value_json['pv1']['Power'] | float)}}", "unit_of_meas": "W", "icon": "mdi:gauge", "dev": {"name": "Module PV1", "sa": "Module PV1", "via_device": "inverter_123", "ids": ["input_pv1_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+
+        elif id == 'power_pv2_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Power", "stat_t": "tsun/garagendach/input", "dev_cla": "power", "stat_cla": "measurement", "uniq_id": "power_pv2_123", "val_tpl": "{{ (value_json['pv2']['Power'] | float)}}", "unit_of_meas": "W", "icon": "mdi:gauge", "dev": {"name": "Module PV2", "sa": "Module PV2", "via_device": "inverter_123", "ids": ["input_pv2_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+
+        elif id == 'signal_123':
+            assert comp == 'sensor'
+            assert  d_json == json.dumps({"name": "Signal Strength", "stat_t": "tsun/garagendach/controller", "dev_cla": None, "stat_cla": "measurement", "uniq_id": "signal_123", "val_tpl": "{{value_json[\'Signal_Strength\'] | int}}", "unit_of_meas": "%", "icon": "mdi:wifi", "dev": {"name": "Controller", "sa": "Controller", "mdl": "RSW-1-10001", "mf": "Raymon", "sw": "RSW_400_V1.00.06", "ids": ["controller_123"]}, "o": {"name": "proxy", "sw": "unknown"}})
+            tests +=1
+    assert tests==5