Add on (#212)

* added service to transfer Add-on config from options.json to config.toml

* added feature to get MQTT config from Homeassistant

current version is MVP. can run as Home Assistant Add-On, config.toml is automatically created from option parameters in the add-on configuration tab.

* fix pylance and flake8 warnings

* prepare building a ha addon

- move build script into root dir
- cp source files in addon build-tree

* ignore proxy source files in addon build tree

* move proxy source files in own directory

* remove duplicates source files from repro

* check for a valis SONAR_TOKEN

* rename add_on path

* prepare for unittests and coverage measurement

* move file cause of the changes pathname

* move the proxy dir to /home/proxy

* build addon with make now

* remove duplicated requirements.txt file from repo

* undo changes

---------

Co-authored-by: Michael Metz <michael.metz@siemens.com>
Co-authored-by: Stefan Allius <stefan.allius@t-online.de>

.github/workflows/python-app.yml

@@ -26,6 +26,7 @@ permissions:
 
 env:
   TZ: "Europe/Berlin"
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
 jobs:
   build:
@@ -53,13 +54,13 @@ jobs:
         flake8 --exit-zero --ignore=C901,E121,E123,E126,E133,E226,E241,E242,E704,W503,W504,W505 --format=pylint --output-file=output_flake.txt --exclude=*.pyc app/src/
     - name: Test with pytest
       run: |
-        python -m pytest app --cov=app/src --cov-report=xml
+        python -m pytest app ha_addon --cov=app/src --cov=ha_addon/rootfs/home --cov-report=xml
         coverage report
     - name: Analyze with SonarCloud
+      if: ${{ env.SONAR_TOKEN != 0 }}
       uses: SonarSource/sonarcloud-github-action@v3.1.0
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
       with:
         projectBaseDir: .
         args:

.gitignore

@@ -3,9 +3,12 @@ __pycache__
 bin/**
 mosquitto/**
 homeassistant/**
+ha_addon/rootfs/home/proxy/*
+ha_addon/rootfs/requirements.txt
 tsun_proxy/**
 Doku/**
 .DS_Store
 .coverage
 .env
+.venv
 coverage.xml

.vscode/settings.json

@@ -3,9 +3,11 @@
         "-vv", 
         "app",
         "--cov=app/src",
+        "--cov=ha_addon/rootfs/home",
         "--cov-report=xml",
         "--cov-report=html",
-        "system_tests"
+        "system_tests",
+        "ha_addon"
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,

Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+# debug dev:
+# 	$(MAKE) -C app $@
+
+clean build:
+	$(MAKE) -C ha_addon $@ 

ha_addon/Dockerfile

@@ -0,0 +1,97 @@
+
+############################################################################
+#
+# TSUN Proxy
+# Homeassistant Add-on
+# 
+# based on https://github.com/s-allius/tsun-gen3-proxy/tree/main
+#
+############################################################################
+
+
+######################
+# 1 Build Image      #
+######################
+
+# opt for suitable build base. I opted for the recommended hassio-addon base
+
+#ARG BUILD_FROM="ghcr.io/hassio-addons/debian-base:latest"
+ARG BUILD_FROM="ghcr.io/hassio-addons/base:latest"
+FROM $BUILD_FROM
+
+
+#######################
+# 2 Modify Image      #
+#######################
+
+
+
+
+#######################
+# 3 Install apps      #
+#######################
+
+
+
+# Installiere Python, pip und virtuelle Umgebungstools
+RUN apk add --no-cache python3 py3-pip py3-virtualenv
+
+# Erstelle ein virtuelles Umfeld und aktiviere es
+RUN python3 -m venv /opt/venv
+
+RUN . /opt/venv/bin/activate
+
+# Stelle sicher, dass das Add-on das virtuelle Umfeld nutzt
+ENV PATH="/opt/venv/bin:$PATH"
+
+
+
+#######################
+# 4 Install libraries #
+#######################
+
+
+# Kopiere die requirements.txt Datei in das Image
+COPY rootfs/requirements.txt /tmp/requirements.txt
+
+#  installiere die Pakete aus requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+
+#######################
+# 5 copy data         #
+#######################
+
+
+# Add rootfs
+COPY rootfs/ /
+
+# make run.sh executable
+RUN chmod a+x /run.sh
+
+
+# no idea whether needed or not
+ENV SERVICE_NAME="tsun-proxy"
+ENV UID=1000
+ENV GID=1000
+ENV VERSION="0.0"
+
+
+
+#######################
+# 6 run app           #
+#######################
+
+
+# command to run on container start
+CMD [ "/run.sh" ]
+
+
+
+#######################
+
+# Labels
+LABEL  \
+    io.hass.version="VERSION"  \
+    io.hass.type="addon"  \
+    io.hass.arch="armhf|aarch64|i386|amd64"

ha_addon/Makefile

@@ -0,0 +1,45 @@
+SHELL = /bin/sh
+
+# Folders
+SRC=../app
+SRC_PROXY=$(SRC)/src
+CNF_PROXY=$(SRC)/config
+
+DST=rootfs
+DST_PROXY=$(DST)/home/proxy
+
+# collect source files 
+SRC_FILES := $(wildcard $(SRC_PROXY)/*.py)\
+ $(wildcard $(SRC_PROXY)/*.ini)\
+ $(wildcard $(SRC_PROXY)/gen3/*.py)\
+ $(wildcard $(SRC_PROXY)/gen3plus/*.py)
+CNF_FILES := $(wildcard $(CNF_PROXY)/*.toml)
+
+# determine destination files
+TARGET_FILES = $(SRC_FILES:$(SRC_PROXY)/%=$(DST_PROXY)/%)
+CONFIG_FILES = $(CNF_FILES:$(CNF_PROXY)/%=$(DST_PROXY)/%)
+
+build: rootfs
+
+clean:
+	rm -r -f $(DST_PROXY)
+	rm -f $(DST)/requirements.txt
+
+rootfs: $(TARGET_FILES) $(CONFIG_FILES) $(DST)/requirements.txt
+
+.PHONY: build clean rootfs
+
+
+$(CONFIG_FILES): $(DST_PROXY)/% : $(CNF_PROXY)/%
+	@echo Copy $< to $@
+	@mkdir -p $(@D)
+	@cp $< $@
+
+$(TARGET_FILES): $(DST_PROXY)/% : $(SRC_PROXY)/%
+	@echo Copy $< to $@
+	@mkdir -p $(@D)
+	@cp $< $@
+
+$(DST)/requirements.txt : $(SRC)/requirements.txt
+	@echo Copy $< to $@
+	@cp $< $@

ha_addon/config.yaml

@@ -0,0 +1,78 @@
+name: "TSUN-Proxy"
+description: "MQTT Proxy for TSUN Photovoltaic Inverters"
+version: "0.0.7"
+slug: "tsun-proxy"
+init: false
+arch:
+  - aarch64
+  - amd64
+  - armhf
+  - armv7
+  - i386
+startup: services
+homeassistant_api: true
+services:
+  - mqtt:want
+ports:
+  8127/tcp: 8127
+  5005/tcp: 5005
+  10000/tcp: 10000
+
+# Definition of parameters in the configuration tab of the addon
+# parameters are available within the container as /data/options.json
+# and should become picked up by the proxy - current workarround as a transfer script
+# TODO: add further schema for remaining config parameters
+# TODO: implement direct reading of the configuration file
+schema:
+  inverters:
+    - serial: str
+      node_id: str
+      suggested_area: str
+      modbus_polling: bool
+      #strings:                   # leider funktioniert es nicht die folgenden 3 parameter im schema aufzulisten. möglicherweise wird die verschachtelung nicht unterstützt.
+      #  - string: str
+      #    type: str
+      #    manufacturer: str
+      # daher diese variante
+      pv1_manufacturer: str
+      pv1_type: str
+      pv2_manufacturer: str
+      pv2_type: str
+  tsun.enabled: bool
+  solarman.enabled: bool
+  inverters.allow_all: bool
+  # optionale parameter
+  # TODO besser strukturieren und vervollständigen
+  mqtt.host: str?
+  mqtt.port: int?
+  mqtt.user: str?
+  mqtt.passwd: password?
+  ha.auto_conf_prefix: str? # suggeriert optionale konfigurationsoption -> es darf jedoch kein default unter "options" angegeben werden
+  ha.discovery_prefix: str? # dito
+  ha.entity_prefix: str? #dito
+  ha.proxy_node_id: str? #dito
+  ha.proxy_unique_id: str? #dito
+
+# set default options for mandatory parameters
+# for optional parameters do not define any default value in the options dictionary.
+# If any default value is given, the option becomes a required value.
+options:
+  inverters:
+    - serial: R17E760702080400
+      node_id: PV-Garage
+      suggested_area: Garage
+      modbus_polling: false
+      #strings:
+      #  - string: PV1
+      #    type: SF-M18/144550
+      #    manufacturer: Shinefar
+      #  - string: PV2
+      #    type: SF-M18/144550
+      #    manufacturer: Shinefar
+      pv1_manufacturer: Shinefar
+      pv1_type: SF-M18/144550
+      pv2_manufacturer: Shinefar
+      pv2_type: SF-M18/144550
+  tsun.enabled: true # set default
+  solarman.enabled: true # set default
+  inverters.allow_all: false # set default

ha_addon/rootfs/home/create_config_toml.py

@@ -0,0 +1,65 @@
+import json
+import os
+
+# Dieses file übernimmt die Add-On Konfiguration und schreibt sie in die
+# Konfigurationsdatei des tsun-proxy
+# Die Addon Konfiguration wird in der Datei /data/options.json bereitgestellt
+# Die Konfiguration wird in der Datei /home/proxy/config/config.toml
+# gespeichert
+
+# Übernehme die Umgebungsvariablen
+# alternativ kann auch auf die homeassistant supervisor API zugegriffen werden
+
+data = {}
+data['mqtt.host'] = os.getenv('MQTT_HOST')
+data['mqtt.port'] = os.getenv('MQTT_PORT')
+data['mqtt.user'] = os.getenv('MQTT_USER')
+data['mqtt.passwd'] = os.getenv('MQTT_PASSWORD')
+
+
+# Lese die Add-On Konfiguration aus der Datei /data/options.json
+with open('/data/options.json') as json_file:
+    # with open('options.json') as json_file:
+    options_data = json.load(json_file)
+    data.update(options_data)
+
+
+# Schreibe die Add-On Konfiguration in die Datei /home/proxy/config/config.toml    # noqa: E501
+with open('/home/proxy/config/config.toml', 'w+') as f:
+    # with open('./config/config.toml', 'w+') as f:
+    f.write(f"""
+mqtt.host    = '{data.get('mqtt.host')}' # URL or IP address of the mqtt broker
+mqtt.port    = {data.get('mqtt.port')}
+mqtt.user    = '{data.get('mqtt.user')}'
+mqtt.passwd  = '{data.get('mqtt.passwd')}'
+
+
+ha.auto_conf_prefix = '{data.get('ha.auto_conf_prefix', 'homeassistant')}'     # MQTT prefix for subscribing for homeassistant status updates    # noqa: E501
+ha.discovery_prefix = '{data.get('ha.discovery_prefix', 'homeassistant')}'     # MQTT prefix for discovery topic                                 # noqa: E501
+ha.entity_prefix    = '{data.get('ha.entity_prefix', 'tsun')}'              # MQTT topic prefix for publishing inverter values                   # noqa: E501
+ha.proxy_node_id    = '{data.get('ha.proxy_node_id', 'proxy')}'             # MQTT node id, for the proxy_node_id
+ha.proxy_unique_id  = '{data.get('ha.proxy_unique_id', 'P170000000000001')}'  # MQTT unique id, to identify a proxy instance
+
+
+tsun.enabled = {str(data.get('tsun.enabled', True)).lower()}
+tsun.host    = '{data.get('tsun.host', 'logger.talent-monitoring.com')}'
+tsun.port    = {data.get('tsun.port', 5005)}
+
+
+solarman.enabled = {str(data.get('solarman.enabled', True)).lower()}
+solarman.host    = '{data.get('solarman.host', 'iot.talent-monitoring.com')}'
+solarman.port    = {data.get('solarman.port', 10000)}
+
+
+inverters.allow_all = {str(data.get('inverters.allow_all', False)).lower()}
+""")
+
+    for inverter in data['inverters']:
+        f.write(f"""
+[inverters."{inverter['serial']}"]
+node_id = '{inverter['node_id']}'
+suggested_area = '{inverter['suggested_area']}'
+modbus_polling = {str(inverter['modbus_polling']).lower()}
+pv1 = {{type = '{inverter['pv1_type']}', manufacturer = '{inverter['pv1_manufacturer']}'}}   # Optional, PV module descr    # noqa: E501
+pv2 = {{type = '{inverter['pv2_type']}', manufacturer = '{inverter['pv2_manufacturer']}'}}   # Optional, PV module descr    # noqa: E501
+""")

ha_addon/rootfs/home/options.json

@@ -0,0 +1,19 @@
+
+
+{
+  "inverters": [
+    {
+      "serial": "R17E760702080400",
+      "node_id": "PV-Garage",
+      "suggested_area": "Garage",
+      "modbus_polling": false,
+      "pv1_manufacturer": "Shinefar",
+      "pv1_type": "SF-M18/144550",
+      "pv2_manufacturer": "Shinefar",
+      "pv2_type": "SF-M18/144550"
+    }
+  ],
+  "tsun.enabled": false,
+  "solarman.enabled": false,
+  "inverters.allow_all": false
+}

ha_addon/rootfs/run.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/with-contenv bashio
+
+echo "Add-on environment started"
+
+echo "check for Home Assistant MQTT"
+MQTT_HOST=$(bashio::services mqtt "host")
+MQTT_PORT=$(bashio::services mqtt "port")
+MQTT_USER=$(bashio::services mqtt "username")
+MQTT_PASSWORD=$(bashio::services mqtt "password")
+
+# wenn host gefunden wurde, dann nachricht ausgeben
+if [ -z "$MQTT_HOST" ]; then
+    echo "MQTT not found"
+else
+    echo "MQTT found"
+    export MQTT_HOST
+    export MQTT_PORT
+    export MQTT_USER
+    export MQTT_PASSWORD
+fi
+
+
+
+cd /home || exit
+
+
+echo "Erstelle config.toml"
+python3 create_config_toml.py
+
+
+cd /home/proxy || exit
+
+echo "Starte Webserver"
+python3 server.py

ha_addon/tests/test_create_config_toml.py

@@ -0,0 +1,6 @@
+# test_with_pytest.py
+# import ha_addon.rootfs.home.create_config_toml
+
+
+def test_config():
+    pass

ha_addon/translations/en.yaml

@@ -0,0 +1,66 @@
+---
+configuration:
+  inverters:
+    name: Inverters
+    description: >-
+      For each GEN3 inverter, the serial number of the inverter must be mapped to an MQTT
+      definition. To do this, the corresponding configuration block is started with
+      <16-digit serial number> so that all subsequent parameters are assigned
+      to this inverter. Further inverter-specific parameters (e.g. polling mode) can be set
+      in the configuration block
+
+      The serial numbers of all GEN3 inverters start with `R17`!
+
+      node_id            # MQTT replacement for inverters serial number
+      suggested_area     # suggested installation area for home-assistant
+      modbus_polling     # Disable optional MODBUS polling
+      pv1                # Optional, PV module descr
+      pv2                # Optional, PV module descr
+
+  tsun.enabled:
+    name: Connection to TSUN Cloud
+    description: >-
+      disable connecting to the tsun cloud avoids updates.
+      The Inverter become isolated from Internet if switched on.
+  solarman.enabled:
+    name: Connection to Solarman Cloud
+    description: >-
+      disables connecting to the Solarman cloud avoids updates.
+      The Inverter become isolated from Internet if switched on.
+  inverters.allow_all:
+    name: Allow all connections from all inverters
+    description: >-
+      The proxy only usually accepts connections from known inverters.
+      This can be switched off for test purposes and unknown serial
+      numbers are also accepted.
+  mqtt.host:
+    name: MQTT Broker Host
+    description: >-
+      Hostname or IP address of the MQTT broker. if not set, the addon will try to connect to the Home Assistant MQTT broker
+  mqtt.port:
+    name: MQTT Broker Port
+    description: >-
+      Port of the MQTT broker. if not set, the addon will try to connect to the Home Assistant MQTT broker
+  mqtt.user:
+    name: MQTT Broker User
+    description: >-
+      User for the MQTT broker. if not set, the addon will try to connect to the Home Assistant MQTT broker
+  mqtt.passwd:
+    name: MQTT Broker Password
+    description: >-
+      Password for the MQTT broker. if not set, the addon will try to connect to the Home Assistant MQTT broker
+  ha.auto_conf_prefix:
+    name: MQTT prefix for subscribing for homeassistant status updates
+  ha.discovery_prefix:
+    name: MQTT prefix for discovery topic
+  ha.entity_prefix:
+    name: MQTT topic prefix for publishing inverter values
+  ha.proxy_node_id:
+    name: MQTT node id, for the proxy_node_id
+  ha.proxy_unique_id:
+    name: MQTT unique id, to identify a proxy instance
+
+network:
+  8127/tcp: x...
+  5005/tcp: listening Port for TSUN GEN3 Devices
+  10000/tcp: listening Port for TSUN GEN3PLUS Devices