sv/tsun-gen3-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

S allius/issue217 2 (#230)

Browse Source 
* add some reader classes to get the configuration

* adapt unittests

* get config from json or toml file

* loop over all config readers to get the configuration

* rename config test files

* use relative paths for coverage test in vscode

* do not throw an error for missing config files

* remove obsolete tests

* use dotted key notation for pv sub dictonary

* log config reading progress

* remove create_config_toml.py

* remove obsolete tests for the ha_addon

* disable mosquitto tests if the server is down

* ignore main method for test coverage

* increase test coverage

* pytest-cov: use relative_files only on github, so coverage will work with vscode locally

* remove unneeded imports

* add missing test cases

* disable branch coverage, cause its not reachable
This commit is contained in:
Stefan Allius
2024-12-08 13:25:04 +01:00
committed by GitHub
parent ac7b02bde9
commit b335881500
22 changed files with 942 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
app/src/cnf/config.py
View File

@@ -1,24 +1,47 @@
'''Config module handles the proxy configuration in the config.toml file'''
'''Config module handles the proxy configuration'''
import tomllib
import shutil
import logging
from abc import ABC, abstractmethod
from schema import Schema, And, Or, Use, Optional
class ConfigIfc(ABC):
'''Abstract basis class for config readers'''
def __init__(self):
Config.add(self)
@abstractmethod
def get_config(cls) -> dict: # pragma: no cover
def get_config(self) -> dict: # pragma: no cover
'''get the unverified config from the reader'''
pass
@abstractmethod
def descr(self) -> str: # pragma: no cover
'''return a descriction of the source, e.g. the file name'''
pass
def _extend_key(self, conf, key, val):
'''split a dotted dict key into a hierarchical dict tree '''
lst = key.split('.')
d = conf
for i, idx in enumerate(lst, 1): # pragma: no branch
if i == len(lst):
d[idx] = val
break
if idx not in d:
d[idx] = {}
d = d[idx]
class Config():
'''Static class Config is reads and sanitize the config.
'''Static class Config build and sanitize the internal config dictenary.
Read config.toml file and sanitize it with read().
Get named parts of the config with get()'''
act_config = {}
def_config = {}
Using config readers, a partial configuration is added to config.
Config readers are a derivation of the abstract ConfigIfc reader.
When a config reader is instantiated, theits `get_config` method is
called automatically and afterwards the config will be merged.
'''
conf_schema = Schema({
'tsun': {
@@ -34,8 +57,10 @@ class Config():
'mqtt': {
'host': Use(str),
'port': And(Use(int), lambda n: 1024 <= n <= 65535),
'user': And(Use(str), Use(lambda s: s if len(s) > 0 else None)),
'passwd': And(Use(str), Use(lambda s: s if len(s) > 0 else None))
'user': Or(None, And(Use(str),
Use(lambda s: s if len(s) > 0 else None))),
'passwd': Or(None, And(Use(str),
Use(lambda s: s if len(s) > 0 else None)))
},
'ha': {
'auto_conf_prefix': Use(str),
@@ -99,52 +124,74 @@ class Config():
)
@classmethod
def init(cls, ifc: ConfigIfc, path='') -> None | str:
cls.ifc = ifc
cls.act_config = {}
def init(cls, def_reader: ConfigIfc) -> None | str:
'''Initialise the Proxy-Config
Copy the internal default config file into the config directory
and initialise the Config with the default configuration '''
cls.err = None
cls.def_config = {}
return cls.read(path)
try:
# make the default config transparaent by copying it
# in the config.example file
logging.debug('Copy Default Config to config.example.toml')
shutil.copy2("default_config.toml",
"config/config.example.toml")
except Exception:
pass
# read example config file as default configuration
try:
def_config = def_reader.get_config()
cls.def_config = cls.conf_schema.validate(def_config)
logging.info(f'Read from {def_reader.descr()} => ok')
except Exception as error:
cls.err = f'Config.read: {error}'
logging.error(
f"Can't read from {def_reader.descr()} => error\n {error}")
cls.act_config = cls.def_config.copy()
@classmethod
def read(cls, path) -> None | str:
'''Read config file, merge it with the default config
def add(cls, reader: ConfigIfc):
'''Merge the config from the Config Reader into the config
Checks if a default config exists. If no default configuration exists,
the Config.init method has not yet been called.This is normal for the very
first Config Reader which creates the default config and must be ignored
here. The default config reader is handled in the Config.init method'''
if hasattr(cls, 'def_config'):
cls.__parse(reader)
@classmethod
def get_error(cls) -> None | str:
'''return the last error as a string or None if there is no error'''
return cls.err
@classmethod
def __parse(cls, reader) -> None | str:
'''Read config from the reader, merge it with the default config
and sanitize the result'''
err = None
config = {}
logger = logging.getLogger('data')
res = 'ok'
try:
# read example config file as default configuration
cls.def_config = {}
with open(f"{path}default_config.toml", "rb") as f:
def_config = tomllib.load(f)
cls.def_config = cls.conf_schema.validate(def_config)
# overwrite the default values, with values from
# the config.toml file
usr_config = cls.ifc.get_config()
# merge the default and the user config
config = def_config.copy()
rd_config = reader.get_config()
config = cls.act_config.copy()
for key in ['tsun', 'solarman', 'mqtt', 'ha', 'inverters',
'gen3plus']:
if key in usr_config:
config[key] |= usr_config[key]
try:
cls.act_config = cls.conf_schema.validate(config)
except Exception as error:
err = f'Config.read: {error}'
logging.error(err)
# logging.debug(f'Readed config: "{cls.act_config}" ')
if key in rd_config:
config[key] = config[key] | rd_config[key]
cls.act_config = cls.conf_schema.validate(config)
except FileNotFoundError:
res = 'n/a'
except Exception as error:
err = f'Config.read: {error}'
logger.error(err)
cls.act_config = {}
cls.err = f'error: {error}'
logging.error(
f"Can't read from {reader.descr()} => error\n {error}")
return err
logging.info(f'Read from {reader.descr()} => {res}')
return cls.err
@classmethod
def get(cls, member: str = None):

34
app/src/cnf/config_ifc_proxy.py
View File

@@ -1,34 +0,0 @@
'''Config module handles the proxy configuration in the config.toml file'''
import shutil
import tomllib
import logging
from cnf.config import ConfigIfc
class ConfigIfcProxy(ConfigIfc):
def __init__(self): # pragma: no cover
try:
# make the default config transparaent by copying it
# in the config.example file
logging.info('Copy Default Config to config.example.toml')
shutil.copy2("default_config.toml",
"config/config.example.toml")
except Exception:
pass
def get_config(self, cnf_file="config/config.toml") -> dict:
usr_config = {}
try:
with open(cnf_file, "rb") as f:
usr_config = tomllib.load(f)
except Exception as error:
err = f'Config.read: {error}'
logging.error(err)
logging.info(
'\n To create the missing config.toml file, '
'you can rename the template config.example.toml\n'
' and customize it for your scenario.\n')
return usr_config

25
app/src/cnf/config_read_env.py Normal file
View File

@@ -0,0 +1,25 @@
'''Config Reader module which handles config values from the environment'''
import os
from cnf.config import ConfigIfc
class ConfigReadEnv(ConfigIfc):
'''Reader for environment values of the configuration'''
def get_config(self) -> dict:
conf = {}
data = [
('mqtt.host', 'MQTT_HOST'),
('mqtt.port', 'MQTT_PORT'),
('mqtt.user', 'MQTT_USER'),
('mqtt.passwd', 'MQTT_PASSWORD'),
]
for key, env_var in data:
val = os.getenv(env_var)
if val:
self._extend_key(conf, key, val)
return conf
def descr(self):
return "Read environment"

46
app/src/cnf/config_read_json.py Normal file
View File

@@ -0,0 +1,46 @@
'''Config Reader module which handles *.json config files'''
import json
from cnf.config import ConfigIfc
class ConfigReadJson(ConfigIfc):
'''Reader for json config files'''
def __init__(self, cnf_file='/data/options.json'):
'''Read a json file and add the settings to the config'''
if not isinstance(cnf_file, str):
return
self.cnf_file = cnf_file
super().__init__()
def convert_inv(self, conf, inv):
if 'serial' in inv:
snr = inv['serial']
del inv['serial']
conf[snr] = {}
for key, val in inv.items():
self._extend_key(conf[snr], key, val)
def convert_inv_arr(self, conf, key, val: list):
if key not in conf:
conf[key] = {}
for elm in val:
self.convert_inv(conf[key], elm)
def convert_to_obj(self, data):
conf = {}
for key, val in data.items():
if key == 'inverters' and isinstance(val, list):
self.convert_inv_arr(conf, key, val)
else:
self._extend_key(conf, key, val)
return conf
def get_config(self) -> dict:
with open(self.cnf_file) as f:
data = json.load(f)
return self.convert_to_obj(data)
def descr(self):
return self.cnf_file

21
app/src/cnf/config_read_toml.py Normal file
View File

@@ -0,0 +1,21 @@
'''Config Reader module which handles *.toml config files'''
import tomllib
from cnf.config import ConfigIfc
class ConfigReadToml(ConfigIfc):
'''Reader for toml config files'''
def __init__(self, cnf_file):
'''Read a toml file and add the settings to the config'''
if not isinstance(cnf_file, str):
return
self.cnf_file = cnf_file
super().__init__()
def get_config(self) -> dict:
with open(self.cnf_file, "rb") as f:
return tomllib.load(f)
def descr(self):
return self.cnf_file

37
app/src/server.py
View File

@@ -2,6 +2,7 @@ import logging
import asyncio
import signal
import os
import argparse
from asyncio import StreamReader, StreamWriter
from aiohttp import web
from logging import config # noqa F401
@@ -11,7 +12,9 @@ from gen3.inverter_g3 import InverterG3
from gen3plus.inverter_g3p import InverterG3P
from scheduler import Schedule
from cnf.config import Config
from cnf.config_ifc_proxy import ConfigIfcProxy
from cnf.config_read_env import ConfigReadEnv
from cnf.config_read_toml import ConfigReadToml
from cnf.config_read_json import ConfigReadJson
from modbus_tcp import ModbusTcp
routes = web.RouteTableDef()
@@ -117,6 +120,8 @@ def get_log_level() -> int:
'''checks if LOG_LVL is set in the environment and returns the
corresponding logging.LOG_LEVEL'''
log_level = os.getenv('LOG_LVL', 'INFO')
logging.info(f"LOG_LVL : {log_level}")
if log_level == 'DEBUG':
log_level = logging.DEBUG
elif log_level == 'WARN':
@@ -126,7 +131,17 @@ def get_log_level() -> int:
return log_level
if __name__ == "__main__":
if __name__ == "__main__": # pragma: no cover
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--config_path', type=str,
default='./config/',
help='set path for the configuration files')
parser.add_argument('-j', '--json_config', type=str,
help='read user config from json-file')
parser.add_argument('-t', '--toml_config', type=str,
help='read user config from toml-file')
parser.add_argument('--add_on', action='store_true')
args = parser.parse_args()
#
# Setup our daily, rotating logger
#
@@ -135,9 +150,14 @@ if __name__ == "__main__":
logging.config.fileConfig('logging.ini')
logging.info(f'Server "{serv_name} - {version}" will be started')
logging.info(f"AddOn: {args.add_on}")
logging.info(f"config_path: {args.config_path}")
logging.info(f"json_config: {args.json_config}")
logging.info(f"toml_config: {args.toml_config}")
log_level = get_log_level()
logging.info('******')
# set lowest-severity for 'root', 'msg', 'conn' and 'data' logger
log_level = get_log_level()
logging.getLogger().setLevel(log_level)
logging.getLogger('msg').setLevel(log_level)
logging.getLogger('conn').setLevel(log_level)
@@ -150,9 +170,18 @@ if __name__ == "__main__":
asyncio.set_event_loop(loop)
# read config file
ConfigErr = Config.init(ConfigIfcProxy())
Config.init(ConfigReadToml("default_config.toml"))
ConfigReadEnv()
ConfigReadJson(args.config_path + "config.json")
ConfigReadToml(args.config_path + "config.toml")
ConfigReadJson(args.json_config)
ConfigReadToml(args.toml_config)
ConfigErr = Config.get_error()
if ConfigErr is not None:
logging.info(f'ConfigErr: {ConfigErr}')
logging.info('******')
Proxy.class_init()
Schedule.start()
ModbusTcp(loop)