S allius/issue205 (#207)

* Add SolarmanEmu class

* Forward a device ind to establish the EMU connection

* Move SolarmanEmu class into a dedicated file

* Add cloud connection counter

* Send inverter data in emulator mode

* Improve emulator mode

- parse more values from MQTT register
- differ between inverter and logger serial no

* Add some unit tests for SolarmanEmu class

* Send seconds since last sync in data packets

* Increase test coverage
This commit is contained in:
Stefan Allius
2024-11-13 22:03:28 +01:00
committed by GitHub
parent 78a35b5513
commit 5ced5ff06a
19 changed files with 767 additions and 256 deletions

View File

@@ -10,14 +10,14 @@ if __name__ == "app.src.gen3plus.solarman_v5":
from app.src.modbus import Modbus
from app.src.config import Config
from app.src.gen3plus.infos_g3p import InfosG3P
from app.src.infos import Register
from app.src.infos import Register, Fmt
else: # pragma: no cover
from async_ifc import AsyncIfc
from messages import hex_dump_memory, Message, State
from config import Config
from modbus import Modbus
from gen3plus.infos_g3p import InfosG3P
from infos import Register
from infos import Register, Fmt
logger = logging.getLogger('msg')
@@ -48,7 +48,209 @@ class Sequence():
return f'{self.rcv_idx:02x}:{self.snd_idx:02x}'
class SolarmanV5(Message):
class SolarmanBase(Message):
def __init__(self, addr, ifc: "AsyncIfc", server_side: bool,
_send_modbus_cb, mb_timeout: int):
super().__init__('G3P', ifc, server_side, _send_modbus_cb,
mb_timeout)
ifc.rx_set_cb(self.read)
ifc.prot_set_timeout_cb(self._timeout)
ifc.prot_set_init_new_client_conn_cb(self._init_new_client_conn)
ifc.prot_set_update_header_cb(self.__update_header)
self.addr = addr
self.conn_no = ifc.get_conn_no()
self.header_len = 11 # overwrite construcor in class Message
self.control = 0
self.seq = Sequence(server_side)
self.snr = 0
self.time_ofs = 0
def read(self) -> float:
'''process all received messages in the _recv_buffer'''
self._read()
while True:
if not self.header_valid:
self.__parse_header(self.ifc.rx_peek(),
self.ifc.rx_len())
if self.header_valid and self.ifc.rx_len() >= \
(self.header_len + self.data_len+2):
self.__process_complete_received_msg()
self.__flush_recv_msg()
else:
return 0 # wait 0s before sending a response
'''
Our public methods
'''
def _flow_str(self, server_side: bool, type: str): # noqa: F821
switch = {
'rx': ' <',
'tx': ' >',
'forwrd': '<< ',
'drop': ' xx',
'rxS': '> ',
'txS': '< ',
'forwrdS': ' >>',
'dropS': 'xx ',
}
if server_side:
type += 'S'
return switch.get(type, '???')
def get_fnc_handler(self, ctrl):
fnc = self.switch.get(ctrl, self.msg_unknown)
if callable(fnc):
return fnc, repr(fnc.__name__)
else:
return self.msg_unknown, repr(fnc)
def _build_header(self, ctrl) -> None:
'''build header for new transmit message'''
self.send_msg_ofs = self.ifc.tx_len()
self.ifc.tx_add(struct.pack(
'<BHHHL', 0xA5, 0, ctrl, self.seq.get_send(), self.snr))
_fnc, _str = self.get_fnc_handler(ctrl)
logger.info(self._flow_str(self.server_side, 'tx') +
f' Ctl: {int(ctrl):#04x} Msg: {_str}')
def _finish_send_msg(self) -> None:
'''finish the transmit message, set lenght and checksum'''
_len = self.ifc.tx_len() - self.send_msg_ofs
struct.pack_into('<H', self.ifc.tx_peek(), self.send_msg_ofs+1,
_len-11)
check = sum(self.ifc.tx_peek()[
self.send_msg_ofs+1:self.send_msg_ofs + _len]) & 0xff
self.ifc.tx_add(struct.pack('<BB', check, 0x15)) # crc & stop
def _timestamp(self):
# utc as epoche
return int(time.time()) # pragma: no cover
def _emu_timestamp(self):
'''timestamp for an emulated inverter (realtime - 1 day)'''
one_day = 24*60*60
return self._timestamp()-one_day
'''
Our private methods
'''
def __update_header(self, _forward_buffer):
'''update header for message before forwarding,
set sequence and checksum'''
_len = len(_forward_buffer)
ofs = 0
while ofs < _len:
result = struct.unpack_from('<BH', _forward_buffer, ofs)
data_len = result[1] # len of variable id string
struct.pack_into('<H', _forward_buffer, ofs+5,
self.seq.get_send())
check = sum(_forward_buffer[ofs+1:ofs+data_len+11]) & 0xff
struct.pack_into('<B', _forward_buffer, ofs+data_len+11, check)
ofs += (13 + data_len)
def __process_complete_received_msg(self):
log_lvl = self.log_lvl.get(self.control, logging.WARNING)
if callable(log_lvl):
log_lvl = log_lvl()
self.ifc.rx_log(log_lvl, f'Received from {self.addr}:')
# self._recv_buffer, self.header_len +
# self.data_len+2)
if self.__trailer_is_ok(self.ifc.rx_peek(), self.header_len
+ self.data_len + 2):
if self.state == State.init:
self.state = State.received
self._set_serial_no(self.snr)
self.__dispatch_msg()
def __parse_header(self, buf: bytes, buf_len: int) -> None:
if (buf_len < self.header_len): # enough bytes for complete header?
return
result = struct.unpack_from('<BHHHL', buf, 0)
# store parsed header values in the class
start = result[0] # start byte
self.data_len = result[1] # len of variable id string
self.control = result[2]
self.seq.set_recv(result[3])
self.snr = result[4]
if start != 0xA5:
hex_dump_memory(logging.ERROR,
'Drop packet w invalid start byte from'
f' {self.addr}:', buf, buf_len)
self.inc_counter('Invalid_Msg_Format')
# erase broken recv buffer
self.ifc.rx_clear()
return
self.header_valid = True
def __trailer_is_ok(self, buf: bytes, buf_len: int) -> bool:
crc = buf[self.data_len+11]
stop = buf[self.data_len+12]
if stop != 0x15:
hex_dump_memory(logging.ERROR,
'Drop packet w invalid stop byte from '
f'{self.addr}:', buf, buf_len)
self.inc_counter('Invalid_Msg_Format')
if self.ifc.rx_len() > (self.data_len+13):
next_start = buf[self.data_len+13]
if next_start != 0xa5:
# erase broken recv buffer
self.ifc.rx_clear()
return False
check = sum(buf[1:buf_len-2]) & 0xff
if check != crc:
self.inc_counter('Invalid_Msg_Format')
logger.debug(f'CRC {int(crc):#02x} {int(check):#08x}'
f' Stop:{int(stop):#02x}')
# start & stop byte are valid, discard only this message
return False
return True
def __flush_recv_msg(self) -> None:
self.ifc.rx_get(self.header_len + self.data_len+2)
self.header_valid = False
def __dispatch_msg(self) -> None:
_fnc, _str = self.get_fnc_handler(self.control)
if self.unique_id:
logger.info(self._flow_str(self.server_side, 'rx') +
f' Ctl: {int(self.control):#04x}' +
f' Msg: {_str}')
_fnc()
else:
logger.info(self._flow_str(self.server_side, 'drop') +
f' Ctl: {int(self.control):#04x}' +
f' Msg: {_str}')
'''
Message handler methods
'''
def msg_response(self):
data = self.ifc.rx_peek()[self.header_len:]
result = struct.unpack_from('<BBLL', data, 0)
ftype = result[0] # always 2
valid = result[1] == 1 # status
ts = result[2]
set_hb = result[3] # always 60 or 120
logger.debug(f'ftype:{ftype} accepted:{valid}'
f' ts:{ts:08x} nextHeartbeat: {set_hb}s')
dt = datetime.fromtimestamp(ts)
logger.debug(f'ts: {dt.strftime("%Y-%m-%d %H:%M:%S")}')
return ftype, valid, ts, set_hb
class SolarmanV5(SolarmanBase):
AT_CMD = 1
MB_RTU_CMD = 2
MB_CLIENT_DATA_UP = 30
@@ -58,24 +260,15 @@ class SolarmanV5(Message):
def __init__(self, addr, ifc: "AsyncIfc",
server_side: bool, client_mode: bool):
super().__init__('G3P', ifc, server_side, self.send_modbus_cb,
super().__init__(addr, ifc, server_side, self.send_modbus_cb,
mb_timeout=8)
ifc.rx_set_cb(self.read)
ifc.prot_set_timeout_cb(self._timeout)
ifc.prot_set_init_new_client_conn_cb(self._init_new_client_conn)
ifc.prot_set_update_header_cb(self._update_header)
self.addr = addr
self.conn_no = ifc.get_conn_no()
self.header_len = 11 # overwrite construcor in class Message
self.control = 0
self.seq = Sequence(server_side)
self.snr = 0
self.db = InfosG3P(client_mode)
self.time_ofs = 0
self.forward_at_cmd_resp = False
self.no_forwarding = False
'''not allowed to connect to TSUN cloud by connection type'''
self.establish_inv_emu = False
'''create an Solarman EMU instance to send data to the TSUN cloud'''
self.switch = {
0x4210: self.msg_data_ind, # real time data
@@ -135,7 +328,7 @@ class SolarmanV5(Message):
if 'at_acl' in g3p_cnf: # pragma: no cover
self.at_acl = g3p_cnf['at_acl']
self.sensor_list = 0x0000
self.sensor_list = 0
'''
Our puplic methods
@@ -150,16 +343,24 @@ class SolarmanV5(Message):
super().close()
async def send_start_cmd(self, snr: int, host: str,
forward: bool,
start_timeout=MB_CLIENT_DATA_UP):
self.no_forwarding = True
self.establish_inv_emu = forward
self.snr = snr
self.__set_serial_no(snr)
self._set_serial_no(snr)
self.mb_timeout = start_timeout
self.db.set_db_def_value(Register.IP_ADDRESS, host)
self.db.set_db_def_value(Register.POLLING_INTERVAL,
self.mb_timeout)
self.db.set_db_def_value(Register.DATA_UP_INTERVAL,
300)
self.db.set_db_def_value(Register.COLLECT_INTERVAL,
1)
self.db.set_db_def_value(Register.HEARTBEAT_INTERVAL,
120)
self.db.set_db_def_value(Register.SENSOR_LIST,
Fmt.hex4((self.sensor_list, )))
self.new_data['controller'] = True
self.state = State.up
@@ -174,6 +375,15 @@ class SolarmanV5(Message):
self.db.set_db_def_value(Register.POLLING_INTERVAL,
self.mb_timeout)
def establish_emu(self):
_len = 223
build_msg = self.db.build(_len, 0x41, 2)
struct.pack_into(
'<BHHHLBL', build_msg, 0, 0xA5, _len-11, 0x4110,
0, self.snr, 2, self._emu_timestamp())
self.ifc.fwd_add(build_msg)
self.ifc.fwd_add(struct.pack('<BB', 0, 0x15)) # crc & stop
def __set_config_parms(self, inv: dict):
'''init connection with params from the configuration'''
self.node_id = inv['node_id']
@@ -183,7 +393,7 @@ class SolarmanV5(Message):
if self.mb:
self.mb.set_node_id(self.node_id)
def __set_serial_no(self, snr: int):
def _set_serial_no(self, snr: int):
'''check the serial number and configure the inverter connection'''
serial_no = str(snr)
if self.unique_id == serial_no:
@@ -200,7 +410,8 @@ class SolarmanV5(Message):
self.db.set_pv_module_details(inv)
logger.debug(f'SerialNo {serial_no} allowed! area:{self.sug_area}') # noqa: E501
self.db.set_db_def_value(Register.COLLECTOR_SNR, key)
self.db.set_db_def_value(Register.COLLECTOR_SNR, snr)
self.db.set_db_def_value(Register.SERIAL_NUMBER, key)
break
else:
self.node_id = ''
@@ -214,35 +425,6 @@ class SolarmanV5(Message):
self.unique_id = serial_no
def read(self) -> float:
'''process all received messages in the _recv_buffer'''
self._read()
while True:
if not self.header_valid:
self.__parse_header(self.ifc.rx_peek(),
self.ifc.rx_len())
if self.header_valid and self.ifc.rx_len() >= \
(self.header_len + self.data_len+2):
self.__process_complete_received_msg()
self.__flush_recv_msg()
else:
return 0 # wait 0s before sending a response
def __process_complete_received_msg(self):
log_lvl = self.log_lvl.get(self.control, logging.WARNING)
if callable(log_lvl):
log_lvl = log_lvl()
self.ifc.rx_log(log_lvl, f'Received from {self.addr}:')
# self._recv_buffer, self.header_len +
# self.data_len+2)
if self.__trailer_is_ok(self.ifc.rx_peek(), self.header_len
+ self.data_len + 2):
if self.state == State.init:
self.state = State.received
self.__set_serial_no(self.snr)
self.__dispatch_msg()
def forward(self, buffer, buflen) -> None:
'''add the actual receive msg to the forwarding queue'''
if self.no_forwarding:
@@ -252,158 +434,34 @@ class SolarmanV5(Message):
self.ifc.fwd_add(buffer[:buflen])
self.ifc.fwd_log(logging.DEBUG, 'Store for forwarding:')
fnc = self.switch.get(self.control, self.msg_unknown)
logger.info(self.__flow_str(self.server_side, 'forwrd') +
_, _str = self.get_fnc_handler(self.control)
logger.info(self._flow_str(self.server_side, 'forwrd') +
f' Ctl: {int(self.control):#04x}'
f' Msg: {fnc.__name__!r}')
f' Msg: {_str}')
def _init_new_client_conn(self) -> bool:
return False
'''
Our private methods
'''
def __flow_str(self, server_side: bool, type: str): # noqa: F821
switch = {
'rx': ' <',
'tx': ' >',
'forwrd': '<< ',
'drop': ' xx',
'rxS': '> ',
'txS': '< ',
'forwrdS': ' >>',
'dropS': 'xx ',
}
if server_side:
type += 'S'
return switch.get(type, '???')
def _timestamp(self):
# utc as epoche
return int(time.time()) # pragma: no cover
def _heartbeat(self) -> int:
return 60 # pragma: no cover
def __parse_header(self, buf: bytes, buf_len: int) -> None:
if (buf_len < self.header_len): # enough bytes for complete header?
return
result = struct.unpack_from('<BHHHL', buf, 0)
# store parsed header values in the class
start = result[0] # start byte
self.data_len = result[1] # len of variable id string
self.control = result[2]
self.seq.set_recv(result[3])
self.snr = result[4]
if start != 0xA5:
hex_dump_memory(logging.ERROR,
'Drop packet w invalid start byte from'
f' {self.addr}:', buf, buf_len)
self.inc_counter('Invalid_Msg_Format')
# erase broken recv buffer
self.ifc.rx_clear()
return
self.header_valid = True
def __trailer_is_ok(self, buf: bytes, buf_len: int) -> bool:
crc = buf[self.data_len+11]
stop = buf[self.data_len+12]
if stop != 0x15:
hex_dump_memory(logging.ERROR,
'Drop packet w invalid stop byte from '
f'{self.addr}:', buf, buf_len)
self.inc_counter('Invalid_Msg_Format')
if self.ifc.rx_len() > (self.data_len+13):
next_start = buf[self.data_len+13]
if next_start != 0xa5:
# erase broken recv buffer
self.ifc.rx_clear()
return False
check = sum(buf[1:buf_len-2]) & 0xff
if check != crc:
self.inc_counter('Invalid_Msg_Format')
logger.debug(f'CRC {int(crc):#02x} {int(check):#08x}'
f' Stop:{int(stop):#02x}')
# start & stop byte are valid, discard only this message
return False
return True
def __build_header(self, ctrl) -> None:
'''build header for new transmit message'''
self.send_msg_ofs = self.ifc.tx_len()
self.ifc.tx_add(struct.pack(
'<BHHHL', 0xA5, 0, ctrl, self.seq.get_send(), self.snr))
fnc = self.switch.get(ctrl, self.msg_unknown)
logger.info(self.__flow_str(self.server_side, 'tx') +
f' Ctl: {int(ctrl):#04x} Msg: {fnc.__name__!r}')
def __finish_send_msg(self) -> None:
'''finish the transmit message, set lenght and checksum'''
_len = self.ifc.tx_len() - self.send_msg_ofs
struct.pack_into('<H', self.ifc.tx_peek(), self.send_msg_ofs+1,
_len-11)
check = sum(self.ifc.tx_peek()[
self.send_msg_ofs+1:self.send_msg_ofs + _len]) & 0xff
self.ifc.tx_add(struct.pack('<BB', check, 0x15)) # crc & stop
def _update_header(self, _forward_buffer):
'''update header for message before forwarding,
set sequence and checksum'''
_len = len(_forward_buffer)
ofs = 0
while ofs < _len:
result = struct.unpack_from('<BH', _forward_buffer, ofs)
data_len = result[1] # len of variable id string
struct.pack_into('<H', _forward_buffer, ofs+5,
self.seq.get_send())
check = sum(_forward_buffer[ofs+1:ofs+data_len+11]) & 0xff
struct.pack_into('<B', _forward_buffer, ofs+data_len+11, check)
ofs += (13 + data_len)
def __dispatch_msg(self) -> None:
fnc = self.switch.get(self.control, self.msg_unknown)
if self.unique_id:
logger.info(self.__flow_str(self.server_side, 'rx') +
f' Ctl: {int(self.control):#04x}' +
f' Msg: {fnc.__name__!r}')
fnc()
else:
logger.info(self.__flow_str(self.server_side, 'drop') +
f' Ctl: {int(self.control):#04x}' +
f' Msg: {fnc.__name__!r}')
def __flush_recv_msg(self) -> None:
self.ifc.rx_get(self.header_len + self.data_len+2)
self.header_valid = False
def __send_ack_rsp(self, msgtype, ftype, ack=1):
self.__build_header(msgtype)
self._build_header(msgtype)
self.ifc.tx_add(struct.pack('<BBLL', ftype, ack,
self._timestamp(),
self._heartbeat()))
self.__finish_send_msg()
self._finish_send_msg()
def send_modbus_cb(self, pdu: bytearray, log_lvl: int, state: str):
if self.state != State.up:
logger.warning(f'[{self.node_id}] ignore MODBUS cmd,'
' cause the state is not UP anymore')
return
self.__build_header(0x4510)
self._build_header(0x4510)
self.ifc.tx_add(struct.pack('<BHLLL', self.MB_RTU_CMD,
self.sensor_list, 0, 0, 0))
self.ifc.tx_add(pdu)
self.__finish_send_msg()
self._finish_send_msg()
self.ifc.tx_log(log_lvl, f'Send Modbus {state}:{self.addr}:')
self.ifc.tx_flush()
@@ -436,11 +494,11 @@ class SolarmanV5(Message):
return
self.forward_at_cmd_resp = False
self.__build_header(0x4510)
self._build_header(0x4510)
self.ifc.tx_add(struct.pack(f'<BHLLL{len(at_cmd)}sc', self.AT_CMD,
0x0002, 0, 0, 0,
at_cmd.encode('utf-8'), b'\r'))
self.__finish_send_msg()
self._finish_send_msg()
self.ifc.tx_log(logging.INFO, 'Send AT Command:')
try:
self.ifc.tx_flush()
@@ -607,6 +665,18 @@ class SolarmanV5(Message):
return
self.__forward_msg()
def __parse_modbus_rsp(self, data):
inv_update = False
self.modbus_elms = 0
for key, update, _ in self.mb.recv_resp(self.db, data[14:]):
self.modbus_elms += 1
if update:
if key == 'inverter':
inv_update = True
self._set_mqtt_timestamp(key, self._timestamp())
self.new_data[key] = True
return inv_update
def __modbus_command_rsp(self, data):
'''precess MODBUS RTU response'''
valid = data[1]
@@ -614,18 +684,13 @@ class SolarmanV5(Message):
# logger.debug(f'modbus_len:{modbus_msg_len} accepted:{valid}')
if valid == 1 and modbus_msg_len > 4:
# logger.info(f'first byte modbus:{data[14]}')
inv_update = False
self.modbus_elms = 0
for key, update, _ in self.mb.recv_resp(self.db, data[14:]):
self.modbus_elms += 1
if update:
if key == 'inverter':
inv_update = True
self._set_mqtt_timestamp(key, self._timestamp())
self.new_data[key] = True
inv_update = self.__parse_modbus_rsp(data)
if inv_update:
self.__build_model_name()
if self.establish_inv_emu and not self.ifc.remote.stream:
self.establish_emu()
def msg_hbeat_ind(self):
data = self.ifc.rx_peek()[self.header_len:]
result = struct.unpack_from('<B', data, 0)
@@ -647,16 +712,3 @@ class SolarmanV5(Message):
self.__forward_msg()
self.__send_ack_rsp(0x1810, ftype)
def msg_response(self):
data = self.ifc.rx_peek()[self.header_len:]
result = struct.unpack_from('<BBLL', data, 0)
ftype = result[0] # always 2
valid = result[1] == 1 # status
ts = result[2]
set_hb = result[3] # always 60 or 120
logger.debug(f'ftype:{ftype} accepted:{valid}'
f' ts:{ts:08x} nextHeartbeat: {set_hb}s')
dt = datetime.fromtimestamp(ts)
logger.debug(f'ts: {dt.strftime("%Y-%m-%d %H:%M:%S")}')