From 2336955bb880aee4257fdcf6e75cb21cad7a0966 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 29 Sep 2024 21:11:53 +0200 Subject: [PATCH 01/20] fix client loop closing --- app/src/async_stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 9b189b4..6e77f71 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -391,18 +391,18 @@ class AsyncStreamClient(AsyncStream): 'Client loop stopped for' f' l{self.l_addr}') - server_stream = self.remote.stream + server_ifc = self.remote.ifc # if the client connection closes, we don't touch the server # connection. Instead we erase the client connection stream, # thus on the next received packet from the inverter, we can # establish a new connection to the TSUN cloud - if server_stream.remote.ifc == self: + if server_ifc.remote.ifc == self: # logging.debug(f'Client l{client_stream.l_addr} refs:' # f' {gc.get_referrers(client_stream)}') # than erase client connection - server_stream.remote.stream = None # erases stream and ifc link + server_ifc.remote.stream = None # erases stream and ifc link # erase backlink to inverter self.remote.stream = None From b3068a256c047c1d205d91e8edb8d4bd794c6ef9 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 30 Sep 2024 19:12:49 +0200 Subject: [PATCH 02/20] don't overwrite self.remote in constructor --- app/src/gen3/inverter_g3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index 5b12147..c1eaf61 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -26,7 +26,6 @@ class InverterG3(InverterBase): self.async_create_remote, self.remote) - self.remote = StreamPtr(None) self.local = StreamPtr( ConnectionG3(addr, ifc, True) ) From 8f695518bd56cd512efc0a537035949a5751ccc6 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 30 Sep 2024 19:13:29 +0200 Subject: [PATCH 03/20] update class diagramm --- app/proxy.svg | 6 +++--- app/proxy.yuml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/proxy.svg b/app/proxy.svg index dea173f..b432119 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -315,19 +315,19 @@ A8->A9 - + A9->A10 - + A9->A11 - + diff --git a/app/proxy.yuml b/app/proxy.yuml index d463419..d8bfec7 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -18,9 +18,9 @@ [AsyncStreamServer|async_create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] [AsyncStreamClient||client_loop();_async_forward())] [<>]^-.-[AsyncIfcImpl] -[AsyncIfcImpl]<-[AsyncStream] -[AsyncStream]<-[AsyncStreamServer] -[AsyncStream]<-[AsyncStreamClient] +[AsyncIfcImpl]^[AsyncStream] +[AsyncStream]^[AsyncStreamServer] +[AsyncStream]^[AsyncStreamClient] [ConnectionG3||] From f2ade43410aa37be7f512f5af15d03ccea0ce3cf Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 30 Sep 2024 19:14:50 +0200 Subject: [PATCH 04/20] fixes - fixes null pointer accesses - initalize AsyncStreamClient with proper StreamPtr instance --- app/src/inverter_base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index 5784a2b..ac9ecd7 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -29,13 +29,15 @@ class InverterBase(Inverter): port = tsun['port'] addr = (host, port) stream = self.local.stream + if not stream: + return try: logging.info(f'[{stream.node_id}] Connect to {addr}') connect = asyncio.open_connection(host, port) reader, writer = await connect ifc = AsyncStreamClient(reader, writer, - self.remote) + self.local) if hasattr(stream, 'id_str'): self.remote.stream = conn_class( @@ -60,7 +62,7 @@ class InverterBase(Inverter): async def async_publ_mqtt(self) -> None: '''publish data to MQTT broker''' stream = self.local.stream - if not stream.unique_id: + if not stream or not stream.unique_id: return # check if new inverter or collector infos are available or when the # home assistant has changed the status back to online From a1441fb4fdcb50cb71a4df75879b731f4d762c7a Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Mon, 30 Sep 2024 19:17:06 +0200 Subject: [PATCH 05/20] add close callback --- app/src/async_ifc.py | 4 ++++ app/src/async_stream.py | 21 ++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/app/src/async_ifc.py b/app/src/async_ifc.py index 144f270..99204a9 100644 --- a/app/src/async_ifc.py +++ b/app/src/async_ifc.py @@ -117,3 +117,7 @@ class AsyncIfc(ABC): @abstractmethod def prot_set_update_header_cb(self, callback): pass # pragma: no cover + + @abstractmethod + def prot_set_close_header_cb(self, callback): + pass # pragma: no cover diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 6e77f71..e08bdb4 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -35,12 +35,14 @@ class AsyncIfcImpl(AsyncIfc): self.timeout_cb = None self.init_new_client_conn_cb = None self.update_header_cb = None + self.close_cb = None def close(self): self.timeout_cb = None self.fwd_fifo.reg_trigger(None) self.tx_fifo.reg_trigger(None) self.rx_fifo.reg_trigger(None) + self.close_cb = None def set_node_id(self, value: str): self.node_id = value @@ -124,12 +126,18 @@ class AsyncIfcImpl(AsyncIfc): def prot_set_update_header_cb(self, callback): self.update_header_cb = callback + def prot_set_close_header_cb(self, callback): + self.close_cb = callback + class StreamPtr(): '''Descr StreamPtr''' def __init__(self, _stream): self.stream = _stream + def __str__(self) -> str: + return f'ifc:{self._ifc}, stream: {self._stream}' + @property def ifc(self): return self._ifc @@ -238,15 +246,19 @@ class AsyncStream(AsyncIfcImpl): await self._writer.wait_closed() def close(self) -> None: + logging.info(f'AsyncStream.close1() l{self.l_addr} | r{self.r_addr}') """close handler for a no waiting disconnect hint: must be called before releasing the connection instance """ + close_cb = self.close_cb super().close() + if close_cb: + close_cb() self._reader.feed_eof() # abort awaited read if self._writer.is_closing(): return - logger.debug(f'AsyncStream.close() l{self.l_addr} | r{self.r_addr}') + logger.info(f'AsyncStream.close2() l{self.l_addr} | r{self.r_addr}') self._writer.close() def healthy(self) -> bool: @@ -315,7 +327,7 @@ class AsyncStream(AsyncIfcImpl): f"{traceback.format_exc()}") def __del__(self): - logger.debug( + logger.info( f"AsyncStream.__del__ l{self.l_addr} | r{self.r_addr}") @@ -374,6 +386,8 @@ class AsyncStreamServer(AsyncStream): hint: must be called before releasing the connection instance """ + logging.info( + f'AsyncStreamServer.close() l{self.l_addr} | r{self.r_addr}') self.async_create_remote = None self.async_publ_mqtt = None super().close() @@ -386,6 +400,7 @@ class AsyncStreamClient(AsyncStream): async def client_loop(self, _: str) -> None: '''Loop for receiving messages from the TSUN cloud (client-side)''' + logging.info(f'AsynStream.client_loop{self} rem-> {self.remote}') await self.loop() logger.info(f'[{self.node_id}:{self.conn_no}] ' 'Client loop stopped for' @@ -398,7 +413,7 @@ class AsyncStreamClient(AsyncStream): # thus on the next received packet from the inverter, we can # establish a new connection to the TSUN cloud - if server_ifc.remote.ifc == self: + if server_ifc and server_ifc.remote.ifc == self: # logging.debug(f'Client l{client_stream.l_addr} refs:' # f' {gc.get_referrers(client_stream)}') # than erase client connection From 39aba31bbdf2a168e5fbcd1e5c9f65ff57d5404b Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Tue, 1 Oct 2024 19:50:42 +0200 Subject: [PATCH 06/20] refactor close handling --- app/src/async_ifc.py | 4 - app/src/async_stream.py | 68 ++++----------- app/src/gen3/connection_g3.py | 3 - app/src/gen3/inverter_g3.py | 5 -- app/src/gen3/talent.py | 9 +- app/src/gen3plus/connection_g3p.py | 4 - app/src/gen3plus/inverter_g3p.py | 5 -- app/src/gen3plus/solarman_v5.py | 6 +- app/src/inverter_base.py | 30 ++++++- app/src/modbus_tcp.py | 3 +- app/src/server.py | 3 +- app/tests/test_inverter_g3.py | 134 +++++++++++----------------- app/tests/test_inverter_g3p.py | 135 +++++++++++------------------ app/tests/test_solarman.py | 12 ++- app/tests/test_talent.py | 16 ++-- 15 files changed, 173 insertions(+), 264 deletions(-) diff --git a/app/src/async_ifc.py b/app/src/async_ifc.py index 99204a9..144f270 100644 --- a/app/src/async_ifc.py +++ b/app/src/async_ifc.py @@ -117,7 +117,3 @@ class AsyncIfc(ABC): @abstractmethod def prot_set_update_header_cb(self, callback): pass # pragma: no cover - - @abstractmethod - def prot_set_close_header_cb(self, callback): - pass # pragma: no cover diff --git a/app/src/async_stream.py b/app/src/async_stream.py index e08bdb4..3d9957d 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -35,14 +35,12 @@ class AsyncIfcImpl(AsyncIfc): self.timeout_cb = None self.init_new_client_conn_cb = None self.update_header_cb = None - self.close_cb = None def close(self): self.timeout_cb = None self.fwd_fifo.reg_trigger(None) self.tx_fifo.reg_trigger(None) self.rx_fifo.reg_trigger(None) - self.close_cb = None def set_node_id(self, value: str): self.node_id = value @@ -126,9 +124,6 @@ class AsyncIfcImpl(AsyncIfc): def prot_set_update_header_cb(self, callback): self.update_header_cb = callback - def prot_set_close_header_cb(self, callback): - self.close_cb = callback - class StreamPtr(): '''Descr StreamPtr''' @@ -212,7 +207,6 @@ class AsyncStream(AsyncIfcImpl): f'connection timeout ({dead_conn_to}s) ' f'for {self.l_addr}') await self.disc() - self.close() return self except OSError as error: @@ -220,14 +214,12 @@ class AsyncStream(AsyncIfcImpl): f'{error} for l{self.l_addr} | ' f'r{self.r_addr}') await self.disc() - self.close() return self except RuntimeError as error: logger.info(f'[{self.node_id}:{self.conn_no}] ' f'{error} for {self.l_addr}') await self.disc() - self.close() return self except Exception: @@ -251,10 +243,7 @@ class AsyncStream(AsyncIfcImpl): hint: must be called before releasing the connection instance """ - close_cb = self.close_cb super().close() - if close_cb: - close_cb() self._reader.feed_eof() # abort awaited read if self._writer.is_closing(): return @@ -304,21 +293,22 @@ class AsyncStream(AsyncIfcImpl): except OSError as error: if self.remote.stream: - rmt = self.remote.stream - self.remote.stream = None - logger.error(f'[{rmt.node_id}:{rmt.conn_no}] Fwd: {error} for ' - f'l{rmt._ifc.l_addr} | r{rmt._ifc.r_addr}') - await rmt._ifc.disc() - rmt._ifc.close() + rmt = self.remote + logger.error(f'[{rmt.stream.node_id}:{rmt.stream.conn_no}] ' + f'Fwd: {error} for ' + f'l{rmt.ifc.l_addr} | r{rmt.ifc.r_addr}') + await rmt.ifc.disc() + if rmt.ifc.close_cb: + rmt.ifc.close_cb() except RuntimeError as error: if self.remote.stream: - rmt = self.remote.stream - self.remote.stream = None - logger.info(f'[{rmt.node_id}:{rmt.conn_no}] ' - f'Fwd: {error} for {rmt._ifc.l_addr}') - await rmt._ifc.disc() - rmt._ifc.close() + rmt = self.remote + logger.info(f'[{rmt.stream.node_id}:{rmt.stream.conn_no}] ' + f'Fwd: {error} for {rmt.ifc.l_addr}') + await rmt.ifc.disc() + if rmt.ifc.close_cb: + rmt.ifc.close_cb() except Exception: Infos.inc_counter('SW_Exception') @@ -381,46 +371,22 @@ class AsyncStreamServer(AsyncStream): except Exception: pass - def close(self) -> None: - """close handler for a no waiting disconnect - - hint: must be called before releasing the connection instance - """ - logging.info( - f'AsyncStreamServer.close() l{self.l_addr} | r{self.r_addr}') - self.async_create_remote = None - self.async_publ_mqtt = None - super().close() - class AsyncStreamClient(AsyncStream): def __init__(self, reader: StreamReader, writer: StreamWriter, - rstream: "StreamPtr") -> None: + rstream: "StreamPtr", close_cb) -> None: AsyncStream.__init__(self, reader, writer, rstream) + self.close_cb = close_cb async def client_loop(self, _: str) -> None: '''Loop for receiving messages from the TSUN cloud (client-side)''' - logging.info(f'AsynStream.client_loop{self} rem-> {self.remote}') await self.loop() logger.info(f'[{self.node_id}:{self.conn_no}] ' 'Client loop stopped for' f' l{self.l_addr}') - server_ifc = self.remote.ifc - - # if the client connection closes, we don't touch the server - # connection. Instead we erase the client connection stream, - # thus on the next received packet from the inverter, we can - # establish a new connection to the TSUN cloud - - if server_ifc and server_ifc.remote.ifc == self: - # logging.debug(f'Client l{client_stream.l_addr} refs:' - # f' {gc.get_referrers(client_stream)}') - # than erase client connection - server_ifc.remote.stream = None # erases stream and ifc link - - # erase backlink to inverter - self.remote.stream = None + if self.close_cb: + self.close_cb() async def _async_forward(self) -> None: """forward handler transmits data over the remote connection""" diff --git a/app/src/gen3/connection_g3.py b/app/src/gen3/connection_g3.py index 6123693..b1e8b8b 100644 --- a/app/src/gen3/connection_g3.py +++ b/app/src/gen3/connection_g3.py @@ -11,6 +11,3 @@ logger = logging.getLogger('conn') class ConnectionG3(Talent): def __init__(self, addr, ifc, server_side, id_str=b'') -> None: super().__init__(addr, server_side, ifc, id_str) - - def close(self): - super().close() diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index c1eaf61..05f12ec 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -33,8 +33,3 @@ class InverterG3(InverterBase): async def async_create_remote(self) -> None: await InverterBase.async_create_remote( self, 'tsun', ConnectionG3) - - def close(self) -> None: - logging.debug(f'InverterG3.close() {self.addr}') - self.local.stream.close() -# logging.info(f'Inverter refs: {gc.get_referrers(self)}') diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index 4f9b309..f0c3246 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -438,8 +438,8 @@ class Talent(Message): result = struct.unpack_from('!q', self.ifc.rx_peek(), self.header_len) self.ts_offset = result[0]-ts - if self.remote.stream: - self.remote.stream.ts_offset = self.ts_offset + if self.ifc.remote.stream: + self.ifc.remote.stream.ts_offset = self.ts_offset logger.debug(f'tsun-time: {int(result[0]):08x}' f' proxy-time: {ts:08x}' f' offset: {self.ts_offset}') @@ -597,9 +597,8 @@ class Talent(Message): self.header_len+self.data_len] if self.ctrl.is_req(): - if self.remote.stream.mb.recv_req(data[hdr_len:], - self.remote.stream. - msg_forward): + rstream = self.ifc.remote.stream + if rstream.mb.recv_req(data[hdr_len:], rstream.msg_forward): self.inc_counter('Modbus_Command') else: self.inc_counter('Invalid_Msg_Format') diff --git a/app/src/gen3plus/connection_g3p.py b/app/src/gen3plus/connection_g3p.py index b86592e..66d327e 100644 --- a/app/src/gen3plus/connection_g3p.py +++ b/app/src/gen3plus/connection_g3p.py @@ -12,7 +12,3 @@ class ConnectionG3P(SolarmanV5): def __init__(self, addr, ifc, server_side, client_mode: bool = False) -> None: super().__init__(addr, server_side, client_mode, ifc) - - def close(self): - super().close() - # logger.info(f'AsyncStream refs: {gc.get_referrers(self)}') diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index f8fbfc7..a362e4f 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -34,8 +34,3 @@ class InverterG3P(InverterBase): async def async_create_remote(self) -> None: await InverterBase.async_create_remote( self, 'solarman', ConnectionG3P) - - def close(self) -> None: - logging.debug(f'InverterG3P.close() {self.addr}') - self.local.stream.close() -# logger.debug (f'Inverter refs: {gc.get_referrers(self)}') diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index a9dfb79..c0cd827 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -603,9 +603,9 @@ class SolarmanV5(Message): self.forward_at_cmd_resp = True elif ftype == self.MB_RTU_CMD: - if self.remote.stream.mb.recv_req(data[15:], - self.remote.stream. - __forward_msg): + rstream = self.ifc.remote.stream + if rstream.mb.recv_req(data[15:], + rstream.__forward_msg): self.inc_counter('Modbus_Command') else: logger.error('Invalid Modbus Msg') diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index ac9ecd7..0b6fad7 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -22,6 +22,32 @@ class InverterBase(Inverter): def __init__(self): self.__ha_restarts = -1 + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb) -> None: + logging.info(f'Inverter.__exit__() {self.addr}') + self.__del_remote() + if self.local.stream: + self.local.stream.close() + self.local.stream = None + + if self.local.ifc: + self.local.ifc.close() + self.local.ifc = None + + def __del__(self) -> None: + logging.info(f'Inverter.__del__() {self.addr}') + + def __del_remote(self): + if self.remote.stream: + self.remote.stream.close() + self.remote.stream = None + + if self.remote.ifc: + self.remote.ifc.close() + self.remote.ifc = None + async def async_create_remote(self, inv_prot: str, conn_class) -> None: '''Establish a client connection to the TSUN cloud''' tsun = Config.get(inv_prot) @@ -36,8 +62,8 @@ class InverterBase(Inverter): logging.info(f'[{stream.node_id}] Connect to {addr}') connect = asyncio.open_connection(host, port) reader, writer = await connect - ifc = AsyncStreamClient(reader, writer, - self.local) + ifc = AsyncStreamClient( + reader, writer, self.local, self.__del_remote) if hasattr(stream, 'id_str'): self.remote.stream = conn_class( diff --git a/app/src/modbus_tcp.py b/app/src/modbus_tcp.py index bac6243..8bacce3 100644 --- a/app/src/modbus_tcp.py +++ b/app/src/modbus_tcp.py @@ -27,6 +27,7 @@ class ModbusConn(): reader, writer = await connection self.inverter = InverterG3P(reader, writer, self.addr, client_mode=True) + self.inverter.__enter__() stream = self.inverter.local.stream logging.info(f'[{stream.node_id}:{stream.conn_no}] ' f'Connected to {self.addr}') @@ -37,7 +38,7 @@ class ModbusConn(): async def __aexit__(self, exc_type, exc, tb): Infos.dec_counter('Inverter_Cnt') await self.inverter.local.ifc.publish_outstanding_mqtt() - self.inverter.close() + self.inverter.__exit__(exc_type, exc, tb) class ModbusTcp(): diff --git a/app/src/server.py b/app/src/server.py index a40a178..bbd7f83 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -74,7 +74,8 @@ async def handle_client(reader: StreamReader, writer: StreamWriter, inv_class): '''Handles a new incoming connection and starts an async loop''' addr = writer.get_extra_info('peername') - await inv_class(reader, writer, addr).local.ifc.server_loop() + with inv_class(reader, writer, addr) as inv: + await inv.local.ifc.server_loop() async def handle_shutdown(web_task): diff --git a/app/tests/test_inverter_g3.py b/app/tests/test_inverter_g3.py index e30f472..27ee51b 100644 --- a/app/tests/test_inverter_g3.py +++ b/app/tests/test_inverter_g3.py @@ -47,11 +47,6 @@ def patch_conn_init(): with patch.object(ConnectionG3, '__init__', return_value= None) as conn: yield conn -@pytest.fixture -def patch_conn_close(): - with patch.object(ConnectionG3, 'close') as conn: - yield conn - class FakeReader(): def __init__(self): self.on_recv = asyncio.Event() @@ -104,131 +99,102 @@ def patch_open_connection(): yield conn -def test_method_calls(patch_conn_close): - spy2 = patch_conn_close +def test_method_calls(): reader = FakeReader() writer = FakeWriter() addr = ('proxy.local', 10000) - inverter = InverterG3(reader, writer, addr) - assert inverter.local.stream - assert inverter.local.ifc - - inverter.close() - spy2.assert_called_once() + with InverterG3(reader, writer, addr) as inverter: + assert inverter.local.stream + assert inverter.local.ifc @pytest.mark.asyncio -async def test_remote_conn(config_conn, patch_open_connection, patch_conn_close): +async def test_remote_conn(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - spy1 = patch_conn_close - - inverter = InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) - - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream - inverter.close() - spy1.assert_called_once() + with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream @pytest.mark.asyncio -async def test_remote_except(config_conn, patch_open_connection, patch_conn_close): +async def test_remote_except(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - spy1 = patch_conn_close - global test test = TestType.RD_TEST_TIMEOUT - inverter = InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) + with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - - test = TestType.RD_TEST_EXCEPT - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - inverter.close() - spy1.assert_called_once() + test = TestType.RD_TEST_EXCEPT + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None @pytest.mark.asyncio -async def test_mqtt_publish(config_conn, patch_open_connection, patch_conn_close): +async def test_mqtt_publish(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) - stream = inverter.local.stream - await inverter.async_publ_mqtt() # check call with invalid unique_id - stream._Talent__set_serial_no(serial_no= "123344") - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == False + with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + stream = inverter.local.stream + await inverter.async_publ_mqtt() # check call with invalid unique_id + stream._Talent__set_serial_no(serial_no= "123344") - stream.new_data['env'] = True - stream.db.db['env'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['env'] == False + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == False - Infos.new_stat_data['proxy'] = True - await inverter.async_publ_mqtt() - assert Infos.new_stat_data['proxy'] == False + stream.new_data['env'] = True + stream.db.db['env'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['env'] == False - inverter.close() - spy1.assert_called_once() + Infos.new_stat_data['proxy'] = True + await inverter.async_publ_mqtt() + assert Infos.new_stat_data['proxy'] == False @pytest.mark.asyncio -async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err, patch_conn_close): +async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): _ = config_conn _ = patch_open_connection _ = patch_mqtt_err assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) - stream = inverter.local.stream - stream._Talent__set_serial_no(serial_no= "123344") - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True - - inverter.close() - spy1.assert_called_once() + with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + stream = inverter.local.stream + stream._Talent__set_serial_no(serial_no= "123344") + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True @pytest.mark.asyncio -async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except, patch_conn_close): +async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except): _ = config_conn _ = patch_open_connection _ = patch_mqtt_except assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) - stream = inverter.local.stream - stream._Talent__set_serial_no(serial_no= "123344") - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True + with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + stream = inverter.local.stream + stream._Talent__set_serial_no(serial_no= "123344") - inverter.close() - spy1.assert_called_once() + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True diff --git a/app/tests/test_inverter_g3p.py b/app/tests/test_inverter_g3p.py index 5a4e5a4..ff5564a 100644 --- a/app/tests/test_inverter_g3p.py +++ b/app/tests/test_inverter_g3p.py @@ -48,11 +48,6 @@ def patch_conn_init(): with patch.object(ConnectionG3P, '__init__', return_value= None) as conn: yield conn -@pytest.fixture -def patch_conn_close(): - with patch.object(ConnectionG3P, 'close') as conn: - yield conn - class FakeReader(): def __init__(self): self.on_recv = asyncio.Event() @@ -104,132 +99,102 @@ def patch_open_connection(): with patch.object(asyncio, 'open_connection', new_open) as conn: yield conn - -def test_method_calls(patch_conn_close): - spy2 = patch_conn_close +def test_method_calls(): reader = FakeReader() writer = FakeWriter() addr = ('proxy.local', 10000) - inverter = InverterG3P(reader, writer, addr, client_mode=False) - assert inverter.local.stream - assert inverter.local.ifc - - inverter.close() - spy2.assert_called_once() + with InverterG3P(reader, writer, addr, client_mode=False) as inverter: + assert inverter.local.stream + assert inverter.local.ifc @pytest.mark.asyncio -async def test_remote_conn(config_conn, patch_open_connection, patch_conn_close): +async def test_remote_conn(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - spy1 = patch_conn_close - - inverter = InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) - - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream - inverter.close() - spy1.assert_called_once() + with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream @pytest.mark.asyncio -async def test_remote_except(config_conn, patch_open_connection, patch_conn_close): +async def test_remote_except(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - - spy1 = patch_conn_close global test test = TestType.RD_TEST_TIMEOUT - inverter = InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) + with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - - test = TestType.RD_TEST_EXCEPT - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - inverter.close() - spy1.assert_called_once() + test = TestType.RD_TEST_EXCEPT + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None @pytest.mark.asyncio -async def test_mqtt_publish(config_conn, patch_open_connection, patch_conn_close): +async def test_mqtt_publish(config_conn, patch_open_connection): _ = config_conn _ = patch_open_connection assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) - stream = inverter.local.stream - await inverter.async_publ_mqtt() # check call with invalid unique_id - stream._SolarmanV5__set_serial_no(snr= 123344) - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == False + with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + stream = inverter.local.stream + await inverter.async_publ_mqtt() # check call with invalid unique_id + stream._SolarmanV5__set_serial_no(snr= 123344) - stream.new_data['env'] = True - stream.db.db['env'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['env'] == False + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == False - Infos.new_stat_data['proxy'] = True - await inverter.async_publ_mqtt() - assert Infos.new_stat_data['proxy'] == False + stream.new_data['env'] = True + stream.db.db['env'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['env'] == False - inverter.close() - spy1.assert_called_once() + Infos.new_stat_data['proxy'] = True + await inverter.async_publ_mqtt() + assert Infos.new_stat_data['proxy'] == False @pytest.mark.asyncio -async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err, patch_conn_close): +async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): _ = config_conn _ = patch_open_connection _ = patch_mqtt_err assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) - stream = inverter.local.stream - stream._SolarmanV5__set_serial_no(snr= 123344) - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True - - inverter.close() - spy1.assert_called_once() + with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + stream = inverter.local.stream + stream._SolarmanV5__set_serial_no(snr= 123344) + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True @pytest.mark.asyncio -async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except, patch_conn_close): +async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except): _ = config_conn _ = patch_open_connection _ = patch_mqtt_except assert asyncio.get_running_loop() - spy1 = patch_conn_close - Inverter.class_init() - inverter = InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) - stream = inverter.local.stream - stream._SolarmanV5__set_serial_no(snr= 123344) - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True + with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + stream = inverter.local.stream + stream._SolarmanV5__set_serial_no(snr= 123344) - inverter.close() - spy1.assert_called_once() + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 6da7d23..795b32f 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -32,13 +32,17 @@ class Mqtt(): self.data = data +class FakeIfc(AsyncIfcImpl): + def __init__(self): + super().__init__() + self.remote = StreamPtr(None) + class MemoryStream(SolarmanV5): def __init__(self, msg, chunks = (0,), server_side: bool = True): - _ifc = AsyncIfcImpl() + _ifc = FakeIfc() super().__init__(('test.local', 1234), server_side, client_mode=False, ifc=_ifc) if server_side: self.mb.timeout = 0.4 # overwrite for faster testing - self.remote = StreamPtr(None) self.mb_first_timeout = 0.5 self.mb_timeout = 0.5 self.sent_pdu = b'' @@ -101,8 +105,8 @@ class MemoryStream(SolarmanV5): def createClientStream(self, msg, chunks = (0,)): c = MemoryStream(msg, chunks, False) - self.remote.stream = c - c. remote.stream = self + self.ifc.remote.stream = c + c.ifc.remote.stream = self return c def _SolarmanV5__flush_recv_msg(self) -> None: diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index 38b0edd..d082885 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -16,14 +16,17 @@ Infos.static_init() tracer = logging.getLogger('tracer') +class FakeIfc(AsyncIfcImpl): + def __init__(self): + super().__init__() + self.remote = StreamPtr(None) class MemoryStream(Talent): def __init__(self, msg, chunks = (0,), server_side: bool = True): - self.ifc = AsyncIfcImpl() + self.ifc = FakeIfc() super().__init__(('test.local', 1234), server_side, self.ifc) if server_side: self.mb.timeout = 0.4 # overwrite for faster testing - self.remote = StreamPtr(None) self.mb_first_timeout = 0.5 self.mb_timeout = 0.5 self.sent_pdu = b'' @@ -37,7 +40,6 @@ class MemoryStream(Talent): self.addr = 'Test: SrvSide' self.send_msg_ofs = 0 self.msg_recvd = [] - self.remote.stream = None def write_cb(self): self.sent_pdu = self.ifc.tx_fifo.get() @@ -73,8 +75,8 @@ class MemoryStream(Talent): def createClientStream(self, msg, chunks = (0,)): c = MemoryStream(msg, chunks, False) - self.remote.stream = c - c. remote.stream = self + self.ifc.remote.stream = c + c.ifc.remote.stream = self return c def _Talent__flush_recv_msg(self) -> None: @@ -1059,7 +1061,7 @@ def test_msg_time_resp(config_tsun_inv1, msg_time_rsp): m = MemoryStream(msg_time_rsp, (0,), False) s = MemoryStream(b'', (0,), True) assert s.ts_offset==0 - m.remote.stream = s + m.ifc.remote.stream = s m.db.stat['proxy']['Unknown_Ctrl'] = 0 m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed @@ -1075,7 +1077,7 @@ def test_msg_time_resp(config_tsun_inv1, msg_time_rsp): assert m.ifc.fwd_fifo.get()==b'' assert m.ifc.tx_fifo.get()==b'' assert m.db.stat['proxy']['Unknown_Ctrl'] == 0 - m.remote.stream = None + m.ifc.remote.stream = None s.close() m.close() From cfe2c9cb9d994a4181f76815497007fcfce08931 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Wed, 2 Oct 2024 23:40:42 +0200 Subject: [PATCH 07/20] remove connection classes --- app/proxy.svg | 881 ++++++++++++++--------------- app/proxy.yuml | 27 +- app/src/async_stream.py | 25 +- app/src/gen3/connection_g3.py | 13 - app/src/gen3/inverter_g3.py | 8 +- app/src/gen3/talent.py | 9 +- app/src/gen3plus/connection_g3p.py | 14 - app/src/gen3plus/inverter_g3p.py | 8 +- app/src/gen3plus/solarman_v5.py | 10 +- app/src/inverter.py | 10 +- app/src/inverter_base.py | 43 +- app/src/iter_registry.py | 8 + app/src/messages.py | 12 +- app/src/server.py | 27 +- app/tests/test_connection_g3.py | 100 ---- app/tests/test_connection_g3p.py | 105 ---- app/tests/test_inverter_g3.py | 38 +- app/tests/test_inverter_g3p.py | 6 - app/tests/test_solarman.py | 10 +- app/tests/test_talent.py | 8 +- 20 files changed, 582 insertions(+), 780 deletions(-) delete mode 100644 app/src/gen3/connection_g3.py delete mode 100644 app/src/gen3plus/connection_g3p.py create mode 100644 app/src/iter_registry.py delete mode 100644 app/tests/test_connection_g3.py delete mode 100644 app/tests/test_connection_g3p.py diff --git a/app/proxy.svg b/app/proxy.svg index b432119..88a5606 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,526 +4,519 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -Mqtt -<<Singleton>> - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() - - - -A2 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt -__ha_restarts - -async_create_remote(inv_prot, conn_class)async_publ_mqtt() - - - -A1->A2 - - - + +IterRegistry + + +__iter__ A3 - -InverterG3 - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() + +Inverter + +cls.db_stat +cls.entity_prfx +cls.discovery_prfx +cls.proxy_node_id +cls.proxy_unique_id +cls.mqtt:Mqtt +__ha_restarts + +async_create_remote(inv_prot, conn_class)async_publ_mqtt() - - -A2->A3 - - - - - -A4 - -InverterG3P - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() - - - -A2->A4 - - - - - -A10 - -AsyncStreamServer - -async_create_remote - -<async>server_loop() -<async>_async_forward() -<async>publish_outstanding_mqtt() -close() - - - -A3->A10 - - - -local - - - -A11 - -AsyncStreamClient - - -<async>client_loop() -<async>_async_forward()) - - - -A3->A11 - - -remote - - - -A4->A10 - - - -local - - - -A4->A11 - - -remote - - - -A5 - -IterRegistry - - -__iter__ - - - -A6 - -Message - -server_side:bool -header_valid:bool -header_len:unsigned -data_len:unsigned -unique_id -node_id -sug_area -_recv_buffer:bytearray -_send_buffer:bytearray -_forward_buffer:bytearray -db:Infos -new_data:list -state - -_read():void<abstract> -close():void -inc_counter():void -dec_counter():void - - - -A5->A6 - - - - - -A17 - -Talent - -ifc:AsyncIfc -conn_no -addr -await_conn_resp_cnt -id_str -contact_name -contact_mail -db:InfosG3 -mb:Modbus -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -healthy() -close() - - - -A6->A17 - - + + +A1->A3 + + A18 - -SolarmanV5 - -ifc:AsyncIfc -conn_no -addr -control -serial -snr -db:InfosG3P -mb:Modbus -switch - -msg_unknown() -healthy() -close() + +Message + +node_id + +inc_counter() +dec_counter() - - -A6->A18 - - + + +A1->A18 + + + + + +A2 + +Mqtt +<<Singleton>> + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() + + + +A3->A2 + + + + + + +A4 + +InverterBase + + +<async>disc(shutdown_started) +<async>async_create_remote() +healthy() + + + +A3->A4 + + + + + +A5 + +InverterG3 + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() + + + +A4->A5 + + + + + +A6 + +InverterG3P + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() + + + +A4->A6 + + + + + +A10 + +AsyncStreamServer + +async_create_remote + +<async>server_loop() +<async>_async_forward() +<async>publish_outstanding_mqtt() +close() + + + +A5->A10 + + + +local + + + +A11 + +AsyncStreamClient + + +<async>client_loop() +<async>_async_forward()) + + + +A5->A11 + + +remote + + + +A6->A10 + + + +local + + + +A6->A11 + + +remote A7 - -<<AsyncIfc>> - - -set_node_id() -get_conn_no() -tx_add() -tx_flush() -tx_get() -tx_peek() -tx_log() -tx_clear() -tx_len() -fwd_add() -fwd_flush() -fwd_log() -fwd_clear() -rx_get() -rx_peek() -rx_log() -rx_clear() -rx_len() -rx_set_cb() -prot_set_timeout_cb() + +<<AsyncIfc>> + + +set_node_id() +get_conn_no() +tx_add() +tx_flush() +tx_get() +tx_peek() +tx_log() +tx_clear() +tx_len() +fwd_add() +fwd_flush() +fwd_log() +fwd_clear() +rx_get() +rx_peek() +rx_log() +rx_clear() +rx_len() +rx_set_cb() +prot_set_timeout_cb() A8 - -AsyncIfcImpl - -fwd_fifo:ByteFifo -tx_fifo:ByteFifo -rx_fifo:ByteFifo -conn_no:Count -node_id -timeout_cb + +AsyncIfcImpl + +fwd_fifo:ByteFifo +tx_fifo:ByteFifo +rx_fifo:ByteFifo +conn_no:Count +node_id +timeout_cb - + A7->A8 - - + + A9 - -AsyncStream - -reader -writer -addr -r_addr -l_addr - -<async>loop -disc() -close() -healthy() -__async_read() -__async_write() -__async_forward() + +AsyncStream + +reader +writer +addr +r_addr +l_addr + +<async>loop +disc() +close() +healthy() +__async_read() +__async_write() +__async_forward() - + A8->A9 - - + + - + A9->A10 - - + + - + A9->A11 - - + + A12 - -ConnectionG3 - - + +Talent + +ifc:AsyncIfc +conn_no +addr +await_conn_resp_cnt +id_str +contact_name +contact_mail +db:InfosG3 +mb:Modbus +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +healthy() +close() - - -A12->A3 - - -remote + + +A12->A5 + + +remote - - -A12->A3 - - - -local + + +A12->A5 + + + +local - - -A13 - -ConnectionG3P - - - - - -A13->A4 - - -remote - - - -A13->A4 - - - -local - - - -A14 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -ha_remove -update_db -set_db_def_value -get_db_value -ignore_this_device + + +A12->A7 + + +use A15 - -InfosG3 - - -ha_confs() -parse() + +InfosG3 + + +ha_confs() +parse() - - -A14->A15 - - + + +A12->A15 + + + + + +A17 + +ConnectionG3 + + + +A12->A17 + + + + + +A13 + +SolarmanV5 + +ifc:AsyncIfc +conn_no +addr +control +serial +snr +db:InfosG3P +mb:Modbus +switch + +msg_unknown() +healthy() +close() + + + +A13->A6 + + +remote + + + +A13->A6 + + + +local + + + +A13->A7 + + +use A16 - -InfosG3P - - -ha_confs() -parse() + +InfosG3P + + +ha_confs() +parse() + + + +A13->A16 + + + + + +A14 + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +ha_remove +update_db +set_db_def_value +get_db_value +ignore_this_device + + + +A14->A15 + + - -A14->A16 - - - - - -A17->A7 - - -use - - -A17->A12 - - +A14->A16 + + - - -A17->A15 - - - - - -A18->A7 - - -use + + +A18->A12 + + - + A18->A13 - - - - - -A18->A16 - - + + A19 - -Modbus - -que -snd_handler -rsp_handler -timeout -max_retires -last_xxx -err -retry_cnt -req_pend -tim - -build_msg() -recv_req() -recv_resp() -close() + +Modbus + +que +snd_handler +rsp_handler +timeout +max_retires +last_xxx +err +retry_cnt +req_pend +tim + +build_msg() +recv_req() +recv_resp() +close() - + + +A19->A12 + + +has +1 + + -A19->A17 - - -has -1 - - - -A19->A18 - - -has -1 +A19->A13 + + +has +1 A20 - -ModbusConn - -host -port -addr -stream:InverterG3P - + +ModbusConn + +host +port +addr +stream:InverterG3P + - - -A20->A4 - - -1 -has + + +A20->A6 + + +1 +has diff --git a/app/proxy.yuml b/app/proxy.yuml index d8bfec7..85bf0d0 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -3,14 +3,17 @@ // {generate:true} [note: You can stick notes on diagrams too!{bg:cornsilk}] +[IterRegistry||__iter__] [Mqtt;<>|ha_restarts;__client;__cb_MqttIsUp|publish();close()] [Inverter|cls.db_stat;cls.entity_prfx;cls.discovery_prfx;cls.proxy_node_id;cls.proxy_unique_id;cls.mqtt:Mqtt;;__ha_restarts|async_create_remote(inv_prot, conn_class)async_publ_mqtt()] -[Inverter]^[InverterG3|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] -[Inverter]^[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] -[Mqtt;<>]<-++[Inverter] +[InverterBase||disc(shutdown_started);async_create_remote();healthy()] +[Inverter]^[InverterBase] +[InverterBase]^[InverterG3|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] +[InverterBase]^[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] +[Inverter]++->[Mqtt;<>] -[IterRegistry||__iter__]^[Message|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list;state|_read():void;close():void;inc_counter():void;dec_counter():void] +[IterRegistry]^[Inverter|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list;state|_read():void;close():void;inc_counter():void;dec_counter():void] [<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_flush();fwd_log();fwd_clear();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] [AsyncIfcImpl|fwd_fifo:ByteFifo;tx_fifo:ByteFifo;rx_fifo:ByteFifo;conn_no:Count;node_id;timeout_cb] @@ -23,31 +26,29 @@ [AsyncStream]^[AsyncStreamClient] -[ConnectionG3||] -[ConnectionG3][AsyncStreamClient] -[ConnectionG3]<-local++[InverterG3] +[Talent]<-local++[InverterG3] [InverterG3]++local->[AsyncStreamServer] -[ConnectionG3P||] -[ConnectionG3P][AsyncStreamClient] -[ConnectionG3P]<-local++[InverterG3P] +[SolarmanV5]<-local++[InverterG3P] [InverterG3P]++local->[AsyncStreamServer] [Infos|stat;new_stat_data;info_dev|static_init();dev_value();inc_counter();dec_counter();ha_proxy_conf;ha_conf;ha_remove;update_db;set_db_def_value;get_db_value;ignore_this_device] [Infos]^[InfosG3||ha_confs();parse()] [Infos]^[InfosG3P||ha_confs();parse()] -[Talent|ifc:AsyncIfc;conn_no;addr;;await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;healthy();close()] [Talent]^[ConnectionG3] [Talent]use->[<>] [Talent]->[InfosG3] -[SolarmanV5|ifc:AsyncIfc;conn_no;addr;;control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;healthy();close()] -[SolarmanV5]^[ConnectionG3P] [SolarmanV5]use->[<>] [SolarmanV5]->[InfosG3P] +[IterRegistry]^[Message|node_id|inc_counter();dec_counter()] [Message]^[Talent] [Message]^[SolarmanV5] diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 3d9957d..268a612 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -127,8 +127,9 @@ class AsyncIfcImpl(AsyncIfc): class StreamPtr(): '''Descr StreamPtr''' - def __init__(self, _stream): + def __init__(self, _stream, _ifc=None): self.stream = _stream + self.ifc = _ifc def __str__(self) -> str: return f'ifc:{self._ifc}, stream: {self._stream}' @@ -137,6 +138,10 @@ class StreamPtr(): def ifc(self): return self._ifc + @ifc.setter + def ifc(self, value): + self._ifc = value + @property def stream(self): return self._stream @@ -144,10 +149,6 @@ class StreamPtr(): @stream.setter def stream(self, value): self._stream = value - if value: - self._ifc = value.ifc - else: - self._ifc = None class AsyncStream(AsyncIfcImpl): @@ -231,6 +232,7 @@ class AsyncStream(AsyncIfcImpl): async def disc(self) -> None: """Async disc handler for graceful disconnect""" + self.remote = None if self._writer.is_closing(): return logger.debug(f'AsyncStream.disc() l{self.l_addr} | r{self.r_addr}') @@ -329,6 +331,12 @@ class AsyncStreamServer(AsyncStream): self.async_create_remote = async_create_remote self.async_publ_mqtt = async_publ_mqtt + def close(self) -> None: + logging.info('AsyncStreamServer.close()') + self.async_create_remote = None + self.async_publ_mqtt = None + super().close() + async def server_loop(self) -> None: '''Loop for receiving messages from the inverter (server-side)''' logger.info(f'[{self.node_id}:{self.conn_no}] ' @@ -343,7 +351,7 @@ class AsyncStreamServer(AsyncStream): # if the server connection closes, we also have to disconnect # the connection to te TSUN cloud - if self.remote.stream: + if self.remote and self.remote.stream: logger.info(f'[{self.node_id}:{self.conn_no}] disc client ' f'connection: [{self.remote.ifc.node_id}:' f'{self.remote.ifc.conn_no}]') @@ -378,6 +386,11 @@ class AsyncStreamClient(AsyncStream): AsyncStream.__init__(self, reader, writer, rstream) self.close_cb = close_cb + def close(self) -> None: + logging.info('AsyncStreamClient.close()') + self.close_cb = None + super().close() + async def client_loop(self, _: str) -> None: '''Loop for receiving messages from the TSUN cloud (client-side)''' await self.loop() diff --git a/app/src/gen3/connection_g3.py b/app/src/gen3/connection_g3.py deleted file mode 100644 index b1e8b8b..0000000 --- a/app/src/gen3/connection_g3.py +++ /dev/null @@ -1,13 +0,0 @@ -import logging - -if __name__ == "app.src.gen3.connection_g3": - from app.src.gen3.talent import Talent -else: # pragma: no cover - from gen3.talent import Talent - -logger = logging.getLogger('conn') - - -class ConnectionG3(Talent): - def __init__(self, addr, ifc, server_side, id_str=b'') -> None: - super().__init__(addr, server_side, ifc, id_str) diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index 05f12ec..b0d64fd 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -5,12 +5,12 @@ if __name__ == "app.src.gen3.inverter_g3": from app.src.inverter_base import InverterBase from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamServer - from app.src.gen3.connection_g3 import ConnectionG3 + from app.src.gen3.talent import Talent else: # pragma: no cover from inverter_base import InverterBase from async_stream import StreamPtr from async_stream import AsyncStreamServer - from gen3.connection_g3 import ConnectionG3 + from gen3.talent import Talent logger_mqtt = logging.getLogger('mqtt') @@ -27,9 +27,9 @@ class InverterG3(InverterBase): self.remote) self.local = StreamPtr( - ConnectionG3(addr, ifc, True) + Talent(addr, ifc, True, False), ifc ) async def async_create_remote(self) -> None: await InverterBase.async_create_remote( - self, 'tsun', ConnectionG3) + self, 'tsun', Talent) diff --git a/app/src/gen3/talent.py b/app/src/gen3/talent.py index f0c3246..a105770 100644 --- a/app/src/gen3/talent.py +++ b/app/src/gen3/talent.py @@ -46,7 +46,8 @@ class Talent(Message): MB_REGULAR_TIMEOUT = 60 TXT_UNKNOWN_CTRL = 'Unknown Ctrl' - def __init__(self, addr, server_side: bool, ifc: "AsyncIfc", id_str=b''): + def __init__(self, addr, ifc: "AsyncIfc", server_side: bool, + client_mode: bool = False, id_str=b''): super().__init__(server_side, self.send_modbus_cb, mb_timeout=15) ifc.rx_set_cb(self.read) ifc.prot_set_timeout_cb(self._timeout) @@ -95,10 +96,6 @@ class Talent(Message): ''' Our puplic methods ''' - def healthy(self) -> bool: - logger.debug('Talent healthy()') - return self.ifc.healthy() - def close(self) -> None: logging.debug('Talent.close()') if self.server_side: @@ -116,11 +113,11 @@ class Talent(Message): self.log_lvl.clear() self.state = State.closed self.mb_timer.close() - self.ifc.close() self.ifc.rx_set_cb(None) self.ifc.prot_set_timeout_cb(None) self.ifc.prot_set_init_new_client_conn_cb(None) self.ifc.prot_set_update_header_cb(None) + self.ifc = None super().close() def __set_serial_no(self, serial_no: str): diff --git a/app/src/gen3plus/connection_g3p.py b/app/src/gen3plus/connection_g3p.py deleted file mode 100644 index 66d327e..0000000 --- a/app/src/gen3plus/connection_g3p.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging - -if __name__ == "app.src.gen3plus.connection_g3p": - from app.src.gen3plus.solarman_v5 import SolarmanV5 -else: # pragma: no cover - from gen3plus.solarman_v5 import SolarmanV5 - -logger = logging.getLogger('conn') - - -class ConnectionG3P(SolarmanV5): - def __init__(self, addr, ifc, server_side, - client_mode: bool = False) -> None: - super().__init__(addr, server_side, client_mode, ifc) diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index a362e4f..21474eb 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -5,12 +5,12 @@ if __name__ == "app.src.gen3plus.inverter_g3p": from app.src.inverter_base import InverterBase from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamServer - from app.src.gen3plus.connection_g3p import ConnectionG3P + from app.src.gen3plus.solarman_v5 import SolarmanV5 else: # pragma: no cover from inverter_base import InverterBase from async_stream import StreamPtr from async_stream import AsyncStreamServer - from gen3plus.connection_g3p import ConnectionG3P + from gen3plus.solarman_v5 import SolarmanV5 logger_mqtt = logging.getLogger('mqtt') @@ -28,9 +28,9 @@ class InverterG3P(InverterBase): self.remote) self.local = StreamPtr( - ConnectionG3P(addr, ifc, True, client_mode) + SolarmanV5(addr, ifc, True, client_mode), ifc ) async def async_create_remote(self) -> None: await InverterBase.async_create_remote( - self, 'solarman', ConnectionG3P) + self, 'solarman', SolarmanV5) diff --git a/app/src/gen3plus/solarman_v5.py b/app/src/gen3plus/solarman_v5.py index c0cd827..51eb3a1 100644 --- a/app/src/gen3plus/solarman_v5.py +++ b/app/src/gen3plus/solarman_v5.py @@ -62,8 +62,8 @@ class SolarmanV5(Message): HDR_FMT = ' bool: - logger.debug('SolarmanV5 healthy()') - return self.ifc.healthy() - def close(self) -> None: logging.debug('Solarman.close()') if self.server_side: @@ -174,11 +170,11 @@ class SolarmanV5(Message): self.log_lvl.clear() self.state = State.closed self.mb_timer.close() - self.ifc.close() self.ifc.rx_set_cb(None) self.ifc.prot_set_timeout_cb(None) self.ifc.prot_set_init_new_client_conn_cb(None) self.ifc.prot_set_update_header_cb(None) + self.ifc = None super().close() async def send_start_cmd(self, snr: int, host: str, diff --git a/app/src/inverter.py b/app/src/inverter.py index dc224ee..7cb4a7a 100644 --- a/app/src/inverter.py +++ b/app/src/inverter.py @@ -1,12 +1,15 @@ import asyncio +import weakref import logging import json if __name__ == "app.src.inverter": + from app.src.iter_registry import IterRegistry from app.src.config import Config from app.src.mqtt import Mqtt from app.src.infos import Infos else: # pragma: no cover + from iter_registry import IterRegistry from config import Config from mqtt import Mqtt from infos import Infos @@ -14,7 +17,7 @@ else: # pragma: no cover logger_mqtt = logging.getLogger('mqtt') -class Inverter(): +class Inverter(metaclass=IterRegistry): '''class Inverter is a baseclass The class has some class method for managing common resources like a @@ -37,6 +40,8 @@ class Inverter(): async_create_remote(): Establish a client connection to the TSUN cloud async_publ_mqtt(): Publish data to MQTT broker ''' + _registry = [] + @classmethod def class_init(cls) -> None: logging.debug('Inverter.class_init') @@ -104,3 +109,6 @@ class Inverter(): logging.info('Close MQTT Task') loop.run_until_complete(cls.mqtt.close()) cls.mqtt = None + + def __init__(self): + self._registry.append(weakref.ref(self)) diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index 0b6fad7..828846b 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -20,24 +20,23 @@ logger_mqtt = logging.getLogger('mqtt') class InverterBase(Inverter): def __init__(self): + super().__init__() self.__ha_restarts = -1 def __enter__(self): return self def __exit__(self, exc_type, exc, tb) -> None: - logging.info(f'Inverter.__exit__() {self.addr}') + logging.debug(f'InverterBase.__exit__() {self.addr}') self.__del_remote() - if self.local.stream: - self.local.stream.close() - self.local.stream = None - if self.local.ifc: - self.local.ifc.close() - self.local.ifc = None + self.local.stream.close() + self.local.stream = None + self.local.ifc.close() + self.local.ifc = None def __del__(self) -> None: - logging.info(f'Inverter.__del__() {self.addr}') + logging.debug(f'InverterBase.__del__() {self.addr}') def __del_remote(self): if self.remote.stream: @@ -48,6 +47,25 @@ class InverterBase(Inverter): self.remote.ifc.close() self.remote.ifc = None + async def disc(self, shutdown_started=False) -> None: + if self.remote.stream: + self.remote.stream.shutdown_started = shutdown_started + if self.remote.ifc: + await self.remote.ifc.disc() + if self.local.stream: + self.local.stream.shutdown_started = shutdown_started + if self.local.ifc: + await self.local.ifc.disc() + + def healthy(self) -> bool: + logging.debug('Inverter healthy()') + + if self.local.ifc and not self.local.ifc.healthy(): + return False + if self.remote.ifc and not self.remote.ifc.healthy(): + return False + return True + async def async_create_remote(self, inv_prot: str, conn_class) -> None: '''Establish a client connection to the TSUN cloud''' tsun = Config.get(inv_prot) @@ -55,8 +73,6 @@ class InverterBase(Inverter): port = tsun['port'] addr = (host, port) stream = self.local.stream - if not stream: - return try: logging.info(f'[{stream.node_id}] Connect to {addr}') @@ -65,12 +81,15 @@ class InverterBase(Inverter): ifc = AsyncStreamClient( reader, writer, self.local, self.__del_remote) + self.remote.ifc = ifc if hasattr(stream, 'id_str'): self.remote.stream = conn_class( - addr, ifc, False, stream.id_str) + addr, ifc, server_side=False, + client_mode=False, id_str=stream.id_str) else: self.remote.stream = conn_class( - addr, ifc, False) + addr, ifc, server_side=False, + client_mode=False) logging.info(f'[{self.remote.stream.node_id}:' f'{self.remote.stream.conn_no}] ' diff --git a/app/src/iter_registry.py b/app/src/iter_registry.py new file mode 100644 index 0000000..b84e16f --- /dev/null +++ b/app/src/iter_registry.py @@ -0,0 +1,8 @@ + + +class IterRegistry(type): + def __iter__(cls): + for ref in cls._registry: + obj = ref() + if obj is not None: + yield obj diff --git a/app/src/messages.py b/app/src/messages.py index 0fe66b9..c68b125 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -1,13 +1,15 @@ import logging import weakref -from typing import Callable, Generator +from typing import Callable from enum import Enum if __name__ == "app.src.messages": + from app.src.iter_registry import IterRegistry from app.src.infos import Infos, Register from app.src.modbus import Modbus else: # pragma: no cover + from iter_registry import IterRegistry from infos import Infos, Register from modbus import Modbus @@ -66,14 +68,6 @@ def hex_dump_memory(level, info, data, data_len): tracer.log(level, '\n'.join(lines)) -class IterRegistry(type): - def __iter__(cls) -> Generator['Message', None, None]: - for ref in cls._registry: - obj = ref() - if obj is not None: - yield obj - - class State(Enum): '''state of the logical connection''' init = 0 diff --git a/app/src/server.py b/app/src/server.py index bbd7f83..a475845 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -5,7 +5,6 @@ import os from asyncio import StreamReader, StreamWriter from aiohttp import web from logging import config # noqa F401 -from messages import Message from inverter import Inverter from gen3.inverter_g3 import InverterG3 from gen3plus.inverter_g3p import InverterG3P @@ -38,9 +37,9 @@ async def healthy(request): if proxy_is_up: # logging.info('web reqeust healthy()') - for stream in Message: + for inverter in Inverter: try: - res = stream.healthy() + res = inverter.healthy() if not res: return web.Response(status=503, text="I have a problem") except Exception as err: @@ -88,25 +87,13 @@ async def handle_shutdown(web_task): # # first, disc all open TCP connections gracefully # - for stream in Message: - stream.shutdown_started = True - try: - await asyncio.wait_for(stream.disc(), 2) - except Exception: - pass + for inverter in Inverter: + await inverter.disc(True) + logging.info('Proxy disconnecting done') # - # second, close all open TCP connections - # - for stream in Message: - stream.close() - - await asyncio.sleep(0.1) # give time for closing - logging.info('Proxy closing done') - - # - # third, cancel the web server + # second, cancel the web server # web_task.cancel() await web_task @@ -167,7 +154,7 @@ if __name__ == "__main__": logging.info(f'ConfigErr: {ConfigErr}') Inverter.class_init() Schedule.start() - mb_tcp = ModbusTcp(loop) + ModbusTcp(loop) # # Create tasks for our listening servers. These must be tasks! If we call diff --git a/app/tests/test_connection_g3.py b/app/tests/test_connection_g3.py deleted file mode 100644 index 167a655..0000000 --- a/app/tests/test_connection_g3.py +++ /dev/null @@ -1,100 +0,0 @@ -# test_with_pytest.py -import pytest -import asyncio - -from itertools import count -from mock import patch -from app.src.async_stream import StreamPtr -from app.src.async_stream import AsyncStream, AsyncStreamServer, AsyncIfcImpl -from app.src.gen3.connection_g3 import ConnectionG3 -from app.src.gen3.talent import Talent - - -class FakeInverter(): - async def async_publ_mqtt(self) -> None: - pass # dummy funcion - - async def async_create_remote(self, inv_prot: str, conn_class) -> None: - pass # dummy function - - def __init__ (self): - self.remote = StreamPtr(None) - self.local = StreamPtr(None) - - -@pytest.fixture -def patch_async_init(): - with patch.object(AsyncStream, '__init__') as conn: - yield conn - -@pytest.fixture -def patch_talent_init(): - with patch.object(Talent, '__init__') as conn: - yield conn - -@pytest.fixture -def patch_healthy(): - with patch.object(AsyncStream, 'healthy') as conn: - yield conn - -@pytest.fixture -def patch_async_close(): - with patch.object(AsyncStream, 'close') as conn: - yield conn - -@pytest.fixture -def patch_talent_close(): - with patch.object(Talent, 'close') as conn: - yield conn - -class FakeReader(): - def __init__(self): - self.on_recv = asyncio.Event() - async def read(self, max_len: int): - await self.on_recv.wait() - return b'' - def feed_eof(self): - return - - -class FakeWriter(): - def write(self, buf: bytes): - return - def get_extra_info(self, sel: str): - if sel == 'peername': - return 'remote.intern' - elif sel == 'sockname': - return 'sock:1234' - assert False - def is_closing(self): - return False - def close(self): - return - async def wait_closed(self): - return - - - -def test_method_calls(patch_healthy, patch_async_close): - AsyncIfcImpl._ids = count(5) - spy3 = patch_healthy - spy4 = patch_async_close - reader = FakeReader() - writer = FakeWriter() - id_str = "id_string" - addr = ('proxy.local', 10000) - inv = FakeInverter() - ifc = AsyncStreamServer(reader, writer, - inv.async_publ_mqtt, - inv.async_create_remote, - inv.remote) - - conn = ConnectionG3(addr, ifc, server_side=True, id_str=id_str) - assert 5 == conn.conn_no - assert 5 == conn.ifc.get_conn_no() - conn.healthy() - - spy3.assert_called_once() - - conn.close() - spy4.assert_called_once() diff --git a/app/tests/test_connection_g3p.py b/app/tests/test_connection_g3p.py deleted file mode 100644 index 7ab1b43..0000000 --- a/app/tests/test_connection_g3p.py +++ /dev/null @@ -1,105 +0,0 @@ -# test_with_pytest.py -import pytest -import asyncio - -from itertools import count -from mock import patch -from app.src.singleton import Singleton -from app.src.async_stream import StreamPtr -from app.src.async_stream import AsyncStream, AsyncStreamServer, AsyncIfcImpl -from app.src.gen3plus.connection_g3p import ConnectionG3P -from app.src.gen3plus.solarman_v5 import SolarmanV5 - - -class FakeInverter(): - async def async_publ_mqtt(self) -> None: - pass # dummy funcion - - async def async_create_remote(self, inv_prot: str, conn_class) -> None: - pass # dummy function - - def __init__ (self): - self.remote = StreamPtr(None) - self.local = StreamPtr(None) - - -@pytest.fixture -def patch_async_init(): - with patch.object(AsyncStream, '__init__', return_value= None) as conn: - yield conn - -@pytest.fixture -def patch_solarman_init(): - with patch.object(SolarmanV5, '__init__') as conn: - yield conn - -@pytest.fixture(scope="module", autouse=True) -def module_init(): - Singleton._instances.clear() - yield - -@pytest.fixture -def patch_healthy(): - with patch.object(AsyncStream, 'healthy') as conn: - yield conn - -@pytest.fixture -def patch_async_close(): - with patch.object(AsyncStream, 'close') as conn: - yield conn - -@pytest.fixture -def patch_solarman_close(): - with patch.object(SolarmanV5, 'close') as conn: - yield conn - -class FakeReader(): - def __init__(self): - self.on_recv = asyncio.Event() - async def read(self, max_len: int): - await self.on_recv.wait() - return b'' - def feed_eof(self): - return - - -class FakeWriter(): - def write(self, buf: bytes): - return - def get_extra_info(self, sel: str): - if sel == 'peername': - return 'remote.intern' - elif sel == 'sockname': - return 'sock:1234' - assert False - def is_closing(self): - return False - def close(self): - return - async def wait_closed(self): - return - - - -def test_method_calls(patch_healthy, patch_async_close): - AsyncIfcImpl._ids = count(5) - spy3 = patch_healthy - spy4 = patch_async_close - reader = FakeReader() - writer = FakeWriter() - addr = ('proxy.local', 10000) - inv = FakeInverter() - ifc = AsyncStreamServer(reader, writer, - inv.async_publ_mqtt, - inv.async_create_remote, - inv.remote) - conn = ConnectionG3P(addr, ifc, server_side=True, client_mode=False) - assert 5 == conn.conn_no - assert 5 == conn.ifc.get_conn_no() - conn.healthy() - - spy3.assert_called_once() - - conn.close() - spy4.assert_called_once() - diff --git a/app/tests/test_inverter_g3.py b/app/tests/test_inverter_g3.py index 27ee51b..f4721c7 100644 --- a/app/tests/test_inverter_g3.py +++ b/app/tests/test_inverter_g3.py @@ -1,6 +1,7 @@ # test_with_pytest.py import pytest import asyncio +import sys,gc from mock import patch from enum import Enum @@ -8,8 +9,8 @@ from app.src.infos import Infos from app.src.config import Config from app.src.inverter import Inverter from app.src.singleton import Singleton -from app.src.gen3.connection_g3 import ConnectionG3 from app.src.gen3.inverter_g3 import InverterG3 +from app.src.async_stream import AsyncStream from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname @@ -42,11 +43,6 @@ def module_init(): Singleton._instances.clear() yield -@pytest.fixture -def patch_conn_init(): - with patch.object(ConnectionG3, '__init__', return_value= None) as conn: - yield conn - class FakeReader(): def __init__(self): self.on_recv = asyncio.Event() @@ -98,14 +94,28 @@ def patch_open_connection(): with patch.object(asyncio, 'open_connection', new_open) as conn: yield conn +@pytest.fixture +def patch_healthy(): + with patch.object(AsyncStream, 'healthy') as conn: + yield conn -def test_method_calls(): +def test_method_calls(patch_healthy): + spy = patch_healthy reader = FakeReader() writer = FakeWriter() addr = ('proxy.local', 10000) with InverterG3(reader, writer, addr) as inverter: assert inverter.local.stream assert inverter.local.ifc + for inv in Inverter: + inv.healthy() + del inv + spy.assert_called_once() + del inverter + cnt = 0 + for inv in Inverter: + cnt += 1 + assert cnt == 0 @pytest.mark.asyncio async def test_remote_conn(config_conn, patch_open_connection): @@ -117,6 +127,13 @@ async def test_remote_conn(config_conn, patch_open_connection): await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream + del inverter + + cnt = 0 + for inv in Inverter: + print(f'Inverter refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 @pytest.mark.asyncio async def test_remote_except(config_conn, patch_open_connection): @@ -136,6 +153,13 @@ async def test_remote_except(config_conn, patch_open_connection): await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None + del inverter + + cnt = 0 + for inv in Inverter: + print(f'Inverter refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 @pytest.mark.asyncio async def test_mqtt_publish(config_conn, patch_open_connection): diff --git a/app/tests/test_inverter_g3p.py b/app/tests/test_inverter_g3p.py index ff5564a..2238b66 100644 --- a/app/tests/test_inverter_g3p.py +++ b/app/tests/test_inverter_g3p.py @@ -8,7 +8,6 @@ from app.src.infos import Infos from app.src.config import Config from app.src.inverter import Inverter from app.src.singleton import Singleton -from app.src.gen3plus.connection_g3p import ConnectionG3P from app.src.gen3plus.inverter_g3p import InverterG3P from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname @@ -43,11 +42,6 @@ def module_init(): Singleton._instances.clear() yield -@pytest.fixture -def patch_conn_init(): - with patch.object(ConnectionG3P, '__init__', return_value= None) as conn: - yield conn - class FakeReader(): def __init__(self): self.on_recv = asyncio.Event() diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index 795b32f..e4e8edc 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -40,7 +40,7 @@ class FakeIfc(AsyncIfcImpl): class MemoryStream(SolarmanV5): def __init__(self, msg, chunks = (0,), server_side: bool = True): _ifc = FakeIfc() - super().__init__(('test.local', 1234), server_side, client_mode=False, ifc=_ifc) + super().__init__(('test.local', 1234), _ifc, server_side, client_mode=False) if server_side: self.mb.timeout = 0.4 # overwrite for faster testing self.mb_first_timeout = 0.5 @@ -1240,9 +1240,9 @@ def test_build_logger_modell(config_tsun_allow_all, device_ind_msg): def test_msg_iterator(): Message._registry.clear() - m1 = SolarmanV5(('test1.local', 1234), server_side=True, client_mode=False, ifc=AsyncIfcImpl()) - m2 = SolarmanV5(('test2.local', 1234), server_side=True, client_mode=False, ifc=AsyncIfcImpl()) - m3 = SolarmanV5(('test3.local', 1234), server_side=True, client_mode=False, ifc=AsyncIfcImpl()) + m1 = SolarmanV5(('test1.local', 1234), ifc=AsyncIfcImpl(), server_side=True, client_mode=False) + m2 = SolarmanV5(('test2.local', 1234), ifc=AsyncIfcImpl(), server_side=True, client_mode=False) + m3 = SolarmanV5(('test3.local', 1234), ifc=AsyncIfcImpl(), server_side=True, client_mode=False) m3.close() del m3 test1 = 0 @@ -1260,7 +1260,7 @@ def test_msg_iterator(): assert test2 == 1 def test_proxy_counter(): - m = SolarmanV5(('test.local', 1234), server_side=True, client_mode=False, ifc=AsyncIfcImpl()) + m = SolarmanV5(('test.local', 1234), ifc=AsyncIfcImpl(), server_side=True, client_mode=False) assert m.new_data == {} m.db.stat['proxy']['Unknown_Msg'] = 0 Infos.new_stat_data['proxy'] = False diff --git a/app/tests/test_talent.py b/app/tests/test_talent.py index d082885..ae156bb 100644 --- a/app/tests/test_talent.py +++ b/app/tests/test_talent.py @@ -24,7 +24,7 @@ class FakeIfc(AsyncIfcImpl): class MemoryStream(Talent): def __init__(self, msg, chunks = (0,), server_side: bool = True): self.ifc = FakeIfc() - super().__init__(('test.local', 1234), server_side, self.ifc) + super().__init__(('test.local', 1234), self.ifc, server_side) if server_side: self.mb.timeout = 0.4 # overwrite for faster testing self.mb_first_timeout = 0.5 @@ -1641,9 +1641,9 @@ def test_ctrl_byte(): def test_msg_iterator(): - m1 = Talent(('test1.local', 1234), server_side=True, ifc=AsyncIfcImpl()) - m2 = Talent(('test2.local', 1234), server_side=True, ifc=AsyncIfcImpl()) - m3 = Talent(('test3.local', 1234), server_side=True, ifc=AsyncIfcImpl()) + m1 = Talent(('test1.local', 1234), ifc=AsyncIfcImpl(), server_side=True) + m2 = Talent(('test2.local', 1234), ifc=AsyncIfcImpl(), server_side=True) + m3 = Talent(('test3.local', 1234), ifc=AsyncIfcImpl(), server_side=True) m3.close() del m3 test1 = 0 From 22d59ed659ce0ad36b62d5b17abfa614e79857e5 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 3 Oct 2024 15:08:07 +0200 Subject: [PATCH 08/20] move more code into InverterBase class --- app/proxy.svg | 574 +++++++------------------------ app/proxy.yuml | 41 +-- app/proxy_2.svg | 418 ++++++++++++++++++++++ app/proxy_2.yuml | 49 +++ app/src/gen3/inverter_g3.py | 26 +- app/src/gen3plus/inverter_g3p.py | 27 +- app/src/inverter_base.py | 30 +- app/src/modbus_tcp.py | 2 +- app/src/server.py | 3 +- app/tests/test_inverter_base.py | 241 +++++++++++++ app/tests/test_inverter_g3.py | 15 +- app/tests/test_inverter_g3p.py | 15 +- app/tests/test_modbus_tcp.py | 10 +- 13 files changed, 892 insertions(+), 559 deletions(-) create mode 100644 app/proxy_2.svg create mode 100644 app/proxy_2.yuml create mode 100644 app/tests/test_inverter_base.py diff --git a/app/proxy.svg b/app/proxy.svg index 88a5606..6493fc7 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,519 +4,193 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -IterRegistry - - -__iter__ + +IterRegistry + + +__iter__ A3 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt -__ha_restarts - -async_create_remote(inv_prot, conn_class)async_publ_mqtt() + +Inverter + +cls.db_stat +cls.entity_prfx +cls.discovery_prfx +cls.proxy_node_id +cls.proxy_unique_id +cls.mqtt:Mqtt +__ha_restarts + +async_create_remote(inv_prot, conn_class)async_publ_mqtt() A1->A3 - - - - - -A18 - -Message - -node_id - -inc_counter() -dec_counter() - - - -A1->A18 - - + + A2 - -Mqtt -<<Singleton>> - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() + +Mqtt +<<Singleton>> + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() A3->A2 - - - + + + A4 - -InverterBase - - -<async>disc(shutdown_started) -<async>async_create_remote() -healthy() + +InverterBase + + +<async>disc(shutdown_started) +<async>async_create_remote() +healthy() A3->A4 - - + + A5 - -InverterG3 - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() + +InverterG3 + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() A4->A5 - - + + A6 - -InverterG3P - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() + +InverterG3P + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() A4->A6 - - - - - -A10 - -AsyncStreamServer - -async_create_remote - -<async>server_loop() -<async>_async_forward() -<async>publish_outstanding_mqtt() -close() - - - -A5->A10 - - - -local - - - -A11 - -AsyncStreamClient - - -<async>client_loop() -<async>_async_forward()) - - - -A5->A11 - - -remote - - - -A6->A10 - - - -local - - - -A6->A11 - - -remote + + A7 - -<<AsyncIfc>> - - -set_node_id() -get_conn_no() -tx_add() -tx_flush() -tx_get() -tx_peek() -tx_log() -tx_clear() -tx_len() -fwd_add() -fwd_flush() -fwd_log() -fwd_clear() -rx_get() -rx_peek() -rx_log() -rx_clear() -rx_len() -rx_set_cb() -prot_set_timeout_cb() + +<<AsyncIfc>> + + + +A5->A7 + + +1..2 A8 - -AsyncIfcImpl - -fwd_fifo:ByteFifo -tx_fifo:ByteFifo -rx_fifo:ByteFifo -conn_no:Count -node_id -timeout_cb + +<<MessageProt>> - + -A7->A8 - - +A5->A8 + + +1..2 + + + +A6->A7 + + +1..2 + + + +A6->A8 + + +1..2 + + + +A8->A7 + + +use A9 - -AsyncStream - -reader -writer -addr -r_addr -l_addr - -<async>loop -disc() -close() -healthy() -__async_read() -__async_write() -__async_forward() + +ModbusConn + +host +port +addr +stream:InverterG3P + - - -A8->A9 - - - - - -A9->A10 - - - - - -A9->A11 - - - - - -A12 - -Talent - -ifc:AsyncIfc -conn_no -addr -await_conn_resp_cnt -id_str -contact_name -contact_mail -db:InfosG3 -mb:Modbus -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -healthy() -close() - - - -A12->A5 - - -remote - - - -A12->A5 - - - -local - - - -A12->A7 - - -use - - - -A15 - -InfosG3 - - -ha_confs() -parse() - - - -A12->A15 - - - - - -A17 - -ConnectionG3 - - - -A12->A17 - - - - - -A13 - -SolarmanV5 - -ifc:AsyncIfc -conn_no -addr -control -serial -snr -db:InfosG3P -mb:Modbus -switch - -msg_unknown() -healthy() -close() - - - -A13->A6 - - -remote - - - -A13->A6 - - - -local - - - -A13->A7 - - -use - - - -A16 - -InfosG3P - - -ha_confs() -parse() - - - -A13->A16 - - - - - -A14 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -ha_remove -update_db -set_db_def_value -get_db_value -ignore_this_device - - - -A14->A15 - - - - - -A14->A16 - - - - - -A18->A12 - - - - - -A18->A13 - - - - - -A19 - -Modbus - -que -snd_handler -rsp_handler -timeout -max_retires -last_xxx -err -retry_cnt -req_pend -tim - -build_msg() -recv_req() -recv_resp() -close() - - - -A19->A12 - - -has -1 - - - -A19->A13 - - -has -1 - - - -A20 - -ModbusConn - -host -port -addr -stream:InverterG3P - - - - -A20->A6 - - -1 -has + + +A9->A6 + + +1 +has diff --git a/app/proxy.yuml b/app/proxy.yuml index 85bf0d0..daf04aa 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -15,46 +15,17 @@ [IterRegistry]^[Inverter|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list;state|_read():void;close():void;inc_counter():void;dec_counter():void] -[<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_flush();fwd_log();fwd_clear();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] -[AsyncIfcImpl|fwd_fifo:ByteFifo;tx_fifo:ByteFifo;rx_fifo:ByteFifo;conn_no:Count;node_id;timeout_cb] -[AsyncStream|reader;writer;addr;r_addr;l_addr|;loop;disc();close();healthy();;__async_read();__async_write();__async_forward()] -[AsyncStreamServer|async_create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] -[AsyncStreamClient||client_loop();_async_forward())] -[<>]^-.-[AsyncIfcImpl] -[AsyncIfcImpl]^[AsyncStream] -[AsyncStream]^[AsyncStreamServer] -[AsyncStream]^[AsyncStreamClient] +[<>] -[Talent|ifc:AsyncIfc;conn_no;addr;;await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;healthy();close()] -[Talent][AsyncStreamClient] -[Talent]<-local++[InverterG3] -[InverterG3]++local->[AsyncStreamServer] +[InverterG3]-1..2>[<>] +[InverterG3]-1..2>[<>] -[SolarmanV5|ifc:AsyncIfc;conn_no;addr;;control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;healthy();close()] -[SolarmanV5][AsyncStreamClient] -[SolarmanV5]<-local++[InverterG3P] -[InverterG3P]++local->[AsyncStreamServer] +[InverterG3P]-1..2>[<>] +[InverterG3P]-1..2>[<>] -[Infos|stat;new_stat_data;info_dev|static_init();dev_value();inc_counter();dec_counter();ha_proxy_conf;ha_conf;ha_remove;update_db;set_db_def_value;get_db_value;ignore_this_device] -[Infos]^[InfosG3||ha_confs();parse()] -[Infos]^[InfosG3P||ha_confs();parse()] +[<>]use-.->[<>] -[Talent]^[ConnectionG3] -[Talent]use->[<>] -[Talent]->[InfosG3] -[SolarmanV5]use->[<>] -[SolarmanV5]->[InfosG3P] - -[IterRegistry]^[Message|node_id|inc_counter();dec_counter()] -[Message]^[Talent] -[Message]^[SolarmanV5] - -[Modbus|que;;snd_handler;rsp_handler;timeout;max_retires;last_xxx;err;retry_cnt;req_pend;tim|build_msg();recv_req();recv_resp();close()] -[Modbus]<1-has[SolarmanV5] -[Modbus]<1-has[Talent] [ModbusConn|host;port;addr;stream:InverterG3P;|]has-1>[InverterG3P] diff --git a/app/proxy_2.svg b/app/proxy_2.svg new file mode 100644 index 0000000..63e6657 --- /dev/null +++ b/app/proxy_2.svg @@ -0,0 +1,418 @@ + + + + + + +G + + + +A0 + + + +You can stick notes +on diagrams too! + + + +A1 + +IterRegistry + + +__iter__ + + + +A14 + +Message + +node_id + +inc_counter() +dec_counter() + + + +A1->A14 + + + + + +A2 + +InverterG3 + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() + + + +A7 + +AsyncStreamServer + +async_create_remote + +<async>server_loop() +<async>_async_forward() +<async>publish_outstanding_mqtt() +close() + + + +A2->A7 + + + +local + + + +A8 + +AsyncStreamClient + + +<async>client_loop() +<async>_async_forward()) + + + +A2->A8 + + +remote + + + +A3 + +InverterG3P + +addr +remote:StreamPtr +local:StreamPtr + +async_create_remote() +close() + + + +A3->A7 + + + +local + + + +A3->A8 + + +remote + + + +A4 + +<<AsyncIfc>> + + +set_node_id() +get_conn_no() +tx_add() +tx_flush() +tx_get() +tx_peek() +tx_log() +tx_clear() +tx_len() +fwd_add() +fwd_flush() +fwd_log() +fwd_clear() +rx_get() +rx_peek() +rx_log() +rx_clear() +rx_len() +rx_set_cb() +prot_set_timeout_cb() + + + +A5 + +AsyncIfcImpl + +fwd_fifo:ByteFifo +tx_fifo:ByteFifo +rx_fifo:ByteFifo +conn_no:Count +node_id +timeout_cb + + + +A4->A5 + + + + + +A6 + +AsyncStream + +reader +writer +addr +r_addr +l_addr + +<async>loop +disc() +close() +healthy() +__async_read() +__async_write() +__async_forward() + + + +A5->A6 + + + + + +A6->A7 + + + + + +A6->A8 + + + + + +A9 + +Talent + +ifc:AsyncIfc +conn_no +addr +await_conn_resp_cnt +id_str +contact_name +contact_mail +db:InfosG3 +mb:Modbus +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +healthy() +close() + + + +A9->A2 + + +remote + + + +A9->A2 + + + +local + + + +A9->A4 + + +use + + + +A12 + +InfosG3 + + +ha_confs() +parse() + + + +A9->A12 + + + + + +A10 + +SolarmanV5 + +ifc:AsyncIfc +conn_no +addr +control +serial +snr +db:InfosG3P +mb:Modbus +switch + +msg_unknown() +healthy() +close() + + + +A10->A3 + + +remote + + + +A10->A3 + + + +local + + + +A10->A4 + + +use + + + +A13 + +InfosG3P + + +ha_confs() +parse() + + + +A10->A13 + + + + + +A11 + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +ha_remove +update_db +set_db_def_value +get_db_value +ignore_this_device + + + +A11->A12 + + + + + +A11->A13 + + + + + +A14->A9 + + + + + +A14->A10 + + + + + +A15 + +Modbus + +que +snd_handler +rsp_handler +timeout +max_retires +last_xxx +err +retry_cnt +req_pend +tim + +build_msg() +recv_req() +recv_resp() +close() + + + +A15->A9 + + +has +1 + + + +A15->A10 + + +has +1 + + + diff --git a/app/proxy_2.yuml b/app/proxy_2.yuml new file mode 100644 index 0000000..2d96500 --- /dev/null +++ b/app/proxy_2.yuml @@ -0,0 +1,49 @@ +// {type:class} +// {direction:topDown} +// {generate:true} + +[note: You can stick notes on diagrams too!{bg:cornsilk}] +[IterRegistry||__iter__] + +[InverterG3|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] +[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] + +[<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_flush();fwd_log();fwd_clear();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] +[AsyncIfcImpl|fwd_fifo:ByteFifo;tx_fifo:ByteFifo;rx_fifo:ByteFifo;conn_no:Count;node_id;timeout_cb] +[AsyncStream|reader;writer;addr;r_addr;l_addr|;loop;disc();close();healthy();;__async_read();__async_write();__async_forward()] +[AsyncStreamServer|async_create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] +[AsyncStreamClient||client_loop();_async_forward())] +[<>]^-.-[AsyncIfcImpl] +[AsyncIfcImpl]^[AsyncStream] +[AsyncStream]^[AsyncStreamServer] +[AsyncStream]^[AsyncStreamClient] + + +[Talent|ifc:AsyncIfc;conn_no;addr;;await_conn_resp_cnt;id_str;contact_name;contact_mail;db:InfosG3;mb:Modbus;switch|msg_contact_info();msg_ota_update();msg_get_time();msg_collector_data();msg_inverter_data();msg_unknown();;healthy();close()] +[Talent][AsyncStreamClient] +[Talent]<-local++[InverterG3] +[InverterG3]++local->[AsyncStreamServer] + +[SolarmanV5|ifc:AsyncIfc;conn_no;addr;;control;serial;snr;db:InfosG3P;mb:Modbus;switch|msg_unknown();;healthy();close()] +[SolarmanV5][AsyncStreamClient] +[SolarmanV5]<-local++[InverterG3P] +[InverterG3P]++local->[AsyncStreamServer] + +[Infos|stat;new_stat_data;info_dev|static_init();dev_value();inc_counter();dec_counter();ha_proxy_conf;ha_conf;ha_remove;update_db;set_db_def_value;get_db_value;ignore_this_device] +[Infos]^[InfosG3||ha_confs();parse()] +[Infos]^[InfosG3P||ha_confs();parse()] + +[Talent]use->[<>] +[Talent]->[InfosG3] +[SolarmanV5]use->[<>] +[SolarmanV5]->[InfosG3P] + +[IterRegistry]^[Message|node_id|inc_counter();dec_counter()] +[Message]^[Talent] +[Message]^[SolarmanV5] + +[Modbus|que;;snd_handler;rsp_handler;timeout;max_retires;last_xxx;err;retry_cnt;req_pend;tim|build_msg();recv_req();recv_resp();close()] +[Modbus]<1-has[SolarmanV5] +[Modbus]<1-has[Talent] diff --git a/app/src/gen3/inverter_g3.py b/app/src/gen3/inverter_g3.py index b0d64fd..de9c519 100644 --- a/app/src/gen3/inverter_g3.py +++ b/app/src/gen3/inverter_g3.py @@ -1,35 +1,13 @@ -import logging from asyncio import StreamReader, StreamWriter if __name__ == "app.src.gen3.inverter_g3": from app.src.inverter_base import InverterBase - from app.src.async_stream import StreamPtr - from app.src.async_stream import AsyncStreamServer from app.src.gen3.talent import Talent else: # pragma: no cover from inverter_base import InverterBase - from async_stream import StreamPtr - from async_stream import AsyncStreamServer from gen3.talent import Talent -logger_mqtt = logging.getLogger('mqtt') - - class InverterG3(InverterBase): - def __init__(self, reader: StreamReader, writer: StreamWriter, addr): - super().__init__() - self.addr = addr - self.remote = StreamPtr(None) - ifc = AsyncStreamServer(reader, writer, - self.async_publ_mqtt, - self.async_create_remote, - self.remote) - - self.local = StreamPtr( - Talent(addr, ifc, True, False), ifc - ) - - async def async_create_remote(self) -> None: - await InverterBase.async_create_remote( - self, 'tsun', Talent) + def __init__(self, reader: StreamReader, writer: StreamWriter): + super().__init__(reader, writer, 'tsun', Talent) diff --git a/app/src/gen3plus/inverter_g3p.py b/app/src/gen3plus/inverter_g3p.py index 21474eb..cc27bb4 100644 --- a/app/src/gen3plus/inverter_g3p.py +++ b/app/src/gen3plus/inverter_g3p.py @@ -1,36 +1,15 @@ -import logging from asyncio import StreamReader, StreamWriter if __name__ == "app.src.gen3plus.inverter_g3p": from app.src.inverter_base import InverterBase - from app.src.async_stream import StreamPtr - from app.src.async_stream import AsyncStreamServer from app.src.gen3plus.solarman_v5 import SolarmanV5 else: # pragma: no cover from inverter_base import InverterBase - from async_stream import StreamPtr - from async_stream import AsyncStreamServer from gen3plus.solarman_v5 import SolarmanV5 -logger_mqtt = logging.getLogger('mqtt') - - class InverterG3P(InverterBase): - def __init__(self, reader: StreamReader, writer: StreamWriter, addr, + def __init__(self, reader: StreamReader, writer: StreamWriter, client_mode: bool = False): - super().__init__() - self.addr = addr - self.remote = StreamPtr(None) - ifc = AsyncStreamServer(reader, writer, - self.async_publ_mqtt, - self.async_create_remote, - self.remote) - - self.local = StreamPtr( - SolarmanV5(addr, ifc, True, client_mode), ifc - ) - - async def async_create_remote(self) -> None: - await InverterBase.async_create_remote( - self, 'solarman', SolarmanV5) + super().__init__(reader, writer, 'solarman', + SolarmanV5, client_mode) diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index 828846b..026b91b 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -3,15 +3,20 @@ import logging import traceback import json from aiomqtt import MqttCodeError +from asyncio import StreamReader, StreamWriter if __name__ == "app.src.inverter_base": from app.src.inverter import Inverter + from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamClient + from app.src.async_stream import AsyncStreamServer from app.src.config import Config from app.src.infos import Infos else: # pragma: no cover from inverter import Inverter + from async_stream import StreamPtr from async_stream import AsyncStreamClient + from async_stream import AsyncStreamServer from config import Config from infos import Infos @@ -19,9 +24,23 @@ logger_mqtt = logging.getLogger('mqtt') class InverterBase(Inverter): - def __init__(self): + def __init__(self, reader: StreamReader, writer: StreamWriter, + config_id: str, prot_class, + client_mode: bool = False): super().__init__() + self.addr = writer.get_extra_info('peername') + self.config_id = config_id + self.prot_class = prot_class self.__ha_restarts = -1 + self.remote = StreamPtr(None) + ifc = AsyncStreamServer(reader, writer, + self.async_publ_mqtt, + self.async_create_remote, + self.remote) + + self.local = StreamPtr( + self.prot_class(self.addr, ifc, True, client_mode), ifc + ) def __enter__(self): return self @@ -66,9 +85,10 @@ class InverterBase(Inverter): return False return True - async def async_create_remote(self, inv_prot: str, conn_class) -> None: + async def async_create_remote(self) -> None: '''Establish a client connection to the TSUN cloud''' - tsun = Config.get(inv_prot) + + tsun = Config.get(self.config_id) host = tsun['host'] port = tsun['port'] addr = (host, port) @@ -83,11 +103,11 @@ class InverterBase(Inverter): self.remote.ifc = ifc if hasattr(stream, 'id_str'): - self.remote.stream = conn_class( + self.remote.stream = self.prot_class( addr, ifc, server_side=False, client_mode=False, id_str=stream.id_str) else: - self.remote.stream = conn_class( + self.remote.stream = self.prot_class( addr, ifc, server_side=False, client_mode=False) diff --git a/app/src/modbus_tcp.py b/app/src/modbus_tcp.py index 8bacce3..f6e8245 100644 --- a/app/src/modbus_tcp.py +++ b/app/src/modbus_tcp.py @@ -25,7 +25,7 @@ class ModbusConn(): '''Establish a client connection to the TSUN cloud''' connection = asyncio.open_connection(self.host, self.port) reader, writer = await connection - self.inverter = InverterG3P(reader, writer, self.addr, + self.inverter = InverterG3P(reader, writer, client_mode=True) self.inverter.__enter__() stream = self.inverter.local.stream diff --git a/app/src/server.py b/app/src/server.py index a475845..0ff5202 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -72,8 +72,7 @@ async def webserver(addr, port): async def handle_client(reader: StreamReader, writer: StreamWriter, inv_class): '''Handles a new incoming connection and starts an async loop''' - addr = writer.get_extra_info('peername') - with inv_class(reader, writer, addr) as inv: + with inv_class(reader, writer) as inv: await inv.local.ifc.server_loop() diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py new file mode 100644 index 0000000..0a4099f --- /dev/null +++ b/app/tests/test_inverter_base.py @@ -0,0 +1,241 @@ +# test_with_pytest.py +import pytest +import asyncio +import sys,gc +import weakref + +from mock import patch +from enum import Enum +from app.src.infos import Infos +from app.src.config import Config +from app.src.inverter import Inverter +from app.src.singleton import Singleton +from app.src.protocol_ifc import ProtocolIfc +from app.src.inverter_base import InverterBase +from app.src.messages import Message +from app.src.async_stream import AsyncStream + +from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname + +pytest_plugins = ('pytest_asyncio',) + +# initialize the proxy statistics +Infos.static_init() + +@pytest.fixture +def config_conn(): + Config.act_config = { + 'mqtt':{ + 'host': test_hostname, + 'port': test_port, + 'user': '', + 'passwd': '' + }, + 'ha':{ + 'auto_conf_prefix': 'homeassistant', + 'discovery_prefix': 'homeassistant', + 'entity_prefix': 'tsun', + 'proxy_node_id': 'test_1', + 'proxy_unique_id': '' + }, + 'tsun':{'enabled': True, 'host': 'test_cloud.local', 'port': 1234}, 'inverters':{'allow_all':True} + } + +@pytest.fixture(scope="module", autouse=True) +def module_init(): + Singleton._instances.clear() + yield + +class FakeProtocol(ProtocolIfc): + def __init__(self, addr, ifc, server_side: bool, + client_mode: bool = False, id_str=b''): + # self._registry.append(weakref.ref(self)) + pass # empty mockup + + def close(self): + pass # empty mockup + +class FakeReader(): + def __init__(self): + self.on_recv = asyncio.Event() + async def read(self, max_len: int): + await self.on_recv.wait() + return b'' + def feed_eof(self): + return + + +class FakeWriter(): + def write(self, buf: bytes): + return + def get_extra_info(self, sel: str): + if sel == 'peername': + return 'remote.intern' + elif sel == 'sockname': + return 'sock:1234' + assert False + def is_closing(self): + return False + def close(self): + return + async def wait_closed(self): + return + +class TestType(Enum): + RD_TEST_0_BYTES = 1 + RD_TEST_TIMEOUT = 2 + RD_TEST_EXCEPT = 3 + + +test = TestType.RD_TEST_0_BYTES + +@pytest.fixture +def patch_open_connection(): + async def new_conn(conn): + await asyncio.sleep(0) + return FakeReader(), FakeWriter() + + def new_open(host: str, port: int): + global test + if test == TestType.RD_TEST_TIMEOUT: + raise ConnectionRefusedError + elif test == TestType.RD_TEST_EXCEPT: + raise ValueError("Value cannot be negative") # Compliant + return new_conn(None) + + with patch.object(asyncio, 'open_connection', new_open) as conn: + yield conn + +@pytest.fixture +def get_test_inverter(): + reader = FakeReader() + writer = FakeWriter() + with InverterBase(reader, writer, 'tsun', FakeProtocol) as inverter: + yield inverter + +@pytest.fixture +def patch_healthy(): + with patch.object(AsyncStream, 'healthy') as conn: + yield conn + +def test_method_calls(get_test_inverter, patch_healthy): + spy = patch_healthy + with get_test_inverter as inverter: + assert inverter.local.stream + assert inverter.local.ifc + for inv in Inverter: + inv.healthy() + del inv + spy.assert_called_once() + del inverter + cnt = 0 + for inv in Inverter: + print(f'inv:{gc.get_referrers()}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_remote_conn(config_conn, patch_open_connection): + _ = config_conn + _ = patch_open_connection + assert asyncio.get_running_loop() + + with InverterBase(FakeReader(), FakeWriter()) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream + del inverter + + cnt = 0 + for inv in Inverter: + print(f'Inverter refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_remote_except(config_conn, patch_open_connection): + _ = config_conn + _ = patch_open_connection + assert asyncio.get_running_loop() + + global test + test = TestType.RD_TEST_TIMEOUT + + with InverterBase(FakeReader(), FakeWriter()) as inverter: + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None + + test = TestType.RD_TEST_EXCEPT + await inverter.async_create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream==None + del inverter + + cnt = 0 + for inv in Inverter: + print(f'Inverter refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_mqtt_publish(config_conn, patch_open_connection): + _ = config_conn + _ = patch_open_connection + assert asyncio.get_running_loop() + + Inverter.class_init() + + with InverterBase(FakeReader(), FakeWriter()) as inverter: + stream = inverter.local.stream + await inverter.async_publ_mqtt() # check call with invalid unique_id + stream._Talent__set_serial_no(serial_no= "123344") + + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == False + + stream.new_data['env'] = True + stream.db.db['env'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['env'] == False + + Infos.new_stat_data['proxy'] = True + await inverter.async_publ_mqtt() + assert Infos.new_stat_data['proxy'] == False + +@pytest.mark.asyncio +async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): + _ = config_conn + _ = patch_open_connection + _ = patch_mqtt_err + assert asyncio.get_running_loop() + + Inverter.class_init() + + with InverterBase(FakeReader(), FakeWriter()) as inverter: + stream = inverter.local.stream + stream._Talent__set_serial_no(serial_no= "123344") + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True + +@pytest.mark.asyncio +async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except): + _ = config_conn + _ = patch_open_connection + _ = patch_mqtt_except + assert asyncio.get_running_loop() + + Inverter.class_init() + + with InverterBase(FakeReader(), FakeWriter()) as inverter: + stream = inverter.local.stream + stream._Talent__set_serial_no(serial_no= "123344") + + stream.new_data['inverter'] = True + stream.db.db['inverter'] = {} + await inverter.async_publ_mqtt() + assert stream.new_data['inverter'] == True diff --git a/app/tests/test_inverter_g3.py b/app/tests/test_inverter_g3.py index f4721c7..2fbe4e8 100644 --- a/app/tests/test_inverter_g3.py +++ b/app/tests/test_inverter_g3.py @@ -103,8 +103,9 @@ def test_method_calls(patch_healthy): spy = patch_healthy reader = FakeReader() writer = FakeWriter() - addr = ('proxy.local', 10000) - with InverterG3(reader, writer, addr) as inverter: + Inverter._registry.clear() + + with InverterG3(reader, writer) as inverter: assert inverter.local.stream assert inverter.local.ifc for inv in Inverter: @@ -123,7 +124,7 @@ async def test_remote_conn(config_conn, patch_open_connection): _ = patch_open_connection assert asyncio.get_running_loop() - with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + with InverterG3(FakeReader(), FakeWriter()) as inverter: await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream @@ -144,7 +145,7 @@ async def test_remote_except(config_conn, patch_open_connection): global test test = TestType.RD_TEST_TIMEOUT - with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + with InverterG3(FakeReader(), FakeWriter()) as inverter: await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None @@ -169,7 +170,7 @@ async def test_mqtt_publish(config_conn, patch_open_connection): Inverter.class_init() - with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream await inverter.async_publ_mqtt() # check call with invalid unique_id stream._Talent__set_serial_no(serial_no= "123344") @@ -197,7 +198,7 @@ async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): Inverter.class_init() - with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream stream._Talent__set_serial_no(serial_no= "123344") stream.new_data['inverter'] = True @@ -214,7 +215,7 @@ async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except Inverter.class_init() - with InverterG3(FakeReader(), FakeWriter(), ('proxy.local', 10000)) as inverter: + with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream stream._Talent__set_serial_no(serial_no= "123344") diff --git a/app/tests/test_inverter_g3p.py b/app/tests/test_inverter_g3p.py index 2238b66..603d356 100644 --- a/app/tests/test_inverter_g3p.py +++ b/app/tests/test_inverter_g3p.py @@ -96,8 +96,9 @@ def patch_open_connection(): def test_method_calls(): reader = FakeReader() writer = FakeWriter() - addr = ('proxy.local', 10000) - with InverterG3P(reader, writer, addr, client_mode=False) as inverter: + Inverter._registry.clear() + + with InverterG3P(reader, writer, client_mode=False) as inverter: assert inverter.local.stream assert inverter.local.ifc @@ -107,7 +108,7 @@ async def test_remote_conn(config_conn, patch_open_connection): _ = patch_open_connection assert asyncio.get_running_loop() - with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream @@ -121,7 +122,7 @@ async def test_remote_except(config_conn, patch_open_connection): global test test = TestType.RD_TEST_TIMEOUT - with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: await inverter.async_create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None @@ -139,7 +140,7 @@ async def test_mqtt_publish(config_conn, patch_open_connection): Inverter.class_init() - with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream await inverter.async_publ_mqtt() # check call with invalid unique_id stream._SolarmanV5__set_serial_no(snr= 123344) @@ -167,7 +168,7 @@ async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): Inverter.class_init() - with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream stream._SolarmanV5__set_serial_no(snr= 123344) stream.new_data['inverter'] = True @@ -184,7 +185,7 @@ async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except Inverter.class_init() - with InverterG3P(FakeReader(), FakeWriter(), ('proxy.local', 10000), client_mode=False) as inverter: + with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream stream._SolarmanV5__set_serial_no(snr= 123344) diff --git a/app/tests/test_modbus_tcp.py b/app/tests/test_modbus_tcp.py index 22887e8..5611a03 100644 --- a/app/tests/test_modbus_tcp.py +++ b/app/tests/test_modbus_tcp.py @@ -93,11 +93,13 @@ class FakeReader(): class FakeWriter(): + def __init__(self, conn='remote.intern'): + self.conn = conn def write(self, buf: bytes): return def get_extra_info(self, sel: str): if sel == 'peername': - return 'remote.intern' + return self.conn elif sel == 'sockname': return 'sock:1234' assert False @@ -113,13 +115,13 @@ class FakeWriter(): def patch_open(): async def new_conn(conn): await asyncio.sleep(0) - return FakeReader(), FakeWriter() + return FakeReader(), FakeWriter(conn) def new_open(host: str, port: int): global test if test == TestType.RD_TEST_TIMEOUT: raise TimeoutError - return new_conn(None) + return new_conn(f'{host}:{port}') with patch.object(asyncio, 'open_connection', new_open) as conn: yield conn @@ -153,7 +155,7 @@ async def test_modbus_conn(patch_open): async with ModbusConn('test.local', 1234) as inverter: stream = inverter.local.stream assert stream.node_id == 'G3P' - assert stream.addr == ('test.local', 1234) + assert stream.addr == ('test.local:1234') assert type(stream.ifc._reader) is FakeReader assert type(stream.ifc._writer) is FakeWriter assert Infos.stat['proxy']['Inverter_Cnt'] == 1 From 84034127e3653ff8daba220f6675d4f6df153248 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Thu, 3 Oct 2024 15:11:22 +0200 Subject: [PATCH 09/20] remove test_inverter_base.py --- app/tests/test_inverter_base.py | 241 -------------------------------- 1 file changed, 241 deletions(-) delete mode 100644 app/tests/test_inverter_base.py diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py deleted file mode 100644 index 0a4099f..0000000 --- a/app/tests/test_inverter_base.py +++ /dev/null @@ -1,241 +0,0 @@ -# test_with_pytest.py -import pytest -import asyncio -import sys,gc -import weakref - -from mock import patch -from enum import Enum -from app.src.infos import Infos -from app.src.config import Config -from app.src.inverter import Inverter -from app.src.singleton import Singleton -from app.src.protocol_ifc import ProtocolIfc -from app.src.inverter_base import InverterBase -from app.src.messages import Message -from app.src.async_stream import AsyncStream - -from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname - -pytest_plugins = ('pytest_asyncio',) - -# initialize the proxy statistics -Infos.static_init() - -@pytest.fixture -def config_conn(): - Config.act_config = { - 'mqtt':{ - 'host': test_hostname, - 'port': test_port, - 'user': '', - 'passwd': '' - }, - 'ha':{ - 'auto_conf_prefix': 'homeassistant', - 'discovery_prefix': 'homeassistant', - 'entity_prefix': 'tsun', - 'proxy_node_id': 'test_1', - 'proxy_unique_id': '' - }, - 'tsun':{'enabled': True, 'host': 'test_cloud.local', 'port': 1234}, 'inverters':{'allow_all':True} - } - -@pytest.fixture(scope="module", autouse=True) -def module_init(): - Singleton._instances.clear() - yield - -class FakeProtocol(ProtocolIfc): - def __init__(self, addr, ifc, server_side: bool, - client_mode: bool = False, id_str=b''): - # self._registry.append(weakref.ref(self)) - pass # empty mockup - - def close(self): - pass # empty mockup - -class FakeReader(): - def __init__(self): - self.on_recv = asyncio.Event() - async def read(self, max_len: int): - await self.on_recv.wait() - return b'' - def feed_eof(self): - return - - -class FakeWriter(): - def write(self, buf: bytes): - return - def get_extra_info(self, sel: str): - if sel == 'peername': - return 'remote.intern' - elif sel == 'sockname': - return 'sock:1234' - assert False - def is_closing(self): - return False - def close(self): - return - async def wait_closed(self): - return - -class TestType(Enum): - RD_TEST_0_BYTES = 1 - RD_TEST_TIMEOUT = 2 - RD_TEST_EXCEPT = 3 - - -test = TestType.RD_TEST_0_BYTES - -@pytest.fixture -def patch_open_connection(): - async def new_conn(conn): - await asyncio.sleep(0) - return FakeReader(), FakeWriter() - - def new_open(host: str, port: int): - global test - if test == TestType.RD_TEST_TIMEOUT: - raise ConnectionRefusedError - elif test == TestType.RD_TEST_EXCEPT: - raise ValueError("Value cannot be negative") # Compliant - return new_conn(None) - - with patch.object(asyncio, 'open_connection', new_open) as conn: - yield conn - -@pytest.fixture -def get_test_inverter(): - reader = FakeReader() - writer = FakeWriter() - with InverterBase(reader, writer, 'tsun', FakeProtocol) as inverter: - yield inverter - -@pytest.fixture -def patch_healthy(): - with patch.object(AsyncStream, 'healthy') as conn: - yield conn - -def test_method_calls(get_test_inverter, patch_healthy): - spy = patch_healthy - with get_test_inverter as inverter: - assert inverter.local.stream - assert inverter.local.ifc - for inv in Inverter: - inv.healthy() - del inv - spy.assert_called_once() - del inverter - cnt = 0 - for inv in Inverter: - print(f'inv:{gc.get_referrers()}') - cnt += 1 - assert cnt == 0 - -@pytest.mark.asyncio -async def test_remote_conn(config_conn, patch_open_connection): - _ = config_conn - _ = patch_open_connection - assert asyncio.get_running_loop() - - with InverterBase(FakeReader(), FakeWriter()) as inverter: - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream - del inverter - - cnt = 0 - for inv in Inverter: - print(f'Inverter refs:{gc.get_referrers(inv)}') - cnt += 1 - assert cnt == 0 - -@pytest.mark.asyncio -async def test_remote_except(config_conn, patch_open_connection): - _ = config_conn - _ = patch_open_connection - assert asyncio.get_running_loop() - - global test - test = TestType.RD_TEST_TIMEOUT - - with InverterBase(FakeReader(), FakeWriter()) as inverter: - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - - test = TestType.RD_TEST_EXCEPT - await inverter.async_create_remote() - await asyncio.sleep(0) - assert inverter.remote.stream==None - del inverter - - cnt = 0 - for inv in Inverter: - print(f'Inverter refs:{gc.get_referrers(inv)}') - cnt += 1 - assert cnt == 0 - -@pytest.mark.asyncio -async def test_mqtt_publish(config_conn, patch_open_connection): - _ = config_conn - _ = patch_open_connection - assert asyncio.get_running_loop() - - Inverter.class_init() - - with InverterBase(FakeReader(), FakeWriter()) as inverter: - stream = inverter.local.stream - await inverter.async_publ_mqtt() # check call with invalid unique_id - stream._Talent__set_serial_no(serial_no= "123344") - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == False - - stream.new_data['env'] = True - stream.db.db['env'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['env'] == False - - Infos.new_stat_data['proxy'] = True - await inverter.async_publ_mqtt() - assert Infos.new_stat_data['proxy'] == False - -@pytest.mark.asyncio -async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): - _ = config_conn - _ = patch_open_connection - _ = patch_mqtt_err - assert asyncio.get_running_loop() - - Inverter.class_init() - - with InverterBase(FakeReader(), FakeWriter()) as inverter: - stream = inverter.local.stream - stream._Talent__set_serial_no(serial_no= "123344") - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True - -@pytest.mark.asyncio -async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except): - _ = config_conn - _ = patch_open_connection - _ = patch_mqtt_except - assert asyncio.get_running_loop() - - Inverter.class_init() - - with InverterBase(FakeReader(), FakeWriter()) as inverter: - stream = inverter.local.stream - stream._Talent__set_serial_no(serial_no= "123344") - - stream.new_data['inverter'] = True - stream.db.db['inverter'] = {} - await inverter.async_publ_mqtt() - assert stream.new_data['inverter'] == True From cd2f41a713e29de321ea8c9379c5769ed406ae6b Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 4 Oct 2024 01:35:44 +0200 Subject: [PATCH 10/20] add abstract inverter interface class --- app/src/inverter.py | 10 +-- app/src/inverter_base.py | 40 ++++++++- app/src/iter_registry.py | 10 +++ app/tests/test_inverter_base.py | 153 ++++++++++++++++++++++++++++++++ app/tests/test_inverter_g3.py | 15 ++-- app/tests/test_inverter_g3p.py | 3 +- 6 files changed, 212 insertions(+), 19 deletions(-) create mode 100644 app/tests/test_inverter_base.py diff --git a/app/src/inverter.py b/app/src/inverter.py index 7cb4a7a..dc224ee 100644 --- a/app/src/inverter.py +++ b/app/src/inverter.py @@ -1,15 +1,12 @@ import asyncio -import weakref import logging import json if __name__ == "app.src.inverter": - from app.src.iter_registry import IterRegistry from app.src.config import Config from app.src.mqtt import Mqtt from app.src.infos import Infos else: # pragma: no cover - from iter_registry import IterRegistry from config import Config from mqtt import Mqtt from infos import Infos @@ -17,7 +14,7 @@ else: # pragma: no cover logger_mqtt = logging.getLogger('mqtt') -class Inverter(metaclass=IterRegistry): +class Inverter(): '''class Inverter is a baseclass The class has some class method for managing common resources like a @@ -40,8 +37,6 @@ class Inverter(metaclass=IterRegistry): async_create_remote(): Establish a client connection to the TSUN cloud async_publ_mqtt(): Publish data to MQTT broker ''' - _registry = [] - @classmethod def class_init(cls) -> None: logging.debug('Inverter.class_init') @@ -109,6 +104,3 @@ class Inverter(metaclass=IterRegistry): logging.info('Close MQTT Task') loop.run_until_complete(cls.mqtt.close()) cls.mqtt = None - - def __init__(self): - self._registry.append(weakref.ref(self)) diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index 026b91b..bd7c700 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -1,3 +1,5 @@ +from abc import abstractmethod +import weakref import asyncio import logging import traceback @@ -6,6 +8,7 @@ from aiomqtt import MqttCodeError from asyncio import StreamReader, StreamWriter if __name__ == "app.src.inverter_base": + from app.src.iter_registry import AbstractIterMeta from app.src.inverter import Inverter from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamClient @@ -13,6 +16,7 @@ if __name__ == "app.src.inverter_base": from app.src.config import Config from app.src.infos import Infos else: # pragma: no cover + from iter_registry import AbstractIterMeta from inverter import Inverter from async_stream import StreamPtr from async_stream import AsyncStreamClient @@ -23,11 +27,43 @@ else: # pragma: no cover logger_mqtt = logging.getLogger('mqtt') -class InverterBase(Inverter): +class InverterIfc(metaclass=AbstractIterMeta): + + @abstractmethod + def __init__(self, reader: StreamReader, writer: StreamWriter, + config_id: str, prot_class, + client_mode: bool): + pass # pragma: no cover + + @abstractmethod + def __enter__(self): + pass # pragma: no cover + + @abstractmethod + def __exit__(self, exc_type, exc, tb): + pass # pragma: no cover + + @abstractmethod + def healthy(self) -> bool: + pass # pragma: no cover + + @abstractmethod + async def disc(self, shutdown_started=False) -> None: + pass # pragma: no cover + + @abstractmethod + async def async_create_remote(self) -> None: + pass # pragma: no cover + + +class InverterBase(InverterIfc, Inverter): + _registry = [] + def __init__(self, reader: StreamReader, writer: StreamWriter, config_id: str, prot_class, client_mode: bool = False): - super().__init__() + Inverter.__init__(self) + self._registry.append(weakref.ref(self)) self.addr = writer.get_extra_info('peername') self.config_id = config_id self.prot_class = prot_class diff --git a/app/src/iter_registry.py b/app/src/iter_registry.py index b84e16f..884d849 100644 --- a/app/src/iter_registry.py +++ b/app/src/iter_registry.py @@ -1,3 +1,4 @@ +from abc import ABCMeta class IterRegistry(type): @@ -6,3 +7,12 @@ class IterRegistry(type): obj = ref() if obj is not None: yield obj + + +class AbstractIterMeta(ABCMeta): + def __iter__(cls): + for ref in cls._registry: + obj = ref() + print(f'obj: {obj}') + if obj is not None: + yield obj diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py new file mode 100644 index 0000000..e1f2f33 --- /dev/null +++ b/app/tests/test_inverter_base.py @@ -0,0 +1,153 @@ +# test_with_pytest.py +import pytest +import asyncio +import sys,gc + +from mock import patch +from enum import Enum +from app.src.infos import Infos +from app.src.config import Config +from app.src.gen3.talent import Talent +from app.src.inverter_base import InverterBase +from app.src.singleton import Singleton +from app.src.protocol_ifc import ProtocolIfcImpl +from app.src.async_stream import AsyncStream, AsyncIfcImpl + +from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname + +pytest_plugins = ('pytest_asyncio',) + +# initialize the proxy statistics +Infos.static_init() + +@pytest.fixture +def config_conn(): + Config.act_config = { + 'mqtt':{ + 'host': test_hostname, + 'port': test_port, + 'user': '', + 'passwd': '' + }, + 'ha':{ + 'auto_conf_prefix': 'homeassistant', + 'discovery_prefix': 'homeassistant', + 'entity_prefix': 'tsun', + 'proxy_node_id': 'test_1', + 'proxy_unique_id': '' + }, + 'tsun':{'enabled': True, 'host': 'test_cloud.local', 'port': 1234}, 'inverters':{'allow_all':True} + } + +@pytest.fixture(scope="module", autouse=True) +def module_init(): + Singleton._instances.clear() + yield + +class FakeReader(): + def __init__(self): + self.on_recv = asyncio.Event() + async def read(self, max_len: int): + await self.on_recv.wait() + return b'' + def feed_eof(self): + return + + +class FakeWriter(): + def write(self, buf: bytes): + return + def get_extra_info(self, sel: str): + if sel == 'peername': + return 'remote.intern' + elif sel == 'sockname': + return 'sock:1234' + assert False + def is_closing(self): + return False + def close(self): + return + async def wait_closed(self): + return + +class TestType(Enum): + RD_TEST_0_BYTES = 1 + RD_TEST_TIMEOUT = 2 + RD_TEST_EXCEPT = 3 + + +test = TestType.RD_TEST_0_BYTES + +@pytest.fixture +def patch_open_connection(): + async def new_conn(conn): + await asyncio.sleep(0) + return FakeReader(), FakeWriter() + + def new_open(host: str, port: int): + global test + if test == TestType.RD_TEST_TIMEOUT: + raise ConnectionRefusedError + elif test == TestType.RD_TEST_EXCEPT: + raise ValueError("Value cannot be negative") # Compliant + return new_conn(None) + + with patch.object(asyncio, 'open_connection', new_open) as conn: + yield conn + + +@pytest.fixture +def patch_healthy(): + with patch.object(AsyncStream, 'healthy') as conn: + yield conn + +def test_protocol_iter(): + ProtocolIfcImpl._registry.clear() + cnt = 0 + ifc = AsyncIfcImpl() + prot = ProtocolIfcImpl(('test.intern', 123), ifc, True) + for p in ProtocolIfcImpl: + assert p == prot + cnt += 1 + del p + del prot + assert cnt == 1 + for p in ProtocolIfcImpl: + assert False + +def test_inverter_iter(): + InverterBase._registry.clear() + cnt = 0 + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + for inv in InverterBase: + assert inv == inverter + cnt += 1 + del inv + del inverter + assert cnt == 1 + + for inv in InverterBase: + assert False + +def test_method_calls(patch_healthy): + spy = patch_healthy + InverterBase._registry.clear() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + assert inverter.local.stream + assert inverter.local.ifc + # inverter.healthy() + for inv in InverterBase: + inv.healthy() + del inv + spy.assert_called_once() + del inverter + cnt = 0 + for inv in InverterBase: + cnt += 1 + assert cnt == 0 diff --git a/app/tests/test_inverter_g3.py b/app/tests/test_inverter_g3.py index 2fbe4e8..702186a 100644 --- a/app/tests/test_inverter_g3.py +++ b/app/tests/test_inverter_g3.py @@ -8,6 +8,7 @@ from enum import Enum from app.src.infos import Infos from app.src.config import Config from app.src.inverter import Inverter +from app.src.inverter_base import InverterBase from app.src.singleton import Singleton from app.src.gen3.inverter_g3 import InverterG3 from app.src.async_stream import AsyncStream @@ -103,18 +104,18 @@ def test_method_calls(patch_healthy): spy = patch_healthy reader = FakeReader() writer = FakeWriter() - Inverter._registry.clear() - + InverterBase._registry.clear() + with InverterG3(reader, writer) as inverter: assert inverter.local.stream assert inverter.local.ifc - for inv in Inverter: + for inv in InverterBase: inv.healthy() del inv spy.assert_called_once() del inverter cnt = 0 - for inv in Inverter: + for inv in InverterBase: cnt += 1 assert cnt == 0 @@ -131,7 +132,7 @@ async def test_remote_conn(config_conn, patch_open_connection): del inverter cnt = 0 - for inv in Inverter: + for inv in InverterBase: print(f'Inverter refs:{gc.get_referrers(inv)}') cnt += 1 assert cnt == 0 @@ -157,8 +158,8 @@ async def test_remote_except(config_conn, patch_open_connection): del inverter cnt = 0 - for inv in Inverter: - print(f'Inverter refs:{gc.get_referrers(inv)}') + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') cnt += 1 assert cnt == 0 diff --git a/app/tests/test_inverter_g3p.py b/app/tests/test_inverter_g3p.py index 603d356..f25b768 100644 --- a/app/tests/test_inverter_g3p.py +++ b/app/tests/test_inverter_g3p.py @@ -7,6 +7,7 @@ from enum import Enum from app.src.infos import Infos from app.src.config import Config from app.src.inverter import Inverter +from app.src.inverter_base import InverterBase from app.src.singleton import Singleton from app.src.gen3plus.inverter_g3p import InverterG3P @@ -96,7 +97,7 @@ def patch_open_connection(): def test_method_calls(): reader = FakeReader() writer = FakeWriter() - Inverter._registry.clear() + InverterBase._registry.clear() with InverterG3P(reader, writer, client_mode=False) as inverter: assert inverter.local.stream From 949f3c96083fa84b88f4dc0ad110b4ed440ddf79 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 4 Oct 2024 01:41:25 +0200 Subject: [PATCH 11/20] initial commit --- app/src/protocol_ifc.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 app/src/protocol_ifc.py diff --git a/app/src/protocol_ifc.py b/app/src/protocol_ifc.py new file mode 100644 index 0000000..90ee8f1 --- /dev/null +++ b/app/src/protocol_ifc.py @@ -0,0 +1,43 @@ +from abc import abstractmethod +import weakref + +if __name__ == "app.src.protocol_ifc": + from app.src.iter_registry import AbstractIterMeta + from app.src.async_ifc import AsyncIfc +else: # pragma: no cover + from iter_registry import AbstractIterMeta + from async_ifc import AsyncIfc + + +class ProtocolIfc(metaclass=AbstractIterMeta): + + @abstractmethod + def __init__(self, addr, ifc: "AsyncIfc", server_side: bool, + client_mode: bool = False, id_str=b''): + pass # pragma: no cover + + @abstractmethod + def close(self): + pass # pragma: no cover + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + self.close() + + +class Test(): + def test_method(self): + return self + + +class ProtocolIfcImpl(ProtocolIfc, Test): + _registry = [] + + def __init__(self, addr, ifc: "AsyncIfc", server_side: bool, + client_mode: bool = False, id_str=b''): + self._registry.append(weakref.ref(self)) + + def close(self): + pass # pragma: no cover From 3a94afb48d0d3f012a1ef1c3397ca728a0a73462 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 4 Oct 2024 01:50:24 +0200 Subject: [PATCH 12/20] fix sonar qube warnings --- app/src/inverter_base.py | 2 +- app/src/server.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index bd7c700..da062d3 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -48,7 +48,7 @@ class InverterIfc(metaclass=AbstractIterMeta): pass # pragma: no cover @abstractmethod - async def disc(self, shutdown_started=False) -> None: + async def disc(self, shutdown_started=False) -> None: pass # pragma: no cover @abstractmethod diff --git a/app/src/server.py b/app/src/server.py index 0ff5202..1cca411 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -6,6 +6,7 @@ from asyncio import StreamReader, StreamWriter from aiohttp import web from logging import config # noqa F401 from inverter import Inverter +from inverter_base import InverterIfc from gen3.inverter_g3 import InverterG3 from gen3plus.inverter_g3p import InverterG3P from scheduler import Schedule @@ -37,7 +38,7 @@ async def healthy(request): if proxy_is_up: # logging.info('web reqeust healthy()') - for inverter in Inverter: + for inverter in InverterIfc: try: res = inverter.healthy() if not res: @@ -86,7 +87,7 @@ async def handle_shutdown(web_task): # # first, disc all open TCP connections gracefully # - for inverter in Inverter: + async for inverter in InverterIfc: await inverter.disc(True) logging.info('Proxy disconnecting done') From 00e9a4534dee193d6bc134f9fbe6c75d0a492999 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Fri, 4 Oct 2024 23:37:05 +0200 Subject: [PATCH 13/20] rename class Inverter into Proxy --- app/proxy.svg | 358 +++++++++++------- app/proxy.yuml | 29 +- app/proxy_2.svg | 6 +- app/proxy_2.yuml | 6 +- app/src/async_stream.py | 23 +- app/src/inverter_base.py | 20 +- app/src/iter_registry.py | 1 - app/src/{inverter.py => proxy.py} | 14 +- app/src/server.py | 8 +- app/tests/test_inverter_g3.py | 16 +- app/tests/test_inverter_g3p.py | 14 +- app/tests/test_modbus_tcp.py | 12 +- app/tests/{test_inverter.py => test_proxy.py} | 24 +- app/tests/test_solarman.py | 1 + 14 files changed, 300 insertions(+), 232 deletions(-) rename app/src/{inverter.py => proxy.py} (92%) rename app/tests/{test_inverter.py => test_proxy.py} (79%) diff --git a/app/proxy.svg b/app/proxy.svg index 6493fc7..a8e328d 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -4,193 +4,257 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -IterRegistry - - -__iter__ - - - -A3 - -Inverter - -cls.db_stat -cls.entity_prfx -cls.discovery_prfx -cls.proxy_node_id -cls.proxy_unique_id -cls.mqtt:Mqtt -__ha_restarts - -async_create_remote(inv_prot, conn_class)async_publ_mqtt() - - - -A1->A3 - - - - - -A2 - -Mqtt -<<Singleton>> - -<static>ha_restarts -<static>__client -<static>__cb_MqttIsUp - -<async>publish() -<async>close() - - - -A3->A2 - - - + +<<AbstractIterMeta>> + + +__iter__ A4 - -InverterBase - - -<async>disc(shutdown_started) -<async>async_create_remote() -healthy() + +<<InverterIfc>> + + +healthy()->bool +<async>disc(shutdown_started=False) +<async>create_remote() - + -A3->A4 - - +A1->A4 + + + + + +A2 + +Mqtt +<<Singleton>> + +<static>ha_restarts +<static>__client +<static>__cb_MqttIsUp + +<async>publish() +<async>close() + + + +A3 + +Proxy + +<cls>db_stat +<cls>entity_prfx +<cls>discovery_prfx +<cls>proxy_node_id +<cls>proxy_unique_id +<cls>mqtt:Mqtt +__ha_restarts + +class_init() +class_close() +<async>_cb_mqtt_is_up() +<async>_register_proxy_stat_home_assistant() +<async>_async_publ_mqtt_proxy_stat(key) + + + +A3->A2 + + + A5 - -InverterG3 - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() + +InverterBase + +_registry +__ha_restarts +addr +config_id:str +prot_class:MessageProt +remote:StreamPtr +local:StreamPtr + +healthy()->bool +<async>disc(shutdown_started=False) +<async>create_remote() +<async>async_publ_mqtt() + + + +A3->A5 + + A4->A5 - - + + A6 - -InverterG3P - -addr -remote:StreamPtr -local:StreamPtr - -async_create_remote() -close() + +StreamPtr + + +stream:MessageProt +ifc:AsyncIfc - - -A4->A6 - - + + +A5->A6 + + +2 A7 - -<<AsyncIfc>> + +InverterG3 - + A5->A7 - - -1..2 - - - -A8 - -<<MessageProt>> - - - -A5->A8 - - -1..2 - - - -A6->A7 - - -1..2 - - - -A6->A8 - - -1..2 - - - -A8->A7 - - -use + + A9 - -ModbusConn - -host -port -addr -stream:InverterG3P - + +InverterG3P - + + +A5->A9 + + + + + +A11 + +<<AsyncIfc>> + + -A9->A6 - - -1 -has +A6->A11 + + +1 + + + +A12 + +<<MessageProt>> + + + +A6->A12 + + +1 + + + +A8 + + + +Creates an GEN3 +inverter instance +with +prot_class:Talent + + + +A7->A8 + + + + +A10 + + + +Creates an GEN3PLUS +inverter instance +with +prot_class:SolarmanV5 + + + +A9->A10 + + + + +A12->A11 + + +use + + + +A13 + +ModbusConn + +host +port +addr +stream:InverterG3P + + + + +A13->A9 + + +1 +has + + + +A14 + +ModbusTcp + + + +A14->A13 + + +* +creates diff --git a/app/proxy.yuml b/app/proxy.yuml index daf04aa..9d007cb 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -3,29 +3,34 @@ // {generate:true} [note: You can stick notes on diagrams too!{bg:cornsilk}] -[IterRegistry||__iter__] +[<>||__iter__] [Mqtt;<>|ha_restarts;__client;__cb_MqttIsUp|publish();close()] -[Inverter|cls.db_stat;cls.entity_prfx;cls.discovery_prfx;cls.proxy_node_id;cls.proxy_unique_id;cls.mqtt:Mqtt;;__ha_restarts|async_create_remote(inv_prot, conn_class)async_publ_mqtt()] -[InverterBase||disc(shutdown_started);async_create_remote();healthy()] -[Inverter]^[InverterBase] -[InverterBase]^[InverterG3|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] -[InverterBase]^[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] -[Inverter]++->[Mqtt;<>] +[Proxy|db_stat;entity_prfx;discovery_prfx;proxy_node_id;proxy_unique_id;mqtt:Mqtt;;__ha_restarts|class_init();class_close();;_cb_mqtt_is_up();_register_proxy_stat_home_assistant();_async_publ_mqtt_proxy_stat(key)] -[IterRegistry]^[Inverter|server_side:bool;header_valid:bool;header_len:unsigned;data_len:unsigned;unique_id;node_id;sug_area;_recv_buffer:bytearray;_send_buffer:bytearray;_forward_buffer:bytearray;db:Infos;new_data:list;state|_read():void;close():void;inc_counter():void;dec_counter():void] +[<>||healthy()->bool;disc(shutdown_started=False);create_remote();] +[<>]^-.-[<>] +[InverterBase|_registry;__ha_restarts;;addr;config_id:str;prot_class:MessageProt;remote:StreamPtr;local:StreamPtr;|healthy()->bool;disc(shutdown_started=False);create_remote();async_publ_mqtt()] +[StreamPtr||stream:MessageProt;ifc:AsyncIfc] +[<>]^-.-[InverterBase] +[InverterG3]-[note: Creates an GEN3 inverter instance with prot_class:Talent{bg:cornsilk}] +[InverterG3P]-[note: Creates an GEN3PLUS inverter instance with prot_class:SolarmanV5{bg:cornsilk}] +[InverterBase]^[InverterG3] +[InverterBase]^[InverterG3P] +[Proxy]^[InverterBase] +[InverterBase]-2>[StreamPtr] +[Proxy]++->[Mqtt;<>] [<>] -[InverterG3]-1..2>[<>] -[InverterG3]-1..2>[<>] +[StreamPtr]-1>[<>] +[StreamPtr]-1>[<>] -[InverterG3P]-1..2>[<>] -[InverterG3P]-1..2>[<>] [<>]use-.->[<>] [ModbusConn|host;port;addr;stream:InverterG3P;|]has-1>[InverterG3P] +[ModbusTcp]creates-*>[ModbusConn] diff --git a/app/proxy_2.svg b/app/proxy_2.svg index 63e6657..2aab16e 100644 --- a/app/proxy_2.svg +++ b/app/proxy_2.svg @@ -54,7 +54,7 @@ remote:StreamPtr local:StreamPtr -async_create_remote() +create_remote() close() @@ -63,7 +63,7 @@ AsyncStreamServer -async_create_remote +create_remote <async>server_loop() <async>_async_forward() @@ -105,7 +105,7 @@ remote:StreamPtr local:StreamPtr -async_create_remote() +create_remote() close() diff --git a/app/proxy_2.yuml b/app/proxy_2.yuml index 2d96500..33d4679 100644 --- a/app/proxy_2.yuml +++ b/app/proxy_2.yuml @@ -5,13 +5,13 @@ [note: You can stick notes on diagrams too!{bg:cornsilk}] [IterRegistry||__iter__] -[InverterG3|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] -[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|async_create_remote();;close()] +[InverterG3|addr;remote:StreamPtr;local:StreamPtr|create_remote();;close()] +[InverterG3P|addr;remote:StreamPtr;local:StreamPtr|create_remote();;close()] [<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_flush();fwd_log();fwd_clear();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] [AsyncIfcImpl|fwd_fifo:ByteFifo;tx_fifo:ByteFifo;rx_fifo:ByteFifo;conn_no:Count;node_id;timeout_cb] [AsyncStream|reader;writer;addr;r_addr;l_addr|;loop;disc();close();healthy();;__async_read();__async_write();__async_forward()] -[AsyncStreamServer|async_create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] +[AsyncStreamServer|create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] [AsyncStreamClient||client_loop();_async_forward())] [<>]^-.-[AsyncIfcImpl] [AsyncIfcImpl]^[AsyncStream] diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 268a612..e260272 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -7,12 +7,12 @@ from typing import Self from itertools import count if __name__ == "app.src.async_stream": - from app.src.inverter import Inverter + from app.src.proxy import Proxy from app.src.byte_fifo import ByteFifo from app.src.async_ifc import AsyncIfc from app.src.infos import Infos else: # pragma: no cover - from inverter import Inverter + from proxy import Proxy from byte_fifo import ByteFifo from async_ifc import AsyncIfc from infos import Infos @@ -240,7 +240,7 @@ class AsyncStream(AsyncIfcImpl): await self._writer.wait_closed() def close(self) -> None: - logging.info(f'AsyncStream.close1() l{self.l_addr} | r{self.r_addr}') + logging.debug(f'AsyncStream.close() l{self.l_addr} | r{self.r_addr}') """close handler for a no waiting disconnect hint: must be called before releasing the connection instance @@ -249,7 +249,6 @@ class AsyncStream(AsyncIfcImpl): self._reader.feed_eof() # abort awaited read if self._writer.is_closing(): return - logger.info(f'AsyncStream.close2() l{self.l_addr} | r{self.r_addr}') self._writer.close() def healthy(self) -> bool: @@ -325,15 +324,15 @@ class AsyncStream(AsyncIfcImpl): class AsyncStreamServer(AsyncStream): def __init__(self, reader: StreamReader, writer: StreamWriter, - async_publ_mqtt, async_create_remote, + async_publ_mqtt, create_remote, rstream: "StreamPtr") -> None: AsyncStream.__init__(self, reader, writer, rstream) - self.async_create_remote = async_create_remote + self.create_remote = create_remote self.async_publ_mqtt = async_publ_mqtt def close(self) -> None: - logging.info('AsyncStreamServer.close()') - self.async_create_remote = None + logging.debug('AsyncStreamServer.close()') + self.create_remote = None self.async_publ_mqtt = None super().close() @@ -342,7 +341,7 @@ class AsyncStreamServer(AsyncStream): logger.info(f'[{self.node_id}:{self.conn_no}] ' f'Accept connection from {self.r_addr}') Infos.inc_counter('Inverter_Cnt') - await self.publish_outstanding_mqtt() + await self.publish_outstanding_mqtugt() await self.loop() Infos.dec_counter('Inverter_Cnt') await self.publish_outstanding_mqtt() @@ -360,7 +359,7 @@ class AsyncStreamServer(AsyncStream): async def _async_forward(self) -> None: """forward handler transmits data over the remote connection""" if not self.remote.stream: - await self.async_create_remote() + await self.create_remote() if self.remote.stream and \ self.remote.ifc.init_new_client_conn_cb(): await self.remote.ifc._AsyncStream__async_write() @@ -375,7 +374,7 @@ class AsyncStreamServer(AsyncStream): '''Publish all outstanding MQTT topics''' try: await self.async_publ_mqtt() - await Inverter._async_publ_mqtt_proxy_stat('proxy') + await Proxy._async_publ_mqtt_proxy_stat('proxy') except Exception: pass @@ -387,7 +386,7 @@ class AsyncStreamClient(AsyncStream): self.close_cb = close_cb def close(self) -> None: - logging.info('AsyncStreamClient.close()') + logging.debug('AsyncStreamClient.close()') self.close_cb = None super().close() diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index da062d3..9ed685a 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -9,7 +9,7 @@ from asyncio import StreamReader, StreamWriter if __name__ == "app.src.inverter_base": from app.src.iter_registry import AbstractIterMeta - from app.src.inverter import Inverter + from app.src.proxy import Proxy from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamClient from app.src.async_stream import AsyncStreamServer @@ -17,7 +17,7 @@ if __name__ == "app.src.inverter_base": from app.src.infos import Infos else: # pragma: no cover from iter_registry import AbstractIterMeta - from inverter import Inverter + from proxy import Proxy from async_stream import StreamPtr from async_stream import AsyncStreamClient from async_stream import AsyncStreamServer @@ -28,6 +28,7 @@ logger_mqtt = logging.getLogger('mqtt') class InverterIfc(metaclass=AbstractIterMeta): + _registry = [] @abstractmethod def __init__(self, reader: StreamReader, writer: StreamWriter, @@ -52,17 +53,16 @@ class InverterIfc(metaclass=AbstractIterMeta): pass # pragma: no cover @abstractmethod - async def async_create_remote(self) -> None: + async def create_remote(self) -> None: pass # pragma: no cover -class InverterBase(InverterIfc, Inverter): - _registry = [] +class InverterBase(InverterIfc, Proxy): def __init__(self, reader: StreamReader, writer: StreamWriter, config_id: str, prot_class, client_mode: bool = False): - Inverter.__init__(self) + Proxy.__init__(self) self._registry.append(weakref.ref(self)) self.addr = writer.get_extra_info('peername') self.config_id = config_id @@ -71,7 +71,7 @@ class InverterBase(InverterIfc, Inverter): self.remote = StreamPtr(None) ifc = AsyncStreamServer(reader, writer, self.async_publ_mqtt, - self.async_create_remote, + self.create_remote, self.remote) self.local = StreamPtr( @@ -113,7 +113,7 @@ class InverterBase(InverterIfc, Inverter): await self.local.ifc.disc() def healthy(self) -> bool: - logging.debug('Inverter healthy()') + logging.debug('InverterBase healthy()') if self.local.ifc and not self.local.ifc.healthy(): return False @@ -121,7 +121,7 @@ class InverterBase(InverterIfc, Inverter): return False return True - async def async_create_remote(self) -> None: + async def create_remote(self) -> None: '''Establish a client connection to the TSUN cloud''' tsun = Config.get(self.config_id) @@ -179,7 +179,7 @@ class InverterBase(InverterIfc, Inverter): for key in stream.new_data: await self.__async_publ_mqtt_packet(stream, key) for key in Infos.new_stat_data: - await Inverter._async_publ_mqtt_proxy_stat(key) + await Proxy._async_publ_mqtt_proxy_stat(key) except MqttCodeError as error: logging.error(f'Mqtt except: {error}') diff --git a/app/src/iter_registry.py b/app/src/iter_registry.py index 884d849..34ebb25 100644 --- a/app/src/iter_registry.py +++ b/app/src/iter_registry.py @@ -13,6 +13,5 @@ class AbstractIterMeta(ABCMeta): def __iter__(cls): for ref in cls._registry: obj = ref() - print(f'obj: {obj}') if obj is not None: yield obj diff --git a/app/src/inverter.py b/app/src/proxy.py similarity index 92% rename from app/src/inverter.py rename to app/src/proxy.py index dc224ee..9b75c37 100644 --- a/app/src/inverter.py +++ b/app/src/proxy.py @@ -2,7 +2,7 @@ import asyncio import logging import json -if __name__ == "app.src.inverter": +if __name__ == "app.src.proxy": from app.src.config import Config from app.src.mqtt import Mqtt from app.src.infos import Infos @@ -14,8 +14,8 @@ else: # pragma: no cover logger_mqtt = logging.getLogger('mqtt') -class Inverter(): - '''class Inverter is a baseclass +class Proxy(): + '''class Proxy is a baseclass The class has some class method for managing common resources like a connection to the MQTT broker or proxy error counter which are common @@ -34,12 +34,12 @@ class Inverter(): destroyed methods: - async_create_remote(): Establish a client connection to the TSUN cloud + create_remote(): Establish a client connection to the TSUN cloud async_publ_mqtt(): Publish data to MQTT broker ''' @classmethod def class_init(cls) -> None: - logging.debug('Inverter.class_init') + logging.debug('Proxy.class_init') # initialize the proxy statistics Infos.static_init() cls.db_stat = Infos() @@ -61,7 +61,7 @@ class Inverter(): # reset at midnight when you restart the proxy just before # midnight! inverters = Config.get('inverters') - # logger.debug(f'Inverters: {inverters}') + # logger.debug(f'Proxys: {inverters}') for inv in inverters.values(): if (type(inv) is dict): node_id = inv['node_id'] @@ -100,7 +100,7 @@ class Inverter(): @classmethod def class_close(cls, loop) -> None: # pragma: no cover - logging.debug('Inverter.class_close') + logging.debug('Proxy.class_close') logging.info('Close MQTT Task') loop.run_until_complete(cls.mqtt.close()) cls.mqtt = None diff --git a/app/src/server.py b/app/src/server.py index 1cca411..754b918 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -5,7 +5,7 @@ import os from asyncio import StreamReader, StreamWriter from aiohttp import web from logging import config # noqa F401 -from inverter import Inverter +from proxy import Proxy from inverter_base import InverterIfc from gen3.inverter_g3 import InverterG3 from gen3plus.inverter_g3p import InverterG3P @@ -87,7 +87,7 @@ async def handle_shutdown(web_task): # # first, disc all open TCP connections gracefully # - async for inverter in InverterIfc: + for inverter in InverterIfc: await inverter.disc(True) logging.info('Proxy disconnecting done') @@ -152,7 +152,7 @@ if __name__ == "__main__": ConfigErr = Config.class_init() if ConfigErr is not None: logging.info(f'ConfigErr: {ConfigErr}') - Inverter.class_init() + Proxy.class_init() Schedule.start() ModbusTcp(loop) @@ -185,7 +185,7 @@ if __name__ == "__main__": pass finally: logging.info("Event loop is stopped") - Inverter.class_close(loop) + Proxy.class_close(loop) logging.debug('Close event loop') loop.close() logging.info(f'Finally, exit Server "{serv_name}"') diff --git a/app/tests/test_inverter_g3.py b/app/tests/test_inverter_g3.py index 702186a..45438bb 100644 --- a/app/tests/test_inverter_g3.py +++ b/app/tests/test_inverter_g3.py @@ -7,7 +7,7 @@ from mock import patch from enum import Enum from app.src.infos import Infos from app.src.config import Config -from app.src.inverter import Inverter +from app.src.proxy import Proxy from app.src.inverter_base import InverterBase from app.src.singleton import Singleton from app.src.gen3.inverter_g3 import InverterG3 @@ -126,14 +126,14 @@ async def test_remote_conn(config_conn, patch_open_connection): assert asyncio.get_running_loop() with InverterG3(FakeReader(), FakeWriter()) as inverter: - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream del inverter cnt = 0 for inv in InverterBase: - print(f'Inverter refs:{gc.get_referrers(inv)}') + print(f'InverterBase refs:{gc.get_referrers(inv)}') cnt += 1 assert cnt == 0 @@ -147,12 +147,12 @@ async def test_remote_except(config_conn, patch_open_connection): test = TestType.RD_TEST_TIMEOUT with InverterG3(FakeReader(), FakeWriter()) as inverter: - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None test = TestType.RD_TEST_EXCEPT - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None del inverter @@ -169,7 +169,7 @@ async def test_mqtt_publish(config_conn, patch_open_connection): _ = patch_open_connection assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream @@ -197,7 +197,7 @@ async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): _ = patch_mqtt_err assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream @@ -214,7 +214,7 @@ async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except _ = patch_mqtt_except assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3(FakeReader(), FakeWriter()) as inverter: stream = inverter.local.stream diff --git a/app/tests/test_inverter_g3p.py b/app/tests/test_inverter_g3p.py index f25b768..0f47cbe 100644 --- a/app/tests/test_inverter_g3p.py +++ b/app/tests/test_inverter_g3p.py @@ -6,7 +6,7 @@ from mock import patch from enum import Enum from app.src.infos import Infos from app.src.config import Config -from app.src.inverter import Inverter +from app.src.proxy import Proxy from app.src.inverter_base import InverterBase from app.src.singleton import Singleton from app.src.gen3plus.inverter_g3p import InverterG3P @@ -110,7 +110,7 @@ async def test_remote_conn(config_conn, patch_open_connection): assert asyncio.get_running_loop() with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream @@ -124,12 +124,12 @@ async def test_remote_except(config_conn, patch_open_connection): test = TestType.RD_TEST_TIMEOUT with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None test = TestType.RD_TEST_EXCEPT - await inverter.async_create_remote() + await inverter.create_remote() await asyncio.sleep(0) assert inverter.remote.stream==None @@ -139,7 +139,7 @@ async def test_mqtt_publish(config_conn, patch_open_connection): _ = patch_open_connection assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream @@ -167,7 +167,7 @@ async def test_mqtt_err(config_conn, patch_open_connection, patch_mqtt_err): _ = patch_mqtt_err assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream @@ -184,7 +184,7 @@ async def test_mqtt_except(config_conn, patch_open_connection, patch_mqtt_except _ = patch_mqtt_except assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() with InverterG3P(FakeReader(), FakeWriter(), client_mode=False) as inverter: stream = inverter.local.stream diff --git a/app/tests/test_modbus_tcp.py b/app/tests/test_modbus_tcp.py index 5611a03..a8a8f5a 100644 --- a/app/tests/test_modbus_tcp.py +++ b/app/tests/test_modbus_tcp.py @@ -10,7 +10,7 @@ from app.src.config import Config from app.src.infos import Infos from app.src.mqtt import Mqtt from app.src.messages import Message, State -from app.src.inverter import Inverter +from app.src.proxy import Proxy from app.src.modbus_tcp import ModbusConn, ModbusTcp @@ -175,7 +175,7 @@ async def test_modbus_cnf1(config_conn, patch_open): _ = patch_open global test assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() test = TestType.RD_TEST_TIMEOUT assert Infos.stat['proxy']['Inverter_Cnt'] == 0 @@ -196,7 +196,7 @@ async def test_modbus_cnf2(config_conn, patch_no_mqtt, patch_open): _ = patch_no_mqtt global test assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 @@ -222,7 +222,7 @@ async def test_modbus_cnf3(config_conn, patch_no_mqtt, patch_open): _ = patch_no_mqtt global test assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 @@ -255,7 +255,7 @@ async def test_mqtt_err(config_conn, patch_mqtt_err, patch_open): _ = patch_mqtt_err global test assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 @@ -288,7 +288,7 @@ async def test_mqtt_except(config_conn, patch_mqtt_except, patch_open): _ = patch_mqtt_except global test assert asyncio.get_running_loop() - Inverter.class_init() + Proxy.class_init() test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 diff --git a/app/tests/test_inverter.py b/app/tests/test_proxy.py similarity index 79% rename from app/tests/test_inverter.py rename to app/tests/test_proxy.py index c59ff38..bee01fd 100644 --- a/app/tests/test_inverter.py +++ b/app/tests/test_proxy.py @@ -6,7 +6,7 @@ import logging from mock import patch, Mock from app.src.singleton import Singleton -from app.src.inverter import Inverter +from app.src.proxy import Proxy from app.src.mqtt import Mqtt from app.src.gen3plus.solarman_v5 import SolarmanV5 from app.src.config import Config @@ -63,13 +63,13 @@ def config_conn(test_hostname, test_port): async def test_inverter_cb(config_conn): _ = config_conn - with patch.object(Inverter, '_cb_mqtt_is_up', wraps=Inverter._cb_mqtt_is_up) as spy: - print('call Inverter.class_init') - Inverter.class_init() - assert 'homeassistant/' == Inverter.discovery_prfx - assert 'tsun/' == Inverter.entity_prfx - assert 'test_1/' == Inverter.proxy_node_id - await Inverter._cb_mqtt_is_up() + with patch.object(Proxy, '_cb_mqtt_is_up', wraps=Proxy._cb_mqtt_is_up) as spy: + print('call Proxy.class_init') + Proxy.class_init() + assert 'homeassistant/' == Proxy.discovery_prfx + assert 'tsun/' == Proxy.entity_prfx + assert 'test_1/' == Proxy.proxy_node_id + await Proxy._cb_mqtt_is_up() spy.assert_called_once() @pytest.mark.asyncio @@ -77,8 +77,8 @@ async def test_mqtt_is_up(config_conn): _ = config_conn with patch.object(Mqtt, 'publish') as spy: - Inverter.class_init() - await Inverter._cb_mqtt_is_up() + Proxy.class_init() + await Proxy._cb_mqtt_is_up() spy.assert_called() @pytest.mark.asyncio @@ -86,6 +86,6 @@ async def test_mqtt_proxy_statt_invalid(config_conn): _ = config_conn with patch.object(Mqtt, 'publish') as spy: - Inverter.class_init() - await Inverter._async_publ_mqtt_proxy_stat('InValId_kEy') + Proxy.class_init() + await Proxy._async_publ_mqtt_proxy_stat('InValId_kEy') spy.assert_not_called() diff --git a/app/tests/test_solarman.py b/app/tests/test_solarman.py index e4e8edc..a980744 100644 --- a/app/tests/test_solarman.py +++ b/app/tests/test_solarman.py @@ -682,6 +682,7 @@ def config_tsun_inv1(): Config.act_config = {'solarman':{'enabled': True},'inverters':{'Y170000000000001':{'monitor_sn': 2070233889, 'node_id':'inv1', 'modbus_polling': True, 'suggested_area':'roof', 'sensor_list': 688}}} def test_read_message(device_ind_msg): + Config.act_config = {'solarman':{'enabled': True}} m = MemoryStream(device_ind_msg, (0,)) m.read() # read complete msg, and dispatch msg assert not m.header_valid # must be invalid, since msg was handled and buffer flushed From 0b79a37fe7baa29625a2346e7de5f21cd8366584 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 5 Oct 2024 01:33:05 +0200 Subject: [PATCH 14/20] fix typo --- app/src/async_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index e260272..eccd7d3 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -341,7 +341,7 @@ class AsyncStreamServer(AsyncStream): logger.info(f'[{self.node_id}:{self.conn_no}] ' f'Accept connection from {self.r_addr}') Infos.inc_counter('Inverter_Cnt') - await self.publish_outstanding_mqtugt() + await self.publish_outstanding_mqtt() await self.loop() Infos.dec_counter('Inverter_Cnt') await self.publish_outstanding_mqtt() From 5b68610f783306883a0724876a2c8074d83f76ef Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 5 Oct 2024 01:34:03 +0200 Subject: [PATCH 15/20] move class InverterIfc into a separate file --- app/src/inverter_base.py | 35 ++--------------------------------- app/src/inverter_ifc.py | 40 ++++++++++++++++++++++++++++++++++++++++ app/src/server.py | 2 +- 3 files changed, 43 insertions(+), 34 deletions(-) create mode 100644 app/src/inverter_ifc.py diff --git a/app/src/inverter_base.py b/app/src/inverter_base.py index 9ed685a..d663a75 100644 --- a/app/src/inverter_base.py +++ b/app/src/inverter_base.py @@ -1,4 +1,3 @@ -from abc import abstractmethod import weakref import asyncio import logging @@ -8,7 +7,7 @@ from aiomqtt import MqttCodeError from asyncio import StreamReader, StreamWriter if __name__ == "app.src.inverter_base": - from app.src.iter_registry import AbstractIterMeta + from app.src.inverter_ifc import InverterIfc from app.src.proxy import Proxy from app.src.async_stream import StreamPtr from app.src.async_stream import AsyncStreamClient @@ -16,7 +15,7 @@ if __name__ == "app.src.inverter_base": from app.src.config import Config from app.src.infos import Infos else: # pragma: no cover - from iter_registry import AbstractIterMeta + from inverter_ifc import InverterIfc from proxy import Proxy from async_stream import StreamPtr from async_stream import AsyncStreamClient @@ -27,36 +26,6 @@ else: # pragma: no cover logger_mqtt = logging.getLogger('mqtt') -class InverterIfc(metaclass=AbstractIterMeta): - _registry = [] - - @abstractmethod - def __init__(self, reader: StreamReader, writer: StreamWriter, - config_id: str, prot_class, - client_mode: bool): - pass # pragma: no cover - - @abstractmethod - def __enter__(self): - pass # pragma: no cover - - @abstractmethod - def __exit__(self, exc_type, exc, tb): - pass # pragma: no cover - - @abstractmethod - def healthy(self) -> bool: - pass # pragma: no cover - - @abstractmethod - async def disc(self, shutdown_started=False) -> None: - pass # pragma: no cover - - @abstractmethod - async def create_remote(self) -> None: - pass # pragma: no cover - - class InverterBase(InverterIfc, Proxy): def __init__(self, reader: StreamReader, writer: StreamWriter, diff --git a/app/src/inverter_ifc.py b/app/src/inverter_ifc.py new file mode 100644 index 0000000..55fc1b9 --- /dev/null +++ b/app/src/inverter_ifc.py @@ -0,0 +1,40 @@ +from abc import abstractmethod +import logging +from asyncio import StreamReader, StreamWriter + +if __name__ == "app.src.inverter_ifc": + from app.src.iter_registry import AbstractIterMeta +else: # pragma: no cover + from iter_registry import AbstractIterMeta + +logger_mqtt = logging.getLogger('mqtt') + + +class InverterIfc(metaclass=AbstractIterMeta): + _registry = [] + + @abstractmethod + def __init__(self, reader: StreamReader, writer: StreamWriter, + config_id: str, prot_class, + client_mode: bool): + pass # pragma: no cover + + @abstractmethod + def __enter__(self): + pass # pragma: no cover + + @abstractmethod + def __exit__(self, exc_type, exc, tb): + pass # pragma: no cover + + @abstractmethod + def healthy(self) -> bool: + pass # pragma: no cover + + @abstractmethod + async def disc(self, shutdown_started=False) -> None: + pass # pragma: no cover + + @abstractmethod + async def create_remote(self) -> None: + pass # pragma: no cover diff --git a/app/src/server.py b/app/src/server.py index 754b918..cda8501 100644 --- a/app/src/server.py +++ b/app/src/server.py @@ -6,7 +6,7 @@ from asyncio import StreamReader, StreamWriter from aiohttp import web from logging import config # noqa F401 from proxy import Proxy -from inverter_base import InverterIfc +from inverter_ifc import InverterIfc from gen3.inverter_g3 import InverterG3 from gen3plus.inverter_g3p import InverterG3P from scheduler import Schedule From c7d0a9137160d51d543160c244653dd975367504 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 5 Oct 2024 01:35:04 +0200 Subject: [PATCH 16/20] add more testcases --- app/tests/test_inverter_base.py | 172 +++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 3 deletions(-) diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py index e1f2f33..f944d43 100644 --- a/app/tests/test_inverter_base.py +++ b/app/tests/test_inverter_base.py @@ -11,7 +11,7 @@ from app.src.gen3.talent import Talent from app.src.inverter_base import InverterBase from app.src.singleton import Singleton from app.src.protocol_ifc import ProtocolIfcImpl -from app.src.async_stream import AsyncStream, AsyncIfcImpl +from app.src.async_stream import AsyncStream, AsyncIfcImpl, AsyncStreamClient from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname @@ -101,6 +101,19 @@ def patch_healthy(): with patch.object(AsyncStream, 'healthy') as conn: yield conn +@pytest.fixture +def patch_unhealthy(): + def new_healthy(self): + return False + with patch.object(AsyncStream, 'healthy', new_healthy) as conn: + yield conn +@pytest.fixture +def patch_unhealthy_remote(): + def new_healthy(self): + return False + with patch.object(AsyncStreamClient, 'healthy', new_healthy) as conn: + yield conn + def test_protocol_iter(): ProtocolIfcImpl._registry.clear() cnt = 0 @@ -141,13 +154,166 @@ def test_method_calls(patch_healthy): with InverterBase(reader, writer, 'tsun', Talent) as inverter: assert inverter.local.stream assert inverter.local.ifc - # inverter.healthy() + # call healthy inside the contexter manager for inv in InverterBase: - inv.healthy() + assert inv.healthy() del inv spy.assert_called_once() + + # outside context manager the health function of AsyncStream is not reachable + cnt = 0 + for inv in InverterBase: + assert inv.healthy() + cnt += 1 + del inv + assert cnt == 1 + spy.assert_called_once() # counter don't increase and keep one! + del inverter cnt = 0 for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +def test_unhealthy(patch_unhealthy): + _ = patch_unhealthy + InverterBase._registry.clear() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + assert inverter.local.stream + assert inverter.local.ifc + # call healthy inside the contexter manager + assert not inverter.healthy() + + # outside context manager the unhealth AsyncStream is released + cnt = 0 + for inv in InverterBase: + assert inv.healthy() # inverter is healthy again (without the unhealty AsyncStream) + cnt += 1 + del inv + assert cnt == 1 + + del inverter + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +def test_unhealthy_remote(patch_unhealthy_remote): + _ = patch_unhealthy + InverterBase._registry.clear() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + assert inverter.local.stream + assert inverter.local.ifc + # call healthy inside the contexter manager + assert not inverter.healthy() + + # outside context manager the unhealth AsyncStream is released + cnt = 0 + for inv in InverterBase: + assert inv.healthy() # inverter is healthy again (without the unhealty AsyncStream) + cnt += 1 + del inv + assert cnt == 1 + + del inverter + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_remote_conn(config_conn, patch_open_connection): + _ = config_conn + _ = patch_open_connection + assert asyncio.get_running_loop() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + await inverter.create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream + assert inverter.remote.ifc + # call healthy inside the contexter manager + assert inverter.healthy() + + # call healthy outside the contexter manager (__exit__() was called) + assert inverter.healthy() + del inverter + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_unhealthy_remote(config_conn, patch_open_connection, patch_unhealthy_remote): + _ = config_conn + _ = patch_open_connection + _ = patch_unhealthy_remote + assert asyncio.get_running_loop() + InverterBase._registry.clear() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + assert inverter.local.stream + assert inverter.local.ifc + await inverter.create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream + assert inverter.remote.ifc + assert inverter.local.ifc.healthy() + assert not inverter.remote.ifc.healthy() + # call healthy inside the contexter manager + assert not inverter.healthy() + + # outside context manager the unhealth AsyncStream is released + cnt = 0 + for inv in InverterBase: + assert inv.healthy() # inverter is healthy again (without the unhealty AsyncStream) + cnt += 1 + del inv + assert cnt == 1 + + del inverter + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_remote_disc(config_conn, patch_open_connection): + _ = config_conn + _ = patch_open_connection + assert asyncio.get_running_loop() + reader = FakeReader() + writer = FakeWriter() + + with InverterBase(reader, writer, 'tsun', Talent) as inverter: + await inverter.create_remote() + await asyncio.sleep(0) + assert inverter.remote.stream + # call disc inside the contexter manager + await inverter.disc() + + # call disc outside the contexter manager (__exit__() was called) + await inverter.disc() + del inverter + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') cnt += 1 assert cnt == 0 From 9852f44dfa9094004b9e6395140f7132abd5feae Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sat, 5 Oct 2024 21:11:42 +0200 Subject: [PATCH 17/20] use ProtocolIfc class --- app/proxy.svg | 60 +++++++------- app/proxy.yuml | 6 +- app/proxy_2.svg | 136 ++++++++++++++++---------------- app/src/iter_registry.py | 8 -- app/src/messages.py | 7 +- app/src/protocol_ifc.py | 28 +------ app/tests/test_inverter_base.py | 17 +--- 7 files changed, 108 insertions(+), 154 deletions(-) diff --git a/app/proxy.svg b/app/proxy.svg index a8e328d..649fe8e 100644 --- a/app/proxy.svg +++ b/app/proxy.svg @@ -119,19 +119,19 @@ A6 - -StreamPtr - - -stream:MessageProt -ifc:AsyncIfc + +StreamPtr + + +stream:ProtocolIfc +ifc:AsyncIfc A5->A6 - - -2 + + +2 @@ -160,44 +160,44 @@ A11 - -<<AsyncIfc>> + +<<AsyncIfc>> A6->A11 - - -1 + + +1 A12 - -<<MessageProt>> + +<<ProtocolIfc>> A6->A12 - - -1 + + +1 A8 - - - -Creates an GEN3 -inverter instance -with -prot_class:Talent + + + +Creates an GEN3 +inverter instance +with +prot_class:Talent A7->A8 - + @@ -218,9 +218,9 @@ A12->A11 - - -use + + +use diff --git a/app/proxy.yuml b/app/proxy.yuml index 9d007cb..21c7d11 100644 --- a/app/proxy.yuml +++ b/app/proxy.yuml @@ -11,7 +11,7 @@ [<>||healthy()->bool;disc(shutdown_started=False);create_remote();] [<>]^-.-[<>] [InverterBase|_registry;__ha_restarts;;addr;config_id:str;prot_class:MessageProt;remote:StreamPtr;local:StreamPtr;|healthy()->bool;disc(shutdown_started=False);create_remote();async_publ_mqtt()] -[StreamPtr||stream:MessageProt;ifc:AsyncIfc] +[StreamPtr||stream:ProtocolIfc;ifc:AsyncIfc] [<>]^-.-[InverterBase] [InverterG3]-[note: Creates an GEN3 inverter instance with prot_class:Talent{bg:cornsilk}] [InverterG3P]-[note: Creates an GEN3PLUS inverter instance with prot_class:SolarmanV5{bg:cornsilk}] @@ -24,11 +24,11 @@ [<>] -[StreamPtr]-1>[<>] +[StreamPtr]-1>[<>] [StreamPtr]-1>[<>] -[<>]use-.->[<>] +[<>]use-.->[<>] [ModbusConn|host;port;addr;stream:InverterG3P;|]has-1>[InverterG3P] diff --git a/app/proxy_2.svg b/app/proxy_2.svg index 2aab16e..35c5479 100644 --- a/app/proxy_2.svg +++ b/app/proxy_2.svg @@ -4,11 +4,11 @@ - + G - + A0 @@ -47,81 +47,81 @@ A2 - -InverterG3 - -addr -remote:StreamPtr -local:StreamPtr - -create_remote() -close() + +InverterG3 + +addr +remote:StreamPtr +local:StreamPtr + +create_remote() +close() A7 - -AsyncStreamServer - -create_remote - -<async>server_loop() -<async>_async_forward() -<async>publish_outstanding_mqtt() -close() + +AsyncStreamServer + +create_remote + +<async>server_loop() +<async>_async_forward() +<async>publish_outstanding_mqtt() +close() A2->A7 - - - -local + + + +local A8 - -AsyncStreamClient - - -<async>client_loop() -<async>_async_forward()) + +AsyncStreamClient + + +<async>client_loop() +<async>_async_forward()) A2->A8 - - -remote + + +remote A3 - -InverterG3P - -addr -remote:StreamPtr -local:StreamPtr - -create_remote() -close() + +InverterG3P + +addr +remote:StreamPtr +local:StreamPtr + +create_remote() +close() A3->A7 - - - -local + + + +local A3->A8 - - -remote + + +remote @@ -199,14 +199,14 @@ A6->A7 - - + + A6->A8 - - + + @@ -237,17 +237,17 @@ A9->A2 - - -remote + + +remote A9->A2 - - - -local + + + +local @@ -295,17 +295,17 @@ A10->A3 - - -remote + + +remote A10->A3 - - - -local + + + +local diff --git a/app/src/iter_registry.py b/app/src/iter_registry.py index 34ebb25..ea0cd73 100644 --- a/app/src/iter_registry.py +++ b/app/src/iter_registry.py @@ -1,14 +1,6 @@ from abc import ABCMeta -class IterRegistry(type): - def __iter__(cls): - for ref in cls._registry: - obj = ref() - if obj is not None: - yield obj - - class AbstractIterMeta(ABCMeta): def __iter__(cls): for ref in cls._registry: diff --git a/app/src/messages.py b/app/src/messages.py index c68b125..bbff315 100644 --- a/app/src/messages.py +++ b/app/src/messages.py @@ -5,11 +5,11 @@ from enum import Enum if __name__ == "app.src.messages": - from app.src.iter_registry import IterRegistry + from app.src.protocol_ifc import ProtocolIfc from app.src.infos import Infos, Register from app.src.modbus import Modbus else: # pragma: no cover - from iter_registry import IterRegistry + from protocol_ifc import ProtocolIfc from infos import Infos, Register from modbus import Modbus @@ -82,8 +82,7 @@ class State(Enum): '''connection closed''' -class Message(metaclass=IterRegistry): - _registry = [] +class Message(ProtocolIfc): MAX_START_TIME = 400 '''maximum time without a received msg in sec''' MAX_INV_IDLE_TIME = 120 diff --git a/app/src/protocol_ifc.py b/app/src/protocol_ifc.py index 90ee8f1..46795e4 100644 --- a/app/src/protocol_ifc.py +++ b/app/src/protocol_ifc.py @@ -1,5 +1,4 @@ from abc import abstractmethod -import weakref if __name__ == "app.src.protocol_ifc": from app.src.iter_registry import AbstractIterMeta @@ -10,34 +9,13 @@ else: # pragma: no cover class ProtocolIfc(metaclass=AbstractIterMeta): - - @abstractmethod - def __init__(self, addr, ifc: "AsyncIfc", server_side: bool, - client_mode: bool = False, id_str=b''): - pass # pragma: no cover - - @abstractmethod - def close(self): - pass # pragma: no cover - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - self.close() - - -class Test(): - def test_method(self): - return self - - -class ProtocolIfcImpl(ProtocolIfc, Test): _registry = [] + @abstractmethod def __init__(self, addr, ifc: "AsyncIfc", server_side: bool, client_mode: bool = False, id_str=b''): - self._registry.append(weakref.ref(self)) + pass # pragma: no cover + @abstractmethod def close(self): pass # pragma: no cover diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py index f944d43..4e3ad92 100644 --- a/app/tests/test_inverter_base.py +++ b/app/tests/test_inverter_base.py @@ -10,8 +10,7 @@ from app.src.config import Config from app.src.gen3.talent import Talent from app.src.inverter_base import InverterBase from app.src.singleton import Singleton -from app.src.protocol_ifc import ProtocolIfcImpl -from app.src.async_stream import AsyncStream, AsyncIfcImpl, AsyncStreamClient +from app.src.async_stream import AsyncStream, AsyncStreamClient from app.tests.test_modbus_tcp import patch_mqtt_err, patch_mqtt_except, test_port, test_hostname @@ -114,20 +113,6 @@ def patch_unhealthy_remote(): with patch.object(AsyncStreamClient, 'healthy', new_healthy) as conn: yield conn -def test_protocol_iter(): - ProtocolIfcImpl._registry.clear() - cnt = 0 - ifc = AsyncIfcImpl() - prot = ProtocolIfcImpl(('test.intern', 123), ifc, True) - for p in ProtocolIfcImpl: - assert p == prot - cnt += 1 - del p - del prot - assert cnt == 1 - for p in ProtocolIfcImpl: - assert False - def test_inverter_iter(): InverterBase._registry.clear() cnt = 0 From e074a39f5aaf71ccba56e71b957c090c6aca70d8 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 6 Oct 2024 17:43:40 +0200 Subject: [PATCH 18/20] add unit tests for AsyncStream class --- app/proxy_2.svg | 334 ++++++++++++++++---------------- app/proxy_2.yuml | 2 +- app/src/async_ifc.py | 10 - app/src/async_stream.py | 17 +- app/src/byte_fifo.py | 3 +- app/tests/test_async_stream.py | 207 ++++++++++++++++++++ app/tests/test_inverter_base.py | 2 +- app/tests/test_modbus_tcp.py | 53 ++--- 8 files changed, 407 insertions(+), 221 deletions(-) create mode 100644 app/tests/test_async_stream.py diff --git a/app/proxy_2.svg b/app/proxy_2.svg index 35c5479..51cdaca 100644 --- a/app/proxy_2.svg +++ b/app/proxy_2.svg @@ -4,45 +4,45 @@ - - + + G - + A0 - - - -You can stick notes -on diagrams too! + + + +You can stick notes +on diagrams too! A1 - -IterRegistry - - -__iter__ + +IterRegistry + + +__iter__ A14 - -Message - -node_id - -inc_counter() -dec_counter() + +Message + +node_id + +inc_counter() +dec_counter() A1->A14 - - + + @@ -126,23 +126,21 @@ A4 - -<<AsyncIfc>> - - -set_node_id() -get_conn_no() -tx_add() -tx_flush() -tx_get() -tx_peek() -tx_log() -tx_clear() -tx_len() -fwd_add() -fwd_flush() -fwd_log() -fwd_clear() + +<<AsyncIfc>> + + +set_node_id() +get_conn_no() +tx_add() +tx_flush() +tx_get() +tx_peek() +tx_log() +tx_clear() +tx_len() +fwd_add() +fwd_log() rx_get() rx_peek() rx_log() @@ -167,8 +165,8 @@ A4->A5 - - + + @@ -211,208 +209,208 @@ A9 - -Talent - -ifc:AsyncIfc -conn_no -addr -await_conn_resp_cnt -id_str -contact_name -contact_mail -db:InfosG3 -mb:Modbus -switch - -msg_contact_info() -msg_ota_update() -msg_get_time() -msg_collector_data() -msg_inverter_data() -msg_unknown() -healthy() -close() + +Talent + +ifc:AsyncIfc +conn_no +addr +await_conn_resp_cnt +id_str +contact_name +contact_mail +db:InfosG3 +mb:Modbus +switch + +msg_contact_info() +msg_ota_update() +msg_get_time() +msg_collector_data() +msg_inverter_data() +msg_unknown() +healthy() +close() A9->A2 - - -remote + + +remote A9->A2 - - - -local + + + +local A9->A4 - - -use + + +use A12 - -InfosG3 - - -ha_confs() -parse() + +InfosG3 + + +ha_confs() +parse() A9->A12 - - + + A10 - -SolarmanV5 - -ifc:AsyncIfc -conn_no -addr -control -serial -snr -db:InfosG3P -mb:Modbus -switch - -msg_unknown() -healthy() -close() + +SolarmanV5 + +ifc:AsyncIfc +conn_no +addr +control +serial +snr +db:InfosG3P +mb:Modbus +switch + +msg_unknown() +healthy() +close() A10->A3 - - -remote + + +remote A10->A3 - - - -local + + + +local A10->A4 - - -use + + +use A13 - -InfosG3P - - -ha_confs() -parse() + +InfosG3P + + +ha_confs() +parse() A10->A13 - - + + A11 - -Infos - -stat -new_stat_data -info_dev - -static_init() -dev_value() -inc_counter() -dec_counter() -ha_proxy_conf -ha_conf -ha_remove -update_db -set_db_def_value -get_db_value -ignore_this_device + +Infos + +stat +new_stat_data +info_dev + +static_init() +dev_value() +inc_counter() +dec_counter() +ha_proxy_conf +ha_conf +ha_remove +update_db +set_db_def_value +get_db_value +ignore_this_device A11->A12 - - + + A11->A13 - - + + A14->A9 - - + + A14->A10 - - + + A15 - -Modbus - -que -snd_handler -rsp_handler -timeout -max_retires -last_xxx -err -retry_cnt -req_pend -tim - -build_msg() -recv_req() -recv_resp() -close() + +Modbus + +que +snd_handler +rsp_handler +timeout +max_retires +last_xxx +err +retry_cnt +req_pend +tim + +build_msg() +recv_req() +recv_resp() +close() A15->A9 - - -has -1 + + +has +1 A15->A10 - - -has -1 + + +has +1 diff --git a/app/proxy_2.yuml b/app/proxy_2.yuml index 33d4679..6a46bae 100644 --- a/app/proxy_2.yuml +++ b/app/proxy_2.yuml @@ -8,7 +8,7 @@ [InverterG3|addr;remote:StreamPtr;local:StreamPtr|create_remote();;close()] [InverterG3P|addr;remote:StreamPtr;local:StreamPtr|create_remote();;close()] -[<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_flush();fwd_log();fwd_clear();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] +[<>||set_node_id();get_conn_no();;tx_add();tx_flush();tx_get();tx_peek();tx_log();tx_clear();tx_len();;fwd_add();fwd_log();rx_get();rx_peek();rx_log();rx_clear();rx_len();rx_set_cb();;prot_set_timeout_cb()] [AsyncIfcImpl|fwd_fifo:ByteFifo;tx_fifo:ByteFifo;rx_fifo:ByteFifo;conn_no:Count;node_id;timeout_cb] [AsyncStream|reader;writer;addr;r_addr;l_addr|;loop;disc();close();healthy();;__async_read();__async_write();__async_forward()] [AsyncStreamServer|create_remote|server_loop();_async_forward();publish_outstanding_mqtt();close()] diff --git a/app/src/async_ifc.py b/app/src/async_ifc.py index 144f270..4e1536e 100644 --- a/app/src/async_ifc.py +++ b/app/src/async_ifc.py @@ -56,21 +56,11 @@ class AsyncIfc(ABC): ''' add data to forward queue''' pass # pragma: no cover - @abstractmethod - def fwd_flush(self): - ''' send forward queue and clears it''' - pass # pragma: no cover - @abstractmethod def fwd_log(self, level, info): ''' log the forward queue''' pass # pragma: no cover - @abstractmethod - def fwd_clear(self): - ''' clear forward queue''' - pass # pragma: no cover - # # RX - QUEUE # diff --git a/app/src/async_stream.py b/app/src/async_stream.py index eccd7d3..3676f96 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -80,18 +80,10 @@ class AsyncIfcImpl(AsyncIfc): ''' add data to forward queue''' self.fwd_fifo += data - def fwd_flush(self): - ''' send forward queue and clears it''' - self.fwd_fifo() - def fwd_log(self, level, info): ''' log the forward queue''' self.fwd_fifo.logging(level, info) - def fwd_clear(self): - ''' clear forward queue''' - self.fwd_fifo.clear() - def rx_get(self, size: int = None) -> bytearray: '''removes size numbers of bytes and return them''' return self.rx_fifo.get(size) @@ -131,9 +123,6 @@ class StreamPtr(): self.stream = _stream self.ifc = _ifc - def __str__(self) -> str: - return f'ifc:{self._ifc}, stream: {self._stream}' - @property def ifc(self): return self._ifc @@ -181,8 +170,8 @@ class AsyncStream(AsyncIfcImpl): self._writer.write(self.tx_fifo.get()) def __timeout(self) -> int: - if self.timeout_cb is callable: - return self.timeout_cb + if self.timeout_cb: + return self.timeout_cb() return 360 async def loop(self) -> Self: @@ -273,7 +262,7 @@ class AsyncStream(AsyncIfcImpl): self.proc_start = time.time() self.rx_fifo += data wait = self.rx_fifo() # call read in parent class - if wait > 0: + if wait and wait > 0: await asyncio.sleep(wait) else: raise RuntimeError("Peer closed.") diff --git a/app/src/byte_fifo.py b/app/src/byte_fifo.py index 90eb8ba..af9cb09 100644 --- a/app/src/byte_fifo.py +++ b/app/src/byte_fifo.py @@ -18,10 +18,11 @@ class ByteFifo: self.__buf.extend(data) return self - def __call__(self) -> None: + def __call__(self): '''triggers the observer''' if callable(self.__trigger_cb): return self.__trigger_cb() + return None def get(self, size: int = None) -> bytearray: '''removes size numbers of byte and return them''' diff --git a/app/tests/test_async_stream.py b/app/tests/test_async_stream.py new file mode 100644 index 0000000..864f2c3 --- /dev/null +++ b/app/tests/test_async_stream.py @@ -0,0 +1,207 @@ +# test_with_pytest.py +import pytest +import asyncio +import gc + +from app.src.infos import Infos +from app.src.inverter_base import InverterBase +from app.src.async_stream import AsyncStreamServer, AsyncStreamClient + +from app.tests.test_modbus_tcp import FakeReader, FakeWriter +from app.tests.test_inverter_base import config_conn, patch_open_connection + +pytest_plugins = ('pytest_asyncio',) + +# initialize the proxy statistics +Infos.static_init() + +def test_timeout_cb(): + reader = FakeReader() + writer = FakeWriter() + def timeout(): + return 13 + + ifc = AsyncStreamClient(reader, writer, None, None) + assert 360 == ifc._AsyncStream__timeout() + ifc.prot_set_timeout_cb(timeout) + assert 13 == ifc._AsyncStream__timeout() + ifc.prot_set_timeout_cb(None) + assert 360 == ifc._AsyncStream__timeout() + + # call healthy outside the contexter manager (__exit__() was called) + assert ifc.healthy() + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_close_cb(): + assert asyncio.get_running_loop() + reader = FakeReader() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 0.1 + def closed(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, closed) + ifc.prot_set_timeout_cb(timeout) + await ifc.client_loop('') + assert cnt == 1 + ifc.prot_set_timeout_cb(timeout) + await ifc.client_loop('') + assert cnt == 1 # check that the closed method would not be called + del ifc + + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, None) + ifc.prot_set_timeout_cb(timeout) + await ifc.client_loop('') + assert cnt == 0 + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_read(): + global test + assert asyncio.get_running_loop() + reader = FakeReader() + reader.test = FakeReader.RD_TEST_13_BYTES + reader.on_recv.set() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 1 + def closed(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + def app_read(): + return 0.01 # async wait of 0.01 + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, closed) + ifc.prot_set_timeout_cb(timeout) + ifc.rx_set_cb(app_read) + await ifc.client_loop('') + print('End loop') + assert 13 == ifc.rx_len() + assert cnt == 1 + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_write(): + global test + assert asyncio.get_running_loop() + reader = FakeReader() + reader.test = FakeReader.RD_TEST_13_BYTES + reader.on_recv.set() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 1 + def closed(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, closed) + ifc.prot_set_timeout_cb(timeout) + ifc.tx_add(b'test-data-resp') + assert 14 == ifc.tx_len() + await ifc.client_loop('') + print('End loop') + assert 13 == ifc.rx_len() + assert 0 == ifc.tx_len() + assert cnt == 1 + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_publ_mqtt_cb(): + assert asyncio.get_running_loop() + reader = FakeReader() + reader.test = FakeReader.RD_TEST_13_BYTES + reader.on_recv.set() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 0.1 + async def publ_mqtt(): + nonlocal cnt + nonlocal ifc + cnt += 1 + + cnt = 0 + ifc = AsyncStreamServer(reader, writer, publ_mqtt, None, None) + assert ifc.async_publ_mqtt + ifc.prot_set_timeout_cb(timeout) + await ifc.server_loop() + assert cnt == 3 # 2 calls in server_loop() and 1 in loop() + assert ifc.async_publ_mqtt + ifc.close() # clears the closed callback + assert not ifc.async_publ_mqtt + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_create_remote_cb(): + assert asyncio.get_running_loop() + reader = FakeReader() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 0.1 + async def create_remote(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + + cnt = 0 + ifc = AsyncStreamServer(reader, writer, None, create_remote, None) + assert ifc.create_remote + await ifc.create_remote() + assert cnt == 1 + ifc.prot_set_timeout_cb(timeout) + await ifc.server_loop() + assert not ifc.create_remote + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 diff --git a/app/tests/test_inverter_base.py b/app/tests/test_inverter_base.py index 4e3ad92..054d729 100644 --- a/app/tests/test_inverter_base.py +++ b/app/tests/test_inverter_base.py @@ -1,7 +1,7 @@ # test_with_pytest.py import pytest import asyncio -import sys,gc +import gc from mock import patch from enum import Enum diff --git a/app/tests/test_modbus_tcp.py b/app/tests/test_modbus_tcp.py index a8a8f5a..6a33727 100644 --- a/app/tests/test_modbus_tcp.py +++ b/app/tests/test_modbus_tcp.py @@ -71,22 +71,25 @@ def config_conn(test_hostname, test_port): } -class TestType(Enum): +class FakeReader(): RD_TEST_0_BYTES = 1 RD_TEST_TIMEOUT = 2 + RD_TEST_13_BYTES = 3 - -test = TestType.RD_TEST_0_BYTES - - -class FakeReader(): def __init__(self): self.on_recv = asyncio.Event() + self.test = self.RD_TEST_0_BYTES + async def read(self, max_len: int): + print(f'fakeReader test: {self.test}') await self.on_recv.wait() - if test == TestType.RD_TEST_0_BYTES: + if self.test == self.RD_TEST_0_BYTES: return b'' - elif test == TestType.RD_TEST_TIMEOUT: + elif self.test == self.RD_TEST_13_BYTES: + print('fakeReader return 13 bytes') + self.test = self.RD_TEST_0_BYTES + return b'test-data-req' + elif self.test == self.RD_TEST_TIMEOUT: raise TimeoutError def feed_eof(self): return @@ -95,8 +98,11 @@ class FakeReader(): class FakeWriter(): def __init__(self, conn='remote.intern'): self.conn = conn + self.closing = False def write(self, buf: bytes): return + async def drain(self): + await asyncio.sleep(0) def get_extra_info(self, sel: str): if sel == 'peername': return self.conn @@ -104,11 +110,11 @@ class FakeWriter(): return 'sock:1234' assert False def is_closing(self): - return False + return self.closing def close(self): - return + self.closing = True async def wait_closed(self): - return + await asyncio.sleep(0) @pytest.fixture @@ -118,14 +124,19 @@ def patch_open(): return FakeReader(), FakeWriter(conn) def new_open(host: str, port: int): - global test - if test == TestType.RD_TEST_TIMEOUT: - raise TimeoutError return new_conn(f'{host}:{port}') with patch.object(asyncio, 'open_connection', new_open) as conn: yield conn +@pytest.fixture +def patch_open_timeout(): + def new_open(host: str, port: int): + raise TimeoutError + + with patch.object(asyncio, 'open_connection', new_open) as conn: + yield conn + @pytest.fixture def patch_no_mqtt(): with patch.object(Mqtt, 'publish') as conn: @@ -170,13 +181,11 @@ async def test_modbus_no_cnf(): assert Infos.stat['proxy']['Inverter_Cnt'] == 0 @pytest.mark.asyncio -async def test_modbus_cnf1(config_conn, patch_open): +async def test_modbus_cnf1(config_conn, patch_open_timeout): _ = config_conn - _ = patch_open - global test + _ = patch_open_timeout assert asyncio.get_running_loop() Proxy.class_init() - test = TestType.RD_TEST_TIMEOUT assert Infos.stat['proxy']['Inverter_Cnt'] == 0 loop = asyncio.get_event_loop() @@ -194,10 +203,8 @@ async def test_modbus_cnf2(config_conn, patch_no_mqtt, patch_open): _ = config_conn _ = patch_open _ = patch_no_mqtt - global test assert asyncio.get_running_loop() Proxy.class_init() - test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 ModbusTcp(asyncio.get_event_loop()) @@ -220,10 +227,8 @@ async def test_modbus_cnf3(config_conn, patch_no_mqtt, patch_open): _ = config_conn _ = patch_open _ = patch_no_mqtt - global test assert asyncio.get_running_loop() Proxy.class_init() - test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 ModbusTcp(asyncio.get_event_loop(), tim_restart= 0) @@ -253,10 +258,8 @@ async def test_mqtt_err(config_conn, patch_mqtt_err, patch_open): _ = config_conn _ = patch_open _ = patch_mqtt_err - global test assert asyncio.get_running_loop() Proxy.class_init() - test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 ModbusTcp(asyncio.get_event_loop(), tim_restart= 0) @@ -286,10 +289,8 @@ async def test_mqtt_except(config_conn, patch_mqtt_except, patch_open): _ = config_conn _ = patch_open _ = patch_mqtt_except - global test assert asyncio.get_running_loop() Proxy.class_init() - test = TestType.RD_TEST_0_BYTES assert Infos.stat['proxy']['Inverter_Cnt'] == 0 ModbusTcp(asyncio.get_event_loop(), tim_restart= 0) From d7628689f02a2d62c4da2fecfa47c5e2b067508c Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 6 Oct 2024 21:15:46 +0200 Subject: [PATCH 19/20] icrease test coverage --- app/src/async_stream.py | 9 ++-- app/tests/test_async_stream.py | 85 ++++++++++++++++++++++++++++++++++ app/tests/test_modbus_tcp.py | 9 ++++ 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 3676f96..21bb6b9 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -179,10 +179,11 @@ class AsyncStream(AsyncIfcImpl): self.proc_start = time.time() while True: try: - proc = time.time() - self.proc_start - if proc > self.proc_max: - self.proc_max = proc - self.proc_start = None + if self.proc_start: + proc = time.time() - self.proc_start + if proc > self.proc_max: + self.proc_max = proc + self.proc_start = None dead_conn_to = self.__timeout() await asyncio.wait_for(self.__async_read(), dead_conn_to) diff --git a/app/tests/test_async_stream.py b/app/tests/test_async_stream.py index 864f2c3..afe0039 100644 --- a/app/tests/test_async_stream.py +++ b/app/tests/test_async_stream.py @@ -2,6 +2,7 @@ import pytest import asyncio import gc +import time from app.src.infos import Infos from app.src.inverter_base import InverterBase @@ -38,6 +39,26 @@ def test_timeout_cb(): cnt += 1 assert cnt == 0 +def test_health(): + reader = FakeReader() + writer = FakeWriter() + + ifc = AsyncStreamClient(reader, writer, None, None) + ifc.proc_start = time.time() + assert ifc.healthy() + ifc.proc_start = time.time() -10 + assert not ifc.healthy() + ifc.proc_start = None + assert ifc.healthy() + + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + @pytest.mark.asyncio async def test_close_cb(): assert asyncio.get_running_loop() @@ -92,6 +113,8 @@ async def test_read(): ifc.close() # clears the closed callback cnt += 1 def app_read(): + nonlocal ifc + ifc.proc_start -= 3 return 0.01 # async wait of 0.01 cnt = 0 ifc = AsyncStreamClient(reader, writer, None, closed) @@ -99,6 +122,7 @@ async def test_read(): ifc.rx_set_cb(app_read) await ifc.client_loop('') print('End loop') + assert ifc.proc_max >= 3 assert 13 == ifc.rx_len() assert cnt == 1 del ifc @@ -205,3 +229,64 @@ async def test_create_remote_cb(): print(f'InverterBase refs:{gc.get_referrers(inv)}') cnt += 1 assert cnt == 0 + +@pytest.mark.asyncio +async def test_sw_exception(): + global test + assert asyncio.get_running_loop() + reader = FakeReader() + reader.test = FakeReader.RD_TEST_SW_EXCEPT + reader.on_recv.set() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 1 + def closed(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, closed) + ifc.prot_set_timeout_cb(timeout) + await ifc.client_loop('') + print('End loop') + assert cnt == 1 + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 + +@pytest.mark.asyncio +async def test_os_error(): + global test + assert asyncio.get_running_loop() + reader = FakeReader() + reader.test = FakeReader.RD_TEST_OS_ERROR + + reader.on_recv.set() + writer = FakeWriter() + cnt = 0 + def timeout(): + return 1 + def closed(): + nonlocal cnt + nonlocal ifc + ifc.close() # clears the closed callback + cnt += 1 + cnt = 0 + ifc = AsyncStreamClient(reader, writer, None, closed) + ifc.prot_set_timeout_cb(timeout) + await ifc.client_loop('') + print('End loop') + assert cnt == 1 + del ifc + + cnt = 0 + for inv in InverterBase: + print(f'InverterBase refs:{gc.get_referrers(inv)}') + cnt += 1 + assert cnt == 0 diff --git a/app/tests/test_modbus_tcp.py b/app/tests/test_modbus_tcp.py index 6a33727..a39c2cd 100644 --- a/app/tests/test_modbus_tcp.py +++ b/app/tests/test_modbus_tcp.py @@ -75,6 +75,8 @@ class FakeReader(): RD_TEST_0_BYTES = 1 RD_TEST_TIMEOUT = 2 RD_TEST_13_BYTES = 3 + RD_TEST_SW_EXCEPT = 4 + RD_TEST_OS_ERROR = 5 def __init__(self): self.on_recv = asyncio.Event() @@ -91,6 +93,13 @@ class FakeReader(): return b'test-data-req' elif self.test == self.RD_TEST_TIMEOUT: raise TimeoutError + elif self.test == self.RD_TEST_SW_EXCEPT: + self.test = self.RD_TEST_0_BYTES + self.unknown_var += 1 + elif self.test == self.RD_TEST_OS_ERROR: + self.test = self.RD_TEST_0_BYTES + raise ConnectionRefusedError + def feed_eof(self): return From 595b68ba03ccb1535b4d07ecc25cb4c7eea5e854 Mon Sep 17 00:00:00 2001 From: Stefan Allius Date: Sun, 6 Oct 2024 21:20:53 +0200 Subject: [PATCH 20/20] reduce cognitive complexity --- app/src/async_stream.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/src/async_stream.py b/app/src/async_stream.py index 21bb6b9..559e48a 100644 --- a/app/src/async_stream.py +++ b/app/src/async_stream.py @@ -179,11 +179,7 @@ class AsyncStream(AsyncIfcImpl): self.proc_start = time.time() while True: try: - if self.proc_start: - proc = time.time() - self.proc_start - if proc > self.proc_max: - self.proc_max = proc - self.proc_start = None + self.__calc_proc_time() dead_conn_to = self.__timeout() await asyncio.wait_for(self.__async_read(), dead_conn_to) @@ -220,6 +216,13 @@ class AsyncStream(AsyncIfcImpl): f"{traceback.format_exc()}") await asyncio.sleep(0) # be cooperative to other task + def __calc_proc_time(self): + if self.proc_start: + proc = time.time() - self.proc_start + if proc > self.proc_max: + self.proc_max = proc + self.proc_start = None + async def disc(self) -> None: """Async disc handler for graceful disconnect""" self.remote = None