From 355376a1baf68c6d7dde18fd37ef6276f959759b Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 29 Jun 2023 07:26:16 +0200 Subject: [PATCH 1/6] adapted requirements (cryptography/pyopenssl for security, asyncio-mqtt to aiomqtt due to change of package name) --- cbpi/__init__.py | 2 +- cbpi/controller/satellite_controller.py | 2 +- requirements.txt | 5 +++-- setup.py | 5 +++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 2a723f2..3b6c5ac 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.1.10" +__version__ = "4.1.11.a1" __codename__ = "Groundhog Day" diff --git a/cbpi/controller/satellite_controller.py b/cbpi/controller/satellite_controller.py index 3fc4b2e..3e7dfd3 100644 --- a/cbpi/controller/satellite_controller.py +++ b/cbpi/controller/satellite_controller.py @@ -2,7 +2,7 @@ import asyncio import json from re import M -from asyncio_mqtt import Client, MqttError, Will +from aiomqtt import Client, MqttError, Will from contextlib import AsyncExitStack, asynccontextmanager from cbpi import __version__ import logging diff --git a/requirements.txt b/requirements.txt index bdcfc87..a4ac8b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,8 @@ aiohttp-session==2.12.0 aiohttp-swagger==1.0.16 aiojobs==1.1.0 aiosqlite==0.17.0 -cryptography==40.0.1 +cryptography==41.0.1 +pyopenssl==23.2.0 requests==2.31.0 voluptuous==0.13.1 pyfiglet==0.8.post1 @@ -18,7 +19,7 @@ numpy==1.24.1 cbpi4gui click==8.1.3 importlib_metadata==4.11.1 -asyncio-mqtt==0.16.1 +aiomqtt==1.0.0 psutil==5.9.4 zipp>=0.5 colorama==0.4.6 diff --git a/setup.py b/setup.py index 7a2159d..a430f9c 100644 --- a/setup.py +++ b/setup.py @@ -47,14 +47,15 @@ setup(name='cbpi4', "aiohttp-swagger==1.0.16", "aiojobs==1.1.0 ", "aiosqlite==0.17.0", - "cryptography==40.0.1", + "cryptography==41.0.1", + "pyopenssl==23.2.0", "requests==2.31.0", "voluptuous==0.13.1", "pyfiglet==0.8.post1", 'click==8.1.3', 'shortuuid==1.0.11', 'tabulate==0.9.0', - 'asyncio-mqtt==0.16.1', + 'aiomqtt==1.0.0', 'inquirer==3.1.1', 'colorama==0.4.6', 'psutil==5.9.4', From a301276725219ab810fd1b406eb4af4e7b1eec03 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 29 Jun 2023 19:16:27 +0200 Subject: [PATCH 2/6] test: add wait time for influxdb if no connection can be established after several retries -> currently, there seems to be interference with mqtt. --- cbpi/__init__.py | 2 +- .../SensorLogTarget_InfluxDB/__init__.py | 66 ++++++++++++------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 3b6c5ac..705bfac 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.1.11.a1" +__version__ = "4.1.11.a2" __codename__ = "Groundhog Day" diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py index cb9fe75..622c611 100644 --- a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py +++ b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os -from urllib3 import Timeout, PoolManager +from urllib3 import Timeout, PoolManager, Retry import logging from unittest.mock import MagicMock, patch import asyncio @@ -21,6 +21,9 @@ class SensorLogTargetInfluxDB(CBPiExtension): if self.influxdb == "No": return # never run() self._task = asyncio.create_task(self.run()) # one time run() only + self.counter = 0 + self.max_retries = 2 + self.send=True async def run(self): # called by __init__ once on start if influx is enabled @@ -40,7 +43,7 @@ class SensorLogTargetInfluxDB(CBPiExtension): self.influxdbuser = self.cbpi.config.get("INFLUXDBUSER", None) self.influxdbpwd = self.cbpi.config.get("INFLUXDBPWD", None) self.influxdbmeasurement = self.cbpi.config.get("INFLUXDBMEASUREMENT", "measurement") - timeout = Timeout(connect=5.0, read=None) + timeout = Timeout(connect=2.0, read=None) try: sensor=self.cbpi.sensor.find_by_id(id) if sensor is not None: @@ -49,28 +52,45 @@ class SensorLogTargetInfluxDB(CBPiExtension): except Exception as e: logging.error("InfluxDB ID Error: {}".format(e)) - if self.influxdbcloud == "Yes": - self.influxdburl=self.influxdbaddr + "/api/v2/write?org=" + self.influxdbuser + "&bucket=" + self.influxdbname + "&precision=s" - try: - header = {'User-Agent': id, 'Authorization': "Token {}".format(self.influxdbpwd)} - http = PoolManager(timeout=timeout) - req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) - if req.status != 204: - raise Exception(f'InfluxDB Status code {req.status}') - except Exception as e: - logging.error("InfluxDB cloud write Error: {}".format(e)) + if self.influxdbcloud == "Yes" and self.send == True: + if self.counter <= self.max_retries: + self.influxdburl=self.influxdbaddr + "/api/v2/write?org=" + self.influxdbuser + "&bucket=" + self.influxdbname + "&precision=s" + try: + header = {'User-Agent': id, 'Authorization': "Token {}".format(self.influxdbpwd)} + http = PoolManager(timeout=timeout) + req = http.request('POST',self.influxdburl, body=out.encode(), headers = header, retries=Retry(2)) + if req.status != 204: + raise Exception(f'InfluxDB Status code {req.status}') + except Exception as e: + self.counter += 1 + logging.error("InfluxDB cloud write Error #{}: {}".format(self.counter, e)) + else: + logging.warning("Waiting 3 Minutes before connecting to INFLUXDB again") + self.send=False + await asyncio.sleep(180) + self.counter = 0 + self.send = True + + elif self.influxdbcloud == "No" and self.send == True: + if self.counter <= self.max_retries: + self.base64string = base64.b64encode(('%s:%s' % (self.influxdbuser,self.influxdbpwd)).encode()) + self.influxdburl= self.influxdbaddr + '/write?db=' + self.influxdbname + try: + header = {'User-Agent': id, 'Content-Type': 'application/x-www-form-urlencoded','Authorization': 'Basic %s' % self.base64string.decode('utf-8')} + http = PoolManager(timeout=timeout) + req = http.request('POST',self.influxdburl, body=out.encode(), headers = header, retries=Retry(2)) + if req.status != 204: + raise Exception(f'InfluxDB Status code {req.status}') + except Exception as e: + self.counter += 1 + logging.error("InfluxDB write Error #{}: {}".format(self.counter, e)) + else: + logging.warning("Waiting 3 Minutes before connecting to INFLUXDB again") + self.send=False + await asyncio.sleep(180) + self.counter = 0 + self.send = True - else: - self.base64string = base64.b64encode(('%s:%s' % (self.influxdbuser,self.influxdbpwd)).encode()) - self.influxdburl= self.influxdbaddr + '/write?db=' + self.influxdbname - try: - header = {'User-Agent': id, 'Content-Type': 'application/x-www-form-urlencoded','Authorization': 'Basic %s' % self.base64string.decode('utf-8')} - http = PoolManager(timeout=timeout) - req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) - if req.status != 204: - raise Exception(f'InfluxDB Status code {req.status}') - except Exception as e: - logging.error("InfluxDB write Error: {}".format(e)) def setup(cbpi): cbpi.plugin.register("SensorLogTargetInfluxDB", SensorLogTargetInfluxDB) From d6c2e465acfdc11f4138f3d5364e24fd484a92bd Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:45:28 +0200 Subject: [PATCH 3/6] change requirements (depemdabot recommendation) --- cbpi/__init__.py | 2 +- requirements.txt | 4 ++-- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 705bfac..78e3a7b 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.1.11.a2" +__version__ = "4.1.11.a3" __codename__ = "Groundhog Day" diff --git a/requirements.txt b/requirements.txt index a4ac8b6..dfafb21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ typing-extensions>=4 -aiohttp==3.8.3 +aiohttp==3.8.5 aiohttp-auth==0.1.1 aiohttp-route-decorator==0.1.4 aiohttp-security==0.4.0 @@ -7,7 +7,7 @@ aiohttp-session==2.12.0 aiohttp-swagger==1.0.16 aiojobs==1.1.0 aiosqlite==0.17.0 -cryptography==41.0.1 +cryptography==41.0.2 pyopenssl==23.2.0 requests==2.31.0 voluptuous==0.13.1 diff --git a/setup.py b/setup.py index a430f9c..7d9854d 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup(name='cbpi4', long_description_content_type='text/markdown', install_requires=[ "typing-extensions>=4", - "aiohttp==3.8.4", + "aiohttp==3.8.5", "aiohttp-auth==0.1.1", "aiohttp-route-decorator==0.1.4", "aiohttp-security==0.4.0", @@ -47,7 +47,7 @@ setup(name='cbpi4', "aiohttp-swagger==1.0.16", "aiojobs==1.1.0 ", "aiosqlite==0.17.0", - "cryptography==41.0.1", + "cryptography==41.0.2", "pyopenssl==23.2.0", "requests==2.31.0", "voluptuous==0.13.1", From 2eece208eb98c25902c2f7d7f6454fc10e7185b5 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Sat, 29 Jul 2023 15:10:28 +0200 Subject: [PATCH 4/6] activate break to mqtt listen loop on cancel -> prevent sigkill on stop or restart service --- cbpi/controller/satellite_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpi/controller/satellite_controller.py b/cbpi/controller/satellite_controller.py index 3e7dfd3..7877666 100644 --- a/cbpi/controller/satellite_controller.py +++ b/cbpi/controller/satellite_controller.py @@ -73,7 +73,7 @@ class SatelliteController: except asyncio.CancelledError: # Cancel self.logger.warning("MQTT Listening Cancelled") - #break + break except MqttError as e: self.logger.error("MQTT Exception: {}".format(e)) except Exception as e: From 4dee07fc264335c05167ecadf7e9891f46477802 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Sat, 29 Jul 2023 15:11:26 +0200 Subject: [PATCH 5/6] update version # for previous push --- cbpi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 78e3a7b..2bc710e 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.1.11.a3" +__version__ = "4.1.11.a4" __codename__ = "Groundhog Day" From 17d49d5a949c27e54b7c459da7246f2d0c718b6f Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Sun, 30 Jul 2023 12:18:28 +0200 Subject: [PATCH 6/6] bump version --- cbpi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 2bc710e..7eeec87 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.1.11.a4" +__version__ = "4.1.11" __codename__ = "Groundhog Day"