From f01bdb94bde622d07bcca788ca18699688e6538a Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 19 Apr 2023 14:07:26 +0200 Subject: [PATCH 1/7] implements the log_data() hook --- cbpi/controller/log_file_controller.py | 50 +++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index c752216..9c61e85 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -13,6 +13,7 @@ from cbpi.api import * from cbpi.api.config import ConfigType from cbpi.api.base import CBPiBase import asyncio +import shortuuid class LogController: @@ -28,28 +29,59 @@ class LogController: self.datalogger = {} self.logsFolderPath = self.cbpi.config_folder.logsFolderPath self.logger.info("Log folder path : " + self.logsFolderPath) + self.sensor_data_listeners = {} + + def add_sensor_data_listener(self, method): + listener_id = shortuuid.uuid() + self.sensor_data_listeners[listener_id] = method + return listener_id + + def remove_sensor_data_listener(self, listener_id): + try: + del self.sensor_data_listener[listener_id] + except: + self.logger.error("Failed to remove listener {}".format(listener_id)) - def log_data(self, name: str, value: str) -> None: + async def _call_sensor_data_listeners(self, id, value, formatted_time, name): + for id, method in self.sensor_data_listeners.items(): + asyncio.create_task(method(self.cbpi, id, value, formatted_time, name)) + + def log_data(self, id: str, value: str) -> None: + # check which default log targets are enabled: self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") self.influxdb = self.cbpi.config.get("INFLUXDB", "No") + formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) + # ^^ both legacy log targets should probably be implemented as a core plugin each unsing the hook instead + + # CSV target: if self.logfiles == "Yes": - if name not in self.datalogger: + if id not in self.datalogger: max_bytes = int(self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 100000)) backup_count = int(self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3)) - data_logger = logging.getLogger('cbpi.sensor.%s' % name) + data_logger = logging.getLogger('cbpi.sensor.%s' % id) data_logger.propagate = False data_logger.setLevel(logging.DEBUG) - handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{name}.log"), maxBytes=max_bytes, backupCount=backup_count) + handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) data_logger.addHandler(handler) - self.datalogger[name] = data_logger - - formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) - self.datalogger[name].info("%s,%s" % (formatted_time, str(value))) + self.datalogger[id] = data_logger + self.datalogger[id].info("%s,%s" % (formatted_time, str(value))) + + # influx target: if self.influxdb == "Yes": ## Write to influxdb in an asyncio task - self._task = asyncio.create_task(self.log_influx(name,value)) + self._task = asyncio.create_task(self.log_influx(id,value)) + + # all plugin targets: + if self.sensor_data_listeners: # true if there are listners + try: + sensor=self.cbpi.sensor.find_by_id(id) + if sensor is not None: + name = sensor.name.replace(" ", "_") + asyncio.create_task(self._call_sensor_data_listeners(id, value, formatted_time, name)) + except Exception as e: + logging.error("sensor logging listener exception: {}".format(e)) async def log_influx(self, name:str, value:str): self.influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", "No") From 4eebb17291442213870a68eaf86fdf7f51d76288 Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 19 Apr 2023 16:47:48 +0200 Subject: [PATCH 2/7] moves influxDB integration into extensions folder --- cbpi/controller/log_file_controller.py | 46 +---------- .../SensorLogTarget_InfluxDB/__init__.py | 78 +++++++++++++++++++ .../SensorLogTarget_InfluxDB/config.yaml | 3 + 3 files changed, 82 insertions(+), 45 deletions(-) create mode 100644 cbpi/extension/SensorLogTarget_InfluxDB/__init__.py create mode 100644 cbpi/extension/SensorLogTarget_InfluxDB/config.yaml diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 9c61e85..b55c7eb 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -49,7 +49,6 @@ class LogController: def log_data(self, id: str, value: str) -> None: # check which default log targets are enabled: self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") - self.influxdb = self.cbpi.config.get("INFLUXDB", "No") formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) # ^^ both legacy log targets should probably be implemented as a core plugin each unsing the hook instead @@ -68,11 +67,6 @@ class LogController: self.datalogger[id].info("%s,%s" % (formatted_time, str(value))) - # influx target: - if self.influxdb == "Yes": - ## Write to influxdb in an asyncio task - self._task = asyncio.create_task(self.log_influx(id,value)) - # all plugin targets: if self.sensor_data_listeners: # true if there are listners try: @@ -81,45 +75,7 @@ class LogController: name = sensor.name.replace(" ", "_") asyncio.create_task(self._call_sensor_data_listeners(id, value, formatted_time, name)) except Exception as e: - logging.error("sensor logging listener exception: {}".format(e)) - - async def log_influx(self, name:str, value:str): - self.influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", "No") - self.influxdbaddr = self.cbpi.config.get("INFLUXDBADDR", None) - self.influxdbname = self.cbpi.config.get("INFLUXDBNAME", None) - self.influxdbuser = self.cbpi.config.get("INFLUXDBUSER", None) - self.influxdbpwd = self.cbpi.config.get("INFLUXDBPWD", None) - self.influxdbmeasurement = self.cbpi.config.get("INFLUXDBMEASUREMENT", "measurement") - id = name - timeout = Timeout(connect=5.0, read=None) - try: - sensor=self.cbpi.sensor.find_by_id(name) - if sensor is not None: - itemname=sensor.name.replace(" ", "_") - out=str(self.influxdbmeasurement)+",source=" + itemname + ",itemID=" + str(id) + " value="+str(value) - except Exception as e: - logging.error("InfluxDB ID Error: {}".format(e)) - - if self.influxdbcloud == "Yes": - self.influxdburl=self.influxdbaddr + "/api/v2/write?org=" + self.influxdbuser + "&bucket=" + self.influxdbname + "&precision=s" - try: - header = {'User-Agent': name, 'Authorization': "Token {}".format(self.influxdbpwd)} - http = PoolManager(timeout=timeout) - req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) - except Exception as e: - logging.error("InfluxDB cloud write Error: {}".format(e)) - - else: - self.base64string = base64.b64encode(('%s:%s' % (self.influxdbuser,self.influxdbpwd)).encode()) - self.influxdburl= self.influxdbaddr + '/write?db=' + self.influxdbname - try: - header = {'User-Agent': name, 'Content-Type': 'application/x-www-form-urlencoded','Authorization': 'Basic %s' % self.base64string.decode('utf-8')} - http = PoolManager(timeout=timeout) - req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) - except Exception as e: - logging.error("InfluxDB write Error: {}".format(e)) - - + logging.error("sensor logging listener exception: {}".format(e)) async def get_data(self, names, sample_rate='60s'): logging.info("Start Log for {}".format(names)) diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py new file mode 100644 index 0000000..2a35d67 --- /dev/null +++ b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py @@ -0,0 +1,78 @@ + +# -*- coding: utf-8 -*- +import os +from urllib3 import Timeout, PoolManager +import logging +from unittest.mock import MagicMock, patch +import asyncio +import random +from cbpi.api import * +from cbpi.api.config import ConfigType +import urllib3 +import base64 + +logger = logging.getLogger(__name__) + +# ToDo: +# - make log_data(id, value) to use id explicitly so there is no abiguity +# - create data legend for listener method call parameters including id, value, timestamp, name, cleanname +# - clean up data preperations for universal use +# - move influxDB logic to the plugin +# - + +class SensorLogTargetInfluxDB(CBPiExtension): + + def __init__(self, cbpi): # called from cbpi on start + self.cbpi = cbpi + self.influxdb = self.cbpi.config.get("INFLUXDB", "No") + if self.influxdb == "No": + return # never run() + self._task = asyncio.create_task(self.run()) # one time run() only + + + async def run(self): # called by __init__ once on start if influx is enabled + self.listener_ID = self.cbpi.log.add_sensor_data_listener(self.log_data_to_InfluxDB) + logger.info("InfluxDB sensor log target listener ID: {}".format(self.listener_ID)) + + async def log_data_to_InfluxDB(self, cbpi, id:str, value:str, timestamp, name): # called by log_data() hook from the log file controller + self.influxdb = self.cbpi.config.get("INFLUXDB", "No") + if self.influxdb == "No": + # We intentionally do not unsubscribe the listener here because then we had no way of resubscribing him without a restart of cbpi + # as long as cbpi was STARTED with INFLUXDB set to Yes this function is still subscribed, so changes can be made on the fly. + return + self.influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", "No") + self.influxdbaddr = self.cbpi.config.get("INFLUXDBADDR", None) + self.influxdbname = self.cbpi.config.get("INFLUXDBNAME", None) + self.influxdbuser = self.cbpi.config.get("INFLUXDBUSER", None) + self.influxdbpwd = self.cbpi.config.get("INFLUXDBPWD", None) + self.influxdbmeasurement = self.cbpi.config.get("INFLUXDBMEASUREMENT", "measurement") + timeout = Timeout(connect=5.0, read=None) + try: + sensor=self.cbpi.sensor.find_by_id(id) + if sensor is not None: + itemname=sensor.name.replace(" ", "_") + out=str(self.influxdbmeasurement)+",source=" + itemname + ",itemID=" + str(id) + " value="+str(value) + except Exception as e: + logging.error("InfluxDB ID Error: {}".format(e)) + + if self.influxdbcloud == "Yes": + self.influxdburl=self.influxdbaddr + "/api/v2/write?org=" + self.influxdbuser + "&bucket=" + self.influxdbname + "&precision=s" + try: + header = {'User-Agent': id, 'Authorization': "Token {}".format(self.influxdbpwd)} + http = PoolManager(timeout=timeout) + req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) + except Exception as e: + logging.error("InfluxDB cloud write Error: {}".format(e)) + + else: + self.base64string = base64.b64encode(('%s:%s' % (self.influxdbuser,self.influxdbpwd)).encode()) + self.influxdburl= self.influxdbaddr + '/write?db=' + self.influxdbname + try: + header = {'User-Agent': id, 'Content-Type': 'application/x-www-form-urlencoded','Authorization': 'Basic %s' % self.base64string.decode('utf-8')} + http = PoolManager(timeout=timeout) + req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) + except Exception as e: + logging.error("InfluxDB write Error: {}".format(e)) + +def setup(cbpi): + cbpi.plugin.register("SensorLogTargetInfluxDB", SensorLogTargetInfluxDB) diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/config.yaml b/cbpi/extension/SensorLogTarget_InfluxDB/config.yaml new file mode 100644 index 0000000..13cb8df --- /dev/null +++ b/cbpi/extension/SensorLogTarget_InfluxDB/config.yaml @@ -0,0 +1,3 @@ +name: SensorLogTargetInfluxDB +version: 4 +active: true From 384a8d5422d6a1e6d5ab165206c1127442aeeb6c Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 19 Apr 2023 17:33:18 +0200 Subject: [PATCH 3/7] moves CSV Sensor logging into core extension --- cbpi/controller/log_file_controller.py | 27 +--------- cbpi/extension/ConfigUpdate/__init__.py | 4 +- .../extension/SensorLogTarget_CSV/__init__.py | 52 +++++++++++++++++++ .../extension/SensorLogTarget_CSV/config.yaml | 3 ++ .../SensorLogTarget_InfluxDB/__init__.py | 1 + 5 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 cbpi/extension/SensorLogTarget_CSV/__init__.py create mode 100644 cbpi/extension/SensorLogTarget_CSV/config.yaml diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index b55c7eb..634f110 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -46,33 +46,14 @@ class LogController: for id, method in self.sensor_data_listeners.items(): asyncio.create_task(method(self.cbpi, id, value, formatted_time, name)) - def log_data(self, id: str, value: str) -> None: - # check which default log targets are enabled: - self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") - formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) - # ^^ both legacy log targets should probably be implemented as a core plugin each unsing the hook instead - - # CSV target: - if self.logfiles == "Yes": - if id not in self.datalogger: - max_bytes = int(self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 100000)) - backup_count = int(self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3)) - - data_logger = logging.getLogger('cbpi.sensor.%s' % id) - data_logger.propagate = False - data_logger.setLevel(logging.DEBUG) - handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) - data_logger.addHandler(handler) - self.datalogger[id] = data_logger - - self.datalogger[id].info("%s,%s" % (formatted_time, str(value))) - + def log_data(self, id: str, value: str) -> None: # all plugin targets: if self.sensor_data_listeners: # true if there are listners try: sensor=self.cbpi.sensor.find_by_id(id) if sensor is not None: name = sensor.name.replace(" ", "_") + formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) asyncio.create_task(self._call_sensor_data_listeners(id, value, formatted_time, name)) except Exception as e: logging.error("sensor logging listener exception: {}".format(e)) @@ -171,10 +152,6 @@ class LogController: def clear_log(self, name:str ) -> str: all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) - if name in self.datalogger: - self.datalogger[name].removeHandler(self.datalogger[name].handlers[0]) - del self.datalogger[name] - for f in all_filenames: try: os.remove(f) diff --git a/cbpi/extension/ConfigUpdate/__init__.py b/cbpi/extension/ConfigUpdate/__init__.py index 0aab218..ebc7a85 100644 --- a/cbpi/extension/ConfigUpdate/__init__.py +++ b/cbpi/extension/ConfigUpdate/__init__.py @@ -205,7 +205,7 @@ class ConfigUpdate(CBPiExtension): if logfiles is None: logger.info("INIT CSV logfiles") try: - await self.cbpi.config.add("CSVLOGFILES", "Yes", ConfigType.SELECT, "Write sensor data to csv logfiles", + await self.cbpi.config.add("CSVLOGFILES", "Yes", ConfigType.SELECT, "Write sensor data to csv logfiles (enabling requires restart)", [{"label": "Yes", "value": "Yes"}, {"label": "No", "value": "No"}]) except: @@ -215,7 +215,7 @@ class ConfigUpdate(CBPiExtension): if influxdb is None: logger.info("INIT Influxdb") try: - await self.cbpi.config.add("INFLUXDB", "No", ConfigType.SELECT, "Write sensor data to influxdb", + await self.cbpi.config.add("INFLUXDB", "No", ConfigType.SELECT, "Write sensor data to influxdb (enabling requires restart)", [{"label": "Yes", "value": "Yes"}, {"label": "No", "value": "No"}]) except: diff --git a/cbpi/extension/SensorLogTarget_CSV/__init__.py b/cbpi/extension/SensorLogTarget_CSV/__init__.py new file mode 100644 index 0000000..6983021 --- /dev/null +++ b/cbpi/extension/SensorLogTarget_CSV/__init__.py @@ -0,0 +1,52 @@ + +# -*- coding: utf-8 -*- +import os +from logging.handlers import RotatingFileHandler +import logging +from unittest.mock import MagicMock, patch +import asyncio +import random +from cbpi.api import * +from cbpi.api.config import ConfigType +import urllib3 +import base64 + +logger = logging.getLogger(__name__) + +class SensorLogTargetCSV(CBPiExtension): + + def __init__(self, cbpi): # called from cbpi on start + self.cbpi = cbpi + self.datalogger = {} + self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") + if self.logfiles == "No": + return # never run() + self._task = asyncio.create_task(self.run()) # one time run() only + + + async def run(self): # called by __init__ once on start if CSV is enabled + self.listener_ID = self.cbpi.log.add_sensor_data_listener(self.log_data_to_CSV) + logger.info("CSV sensor log target listener ID: {}".format(self.listener_ID)) + + async def log_data_to_CSV(self, cbpi, id:str, value:str, formatted_time, name): # called by log_data() hook from the log file controller + self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") + if self.logfiles == "No": + # We intentionally do not unsubscribe the listener here because then we had no way of resubscribing him without a restart of cbpi + # as long as cbpi was STARTED with CSVLOGFILES set to Yes this function is still subscribed, so changes can be made on the fly. + # but after initially enabling this logging target a restart is required. + return + if id not in self.datalogger: + max_bytes = int(self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 100000)) + backup_count = int(self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3)) + + data_logger = logging.getLogger('cbpi.sensor.%s' % id) + data_logger.propagate = False + data_logger.setLevel(logging.DEBUG) + handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) + data_logger.addHandler(handler) + self.datalogger[id] = data_logger + + self.datalogger[id].info("%s,%s" % (formatted_time, str(value))) + +def setup(cbpi): + cbpi.plugin.register("SensorLogTargetCSV", SensorLogTargetCSV) diff --git a/cbpi/extension/SensorLogTarget_CSV/config.yaml b/cbpi/extension/SensorLogTarget_CSV/config.yaml new file mode 100644 index 0000000..d396a0c --- /dev/null +++ b/cbpi/extension/SensorLogTarget_CSV/config.yaml @@ -0,0 +1,3 @@ +name: SensorLogTargetCSV +version: 4 +active: true diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py index 2a35d67..44d68f9 100644 --- a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py +++ b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py @@ -39,6 +39,7 @@ class SensorLogTargetInfluxDB(CBPiExtension): if self.influxdb == "No": # We intentionally do not unsubscribe the listener here because then we had no way of resubscribing him without a restart of cbpi # as long as cbpi was STARTED with INFLUXDB set to Yes this function is still subscribed, so changes can be made on the fly. + # but after initially enabling this logging target a restart is required. return self.influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", "No") self.influxdbaddr = self.cbpi.config.get("INFLUXDBADDR", None) From 7fd6361d3036394f5df790fc8418d34ac6bd670e Mon Sep 17 00:00:00 2001 From: prash3r Date: Sun, 14 May 2023 15:55:41 +0200 Subject: [PATCH 4/7] hookable log data debugging fixes --- cbpi/controller/log_file_controller.py | 6 +++--- cbpi/extension/SensorLogTarget_CSV/__init__.py | 2 +- cbpi/extension/SensorLogTarget_InfluxDB/__init__.py | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 634f110..14a9c02 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -42,9 +42,9 @@ class LogController: except: self.logger.error("Failed to remove listener {}".format(listener_id)) - async def _call_sensor_data_listeners(self, id, value, formatted_time, name): - for id, method in self.sensor_data_listeners.items(): - asyncio.create_task(method(self.cbpi, id, value, formatted_time, name)) + async def _call_sensor_data_listeners(self, sensor_id, value, formatted_time, name): + for listener_id, method in self.sensor_data_listeners.items(): + asyncio.create_task(method(self.cbpi, sensor_id, value, formatted_time, name)) def log_data(self, id: str, value: str) -> None: # all plugin targets: diff --git a/cbpi/extension/SensorLogTarget_CSV/__init__.py b/cbpi/extension/SensorLogTarget_CSV/__init__.py index 6983021..8fbb468 100644 --- a/cbpi/extension/SensorLogTarget_CSV/__init__.py +++ b/cbpi/extension/SensorLogTarget_CSV/__init__.py @@ -42,7 +42,7 @@ class SensorLogTargetCSV(CBPiExtension): data_logger = logging.getLogger('cbpi.sensor.%s' % id) data_logger.propagate = False data_logger.setLevel(logging.DEBUG) - handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) + handler = RotatingFileHandler(os.path.join(self.cbpi.log.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) data_logger.addHandler(handler) self.datalogger[id] = data_logger diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py index 44d68f9..879e057 100644 --- a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py +++ b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py @@ -62,6 +62,8 @@ class SensorLogTargetInfluxDB(CBPiExtension): header = {'User-Agent': id, 'Authorization': "Token {}".format(self.influxdbpwd)} http = PoolManager(timeout=timeout) req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) + if req.status != 204: + raise Exception(f'InfluxDB Status code {req.status}') except Exception as e: logging.error("InfluxDB cloud write Error: {}".format(e)) @@ -72,6 +74,8 @@ class SensorLogTargetInfluxDB(CBPiExtension): header = {'User-Agent': id, 'Content-Type': 'application/x-www-form-urlencoded','Authorization': 'Basic %s' % self.base64string.decode('utf-8')} http = PoolManager(timeout=timeout) req = http.request('POST',self.influxdburl, body=out.encode(), headers = header) + if req.status != 204: + raise Exception(f'InfluxDB Status code {req.status}') except Exception as e: logging.error("InfluxDB write Error: {}".format(e)) From 125bd071628934029102890424720700a213fac0 Mon Sep 17 00:00:00 2001 From: prash3r Date: Sun, 14 May 2023 17:43:48 +0200 Subject: [PATCH 5/7] modified tests for hookable log data --- tests/cbpi-test-config/config.json | 14 +++----------- tests/cbpi-test-config/sensor.json | 10 +++++++++- tests/test_logger.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/cbpi-test-config/config.json b/tests/cbpi-test-config/config.json index da7fab4..298b8b0 100644 --- a/tests/cbpi-test-config/config.json +++ b/tests/cbpi-test-config/config.json @@ -80,7 +80,7 @@ "options": null, "source": "hidden", "type": "string", - "value": "4.1.8.a11" + "value": "4.1.10.a1" }, "CSVLOGFILES": { "description": "Write sensor data to csv logfiles", @@ -117,7 +117,7 @@ "value": "No" }, "INFLUXDBADDR": { - "description": "IP Address of your influxdb server (If INFLUXDBCLOUD set to Yes use URL Address of your influxdb cloud server)", + "description": "URL Address of your influxdb server incl. http:// and port, e.g. http://localhost:8086 (If INFLUXDBCLOUD set to Yes use URL Address of your influxdb cloud server)", "name": "INFLUXDBADDR", "options": null, "source": "craftbeerpi", @@ -157,14 +157,6 @@ "type": "string", "value": "cbpi4" }, - "INFLUXDBPORT": { - "description": "Port of your influxdb server", - "name": "INFLUXDBPORT", - "options": null, - "source": "craftbeerpi", - "type": "string", - "value": "8086" - }, "INFLUXDBPWD": { "description": "Password for your influxdb database (only if required)(If INFLUXDBCLOUD set to Yes use token of your influxdb cloud database)", "name": "INFLUXDBPWD", @@ -174,7 +166,7 @@ "value": " " }, "INFLUXDBUSER": { - "description": "User name for your influxdb database (only if required)(If INFLUXDBCLOUD set to Yes use organisation of your influxdb cloud database)", + "description": "User Name for your influxdb database (only if required)(If INFLUXDBCLOUD set to Yes use organisation of your influxdb cloud database)", "name": "INFLUXDBUSER", "options": null, "source": "craftbeerpi", diff --git a/tests/cbpi-test-config/sensor.json b/tests/cbpi-test-config/sensor.json index ce96464..4cf8966 100644 --- a/tests/cbpi-test-config/sensor.json +++ b/tests/cbpi-test-config/sensor.json @@ -1,3 +1,11 @@ { - "data": [] + "data": [ + { + "id": "unconfigured_test_sensor_ID", + "name": "unconfigured_mqtt_sensor", + "props": {}, + "state": false, + "type": "MQTTSensor" + } + ] } \ No newline at end of file diff --git a/tests/test_logger.py b/tests/test_logger.py index 50d729b..633ba00 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -10,10 +10,10 @@ class LoggerTestCase(CraftBeerPiTestCase): async def test_log_data(self): os.makedirs(os.path.join(".", "tests", "logs"), exist_ok=True) - log_name = "test" + log_name = "unconfigured_test_sensor_ID" #clear all logs self.cbpi.log.clear_log(log_name) - assert len(glob.glob(os.path.join(".", "tests", "logs", f"sensor_{log_name}.log*"))) == 0 + assert len(glob.glob(os.path.join(self.cbpi.log.logsFolderPath, f"sensor_{log_name}.log*"))) == 0 # write log entries for i in range(5): From 955409d81aa946e84ba024601188d7fe5c6b91e0 Mon Sep 17 00:00:00 2001 From: prash3r Date: Tue, 16 May 2023 12:23:24 +0200 Subject: [PATCH 6/7] repaires log clearence from analytics page --- .gitignore | 3 ++- .vscode/launch.json | 6 +++++- cbpi/controller/log_file_controller.py | 6 ++++++ cbpi/extension/SensorLogTarget_CSV/__init__.py | 7 +++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d23da45..6157d3e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ logs/ .coverage .devcontainer/cbpi-dev-config/* cbpi4-* -temp* \ No newline at end of file +temp* +*.patch \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index e24e1e9..0b0fdc4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,11 @@ "type": "python", "request": "launch", "module": "run", - "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "start"], + "args": [ + "--config-folder-path=./.devcontainer/cbpi-dev-config", + "--debug-log-level=20", + "start" + ], "preLaunchTask": "copy default cbpi config files if dev config files dont exist" }, diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 14a9c02..fecae47 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -151,6 +151,12 @@ class LogController: def clear_log(self, name:str ) -> str: all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) + + logging.info(f'Deleting logfiles for sensor {name}.') + + if name in self.datalogger: + self.datalogger[name].removeHandler(self.datalogger[name].handlers[0]) + del self.datalogger[name] for f in all_filenames: try: diff --git a/cbpi/extension/SensorLogTarget_CSV/__init__.py b/cbpi/extension/SensorLogTarget_CSV/__init__.py index 8fbb468..6360cf7 100644 --- a/cbpi/extension/SensorLogTarget_CSV/__init__.py +++ b/cbpi/extension/SensorLogTarget_CSV/__init__.py @@ -17,7 +17,6 @@ class SensorLogTargetCSV(CBPiExtension): def __init__(self, cbpi): # called from cbpi on start self.cbpi = cbpi - self.datalogger = {} self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") if self.logfiles == "No": return # never run() @@ -35,7 +34,7 @@ class SensorLogTargetCSV(CBPiExtension): # as long as cbpi was STARTED with CSVLOGFILES set to Yes this function is still subscribed, so changes can be made on the fly. # but after initially enabling this logging target a restart is required. return - if id not in self.datalogger: + if id not in self.cbpi.log.datalogger: max_bytes = int(self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 100000)) backup_count = int(self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3)) @@ -44,9 +43,9 @@ class SensorLogTargetCSV(CBPiExtension): data_logger.setLevel(logging.DEBUG) handler = RotatingFileHandler(os.path.join(self.cbpi.log.logsFolderPath, f"sensor_{id}.log"), maxBytes=max_bytes, backupCount=backup_count) data_logger.addHandler(handler) - self.datalogger[id] = data_logger + self.cbpi.log.datalogger[id] = data_logger - self.datalogger[id].info("%s,%s" % (formatted_time, str(value))) + self.cbpi.log.datalogger[id].info("%s,%s" % (formatted_time, str(value))) def setup(cbpi): cbpi.plugin.register("SensorLogTargetCSV", SensorLogTargetCSV) From 819c9f33b9831068ff5b69f0635969bf099dbfe0 Mon Sep 17 00:00:00 2001 From: prash3r Date: Tue, 16 May 2023 13:40:59 +0200 Subject: [PATCH 7/7] removes unnecessary comments previously used as ToDo-List --- cbpi/extension/SensorLogTarget_InfluxDB/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py index 879e057..cb9fe75 100644 --- a/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py +++ b/cbpi/extension/SensorLogTarget_InfluxDB/__init__.py @@ -13,13 +13,6 @@ import base64 logger = logging.getLogger(__name__) -# ToDo: -# - make log_data(id, value) to use id explicitly so there is no abiguity -# - create data legend for listener method call parameters including id, value, timestamp, name, cleanname -# - clean up data preperations for universal use -# - move influxDB logic to the plugin -# - - class SensorLogTargetInfluxDB(CBPiExtension): def __init__(self, cbpi): # called from cbpi on start