From ffacde41aa899407b728d943f7a4612d8f6a55d9 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 16:50:24 +0200 Subject: [PATCH 01/12] test requirement python-systemd --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fab42ac..fb33bd4 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,8 @@ setup(name='cbpi4', 'importlib_metadata', 'numpy==1.26.4', 'pandas==2.2.2'] + ( - ['rpi-lgpio'] if raspberrypi else [] ), + ['rpi-lgpio'] if raspberrypi else [] ) + ( + ['python-systemd'] if localsystem == "Linux" else [] ), dependency_links=[ 'https://testpypi.python.org/pypi', From fd2a8b2988bb3da16a79abd4b6af9104d76bebf6 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 16:56:05 +0200 Subject: [PATCH 02/12] change requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fb33bd4..c6cafe3 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup(name='cbpi4', 'numpy==1.26.4', 'pandas==2.2.2'] + ( ['rpi-lgpio'] if raspberrypi else [] ) + ( - ['python-systemd'] if localsystem == "Linux" else [] ), + ['systemd-python'] if localsystem == "Linux" else [] ), dependency_links=[ 'https://testpypi.python.org/pypi', From 3f74bd9de55f75d651299c2aba40f1e6a12924f3 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 19:25:56 +0200 Subject: [PATCH 03/12] test on log download --- cbpi/controller/system_controller.py | 79 +++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index ebf6f42..6e890e4 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -15,6 +15,14 @@ import zipfile import socket import importlib from tabulate import tabulate +from datetime import datetime, timedelta + +try: + from systemd import journal + systemd_available=True +except Exception: + logger.warning("Failed to load systemd library. logfile download not available") + systemd_available=False class SystemController: @@ -64,7 +72,7 @@ class SystemController: print(e) return tabulate(result, headers="keys") - async def downloadlog(self, logtime): + async def downloadlog_old(self, logtime): filename = "cbpi4.log" fullname = pathlib.Path(os.path.join(".",filename)) pluginname = "cbpi4_plugins.txt" @@ -123,6 +131,75 @@ class SystemController: logging.info(e) self.cbpi.notify("Error", "Removal of original files failed: {}".format(e), NotificationType.ERROR) + async def downloadlog(self, logtime): + filename = "cbpi4.log" + fullname = pathlib.Path(os.path.join(".",filename)) + pluginname = "cbpi4_plugins.txt" + fullpluginname = pathlib.Path(os.path.join(".",pluginname)) + actorname = "cbpi4_actors.txt" + fullactorname = pathlib.Path(os.path.join(".",actorname)) + sensorname = "cbpi4_sensors.txt" + fullsensorname = pathlib.Path(os.path.join(".",sensorname)) + kettlename = "cbpi4_kettles.txt" + fullkettlename = pathlib.Path(os.path.join(".",kettlename)) + + output_filename="cbpi4_log.zip" + + if logtime == "b": + os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) + else: + if systemd_available: + result=[] + #os.system('journalctl --since \"{} hours ago\" -u craftbeerpi.service --output cat > {}'.format(logtime, fullname)) + j = journal.Reader() + j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") + since = datetime.now() - timedelta(hours=int(logtime)) + j.seek_realtime(since) + for entry in j: + result.append(entry) + logging.error(result) + + + plugins = await self.plugins_list() + + with open(fullpluginname, 'w') as f: + f.write(plugins) + + #os.system('echo "{}" >> {}'.format(plugins,fullpluginname)) + + try: + actors = self.cbpi.actor.get_state() + json.dump(actors['data'],open(fullactorname,'w'),indent=4, sort_keys=True) + sensors = self.cbpi.sensor.get_state() + json.dump(sensors['data'],open(fullsensorname,'w'),indent=4, sort_keys=True) + kettles = self.cbpi.kettle.get_state() + json.dump(kettles['data'],open(fullkettlename,'w'),indent=4, sort_keys=True) + except Exception as e: + logging.info(e) + self.cbpi.notify("Error", "Creation of files failed: {}".format(e), NotificationType.ERROR) + + try: + zipObj=zipfile.ZipFile(output_filename , 'w', zipfile.ZIP_DEFLATED) + zipObj.write(fullname) + zipObj.write(fullpluginname) + zipObj.write(fullactorname) + zipObj.write(fullsensorname) + zipObj.write(fullkettlename) + zipObj.close() + except Exception as e: + logging.info(e) + self.cbpi.notify("Error", "Zip creation failed: {}".format(e), NotificationType.ERROR) + + try: + os.remove(fullname) + os.remove(fullpluginname) + os.remove(fullactorname) + os.remove(fullsensorname) + os.remove(fullkettlename) + except Exception as e: + logging.info(e) + self.cbpi.notify("Error", "Removal of original files failed: {}".format(e), NotificationType.ERROR) + def allowed_file(self, filename, extension): return '.' in filename and filename.rsplit('.', 1)[1] in set([extension]) From 5d1c1eef2ea99a8360c13e79698e948e6b3b98e2 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 19:31:59 +0200 Subject: [PATCH 04/12] fix --- cbpi/__init__.py | 2 +- cbpi/controller/system_controller.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 875186b..ddc10e3 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.4.1.a3" +__version__ = "4.4.1.a4" __codename__ = "Yeast Starter" diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 6e890e4..47f192e 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -156,7 +156,7 @@ class SystemController: since = datetime.now() - timedelta(hours=int(logtime)) j.seek_realtime(since) for entry in j: - result.append(entry) + result.append(entry['MESSAGE']) logging.error(result) From e12d437ed0df67e5ce86bce0d4ff5558462ab415 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 19:35:36 +0200 Subject: [PATCH 05/12] add write logfile --- cbpi/controller/system_controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 47f192e..47de359 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -159,6 +159,8 @@ class SystemController: result.append(entry['MESSAGE']) logging.error(result) + with open(fullname, 'w') as f: + f.write(result) plugins = await self.plugins_list() From d7ff5ca712d187870a65b760c19184cb1df3905b Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 19:37:22 +0200 Subject: [PATCH 06/12] check error --- cbpi/controller/system_controller.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 47de359..70c804a 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -158,9 +158,11 @@ class SystemController: for entry in j: result.append(entry['MESSAGE']) logging.error(result) - - with open(fullname, 'w') as f: - f.write(result) + try: + with open(fullname, 'w') as f: + f.write(result) + except Exception as e: + logging.error(e) plugins = await self.plugins_list() From 00ac853488183a833dd1500032d6786daa02632e Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 19:55:29 +0200 Subject: [PATCH 07/12] write logfile line by line --- cbpi/controller/system_controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 70c804a..89387fc 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -157,15 +157,14 @@ class SystemController: j.seek_realtime(since) for entry in j: result.append(entry['MESSAGE']) - logging.error(result) try: with open(fullname, 'w') as f: - f.write(result) + for line in result: + f.write(f"{line}\n") except Exception as e: logging.error(e) plugins = await self.plugins_list() - with open(fullpluginname, 'w') as f: f.write(plugins) From 1f56fa8fe1dd2fe8491c1b96c429556c52afc164 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Wed, 1 May 2024 20:36:29 +0200 Subject: [PATCH 08/12] fixas 'last boot log' was always overwritten --- cbpi/controller/system_controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 89387fc..c5b4256 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -157,12 +157,12 @@ class SystemController: j.seek_realtime(since) for entry in j: result.append(entry['MESSAGE']) - try: - with open(fullname, 'w') as f: - for line in result: - f.write(f"{line}\n") - except Exception as e: - logging.error(e) + try: + with open(fullname, 'w') as f: + for line in result: + f.write(f"{line}\n") + except Exception as e: + logging.error(e) plugins = await self.plugins_list() with open(fullpluginname, 'w') as f: From 4ca441bba39b6973dc02543bfc80b324b795a9bf Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 2 May 2024 07:08:23 +0200 Subject: [PATCH 09/12] test for log since last boot --- cbpi/__init__.py | 2 +- cbpi/controller/system_controller.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index ddc10e3..9c63548 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.4.1.a4" +__version__ = "4.4.1.a5" __codename__ = "Yeast Starter" diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index c5b4256..104ea71 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -146,7 +146,22 @@ class SystemController: output_filename="cbpi4_log.zip" if logtime == "b": - os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) + if systemd_available: + #os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) + j = journal.Reader() + j.add_match(_TRANSPORT="kernel") + result=[] + for entry in j: + message=entry['MESSAGE'] + if message.find("Booting") != -1: + result.append(entry['__REALTIME_TIMESTAMP']) + j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") + j.seek_realtime(result[-1]) + for entry in j: + timestamp=entry['__REALTIME_TIMESTAMP'] + message=entry['MESSAGE'] + print(message) + else: if systemd_available: result=[] From f5c242d7c856e8f7be2cac8e8ca4dcb51d779515 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 2 May 2024 07:32:38 +0200 Subject: [PATCH 10/12] write log since last boot to file --- cbpi/controller/system_controller.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 104ea71..ed13271 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -148,19 +148,25 @@ class SystemController: if logtime == "b": if systemd_available: #os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) - j = journal.Reader() - j.add_match(_TRANSPORT="kernel") + b = journal.Reader() + b.add_match(_TRANSPORT="kernel") result=[] - for entry in j: + for entry in b: message=entry['MESSAGE'] if message.find("Booting") != -1: result.append(entry['__REALTIME_TIMESTAMP']) + j = journal.Reader() j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") j.seek_realtime(result[-1]) + result=[] for entry in j: - timestamp=entry['__REALTIME_TIMESTAMP'] - message=entry['MESSAGE'] - print(message) + result.append(entry['MESSAGE']) + try: + with open(fullname, 'w') as f: + for line in result: + f.write(f"{line}\n") + except Exception as e: + logging.error(e) else: if systemd_available: From 858c71856da9ec61e92a730011fc8300fea4cdee Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 2 May 2024 12:45:44 +0200 Subject: [PATCH 11/12] simplify log download --- cbpi/__init__.py | 2 +- cbpi/controller/system_controller.py | 56 ++++++++-------------------- 2 files changed, 17 insertions(+), 41 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index 9c63548..134dd57 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.4.1.a5" +__version__ = "4.4.1.a6" __codename__ = "Yeast Starter" diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index ed13271..7f45365 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -144,53 +144,29 @@ class SystemController: fullkettlename = pathlib.Path(os.path.join(".",kettlename)) output_filename="cbpi4_log.zip" - - if logtime == "b": - if systemd_available: - #os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) - b = journal.Reader() - b.add_match(_TRANSPORT="kernel") - result=[] - for entry in b: - message=entry['MESSAGE'] - if message.find("Booting") != -1: - result.append(entry['__REALTIME_TIMESTAMP']) - j = journal.Reader() - j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") - j.seek_realtime(result[-1]) - result=[] - for entry in j: - result.append(entry['MESSAGE']) - try: - with open(fullname, 'w') as f: - for line in result: - f.write(f"{line}\n") - except Exception as e: - logging.error(e) - - else: - if systemd_available: - result=[] - #os.system('journalctl --since \"{} hours ago\" -u craftbeerpi.service --output cat > {}'.format(logtime, fullname)) - j = journal.Reader() - j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") + result=[] + if systemd_available: + j = journal.Reader() + if logtime == "b": + j.this_boot() + else: since = datetime.now() - timedelta(hours=int(logtime)) j.seek_realtime(since) - for entry in j: - result.append(entry['MESSAGE']) - try: - with open(fullname, 'w') as f: - for line in result: - f.write(f"{line}\n") - except Exception as e: - logging.error(e) + j.add_match(_SYSTEMD_UNIT="craftbeerpi.service") + + for entry in j: + result.append(entry['MESSAGE']) + try: + with open(fullname, 'w') as f: + for line in result: + f.write(f"{line}\n") + except Exception as e: + logging.error(e) plugins = await self.plugins_list() with open(fullpluginname, 'w') as f: f.write(plugins) - #os.system('echo "{}" >> {}'.format(plugins,fullpluginname)) - try: actors = self.cbpi.actor.get_state() json.dump(actors['data'],open(fullactorname,'w'),indent=4, sort_keys=True) From f5593fdc1f9bbbdc8f989fd004d31930dd4a67f2 Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Thu, 2 May 2024 12:52:22 +0200 Subject: [PATCH 12/12] remove old downloadlog function --- cbpi/controller/system_controller.py | 59 ---------------------------- 1 file changed, 59 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 7f45365..ae8a346 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -72,65 +72,6 @@ class SystemController: print(e) return tabulate(result, headers="keys") - async def downloadlog_old(self, logtime): - filename = "cbpi4.log" - fullname = pathlib.Path(os.path.join(".",filename)) - pluginname = "cbpi4_plugins.txt" - fullpluginname = pathlib.Path(os.path.join(".",pluginname)) - actorname = "cbpi4_actors.txt" - fullactorname = pathlib.Path(os.path.join(".",actorname)) - sensorname = "cbpi4_sensors.txt" - fullsensorname = pathlib.Path(os.path.join(".",sensorname)) - kettlename = "cbpi4_kettles.txt" - fullkettlename = pathlib.Path(os.path.join(".",kettlename)) - - output_filename="cbpi4_log.zip" - - if logtime == "b": - os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) - else: - os.system('journalctl --since \"{} hours ago\" -u craftbeerpi.service --output cat > {}'.format(logtime, fullname)) - - plugins = await self.plugins_list() - - with open(fullpluginname, 'w') as f: - f.write(plugins) - - #os.system('echo "{}" >> {}'.format(plugins,fullpluginname)) - - try: - actors = self.cbpi.actor.get_state() - json.dump(actors['data'],open(fullactorname,'w'),indent=4, sort_keys=True) - sensors = self.cbpi.sensor.get_state() - json.dump(sensors['data'],open(fullsensorname,'w'),indent=4, sort_keys=True) - kettles = self.cbpi.kettle.get_state() - json.dump(kettles['data'],open(fullkettlename,'w'),indent=4, sort_keys=True) - except Exception as e: - logging.info(e) - self.cbpi.notify("Error", "Creation of files failed: {}".format(e), NotificationType.ERROR) - - try: - zipObj=zipfile.ZipFile(output_filename , 'w', zipfile.ZIP_DEFLATED) - zipObj.write(fullname) - zipObj.write(fullpluginname) - zipObj.write(fullactorname) - zipObj.write(fullsensorname) - zipObj.write(fullkettlename) - zipObj.close() - except Exception as e: - logging.info(e) - self.cbpi.notify("Error", "Zip creation failed: {}".format(e), NotificationType.ERROR) - - try: - os.remove(fullname) - os.remove(fullpluginname) - os.remove(fullactorname) - os.remove(fullsensorname) - os.remove(fullkettlename) - except Exception as e: - logging.info(e) - self.cbpi.notify("Error", "Removal of original files failed: {}".format(e), NotificationType.ERROR) - async def downloadlog(self, logtime): filename = "cbpi4.log" fullname = pathlib.Path(os.path.join(".",filename))