From 2f085965c780c5baa9a475f7ae1aa668320828df Mon Sep 17 00:00:00 2001 From: prash3r Date: Fri, 2 Sep 2022 17:00:50 +0200 Subject: [PATCH 01/13] extends cbpi-dev-config to be a complete set. cbpi requires empty folders inside the config folder to fully function, and this commit adds them. There are also some more extensive checks on missing files and folders newly implemented. As well as checking for restored_config.zip is now done before checking for config.yaml. On unsuccessfull restore the zip file is renamed instead of deleted. --- .../{_widgets_are_placed_here => .gitkeep} | 0 .../.gitkeep} | 0 .../cbpi-dev-config/logs/sensors/.gitkeep | 0 .../cbpi-dev-config/recipes/.gitkeep | 0 .devcontainer/cbpi-dev-config/upload/.gitkeep | 0 .gitignore | 3 +- .vscode/launch.json | 10 +- cbpi/configFolder.py | 114 +++++++++++++----- 8 files changed, 97 insertions(+), 30 deletions(-) rename .devcontainer/cbpi-dev-config/dashboard/widgets/{_widgets_are_placed_here => .gitkeep} (100%) rename .devcontainer/cbpi-dev-config/{upload/_uploads_are_placed_here => fermenterrecipes/.gitkeep} (100%) create mode 100644 .devcontainer/cbpi-dev-config/logs/sensors/.gitkeep create mode 100644 .devcontainer/cbpi-dev-config/recipes/.gitkeep create mode 100644 .devcontainer/cbpi-dev-config/upload/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/dashboard/widgets/_widgets_are_placed_here b/.devcontainer/cbpi-dev-config/dashboard/widgets/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/dashboard/widgets/_widgets_are_placed_here rename to .devcontainer/cbpi-dev-config/dashboard/widgets/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/upload/_uploads_are_placed_here b/.devcontainer/cbpi-dev-config/fermenterrecipes/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/upload/_uploads_are_placed_here rename to .devcontainer/cbpi-dev-config/fermenterrecipes/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/logs/sensors/.gitkeep b/.devcontainer/cbpi-dev-config/logs/sensors/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.devcontainer/cbpi-dev-config/recipes/.gitkeep b/.devcontainer/cbpi-dev-config/recipes/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.devcontainer/cbpi-dev-config/upload/.gitkeep b/.devcontainer/cbpi-dev-config/upload/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index bb723ac..20ad518 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ node_modules .DS_Store config/* logs/ -.coverage \ No newline at end of file +.coverage +.devcontainer/cbpi-dev-config/* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index c2f9637..feff4f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,15 @@ "configurations": [ { - "name": "Run CraftBeerPi4", + "name": "setup CraftBeerPi4: create config folder structure", + "type": "python", + "request": "launch", + "module": "run", + "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "setup"] + }, + + { + "name": "run CraftBeerPi4", "type": "python", "request": "launch", "module": "run", diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index 5741d4f..8926a91 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -1,3 +1,4 @@ +from ast import If, Try import os from os import listdir from os.path import isfile, join @@ -33,25 +34,8 @@ class ConfigFolder: return fermenter_recipe_ids def check_for_setup(self): - if self.config_file_exists("config.yaml") is False: - print("***************************************************") - print("CraftBeerPi Config File not found: %s" % self.get_file_path("config.yaml")) - print("Please run 'cbpi setup' before starting the server ") - print("***************************************************") - return False - if self.config_file_exists("upload") is False: - print("***************************************************") - print("CraftBeerPi upload folder not found: %s" % self.get_file_path("upload")) - print("Please run 'cbpi setup' before starting the server ") - print("***************************************************") - return False - # if os.path.exists(os.path.join(".", "config", "fermenterrecipes")) is False: - # print("***************************************************") - # print("CraftBeerPi fermenterrecipes folder not found: %s" % os.path.join(".", "config/fermenterrecipes")) - # print("Please run 'cbpi setup' before starting the server ") - # print("***************************************************") - # return False - backupfile = os.path.join(".", "restored_config.zip") + # is there a restored_config.zip file? if yes restore it first then delte the zip. + backupfile = os.path.join(self._rawPath, "restored_config.zip") if os.path.exists(os.path.join(backupfile)) is True: print("***************************************************") print("Found backup of config. Starting restore") @@ -75,7 +59,7 @@ class ConfigFolder: owner = output_path.owner() group = output_path.group() print("Removing old config folder") - shutil.rmtree(output_path, ignore_errors=True) + shutil.rmtree(output_path, ignore_errors=True) print("Extracting zip file to config folder") zip.extractall(output_path) zip.close() @@ -83,12 +67,79 @@ class ConfigFolder: print(f"Changing owner and group of config folder recursively to {owner}:{group}") self.recursive_chown(output_path, owner, group) print("Removing backup file") - os.remove(backupfile) + print("contents of restored_config.zip file have been restored.") + print("in case of a partial backup you will still be prompted to run 'cbpi setup'.") + # os.remove(backupfile) # since the zip was inside the config folder and the config folder was deleted 10 lines ago this file doesnt exist anymore else: print("Wrong Content in zip file. No restore possible") - print("Removing zip file") - os.remove(backupfile) + print("renaming zip file so it will be ignored on the next start") + try: + os.rename(backupfile, os.path.join(self._rawPath, "UNRESTORABLE_restored_config.zip")) + except: + print("renamed file does exist - deleting instead") + os.remove(backupfile) print("***************************************************") + # possible restored_config.zip has been handeled now lets check if files and folders exist + required_config_content = [ + ['config.yaml', 'file'], + ['actor.json', 'file'], + ['sensor.json', 'file'], + ['kettle.json', 'file'], + ['fermenter_data.json', 'file'], + ['step_data.json', 'file'], + ['config.json', 'file'], + ['craftbeerpi.service', 'file'], + ['chromium.desktop', 'file'], + ['dashboard/cbpi_dashboard_1.json', 'file'], + ['dashboard', 'folder'], + ['dashboard/widgets', 'folder'], + ['fermenterrecipes', 'folder'], + ['logs', 'folder'], + ['logs/sensors', 'folder'], + ['recipes', 'folder'], + ['upload', 'folder'] + ] + for checking in required_config_content: + if self.inform_missing_content(self.check_for_file_or_folder(os.path.join(self._rawPath, checking[0]), checking[1])): + # since there is no complete config we now check if the config folde rmay be completely empty to show hints: + if len(os.listdir(os.path.join(self._rawPath))) == 0 : + print("***************************************************") + print(f"the config folder '{self._rawPath}' seems to be completely empty") + print("you might want to run 'cbpi setup'.print") + print("but you could also place your zipped config backup named") + print("'restored_config.zip' inside the mentioned config folder for") + print("cbpi4 to automatically unpack it") + print("of course you can also place your config files manually") + print("***************************************************") + return False + + def inform_missing_content(self, whatsmissing : str): + if whatsmissing == "": + return False + print("***************************************************") + print(f"CraftBeerPi config content not found: {whatsmissing}") + print("Please run 'cbpi setup' before starting the server ") + print("***************************************************") + return True + + + def check_for_file_or_folder(self, path : str, file_or_folder : str = ""): # file_or_folder should be "file" or "folder" or "" if both is ok + if (file_or_folder == ""): # file and folder is ok + if os.path.exists(path): + return "" + else: + return "file or folder missing: " + path + if (file_or_folder == "file"): # only file is ok + if (os.path.isfile(path)): + return "" + else: + return "file missing: " + path + if (file_or_folder == "folder"): # oly folder is ok + if (os.path.isdir(path)): + return "" + else: + return "folder missing: " + path + return "usage of check_file_or_folder() function wrong. second Argument must either be 'file' or 'folder' or an empty string" def copyDefaultFileIfNotExists(self, file): if self.config_file_exists(file) is False: @@ -115,7 +166,7 @@ class ConfigFolder: print("Config Folder created") def create_home_folder_structure(configFolder): - pathlib.Path(os.path.join(".", 'logs/sensors')).mkdir(parents=True, exist_ok=True) + # pathlib.Path(os.path.join(".", 'logs/sensors')).mkdir(parents=True, exist_ok=True) configFolder.create_folders() print("Folder created") @@ -123,11 +174,18 @@ class ConfigFolder: def create_folders(self): pathlib.Path(self._rawPath).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self._rawPath, 'logs', 'sensors')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'recipes')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self._rawPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'upload')).mkdir(parents=True, exist_ok=True) def recursive_chown(self, path, owner, group): - for dirpath, dirnames, filenames in os.walk(path): - shutil.chown(dirpath, owner, group) - for filename in filenames: - shutil.chown(os.path.join(dirpath, filename), owner, group) \ No newline at end of file + try: + for dirpath, dirnames, filenames in os.walk(path): + shutil.chown(dirpath, owner, group) + for filename in filenames: + shutil.chown(os.path.join(dirpath, filename), owner, group) + except: + print("problems assigning file or folder permissions") + print("if this happend on windows its fine") + print("if this happend in the dev container running inside windows its also fine but you might have to rebuild the container if you run into further problems") From 2076c66eb5b3d926b5af340afceed0e8955bc40b Mon Sep 17 00:00:00 2001 From: prash3r Date: Sat, 3 Sep 2022 13:43:17 +0200 Subject: [PATCH 02/13] uploaded restored_config.zip is now placed in provided config folder and not the working directory where it will be ignored on restart --- cbpi/controller/system_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 8c211fc..24128fe 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -116,8 +116,8 @@ class SystemController: try: content = backup_file.read() if backup_file and self.allowed_file(filename, 'zip'): - self.path = os.path.join(".", "restored_config.zip") - + self.path = os.path.join(self.cbpi.config_folder._rawPath, "restored_config.zip") + f=open(self.path, "wb") f.write(content) f.close() From e61fbdb69cb5290295be937117b07819f18e17c2 Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 7 Sep 2022 12:39:52 +0200 Subject: [PATCH 03/13] extends #62 reorders the .vscode tasks so run is the default hopefully. removes code that was commented out anyway. --- .vscode/launch.json | 16 ++++++++-------- cbpi/configFolder.py | 3 --- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index feff4f1..73c363b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,20 +5,20 @@ "version": "0.2.0", "configurations": [ - { - "name": "setup CraftBeerPi4: create config folder structure", - "type": "python", - "request": "launch", - "module": "run", - "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "setup"] - }, - { "name": "run CraftBeerPi4", "type": "python", "request": "launch", "module": "run", "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "start"] + }, + + { + "name": "setup CraftBeerPi4: create config folder structure", + "type": "python", + "request": "launch", + "module": "run", + "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "setup"] } ] } \ No newline at end of file diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index 8926a91..058271c 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -69,7 +69,6 @@ class ConfigFolder: print("Removing backup file") print("contents of restored_config.zip file have been restored.") print("in case of a partial backup you will still be prompted to run 'cbpi setup'.") - # os.remove(backupfile) # since the zip was inside the config folder and the config folder was deleted 10 lines ago this file doesnt exist anymore else: print("Wrong Content in zip file. No restore possible") print("renaming zip file so it will be ignored on the next start") @@ -166,8 +165,6 @@ class ConfigFolder: print("Config Folder created") def create_home_folder_structure(configFolder): - # pathlib.Path(os.path.join(".", 'logs/sensors')).mkdir(parents=True, exist_ok=True) - configFolder.create_folders() print("Folder created") From 4952861a584224311c4072c78ce15abca8913022 Mon Sep 17 00:00:00 2001 From: prash3r Date: Fri, 9 Sep 2022 18:38:08 +0200 Subject: [PATCH 04/13] logs folder is next to the active config folder. It was wherever 'cbpi setup' was run from. and if no config folder path is provided its still the same. This also adds a file based global logger placed in the logs folder. The sensor loggers respect the new logs folder location. --- .../cbpi-dev-config/logs/sensors/.gitkeep | 0 cbpi/cli.py | 12 ++++++++++- cbpi/configFolder.py | 7 +++---- cbpi/controller/config_controller.py | 2 ++ cbpi/controller/log_file_controller.py | 21 +++++++++++-------- cbpi/craftbeerpi.py | 6 ------ 6 files changed, 28 insertions(+), 20 deletions(-) delete mode 100644 .devcontainer/cbpi-dev-config/logs/sensors/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/logs/sensors/.gitkeep b/.devcontainer/cbpi-dev-config/logs/sensors/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/cbpi/cli.py b/cbpi/cli.py index c1ff9be..1bf8012 100644 --- a/cbpi/cli.py +++ b/cbpi/cli.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path import requests from cbpi.configFolder import ConfigFolder from cbpi.utils.utils import load_config @@ -230,7 +231,16 @@ def main(context, config_folder_path): print("Welcome to CBPi") print("---------------------") level = logging.INFO - logging.basicConfig(level=level, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s') + logger = logging.getLogger() + logger.setLevel(level) + try: + logger.addHandler(logging.handlers.RotatingFileHandler(os.path.join(Path(config_folder_path).parent, 'logs', f"cbpi.log"), maxBytes=1000000, backupCount=3)) + except: + print("there seems to be no log folder - continueing without (maybe you should run 'cbpi setup')") + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') + for handler in logger.handlers: + handler.setLevel(level) + handler.setFormatter(formatter) cbpi_cli = CraftBeerPiCli(ConfigFolder(config_folder_path)) context.obj = cbpi_cli pass diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index 058271c..a29db84 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -90,11 +90,10 @@ class ConfigFolder: ['craftbeerpi.service', 'file'], ['chromium.desktop', 'file'], ['dashboard/cbpi_dashboard_1.json', 'file'], - ['dashboard', 'folder'], ['dashboard/widgets', 'folder'], + ['dashboard', 'folder'], ['fermenterrecipes', 'folder'], - ['logs', 'folder'], - ['logs/sensors', 'folder'], + ['../logs', 'folder'], ['recipes', 'folder'], ['upload', 'folder'] ] @@ -171,7 +170,7 @@ class ConfigFolder: def create_folders(self): pathlib.Path(self._rawPath).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, 'logs', 'sensors')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self._rawPath, '..','logs')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'recipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self._rawPath, 'upload')).mkdir(parents=True, exist_ok=True) diff --git a/cbpi/controller/config_controller.py b/cbpi/controller/config_controller.py index 4afaa0b..2412688 100644 --- a/cbpi/controller/config_controller.py +++ b/cbpi/controller/config_controller.py @@ -1,6 +1,7 @@ from cbpi.api.dataclasses import Config import logging import os +from pathlib import Path from cbpi.api.config import ConfigType from cbpi.utils import load_config @@ -16,6 +17,7 @@ class ConfigController: self.cbpi.register(self) self.path = cbpi.config_folder.get_file_path("config.json") self.path_static = cbpi.config_folder.get_file_path("config.yaml") + self.logger.info("Config folder path : " + os.path.join(Path(self.cbpi.config_folder._rawPath).absolute())) def get_state(self): diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 968929f..05024bb 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -8,6 +8,7 @@ import pandas as pd import zipfile import base64 import urllib3 +from pathlib import Path from cbpi.api import * from cbpi.api.config import ConfigType from cbpi.api.base import CBPiBase @@ -25,6 +26,8 @@ class LogController: self.logger = logging.getLogger(__name__) self.configuration = False self.datalogger = {} + self.logsFolderPath = os.path.join(Path(self.cbpi.config_folder._rawPath).parent.absolute(), 'logs') + self.logger.info("Log folder path : " + self.logsFolderPath) def log_data(self, name: str, value: str) -> None: self.logfiles = self.cbpi.config.get("CSVLOGFILES", "Yes") @@ -37,7 +40,7 @@ class LogController: data_logger = logging.getLogger('cbpi.sensor.%s' % name) data_logger.propagate = False data_logger.setLevel(logging.DEBUG) - handler = RotatingFileHandler('./logs/sensor_%s.log' % name, maxBytes=max_bytes, backupCount=backup_count) + handler = RotatingFileHandler(os.path.join(self.logsFolderPath, f"sensor_{name}.log"), maxBytes=max_bytes, backupCount=backup_count) data_logger.addHandler(handler) self.datalogger[name] = data_logger @@ -115,7 +118,7 @@ class LogController: for name in names: # get all log names - all_filenames = glob.glob('./logs/sensor_%s.log*' % name) + all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") # concat all logs df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', name], header=None) for f in all_filenames]) logging.info("Read all files for {}".format(names)) @@ -157,7 +160,7 @@ class LogController: for id in ids: # df = pd.read_csv("./logs/sensor_%s.log" % id, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime',"Values"], header=None) # concat all logs - all_filenames = glob.glob('./logs/sensor_%s.log*' % id) + all_filenames = glob.glob(os.path.join(self.logsFolderPath,f"./logs/sensor_{id}.log*")) df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', 'Values'], header=None) for f in all_filenames]) df = df.resample('60s').max() df = df.dropna() @@ -173,11 +176,11 @@ class LogController: :return: list of log file names ''' - return [os.path.basename(x) for x in glob.glob('./logs/sensor_%s.log*' % name)] + return [os.path.basename(x) for x in os.path.join(self.logsFolderPath, f"sensor_{name}.log*")] def clear_log(self, name:str ) -> str: - all_filenames = glob.glob('./logs/sensor_%s.log*' % name) + all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") for f in all_filenames: os.remove(f) @@ -193,7 +196,7 @@ class LogController: :return: ''' - return [os.path.basename(x) for x in glob.glob('./logs/*-sensor-%s.zip' % name)] + return [os.path.basename(x) for x in glob.glob(os.path.join(self.logsFolderPath, f"*-sensor-{name}.zip"))] def clear_zip(self, name:str ) -> None: """ @@ -202,7 +205,7 @@ class LogController: :return: None """ - all_filenames = glob.glob('./logs/*-sensor-%s.zip' % name) + all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"*-sensor-{name}.zip")) for f in all_filenames: os.remove(f) @@ -213,9 +216,9 @@ class LogController: """ formatted_time = strftime("%Y-%m-%d-%H_%M_%S", localtime()) - file_name = './logs/%s-sensor-%s.zip' % (formatted_time, name) + file_name = os.path.join(self.logsFolderPath, f"./logs/{formatted_time}-sensor-{name}.zip" % (formatted_time, name)) zip = zipfile.ZipFile(file_name, 'w', zipfile.ZIP_DEFLATED) - all_filenames = glob.glob('./logs/sensor_%s.log*' % name) + all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") for f in all_filenames: zip.write(os.path.join(f)) zip.close() diff --git a/cbpi/craftbeerpi.py b/cbpi/craftbeerpi.py index c4dad9d..d3c60c6 100644 --- a/cbpi/craftbeerpi.py +++ b/cbpi/craftbeerpi.py @@ -301,12 +301,6 @@ class CraftBeerPi: self._swagger_setup() - level = logging.INFO - logger = logging.getLogger() - logger.setLevel(level) - for handler in logger.handlers: - handler.setLevel(level) - return self.app def start(self): From c56d7ccc51b29693914392d01023ecebad0cae14 Mon Sep 17 00:00:00 2001 From: prash3r Date: Mon, 12 Sep 2022 21:54:51 +0200 Subject: [PATCH 05/13] adds --logs-folder-path as cli option --- cbpi/cli.py | 41 +++++++++++++++------- cbpi/configFolder.py | 47 ++++++++++++++------------ cbpi/controller/config_controller.py | 2 +- cbpi/controller/log_file_controller.py | 2 +- cbpi/controller/system_controller.py | 2 +- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/cbpi/cli.py b/cbpi/cli.py index 1bf8012..83a2582 100644 --- a/cbpi/cli.py +++ b/cbpi/cli.py @@ -9,6 +9,7 @@ import os import pkgutil import shutil import click +import pathlib from subprocess import call from colorama import Fore, Back, Style import importlib @@ -16,6 +17,7 @@ from importlib_metadata import metadata from tabulate import tabulate from PyInquirer import prompt, print_json import platform +import subprocess class CraftBeerPiCli(): def __init__(self, config) -> None: @@ -226,24 +228,37 @@ class CraftBeerPiCli(): @click.group() @click.pass_context @click.option('--config-folder-path', '-c', default="./config", type=click.Path(), help="Specify where the config folder is located. Defaults to './config'.") -def main(context, config_folder_path): +@click.option('--logs-folder-path', '-l', default="", type=click.Path(), help="Specify where the log folder is located. Defaults to '../logs' relative from the config folder.") +@click.option('--debug-log-level', '-d', default="30", type=int, help="Specify the log level you want to write to all logs. 0=ALL, 10=DEBUG, 20=INFO 30(default)=WARNING, 40=ERROR, 50=CRITICAL") +def main(context, config_folder_path, logs_folder_path, debug_log_level): print("---------------------") print("Welcome to CBPi") print("---------------------") - level = logging.INFO - logger = logging.getLogger() - logger.setLevel(level) - try: - logger.addHandler(logging.handlers.RotatingFileHandler(os.path.join(Path(config_folder_path).parent, 'logs', f"cbpi.log"), maxBytes=1000000, backupCount=3)) - except: - print("there seems to be no log folder - continueing without (maybe you should run 'cbpi setup')") + if logs_folder_path == "": + logs_folder_path = os.path.join(Path(config_folder_path).absolute().parent, 'logs') formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') - for handler in logger.handlers: - handler.setLevel(level) - handler.setFormatter(formatter) - cbpi_cli = CraftBeerPiCli(ConfigFolder(config_folder_path)) + logging.basicConfig(format=formatter,level=debug_log_level, stream=logging.StreamHandler()) + logger = logging.getLogger() + try: + result = subprocess.run(['journalctl', '--since', '2 hours ago', '-u', 'craftbeerpi.service' ], stdout=subprocess.PIPE) + # journalctl is present, we assume we are running in production. + # We therefore omit the timestamp from the stdout log handler formatter because timestamps are added to the logs by journalctl anyway + logger.handlers[0].setFormatter('%(levelname)s - %(name)s - %(message)s') + except: + # journalctl command seems to not be present. + # We assume we are in the dev container and keep writing timestampts to stdout for vscode terminal output + logger.warning("journalctl command error - assuming dev container execution and writing timestamps to stdout") + try: + if not os.path.isdir(logs_folder_path): + logger.info(f"logs folder '{logs_folder_path}' doesnt exist and we are trying to create it") + pathlib.Path(logs_folder_path).mkdir(parents=True, exist_ok=True) + logger.info(f"logs folder '{logs_folder_path}' successfully created") + logger.addHandler(logging.handlers.RotatingFileHandler(os.path.join(logs_folder_path, f"cbpi.log"), maxBytes=1000000, backupCount=3)) + except Exception as e: + logger.warning("log folder or log file could not be created or accessed. check folder and file permissions or create the logs folder somewhere you have access with a start option like '--log-folder-path=./logs'") + logging.critical(e, exc_info=True) + cbpi_cli = CraftBeerPiCli(ConfigFolder(config_folder_path, logs_folder_path)) context.obj = cbpi_cli - pass @main.command() @click.pass_context diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index a29db84..d3690cb 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -6,36 +6,39 @@ import pathlib import platform import shutil import zipfile +from pathlib import Path import glob class ConfigFolder: - def __init__(self, configFolderPath): - self._rawPath = configFolderPath + def __init__(self, configFolderPath, logsFolderPath): + self.configFolderPath = configFolderPath + self.logsFolderPath = logsFolderPath + self.logger.info("Config folder path : " + os.path.join(Path(self.cbpi.config_folder.configFolderPath).absolute())) def config_file_exists(self, path): return os.path.exists(self.get_file_path(path)) def get_file_path(self, file): - return os.path.join(self._rawPath, file) + return os.path.join(self.configFolderPath, file) def get_upload_file(self, file): - return os.path.join(self._rawPath, 'upload', file) + return os.path.join(self.configFolderPath, 'upload', file) def get_recipe_file_by_id(self, recipe_id): - return os.path.join(self._rawPath, 'recipes', "{}.yaml".format(recipe_id)) + return os.path.join(self.configFolderPath, 'recipes', "{}.yaml".format(recipe_id)) def get_fermenter_recipe_by_id(self, recipe_id): - return os.path.join(self._rawPath, 'fermenterrecipes', "{}.yaml".format(recipe_id)) + return os.path.join(self.configFolderPath, 'fermenterrecipes', "{}.yaml".format(recipe_id)) def get_all_fermenter_recipes(self): - fermenter_recipes_folder = os.path.join(self._rawPath, 'fermenterrecipes') + fermenter_recipes_folder = os.path.join(self.configFolderPath, 'fermenterrecipes') fermenter_recipe_ids = [os.path.splitext(f)[0] for f in listdir(fermenter_recipes_folder) if isfile(join(fermenter_recipes_folder, f)) and f.endswith(".yaml")] return fermenter_recipe_ids def check_for_setup(self): # is there a restored_config.zip file? if yes restore it first then delte the zip. - backupfile = os.path.join(self._rawPath, "restored_config.zip") + backupfile = os.path.join(self.configFolderPath, "restored_config.zip") if os.path.exists(os.path.join(backupfile)) is True: print("***************************************************") print("Found backup of config. Starting restore") @@ -52,7 +55,7 @@ class ConfigFolder: if zip_content == True: print("Found correct content. Starting Restore process") - output_path = pathlib.Path(self._rawPath) + output_path = pathlib.Path(self.configFolderPath) system = platform.system() print(system) if system != "Windows": @@ -73,7 +76,7 @@ class ConfigFolder: print("Wrong Content in zip file. No restore possible") print("renaming zip file so it will be ignored on the next start") try: - os.rename(backupfile, os.path.join(self._rawPath, "UNRESTORABLE_restored_config.zip")) + os.rename(backupfile, os.path.join(self.configFolderPath, "UNRESTORABLE_restored_config.zip")) except: print("renamed file does exist - deleting instead") os.remove(backupfile) @@ -98,11 +101,11 @@ class ConfigFolder: ['upload', 'folder'] ] for checking in required_config_content: - if self.inform_missing_content(self.check_for_file_or_folder(os.path.join(self._rawPath, checking[0]), checking[1])): + if self.inform_missing_content(self.check_for_file_or_folder(os.path.join(self.configFolderPath, checking[0]), checking[1])): # since there is no complete config we now check if the config folde rmay be completely empty to show hints: - if len(os.listdir(os.path.join(self._rawPath))) == 0 : + if len(os.listdir(os.path.join(self.configFolderPath))) == 0 : print("***************************************************") - print(f"the config folder '{self._rawPath}' seems to be completely empty") + print(f"the config folder '{self.configFolderPath}' seems to be completely empty") print("you might want to run 'cbpi setup'.print") print("but you could also place your zipped config backup named") print("'restored_config.zip' inside the mentioned config folder for") @@ -142,7 +145,7 @@ class ConfigFolder: def copyDefaultFileIfNotExists(self, file): if self.config_file_exists(file) is False: srcfile = os.path.join(os.path.dirname(__file__), "config", file) - destfile = os.path.join(self._rawPath, file) + destfile = os.path.join(self.configFolderPath, file) shutil.copy(srcfile, destfile) def create_config_file(self): @@ -156,9 +159,9 @@ class ConfigFolder: self.copyDefaultFileIfNotExists("craftbeerpi.service") self.copyDefaultFileIfNotExists("chromium.desktop") - if os.path.exists(os.path.join(self._rawPath, "dashboard", "cbpi_dashboard_1.json")) is False: + if os.path.exists(os.path.join(self.configFolderPath, "dashboard", "cbpi_dashboard_1.json")) is False: srcfile = os.path.join(os.path.dirname(__file__), "config", "dashboard", "cbpi_dashboard_1.json") - destfile = os.path.join(self._rawPath, "dashboard") + destfile = os.path.join(self.configFolderPath, "dashboard") shutil.copy(srcfile, destfile) print("Config Folder created") @@ -168,12 +171,12 @@ class ConfigFolder: print("Folder created") def create_folders(self): - pathlib.Path(self._rawPath).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, '..','logs')).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, 'recipes')).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) - pathlib.Path(os.path.join(self._rawPath, 'upload')).mkdir(parents=True, exist_ok=True) + pathlib.Path(self.configFolderPath).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self.configFolderPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) + #pathlib.Path(os.path.join(self.configFolderPath, '..','logs')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self.configFolderPath, 'recipes')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self.configFolderPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) + pathlib.Path(os.path.join(self.configFolderPath, 'upload')).mkdir(parents=True, exist_ok=True) def recursive_chown(self, path, owner, group): try: diff --git a/cbpi/controller/config_controller.py b/cbpi/controller/config_controller.py index 2412688..6d62d79 100644 --- a/cbpi/controller/config_controller.py +++ b/cbpi/controller/config_controller.py @@ -17,7 +17,7 @@ class ConfigController: self.cbpi.register(self) self.path = cbpi.config_folder.get_file_path("config.json") self.path_static = cbpi.config_folder.get_file_path("config.yaml") - self.logger.info("Config folder path : " + os.path.join(Path(self.cbpi.config_folder._rawPath).absolute())) + self.logger.info("Config folder path : " + os.path.join(Path(self.cbpi.config_folder.configFolderPath).absolute())) def get_state(self): diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 05024bb..686d5b2 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -26,7 +26,7 @@ class LogController: self.logger = logging.getLogger(__name__) self.configuration = False self.datalogger = {} - self.logsFolderPath = os.path.join(Path(self.cbpi.config_folder._rawPath).parent.absolute(), 'logs') + self.logsFolderPath = self.cbpi.config_folder.logsFolderPath self.logger.info("Log folder path : " + self.logsFolderPath) def log_data(self, name: str, value: str) -> None: diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 24128fe..14d3bf4 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -116,7 +116,7 @@ class SystemController: try: content = backup_file.read() if backup_file and self.allowed_file(filename, 'zip'): - self.path = os.path.join(self.cbpi.config_folder._rawPath, "restored_config.zip") + self.path = os.path.join(self.cbpi.config_folder.configFolderPath, "restored_config.zip") f=open(self.path, "wb") f.write(content) From f924e1a683f2e416539270761ce4384c1a8a2e8c Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 14 Sep 2022 13:54:58 +0200 Subject: [PATCH 06/13] adds journalctl arg '--output cat' (log exports), because cbpi already writes timestamps into the logs and we dont need two timestamps when exporting. This commit also reverts to using the same formatter for all logHandlers. --- cbpi/cli.py | 10 ---------- cbpi/controller/system_controller.py | 4 ++-- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cbpi/cli.py b/cbpi/cli.py index 83a2582..1496764 100644 --- a/cbpi/cli.py +++ b/cbpi/cli.py @@ -17,7 +17,6 @@ from importlib_metadata import metadata from tabulate import tabulate from PyInquirer import prompt, print_json import platform -import subprocess class CraftBeerPiCli(): def __init__(self, config) -> None: @@ -239,15 +238,6 @@ def main(context, config_folder_path, logs_folder_path, debug_log_level): formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') logging.basicConfig(format=formatter,level=debug_log_level, stream=logging.StreamHandler()) logger = logging.getLogger() - try: - result = subprocess.run(['journalctl', '--since', '2 hours ago', '-u', 'craftbeerpi.service' ], stdout=subprocess.PIPE) - # journalctl is present, we assume we are running in production. - # We therefore omit the timestamp from the stdout log handler formatter because timestamps are added to the logs by journalctl anyway - logger.handlers[0].setFormatter('%(levelname)s - %(name)s - %(message)s') - except: - # journalctl command seems to not be present. - # We assume we are in the dev container and keep writing timestampts to stdout for vscode terminal output - logger.warning("journalctl command error - assuming dev container execution and writing timestamps to stdout") try: if not os.path.isdir(logs_folder_path): logger.info(f"logs folder '{logs_folder_path}' doesnt exist and we are trying to create it") diff --git a/cbpi/controller/system_controller.py b/cbpi/controller/system_controller.py index 14d3bf4..b683679 100644 --- a/cbpi/controller/system_controller.py +++ b/cbpi/controller/system_controller.py @@ -56,9 +56,9 @@ class SystemController: output_filename="cbpi4_log.zip" if logtime == "b": - os.system('journalctl -b -u craftbeerpi.service > {}'.format(fullname)) + os.system('journalctl -b -u craftbeerpi.service --output cat > {}'.format(fullname)) else: - os.system('journalctl --since \"{} hours ago\" -u craftbeerpi.service > {}'.format(logtime, fullname)) + os.system('journalctl --since \"{} hours ago\" -u craftbeerpi.service --output cat > {}'.format(logtime, fullname)) os.system('cbpi plugins > {}'.format(fullpluginname)) From 706bbac9f0a84e06102d31d9c176e24e0c22526d Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 14 Sep 2022 19:04:58 +0200 Subject: [PATCH 07/13] keeps default dev config unchanged. this commit moves the default config into a seperate folder. And it adds a preLaunchTask which copies the default dev config to the used dev config. This is done with the cp option '-ru' so if the files in the target dir (current config) are newer they are not overwritten. --- .../actor.json | 0 .../chromium.desktop | 0 .../config.json | 0 .../config.yaml | 0 .../craftbeerpi.service | 0 .../dashboard/cbpi_dashboard_1.json | 0 .../dashboard/widgets/.gitkeep | 0 .../fermenter_data.json | 0 .../fermenterrecipes/.gitkeep | 0 .../kettle.json | 0 .../recipes/.gitkeep | 0 .../sensor.json | 0 .../step_data.json | 0 .../upload/.gitkeep | 0 .vscode/launch.json | 3 ++- .vscode/tasks.json | 20 +++++++++++++++++++ cbpi/configFolder.py | 3 ++- 17 files changed, 24 insertions(+), 2 deletions(-) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/actor.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/chromium.desktop (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/config.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/config.yaml (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/craftbeerpi.service (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/dashboard/cbpi_dashboard_1.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/dashboard/widgets/.gitkeep (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/fermenter_data.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/fermenterrecipes/.gitkeep (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/kettle.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/recipes/.gitkeep (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/sensor.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/step_data.json (100%) rename .devcontainer/{cbpi-dev-config => cbpi-default-dev-config}/upload/.gitkeep (100%) create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/cbpi-dev-config/actor.json b/.devcontainer/cbpi-default-dev-config/actor.json similarity index 100% rename from .devcontainer/cbpi-dev-config/actor.json rename to .devcontainer/cbpi-default-dev-config/actor.json diff --git a/.devcontainer/cbpi-dev-config/chromium.desktop b/.devcontainer/cbpi-default-dev-config/chromium.desktop similarity index 100% rename from .devcontainer/cbpi-dev-config/chromium.desktop rename to .devcontainer/cbpi-default-dev-config/chromium.desktop diff --git a/.devcontainer/cbpi-dev-config/config.json b/.devcontainer/cbpi-default-dev-config/config.json similarity index 100% rename from .devcontainer/cbpi-dev-config/config.json rename to .devcontainer/cbpi-default-dev-config/config.json diff --git a/.devcontainer/cbpi-dev-config/config.yaml b/.devcontainer/cbpi-default-dev-config/config.yaml similarity index 100% rename from .devcontainer/cbpi-dev-config/config.yaml rename to .devcontainer/cbpi-default-dev-config/config.yaml diff --git a/.devcontainer/cbpi-dev-config/craftbeerpi.service b/.devcontainer/cbpi-default-dev-config/craftbeerpi.service similarity index 100% rename from .devcontainer/cbpi-dev-config/craftbeerpi.service rename to .devcontainer/cbpi-default-dev-config/craftbeerpi.service diff --git a/.devcontainer/cbpi-dev-config/dashboard/cbpi_dashboard_1.json b/.devcontainer/cbpi-default-dev-config/dashboard/cbpi_dashboard_1.json similarity index 100% rename from .devcontainer/cbpi-dev-config/dashboard/cbpi_dashboard_1.json rename to .devcontainer/cbpi-default-dev-config/dashboard/cbpi_dashboard_1.json diff --git a/.devcontainer/cbpi-dev-config/dashboard/widgets/.gitkeep b/.devcontainer/cbpi-default-dev-config/dashboard/widgets/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/dashboard/widgets/.gitkeep rename to .devcontainer/cbpi-default-dev-config/dashboard/widgets/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/fermenter_data.json b/.devcontainer/cbpi-default-dev-config/fermenter_data.json similarity index 100% rename from .devcontainer/cbpi-dev-config/fermenter_data.json rename to .devcontainer/cbpi-default-dev-config/fermenter_data.json diff --git a/.devcontainer/cbpi-dev-config/fermenterrecipes/.gitkeep b/.devcontainer/cbpi-default-dev-config/fermenterrecipes/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/fermenterrecipes/.gitkeep rename to .devcontainer/cbpi-default-dev-config/fermenterrecipes/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/kettle.json b/.devcontainer/cbpi-default-dev-config/kettle.json similarity index 100% rename from .devcontainer/cbpi-dev-config/kettle.json rename to .devcontainer/cbpi-default-dev-config/kettle.json diff --git a/.devcontainer/cbpi-dev-config/recipes/.gitkeep b/.devcontainer/cbpi-default-dev-config/recipes/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/recipes/.gitkeep rename to .devcontainer/cbpi-default-dev-config/recipes/.gitkeep diff --git a/.devcontainer/cbpi-dev-config/sensor.json b/.devcontainer/cbpi-default-dev-config/sensor.json similarity index 100% rename from .devcontainer/cbpi-dev-config/sensor.json rename to .devcontainer/cbpi-default-dev-config/sensor.json diff --git a/.devcontainer/cbpi-dev-config/step_data.json b/.devcontainer/cbpi-default-dev-config/step_data.json similarity index 100% rename from .devcontainer/cbpi-dev-config/step_data.json rename to .devcontainer/cbpi-default-dev-config/step_data.json diff --git a/.devcontainer/cbpi-dev-config/upload/.gitkeep b/.devcontainer/cbpi-default-dev-config/upload/.gitkeep similarity index 100% rename from .devcontainer/cbpi-dev-config/upload/.gitkeep rename to .devcontainer/cbpi-default-dev-config/upload/.gitkeep diff --git a/.vscode/launch.json b/.vscode/launch.json index 73c363b..a3c07c1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,8 @@ "type": "python", "request": "launch", "module": "run", - "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "start"] + "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "start"], + "preLaunchTask": "copy default cbpi config files if dev config files dont exist" }, { diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..05db7ee --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,20 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "copy default cbpi config files if dev config files dont exist", + "type": "shell", + "command": "cp -ru ${workspaceFolder}/.devcontainer/cbpi-default-dev-config/. ${workspaceFolder}/.devcontainer/cbpi-dev-config", + "windows": { + "command": "echo 'this pre debug task should only be run inside the docker dev container - doing nothing instead'" + }, + "group": "build", + "presentation": { + "reveal": "silent", + "panel": "shared" + } + } + ] +} \ No newline at end of file diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index d3690cb..fc21aa5 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -14,7 +14,8 @@ class ConfigFolder: def __init__(self, configFolderPath, logsFolderPath): self.configFolderPath = configFolderPath self.logsFolderPath = logsFolderPath - self.logger.info("Config folder path : " + os.path.join(Path(self.cbpi.config_folder.configFolderPath).absolute())) + print("config folder path : " + configFolderPath) + print("logs folder path : " + logsFolderPath) def config_file_exists(self, path): return os.path.exists(self.get_file_path(path)) From ac9b599619c348ebae5c4fa96f4d3df5d3d5f70e Mon Sep 17 00:00:00 2001 From: prash3r Date: Wed, 14 Sep 2022 19:36:51 +0200 Subject: [PATCH 08/13] check_for_setup() now checks for provided logs folder path --- cbpi/configFolder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index fc21aa5..88ee8b8 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -97,7 +97,6 @@ class ConfigFolder: ['dashboard/widgets', 'folder'], ['dashboard', 'folder'], ['fermenterrecipes', 'folder'], - ['../logs', 'folder'], ['recipes', 'folder'], ['upload', 'folder'] ] @@ -174,7 +173,6 @@ class ConfigFolder: def create_folders(self): pathlib.Path(self.configFolderPath).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) - #pathlib.Path(os.path.join(self.configFolderPath, '..','logs')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'recipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'upload')).mkdir(parents=True, exist_ok=True) From 1fa3f8899725223250e36bd4fe92fa0661aa5491 Mon Sep 17 00:00:00 2001 From: prash3r Date: Fri, 16 Sep 2022 10:25:31 +0200 Subject: [PATCH 09/13] adds the 'cbpi create' command to vscode. also initilizes plugin names with dashes instead of underscores. everything now runs as root inside the dev container (otherwise the permissions for cbpi create wouldnt be sufficient). --- .devcontainer/devcontainer.json | 2 +- .gitignore | 4 +++- .vscode/launch.json | 8 ++++++++ cbpi/cli.py | 2 +- cbpi/configFolder.py | 2 ++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 08f216c..ae59030 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,5 +47,5 @@ //"postCreateCommand": "pip3 install -r ./requirements.txt", // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + // "remoteUser": "vscode" } diff --git a/.gitignore b/.gitignore index 20ad518..32b01ba 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,6 @@ node_modules config/* logs/ .coverage -.devcontainer/cbpi-dev-config/* \ No newline at end of file +.devcontainer/cbpi-dev-config/* +cbpi4-* +temp* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index a3c07c1..1dd8992 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,6 +14,14 @@ "preLaunchTask": "copy default cbpi config files if dev config files dont exist" }, + { + "name": "create CraftBeerPi4 plugin", + "type": "python", + "request": "launch", + "module": "run", + "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "create"] + }, + { "name": "setup CraftBeerPi4: create config folder structure", "type": "python", diff --git a/cbpi/cli.py b/cbpi/cli.py index 1496764..fcec9de 100644 --- a/cbpi/cli.py +++ b/cbpi/cli.py @@ -87,7 +87,7 @@ class CraftBeerPiCli(): answers = prompt(questions) - name = "cbpi4_" + answers["name"] + name = "cbpi4-" + str(answers["name"]).replace('_', '-').replace(' ', '-') if os.path.exists(os.path.join(".", name)) is True: print("Cant create Plugin. Folder {} already exists ".format(name)) return diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index 88ee8b8..954f1a3 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -97,6 +97,7 @@ class ConfigFolder: ['dashboard/widgets', 'folder'], ['dashboard', 'folder'], ['fermenterrecipes', 'folder'], + [self.logsFolderPath, 'folder'], ['recipes', 'folder'], ['upload', 'folder'] ] @@ -172,6 +173,7 @@ class ConfigFolder: def create_folders(self): pathlib.Path(self.configFolderPath).mkdir(parents=True, exist_ok=True) + pathlib.Path(self.logsFolderPath).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'dashboard', 'widgets')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'recipes')).mkdir(parents=True, exist_ok=True) pathlib.Path(os.path.join(self.configFolderPath, 'fermenterrecipes')).mkdir(parents=True, exist_ok=True) From fea1ba04f0c2f6d4b7db878a0d58b36b56ce904d Mon Sep 17 00:00:00 2001 From: prash3r Date: Fri, 16 Sep 2022 18:56:52 +0200 Subject: [PATCH 10/13] corrected some more places where ./logs was used instead of the dynamically given logs path. --- cbpi/controller/log_file_controller.py | 7 +++---- cbpi/http_endpoints/http_log.py | 3 ++- cbpi/http_endpoints/http_system.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 686d5b2..05e9d32 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -160,7 +160,7 @@ class LogController: for id in ids: # df = pd.read_csv("./logs/sensor_%s.log" % id, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime',"Values"], header=None) # concat all logs - all_filenames = glob.glob(os.path.join(self.logsFolderPath,f"./logs/sensor_{id}.log*")) + all_filenames = glob.glob(os.path.join(self.logsFolderPath,f"sensor_{id}.log*")) df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', 'Values'], header=None) for f in all_filenames]) df = df.resample('60s').max() df = df.dropna() @@ -179,8 +179,7 @@ class LogController: return [os.path.basename(x) for x in os.path.join(self.logsFolderPath, f"sensor_{name}.log*")] def clear_log(self, name:str ) -> str: - - all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") + all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) for f in all_filenames: os.remove(f) @@ -216,7 +215,7 @@ class LogController: """ formatted_time = strftime("%Y-%m-%d-%H_%M_%S", localtime()) - file_name = os.path.join(self.logsFolderPath, f"./logs/{formatted_time}-sensor-{name}.zip" % (formatted_time, name)) + file_name = os.path.join(self.logsFolderPath, f"{formatted_time}-sensor-{name}.zip" % (formatted_time, name)) zip = zipfile.ZipFile(file_name, 'w', zipfile.ZIP_DEFLATED) all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") for f in all_filenames: diff --git a/cbpi/http_endpoints/http_log.py b/cbpi/http_endpoints/http_log.py index e898460..f452026 100644 --- a/cbpi/http_endpoints/http_log.py +++ b/cbpi/http_endpoints/http_log.py @@ -2,6 +2,7 @@ from cbpi.utils.encoder import ComplexEncoder from aiohttp import web from cbpi.utils.utils import json_dumps from cbpi.api import request_mapping +import os import json class LogHttpEndpoints: @@ -83,7 +84,7 @@ class LogHttpEndpoints: ) await response.prepare(request) log_name = request.match_info['name'] - with open('./logs/%s.zip' % log_name, 'rb') as file: + with open(os.path.join(self.cbpi.logsFolderPath, '%s.zip' % log_name), 'rb') as file: for line in file.readlines(): await response.write(line) diff --git a/cbpi/http_endpoints/http_system.py b/cbpi/http_endpoints/http_system.py index c21c4cf..823ef6f 100644 --- a/cbpi/http_endpoints/http_system.py +++ b/cbpi/http_endpoints/http_system.py @@ -44,7 +44,7 @@ class SystemHttpEndpoints: async def http_get_log(self, request): result = [] file_pattern = re.compile("^(\w+.).log(.?\d*)") - for filename in sorted(os.listdir("./logs"), reverse=True): # + for filename in sorted(os.listdir(self.cbpi.logsFolderPath), reverse=True): if file_pattern.match(filename): result.append(filename) return web.json_response(result) From e0d809c3a31b0b37de525af12ca0c135f425e4ff Mon Sep 17 00:00:00 2001 From: prash3r Date: Sat, 24 Sep 2022 12:58:26 +0200 Subject: [PATCH 11/13] additional requirements.txt for devcontainer only inside the dev container default config there now is a 'additional-dev-requirements.txt' file that allows us to ship additional python packages requirements with a default development config. --- .devcontainer/Dockerfile | 7 ++++--- .../additional-dev-requirements.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/cbpi-default-dev-config/additional-dev-requirements.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 732ba18..41c4890 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,9 +9,10 @@ RUN apt-get install --no-install-recommends -y \ && rm -rf /var/lib/apt/lists/* RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel -# Install craftbeerpi requirements for better caching -COPY ./requirements.txt /workspace/requirements.txt -RUN pip3 install --no-cache-dir -r /workspace/requirements.txt +# Install craftbeerpi requirements and additional-dev-requirements for better caching +COPY ./requirements.txt ./.devcontainer/cbpi-default-dev-config/additional-dev-requirements.txt /workspace/ +RUN cat /workspace/additional-dev-requirements.txt 2>/dev/null 1>> /workspace/requirements.txt \ + && pip3 install --no-cache-dir -r /workspace/requirements.txt # Install current version of cbpi-ui RUN mkdir /opt/downloads \ diff --git a/.devcontainer/cbpi-default-dev-config/additional-dev-requirements.txt b/.devcontainer/cbpi-default-dev-config/additional-dev-requirements.txt new file mode 100644 index 0000000..a1b4c97 --- /dev/null +++ b/.devcontainer/cbpi-default-dev-config/additional-dev-requirements.txt @@ -0,0 +1 @@ +cbpi4-SimulatedSensor==0.0.2 \ No newline at end of file From e7aa0a64c7dd9a7b597d0634305efeb5b5c225a4 Mon Sep 17 00:00:00 2001 From: prash3r Date: Sat, 24 Sep 2022 15:19:34 +0200 Subject: [PATCH 12/13] repairs log_file_controller.py. I seem to have exidentally search and replaced some needed glob.glob calls when adding dinamic log location, which should now be repaired. There also was a mysterious cbpi_dashboard_1.json appearing in the wrong folder which now inside the dashboard folder. I also figured out how to run tests locally, they should now respect the dynamic folder paths. --- .vscode/launch.json | 8 ++++++++ cbpi/configFolder.py | 3 +++ cbpi/controller/dashboard_controller.py | 10 +++++----- cbpi/controller/log_file_controller.py | 8 ++++---- tests/cbpi_config_fixture.py | 3 ++- tests/test_cli.py | 2 +- tests/test_logger.py | 4 ++-- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1dd8992..e24e1e9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -28,6 +28,14 @@ "request": "launch", "module": "run", "args": ["--config-folder-path=./.devcontainer/cbpi-dev-config", "setup"] + }, + + { + "name": "run tests", + "type": "python", + "request": "launch", + "module": "pytest", + "args": ["tests"] } ] } \ No newline at end of file diff --git a/cbpi/configFolder.py b/cbpi/configFolder.py index 954f1a3..ef2169b 100644 --- a/cbpi/configFolder.py +++ b/cbpi/configFolder.py @@ -22,6 +22,9 @@ class ConfigFolder: def get_file_path(self, file): return os.path.join(self.configFolderPath, file) + + def get_dashboard_path(self, file): + return os.path.join(self.configFolderPath, "dashboard", file) def get_upload_file(self, file): return os.path.join(self.configFolderPath, 'upload', file) diff --git a/cbpi/controller/dashboard_controller.py b/cbpi/controller/dashboard_controller.py index 2f39af5..feef806 100644 --- a/cbpi/controller/dashboard_controller.py +++ b/cbpi/controller/dashboard_controller.py @@ -18,14 +18,14 @@ class DashboardController: self.logger = logging.getLogger(__name__) self.cbpi.register(self) - self.path = cbpi.config_folder.get_file_path("cbpi_dashboard_1.json") + self.path = cbpi.config_folder.get_dashboard_path("cbpi_dashboard_1.json") async def init(self): pass async def get_content(self, dashboard_id): try: - self.path = self.cbpi.config_folder.get_file_path("cbpi_dashboard_"+ str(dashboard_id) +".json") + self.path = self.cbpi.config_folder.get_dashboard_path("cbpi_dashboard_"+ str(dashboard_id) +".json") logging.info(self.path) with open(self.path) as json_file: data = json.load(json_file) @@ -35,21 +35,21 @@ class DashboardController: async def add_content(self, dashboard_id, data): print(data) - self.path = self.cbpi.config_folder.get_file_path("cbpi_dashboard_" + str(dashboard_id)+ ".json") + self.path = self.cbpi.config_folder.get_dashboard_path("cbpi_dashboard_" + str(dashboard_id)+ ".json") with open(self.path, 'w') as outfile: json.dump(data, outfile, indent=4, sort_keys=True) self.cbpi.notify(title="Dashboard {}".format(dashboard_id), message="Saved Successfully", type=NotificationType.SUCCESS) return {"status": "OK"} async def delete_content(self, dashboard_id): - self.path = self.cbpi.config_folder.get_file_path("cbpi_dashboard_"+ str(dashboard_id)+ ".json") + self.path = self.cbpi.config_folder.get_dashboard_path("cbpi_dashboard_"+ str(dashboard_id)+ ".json") if os.path.exists(self.path): os.remove(self.path) self.cbpi.notify(title="Dashboard {}".format(dashboard_id), message="Deleted Successfully", type=NotificationType.SUCCESS) async def get_custom_widgets(self): - path = os.path.join(self.cbpi.config_folder.get_file_path("dashboard"), "widgets") + path = self.cbpi.config_folder.get_dashboard_path("widgets") onlyfiles = [os.path.splitext(f)[0] for f in sorted(listdir(path)) if isfile(join(path, f)) and f.endswith(".svg")] return onlyfiles diff --git a/cbpi/controller/log_file_controller.py b/cbpi/controller/log_file_controller.py index 05e9d32..50790e2 100644 --- a/cbpi/controller/log_file_controller.py +++ b/cbpi/controller/log_file_controller.py @@ -118,7 +118,7 @@ class LogController: for name in names: # get all log names - all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") + all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) # concat all logs df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', name], header=None) for f in all_filenames]) logging.info("Read all files for {}".format(names)) @@ -176,7 +176,7 @@ class LogController: :return: list of log file names ''' - return [os.path.basename(x) for x in os.path.join(self.logsFolderPath, f"sensor_{name}.log*")] + return [os.path.basename(x) for x in glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*"))] def clear_log(self, name:str ) -> str: all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) @@ -215,9 +215,9 @@ class LogController: """ formatted_time = strftime("%Y-%m-%d-%H_%M_%S", localtime()) - file_name = os.path.join(self.logsFolderPath, f"{formatted_time}-sensor-{name}.zip" % (formatted_time, name)) + file_name = os.path.join(self.logsFolderPath, f"{formatted_time}-sensor-{name}.zip") zip = zipfile.ZipFile(file_name, 'w', zipfile.ZIP_DEFLATED) - all_filenames = os.path.join(self.logsFolderPath, f"sensor_{name}.log*") + all_filenames = glob.glob(os.path.join(self.logsFolderPath, f"sensor_{name}.log*")) for f in all_filenames: zip.write(os.path.join(f)) zip.close() diff --git a/tests/cbpi_config_fixture.py b/tests/cbpi_config_fixture.py index b9824ba..2510c21 100644 --- a/tests/cbpi_config_fixture.py +++ b/tests/cbpi_config_fixture.py @@ -19,5 +19,6 @@ class CraftBeerPiTestCase(AioHTTPTestCase): def configuration(self): test_directory = os.path.dirname(__file__) test_config_directory = os.path.join(test_directory, 'cbpi-test-config') - configFolder = ConfigFolder(test_config_directory) + test_logs_directory = os.path.join(test_directory, 'logs') + configFolder = ConfigFolder(test_config_directory, test_logs_directory) return configFolder diff --git a/tests/test_cli.py b/tests/test_cli.py index 0a9f705..5de4f03 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %( class CLITest(unittest.TestCase): def test_list(self): - cli = CraftBeerPiCli(ConfigFolder("./cbpi-test-config")) + cli = CraftBeerPiCli(ConfigFolder("./cbpi-test-config", './logs')) # inside tests folder cli.plugins_list() if __name__ == '__main__': diff --git a/tests/test_logger.py b/tests/test_logger.py index d626ab0..2f84642 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -10,11 +10,11 @@ class LoggerTestCase(CraftBeerPiTestCase): @unittest_run_loop async def test_log_data(self): - os.makedirs("./logs", exist_ok=True) + os.makedirs(os.path.join(".", "tests", "logs"), exist_ok=True) log_name = "test" #clear all logs self.cbpi.log.clear_log(log_name) - assert len(glob.glob('./logs/sensor_%s.log*' % log_name)) == 0 + assert len(glob.glob(os.path.join(".", "tests", "logs", f"sensor_{log_name}.log*"))) == 0 # write log entries for i in range(5): From 50fa87d6dff8d7a17d9d48740e3ee4ee4af59c3c Mon Sep 17 00:00:00 2001 From: prash3r Date: Sat, 24 Sep 2022 21:55:36 +0200 Subject: [PATCH 13/13] repairs plugin creation inside the devcontainer strange windows volume behaviour needs a small delay between template folder unzip and rename. reverts user changes of the dev container which werent the cause of this problem. --- .devcontainer/devcontainer.json | 2 +- cbpi/cli.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ae59030..08f216c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,5 +47,5 @@ //"postCreateCommand": "pip3 install -r ./requirements.txt", // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" + "remoteUser": "vscode" } diff --git a/cbpi/cli.py b/cbpi/cli.py index fcec9de..bc3b13d 100644 --- a/cbpi/cli.py +++ b/cbpi/cli.py @@ -17,6 +17,7 @@ from importlib_metadata import metadata from tabulate import tabulate from PyInquirer import prompt, print_json import platform +import time class CraftBeerPiCli(): def __init__(self, config) -> None: @@ -99,8 +100,10 @@ class CraftBeerPiCli(): with ZipFile('temp.zip', 'r') as repo_zip: repo_zip.extractall() + + time.sleep(1) # windows dev container permissions problem otherwise - os.rename("./craftbeerpi4-plugin-template-main", os.path.join(".", name)) + os.rename(os.path.join(".","craftbeerpi4-plugin-template-main"), os.path.join(".", name)) os.rename(os.path.join(".", name, "src"), os.path.join(".", name, name)) import jinja2