From 0c380d5912b092fa8e901cf2b3185fd01be6d7b5 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Thu, 2 Dec 2021 22:21:16 +0100 Subject: [PATCH 1/9] Avoid root when installing cbpi --- Dockerfile | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e747ba..40a61ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apk --no-cache add curl && mkdir /downloads # Download installation files RUN curl https://github.com/avollkopf/craftbeerpi4-ui/archive/main.zip -L -o ./downloads/cbpi-ui.zip -FROM python:3.7 +FROM python:3.9 # Install dependencies RUN apt-get update \ @@ -14,28 +14,39 @@ RUN apt-get install --no-install-recommends -y \ python3-pip \ && rm -rf /var/lib/apt/lists/* -RUN python -m pip install --upgrade pip setuptools wheel +ENV VIRTUAL_ENV=/opt/venv -WORKDIR /cbpi # Create non-root user working directory RUN groupadd -g 1000 -r craftbeerpi \ && useradd -u 1000 -r -s /bin/false -g craftbeerpi craftbeerpi \ - && chown craftbeerpi:craftbeerpi /cbpi - -# Install craftbeerpi from source -COPY . /cbpi-src -RUN pip3 install --no-cache-dir /cbpi-src - -# Install craftbeerpi-ui -COPY --from=download /downloads /downloads -RUN pip3 install --no-cache-dir /downloads/cbpi-ui.zip - -# Clean up installation files -RUN rm -rf /downloads /cbpi-src + && mkdir /cbpi \ + && chown craftbeerpi:craftbeerpi /cbpi \ + && mkdir -p $VIRTUAL_ENV \ + && chown -R craftbeerpi:craftbeerpi ${VIRTUAL_ENV} USER craftbeerpi -RUN cbpi setup +# create virtual environment +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel + +# Install craftbeerpi from source +COPY --chown=craftbeerpi . /cbpi-src +RUN pip3 install --no-cache-dir /cbpi-src + +# Install craftbeerpi-ui +COPY --from=download --chown=craftbeerpi /downloads /downloads +RUN pip3 install --no-cache-dir /downloads/cbpi-ui.zip + +# Clean up installation files +USER root +RUN rm -rf /downloads /cbpi-src +USER craftbeerpi + +WORKDIR /cbpi +RUN ["cbpi", "setup"] EXPOSE 8000 From 623711e772073016c9f831af185e8052402f01c8 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Fri, 3 Dec 2021 00:35:45 +0100 Subject: [PATCH 2/9] Increase possible caching on docker file creation --- Dockerfile | 4 ++++ requirements.txt | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 40a61ac..639e820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,10 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel +# Install craftbeerpi requirements for better caching +COPY --chown=craftbeerpi ./requirements.txt /cbpi-src/ +RUN pip3 install --no-cache-dir -r /cbpi-src/requirements.txt + # Install craftbeerpi from source COPY --chown=craftbeerpi . /cbpi-src RUN pip3 install --no-cache-dir /cbpi-src diff --git a/requirements.txt b/requirements.txt index 739db3a..28e5433 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,10 @@ pyfiglet==0.8.post1 pandas==1.1.5 shortuuid==1.0.1 tabulate==0.8.7 +numpy==1.20.3 cbpi4ui -click -asyncio-mqtt \ No newline at end of file +click==7.1.2 +importlib_metadata==4.8.2 +asyncio-mqtt +psutil==5.8.0 +zipp>=0.5 \ No newline at end of file From f337cb8e9755f5085a7be380f7a97a77faf48a07 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Fri, 3 Dec 2021 00:34:08 +0100 Subject: [PATCH 3/9] Add generic and Tasmota specific MQTT actors --- cbpi/extension/mqtt_actor/__init__.py | 61 ++----------------- .../mqtt_actor/generic_mqtt_actor.py | 37 +++++++++++ cbpi/extension/mqtt_actor/mqtt_actor.py | 57 +++++++++++++++++ .../mqtt_actor/tasmota_mqtt_actor.py | 13 ++++ 4 files changed, 112 insertions(+), 56 deletions(-) create mode 100644 cbpi/extension/mqtt_actor/generic_mqtt_actor.py create mode 100644 cbpi/extension/mqtt_actor/mqtt_actor.py create mode 100644 cbpi/extension/mqtt_actor/tasmota_mqtt_actor.py diff --git a/cbpi/extension/mqtt_actor/__init__.py b/cbpi/extension/mqtt_actor/__init__.py index b7e8cf1..bab41c6 100644 --- a/cbpi/extension/mqtt_actor/__init__.py +++ b/cbpi/extension/mqtt_actor/__init__.py @@ -1,61 +1,8 @@ # -*- coding: utf-8 -*- -import asyncio -import json -from cbpi.api import parameters, Property, CBPiActor from cbpi.api import * - -@parameters([Property.Text(label="Topic", configurable=True, description = "MQTT Topic")]) -class MQTTActor(CBPiActor): - - # Custom property which can be configured by the user - @action("Set Power", parameters=[Property.Number(label="Power", configurable=True,description="Power Setting [0-100]")]) - async def setpower(self,Power = 100 ,**kwargs): - self.power=round(Power) - if self.power < 0: - self.power = 0 - if self.power > 100: - self.power = 100 - await self.set_power(self.power) - - def __init__(self, cbpi, id, props): - super(MQTTActor, self).__init__(cbpi, id, props) - - async def on_start(self): - self.topic = self.props.get("Topic", None) - self.power = 100 - - async def on(self, power=None): - if power is not None: - if power != self.power: - power = min(100, power) - power = max(0, power) - self.power = round(power) - await self.cbpi.satellite.publish(self.topic, json.dumps( - {"state": "on", "power": self.power}), True) - self.state = True - pass - - async def off(self): - self.state = False - await self.cbpi.satellite.publish(self.topic, json.dumps( - {"state": "off", "power": self.power}), True) - pass - - async def run(self): - while self.running: - await asyncio.sleep(1) - - def get_state(self): - return self.state - - async def set_power(self, power): - self.power = round(power) - if self.state == True: - await self.on(power) - else: - await self.off() - await self.cbpi.actor.actor_update(self.id,power) - pass +from .mqtt_actor import MQTTActor +from .generic_mqtt_actor import GenericMqttActor +from .tasmota_mqtt_actor import TasmotaMqttActor def setup(cbpi): ''' @@ -67,3 +14,5 @@ def setup(cbpi): ''' if str(cbpi.static_config.get("mqtt", False)).lower() == "true": cbpi.plugin.register("MQTTActor", MQTTActor) + cbpi.plugin.register("MQTT Actor (Generic)", GenericMqttActor) + cbpi.plugin.register("MQTT Actor (Tasmota)", TasmotaMqttActor) \ No newline at end of file diff --git a/cbpi/extension/mqtt_actor/generic_mqtt_actor.py b/cbpi/extension/mqtt_actor/generic_mqtt_actor.py new file mode 100644 index 0000000..c1a5b17 --- /dev/null +++ b/cbpi/extension/mqtt_actor/generic_mqtt_actor.py @@ -0,0 +1,37 @@ +from cbpi.api import parameters, Property +from . import MQTTActor + +@parameters([ + Property.Text(label="Topic", configurable=True, description = "MQTT Topic"), + Property.Text(label="Payload", configurable=True, description = "Payload that is sent as MQTT message. Available placeholders are {switch_onoff}: [on|off], {switch_10}: [1|0], {power}: [0-100].") +]) +class GenericMqttActor(MQTTActor): + def __init__(self, cbpi, id, props): + MQTTActor.__init__(self, cbpi, id, props) + self.payload = "" + + async def on_start(self): + await MQTTActor.on_start(self) + self.payload = self.props.get("Payload", "{{\"state\": \"{switch_onoff}\", \"power\": {power}}}") + + def normalize_power_value(self, power): + if power is not None: + if power != self.power: + power = min(100, power) + power = max(0, power) + self.power = round(power) + + async def publish_mqtt_message(self, topic, payload): + self.logger.info("Publish '{payload}' to '{topic}'".format(payload = payload, topic = self.topic)) + await self.cbpi.satellite.publish(self.topic, payload, True) + + async def on(self, power=None): + self.normalize_power_value(power) + formatted_payload = self.payload.format(switch_onoff = "on", switch_10 = 1, power = self.power) + await self.publish_mqtt_message(self.topic, formatted_payload) + self.state = True + + async def off(self): + formatted_payload = self.payload.format(switch_onoff = "off", switch_10 = 0, power = self.power) + await self.publish_mqtt_message(self.topic, formatted_payload) + self.state = False \ No newline at end of file diff --git a/cbpi/extension/mqtt_actor/mqtt_actor.py b/cbpi/extension/mqtt_actor/mqtt_actor.py new file mode 100644 index 0000000..e9a6f4a --- /dev/null +++ b/cbpi/extension/mqtt_actor/mqtt_actor.py @@ -0,0 +1,57 @@ +import asyncio +import json +from cbpi.api import parameters, Property, CBPiActor +from cbpi.api import * + +@parameters([Property.Text(label="Topic", configurable=True, description = "MQTT Topic")]) +class MQTTActor(CBPiActor): + + # Custom property which can be configured by the user + @action("Set Power", parameters=[Property.Number(label="Power", configurable=True, description="Power Setting [0-100]")]) + async def setpower(self,Power = 100 ,**kwargs): + self.power=round(Power) + if self.power < 0: + self.power = 0 + if self.power > 100: + self.power = 100 + await self.set_power(self.power) + + def __init__(self, cbpi, id, props): + super(MQTTActor, self).__init__(cbpi, id, props) + + async def on_start(self): + self.topic = self.props.get("Topic", None) + self.power = 100 + + async def on(self, power=None): + if power is not None: + if power != self.power: + power = min(100, power) + power = max(0, power) + self.power = round(power) + await self.cbpi.satellite.publish(self.topic, json.dumps( + {"state": "on", "power": self.power}), True) + self.state = True + pass + + async def off(self): + self.state = False + await self.cbpi.satellite.publish(self.topic, json.dumps( + {"state": "off", "power": self.power}), True) + pass + + async def run(self): + while self.running: + await asyncio.sleep(1) + + def get_state(self): + return self.state + + async def set_power(self, power): + self.power = round(power) + if self.state == True: + await self.on(power) + else: + await self.off() + await self.cbpi.actor.actor_update(self.id,power) + pass \ No newline at end of file diff --git a/cbpi/extension/mqtt_actor/tasmota_mqtt_actor.py b/cbpi/extension/mqtt_actor/tasmota_mqtt_actor.py new file mode 100644 index 0000000..5b0e44b --- /dev/null +++ b/cbpi/extension/mqtt_actor/tasmota_mqtt_actor.py @@ -0,0 +1,13 @@ +from cbpi.api import parameters, Property +from . import GenericMqttActor + +@parameters([ + Property.Text(label="Topic", configurable=True, description = "MQTT Topic"), +]) +class TasmotaMqttActor(GenericMqttActor): + def __init__(self, cbpi, id, props): + GenericMqttActor.__init__(self, cbpi, id, props) + + async def on_start(self): + await GenericMqttActor.on_start(self) + self.payload = "{switch_onoff}" \ No newline at end of file From 7413f79d6ed7c79747220911e9ea8483944c3ee2 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Fri, 3 Dec 2021 11:56:28 +0100 Subject: [PATCH 4/9] Extend exception messages for warning logs --- cbpi/controller/satellite_controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cbpi/controller/satellite_controller.py b/cbpi/controller/satellite_controller.py index 4d29dda..e12223c 100644 --- a/cbpi/controller/satellite_controller.py +++ b/cbpi/controller/satellite_controller.py @@ -33,24 +33,24 @@ class SatelliteController: if self.client is not None and self.client._connected: try: await self.client.publish(topic, message, qos=1, retain=retain) - except: - self.logger.warning("Failed to push data via mqtt") + except Exception as e: + self.logger.warning("Failed to push data via mqtt: {}".format(e)) async def _actor_on(self, messages): async for message in messages: try: topic_key = message.topic.split("/") await self.cbpi.actor.on(topic_key[2]) - except: - self.logger.warning("Failed to process actor on via mqtt") + except Exception as e: + self.logger.warning("Failed to process actor on via mqtt: {}".format(e)) async def _actor_off(self, messages): async for message in messages: try: topic_key = message.topic.split("/") await self.cbpi.actor.off(topic_key[2]) - except: - self.logger.warning("Failed to process actor off via mqtt") + except Exception as e: + self.logger.warning("Failed to process actor off via mqtt: {}".format(e)) async def _actor_power(self, messages): async for message in messages: From d80486abd0e4b79b1d1bb9206fe74149f61a9bae Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Sat, 4 Dec 2021 11:45:53 +0100 Subject: [PATCH 5/9] Revert to python v3.7 to increase build time on gh actions --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 639e820..fa4c692 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apk --no-cache add curl && mkdir /downloads # Download installation files RUN curl https://github.com/avollkopf/craftbeerpi4-ui/archive/main.zip -L -o ./downloads/cbpi-ui.zip -FROM python:3.9 +FROM python:3.7 as base # Install dependencies RUN apt-get update \ From 47271620025f997b9a1aafa1c0bccb84532f9946 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Sat, 4 Dec 2021 12:09:46 +0100 Subject: [PATCH 6/9] Increase caching potential of docker image --- .dockerignore | 50 +++++++++++++++++++++++++++++++++++++ .github/workflows/build.yml | 1 + Dockerfile | 1 + 3 files changed, 52 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..615f5ad --- /dev/null +++ b/.dockerignore @@ -0,0 +1,50 @@ +# Docker +docker-compose.yml +.docker + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +**/__pycache__ +**/*.py[cod] + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Virtual environment +.env/ +.venv/ +venv/ +venv3/ + +*.cover +*.log +.git +.mypy_cache +.pytest_cache +.hypothesis +.idea + +**/*.swp \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0de6dcb..abf5443 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,6 +91,7 @@ jobs: context: . file: ./Dockerfile platforms: linux/amd64,linux/arm64 + target: deploy push: ${{ steps.prep.outputs.publish_image }} tags: ${{ steps.prep.outputs.tags }} labels: | diff --git a/Dockerfile b/Dockerfile index fa4c692..bc14714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel COPY --chown=craftbeerpi ./requirements.txt /cbpi-src/ RUN pip3 install --no-cache-dir -r /cbpi-src/requirements.txt +FROM base as deploy # Install craftbeerpi from source COPY --chown=craftbeerpi . /cbpi-src RUN pip3 install --no-cache-dir /cbpi-src From 4faea7ed2072cda675c596412bf71c69b39ccaa6 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Sat, 4 Dec 2021 12:23:56 +0100 Subject: [PATCH 7/9] Determin the proper image for caching for the build --- .github/workflows/build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index abf5443..77c7091 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,11 +54,17 @@ jobs: PUBLISH_IMAGE=false TAGS="${{ env.image-name }}:dev" + # Define the image that will be used as a cached image + # to speed up the build process + BUILD_CACHE_IMAGE_NAME=${TAGS} + if [[ $GITHUB_REF_NAME == master ]] || [[ $GITHUB_REF_NAME == main ]]; then # when building master/main use :latest tag and the version number # from the cbpi/__init__.py file VERSION=$(grep -o -E "(([0-9]{1,2}[.]?){3}[0-9]+)" cbpi/__init__.py) - TAGS="${{ env.image-name }}:latest,${{ env.image-name }}:v${VERSION}" + LATEST_IMAGE=${{ env.image-name }}:latest + BUILD_CACHE_IMAGE_NAME=${LATEST_IMAGE} + TAGS="${LATEST_IMAGE},${{ env.image-name }}:v${VERSION}" PUBLISH_IMAGE=true elif [[ $GITHUB_REF_NAME == development ]]; then PUBLISH_IMAGE=true @@ -67,6 +73,7 @@ jobs: # Set output parameters. echo ::set-output name=tags::${TAGS} echo ::set-output name=publish_image::${PUBLISH_IMAGE} + echo ::set-output name=build_cache_image_name::${BUILD_CACHE_IMAGE_NAME} - name: Set up QEMU uses: docker/setup-qemu-action@master From 48df3cdffa50ff3df50f63f80a9bb3eb0232f269 Mon Sep 17 00:00:00 2001 From: Philipp Grathwohl Date: Sat, 4 Dec 2021 12:26:48 +0100 Subject: [PATCH 8/9] Use cache to increase build performance --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 77c7091..7dbc85d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,6 +101,8 @@ jobs: target: deploy push: ${{ steps.prep.outputs.publish_image }} tags: ${{ steps.prep.outputs.tags }} + cache-from: type=registry,ref=${{ steps.prep.outputs.build_cache_image_name }} + cache-to: type=inline labels: | org.opencontainers.image.title=${{ github.event.repository.name }} org.opencontainers.image.description=${{ github.event.repository.description }} From b2fe624d0826a9704dbc601bab22b555ccb8c73b Mon Sep 17 00:00:00 2001 From: avollkopf <43980694+avollkopf@users.noreply.github.com> Date: Mon, 6 Dec 2021 17:13:49 +0100 Subject: [PATCH 9/9] Version change - bumped Version to 4.0.0.56 - Uncommented on push_ update in basic_controller2.py which seemd to cause error messages during startup and is not required --- cbpi/__init__.py | 2 +- cbpi/controller/basic_controller2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cbpi/__init__.py b/cbpi/__init__.py index db205e1..6fa1e66 100644 --- a/cbpi/__init__.py +++ b/cbpi/__init__.py @@ -1 +1 @@ -__version__ = "4.0.0.55" +__version__ = "4.0.0.56" diff --git a/cbpi/controller/basic_controller2.py b/cbpi/controller/basic_controller2.py index 750bc73..c89b149 100644 --- a/cbpi/controller/basic_controller2.py +++ b/cbpi/controller/basic_controller2.py @@ -102,7 +102,7 @@ class BasicController: logging.info("{} started {}".format(self.name, id)) - await self.push_udpate() +# await self.push_udpate() except Exception as e: logging.error("{} Cant start {} - {}".format(self.name, id, e))