log http endpoint added

cbpi/controller/job_controller.py

@@ -38,6 +38,8 @@ class JobController(object):
 
         self.cbpi.app.on_startup.append(spawn_job)
 
+
+
     async def start_job(self, method, name, type):
         scheduler = get_scheduler_from_app(self.cbpi.app)
         return await scheduler.spawn(method, name, type)

cbpi/controller/log_file_controller.py

@@ -5,6 +5,8 @@ import os
 from logging.handlers import RotatingFileHandler
 from time import strftime, localtime
 import pandas as pd
+import zipfile
+
 
 class LogController:
 
@@ -50,6 +52,7 @@ class LogController:
         # remove duplicates
         names = set(names)
 
+        print(names)
         result = None
 
         def dateparse(time_in_secs):
@@ -89,6 +92,8 @@ class LogController:
                 data[name] = result[name].interpolate(limit_direction='both', limit=10).tolist()
         else:
             data[name] = result.interpolate().tolist()
+
+        print(data)
         return data
 
 
@@ -98,9 +103,10 @@ class LogController:
         :param name: log name as string. pattern /logs/sensor_%s.log*
         :return: list of log file names
         '''
-        return = glob.glob('./logs/sensor_%s.log*' % name)
 
-    async def clear_log(self, name:str ) -> str:
+        return [os.path.basename(x) for x in glob.glob('./logs/sensor_%s.log*' % name)]
+
+    def clear_log(self, name:str ) -> str:
         '''
 
         :param name: log name as string. pattern /logs/sensor_%s.log*
@@ -108,12 +114,48 @@ class LogController:
         '''
         all_filenames = glob.glob('./logs/sensor_%s.log*' % name)
         for f in all_filenames:
-            print(f)
+
             os.remove(f)
 
         if name in self.datalogger:
             del self.datalogger[name]
 
 
+    def get_all_zip_file_names(self, name: str) -> list:
+
+        '''
+        Return a list of all zip file names
+        :param name: 
+        :return: 
+        '''
+
+        return [os.path.basename(x) for x in glob.glob('./logs/*-sensor-%s.zip' % name)]
+
+    def clear_zip(self, name:str ) -> None:
+        """
+        clear all zip files for a sensor
+        :param name: sensor name
+        :return: None
+        """
+
+        all_filenames = glob.glob('./logs/*-sensor-%s.zip' % name)
+        for f in all_filenames:
+            os.remove(f)
+
+    def zip_log_data(self, name: str) -> str:
+        """
+        :param name: sensor name
+        :return: zip_file_name
+        """
+
+        formatted_time = strftime("%Y-%m-%d-%H_%M_%S", localtime())
+        file_name = './logs/%s-sensor-%s.zip' % (formatted_time, name)
+        zip = zipfile.ZipFile(file_name, 'w', zipfile.ZIP_DEFLATED)
+        all_filenames = glob.glob('./logs/sensor_%s.log*' % name)
+        for f in all_filenames:
+            zip.write(os.path.join(f))
+        zip.close()
+
+        return file_name
 
 

cbpi/craftbeerpi.py

@@ -35,6 +35,7 @@ from cbpi.controller.translation_controller import TranslationController
 from cbpi.http_endpoints.http_translation import TranslationHttpEndpoint
 from cbpi.http_endpoints.http_plugin import PluginHttpEndpoints
 from cbpi.http_endpoints.http_system import SystemHttpEndpoints
+from cbpi.http_endpoints.http_log import LogHttpEndpoints
 
 logger = logging.getLogger(__name__)
 
@@ -104,6 +105,7 @@ class CraftBeerPi():
         self.http_plugin = PluginHttpEndpoints(self)
         self.http_system = SystemHttpEndpoints(self)
         self.notification = NotificationController(self)
+        self.http_log = LogHttpEndpoints(self)
         self.login = Login(self)
 
     def _setup_shutdownhook(self):

cbpi/extension/dummysensor/__init__.py

@@ -31,8 +31,6 @@ class CustomSensor(CBPiSensor):
     def get_state(self):
         return self.state
 
-
-
     def get_value(self):
 
         return self.value

cbpi/extension/mqtt/__init__.py

@@ -21,7 +21,6 @@ class CBPiMqttClient:
 
     async def listen(self, topic, **kwargs):
         if self.client is not None:
-            print(topic, kwargs)
             await self.client.publish(topic, str.encode(json.dumps(kwargs, cls=ComplexEncoder)), QOS_0)
 
 def setup(cbpi):
@@ -32,5 +31,5 @@ def setup(cbpi):
     :param cbpi: the cbpi core
     :return:
     '''
-    print("MQTT REGISTER-------------")
+    client = CBPiMqttClient(cbpi)
-    c = CBPiMqttClient(cbpi)
+

cbpi/extension/mqtt/config.yaml

@@ -1,3 +1,3 @@
 name: MQTT
 version: 4.0
-active: false
+active: False

cbpi/http_endpoints/http_log.py

@@ -0,0 +1,63 @@
+from aiohttp import web
+from cbpi.utils.utils import json_dumps
+from cbpi.api import request_mapping
+
+class LogHttpEndpoints:
+
+    def __init__(self,cbpi):
+        self.cbpi = cbpi
+        self.cbpi.register(self, url_prefix="/log")
+
+    @request_mapping(path="/{name}/zip", method="POST", auth_required=False)
+    async def create_zip_names(self, request):
+        log_name = request.match_info['name']
+        data = self.cbpi.log.zip_log_data(log_name)
+        print(data)
+        return web.json_response(dict(filename=data), dumps=json_dumps)
+
+    @request_mapping(path="/{name}/zip", method="DELETE", auth_required=False)
+    async def clear_zip_names(self, request):
+        log_name = request.match_info['name']
+        self.cbpi.log.clear_zip(log_name)
+        return web.Response(status=204)
+
+    @request_mapping(path="/zip/download/{name}", method="GET", auth_required=False)
+    async def download_zip(self, request):
+        response = web.StreamResponse(
+            status=200,
+            reason='OK',
+            headers={'Content-Type': 'application/zip'},
+        )
+        await response.prepare(request)
+        log_name = request.match_info['name']
+        with open('./logs/%s.zip' % log_name, 'rb') as file:
+            for line in file.readlines():
+                await response.write(line)
+
+        await response.write_eof()
+        return response
+
+    @request_mapping(path="/{name}/zip", method="GET", auth_required=False)
+    async def get_zip_names(self, request):
+        log_name = request.match_info['name']
+        data = self.cbpi.log.get_all_zip_file_names(log_name)
+        return web.json_response(data, dumps=json_dumps)
+
+    @request_mapping(path="/{name}/files", method="GET", auth_required=False)
+    async def get_file_names(self, request):
+        log_name = request.match_info['name']
+        print(log_name)
+        data = self.cbpi.log.get_logfile_names(log_name)
+        return web.json_response(data, dumps=json_dumps)
+
+    @request_mapping(path="/{name}", method="GET", auth_required=False)
+    async def delete_log(self, request):
+        log_name = request.match_info['name']
+        data = await self.cbpi.log.get_data(log_name)
+        return web.json_response(data, dumps=json_dumps)
+
+    @request_mapping(path="/{name}", method="DELETE", auth_required=False)
+    async def delete_all_logs(self, request):
+        log_name = request.match_info['name']
+        await self.cbpi.log.clear_logs(log_name)
+        return web.Response(status=204)

cbpi/utils/encoder.py

@@ -1,3 +1,4 @@
+import datetime
 from json import JSONEncoder
 
 
@@ -8,6 +9,8 @@ class ComplexEncoder(JSONEncoder):
 
             if hasattr(obj, "to_json") and callable(getattr(obj, "to_json")):
                 return obj.to_json()
+            elif isinstance(obj, datetime.datetime):
+                return obj.__str__()
             else:
                 raise TypeError()
         except Exception as e:

cbpi/websocket.py

@@ -1,11 +1,9 @@
 import logging
-import re
 import weakref
 from collections import defaultdict
 
 import aiohttp
 from aiohttp import web
-from cbpi.api import *
 from voluptuous import Schema
 
 from cbpi.utils import json_dumps

requirements.txt

@@ -1,3 +1,4 @@
+pandas
 aiohttp==3.4.4
 aiohttp-auth==0.1.1
 aiohttp-route-decorator==0.1.4

tests/test_logger.py

@@ -18,7 +18,7 @@ class UtilsTestCase(AioHTTPTestCase):
 
         log_name = "test"
         #clear all logs
-        await self.cbpi.log.clear_log(log_name)
+        self.cbpi.log.clear_log(log_name)
         assert len(glob.glob('./logs/sensor_%s.log*' % log_name)) == 0
 
         # write log entries
@@ -29,11 +29,13 @@ class UtilsTestCase(AioHTTPTestCase):
 
         # read log data
         data = await self.cbpi.log.get_data(log_name, sample_rate='1s')
-
-
         assert len(data["time"]) == 5
 
-        await self.cbpi.log.clear_log(log_name)
+        assert self.cbpi.log.zip_log_data(log_name) is not None
+
+        self.cbpi.log.clear_zip(log_name)
+
+        self.cbpi.log.clear_log(log_name)