Merge pull request #54 from phylax2020/master

Allow changing logfile size and logfile backup count

cbpi/controller/log_file_controller.py

@@ -31,8 +31,8 @@ class LogController:
         self.influxdb = self.cbpi.config.get("INFLUXDB", "No")
         if self.logfiles == "Yes":
             if name not in self.datalogger:
-                max_bytes = self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 1048576)
+                max_bytes = int(self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", 131072))
-                backup_count = self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3)
+                backup_count = int(self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", 3))
     
                 data_logger = logging.getLogger('cbpi.sensor.%s' % name)
                 data_logger.propagate = False
@@ -42,7 +42,7 @@ class LogController:
                 self.datalogger[name] = data_logger
 
             formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime())
-            self.datalogger[name].info("%s,%s" % (formatted_time, value))
+            self.datalogger[name].info("%s,%s" % (formatted_time, str(value)))
         if self.influxdb == "Yes":
             self.influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", "No")
             self.influxdbaddr = self.cbpi.config.get("INFLUXDBADDR", None)
@@ -116,7 +116,6 @@ class LogController:
         for name in names:
             # get all log names
             all_filenames = glob.glob('./logs/sensor_%s.log*' % name)
-
             # concat all logs
             df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', name], header=None) for f in all_filenames])
             logging.info("Read all files for {}".format(names))
@@ -125,19 +124,29 @@ class LogController:
                 df = df[name].resample(sample_rate).max()
             logging.info("Sampled now for {}".format(names))
             df = df.dropna()
+            # take every nth row so that total number of rows does not exceed max_rows * 2
+            max_rows = 500
+            total_rows = df.shape[0]
+            if (total_rows > 0) and (total_rows > max_rows):
+                nth = int(total_rows/max_rows)
+                if nth > 1:
+                    df = df.iloc[::nth]
+                    
             if result is None:
                 result = df
             else:
                 result = pd.merge(result, df, how='outer', left_index=True, right_index=True)
 
         data = {"time": df.index.tolist()}
-
+        
         if len(names) > 1:
             for name in names:
                 data[name] = result[name].interpolate(limit_direction='both', limit=10).tolist()
         else:
             data[name] = result.interpolate().tolist()
+
         logging.info("Send Log for {}".format(names))
+        
         return data
 
     async def get_data2(self, ids) -> dict:
@@ -146,7 +155,12 @@ class LogController:
         
         result = dict()
         for id in ids:
-            df = pd.read_csv("./logs/sensor_%s.log" % id, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime',"Values"], header=None) 
+            # df = pd.read_csv("./logs/sensor_%s.log" % id, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime',"Values"], header=None) 
+            # concat all logs
+            all_filenames = glob.glob('./logs/sensor_%s.log*' % id)
+            df = pd.concat([pd.read_csv(f, parse_dates=True, date_parser=dateparse, index_col='DateTime', names=['DateTime', 'Values'], header=None) for f in all_filenames])
+            df = df.resample('60s').max()
+            df = df.dropna()
             result[id] = {"time": df.index.astype(str).tolist(), "value":df.Values.tolist()}
         return result
 

cbpi/extension/ConfigUpdate/__init__.py

@@ -47,8 +47,9 @@ class ConfigUpdate(CBPiExtension):
         influxdbcloud = self.cbpi.config.get("INFLUXDBCLOUD", None)
         mqttupdate = self.cbpi.config.get("MQTTUpdate", None)
         PRESSURE_UNIT = self.cbpi.config.get("PRESSURE_UNIT", None)
-
+        SENSOR_LOG_BACKUP_COUNT = self.cbpi.config.get("SENSOR_LOG_BACKUP_COUNT", None)
-
+        SENSOR_LOG_MAX_BYTES = self.cbpi.config.get("SENSOR_LOG_MAX_BYTES", None)
+        
         if boil_temp is None:
             logger.info("INIT Boil Temp Setting")
             try:
@@ -285,6 +286,23 @@ class ConfigUpdate(CBPiExtension):
                                                                                                 {"label": "PSI", "value": "PSI"}])
             except:
                 logger.warning('Unable to update config')
+        
+        # check if SENSOR_LOG_BACKUP_COUNT exists in config
+        if SENSOR_LOG_BACKUP_COUNT is None:
+            logger.info("INIT SENSOR_LOG_BACKUP_COUNT")
+            try:
+                await self.cbpi.config.add("SENSOR_LOG_BACKUP_COUNT", 3, ConfigType.NUMBER, "Max. number of backup logs")
+            except:
+                logger.warning('Unable to update database')
+                
+        # check if SENSOR_LOG_MAX_BYTES exists in config
+        if SENSOR_LOG_MAX_BYTES is None:
+            logger.info("INIT SENSOR_LOG_MAX_BYTES")
+            try:
+                await self.cbpi.config.add("SENSOR_LOG_MAX_BYTES", 100000, ConfigType.NUMBER, "Max. number of bytes in sensor logs")
+            except:
+                logger.warning('Unable to update database')
+                
 
 def setup(cbpi):
     cbpi.plugin.register("ConfigUpdate", ConfigUpdate)