yaml as a compressed global array

esphome/components/store_yaml/__init__.py

@@ -25,13 +25,68 @@ CONFIG_SCHEMA = cv.Schema(
 )
 
 
+class BitWriter:
+    def __init__(self, output, next, bits):
+        self.output = output
+        self.buff = 0
+        self.size = 0
+        self.next = next
+        self.bits = bits
+
+    def put_bits(self, value, bits):
+        self.buff = (self.buff << bits) | (value & ((1 << bits) - 1))
+        self.size += bits
+        while self.size >= 8:
+            b = (self.buff >> (self.size - 8)) & 255
+            self.buff &= (1 << (self.size - 8)) - 1
+            self.output.append(b)
+            self.size -= 8
+
+    def put_code(self, code):
+        if self.next == (1 << self.bits):
+            self.bits += 1
+        self.put_bits(code, self.bits)
+
+    def flush(self):
+        if self.size > 0:
+            b = (self.buff << (8 - self.size)) & 255
+
+
+def compress(s):
+    input = s.encode("utf-8")
+    output = bytearray()
+    codes = {}
+    for i in range(256):
+        codes[chr(i)] = i
+    writer = BitWriter(output, len(codes), 8)
+    prev = None
+    for c in input:
+        pc = prev + chr(c) if prev is not None else chr(c)
+        if pc not in codes:
+            writer.put_code(codes[prev])
+            prev = chr(c)
+            codes[pc] = writer.next
+            writer.next += 1
+        else:
+            prev = pc
+    writer.put_code(codes[prev])
+    writer.flush()
+    return output
+
+
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     store_yaml = await cg.register_component(var, config)
     cg.add(store_yaml.set_show_in_dump_config(config[CONF_SHOW_IN_DUMP_CONFIG]))
     yaml = strip_default_ids(copy.deepcopy(CORE.config))
     dump = yaml_util.dump(yaml, show_secrets=config[CONF_SHOW_SECRETS])
-    cg.add(store_yaml.set_yaml(dump))
+    data = compress(dump)
+    size = len(data)
+    bytes_as_int = ", ".join(str(x) for x in data)
+    uint8_t = f"const uint8_t ESPHOME_YAML[{size}] PROGMEM = {{{bytes_as_int}}}"
+    size_t = f"const size_t ESPHOME_YAML_SIZE = {size}"
+    cg.add_global(cg.RawExpression(uint8_t))
+    cg.add_global(cg.RawExpression(size_t))
 
 
 LogAction = store_yaml_ns.class_("LogAction", automation.Action)

esphome/components/store_yaml/store_yaml.cpp

@@ -1,5 +1,7 @@
 #include "store_yaml.h"
 #include "esphome/core/log.h"
+#include <string>
+#include <map>
 
 namespace esphome {
 namespace store_yaml {
@@ -13,27 +15,90 @@ void StoreYamlComponent::dump_config() {
   }
 }
 
-const char *StoreYamlComponent::get_yaml() const { return this->yaml_; }
+class Decompressor {
+  std::map<uint16_t, std::string> codes_;  // TODO: replace string with char + index to next char
+  size_t pos_{0};
+  uint8_t size_{0};
+  uint32_t buff_{0};
+  size_t code_index_;
+  uint8_t code_width_;
+
+ public:
+  Decompressor() {
+    for (int i = 0; i < 256; i++) {
+      this->codes_[i] = std::string(1, (char) i);
+    }
+    this->code_index_ = this->codes_.size();
+    this->code_width_ = 9;  // log2next + 1
+  }
+
+  uint32_t get_bits(size_t bits) {
+    if (this->is_eof())
+      return UINT32_MAX;
+
+    while (this->size_ < bits) {
+      this->buff_ = (this->buff_ << 8) | ESPHOME_YAML[this->pos_++];
+      this->size_ += 8;
+    }
+
+    uint32_t value = (this->buff_ >> (this->size_ - bits)) & ((1 << bits) - 1);
+    this->size_ -= bits;
+    this->buff_ &= (1 << this->size_) - 1;
+    return value;
+  }
+
+  std::string get_code() {
+    if (this->code_index_ == ((1 << this->code_width_) - 1)) {
+      this->code_width_++;
+    }
+    uint32_t c = get_bits(this->code_width_);
+    auto i = this->codes_.find(c);
+    if (i != this->codes_.end()) {
+      return i->second;
+    }
+    if (c != this->codes_.size()) {
+      this->pos_ = ESPHOME_YAML_SIZE;  // error in input, set eof
+    }
+    return std::string();
+  }
+
+  void add_code(const std::string &c) { this->codes_[this->code_index_++] = c; }
+
+  bool is_eof() const { return this->pos_ >= ESPHOME_YAML_SIZE; }
+};
 
 void StoreYamlComponent::log(bool dump_config) const {
-  const char *s = this->yaml_;
+  Decompressor *dec = new Decompressor();
-  while (*s) {
+
-    const char *e = s;
+  std::string c = dec->get_code();
-    while (*e && *e++ != '\n')
+  std::string prev = c;
-      ;
+  std::string yaml = c;
-    const char *tmp = e;
+
-    if (e > s) {
+  while (!dec->is_eof()) {
-      if (e[-1] == '\n')
+    std::string c = dec->get_code();
-        e--;
+    if (c.empty())
-      std::string row = std::string(s, e - s);
+      c = prev + prev[0];
-      if (dump_config) {
+    dec->add_code(prev + c[0]);
-        ESP_LOGCONFIG(TAG, "%s", row.c_str());
+    prev = c;
-      } else {
+    yaml += c;
-        ESP_LOGI(TAG, "%s", row.c_str());
+    // print line by line because the logger cannot handle too much data
+    // it also uses less memory
+    size_t newline = 0;
+    while (newline != std::string::npos) {
+      newline = yaml.find('\n');
+      if (newline != std::string::npos) {
+        std::string row = yaml.substr(0, newline);
+        yaml = std::string(yaml.begin() + newline + 1, yaml.end());
+        if (dump_config) {
+          ESP_LOGCONFIG(TAG, "%s", row.c_str());
+        } else {
+          ESP_LOGI(TAG, "%s", row.c_str());
+        }
       }
     }
-    s = tmp;
   }
+
+  delete dec;
 }
 
 }  // namespace store_yaml

esphome/components/store_yaml/store_yaml.h

@@ -2,20 +2,20 @@
 
 #include "esphome/core/component.h"
 #include "esphome/core/automation.h"
+#include "esphome/core/hal.h"
 
-extern const char ESPHOME_YAML[];
+extern const uint8_t ESPHOME_YAML[] PROGMEM;
+extern const size_t ESPHOME_YAML_SIZE;
 
 namespace esphome {
 namespace store_yaml {
+
 class StoreYamlComponent : public Component {
   bool show_in_dump_config_{false};
-  const char *yaml_;
 
  public:
   void dump_config() override;
   void set_show_in_dump_config(bool show) { this->show_in_dump_config_ = show; }
-  void set_yaml(const char *yaml) { this->yaml_ = yaml; }
-  const char *get_yaml() const;
   void log(bool dump_config = false) const;
 };