Merge remote-tracking branch 'upstream/dev' into dev

esphome/components/host/preferences.cpp

@@ -1,36 +1,87 @@
 #ifdef USE_HOST
 
+#include <filesystem>
+#include <fstream>
 #include "preferences.h"
-#include <cstring>
+#include "esphome/core/application.h"
-#include "esphome/core/preferences.h"
-#include "esphome/core/helpers.h"
-#include "esphome/core/log.h"
-#include "esphome/core/defines.h"
 
 namespace esphome {
 namespace host {
+namespace fs = std::filesystem;
 
 static const char *const TAG = "host.preferences";
 
-class HostPreferences : public ESPPreferences {
+void HostPreferences::setup_() {
- public:
+  if (this->setup_complete_)
-  ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash) override { return {}; }
+    return;
+  this->filename_.append(getenv("HOME"));
+  this->filename_.append("/.esphome");
+  this->filename_.append("/prefs");
+  fs::create_directories(this->filename_);
+  this->filename_.append("/");
+  this->filename_.append(App.get_name());
+  this->filename_.append(".prefs");
+  FILE *fp = fopen(this->filename_.c_str(), "rb");
+  if (fp != nullptr) {
+    while (!feof((fp))) {
+      uint32_t key;
+      uint8_t len;
+      if (fread(&key, sizeof(key), 1, fp) != 1)
+        break;
+      if (fread(&len, sizeof(len), 1, fp) != 1)
+        break;
+      uint8_t data[len];
+      if (fread(data, sizeof(uint8_t), len, fp) != len)
+        break;
+      std::vector vec(data, data + len);
+      this->data[key] = vec;
+    }
+    fclose(fp);
+  }
+  this->setup_complete_ = true;
+}
 
-  ESPPreferenceObject make_preference(size_t length, uint32_t type) override { return {}; }
+bool HostPreferences::sync() {
+  this->setup_();
+  FILE *fp = fopen(this->filename_.c_str(), "wb");
+  std::map<uint32_t, std::vector<uint8_t>>::iterator it;
 
-  bool sync() override { return true; }
+  for (it = this->data.begin(); it != this->data.end(); ++it) {
-  bool reset() override { return true; }
+    fwrite(&it->first, sizeof(uint32_t), 1, fp);
+    uint8_t len = it->second.size();
+    fwrite(&len, sizeof(len), 1, fp);
+    fwrite(it->second.data(), sizeof(uint8_t), it->second.size(), fp);
+  }
+  fclose(fp);
+  return true;
+}
+
+bool HostPreferences::reset() {
+  host_preferences->data.clear();
+  return true;
+}
+
+ESPPreferenceObject HostPreferences::make_preference(size_t length, uint32_t type, bool in_flash) {
+  auto backend = new HostPreferenceBackend(type);
+  return ESPPreferenceObject(backend);
 };
 
 void setup_preferences() {
   auto *pref = new HostPreferences();  // NOLINT(cppcoreguidelines-owning-memory)
+  host_preferences = pref;
   global_preferences = pref;
 }
 
+bool HostPreferenceBackend::save(const uint8_t *data, size_t len) {
+  return host_preferences->save(this->key_, data, len);
+}
+
+bool HostPreferenceBackend::load(uint8_t *data, size_t len) { return host_preferences->load(this->key_, data, len); }
+
+HostPreferences *host_preferences;
 }  // namespace host
 
 ESPPreferences *global_preferences;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
-
 }  // namespace esphome
 
 #endif  // USE_HOST

esphome/components/host/preferences.h

@@ -2,10 +2,63 @@
 
 #ifdef USE_HOST
 
+#include "esphome/core/preferences.h"
+#include <map>
+
 namespace esphome {
 namespace host {
 
+class HostPreferenceBackend : public ESPPreferenceBackend {
+ public:
+  explicit HostPreferenceBackend(uint32_t key) { this->key_ = key; }
+
+  bool save(const uint8_t *data, size_t len) override;
+  bool load(uint8_t *data, size_t len) override;
+
+ protected:
+  uint32_t key_{};
+};
+
+class HostPreferences : public ESPPreferences {
+ public:
+  bool sync() override;
+  bool reset() override;
+
+  ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash) override;
+  ESPPreferenceObject make_preference(size_t length, uint32_t type) override {
+    return make_preference(length, type, false);
+  }
+
+  bool save(uint32_t key, const uint8_t *data, size_t len) {
+    if (len > 255)
+      return false;
+    this->setup_();
+    std::vector vec(data, data + len);
+    this->data[key] = vec;
+    return true;
+  }
+
+  bool load(uint32_t key, uint8_t *data, size_t len) {
+    if (len > 255)
+      return false;
+    this->setup_();
+    if (this->data.count(key) == 0)
+      return false;
+    auto vec = this->data[key];
+    if (vec.size() != len)
+      return false;
+    memcpy(data, vec.data(), len);
+    return true;
+  }
+
+ protected:
+  void setup_();
+  bool setup_complete_{};
+  std::string filename_{};
+  std::map<uint32_t, std::vector<uint8_t>> data{};
+};
 void setup_preferences();
+extern HostPreferences *host_preferences;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 }  // namespace host
 }  // namespace esphome

esphome/components/micro_wake_word/__init__.py

@@ -287,7 +287,7 @@ def _load_model_data(manifest_path: Path):
     except cv.Invalid as e:
         raise EsphomeError(f"Invalid manifest file: {e}") from e
 
-    model_path = urljoin(str(manifest_path), manifest[CONF_MODEL])
+    model_path = manifest_path.parent / manifest[CONF_MODEL]
 
     with open(model_path, "rb") as f:
         model = f.read()

esphome/config.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
 import abc
 import functools
 import heapq
 import logging
 import re
 
-from typing import Optional, Union
+from typing import Union, Any
 
 from contextlib import contextmanager
 import contextvars
@@ -76,7 +77,7 @@ def _path_begins_with(path: ConfigPath, other: ConfigPath) -> bool:
 
 @functools.total_ordering
 class _ValidationStepTask:
-    def __init__(self, priority: float, id_number: int, step: "ConfigValidationStep"):
+    def __init__(self, priority: float, id_number: int, step: ConfigValidationStep):
         self.priority = priority
         self.id_number = id_number
         self.step = step
@@ -130,7 +131,7 @@ class Config(OrderedDict, fv.FinalValidateConfig):
             )
         self.errors.append(error)
 
-    def add_validation_step(self, step: "ConfigValidationStep"):
+    def add_validation_step(self, step: ConfigValidationStep):
         id_num = self._validation_tasks_id
         self._validation_tasks_id += 1
         heapq.heappush(
@@ -172,7 +173,7 @@ class Config(OrderedDict, fv.FinalValidateConfig):
             conf = conf[key]
         conf[path[-1]] = value
 
-    def get_error_for_path(self, path: ConfigPath) -> Optional[vol.Invalid]:
+    def get_error_for_path(self, path: ConfigPath) -> vol.Invalid | None:
         for err in self.errors:
             if self.get_deepest_path(err.path) == path:
                 self.errors.remove(err)
@@ -181,7 +182,7 @@ class Config(OrderedDict, fv.FinalValidateConfig):
 
     def get_deepest_document_range_for_path(
         self, path: ConfigPath, get_key: bool = False
-    ) -> Optional[ESPHomeDataBase]:
+    ) -> ESPHomeDataBase | None:
         data = self
         doc_range = None
         for index, path_item in enumerate(path):
@@ -733,7 +734,9 @@ class PinUseValidationCheck(ConfigValidationStep):
         pins.PIN_SCHEMA_REGISTRY.final_validate(result)
 
 
-def validate_config(config, command_line_substitutions) -> Config:
+def validate_config(
+    config: dict[str, Any], command_line_substitutions: dict[str, Any]
+) -> Config:
     result = Config()
 
     loader.clear_component_meta_finders()
@@ -897,24 +900,23 @@ class InvalidYAMLError(EsphomeError):
         self.base_exc = base_exc
 
 
-def _load_config(command_line_substitutions):
+def _load_config(command_line_substitutions: dict[str, Any]) -> Config:
+    """Load the configuration file."""
     try:
         config = yaml_util.load_yaml(CORE.config_path)
     except EsphomeError as e:
         raise InvalidYAMLError(e) from e
 
     try:
-        result = validate_config(config, command_line_substitutions)
+        return validate_config(config, command_line_substitutions)
     except EsphomeError:
         raise
     except Exception:
         _LOGGER.error("Unexpected exception while reading configuration:")
         raise
 
-    return result
 
-
+def load_config(command_line_substitutions: dict[str, Any]) -> Config:
-def load_config(command_line_substitutions):
     try:
         return _load_config(command_line_substitutions)
     except vol.Invalid as err:

esphome/config_helpers.py

@@ -1,9 +1,4 @@
-import json
-import os
-
 from esphome.const import CONF_ID
-from esphome.core import CORE
-from esphome.helpers import read_file
 
 
 class Extend:
@@ -38,25 +33,6 @@ class Remove:
         return isinstance(b, Remove) and self.value == b.value
 
 
-def read_config_file(path: str) -> str:
-    if CORE.vscode and (
-        not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path)
-    ):
-        print(
-            json.dumps(
-                {
-                    "type": "read_file",
-                    "path": path,
-                }
-            )
-        )
-        data = json.loads(input())
-        assert data["type"] == "file_response"
-        return data["content"]
-
-    return read_file(path)
-
-
 def merge_config(full_old, full_new):
     def merge(old, new):
         if isinstance(new, dict):

esphome/core/time.cpp

@@ -1,4 +1,6 @@
+#ifdef USE_DATETIME
 #include <regex>
+#endif
 
 #include "helpers.h"
 #include "time.h"  // NOLINT
@@ -64,6 +66,8 @@ std::string ESPTime::strftime(const std::string &format) {
   return timestr;
 }
 
+#ifdef USE_DATETIME
+
 bool ESPTime::strptime(const std::string &time_to_parse, ESPTime &esp_time) {
   // clang-format off
   std::regex dt_regex(R"(^
@@ -102,6 +106,8 @@ bool ESPTime::strptime(const std::string &time_to_parse, ESPTime &esp_time) {
   return true;
 }
 
+#endif
+
 void ESPTime::increment_second() {
   this->timestamp++;
   if (!increment_time_value(this->second, 0, 60))

esphome/core/time.h

@@ -67,6 +67,8 @@ struct ESPTime {
            this->day_of_year < 367 && this->month > 0 && this->month < 13;
   }
 
+#ifdef USE_DATETIME
+
   /** Convert a string to ESPTime struct as specified by the format argument.
    * @param time_to_parse null-terminated c string formatet like this: 2020-08-25 05:30:00.
    * @param esp_time an instance of a ESPTime struct
@@ -74,6 +76,8 @@ struct ESPTime {
    */
   static bool strptime(const std::string &time_to_parse, ESPTime &esp_time);
 
+#endif
+
   /// Convert a C tm struct instance with a C unix epoch timestamp to an ESPTime instance.
   static ESPTime from_c_tm(struct tm *c_tm, time_t c_time);
 

esphome/vscode.py

@@ -1,20 +1,22 @@
+from __future__ import annotations
 import json
 import os
+from io import StringIO
+from typing import Any
 
-from typing import Optional
+from esphome.yaml_util import parse_yaml
-
+from esphome.config import validate_config, _format_vol_invalid, Config
-from esphome.config import load_config, _format_vol_invalid, Config
 from esphome.core import CORE, DocumentRange
 import esphome.config_validation as cv
 
 
-def _get_invalid_range(res: Config, invalid: cv.Invalid) -> Optional[DocumentRange]:
+def _get_invalid_range(res: Config, invalid: cv.Invalid) -> DocumentRange | None:
     return res.get_deepest_document_range_for_path(
         invalid.path, invalid.error_message == "extra keys not allowed"
     )
 
 
-def _dump_range(range: Optional[DocumentRange]) -> Optional[dict]:
+def _dump_range(range: DocumentRange | None) -> dict | None:
     if range is None:
         return None
     return {
@@ -56,6 +58,25 @@ class VSCodeResult:
         )
 
 
+def _read_file_content_from_json_on_stdin() -> str:
+    """Read the content of a json encoded file from stdin."""
+    data = json.loads(input())
+    assert data["type"] == "file_response"
+    return data["content"]
+
+
+def _print_file_read_event(path: str) -> None:
+    """Print a file read event."""
+    print(
+        json.dumps(
+            {
+                "type": "read_file",
+                "path": path,
+            }
+        )
+    )
+
+
 def read_config(args):
     while True:
         CORE.reset()
@@ -68,9 +89,17 @@ def read_config(args):
             CORE.config_path = os.path.join(args.configuration, f)
         else:
             CORE.config_path = data["file"]
+
+        file_name = CORE.config_path
+        _print_file_read_event(file_name)
+        raw_yaml = _read_file_content_from_json_on_stdin()
+        command_line_substitutions: dict[str, Any] = (
+            dict(args.substitution) if args.substitution else {}
+        )
         vs = VSCodeResult()
         try:
-            res = load_config(dict(args.substitution) if args.substitution else {})
+            config = parse_yaml(file_name, StringIO(raw_yaml))
+            res = validate_config(config, command_line_substitutions)
         except Exception as err:  # pylint: disable=broad-except
             vs.add_yaml_error(str(err))
         else:

esphome/yaml_util.py

@@ -417,20 +417,25 @@ def load_yaml(fname: str, clear_secrets: bool = True) -> Any:
     return _load_yaml_internal(fname)
 
 
+def parse_yaml(file_name: str, file_handle: TextIOWrapper) -> Any:
+    """Parse a YAML file."""
+    try:
+        return _load_yaml_internal_with_type(ESPHomeLoader, file_name, file_handle)
+    except EsphomeError:
+        # Loading failed, so we now load with the Python loader which has more
+        # readable exceptions
+        # Rewind the stream so we can try again
+        file_handle.seek(0, 0)
+        return _load_yaml_internal_with_type(
+            ESPHomePurePythonLoader, file_name, file_handle
+        )
+
+
 def _load_yaml_internal(fname: str) -> Any:
     """Load a YAML file."""
     try:
         with open(fname, encoding="utf-8") as f_handle:
-            try:
+            return parse_yaml(fname, f_handle)
-                return _load_yaml_internal_with_type(ESPHomeLoader, fname, f_handle)
-            except EsphomeError:
-                # Loading failed, so we now load with the Python loader which has more
-                # readable exceptions
-                # Rewind the stream so we can try again
-                f_handle.seek(0, 0)
-                return _load_yaml_internal_with_type(
-                    ESPHomePurePythonLoader, fname, f_handle
-                )
     except (UnicodeDecodeError, OSError) as err:
         raise EsphomeError(f"Error reading file {fname}: {err}") from err