From 121bd84854fbf8ea33daabdbb320e6508c9ad0d4 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:03:51 +1100 Subject: [PATCH 1/4] Store preferences in disk file on host platform (#6428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: H. Árkosi Róbert Co-authored-by: clydeps --- esphome/components/host/preferences.cpp | 75 +++++++++++++++++++++---- esphome/components/host/preferences.h | 53 +++++++++++++++++ 2 files changed, 116 insertions(+), 12 deletions(-) diff --git a/esphome/components/host/preferences.cpp b/esphome/components/host/preferences.cpp index bf45893e40..7b939cdebb 100644 --- a/esphome/components/host/preferences.cpp +++ b/esphome/components/host/preferences.cpp @@ -1,36 +1,87 @@ #ifdef USE_HOST +#include +#include #include "preferences.h" -#include -#include "esphome/core/preferences.h" -#include "esphome/core/helpers.h" -#include "esphome/core/log.h" -#include "esphome/core/defines.h" +#include "esphome/core/application.h" namespace esphome { namespace host { +namespace fs = std::filesystem; static const char *const TAG = "host.preferences"; -class HostPreferences : public ESPPreferences { - public: - ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash) override { return {}; } +void HostPreferences::setup_() { + if (this->setup_complete_) + return; + this->filename_.append(getenv("HOME")); + this->filename_.append("/.esphome"); + this->filename_.append("/prefs"); + fs::create_directories(this->filename_); + this->filename_.append("/"); + this->filename_.append(App.get_name()); + this->filename_.append(".prefs"); + FILE *fp = fopen(this->filename_.c_str(), "rb"); + if (fp != nullptr) { + while (!feof((fp))) { + uint32_t key; + uint8_t len; + if (fread(&key, sizeof(key), 1, fp) != 1) + break; + if (fread(&len, sizeof(len), 1, fp) != 1) + break; + uint8_t data[len]; + if (fread(data, sizeof(uint8_t), len, fp) != len) + break; + std::vector vec(data, data + len); + this->data[key] = vec; + } + fclose(fp); + } + this->setup_complete_ = true; +} - ESPPreferenceObject make_preference(size_t length, uint32_t type) override { return {}; } +bool HostPreferences::sync() { + this->setup_(); + FILE *fp = fopen(this->filename_.c_str(), "wb"); + std::map>::iterator it; - bool sync() override { return true; } - bool reset() override { return true; } + for (it = this->data.begin(); it != this->data.end(); ++it) { + fwrite(&it->first, sizeof(uint32_t), 1, fp); + uint8_t len = it->second.size(); + fwrite(&len, sizeof(len), 1, fp); + fwrite(it->second.data(), sizeof(uint8_t), it->second.size(), fp); + } + fclose(fp); + return true; +} + +bool HostPreferences::reset() { + host_preferences->data.clear(); + return true; +} + +ESPPreferenceObject HostPreferences::make_preference(size_t length, uint32_t type, bool in_flash) { + auto backend = new HostPreferenceBackend(type); + return ESPPreferenceObject(backend); }; void setup_preferences() { auto *pref = new HostPreferences(); // NOLINT(cppcoreguidelines-owning-memory) + host_preferences = pref; global_preferences = pref; } +bool HostPreferenceBackend::save(const uint8_t *data, size_t len) { + return host_preferences->save(this->key_, data, len); +} + +bool HostPreferenceBackend::load(uint8_t *data, size_t len) { return host_preferences->load(this->key_, data, len); } + +HostPreferences *host_preferences; } // namespace host ESPPreferences *global_preferences; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - } // namespace esphome #endif // USE_HOST diff --git a/esphome/components/host/preferences.h b/esphome/components/host/preferences.h index 7462360ec3..6707366517 100644 --- a/esphome/components/host/preferences.h +++ b/esphome/components/host/preferences.h @@ -2,10 +2,63 @@ #ifdef USE_HOST +#include "esphome/core/preferences.h" +#include + namespace esphome { namespace host { +class HostPreferenceBackend : public ESPPreferenceBackend { + public: + explicit HostPreferenceBackend(uint32_t key) { this->key_ = key; } + + bool save(const uint8_t *data, size_t len) override; + bool load(uint8_t *data, size_t len) override; + + protected: + uint32_t key_{}; +}; + +class HostPreferences : public ESPPreferences { + public: + bool sync() override; + bool reset() override; + + ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash) override; + ESPPreferenceObject make_preference(size_t length, uint32_t type) override { + return make_preference(length, type, false); + } + + bool save(uint32_t key, const uint8_t *data, size_t len) { + if (len > 255) + return false; + this->setup_(); + std::vector vec(data, data + len); + this->data[key] = vec; + return true; + } + + bool load(uint32_t key, uint8_t *data, size_t len) { + if (len > 255) + return false; + this->setup_(); + if (this->data.count(key) == 0) + return false; + auto vec = this->data[key]; + if (vec.size() != len) + return false; + memcpy(data, vec.data(), len); + return true; + } + + protected: + void setup_(); + bool setup_complete_{}; + std::string filename_{}; + std::map> data{}; +}; void setup_preferences(); +extern HostPreferences *host_preferences; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace host } // namespace esphome From f5ac1bd90598701cf579c22aaeb9b740e396cb67 Mon Sep 17 00:00:00 2001 From: ebw44 Date: Tue, 26 Mar 2024 11:20:15 +1300 Subject: [PATCH 2/4] microWakeWord: Fix model path joining (#6426) --- esphome/components/micro_wake_word/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index 209a1412ca..9073d103f1 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -287,7 +287,7 @@ def _load_model_data(manifest_path: Path): except cv.Invalid as e: raise EsphomeError(f"Invalid manifest file: {e}") from e - model_path = urljoin(str(manifest_path), manifest[CONF_MODEL]) + model_path = manifest_path.parent / manifest[CONF_MODEL] with open(model_path, "rb") as f: model = f.read() From 7cb8f99884d6af18efd1b08e47634c488815f1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Poczkodi?= Date: Mon, 25 Mar 2024 23:34:47 +0100 Subject: [PATCH 3/4] Don't compile strptime unless its required (#6424) --- esphome/core/time.cpp | 6 ++++++ esphome/core/time.h | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/esphome/core/time.cpp b/esphome/core/time.cpp index 2e46a611e6..ae4fabac52 100644 --- a/esphome/core/time.cpp +++ b/esphome/core/time.cpp @@ -1,4 +1,6 @@ +#ifdef USE_DATETIME #include +#endif #include "helpers.h" #include "time.h" // NOLINT @@ -64,6 +66,8 @@ std::string ESPTime::strftime(const std::string &format) { return timestr; } +#ifdef USE_DATETIME + bool ESPTime::strptime(const std::string &time_to_parse, ESPTime &esp_time) { // clang-format off std::regex dt_regex(R"(^ @@ -102,6 +106,8 @@ bool ESPTime::strptime(const std::string &time_to_parse, ESPTime &esp_time) { return true; } +#endif + void ESPTime::increment_second() { this->timestamp++; if (!increment_time_value(this->second, 0, 60)) diff --git a/esphome/core/time.h b/esphome/core/time.h index 4300cf26b7..738a0261c7 100644 --- a/esphome/core/time.h +++ b/esphome/core/time.h @@ -67,6 +67,8 @@ struct ESPTime { this->day_of_year < 367 && this->month > 0 && this->month < 13; } +#ifdef USE_DATETIME + /** Convert a string to ESPTime struct as specified by the format argument. * @param time_to_parse null-terminated c string formatet like this: 2020-08-25 05:30:00. * @param esp_time an instance of a ESPTime struct @@ -74,6 +76,8 @@ struct ESPTime { */ static bool strptime(const std::string &time_to_parse, ESPTime &esp_time); +#endif + /// Convert a C tm struct instance with a C unix epoch timestamp to an ESPTime instance. static ESPTime from_c_tm(struct tm *c_tm, time_t c_time); From 2345e7606a1ef2d1ec89e36d487666e9be8b2496 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 Mar 2024 21:24:58 -1000 Subject: [PATCH 4/4] Fix editor live validation (#6431) --- esphome/config.py | 24 ++++++++++++----------- esphome/config_helpers.py | 24 ----------------------- esphome/vscode.py | 41 +++++++++++++++++++++++++++++++++------ esphome/yaml_util.py | 25 ++++++++++++++---------- 4 files changed, 63 insertions(+), 51 deletions(-) diff --git a/esphome/config.py b/esphome/config.py index f5a1ebb8d7..c5764dd4f2 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1,10 +1,11 @@ +from __future__ import annotations import abc import functools import heapq import logging import re -from typing import Optional, Union +from typing import Union, Any from contextlib import contextmanager import contextvars @@ -76,7 +77,7 @@ def _path_begins_with(path: ConfigPath, other: ConfigPath) -> bool: @functools.total_ordering class _ValidationStepTask: - def __init__(self, priority: float, id_number: int, step: "ConfigValidationStep"): + def __init__(self, priority: float, id_number: int, step: ConfigValidationStep): self.priority = priority self.id_number = id_number self.step = step @@ -130,7 +131,7 @@ class Config(OrderedDict, fv.FinalValidateConfig): ) self.errors.append(error) - def add_validation_step(self, step: "ConfigValidationStep"): + def add_validation_step(self, step: ConfigValidationStep): id_num = self._validation_tasks_id self._validation_tasks_id += 1 heapq.heappush( @@ -172,7 +173,7 @@ class Config(OrderedDict, fv.FinalValidateConfig): conf = conf[key] conf[path[-1]] = value - def get_error_for_path(self, path: ConfigPath) -> Optional[vol.Invalid]: + def get_error_for_path(self, path: ConfigPath) -> vol.Invalid | None: for err in self.errors: if self.get_deepest_path(err.path) == path: self.errors.remove(err) @@ -181,7 +182,7 @@ class Config(OrderedDict, fv.FinalValidateConfig): def get_deepest_document_range_for_path( self, path: ConfigPath, get_key: bool = False - ) -> Optional[ESPHomeDataBase]: + ) -> ESPHomeDataBase | None: data = self doc_range = None for index, path_item in enumerate(path): @@ -733,7 +734,9 @@ class PinUseValidationCheck(ConfigValidationStep): pins.PIN_SCHEMA_REGISTRY.final_validate(result) -def validate_config(config, command_line_substitutions) -> Config: +def validate_config( + config: dict[str, Any], command_line_substitutions: dict[str, Any] +) -> Config: result = Config() loader.clear_component_meta_finders() @@ -897,24 +900,23 @@ class InvalidYAMLError(EsphomeError): self.base_exc = base_exc -def _load_config(command_line_substitutions): +def _load_config(command_line_substitutions: dict[str, Any]) -> Config: + """Load the configuration file.""" try: config = yaml_util.load_yaml(CORE.config_path) except EsphomeError as e: raise InvalidYAMLError(e) from e try: - result = validate_config(config, command_line_substitutions) + return validate_config(config, command_line_substitutions) except EsphomeError: raise except Exception: _LOGGER.error("Unexpected exception while reading configuration:") raise - return result - -def load_config(command_line_substitutions): +def load_config(command_line_substitutions: dict[str, Any]) -> Config: try: return _load_config(command_line_substitutions) except vol.Invalid as err: diff --git a/esphome/config_helpers.py b/esphome/config_helpers.py index ac52c6ede2..7b47e097c8 100644 --- a/esphome/config_helpers.py +++ b/esphome/config_helpers.py @@ -1,9 +1,4 @@ -import json -import os - from esphome.const import CONF_ID -from esphome.core import CORE -from esphome.helpers import read_file class Extend: @@ -38,25 +33,6 @@ class Remove: return isinstance(b, Remove) and self.value == b.value -def read_config_file(path: str) -> str: - if CORE.vscode and ( - not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path) - ): - print( - json.dumps( - { - "type": "read_file", - "path": path, - } - ) - ) - data = json.loads(input()) - assert data["type"] == "file_response" - return data["content"] - - return read_file(path) - - def merge_config(full_old, full_new): def merge(old, new): if isinstance(new, dict): diff --git a/esphome/vscode.py b/esphome/vscode.py index cb2f51976f..8198d2659a 100644 --- a/esphome/vscode.py +++ b/esphome/vscode.py @@ -1,20 +1,22 @@ +from __future__ import annotations import json import os +from io import StringIO +from typing import Any -from typing import Optional - -from esphome.config import load_config, _format_vol_invalid, Config +from esphome.yaml_util import parse_yaml +from esphome.config import validate_config, _format_vol_invalid, Config from esphome.core import CORE, DocumentRange import esphome.config_validation as cv -def _get_invalid_range(res: Config, invalid: cv.Invalid) -> Optional[DocumentRange]: +def _get_invalid_range(res: Config, invalid: cv.Invalid) -> DocumentRange | None: return res.get_deepest_document_range_for_path( invalid.path, invalid.error_message == "extra keys not allowed" ) -def _dump_range(range: Optional[DocumentRange]) -> Optional[dict]: +def _dump_range(range: DocumentRange | None) -> dict | None: if range is None: return None return { @@ -56,6 +58,25 @@ class VSCodeResult: ) +def _read_file_content_from_json_on_stdin() -> str: + """Read the content of a json encoded file from stdin.""" + data = json.loads(input()) + assert data["type"] == "file_response" + return data["content"] + + +def _print_file_read_event(path: str) -> None: + """Print a file read event.""" + print( + json.dumps( + { + "type": "read_file", + "path": path, + } + ) + ) + + def read_config(args): while True: CORE.reset() @@ -68,9 +89,17 @@ def read_config(args): CORE.config_path = os.path.join(args.configuration, f) else: CORE.config_path = data["file"] + + file_name = CORE.config_path + _print_file_read_event(file_name) + raw_yaml = _read_file_content_from_json_on_stdin() + command_line_substitutions: dict[str, Any] = ( + dict(args.substitution) if args.substitution else {} + ) vs = VSCodeResult() try: - res = load_config(dict(args.substitution) if args.substitution else {}) + config = parse_yaml(file_name, StringIO(raw_yaml)) + res = validate_config(config, command_line_substitutions) except Exception as err: # pylint: disable=broad-except vs.add_yaml_error(str(err)) else: diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 60705082b6..c7aa78201f 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -417,20 +417,25 @@ def load_yaml(fname: str, clear_secrets: bool = True) -> Any: return _load_yaml_internal(fname) +def parse_yaml(file_name: str, file_handle: TextIOWrapper) -> Any: + """Parse a YAML file.""" + try: + return _load_yaml_internal_with_type(ESPHomeLoader, file_name, file_handle) + except EsphomeError: + # Loading failed, so we now load with the Python loader which has more + # readable exceptions + # Rewind the stream so we can try again + file_handle.seek(0, 0) + return _load_yaml_internal_with_type( + ESPHomePurePythonLoader, file_name, file_handle + ) + + def _load_yaml_internal(fname: str) -> Any: """Load a YAML file.""" try: with open(fname, encoding="utf-8") as f_handle: - try: - return _load_yaml_internal_with_type(ESPHomeLoader, fname, f_handle) - except EsphomeError: - # Loading failed, so we now load with the Python loader which has more - # readable exceptions - # Rewind the stream so we can try again - f_handle.seek(0, 0) - return _load_yaml_internal_with_type( - ESPHomePurePythonLoader, fname, f_handle - ) + return parse_yaml(fname, f_handle) except (UnicodeDecodeError, OSError) as err: raise EsphomeError(f"Error reading file {fname}: {err}") from err