From 7143e9cd9e2073351cfd239e242e2132a36eec0b Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 6 Jun 2024 05:27:06 +1000 Subject: [PATCH 1/3] [config] Allow file: scheme for git external components (#6844) --- esphome/config_validation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 5fc72921e1..7259e3c062 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1949,13 +1949,13 @@ def url(value): except ValueError as e: raise Invalid("Not a valid URL") from e - if not parsed.scheme or not parsed.netloc: - raise Invalid("Expected a URL scheme and host") - return parsed.geturl() + if parsed.scheme and parsed.netloc or parsed.scheme == "file": + return parsed.geturl() + raise Invalid("Expected a file scheme or a URL scheme with host") def git_ref(value): - if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None: + if re.match(r"[a-zA-Z0-9_./-]+", value) is None: raise Invalid("Not a valid git ref") return value From 8ef4aaa70eac95656f2742b7e57d1010cfb235b5 Mon Sep 17 00:00:00 2001 From: Olivier ARCHER Date: Thu, 6 Jun 2024 04:35:28 +0200 Subject: [PATCH 2/3] [ota] http_request update platform (#5586) Co-authored-by: Keith Burzinski Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Co-authored-by: Edward Firmo <94725493+edwardtfn@users.noreply.github.com> --- CODEOWNERS | 1 + .../components/http_request/ota/__init__.py | 189 +++++++++++ .../components/http_request/ota/automation.h | 42 +++ .../http_request/ota/ota_http_request.cpp | 293 ++++++++++++++++++ .../http_request/ota/ota_http_request.h | 72 +++++ .../ota/ota_http_request_arduino.cpp | 134 ++++++++ .../ota/ota_http_request_arduino.h | 42 +++ .../http_request/ota/ota_http_request_idf.cpp | 86 +++++ .../http_request/ota/ota_http_request_idf.h | 24 ++ .../components/http_request/ota/watchdog.cpp | 71 +++++ .../components/http_request/ota/watchdog.h | 27 ++ esphome/core/defines.h | 1 + .../{common.yaml => common_http_request.yaml} | 4 - tests/components/http_request/common_ota.yaml | 36 +++ .../http_request/test-nossl.esp8266.yaml | 38 +++ .../http_request/test.esp32-c3-idf.yaml | 4 + .../http_request/test.esp32-c3.yaml | 7 +- .../http_request/test.esp32-idf.yaml | 4 + tests/components/http_request/test.esp32.yaml | 7 +- .../components/http_request/test.esp8266.yaml | 7 +- .../components/http_request/test.rp2040.yaml | 4 + 21 files changed, 1083 insertions(+), 10 deletions(-) create mode 100644 esphome/components/http_request/ota/__init__.py create mode 100644 esphome/components/http_request/ota/automation.h create mode 100644 esphome/components/http_request/ota/ota_http_request.cpp create mode 100644 esphome/components/http_request/ota/ota_http_request.h create mode 100644 esphome/components/http_request/ota/ota_http_request_arduino.cpp create mode 100644 esphome/components/http_request/ota/ota_http_request_arduino.h create mode 100644 esphome/components/http_request/ota/ota_http_request_idf.cpp create mode 100644 esphome/components/http_request/ota/ota_http_request_idf.h create mode 100644 esphome/components/http_request/ota/watchdog.cpp create mode 100644 esphome/components/http_request/ota/watchdog.h rename tests/components/http_request/{common.yaml => common_http_request.yaml} (95%) create mode 100644 tests/components/http_request/common_ota.yaml create mode 100644 tests/components/http_request/test-nossl.esp8266.yaml create mode 100644 tests/components/http_request/test.esp32-c3-idf.yaml create mode 100644 tests/components/http_request/test.esp32-idf.yaml create mode 100644 tests/components/http_request/test.rp2040.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 82f61c3b80..90574ca9ba 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -171,6 +171,7 @@ esphome/components/host/* @clydebarrow @esphome/core esphome/components/host/time/* @clydebarrow esphome/components/hrxl_maxsonar_wr/* @netmikey esphome/components/hte501/* @Stock-M +esphome/components/http_request/ota/* @oarcher esphome/components/htu31d/* @betterengineering esphome/components/hydreon_rgxx/* @functionpointer esphome/components/hyt271/* @Philippe12 diff --git a/esphome/components/http_request/ota/__init__.py b/esphome/components/http_request/ota/__init__.py new file mode 100644 index 0000000000..6a56fac83a --- /dev/null +++ b/esphome/components/http_request/ota/__init__.py @@ -0,0 +1,189 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome import automation +from esphome.const import ( + CONF_ESP8266_DISABLE_SSL_SUPPORT, + CONF_ID, + CONF_PASSWORD, + CONF_TIMEOUT, + CONF_URL, + CONF_USERNAME, +) +from esphome.components import esp32 +from esphome.components.ota import BASE_OTA_SCHEMA, ota_to_code, OTAComponent +from esphome.core import CORE, coroutine_with_priority +from .. import http_request_ns + +CODEOWNERS = ["@oarcher"] + +AUTO_LOAD = ["md5"] +DEPENDENCIES = ["network"] + +CONF_MD5 = "md5" +CONF_MD5_URL = "md5_url" +CONF_VERIFY_SSL = "verify_ssl" +CONF_WATCHDOG_TIMEOUT = "watchdog_timeout" + +OtaHttpRequestComponent = http_request_ns.class_( + "OtaHttpRequestComponent", OTAComponent +) +OtaHttpRequestComponentArduino = http_request_ns.class_( + "OtaHttpRequestComponentArduino", OtaHttpRequestComponent +) +OtaHttpRequestComponentIDF = http_request_ns.class_( + "OtaHttpRequestComponentIDF", OtaHttpRequestComponent +) +OtaHttpRequestComponentFlashAction = http_request_ns.class_( + "OtaHttpRequestComponentFlashAction", automation.Action +) + + +def validate_ssl_verification(config): + error_message = "" + + if CORE.is_esp32: + if not CORE.using_esp_idf and config[CONF_VERIFY_SSL]: + error_message = "ESPHome supports certificate verification only via ESP-IDF" + + if CORE.is_rp2040 and config[CONF_VERIFY_SSL]: + error_message = "ESPHome does not support certificate verification in Arduino" + + if ( + CORE.is_esp8266 + and not config[CONF_ESP8266_DISABLE_SSL_SUPPORT] + and config[CONF_VERIFY_SSL] + ): + error_message = "ESPHome does not support certificate verification in Arduino" + + if len(error_message) > 0: + raise cv.Invalid( + f"{error_message}. Set '{CONF_VERIFY_SSL}: false' to skip certificate validation and allow less secure HTTPS connections." + ) + + return config + + +def _declare_request_class(value): + if CORE.using_esp_idf: + return cv.declare_id(OtaHttpRequestComponentIDF)(value) + + if CORE.is_esp8266 or CORE.is_esp32 or CORE.is_rp2040: + return cv.declare_id(OtaHttpRequestComponentArduino)(value) + return NotImplementedError + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): _declare_request_class, + cv.SplitDefault(CONF_ESP8266_DISABLE_SSL_SUPPORT, esp8266=False): cv.All( + cv.only_on_esp8266, cv.boolean + ), + cv.Optional(CONF_VERIFY_SSL, default=True): cv.boolean, + cv.Optional( + CONF_TIMEOUT, default="5min" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_WATCHDOG_TIMEOUT): cv.All( + cv.Any(cv.only_on_esp32, cv.only_on_rp2040), + cv.positive_not_null_time_period, + cv.positive_time_period_milliseconds, + ), + } + ) + .extend(BASE_OTA_SCHEMA) + .extend(cv.COMPONENT_SCHEMA), + cv.require_framework_version( + esp8266_arduino=cv.Version(2, 5, 1), + esp32_arduino=cv.Version(0, 0, 0), + esp_idf=cv.Version(0, 0, 0), + rp2040_arduino=cv.Version(0, 0, 0), + ), + validate_ssl_verification, +) + + +@coroutine_with_priority(52.0) +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await ota_to_code(var, config) + + cg.add(var.set_timeout(config[CONF_TIMEOUT])) + + if timeout_ms := config.get(CONF_WATCHDOG_TIMEOUT): + cg.add_define( + "USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT", + timeout_ms, + ) + + if CORE.is_esp8266 and not config[CONF_ESP8266_DISABLE_SSL_SUPPORT]: + cg.add_define("USE_HTTP_REQUEST_ESP8266_HTTPS") + + if CORE.is_esp32: + if CORE.using_esp_idf: + esp32.add_idf_sdkconfig_option( + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", + config.get(CONF_VERIFY_SSL), + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_TLS_INSECURE", + not config.get(CONF_VERIFY_SSL), + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY", + not config.get(CONF_VERIFY_SSL), + ) + else: + cg.add_library("WiFiClientSecure", None) + cg.add_library("HTTPClient", None) + if CORE.is_esp8266: + cg.add_library("ESP8266HTTPClient", None) + if CORE.is_rp2040 and CORE.using_arduino: + cg.add_library("HTTPClient", None) + + await cg.register_component(var, config) + + +OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(OtaHttpRequestComponent), + cv.Optional(CONF_MD5_URL): cv.templatable(cv.url), + cv.Optional(CONF_MD5): cv.templatable(cv.string), + cv.Optional(CONF_PASSWORD): cv.templatable(cv.string), + cv.Optional(CONF_USERNAME): cv.templatable(cv.string), + cv.Required(CONF_URL): cv.templatable(cv.url), + } + ), + cv.has_exactly_one_key(CONF_MD5, CONF_MD5_URL), +) + + +@automation.register_action( + "ota_http_request.flash", + OtaHttpRequestComponentFlashAction, + OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA, +) +async def ota_http_request_action_to_code(config, action_id, template_arg, args): + paren = await cg.get_variable(config[CONF_ID]) + var = cg.new_Pvariable(action_id, template_arg, paren) + + if md5_url := config.get(CONF_MD5_URL): + template_ = await cg.templatable(md5_url, args, cg.std_string) + cg.add(var.set_md5_url(template_)) + + if md5_str := config.get(CONF_MD5): + template_ = await cg.templatable(md5_str, args, cg.std_string) + cg.add(var.set_md5(template_)) + + if password_str := config.get(CONF_PASSWORD): + template_ = await cg.templatable(password_str, args, cg.std_string) + cg.add(var.set_password(template_)) + + if username_str := config.get(CONF_USERNAME): + template_ = await cg.templatable(username_str, args, cg.std_string) + cg.add(var.set_username(template_)) + + template_ = await cg.templatable(config[CONF_URL], args, cg.std_string) + cg.add(var.set_url(template_)) + + return var diff --git a/esphome/components/http_request/ota/automation.h b/esphome/components/http_request/ota/automation.h new file mode 100644 index 0000000000..d4c21f1c72 --- /dev/null +++ b/esphome/components/http_request/ota/automation.h @@ -0,0 +1,42 @@ +#pragma once +#include "ota_http_request.h" + +#include "esphome/core/automation.h" + +namespace esphome { +namespace http_request { + +template class OtaHttpRequestComponentFlashAction : public Action { + public: + OtaHttpRequestComponentFlashAction(OtaHttpRequestComponent *parent) : parent_(parent) {} + TEMPLATABLE_VALUE(std::string, md5_url) + TEMPLATABLE_VALUE(std::string, md5) + TEMPLATABLE_VALUE(std::string, password) + TEMPLATABLE_VALUE(std::string, url) + TEMPLATABLE_VALUE(std::string, username) + + void play(Ts... x) override { + if (this->md5_url_.has_value()) { + this->parent_->set_md5_url(this->md5_url_.value(x...)); + } + if (this->md5_.has_value()) { + this->parent_->set_md5(this->md5_.value(x...)); + } + if (this->password_.has_value()) { + this->parent_->set_password(this->password_.value(x...)); + } + if (this->username_.has_value()) { + this->parent_->set_username(this->username_.value(x...)); + } + this->parent_->set_url(this->url_.value(x...)); + + this->parent_->flash(); + // Normally never reached due to reboot + } + + protected: + OtaHttpRequestComponent *parent_; +}; + +} // namespace http_request +} // namespace esphome diff --git a/esphome/components/http_request/ota/ota_http_request.cpp b/esphome/components/http_request/ota/ota_http_request.cpp new file mode 100644 index 0000000000..cf0816c858 --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request.cpp @@ -0,0 +1,293 @@ +#include "ota_http_request.h" +#include "watchdog.h" + +#include "esphome/core/application.h" +#include "esphome/core/defines.h" +#include "esphome/core/log.h" + +#include "esphome/components/md5/md5.h" +#include "esphome/components/ota/ota_backend_arduino_esp32.h" +#include "esphome/components/ota/ota_backend_arduino_esp8266.h" +#include "esphome/components/ota/ota_backend_arduino_rp2040.h" +#include "esphome/components/ota/ota_backend_esp_idf.h" +#include "esphome/components/ota/ota_backend.h" + +namespace esphome { +namespace http_request { + +void OtaHttpRequestComponent::setup() { +#ifdef USE_OTA_STATE_CALLBACK + ota::register_ota_platform(this); +#endif +} + +void OtaHttpRequestComponent::dump_config() { + ESP_LOGCONFIG(TAG, "Over-The-Air updates via HTTP request:"); + ESP_LOGCONFIG(TAG, " Timeout: %llus", this->timeout_ / 1000); +#ifdef USE_ESP8266 +#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS + ESP_LOGCONFIG(TAG, " ESP8266 SSL support: No"); +#else + ESP_LOGCONFIG(TAG, " ESP8266 SSL support: Yes"); +#endif +#endif +#ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE + ESP_LOGCONFIG(TAG, " TLS server verification: Yes"); +#else + ESP_LOGCONFIG(TAG, " TLS server verification: No"); +#endif +#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT + ESP_LOGCONFIG(TAG, " Watchdog timeout: %ds", USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT / 1000); +#endif +}; + +void OtaHttpRequestComponent::set_md5_url(const std::string &url) { + if (!this->validate_url_(url)) { + this->md5_url_.clear(); // URL was not valid; prevent flashing until it is + return; + } + this->md5_url_ = url; + this->md5_expected_.clear(); // to be retrieved later +} + +void OtaHttpRequestComponent::set_url(const std::string &url) { + if (!this->validate_url_(url)) { + this->url_.clear(); // URL was not valid; prevent flashing until it is + return; + } + this->url_ = url; +} + +bool OtaHttpRequestComponent::check_status() { + // status can be -1, or HTTP status code + if (this->status_ < 100) { + ESP_LOGE(TAG, "HTTP server did not respond (error %d)", this->status_); + return false; + } + if (this->status_ >= 310) { + ESP_LOGE(TAG, "HTTP error %d", this->status_); + return false; + } + ESP_LOGV(TAG, "HTTP status %d", this->status_); + return true; +} + +void OtaHttpRequestComponent::flash() { + if (this->url_.empty()) { + ESP_LOGE(TAG, "URL not set; cannot start update"); + return; + } + + ESP_LOGI(TAG, "Starting update..."); +#ifdef USE_OTA_STATE_CALLBACK + this->state_callback_.call(ota::OTA_STARTED, 0.0f, 0); +#endif + + auto ota_status = this->do_ota_(); + + switch (ota_status) { + case ota::OTA_RESPONSE_OK: +#ifdef USE_OTA_STATE_CALLBACK + this->state_callback_.call(ota::OTA_COMPLETED, 100.0f, ota_status); +#endif + delay(10); + App.safe_reboot(); + break; + + default: +#ifdef USE_OTA_STATE_CALLBACK + this->state_callback_.call(ota::OTA_ERROR, 0.0f, ota_status); +#endif + this->md5_computed_.clear(); // will be reset at next attempt + this->md5_expected_.clear(); // will be reset at next attempt + break; + } +} + +void OtaHttpRequestComponent::cleanup_(std::unique_ptr backend) { + if (this->update_started_) { + ESP_LOGV(TAG, "Aborting OTA backend"); + backend->abort(); + } + ESP_LOGV(TAG, "Aborting HTTP connection"); + this->http_end(); +}; + +uint8_t OtaHttpRequestComponent::do_ota_() { + uint8_t buf[this->http_recv_buffer_ + 1]; + uint32_t last_progress = 0; + uint32_t update_start_time = millis(); + md5::MD5Digest md5_receive; + std::unique_ptr md5_receive_str(new char[33]); + + if (this->md5_expected_.empty() && !this->http_get_md5_()) { + return OTA_MD5_INVALID; + } + + ESP_LOGD(TAG, "MD5 expected: %s", this->md5_expected_.c_str()); + + auto url_with_auth = this->get_url_with_auth_(this->url_); + if (url_with_auth.empty()) { + return OTA_BAD_URL; + } + ESP_LOGVV(TAG, "url_with_auth: %s", url_with_auth.c_str()); + ESP_LOGI(TAG, "Connecting to: %s", this->url_.c_str()); + this->http_init(url_with_auth); + if (!this->check_status()) { + this->http_end(); + return OTA_CONNECTION_ERROR; + } + + // we will compute MD5 on the fly for verification -- Arduino OTA seems to ignore it + md5_receive.init(); + ESP_LOGV(TAG, "MD5Digest initialized"); + + ESP_LOGV(TAG, "OTA backend begin"); + auto backend = ota::make_ota_backend(); + auto error_code = backend->begin(this->body_length_); + if (error_code != ota::OTA_RESPONSE_OK) { + ESP_LOGW(TAG, "backend->begin error: %d", error_code); + this->cleanup_(std::move(backend)); + return error_code; + } + + this->bytes_read_ = 0; + while (this->bytes_read_ < this->body_length_) { + // read a maximum of chunk_size bytes into buf. (real read size returned) + int bufsize = this->http_read(buf, this->http_recv_buffer_); + ESP_LOGVV(TAG, "bytes_read_ = %u, body_length_ = %u, bufsize = %i", this->bytes_read_, this->body_length_, bufsize); + + // feed watchdog and give other tasks a chance to run + App.feed_wdt(); + yield(); + + if (bufsize < 0) { + ESP_LOGE(TAG, "Stream closed"); + this->cleanup_(std::move(backend)); + return OTA_CONNECTION_ERROR; + } else if (bufsize > 0 && bufsize <= this->http_recv_buffer_) { + // add read bytes to MD5 + md5_receive.add(buf, bufsize); + + // write bytes to OTA backend + this->update_started_ = true; + error_code = backend->write(buf, bufsize); + if (error_code != ota::OTA_RESPONSE_OK) { + // error code explanation available at + // https://github.com/esphome/esphome/blob/dev/esphome/components/ota/ota_backend.h + ESP_LOGE(TAG, "Error code (%02X) writing binary data to flash at offset %d and size %d", error_code, + this->bytes_read_ - bufsize, this->body_length_); + this->cleanup_(std::move(backend)); + return error_code; + } + } + + uint32_t now = millis(); + if ((now - last_progress > 1000) or (this->bytes_read_ == this->body_length_)) { + last_progress = now; + float percentage = this->bytes_read_ * 100.0f / this->body_length_; + ESP_LOGD(TAG, "Progress: %0.1f%%", percentage); +#ifdef USE_OTA_STATE_CALLBACK + this->state_callback_.call(ota::OTA_IN_PROGRESS, percentage, 0); +#endif + } + } // while + + ESP_LOGI(TAG, "Done in %.0f seconds", float(millis() - update_start_time) / 1000); + + // verify MD5 is as expected and act accordingly + md5_receive.calculate(); + md5_receive.get_hex(md5_receive_str.get()); + this->md5_computed_ = md5_receive_str.get(); + if (strncmp(this->md5_computed_.c_str(), this->md5_expected_.c_str(), MD5_SIZE) != 0) { + ESP_LOGE(TAG, "MD5 computed: %s - Aborting due to MD5 mismatch", this->md5_computed_.c_str()); + this->cleanup_(std::move(backend)); + return ota::OTA_RESPONSE_ERROR_MD5_MISMATCH; + } else { + backend->set_update_md5(md5_receive_str.get()); + } + + this->http_end(); + + // feed watchdog and give other tasks a chance to run + App.feed_wdt(); + yield(); + delay(100); // NOLINT + + error_code = backend->end(); + if (error_code != ota::OTA_RESPONSE_OK) { + ESP_LOGW(TAG, "Error ending update! error_code: %d", error_code); + this->cleanup_(std::move(backend)); + return error_code; + } + + ESP_LOGI(TAG, "Update complete"); + return ota::OTA_RESPONSE_OK; +} + +std::string OtaHttpRequestComponent::get_url_with_auth_(const std::string &url) { + if (this->username_.empty() || this->password_.empty()) { + return url; + } + + auto start_char = url.find("://"); + if ((start_char == std::string::npos) || (start_char < 4)) { + ESP_LOGE(TAG, "Incorrect URL prefix"); + return {}; + } + + ESP_LOGD(TAG, "Using basic HTTP authentication"); + + start_char += 3; // skip '://' characters + auto url_with_auth = + url.substr(0, start_char) + this->username_ + ":" + this->password_ + "@" + url.substr(start_char); + return url_with_auth; +} + +bool OtaHttpRequestComponent::http_get_md5_() { + if (this->md5_url_.empty()) { + return false; + } + + auto url_with_auth = this->get_url_with_auth_(this->md5_url_); + if (url_with_auth.empty()) { + return false; + } + + ESP_LOGVV(TAG, "url_with_auth: %s", url_with_auth.c_str()); + ESP_LOGI(TAG, "Connecting to: %s", this->md5_url_.c_str()); + this->http_init(url_with_auth); + if (!this->check_status()) { + this->http_end(); + return false; + } + int length = this->body_length_; + if (length < 0) { + this->http_end(); + return false; + } + if (length < MD5_SIZE) { + ESP_LOGE(TAG, "MD5 file must be %u bytes; %u bytes reported by HTTP server. Aborting", MD5_SIZE, + this->body_length_); + this->http_end(); + return false; + } + + this->bytes_read_ = 0; + this->md5_expected_.resize(MD5_SIZE); + auto read_len = this->http_read((uint8_t *) this->md5_expected_.data(), MD5_SIZE); + this->http_end(); + + return read_len == MD5_SIZE; +} + +bool OtaHttpRequestComponent::validate_url_(const std::string &url) { + if ((url.length() < 8) || (url.find("http") != 0) || (url.find("://") == std::string::npos)) { + ESP_LOGE(TAG, "URL is invalid and/or must be prefixed with 'http://' or 'https://'"); + return false; + } + return true; +} + +} // namespace http_request +} // namespace esphome diff --git a/esphome/components/http_request/ota/ota_http_request.h b/esphome/components/http_request/ota/ota_http_request.h new file mode 100644 index 0000000000..9fbdf2ec25 --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request.h @@ -0,0 +1,72 @@ +#pragma once + +#include "esphome/core/component.h" +#include "esphome/core/defines.h" +#include "esphome/components/ota/ota_backend.h" + +#include +#include +#include + +namespace esphome { +namespace http_request { + +static const char *const TAG = "http_request.ota"; +static const uint8_t MD5_SIZE = 32; + +enum OtaHttpRequestError : uint8_t { + OTA_MD5_INVALID = 0x10, + OTA_BAD_URL = 0x11, + OTA_CONNECTION_ERROR = 0x12, +}; + +class OtaHttpRequestComponent : public ota::OTAComponent { + public: + void setup() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + + void set_md5_url(const std::string &md5_url); + void set_md5(const std::string &md5) { this->md5_expected_ = md5; } + void set_password(const std::string &password) { this->password_ = password; } + void set_timeout(const uint64_t timeout) { this->timeout_ = timeout; } + void set_url(const std::string &url); + void set_username(const std::string &username) { this->username_ = username; } + + std::string md5_computed() { return this->md5_computed_; } + std::string md5_expected() { return this->md5_expected_; } + + bool check_status(); + + void flash(); + + virtual void http_init(const std::string &url){}; + virtual int http_read(uint8_t *buf, size_t len) { return 0; }; + virtual void http_end(){}; + + protected: + void cleanup_(std::unique_ptr backend); + uint8_t do_ota_(); + std::string get_url_with_auth_(const std::string &url); + bool http_get_md5_(); + bool secure_() { return this->url_.find("https:") != std::string::npos; }; + bool validate_url_(const std::string &url); + + std::string md5_computed_{}; + std::string md5_expected_{}; + std::string md5_url_{}; + std::string password_{}; + std::string username_{}; + std::string url_{}; + size_t body_length_ = 0; + size_t bytes_read_ = 0; + int status_ = -1; + uint64_t timeout_ = 0; + bool update_started_ = false; + const uint16_t http_recv_buffer_ = 256; // the firmware GET chunk size + const uint16_t max_http_recv_buffer_ = 512; // internal max http buffer size must be > HTTP_RECV_BUFFER_ (TLS + // overhead) and must be a power of two from 512 to 4096 +}; + +} // namespace http_request +} // namespace esphome diff --git a/esphome/components/http_request/ota/ota_http_request_arduino.cpp b/esphome/components/http_request/ota/ota_http_request_arduino.cpp new file mode 100644 index 0000000000..d1dc638d5e --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request_arduino.cpp @@ -0,0 +1,134 @@ +#include "ota_http_request.h" +#include "watchdog.h" + +#ifdef USE_ARDUINO +#include "ota_http_request_arduino.h" +#include "esphome/core/defines.h" +#include "esphome/core/log.h" +#include "esphome/core/application.h" +#include "esphome/components/network/util.h" +#include "esphome/components/md5/md5.h" + +namespace esphome { +namespace http_request { + +struct Header { + const char *name; + const char *value; +}; + +void OtaHttpRequestComponentArduino::http_init(const std::string &url) { + const char *header_keys[] = {"Content-Length", "Content-Type"}; + const size_t header_count = sizeof(header_keys) / sizeof(header_keys[0]); + watchdog::WatchdogManager wdts; + +#ifdef USE_ESP8266 + if (this->stream_ptr_ == nullptr && this->set_stream_ptr_()) { + ESP_LOGE(TAG, "Unable to set client"); + return; + } +#endif // USE_ESP8266 + +#ifdef USE_RP2040 + this->client_.setInsecure(); +#endif + + App.feed_wdt(); + +#if defined(USE_ESP32) || defined(USE_RP2040) + this->status_ = this->client_.begin(url.c_str()); +#endif +#ifdef USE_ESP8266 + this->client_.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); + this->status_ = this->client_.begin(*this->stream_ptr_, url.c_str()); +#endif + + if (!this->status_) { + this->client_.end(); + return; + } + + this->client_.setReuse(true); + + // returned needed headers must be collected before the requests + this->client_.collectHeaders(header_keys, header_count); + + // HTTP GET + this->status_ = this->client_.GET(); + + this->body_length_ = (size_t) this->client_.getSize(); + +#if defined(USE_ESP32) || defined(USE_RP2040) + if (this->stream_ptr_ == nullptr) { + this->set_stream_ptr_(); + } +#endif +} + +int OtaHttpRequestComponentArduino::http_read(uint8_t *buf, const size_t max_len) { +#ifdef USE_ESP8266 +#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 1, 0) // && USE_ARDUINO_VERSION_CODE < VERSION_CODE(?, ?, ?) + if (!this->secure_()) { + ESP_LOGW(TAG, "Using HTTP on Arduino version >= 3.1 is **very** slow. Consider setting framework version to 3.0.2 " + "in your YAML, or use HTTPS"); + } +#endif // USE_ARDUINO_VERSION_CODE +#endif // USE_ESP8266 + + watchdog::WatchdogManager wdts; + + // Since arduino8266 >= 3.1 using this->stream_ptr_ is broken (https://github.com/esp8266/Arduino/issues/9035) + WiFiClient *stream_ptr = this->client_.getStreamPtr(); + if (stream_ptr == nullptr) { + ESP_LOGE(TAG, "Stream pointer vanished!"); + return -1; + } + + int available_data = stream_ptr->available(); + int bufsize = std::min((int) max_len, available_data); + if (bufsize > 0) { + stream_ptr->readBytes(buf, bufsize); + this->bytes_read_ += bufsize; + buf[bufsize] = '\0'; // not fed to ota + } + + return bufsize; +} + +void OtaHttpRequestComponentArduino::http_end() { + watchdog::WatchdogManager wdts; + this->client_.end(); +} + +int OtaHttpRequestComponentArduino::set_stream_ptr_() { +#ifdef USE_ESP8266 +#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS + if (this->secure_()) { + ESP_LOGV(TAG, "ESP8266 HTTPS connection with WiFiClientSecure"); + this->stream_ptr_ = std::make_unique(); + WiFiClientSecure *secure_client = static_cast(this->stream_ptr_.get()); + secure_client->setBufferSizes(this->max_http_recv_buffer_, 512); + secure_client->setInsecure(); + } else { + this->stream_ptr_ = std::make_unique(); + } +#else + ESP_LOGV(TAG, "ESP8266 HTTP connection with WiFiClient"); + if (this->secure_()) { + ESP_LOGE(TAG, "Can't use HTTPS connection with esp8266_disable_ssl_support"); + return -1; + } + this->stream_ptr_ = std::make_unique(); +#endif // USE_HTTP_REQUEST_ESP8266_HTTPS +#endif // USE_ESP8266 + +#if defined(USE_ESP32) || defined(USE_RP2040) + this->stream_ptr_ = std::unique_ptr(this->client_.getStreamPtr()); +#endif + return 0; +} + +} // namespace http_request +} // namespace esphome + +#endif // USE_ARDUINO diff --git a/esphome/components/http_request/ota/ota_http_request_arduino.h b/esphome/components/http_request/ota/ota_http_request_arduino.h new file mode 100644 index 0000000000..02bc046520 --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request_arduino.h @@ -0,0 +1,42 @@ +#pragma once + +#include "ota_http_request.h" + +#ifdef USE_ARDUINO +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/core/defines.h" + +#include +#include +#include + +#if defined(USE_ESP32) || defined(USE_RP2040) +#include +#endif +#ifdef USE_ESP8266 +#include +#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS +#include +#endif +#endif + +namespace esphome { +namespace http_request { + +class OtaHttpRequestComponentArduino : public OtaHttpRequestComponent { + public: + void http_init(const std::string &url) override; + int http_read(uint8_t *buf, size_t len) override; + void http_end() override; + + protected: + int set_stream_ptr_(); + HTTPClient client_{}; + std::unique_ptr stream_ptr_; +}; + +} // namespace http_request +} // namespace esphome + +#endif // USE_ARDUINO diff --git a/esphome/components/http_request/ota/ota_http_request_idf.cpp b/esphome/components/http_request/ota/ota_http_request_idf.cpp new file mode 100644 index 0000000000..9fa565d9bb --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request_idf.cpp @@ -0,0 +1,86 @@ +#include "ota_http_request_idf.h" +#include "watchdog.h" + +#ifdef USE_ESP_IDF +#include "esphome/core/application.h" +#include "esphome/core/defines.h" +#include "esphome/core/log.h" +#include "esphome/components/md5/md5.h" +#include "esphome/components/network/util.h" + +#include "esp_event.h" +#include "esp_http_client.h" +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_netif.h" +#include "esp_system.h" +#include "esp_task_wdt.h" +#include "esp_tls.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "nvs_flash.h" + +#include +#include +#include +#include +#include +#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE +#include "esp_crt_bundle.h" +#endif + +namespace esphome { +namespace http_request { + +void OtaHttpRequestComponentIDF::http_init(const std::string &url) { + App.feed_wdt(); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" + esp_http_client_config_t config = {nullptr}; + config.url = url.c_str(); + config.method = HTTP_METHOD_GET; + config.timeout_ms = (int) this->timeout_; + config.buffer_size = this->max_http_recv_buffer_; + config.auth_type = HTTP_AUTH_TYPE_BASIC; + config.max_authorization_retries = -1; +#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE + if (this->secure_()) { + config.crt_bundle_attach = esp_crt_bundle_attach; + } +#endif +#pragma GCC diagnostic pop + + watchdog::WatchdogManager wdts; + this->client_ = esp_http_client_init(&config); + if ((this->status_ = esp_http_client_open(this->client_, 0)) == ESP_OK) { + this->body_length_ = esp_http_client_fetch_headers(this->client_); + this->status_ = esp_http_client_get_status_code(this->client_); + } +} + +int OtaHttpRequestComponentIDF::http_read(uint8_t *buf, const size_t max_len) { + watchdog::WatchdogManager wdts; + int bufsize = std::min(max_len, this->body_length_ - this->bytes_read_); + + App.feed_wdt(); + int read_len = esp_http_client_read(this->client_, (char *) buf, bufsize); + if (read_len > 0) { + this->bytes_read_ += bufsize; + buf[bufsize] = '\0'; // not fed to ota + } + + return read_len; +} + +void OtaHttpRequestComponentIDF::http_end() { + watchdog::WatchdogManager wdts; + + esp_http_client_close(this->client_); + esp_http_client_cleanup(this->client_); +} + +} // namespace http_request +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/http_request/ota/ota_http_request_idf.h b/esphome/components/http_request/ota/ota_http_request_idf.h new file mode 100644 index 0000000000..9783b2a3e1 --- /dev/null +++ b/esphome/components/http_request/ota/ota_http_request_idf.h @@ -0,0 +1,24 @@ +#pragma once + +#include "ota_http_request.h" + +#ifdef USE_ESP_IDF +#include "esp_http_client.h" + +namespace esphome { +namespace http_request { + +class OtaHttpRequestComponentIDF : public OtaHttpRequestComponent { + public: + void http_init(const std::string &url) override; + int http_read(uint8_t *buf, size_t len) override; + void http_end() override; + + protected: + esp_http_client_handle_t client_{}; +}; + +} // namespace http_request +} // namespace esphome + +#endif // USE_ESP_IDF diff --git a/esphome/components/http_request/ota/watchdog.cpp b/esphome/components/http_request/ota/watchdog.cpp new file mode 100644 index 0000000000..663c9afaac --- /dev/null +++ b/esphome/components/http_request/ota/watchdog.cpp @@ -0,0 +1,71 @@ +#include "watchdog.h" + +#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include +#ifdef USE_ESP32 +#include "esp_idf_version.h" +#include "esp_task_wdt.h" +#endif +#ifdef USE_RP2040 +#include "hardware/watchdog.h" +#include "pico/stdlib.h" +#endif + +namespace esphome { +namespace http_request { +namespace watchdog { + +static const char *const TAG = "watchdog.http_request.ota"; + +WatchdogManager::WatchdogManager() { + this->saved_timeout_ms_ = this->get_timeout_(); + this->set_timeout_(USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT); +} + +WatchdogManager::~WatchdogManager() { this->set_timeout_(this->saved_timeout_ms_); } + +void WatchdogManager::set_timeout_(uint32_t timeout_ms) { + ESP_LOGV(TAG, "Adjusting WDT to %" PRIu32 "ms", timeout_ms); +#ifdef USE_ESP32 +#if ESP_IDF_VERSION_MAJOR >= 5 + esp_task_wdt_config_t wdt_config = { + .timeout_ms = timeout_ms, + .idle_core_mask = 0x03, + .trigger_panic = true, + }; + esp_task_wdt_reconfigure(&wdt_config); +#else + esp_task_wdt_init(timeout_ms, true); +#endif // ESP_IDF_VERSION_MAJOR +#endif // USE_ESP32 + +#ifdef USE_RP2040 + watchdog_enable(timeout_ms, true); +#endif +} + +uint32_t WatchdogManager::get_timeout_() { + uint32_t timeout_ms = 0; + +#ifdef USE_ESP32 + timeout_ms = (uint32_t) CONFIG_ESP_TASK_WDT_TIMEOUT_S * 1000; +#endif // USE_ESP32 + +#ifdef USE_RP2040 + timeout_ms = watchdog_get_count() / 1000; +#endif + + ESP_LOGVV(TAG, "get_timeout: %" PRIu32 "ms", timeout_ms); + + return timeout_ms; +} + +} // namespace watchdog +} // namespace http_request +} // namespace esphome +#endif diff --git a/esphome/components/http_request/ota/watchdog.h b/esphome/components/http_request/ota/watchdog.h new file mode 100644 index 0000000000..0a09dcd6fa --- /dev/null +++ b/esphome/components/http_request/ota/watchdog.h @@ -0,0 +1,27 @@ +#pragma once + +#include "esphome/core/defines.h" + +#include + +namespace esphome { +namespace http_request { +namespace watchdog { + +class WatchdogManager { +#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT + public: + WatchdogManager(); + ~WatchdogManager(); + + private: + uint32_t get_timeout_(); + void set_timeout_(uint32_t timeout_ms); + + uint32_t saved_timeout_ms_{0}; +#endif +}; + +} // namespace watchdog +} // namespace http_request +} // namespace esphome diff --git a/esphome/core/defines.h b/esphome/core/defines.h index c2ad0f641c..76b6af6592 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -33,6 +33,7 @@ #define USE_GRAPH #define USE_GRAPHICAL_DISPLAY_MENU #define USE_HOMEASSISTANT_TIME +#define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000 // NOLINT #define USE_JSON #define USE_LIGHT #define USE_LOCK diff --git a/tests/components/http_request/common.yaml b/tests/components/http_request/common_http_request.yaml similarity index 95% rename from tests/components/http_request/common.yaml rename to tests/components/http_request/common_http_request.yaml index 848fe3f509..b00768c736 100644 --- a/tests/components/http_request/common.yaml +++ b/tests/components/http_request/common_http_request.yaml @@ -28,10 +28,6 @@ esphome: body: "Some data" verify_ssl: false -wifi: - ssid: MySSID - password: password1 - http_request: useragent: esphome/tagreader timeout: 10s diff --git a/tests/components/http_request/common_ota.yaml b/tests/components/http_request/common_ota.yaml new file mode 100644 index 0000000000..10e7d54c3f --- /dev/null +++ b/tests/components/http_request/common_ota.yaml @@ -0,0 +1,36 @@ +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: http_request + verify_ssl: ${verify_ssl} + on_begin: + then: + - logger.log: "OTA start" + on_progress: + then: + - logger.log: + format: "OTA progress %0.1f%%" + args: ["x"] + on_end: + then: + - logger.log: "OTA end" + on_error: + then: + - logger.log: + format: "OTA update error %d" + args: ["x"] + on_state_change: + then: + lambda: 'ESP_LOGD("ota", "State %d", state);' + +button: + - platform: template + name: Firmware update + on_press: + then: + - ota_http_request.flash: + md5_url: http://my.ha.net:8123/local/esphome/firmware.md5 + url: http://my.ha.net:8123/local/esphome/firmware.bin + - logger.log: "This message should be not displayed (reboot)" diff --git a/tests/components/http_request/test-nossl.esp8266.yaml b/tests/components/http_request/test-nossl.esp8266.yaml new file mode 100644 index 0000000000..65116d5550 --- /dev/null +++ b/tests/components/http_request/test-nossl.esp8266.yaml @@ -0,0 +1,38 @@ +<<: !include common_http_request.yaml + +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: http_request + esp8266_disable_ssl_support: true + on_begin: + then: + - logger.log: "OTA start" + on_progress: + then: + - logger.log: + format: "OTA progress %0.1f%%" + args: ["x"] + on_end: + then: + - logger.log: "OTA end" + on_error: + then: + - logger.log: + format: "OTA update error %d" + args: ["x"] + on_state_change: + then: + lambda: 'ESP_LOGD("ota", "State %d", state);' + +button: + - platform: template + name: Firmware update + on_press: + then: + - ota_http_request.flash: + md5_url: http://my.ha.net:8123/local/esphome/firmware.md5 + url: http://my.ha.net:8123/local/esphome/firmware.bin + - logger.log: "This message should be not displayed (reboot)" diff --git a/tests/components/http_request/test.esp32-c3-idf.yaml b/tests/components/http_request/test.esp32-c3-idf.yaml new file mode 100644 index 0000000000..da629e83a9 --- /dev/null +++ b/tests/components/http_request/test.esp32-c3-idf.yaml @@ -0,0 +1,4 @@ +substitutions: + verify_ssl: "true" + +<<: !include common_ota.yaml diff --git a/tests/components/http_request/test.esp32-c3.yaml b/tests/components/http_request/test.esp32-c3.yaml index 25cb37a0b4..1f597bb500 100644 --- a/tests/components/http_request/test.esp32-c3.yaml +++ b/tests/components/http_request/test.esp32-c3.yaml @@ -1,2 +1,5 @@ -packages: - common: !include common.yaml +substitutions: + verify_ssl: "false" + +<<: !include common_http_request.yaml +<<: !include common_ota.yaml diff --git a/tests/components/http_request/test.esp32-idf.yaml b/tests/components/http_request/test.esp32-idf.yaml new file mode 100644 index 0000000000..da629e83a9 --- /dev/null +++ b/tests/components/http_request/test.esp32-idf.yaml @@ -0,0 +1,4 @@ +substitutions: + verify_ssl: "true" + +<<: !include common_ota.yaml diff --git a/tests/components/http_request/test.esp32.yaml b/tests/components/http_request/test.esp32.yaml index 25cb37a0b4..1f597bb500 100644 --- a/tests/components/http_request/test.esp32.yaml +++ b/tests/components/http_request/test.esp32.yaml @@ -1,2 +1,5 @@ -packages: - common: !include common.yaml +substitutions: + verify_ssl: "false" + +<<: !include common_http_request.yaml +<<: !include common_ota.yaml diff --git a/tests/components/http_request/test.esp8266.yaml b/tests/components/http_request/test.esp8266.yaml index 25cb37a0b4..1f597bb500 100644 --- a/tests/components/http_request/test.esp8266.yaml +++ b/tests/components/http_request/test.esp8266.yaml @@ -1,2 +1,5 @@ -packages: - common: !include common.yaml +substitutions: + verify_ssl: "false" + +<<: !include common_http_request.yaml +<<: !include common_ota.yaml diff --git a/tests/components/http_request/test.rp2040.yaml b/tests/components/http_request/test.rp2040.yaml new file mode 100644 index 0000000000..077e4d82da --- /dev/null +++ b/tests/components/http_request/test.rp2040.yaml @@ -0,0 +1,4 @@ +substitutions: + verify_ssl: "false" + +<<: !include common_ota.yaml From ccab57fc583189976248b7574eb4ca448860b16a Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 7 Jun 2024 16:30:49 +1200 Subject: [PATCH 3/3] [logger] Fix defines for development (#6870) * [logger] Fix defines for development * Set debugging flags for rp2040 --- esphome/components/logger/__init__.py | 27 ++++++++++++++++++++------- esphome/core/defines.h | 9 +++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index c05f3d54aa..99aa39c4ba 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -112,11 +112,18 @@ HARDWARE_UART_TO_UART_SELECTION = { } HARDWARE_UART_TO_SERIAL = { - UART0: cg.global_ns.Serial, - UART0_SWAP: cg.global_ns.Serial, - UART1: cg.global_ns.Serial1, - UART2: cg.global_ns.Serial2, - DEFAULT: cg.global_ns.Serial, + PLATFORM_ESP8266: { + UART0: cg.global_ns.Serial, + UART0_SWAP: cg.global_ns.Serial, + UART1: cg.global_ns.Serial1, + UART2: cg.global_ns.Serial2, + DEFAULT: cg.global_ns.Serial, + }, + PLATFORM_RP2040: { + UART0: cg.global_ns.Serial1, + UART1: cg.global_ns.Serial2, + USB_CDC: cg.global_ns.Serial, + }, } is_log_level = cv.one_of(*LOG_LEVELS, upper=True) @@ -244,8 +251,14 @@ async def to_code(config): is_at_least_very_verbose = this_severity >= very_verbose_severity has_serial_logging = baud_rate != 0 - if CORE.is_esp8266 and has_serial_logging and is_at_least_verbose: - debug_serial_port = HARDWARE_UART_TO_SERIAL[config.get(CONF_HARDWARE_UART)] + if ( + (CORE.is_esp8266 or CORE.is_rp2040) + and has_serial_logging + and is_at_least_verbose + ): + debug_serial_port = HARDWARE_UART_TO_SERIAL[CORE.target_platform][ + config.get(CONF_HARDWARE_UART) + ] cg.add_build_flag(f"-DDEBUG_ESP_PORT={debug_serial_port}") cg.add_build_flag("-DLWIP_DEBUG") DEBUG_COMPONENTS = { diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 76b6af6592..affcd78089 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -101,6 +101,14 @@ #ifdef USE_ESP_IDF #define USE_ESP_IDF_VERSION_CODE VERSION_CODE(4, 4, 2) #endif + +#if defined(USE_ESP32_VARIANT_ESP32S2) +#define USE_LOGGER_USB_CDC +#elif defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3) || \ + defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32H2) +#define USE_LOGGER_USB_CDC +#define USE_LOGGER_USB_SERIAL_JTAG +#endif #endif // ESP8266-specific feature flags @@ -123,6 +131,7 @@ #ifdef USE_RP2040 #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 3, 0) +#define USE_LOGGER_USB_CDC #define USE_SOCKET_IMPL_LWIP_TCP #define USE_SPI #endif