Merge branch 'esphome:dev' into dev

CODEOWNERS

@@ -171,6 +171,7 @@ esphome/components/host/* @clydebarrow @esphome/core
 esphome/components/host/time/* @clydebarrow
 esphome/components/hrxl_maxsonar_wr/* @netmikey
 esphome/components/hte501/* @Stock-M
+esphome/components/http_request/ota/* @oarcher
 esphome/components/htu31d/* @betterengineering
 esphome/components/hydreon_rgxx/* @functionpointer
 esphome/components/hyt271/* @Philippe12

esphome/components/http_request/ota/__init__.py

@@ -0,0 +1,189 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome import automation
+from esphome.const import (
+    CONF_ESP8266_DISABLE_SSL_SUPPORT,
+    CONF_ID,
+    CONF_PASSWORD,
+    CONF_TIMEOUT,
+    CONF_URL,
+    CONF_USERNAME,
+)
+from esphome.components import esp32
+from esphome.components.ota import BASE_OTA_SCHEMA, ota_to_code, OTAComponent
+from esphome.core import CORE, coroutine_with_priority
+from .. import http_request_ns
+
+CODEOWNERS = ["@oarcher"]
+
+AUTO_LOAD = ["md5"]
+DEPENDENCIES = ["network"]
+
+CONF_MD5 = "md5"
+CONF_MD5_URL = "md5_url"
+CONF_VERIFY_SSL = "verify_ssl"
+CONF_WATCHDOG_TIMEOUT = "watchdog_timeout"
+
+OtaHttpRequestComponent = http_request_ns.class_(
+    "OtaHttpRequestComponent", OTAComponent
+)
+OtaHttpRequestComponentArduino = http_request_ns.class_(
+    "OtaHttpRequestComponentArduino", OtaHttpRequestComponent
+)
+OtaHttpRequestComponentIDF = http_request_ns.class_(
+    "OtaHttpRequestComponentIDF", OtaHttpRequestComponent
+)
+OtaHttpRequestComponentFlashAction = http_request_ns.class_(
+    "OtaHttpRequestComponentFlashAction", automation.Action
+)
+
+
+def validate_ssl_verification(config):
+    error_message = ""
+
+    if CORE.is_esp32:
+        if not CORE.using_esp_idf and config[CONF_VERIFY_SSL]:
+            error_message = "ESPHome supports certificate verification only via ESP-IDF"
+
+    if CORE.is_rp2040 and config[CONF_VERIFY_SSL]:
+        error_message = "ESPHome does not support certificate verification in Arduino"
+
+    if (
+        CORE.is_esp8266
+        and not config[CONF_ESP8266_DISABLE_SSL_SUPPORT]
+        and config[CONF_VERIFY_SSL]
+    ):
+        error_message = "ESPHome does not support certificate verification in Arduino"
+
+    if len(error_message) > 0:
+        raise cv.Invalid(
+            f"{error_message}. Set '{CONF_VERIFY_SSL}: false' to skip certificate validation and allow less secure HTTPS connections."
+        )
+
+    return config
+
+
+def _declare_request_class(value):
+    if CORE.using_esp_idf:
+        return cv.declare_id(OtaHttpRequestComponentIDF)(value)
+
+    if CORE.is_esp8266 or CORE.is_esp32 or CORE.is_rp2040:
+        return cv.declare_id(OtaHttpRequestComponentArduino)(value)
+    return NotImplementedError
+
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): _declare_request_class,
+            cv.SplitDefault(CONF_ESP8266_DISABLE_SSL_SUPPORT, esp8266=False): cv.All(
+                cv.only_on_esp8266, cv.boolean
+            ),
+            cv.Optional(CONF_VERIFY_SSL, default=True): cv.boolean,
+            cv.Optional(
+                CONF_TIMEOUT, default="5min"
+            ): cv.positive_time_period_milliseconds,
+            cv.Optional(CONF_WATCHDOG_TIMEOUT): cv.All(
+                cv.Any(cv.only_on_esp32, cv.only_on_rp2040),
+                cv.positive_not_null_time_period,
+                cv.positive_time_period_milliseconds,
+            ),
+        }
+    )
+    .extend(BASE_OTA_SCHEMA)
+    .extend(cv.COMPONENT_SCHEMA),
+    cv.require_framework_version(
+        esp8266_arduino=cv.Version(2, 5, 1),
+        esp32_arduino=cv.Version(0, 0, 0),
+        esp_idf=cv.Version(0, 0, 0),
+        rp2040_arduino=cv.Version(0, 0, 0),
+    ),
+    validate_ssl_verification,
+)
+
+
+@coroutine_with_priority(52.0)
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await ota_to_code(var, config)
+
+    cg.add(var.set_timeout(config[CONF_TIMEOUT]))
+
+    if timeout_ms := config.get(CONF_WATCHDOG_TIMEOUT):
+        cg.add_define(
+            "USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT",
+            timeout_ms,
+        )
+
+    if CORE.is_esp8266 and not config[CONF_ESP8266_DISABLE_SSL_SUPPORT]:
+        cg.add_define("USE_HTTP_REQUEST_ESP8266_HTTPS")
+
+    if CORE.is_esp32:
+        if CORE.using_esp_idf:
+            esp32.add_idf_sdkconfig_option(
+                "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE",
+                config.get(CONF_VERIFY_SSL),
+            )
+            esp32.add_idf_sdkconfig_option(
+                "CONFIG_ESP_TLS_INSECURE",
+                not config.get(CONF_VERIFY_SSL),
+            )
+            esp32.add_idf_sdkconfig_option(
+                "CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY",
+                not config.get(CONF_VERIFY_SSL),
+            )
+        else:
+            cg.add_library("WiFiClientSecure", None)
+            cg.add_library("HTTPClient", None)
+    if CORE.is_esp8266:
+        cg.add_library("ESP8266HTTPClient", None)
+    if CORE.is_rp2040 and CORE.using_arduino:
+        cg.add_library("HTTPClient", None)
+
+    await cg.register_component(var, config)
+
+
+OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.use_id(OtaHttpRequestComponent),
+            cv.Optional(CONF_MD5_URL): cv.templatable(cv.url),
+            cv.Optional(CONF_MD5): cv.templatable(cv.string),
+            cv.Optional(CONF_PASSWORD): cv.templatable(cv.string),
+            cv.Optional(CONF_USERNAME): cv.templatable(cv.string),
+            cv.Required(CONF_URL): cv.templatable(cv.url),
+        }
+    ),
+    cv.has_exactly_one_key(CONF_MD5, CONF_MD5_URL),
+)
+
+
+@automation.register_action(
+    "ota_http_request.flash",
+    OtaHttpRequestComponentFlashAction,
+    OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA,
+)
+async def ota_http_request_action_to_code(config, action_id, template_arg, args):
+    paren = await cg.get_variable(config[CONF_ID])
+    var = cg.new_Pvariable(action_id, template_arg, paren)
+
+    if md5_url := config.get(CONF_MD5_URL):
+        template_ = await cg.templatable(md5_url, args, cg.std_string)
+        cg.add(var.set_md5_url(template_))
+
+    if md5_str := config.get(CONF_MD5):
+        template_ = await cg.templatable(md5_str, args, cg.std_string)
+        cg.add(var.set_md5(template_))
+
+    if password_str := config.get(CONF_PASSWORD):
+        template_ = await cg.templatable(password_str, args, cg.std_string)
+        cg.add(var.set_password(template_))
+
+    if username_str := config.get(CONF_USERNAME):
+        template_ = await cg.templatable(username_str, args, cg.std_string)
+        cg.add(var.set_username(template_))
+
+    template_ = await cg.templatable(config[CONF_URL], args, cg.std_string)
+    cg.add(var.set_url(template_))
+
+    return var

esphome/components/http_request/ota/automation.h

@@ -0,0 +1,42 @@
+#pragma once
+#include "ota_http_request.h"
+
+#include "esphome/core/automation.h"
+
+namespace esphome {
+namespace http_request {
+
+template<typename... Ts> class OtaHttpRequestComponentFlashAction : public Action<Ts...> {
+ public:
+  OtaHttpRequestComponentFlashAction(OtaHttpRequestComponent *parent) : parent_(parent) {}
+  TEMPLATABLE_VALUE(std::string, md5_url)
+  TEMPLATABLE_VALUE(std::string, md5)
+  TEMPLATABLE_VALUE(std::string, password)
+  TEMPLATABLE_VALUE(std::string, url)
+  TEMPLATABLE_VALUE(std::string, username)
+
+  void play(Ts... x) override {
+    if (this->md5_url_.has_value()) {
+      this->parent_->set_md5_url(this->md5_url_.value(x...));
+    }
+    if (this->md5_.has_value()) {
+      this->parent_->set_md5(this->md5_.value(x...));
+    }
+    if (this->password_.has_value()) {
+      this->parent_->set_password(this->password_.value(x...));
+    }
+    if (this->username_.has_value()) {
+      this->parent_->set_username(this->username_.value(x...));
+    }
+    this->parent_->set_url(this->url_.value(x...));
+
+    this->parent_->flash();
+    // Normally never reached due to reboot
+  }
+
+ protected:
+  OtaHttpRequestComponent *parent_;
+};
+
+}  // namespace http_request
+}  // namespace esphome

esphome/components/http_request/ota/ota_http_request.cpp

@@ -0,0 +1,293 @@
+#include "ota_http_request.h"
+#include "watchdog.h"
+
+#include "esphome/core/application.h"
+#include "esphome/core/defines.h"
+#include "esphome/core/log.h"
+
+#include "esphome/components/md5/md5.h"
+#include "esphome/components/ota/ota_backend_arduino_esp32.h"
+#include "esphome/components/ota/ota_backend_arduino_esp8266.h"
+#include "esphome/components/ota/ota_backend_arduino_rp2040.h"
+#include "esphome/components/ota/ota_backend_esp_idf.h"
+#include "esphome/components/ota/ota_backend.h"
+
+namespace esphome {
+namespace http_request {
+
+void OtaHttpRequestComponent::setup() {
+#ifdef USE_OTA_STATE_CALLBACK
+  ota::register_ota_platform(this);
+#endif
+}
+
+void OtaHttpRequestComponent::dump_config() {
+  ESP_LOGCONFIG(TAG, "Over-The-Air updates via HTTP request:");
+  ESP_LOGCONFIG(TAG, "  Timeout: %llus", this->timeout_ / 1000);
+#ifdef USE_ESP8266
+#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS
+  ESP_LOGCONFIG(TAG, "  ESP8266 SSL support: No");
+#else
+  ESP_LOGCONFIG(TAG, "  ESP8266 SSL support: Yes");
+#endif
+#endif
+#ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
+  ESP_LOGCONFIG(TAG, "  TLS server verification: Yes");
+#else
+  ESP_LOGCONFIG(TAG, "  TLS server verification: No");
+#endif
+#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT
+  ESP_LOGCONFIG(TAG, "  Watchdog timeout: %ds", USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT / 1000);
+#endif
+};
+
+void OtaHttpRequestComponent::set_md5_url(const std::string &url) {
+  if (!this->validate_url_(url)) {
+    this->md5_url_.clear();  // URL was not valid; prevent flashing until it is
+    return;
+  }
+  this->md5_url_ = url;
+  this->md5_expected_.clear();  // to be retrieved later
+}
+
+void OtaHttpRequestComponent::set_url(const std::string &url) {
+  if (!this->validate_url_(url)) {
+    this->url_.clear();  // URL was not valid; prevent flashing until it is
+    return;
+  }
+  this->url_ = url;
+}
+
+bool OtaHttpRequestComponent::check_status() {
+  // status can be -1, or HTTP status code
+  if (this->status_ < 100) {
+    ESP_LOGE(TAG, "HTTP server did not respond (error %d)", this->status_);
+    return false;
+  }
+  if (this->status_ >= 310) {
+    ESP_LOGE(TAG, "HTTP error %d", this->status_);
+    return false;
+  }
+  ESP_LOGV(TAG, "HTTP status %d", this->status_);
+  return true;
+}
+
+void OtaHttpRequestComponent::flash() {
+  if (this->url_.empty()) {
+    ESP_LOGE(TAG, "URL not set; cannot start update");
+    return;
+  }
+
+  ESP_LOGI(TAG, "Starting update...");
+#ifdef USE_OTA_STATE_CALLBACK
+  this->state_callback_.call(ota::OTA_STARTED, 0.0f, 0);
+#endif
+
+  auto ota_status = this->do_ota_();
+
+  switch (ota_status) {
+    case ota::OTA_RESPONSE_OK:
+#ifdef USE_OTA_STATE_CALLBACK
+      this->state_callback_.call(ota::OTA_COMPLETED, 100.0f, ota_status);
+#endif
+      delay(10);
+      App.safe_reboot();
+      break;
+
+    default:
+#ifdef USE_OTA_STATE_CALLBACK
+      this->state_callback_.call(ota::OTA_ERROR, 0.0f, ota_status);
+#endif
+      this->md5_computed_.clear();  // will be reset at next attempt
+      this->md5_expected_.clear();  // will be reset at next attempt
+      break;
+  }
+}
+
+void OtaHttpRequestComponent::cleanup_(std::unique_ptr<ota::OTABackend> backend) {
+  if (this->update_started_) {
+    ESP_LOGV(TAG, "Aborting OTA backend");
+    backend->abort();
+  }
+  ESP_LOGV(TAG, "Aborting HTTP connection");
+  this->http_end();
+};
+
+uint8_t OtaHttpRequestComponent::do_ota_() {
+  uint8_t buf[this->http_recv_buffer_ + 1];
+  uint32_t last_progress = 0;
+  uint32_t update_start_time = millis();
+  md5::MD5Digest md5_receive;
+  std::unique_ptr<char[]> md5_receive_str(new char[33]);
+
+  if (this->md5_expected_.empty() && !this->http_get_md5_()) {
+    return OTA_MD5_INVALID;
+  }
+
+  ESP_LOGD(TAG, "MD5 expected: %s", this->md5_expected_.c_str());
+
+  auto url_with_auth = this->get_url_with_auth_(this->url_);
+  if (url_with_auth.empty()) {
+    return OTA_BAD_URL;
+  }
+  ESP_LOGVV(TAG, "url_with_auth: %s", url_with_auth.c_str());
+  ESP_LOGI(TAG, "Connecting to: %s", this->url_.c_str());
+  this->http_init(url_with_auth);
+  if (!this->check_status()) {
+    this->http_end();
+    return OTA_CONNECTION_ERROR;
+  }
+
+  // we will compute MD5 on the fly for verification -- Arduino OTA seems to ignore it
+  md5_receive.init();
+  ESP_LOGV(TAG, "MD5Digest initialized");
+
+  ESP_LOGV(TAG, "OTA backend begin");
+  auto backend = ota::make_ota_backend();
+  auto error_code = backend->begin(this->body_length_);
+  if (error_code != ota::OTA_RESPONSE_OK) {
+    ESP_LOGW(TAG, "backend->begin error: %d", error_code);
+    this->cleanup_(std::move(backend));
+    return error_code;
+  }
+
+  this->bytes_read_ = 0;
+  while (this->bytes_read_ < this->body_length_) {
+    // read a maximum of chunk_size bytes into buf. (real read size returned)
+    int bufsize = this->http_read(buf, this->http_recv_buffer_);
+    ESP_LOGVV(TAG, "bytes_read_ = %u, body_length_ = %u, bufsize = %i", this->bytes_read_, this->body_length_, bufsize);
+
+    // feed watchdog and give other tasks a chance to run
+    App.feed_wdt();
+    yield();
+
+    if (bufsize < 0) {
+      ESP_LOGE(TAG, "Stream closed");
+      this->cleanup_(std::move(backend));
+      return OTA_CONNECTION_ERROR;
+    } else if (bufsize > 0 && bufsize <= this->http_recv_buffer_) {
+      // add read bytes to MD5
+      md5_receive.add(buf, bufsize);
+
+      // write bytes to OTA backend
+      this->update_started_ = true;
+      error_code = backend->write(buf, bufsize);
+      if (error_code != ota::OTA_RESPONSE_OK) {
+        // error code explanation available at
+        // https://github.com/esphome/esphome/blob/dev/esphome/components/ota/ota_backend.h
+        ESP_LOGE(TAG, "Error code (%02X) writing binary data to flash at offset %d and size %d", error_code,
+                 this->bytes_read_ - bufsize, this->body_length_);
+        this->cleanup_(std::move(backend));
+        return error_code;
+      }
+    }
+
+    uint32_t now = millis();
+    if ((now - last_progress > 1000) or (this->bytes_read_ == this->body_length_)) {
+      last_progress = now;
+      float percentage = this->bytes_read_ * 100.0f / this->body_length_;
+      ESP_LOGD(TAG, "Progress: %0.1f%%", percentage);
+#ifdef USE_OTA_STATE_CALLBACK
+      this->state_callback_.call(ota::OTA_IN_PROGRESS, percentage, 0);
+#endif
+    }
+  }  // while
+
+  ESP_LOGI(TAG, "Done in %.0f seconds", float(millis() - update_start_time) / 1000);
+
+  // verify MD5 is as expected and act accordingly
+  md5_receive.calculate();
+  md5_receive.get_hex(md5_receive_str.get());
+  this->md5_computed_ = md5_receive_str.get();
+  if (strncmp(this->md5_computed_.c_str(), this->md5_expected_.c_str(), MD5_SIZE) != 0) {
+    ESP_LOGE(TAG, "MD5 computed: %s - Aborting due to MD5 mismatch", this->md5_computed_.c_str());
+    this->cleanup_(std::move(backend));
+    return ota::OTA_RESPONSE_ERROR_MD5_MISMATCH;
+  } else {
+    backend->set_update_md5(md5_receive_str.get());
+  }
+
+  this->http_end();
+
+  // feed watchdog and give other tasks a chance to run
+  App.feed_wdt();
+  yield();
+  delay(100);  // NOLINT
+
+  error_code = backend->end();
+  if (error_code != ota::OTA_RESPONSE_OK) {
+    ESP_LOGW(TAG, "Error ending update! error_code: %d", error_code);
+    this->cleanup_(std::move(backend));
+    return error_code;
+  }
+
+  ESP_LOGI(TAG, "Update complete");
+  return ota::OTA_RESPONSE_OK;
+}
+
+std::string OtaHttpRequestComponent::get_url_with_auth_(const std::string &url) {
+  if (this->username_.empty() || this->password_.empty()) {
+    return url;
+  }
+
+  auto start_char = url.find("://");
+  if ((start_char == std::string::npos) || (start_char < 4)) {
+    ESP_LOGE(TAG, "Incorrect URL prefix");
+    return {};
+  }
+
+  ESP_LOGD(TAG, "Using basic HTTP authentication");
+
+  start_char += 3;  // skip '://' characters
+  auto url_with_auth =
+      url.substr(0, start_char) + this->username_ + ":" + this->password_ + "@" + url.substr(start_char);
+  return url_with_auth;
+}
+
+bool OtaHttpRequestComponent::http_get_md5_() {
+  if (this->md5_url_.empty()) {
+    return false;
+  }
+
+  auto url_with_auth = this->get_url_with_auth_(this->md5_url_);
+  if (url_with_auth.empty()) {
+    return false;
+  }
+
+  ESP_LOGVV(TAG, "url_with_auth: %s", url_with_auth.c_str());
+  ESP_LOGI(TAG, "Connecting to: %s", this->md5_url_.c_str());
+  this->http_init(url_with_auth);
+  if (!this->check_status()) {
+    this->http_end();
+    return false;
+  }
+  int length = this->body_length_;
+  if (length < 0) {
+    this->http_end();
+    return false;
+  }
+  if (length < MD5_SIZE) {
+    ESP_LOGE(TAG, "MD5 file must be %u bytes; %u bytes reported by HTTP server. Aborting", MD5_SIZE,
+             this->body_length_);
+    this->http_end();
+    return false;
+  }
+
+  this->bytes_read_ = 0;
+  this->md5_expected_.resize(MD5_SIZE);
+  auto read_len = this->http_read((uint8_t *) this->md5_expected_.data(), MD5_SIZE);
+  this->http_end();
+
+  return read_len == MD5_SIZE;
+}
+
+bool OtaHttpRequestComponent::validate_url_(const std::string &url) {
+  if ((url.length() < 8) || (url.find("http") != 0) || (url.find("://") == std::string::npos)) {
+    ESP_LOGE(TAG, "URL is invalid and/or must be prefixed with 'http://' or 'https://'");
+    return false;
+  }
+  return true;
+}
+
+}  // namespace http_request
+}  // namespace esphome

esphome/components/http_request/ota/ota_http_request.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/defines.h"
+#include "esphome/components/ota/ota_backend.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace esphome {
+namespace http_request {
+
+static const char *const TAG = "http_request.ota";
+static const uint8_t MD5_SIZE = 32;
+
+enum OtaHttpRequestError : uint8_t {
+  OTA_MD5_INVALID = 0x10,
+  OTA_BAD_URL = 0x11,
+  OTA_CONNECTION_ERROR = 0x12,
+};
+
+class OtaHttpRequestComponent : public ota::OTAComponent {
+ public:
+  void setup() override;
+  void dump_config() override;
+  float get_setup_priority() const override { return setup_priority::AFTER_WIFI; }
+
+  void set_md5_url(const std::string &md5_url);
+  void set_md5(const std::string &md5) { this->md5_expected_ = md5; }
+  void set_password(const std::string &password) { this->password_ = password; }
+  void set_timeout(const uint64_t timeout) { this->timeout_ = timeout; }
+  void set_url(const std::string &url);
+  void set_username(const std::string &username) { this->username_ = username; }
+
+  std::string md5_computed() { return this->md5_computed_; }
+  std::string md5_expected() { return this->md5_expected_; }
+
+  bool check_status();
+
+  void flash();
+
+  virtual void http_init(const std::string &url){};
+  virtual int http_read(uint8_t *buf, size_t len) { return 0; };
+  virtual void http_end(){};
+
+ protected:
+  void cleanup_(std::unique_ptr<ota::OTABackend> backend);
+  uint8_t do_ota_();
+  std::string get_url_with_auth_(const std::string &url);
+  bool http_get_md5_();
+  bool secure_() { return this->url_.find("https:") != std::string::npos; };
+  bool validate_url_(const std::string &url);
+
+  std::string md5_computed_{};
+  std::string md5_expected_{};
+  std::string md5_url_{};
+  std::string password_{};
+  std::string username_{};
+  std::string url_{};
+  size_t body_length_ = 0;
+  size_t bytes_read_ = 0;
+  int status_ = -1;
+  uint64_t timeout_ = 0;
+  bool update_started_ = false;
+  const uint16_t http_recv_buffer_ = 256;      // the firmware GET chunk size
+  const uint16_t max_http_recv_buffer_ = 512;  // internal max http buffer size must be > HTTP_RECV_BUFFER_ (TLS
+                                               // overhead) and must be a power of two from 512 to 4096
+};
+
+}  // namespace http_request
+}  // namespace esphome

esphome/components/http_request/ota/ota_http_request_arduino.cpp

@@ -0,0 +1,134 @@
+#include "ota_http_request.h"
+#include "watchdog.h"
+
+#ifdef USE_ARDUINO
+#include "ota_http_request_arduino.h"
+#include "esphome/core/defines.h"
+#include "esphome/core/log.h"
+#include "esphome/core/application.h"
+#include "esphome/components/network/util.h"
+#include "esphome/components/md5/md5.h"
+
+namespace esphome {
+namespace http_request {
+
+struct Header {
+  const char *name;
+  const char *value;
+};
+
+void OtaHttpRequestComponentArduino::http_init(const std::string &url) {
+  const char *header_keys[] = {"Content-Length", "Content-Type"};
+  const size_t header_count = sizeof(header_keys) / sizeof(header_keys[0]);
+  watchdog::WatchdogManager wdts;
+
+#ifdef USE_ESP8266
+  if (this->stream_ptr_ == nullptr && this->set_stream_ptr_()) {
+    ESP_LOGE(TAG, "Unable to set client");
+    return;
+  }
+#endif  // USE_ESP8266
+
+#ifdef USE_RP2040
+  this->client_.setInsecure();
+#endif
+
+  App.feed_wdt();
+
+#if defined(USE_ESP32) || defined(USE_RP2040)
+  this->status_ = this->client_.begin(url.c_str());
+#endif
+#ifdef USE_ESP8266
+  this->client_.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
+  this->status_ = this->client_.begin(*this->stream_ptr_, url.c_str());
+#endif
+
+  if (!this->status_) {
+    this->client_.end();
+    return;
+  }
+
+  this->client_.setReuse(true);
+
+  // returned needed headers must be collected before the requests
+  this->client_.collectHeaders(header_keys, header_count);
+
+  // HTTP GET
+  this->status_ = this->client_.GET();
+
+  this->body_length_ = (size_t) this->client_.getSize();
+
+#if defined(USE_ESP32) || defined(USE_RP2040)
+  if (this->stream_ptr_ == nullptr) {
+    this->set_stream_ptr_();
+  }
+#endif
+}
+
+int OtaHttpRequestComponentArduino::http_read(uint8_t *buf, const size_t max_len) {
+#ifdef USE_ESP8266
+#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 1, 0)  // && USE_ARDUINO_VERSION_CODE < VERSION_CODE(?, ?, ?)
+  if (!this->secure_()) {
+    ESP_LOGW(TAG, "Using HTTP on Arduino version >= 3.1 is **very** slow. Consider setting framework version to 3.0.2 "
+                  "in your YAML, or use HTTPS");
+  }
+#endif  // USE_ARDUINO_VERSION_CODE
+#endif  // USE_ESP8266
+
+  watchdog::WatchdogManager wdts;
+
+  // Since arduino8266 >= 3.1 using this->stream_ptr_ is broken (https://github.com/esp8266/Arduino/issues/9035)
+  WiFiClient *stream_ptr = this->client_.getStreamPtr();
+  if (stream_ptr == nullptr) {
+    ESP_LOGE(TAG, "Stream pointer vanished!");
+    return -1;
+  }
+
+  int available_data = stream_ptr->available();
+  int bufsize = std::min((int) max_len, available_data);
+  if (bufsize > 0) {
+    stream_ptr->readBytes(buf, bufsize);
+    this->bytes_read_ += bufsize;
+    buf[bufsize] = '\0';  // not fed to ota
+  }
+
+  return bufsize;
+}
+
+void OtaHttpRequestComponentArduino::http_end() {
+  watchdog::WatchdogManager wdts;
+  this->client_.end();
+}
+
+int OtaHttpRequestComponentArduino::set_stream_ptr_() {
+#ifdef USE_ESP8266
+#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS
+  if (this->secure_()) {
+    ESP_LOGV(TAG, "ESP8266 HTTPS connection with WiFiClientSecure");
+    this->stream_ptr_ = std::make_unique<WiFiClientSecure>();
+    WiFiClientSecure *secure_client = static_cast<WiFiClientSecure *>(this->stream_ptr_.get());
+    secure_client->setBufferSizes(this->max_http_recv_buffer_, 512);
+    secure_client->setInsecure();
+  } else {
+    this->stream_ptr_ = std::make_unique<WiFiClient>();
+  }
+#else
+  ESP_LOGV(TAG, "ESP8266 HTTP connection with WiFiClient");
+  if (this->secure_()) {
+    ESP_LOGE(TAG, "Can't use HTTPS connection with esp8266_disable_ssl_support");
+    return -1;
+  }
+  this->stream_ptr_ = std::make_unique<WiFiClient>();
+#endif  // USE_HTTP_REQUEST_ESP8266_HTTPS
+#endif  // USE_ESP8266
+
+#if defined(USE_ESP32) || defined(USE_RP2040)
+  this->stream_ptr_ = std::unique_ptr<WiFiClient>(this->client_.getStreamPtr());
+#endif
+  return 0;
+}
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_ARDUINO

esphome/components/http_request/ota/ota_http_request_arduino.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "ota_http_request.h"
+
+#ifdef USE_ARDUINO
+#include "esphome/core/automation.h"
+#include "esphome/core/component.h"
+#include "esphome/core/defines.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#if defined(USE_ESP32) || defined(USE_RP2040)
+#include <HTTPClient.h>
+#endif
+#ifdef USE_ESP8266
+#include <ESP8266HTTPClient.h>
+#ifdef USE_HTTP_REQUEST_ESP8266_HTTPS
+#include <WiFiClientSecure.h>
+#endif
+#endif
+
+namespace esphome {
+namespace http_request {
+
+class OtaHttpRequestComponentArduino : public OtaHttpRequestComponent {
+ public:
+  void http_init(const std::string &url) override;
+  int http_read(uint8_t *buf, size_t len) override;
+  void http_end() override;
+
+ protected:
+  int set_stream_ptr_();
+  HTTPClient client_{};
+  std::unique_ptr<WiFiClient> stream_ptr_;
+};
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_ARDUINO

esphome/components/http_request/ota/ota_http_request_idf.cpp

@@ -0,0 +1,86 @@
+#include "ota_http_request_idf.h"
+#include "watchdog.h"
+
+#ifdef USE_ESP_IDF
+#include "esphome/core/application.h"
+#include "esphome/core/defines.h"
+#include "esphome/core/log.h"
+#include "esphome/components/md5/md5.h"
+#include "esphome/components/network/util.h"
+
+#include "esp_event.h"
+#include "esp_http_client.h"
+#include "esp_idf_version.h"
+#include "esp_log.h"
+#include "esp_netif.h"
+#include "esp_system.h"
+#include "esp_task_wdt.h"
+#include "esp_tls.h"
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "nvs_flash.h"
+
+#include <cctype>
+#include <cinttypes>
+#include <cstdlib>
+#include <cstring>
+#include <sys/param.h>
+#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
+#include "esp_crt_bundle.h"
+#endif
+
+namespace esphome {
+namespace http_request {
+
+void OtaHttpRequestComponentIDF::http_init(const std::string &url) {
+  App.feed_wdt();
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+  esp_http_client_config_t config = {nullptr};
+  config.url = url.c_str();
+  config.method = HTTP_METHOD_GET;
+  config.timeout_ms = (int) this->timeout_;
+  config.buffer_size = this->max_http_recv_buffer_;
+  config.auth_type = HTTP_AUTH_TYPE_BASIC;
+  config.max_authorization_retries = -1;
+#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
+  if (this->secure_()) {
+    config.crt_bundle_attach = esp_crt_bundle_attach;
+  }
+#endif
+#pragma GCC diagnostic pop
+
+  watchdog::WatchdogManager wdts;
+  this->client_ = esp_http_client_init(&config);
+  if ((this->status_ = esp_http_client_open(this->client_, 0)) == ESP_OK) {
+    this->body_length_ = esp_http_client_fetch_headers(this->client_);
+    this->status_ = esp_http_client_get_status_code(this->client_);
+  }
+}
+
+int OtaHttpRequestComponentIDF::http_read(uint8_t *buf, const size_t max_len) {
+  watchdog::WatchdogManager wdts;
+  int bufsize = std::min(max_len, this->body_length_ - this->bytes_read_);
+
+  App.feed_wdt();
+  int read_len = esp_http_client_read(this->client_, (char *) buf, bufsize);
+  if (read_len > 0) {
+    this->bytes_read_ += bufsize;
+    buf[bufsize] = '\0';  // not fed to ota
+  }
+
+  return read_len;
+}
+
+void OtaHttpRequestComponentIDF::http_end() {
+  watchdog::WatchdogManager wdts;
+
+  esp_http_client_close(this->client_);
+  esp_http_client_cleanup(this->client_);
+}
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_ESP_IDF

esphome/components/http_request/ota/ota_http_request_idf.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "ota_http_request.h"
+
+#ifdef USE_ESP_IDF
+#include "esp_http_client.h"
+
+namespace esphome {
+namespace http_request {
+
+class OtaHttpRequestComponentIDF : public OtaHttpRequestComponent {
+ public:
+  void http_init(const std::string &url) override;
+  int http_read(uint8_t *buf, size_t len) override;
+  void http_end() override;
+
+ protected:
+  esp_http_client_handle_t client_{};
+};
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_ESP_IDF

esphome/components/http_request/ota/watchdog.cpp

@@ -0,0 +1,71 @@
+#include "watchdog.h"
+
+#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT
+
+#include "esphome/core/application.h"
+#include "esphome/core/log.h"
+
+#include <cinttypes>
+#include <cstdint>
+#ifdef USE_ESP32
+#include "esp_idf_version.h"
+#include "esp_task_wdt.h"
+#endif
+#ifdef USE_RP2040
+#include "hardware/watchdog.h"
+#include "pico/stdlib.h"
+#endif
+
+namespace esphome {
+namespace http_request {
+namespace watchdog {
+
+static const char *const TAG = "watchdog.http_request.ota";
+
+WatchdogManager::WatchdogManager() {
+  this->saved_timeout_ms_ = this->get_timeout_();
+  this->set_timeout_(USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT);
+}
+
+WatchdogManager::~WatchdogManager() { this->set_timeout_(this->saved_timeout_ms_); }
+
+void WatchdogManager::set_timeout_(uint32_t timeout_ms) {
+  ESP_LOGV(TAG, "Adjusting WDT to %" PRIu32 "ms", timeout_ms);
+#ifdef USE_ESP32
+#if ESP_IDF_VERSION_MAJOR >= 5
+  esp_task_wdt_config_t wdt_config = {
+      .timeout_ms = timeout_ms,
+      .idle_core_mask = 0x03,
+      .trigger_panic = true,
+  };
+  esp_task_wdt_reconfigure(&wdt_config);
+#else
+  esp_task_wdt_init(timeout_ms, true);
+#endif  // ESP_IDF_VERSION_MAJOR
+#endif  // USE_ESP32
+
+#ifdef USE_RP2040
+  watchdog_enable(timeout_ms, true);
+#endif
+}
+
+uint32_t WatchdogManager::get_timeout_() {
+  uint32_t timeout_ms = 0;
+
+#ifdef USE_ESP32
+  timeout_ms = (uint32_t) CONFIG_ESP_TASK_WDT_TIMEOUT_S * 1000;
+#endif  // USE_ESP32
+
+#ifdef USE_RP2040
+  timeout_ms = watchdog_get_count() / 1000;
+#endif
+
+  ESP_LOGVV(TAG, "get_timeout: %" PRIu32 "ms", timeout_ms);
+
+  return timeout_ms;
+}
+
+}  // namespace watchdog
+}  // namespace http_request
+}  // namespace esphome
+#endif

esphome/components/http_request/ota/watchdog.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+
+#include <cstdint>
+
+namespace esphome {
+namespace http_request {
+namespace watchdog {
+
+class WatchdogManager {
+#ifdef USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT
+ public:
+  WatchdogManager();
+  ~WatchdogManager();
+
+ private:
+  uint32_t get_timeout_();
+  void set_timeout_(uint32_t timeout_ms);
+
+  uint32_t saved_timeout_ms_{0};
+#endif
+};
+
+}  // namespace watchdog
+}  // namespace http_request
+}  // namespace esphome

esphome/components/logger/__init__.py

@@ -112,11 +112,18 @@ HARDWARE_UART_TO_UART_SELECTION = {
 }
 
 HARDWARE_UART_TO_SERIAL = {
+    PLATFORM_ESP8266: {
         UART0: cg.global_ns.Serial,
         UART0_SWAP: cg.global_ns.Serial,
         UART1: cg.global_ns.Serial1,
         UART2: cg.global_ns.Serial2,
         DEFAULT: cg.global_ns.Serial,
+    },
+    PLATFORM_RP2040: {
+        UART0: cg.global_ns.Serial1,
+        UART1: cg.global_ns.Serial2,
+        USB_CDC: cg.global_ns.Serial,
+    },
 }
 
 is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
@@ -244,8 +251,14 @@ async def to_code(config):
     is_at_least_very_verbose = this_severity >= very_verbose_severity
     has_serial_logging = baud_rate != 0
 
-    if CORE.is_esp8266 and has_serial_logging and is_at_least_verbose:
-        debug_serial_port = HARDWARE_UART_TO_SERIAL[config.get(CONF_HARDWARE_UART)]
+    if (
+        (CORE.is_esp8266 or CORE.is_rp2040)
+        and has_serial_logging
+        and is_at_least_verbose
+    ):
+        debug_serial_port = HARDWARE_UART_TO_SERIAL[CORE.target_platform][
+            config.get(CONF_HARDWARE_UART)
+        ]
         cg.add_build_flag(f"-DDEBUG_ESP_PORT={debug_serial_port}")
         cg.add_build_flag("-DLWIP_DEBUG")
         DEBUG_COMPONENTS = {

esphome/config_validation.py

@@ -1949,13 +1949,13 @@ def url(value):
     except ValueError as e:
         raise Invalid("Not a valid URL") from e
 
-    if not parsed.scheme or not parsed.netloc:
-        raise Invalid("Expected a URL scheme and host")
+    if parsed.scheme and parsed.netloc or parsed.scheme == "file":
         return parsed.geturl()
+    raise Invalid("Expected a file scheme or a URL scheme with host")
 
 
 def git_ref(value):
-    if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
+    if re.match(r"[a-zA-Z0-9_./-]+", value) is None:
         raise Invalid("Not a valid git ref")
     return value
 

esphome/core/defines.h

@@ -33,6 +33,7 @@
 #define USE_GRAPH
 #define USE_GRAPHICAL_DISPLAY_MENU
 #define USE_HOMEASSISTANT_TIME
+#define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000  // NOLINT
 #define USE_JSON
 #define USE_LIGHT
 #define USE_LOCK
@@ -100,6 +101,14 @@
 #ifdef USE_ESP_IDF
 #define USE_ESP_IDF_VERSION_CODE VERSION_CODE(4, 4, 2)
 #endif
+
+#if defined(USE_ESP32_VARIANT_ESP32S2)
+#define USE_LOGGER_USB_CDC
+#elif defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3) || \
+    defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32H2)
+#define USE_LOGGER_USB_CDC
+#define USE_LOGGER_USB_SERIAL_JTAG
+#endif
 #endif
 
 // ESP8266-specific feature flags
@@ -122,6 +131,7 @@
 
 #ifdef USE_RP2040
 #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 3, 0)
+#define USE_LOGGER_USB_CDC
 #define USE_SOCKET_IMPL_LWIP_TCP
 #define USE_SPI
 #endif

tests/components/http_request/common_http_request.yaml

@@ -28,10 +28,6 @@ esphome:
           body: "Some data"
           verify_ssl: false
 
-wifi:
-  ssid: MySSID
-  password: password1
-
 http_request:
   useragent: esphome/tagreader
   timeout: 10s

tests/components/http_request/common_ota.yaml

@@ -0,0 +1,36 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+ota:
+  - platform: http_request
+    verify_ssl: ${verify_ssl}
+    on_begin:
+      then:
+        - logger.log: "OTA start"
+    on_progress:
+      then:
+        - logger.log:
+            format: "OTA progress %0.1f%%"
+            args: ["x"]
+    on_end:
+      then:
+        - logger.log: "OTA end"
+    on_error:
+      then:
+        - logger.log:
+            format: "OTA update error %d"
+            args: ["x"]
+    on_state_change:
+      then:
+        lambda: 'ESP_LOGD("ota", "State %d", state);'
+
+button:
+  - platform: template
+    name: Firmware update
+    on_press:
+      then:
+        - ota_http_request.flash:
+            md5_url: http://my.ha.net:8123/local/esphome/firmware.md5
+            url: http://my.ha.net:8123/local/esphome/firmware.bin
+        - logger.log: "This message should be not displayed (reboot)"

tests/components/http_request/test-nossl.esp8266.yaml

@@ -0,0 +1,38 @@
+<<: !include common_http_request.yaml
+
+wifi:
+  ssid: MySSID
+  password: password1
+
+ota:
+  - platform: http_request
+    esp8266_disable_ssl_support: true
+    on_begin:
+      then:
+        - logger.log: "OTA start"
+    on_progress:
+      then:
+        - logger.log:
+            format: "OTA progress %0.1f%%"
+            args: ["x"]
+    on_end:
+      then:
+        - logger.log: "OTA end"
+    on_error:
+      then:
+        - logger.log:
+            format: "OTA update error %d"
+            args: ["x"]
+    on_state_change:
+      then:
+        lambda: 'ESP_LOGD("ota", "State %d", state);'
+
+button:
+  - platform: template
+    name: Firmware update
+    on_press:
+      then:
+        - ota_http_request.flash:
+            md5_url: http://my.ha.net:8123/local/esphome/firmware.md5
+            url: http://my.ha.net:8123/local/esphome/firmware.bin
+        - logger.log: "This message should be not displayed (reboot)"

tests/components/http_request/test.esp32-c3-idf.yaml

@@ -0,0 +1,4 @@
+substitutions:
+  verify_ssl: "true"
+
+<<: !include common_ota.yaml

tests/components/http_request/test.esp32-c3.yaml

@@ -1,2 +1,5 @@
-packages:
-  common: !include common.yaml
+substitutions:
+  verify_ssl: "false"
+
+<<: !include common_http_request.yaml
+<<: !include common_ota.yaml

tests/components/http_request/test.esp32-idf.yaml

@@ -0,0 +1,4 @@
+substitutions:
+  verify_ssl: "true"
+
+<<: !include common_ota.yaml

tests/components/http_request/test.esp32.yaml

@@ -1,2 +1,5 @@
-packages:
-  common: !include common.yaml
+substitutions:
+  verify_ssl: "false"
+
+<<: !include common_http_request.yaml
+<<: !include common_ota.yaml

tests/components/http_request/test.esp8266.yaml

@@ -1,2 +1,5 @@
-packages:
-  common: !include common.yaml
+substitutions:
+  verify_ssl: "false"
+
+<<: !include common_http_request.yaml
+<<: !include common_ota.yaml

tests/components/http_request/test.rp2040.yaml

@@ -0,0 +1,4 @@
+substitutions:
+  verify_ssl: "false"
+
+<<: !include common_ota.yaml