From 4ac49907ca339345f934467ff73f56654b7afc49 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Wed, 15 Nov 2023 21:29:50 -0600
Subject: [PATCH 01/25] Add more VA triggers (#5762)

---
 .../components/voice_assistant/__init__.py    | 61 ++++++++++++++++---
 .../voice_assistant/voice_assistant.cpp       | 39 ++++++++----
 .../voice_assistant/voice_assistant.h         | 16 +++--
 3 files changed, 91 insertions(+), 25 deletions(-)

diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py
index 3270b9f370..5715604605 100644
--- a/esphome/components/voice_assistant/__init__.py
+++ b/esphome/components/voice_assistant/__init__.py
@@ -18,20 +18,25 @@ DEPENDENCIES = ["api", "microphone"]
 
 CODEOWNERS = ["@jesserockz"]
 
-CONF_SILENCE_DETECTION = "silence_detection"
-CONF_ON_LISTENING = "on_listening"
-CONF_ON_START = "on_start"
-CONF_ON_WAKE_WORD_DETECTED = "on_wake_word_detected"
-CONF_ON_STT_END = "on_stt_end"
-CONF_ON_TTS_START = "on_tts_start"
-CONF_ON_TTS_END = "on_tts_end"
 CONF_ON_END = "on_end"
 CONF_ON_ERROR = "on_error"
+CONF_ON_INTENT_END = "on_intent_end"
+CONF_ON_INTENT_START = "on_intent_start"
+CONF_ON_LISTENING = "on_listening"
+CONF_ON_START = "on_start"
+CONF_ON_STT_END = "on_stt_end"
+CONF_ON_STT_VAD_END = "on_stt_vad_end"
+CONF_ON_STT_VAD_START = "on_stt_vad_start"
+CONF_ON_TTS_END = "on_tts_end"
+CONF_ON_TTS_START = "on_tts_start"
+CONF_ON_WAKE_WORD_DETECTED = "on_wake_word_detected"
+
+CONF_SILENCE_DETECTION = "silence_detection"
 CONF_USE_WAKE_WORD = "use_wake_word"
 CONF_VAD_THRESHOLD = "vad_threshold"
 
-CONF_NOISE_SUPPRESSION_LEVEL = "noise_suppression_level"
 CONF_AUTO_GAIN = "auto_gain"
+CONF_NOISE_SUPPRESSION_LEVEL = "noise_suppression_level"
 CONF_VOLUME_MULTIPLIER = "volume_multiplier"
 
 
@@ -88,6 +93,18 @@ CONFIG_SCHEMA = cv.All(
             cv.Optional(CONF_ON_CLIENT_DISCONNECTED): automation.validate_automation(
                 single=True
             ),
+            cv.Optional(CONF_ON_INTENT_START): automation.validate_automation(
+                single=True
+            ),
+            cv.Optional(CONF_ON_INTENT_END): automation.validate_automation(
+                single=True
+            ),
+            cv.Optional(CONF_ON_STT_VAD_START): automation.validate_automation(
+                single=True
+            ),
+            cv.Optional(CONF_ON_STT_VAD_END): automation.validate_automation(
+                single=True
+            ),
         }
     ).extend(cv.COMPONENT_SCHEMA),
 )
@@ -177,6 +194,34 @@ async def to_code(config):
             config[CONF_ON_CLIENT_DISCONNECTED],
         )
 
+    if CONF_ON_INTENT_START in config:
+        await automation.build_automation(
+            var.get_intent_start_trigger(),
+            [],
+            config[CONF_ON_INTENT_START],
+        )
+
+    if CONF_ON_INTENT_END in config:
+        await automation.build_automation(
+            var.get_intent_end_trigger(),
+            [],
+            config[CONF_ON_INTENT_END],
+        )
+
+    if CONF_ON_STT_VAD_START in config:
+        await automation.build_automation(
+            var.get_stt_vad_start_trigger(),
+            [],
+            config[CONF_ON_STT_VAD_START],
+        )
+
+    if CONF_ON_STT_VAD_END in config:
+        await automation.build_automation(
+            var.get_stt_vad_end_trigger(),
+            [],
+            config[CONF_ON_STT_VAD_END],
+        )
+
     cg.add_define("USE_VOICE_ASSISTANT")
 
 
diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index fc5dd6e4e4..7ebbe762b3 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -31,7 +31,7 @@ void VoiceAssistant::setup() {
 
   this->socket_ = socket::socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
   if (socket_ == nullptr) {
-    ESP_LOGW(TAG, "Could not create socket.");
+    ESP_LOGW(TAG, "Could not create socket");
     this->mark_failed();
     return;
   }
@@ -69,7 +69,7 @@ void VoiceAssistant::setup() {
     ExternalRAMAllocator<uint8_t> speaker_allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
     this->speaker_buffer_ = speaker_allocator.allocate(SPEAKER_BUFFER_SIZE);
     if (this->speaker_buffer_ == nullptr) {
-      ESP_LOGW(TAG, "Could not allocate speaker buffer.");
+      ESP_LOGW(TAG, "Could not allocate speaker buffer");
       this->mark_failed();
       return;
     }
@@ -79,7 +79,7 @@ void VoiceAssistant::setup() {
   ExternalRAMAllocator<int16_t> allocator(ExternalRAMAllocator<int16_t>::ALLOW_FAILURE);
   this->input_buffer_ = allocator.allocate(INPUT_BUFFER_SIZE);
   if (this->input_buffer_ == nullptr) {
-    ESP_LOGW(TAG, "Could not allocate input buffer.");
+    ESP_LOGW(TAG, "Could not allocate input buffer");
     this->mark_failed();
     return;
   }
@@ -89,7 +89,7 @@ void VoiceAssistant::setup() {
 
   this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t));
   if (this->ring_buffer_ == nullptr) {
-    ESP_LOGW(TAG, "Could not allocate ring buffer.");
+    ESP_LOGW(TAG, "Could not allocate ring buffer");
     this->mark_failed();
     return;
   }
@@ -98,7 +98,7 @@ void VoiceAssistant::setup() {
   ExternalRAMAllocator<uint8_t> send_allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
   this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE);
   if (send_buffer_ == nullptr) {
-    ESP_LOGW(TAG, "Could not allocate send buffer.");
+    ESP_LOGW(TAG, "Could not allocate send buffer");
     this->mark_failed();
     return;
   }
@@ -221,8 +221,8 @@ void VoiceAssistant::loop() {
       msg.audio_settings = audio_settings;
 
       if (this->api_client_ == nullptr || !this->api_client_->send_voice_assistant_request(msg)) {
-        ESP_LOGW(TAG, "Could not request start.");
-        this->error_trigger_->trigger("not-connected", "Could not request start.");
+        ESP_LOGW(TAG, "Could not request start");
+        this->error_trigger_->trigger("not-connected", "Could not request start");
         this->continuous_ = false;
         this->set_state_(State::IDLE, State::IDLE);
         break;
@@ -280,7 +280,7 @@ void VoiceAssistant::loop() {
             this->speaker_buffer_size_ += len;
           }
         } else {
-          ESP_LOGW(TAG, "Receive buffer full.");
+          ESP_LOGW(TAG, "Receive buffer full");
         }
         if (this->speaker_buffer_size_ > 0) {
           size_t written = this->speaker_->play(this->speaker_buffer_, this->speaker_buffer_size_);
@@ -290,7 +290,7 @@ void VoiceAssistant::loop() {
             this->speaker_buffer_index_ -= written;
             this->set_timeout("speaker-timeout", 2000, [this]() { this->speaker_->stop(); });
           } else {
-            ESP_LOGW(TAG, "Speaker buffer full.");
+            ESP_LOGW(TAG, "Speaker buffer full");
           }
         }
         if (this->wait_for_stream_end_) {
@@ -513,7 +513,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_START:
-      ESP_LOGD(TAG, "STT Started");
+      ESP_LOGD(TAG, "STT started");
       this->listening_trigger_->trigger();
       break;
     case api::enums::VOICE_ASSISTANT_STT_END: {
@@ -525,19 +525,24 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         }
       }
       if (text.empty()) {
-        ESP_LOGW(TAG, "No text in STT_END event.");
+        ESP_LOGW(TAG, "No text in STT_END event");
         return;
       }
       ESP_LOGD(TAG, "Speech recognised as: \"%s\"", text.c_str());
       this->stt_end_trigger_->trigger(text);
       break;
     }
+    case api::enums::VOICE_ASSISTANT_INTENT_START:
+      ESP_LOGD(TAG, "Intent started");
+      this->intent_start_trigger_->trigger();
+      break;
     case api::enums::VOICE_ASSISTANT_INTENT_END: {
       for (auto arg : msg.data) {
         if (arg.name == "conversation_id") {
           this->conversation_id_ = std::move(arg.value);
         }
       }
+      this->intent_end_trigger_->trigger();
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_START: {
@@ -548,7 +553,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         }
       }
       if (text.empty()) {
-        ESP_LOGW(TAG, "No text in TTS_START event.");
+        ESP_LOGW(TAG, "No text in TTS_START event");
         return;
       }
       ESP_LOGD(TAG, "Response: \"%s\"", text.c_str());
@@ -566,7 +571,7 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
         }
       }
       if (url.empty()) {
-        ESP_LOGW(TAG, "No url in TTS_END event.");
+        ESP_LOGW(TAG, "No url in TTS_END event");
         return;
       }
       ESP_LOGD(TAG, "Response URL: \"%s\"", url.c_str());
@@ -634,6 +639,14 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
       this->set_state_(State::RESPONSE_FINISHED, State::IDLE);
       break;
     }
+    case api::enums::VOICE_ASSISTANT_STT_VAD_START:
+      ESP_LOGD(TAG, "Starting STT by VAD");
+      this->stt_vad_start_trigger_->trigger();
+      break;
+    case api::enums::VOICE_ASSISTANT_STT_VAD_END:
+      ESP_LOGD(TAG, "STT by VAD end");
+      this->stt_vad_end_trigger_->trigger();
+      break;
     default:
       ESP_LOGD(TAG, "Unhandled event type: %d", msg.event_type);
       break;
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index a265522bca..a985bc4678 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -100,13 +100,17 @@ class VoiceAssistant : public Component {
   void set_auto_gain(uint8_t auto_gain) { this->auto_gain_ = auto_gain; }
   void set_volume_multiplier(float volume_multiplier) { this->volume_multiplier_ = volume_multiplier; }
 
+  Trigger<> *get_intent_end_trigger() const { return this->intent_end_trigger_; }
+  Trigger<> *get_intent_start_trigger() const { return this->intent_start_trigger_; }
   Trigger<> *get_listening_trigger() const { return this->listening_trigger_; }
+  Trigger<> *get_end_trigger() const { return this->end_trigger_; }
   Trigger<> *get_start_trigger() const { return this->start_trigger_; }
+  Trigger<> *get_stt_vad_end_trigger() const { return this->stt_vad_end_trigger_; }
+  Trigger<> *get_stt_vad_start_trigger() const { return this->stt_vad_start_trigger_; }
   Trigger<> *get_wake_word_detected_trigger() const { return this->wake_word_detected_trigger_; }
   Trigger<std::string> *get_stt_end_trigger() const { return this->stt_end_trigger_; }
-  Trigger<std::string> *get_tts_start_trigger() const { return this->tts_start_trigger_; }
   Trigger<std::string> *get_tts_end_trigger() const { return this->tts_end_trigger_; }
-  Trigger<> *get_end_trigger() const { return this->end_trigger_; }
+  Trigger<std::string> *get_tts_start_trigger() const { return this->tts_start_trigger_; }
   Trigger<std::string, std::string> *get_error_trigger() const { return this->error_trigger_; }
 
   Trigger<> *get_client_connected_trigger() const { return this->client_connected_trigger_; }
@@ -124,13 +128,17 @@ class VoiceAssistant : public Component {
   std::unique_ptr<socket::Socket> socket_ = nullptr;
   struct sockaddr_storage dest_addr_;
 
+  Trigger<> *intent_end_trigger_ = new Trigger<>();
+  Trigger<> *intent_start_trigger_ = new Trigger<>();
   Trigger<> *listening_trigger_ = new Trigger<>();
+  Trigger<> *end_trigger_ = new Trigger<>();
   Trigger<> *start_trigger_ = new Trigger<>();
+  Trigger<> *stt_vad_start_trigger_ = new Trigger<>();
+  Trigger<> *stt_vad_end_trigger_ = new Trigger<>();
   Trigger<> *wake_word_detected_trigger_ = new Trigger<>();
   Trigger<std::string> *stt_end_trigger_ = new Trigger<std::string>();
-  Trigger<std::string> *tts_start_trigger_ = new Trigger<std::string>();
   Trigger<std::string> *tts_end_trigger_ = new Trigger<std::string>();
-  Trigger<> *end_trigger_ = new Trigger<>();
+  Trigger<std::string> *tts_start_trigger_ = new Trigger<std::string>();
   Trigger<std::string, std::string> *error_trigger_ = new Trigger<std::string, std::string>();
 
   Trigger<> *client_connected_trigger_ = new Trigger<>();

From 255483de63f9c1fd846cbbdaae98be96df41d6cb Mon Sep 17 00:00:00 2001
From: Mat931 <49403702+Mat931@users.noreply.github.com>
Date: Thu, 16 Nov 2023 07:45:08 +0000
Subject: [PATCH 02/25] Fix MY9231 flicker (#5765)

---
 esphome/components/my9231/my9231.cpp | 26 +++++++++++++++++++++-----
 esphome/components/my9231/my9231.h   |  1 +
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/esphome/components/my9231/my9231.cpp b/esphome/components/my9231/my9231.cpp
index a97587b7be..c511591856 100644
--- a/esphome/components/my9231/my9231.cpp
+++ b/esphome/components/my9231/my9231.cpp
@@ -1,5 +1,6 @@
 #include "my9231.h"
 #include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
 
 namespace esphome {
 namespace my9231 {
@@ -51,7 +52,11 @@ void MY9231OutputComponent::setup() {
       MY9231_CMD_SCATTER_APDM | MY9231_CMD_FREQUENCY_DIVIDE_1 | MY9231_CMD_REACTION_FAST | MY9231_CMD_ONE_SHOT_DISABLE;
   ESP_LOGV(TAG, "  Command: 0x%02X", command);
 
-  this->init_chips_(command);
+  {
+    InterruptLock lock;
+    this->send_dcki_pulses_(32 * this->num_chips_);
+    this->init_chips_(command);
+  }
   ESP_LOGV(TAG, "  Chips initialized.");
 }
 void MY9231OutputComponent::dump_config() {
@@ -66,11 +71,14 @@ void MY9231OutputComponent::loop() {
   if (!this->update_)
     return;
 
-  for (auto pwm_amount : this->pwm_amounts_) {
-    this->write_word_(pwm_amount, this->bit_depth_);
+  {
+    InterruptLock lock;
+    for (auto pwm_amount : this->pwm_amounts_) {
+      this->write_word_(pwm_amount, this->bit_depth_);
+    }
+    // Send 8 DI pulses. After 8 falling edges, the duty data are store.
+    this->send_di_pulses_(8);
   }
-  // Send 8 DI pulses. After 8 falling edges, the duty data are store.
-  this->send_di_pulses_(8);
   this->update_ = false;
 }
 void MY9231OutputComponent::set_channel_value_(uint8_t channel, uint16_t value) {
@@ -92,6 +100,7 @@ void MY9231OutputComponent::init_chips_(uint8_t command) {
   // Send 16 DI pulse. After 14 falling edges, the command data are
   // stored and after 16 falling edges the duty mode is activated.
   this->send_di_pulses_(16);
+  delayMicroseconds(12);
 }
 void MY9231OutputComponent::write_word_(uint16_t value, uint8_t bits) {
   for (uint8_t i = bits; i > 0; i--) {
@@ -106,6 +115,13 @@ void MY9231OutputComponent::send_di_pulses_(uint8_t count) {
     this->pin_di_->digital_write(false);
   }
 }
+void MY9231OutputComponent::send_dcki_pulses_(uint8_t count) {
+  delayMicroseconds(12);
+  for (uint8_t i = 0; i < count; i++) {
+    this->pin_dcki_->digital_write(true);
+    this->pin_dcki_->digital_write(false);
+  }
+}
 
 }  // namespace my9231
 }  // namespace esphome
diff --git a/esphome/components/my9231/my9231.h b/esphome/components/my9231/my9231.h
index a777dcc960..77c1259853 100644
--- a/esphome/components/my9231/my9231.h
+++ b/esphome/components/my9231/my9231.h
@@ -49,6 +49,7 @@ class MY9231OutputComponent : public Component {
   void init_chips_(uint8_t command);
   void write_word_(uint16_t value, uint8_t bits);
   void send_di_pulses_(uint8_t count);
+  void send_dcki_pulses_(uint8_t count);
 
   GPIOPin *pin_di_;
   GPIOPin *pin_dcki_;

From 445b13dbc6330df2361e06fb9499ae31c6a1153b Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Thu, 16 Nov 2023 20:55:28 +1300
Subject: [PATCH 03/25] Bump version to 2023.11.1

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index f937ecf068..7ceaffbe57 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.0"
+__version__ = "2023.11.1"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From c36944326368ca3d323eeb41ba3146d052c30629 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 15 Nov 2023 15:28:04 -0600
Subject: [PATCH 04/25] Bump aioesphomeapi from 18.4.0 to 18.4.1 (#5767)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 1d53c2761c..b1ca8e5058 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.4.0
+aioesphomeapi==18.4.1
 zeroconf==0.123.0
 
 # esp-idf requires this, but doesn't bundle it by default

From e5e3b253bca349ee354ee76565b11ea571bbad4b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 16 Nov 2023 23:35:42 -0600
Subject: [PATCH 05/25] Bump aioesphomeapi from 18.4.1 to 18.5.2 (#5780)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b1ca8e5058..7a3bb421ed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.4.1
+aioesphomeapi==18.5.2
 zeroconf==0.123.0
 
 # esp-idf requires this, but doesn't bundle it by default

From 625ce2b8ebb88367174a1a73b684fa690261471d Mon Sep 17 00:00:00 2001
From: Samuel Sieb <samuel-github@sieb.net>
Date: Fri, 17 Nov 2023 01:16:03 -0800
Subject: [PATCH 06/25] fix 32-bit arm (#5781)

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7ca633a982..a892e1df38 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -68,7 +68,7 @@ ENV \
 # See: https://unix.stackexchange.com/questions/553743/correct-way-to-add-lib-ld-linux-so-3-in-debian
 RUN \
     if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
-        ln -s /lib/arm-linux-gnueabihf/ld-linux.so.3 /lib/ld-linux.so.3; \
+        ln -s /lib/arm-linux-gnueabihf/ld-linux-armhf.so.3 /lib/ld-linux.so.3; \
     fi
 
 RUN \

From 22eef036c7a74072534e00b7071775fc8b20d1a8 Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Sat, 18 Nov 2023 02:00:59 -0600
Subject: [PATCH 07/25] Add 2MB option for partitions.csv generation and
 restore use of user-defined partitions (#5779)

---
 esphome/components/esp32/__init__.py | 58 ++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 8 deletions(-)

diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index 9b83d144f8..fd5e9377dd 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -3,23 +3,26 @@ from typing import Union, Optional
 from pathlib import Path
 import logging
 import os
+import esphome.final_validate as fv
 
 from esphome.helpers import copy_file_if_changed, write_file_if_changed, mkdir_p
 from esphome.const import (
+    CONF_ADVANCED,
     CONF_BOARD,
     CONF_COMPONENTS,
+    CONF_ESPHOME,
     CONF_FRAMEWORK,
+    CONF_IGNORE_EFUSE_MAC_CRC,
     CONF_NAME,
+    CONF_PATH,
+    CONF_PLATFORMIO_OPTIONS,
+    CONF_REF,
+    CONF_REFRESH,
     CONF_SOURCE,
     CONF_TYPE,
+    CONF_URL,
     CONF_VARIANT,
     CONF_VERSION,
-    CONF_ADVANCED,
-    CONF_REFRESH,
-    CONF_PATH,
-    CONF_URL,
-    CONF_REF,
-    CONF_IGNORE_EFUSE_MAC_CRC,
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
     KEY_NAME,
@@ -327,6 +330,32 @@ def _detect_variant(value):
     return value
 
 
+def final_validate(config):
+    if CONF_PLATFORMIO_OPTIONS not in fv.full_config.get()[CONF_ESPHOME]:
+        return config
+
+    pio_flash_size_key = "board_upload.flash_size"
+    pio_partitions_key = "board_build.partitions"
+    if (
+        CONF_PARTITIONS in config
+        and pio_partitions_key
+        in fv.full_config.get()[CONF_ESPHOME][CONF_PLATFORMIO_OPTIONS]
+    ):
+        raise cv.Invalid(
+            f"Do not specify '{pio_partitions_key}' in '{CONF_PLATFORMIO_OPTIONS}' with '{CONF_PARTITIONS}' in esp32"
+        )
+
+    if (
+        pio_flash_size_key
+        in fv.full_config.get()[CONF_ESPHOME][CONF_PLATFORMIO_OPTIONS]
+    ):
+        raise cv.Invalid(
+            f"Please specify {CONF_FLASH_SIZE} within esp32 configuration only"
+        )
+
+    return config
+
+
 CONF_PLATFORM_VERSION = "platform_version"
 
 ARDUINO_FRAMEWORK_SCHEMA = cv.All(
@@ -387,6 +416,7 @@ FRAMEWORK_SCHEMA = cv.typed_schema(
 
 
 FLASH_SIZES = [
+    "2MB",
     "4MB",
     "8MB",
     "16MB",
@@ -394,6 +424,7 @@ FLASH_SIZES = [
 ]
 
 CONF_FLASH_SIZE = "flash_size"
+CONF_PARTITIONS = "partitions"
 CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
@@ -401,6 +432,7 @@ CONFIG_SCHEMA = cv.All(
             cv.Optional(CONF_FLASH_SIZE, default="4MB"): cv.one_of(
                 *FLASH_SIZES, upper=True
             ),
+            cv.Optional(CONF_PARTITIONS): cv.file_,
             cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True),
             cv.Optional(CONF_FRAMEWORK, default={}): FRAMEWORK_SCHEMA,
         }
@@ -410,6 +442,9 @@ CONFIG_SCHEMA = cv.All(
 )
 
 
+FINAL_VALIDATE_SCHEMA = cv.Schema(final_validate)
+
+
 async def to_code(config):
     cg.add_platformio_option("board", config[CONF_BOARD])
     cg.add_platformio_option("board_upload.flash_size", config[CONF_FLASH_SIZE])
@@ -462,7 +497,10 @@ async def to_code(config):
         add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0", False)
         add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1", False)
 
-        cg.add_platformio_option("board_build.partitions", "partitions.csv")
+        if CONF_PARTITIONS in config:
+            cg.add_platformio_option("board_build.partitions", config[CONF_PARTITIONS])
+        else:
+            cg.add_platformio_option("board_build.partitions", "partitions.csv")
 
         for name, value in conf[CONF_SDKCONFIG_OPTIONS].items():
             add_idf_sdkconfig_option(name, RawSdkconfigValue(value))
@@ -507,7 +545,10 @@ async def to_code(config):
             [f"platformio/framework-arduinoespressif32@{conf[CONF_SOURCE]}"],
         )
 
-        cg.add_platformio_option("board_build.partitions", "partitions.csv")
+        if CONF_PARTITIONS in config:
+            cg.add_platformio_option("board_build.partitions", config[CONF_PARTITIONS])
+        else:
+            cg.add_platformio_option("board_build.partitions", "partitions.csv")
 
         cg.add_define(
             "USE_ARDUINO_VERSION_CODE",
@@ -518,6 +559,7 @@ async def to_code(config):
 
 
 APP_PARTITION_SIZES = {
+    "2MB": 0x0C0000,  # 768 KB
     "4MB": 0x1C0000,  # 1792 KB
     "8MB": 0x3C0000,  # 3840 KB
     "16MB": 0x7C0000,  # 7936 KB

From 8fb6b8f1a2a113cf3a47e4f294fb6b3b20ef3c6c Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:15:56 +1300
Subject: [PATCH 08/25] Bump version to 2023.11.2

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 7ceaffbe57..0abb6a8e2d 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.1"
+__version__ = "2023.11.2"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 47d42afda342b6b5f8dc45d0bdcd5742c5f9a2cd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Tue, 21 Nov 2023 01:15:32 +0100
Subject: [PATCH 09/25] dashboard: Fix online status when api is disabled
 (#5791)

---
 esphome/dashboard/dashboard.py | 13 +++++++++++--
 esphome/zeroconf.py            |  6 +++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py
index 050564d21e..dd51382db8 100644
--- a/esphome/dashboard/dashboard.py
+++ b/esphome/dashboard/dashboard.py
@@ -973,6 +973,7 @@ class MDNSStatusThread(threading.Thread):
         self.host_name_to_filename: dict[str, str] = {}
         # This is a set of host names to track (i.e no_mdns = false)
         self.host_name_with_mdns_enabled: set[set] = set()
+        self.zc: EsphomeZeroconf | None = None
         self._refresh_hosts()
 
     def _refresh_hosts(self):
@@ -996,7 +997,14 @@ class MDNSStatusThread(threading.Thread):
             # If we just adopted/imported this host, we likely
             # already have a state for it, so we should make sure
             # to set it so the dashboard shows it as online
-            if name in host_mdns_state:
+            if self.zc and (
+                entry.loaded_integrations and "api" not in entry.loaded_integrations
+            ):
+                # No api available so we have to poll since
+                # the device won't respond to a request to ._esphomelib._tcp.local.
+                PING_RESULT[filename] = bool(self.zc.resolve_host(entry.name))
+            elif name in host_mdns_state:
+                # We already have a state for this host
                 PING_RESULT[filename] = host_mdns_state[name]
 
             # Make sure the mapping is up to date
@@ -1007,7 +1015,8 @@ class MDNSStatusThread(threading.Thread):
     def run(self):
         global IMPORT_RESULT
 
-        zc = EsphomeZeroconf()
+        self.zc = EsphomeZeroconf()
+        zc = self.zc
         host_mdns_state = self.host_mdns_state
         host_name_to_filename = self.host_name_to_filename
         host_name_with_mdns_enabled = self.host_name_with_mdns_enabled
diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py
index f4cb7f080b..2585d5adc0 100644
--- a/esphome/zeroconf.py
+++ b/esphome/zeroconf.py
@@ -150,7 +150,11 @@ class EsphomeZeroconf(Zeroconf):
     def resolve_host(self, host: str, timeout: float = 3.0) -> str | None:
         """Resolve a host name to an IP address."""
         name = host.partition(".")[0]
-        info = HostResolver(ESPHOME_SERVICE_TYPE, f"{name}.{ESPHOME_SERVICE_TYPE}")
+        info = HostResolver(
+            ESPHOME_SERVICE_TYPE,
+            f"{name}.{ESPHOME_SERVICE_TYPE}",
+            server=f"{name}.local.",
+        )
         if (
             info.load_from_cache(self)
             or (timeout and info.request(self, timeout * 1000))

From 1c4b06700fe3b89bb2bc1f4bcbb6f9af655c286e Mon Sep 17 00:00:00 2001
From: Cody Cutrer <cody@cutrer.us>
Date: Mon, 20 Nov 2023 16:59:38 -0700
Subject: [PATCH 10/25] include payload_open when a lock supports OPEN (#5809)

---
 esphome/components/mqtt/mqtt_lock.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/esphome/components/mqtt/mqtt_lock.cpp b/esphome/components/mqtt/mqtt_lock.cpp
index 197d0c32d4..f4a5126d0c 100644
--- a/esphome/components/mqtt/mqtt_lock.cpp
+++ b/esphome/components/mqtt/mqtt_lock.cpp
@@ -40,6 +40,8 @@ const EntityBase *MQTTLockComponent::get_entity() const { return this->lock_; }
 void MQTTLockComponent::send_discovery(JsonObject root, mqtt::SendDiscoveryConfig &config) {
   if (this->lock_->traits.get_assumed_state())
     root[MQTT_OPTIMISTIC] = true;
+  if (this->lock_->traits.get_supports_open())
+    root[MQTT_PAYLOAD_OPEN] = "OPEN";
 }
 bool MQTTLockComponent::send_initial_state() { return this->publish_state(); }
 

From d0ac202a3f81822c16aff0cef3a53a5938cd018a Mon Sep 17 00:00:00 2001
From: CVan <cvandesande@opendmz.com>
Date: Mon, 20 Nov 2023 19:19:36 -0500
Subject: [PATCH 11/25] fix: compile errors with fonts (#5808)

---
 docker/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a892e1df38..1bf754464d 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -48,6 +48,8 @@ RUN \
           libfreetype-dev=2.12.1+dfsg-5 \
           libssl-dev=3.0.11-1~deb12u2 \
           libffi-dev=3.4.4-1 \
+          libopenjp2-7=2.5.0-2 \
+          libtiff6=4.5.0-6 \
           cargo=0.66.0+ds1-1 \
           pkg-config=1.8.1-1 \
           gcc-arm-linux-gnueabihf=4:12.2.0-3; \

From 10ca05b686cf0ffa5d1483bf38aa5e95e2fde77d Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Tue, 21 Nov 2023 14:24:47 +1300
Subject: [PATCH 12/25] Early return when there are no wifi scan results
 (#5797)

---
 esphome/components/wifi/wifi_component_esp_idf.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp
index 34ecaf887d..56cae1b1cc 100644
--- a/esphome/components/wifi/wifi_component_esp_idf.cpp
+++ b/esphome/components/wifi/wifi_component_esp_idf.cpp
@@ -674,6 +674,11 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) {
       return;
     }
 
+    if (it.number == 0) {
+      // no results
+      return;
+    }
+
     uint16_t number = it.number;
     std::vector<wifi_ap_record_t> records(number);
     err = esp_wifi_scan_get_ap_records(&number, records.data());

From b421fccc08c5824df5cc4fed9b3d8602344e4ffa Mon Sep 17 00:00:00 2001
From: Keith Burzinski <kbx81x@gmail.com>
Date: Tue, 21 Nov 2023 15:09:14 -0600
Subject: [PATCH 13/25] Add some additional VA triggers, part 2 (#5811)

---
 .../components/voice_assistant/__init__.py    | 34 +++++++++++++++++++
 .../voice_assistant/voice_assistant.cpp       |  6 ++++
 .../voice_assistant/voice_assistant.h         |  8 +++++
 3 files changed, 48 insertions(+)

diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py
index 5715604605..d05f39072c 100644
--- a/esphome/components/voice_assistant/__init__.py
+++ b/esphome/components/voice_assistant/__init__.py
@@ -29,6 +29,8 @@ CONF_ON_STT_VAD_END = "on_stt_vad_end"
 CONF_ON_STT_VAD_START = "on_stt_vad_start"
 CONF_ON_TTS_END = "on_tts_end"
 CONF_ON_TTS_START = "on_tts_start"
+CONF_ON_TTS_STREAM_START = "on_tts_stream_start"
+CONF_ON_TTS_STREAM_END = "on_tts_stream_end"
 CONF_ON_WAKE_WORD_DETECTED = "on_wake_word_detected"
 
 CONF_SILENCE_DETECTION = "silence_detection"
@@ -56,6 +58,17 @@ IsRunningCondition = voice_assistant_ns.class_(
     "IsRunningCondition", automation.Condition, cg.Parented.template(VoiceAssistant)
 )
 
+
+def tts_stream_validate(config):
+    if CONF_SPEAKER not in config and (
+        CONF_ON_TTS_STREAM_START in config or CONF_ON_TTS_STREAM_END in config
+    ):
+        raise cv.Invalid(
+            f"{CONF_SPEAKER} is required when using {CONF_ON_TTS_STREAM_START} and/or {CONF_ON_TTS_STREAM_END}"
+        )
+    return config
+
+
 CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
@@ -105,8 +118,15 @@ CONFIG_SCHEMA = cv.All(
             cv.Optional(CONF_ON_STT_VAD_END): automation.validate_automation(
                 single=True
             ),
+            cv.Optional(CONF_ON_TTS_STREAM_START): automation.validate_automation(
+                single=True
+            ),
+            cv.Optional(CONF_ON_TTS_STREAM_END): automation.validate_automation(
+                single=True
+            ),
         }
     ).extend(cv.COMPONENT_SCHEMA),
+    tts_stream_validate,
 )
 
 
@@ -222,6 +242,20 @@ async def to_code(config):
             config[CONF_ON_STT_VAD_END],
         )
 
+    if CONF_ON_TTS_STREAM_START in config:
+        await automation.build_automation(
+            var.get_tts_stream_start_trigger(),
+            [],
+            config[CONF_ON_TTS_STREAM_START],
+        )
+
+    if CONF_ON_TTS_STREAM_END in config:
+        await automation.build_automation(
+            var.get_tts_stream_end_trigger(),
+            [],
+            config[CONF_ON_TTS_STREAM_END],
+        )
+
     cg.add_define("USE_VOICE_ASSISTANT")
 
 
diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index 7ebbe762b3..9b13a71039 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -632,11 +632,17 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_START: {
 #ifdef USE_SPEAKER
       this->wait_for_stream_end_ = true;
+      ESP_LOGD(TAG, "TTS stream start");
+      this->tts_stream_start_trigger_->trigger();
 #endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_TTS_STREAM_END: {
       this->set_state_(State::RESPONSE_FINISHED, State::IDLE);
+#ifdef USE_SPEAKER
+      ESP_LOGD(TAG, "TTS stream end");
+      this->tts_stream_end_trigger_->trigger();
+#endif
       break;
     }
     case api::enums::VOICE_ASSISTANT_STT_VAD_START:
diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h
index a985bc4678..f6dcd1c563 100644
--- a/esphome/components/voice_assistant/voice_assistant.h
+++ b/esphome/components/voice_assistant/voice_assistant.h
@@ -107,6 +107,10 @@ class VoiceAssistant : public Component {
   Trigger<> *get_start_trigger() const { return this->start_trigger_; }
   Trigger<> *get_stt_vad_end_trigger() const { return this->stt_vad_end_trigger_; }
   Trigger<> *get_stt_vad_start_trigger() const { return this->stt_vad_start_trigger_; }
+#ifdef USE_SPEAKER
+  Trigger<> *get_tts_stream_start_trigger() const { return this->tts_stream_start_trigger_; }
+  Trigger<> *get_tts_stream_end_trigger() const { return this->tts_stream_end_trigger_; }
+#endif
   Trigger<> *get_wake_word_detected_trigger() const { return this->wake_word_detected_trigger_; }
   Trigger<std::string> *get_stt_end_trigger() const { return this->stt_end_trigger_; }
   Trigger<std::string> *get_tts_end_trigger() const { return this->tts_end_trigger_; }
@@ -135,6 +139,10 @@ class VoiceAssistant : public Component {
   Trigger<> *start_trigger_ = new Trigger<>();
   Trigger<> *stt_vad_start_trigger_ = new Trigger<>();
   Trigger<> *stt_vad_end_trigger_ = new Trigger<>();
+#ifdef USE_SPEAKER
+  Trigger<> *tts_stream_start_trigger_ = new Trigger<>();
+  Trigger<> *tts_stream_end_trigger_ = new Trigger<>();
+#endif
   Trigger<> *wake_word_detected_trigger_ = new Trigger<>();
   Trigger<std::string> *stt_end_trigger_ = new Trigger<std::string>();
   Trigger<std::string> *tts_end_trigger_ = new Trigger<std::string>();

From 3f40e32eba514e3561c092c49d16d3032fd84305 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Wed, 22 Nov 2023 11:08:47 +1300
Subject: [PATCH 14/25] Bump version to 2023.11.3

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 0abb6a8e2d..5f6e9f5eb1 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.2"
+__version__ = "2023.11.3"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 1204b4f1bd96b636086d7118e7f79550d86c37e0 Mon Sep 17 00:00:00 2001
From: Landon Rohatensky <landonroha@gmail.com>
Date: Thu, 23 Nov 2023 11:10:33 -0800
Subject: [PATCH 15/25] Allow images to be downloaded from URLs (#5214)

Co-authored-by: guillempages <guillempages@users.noreply.github.com>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
---
 docker/Dockerfile                    |   1 +
 esphome/components/image/__init__.py | 121 +++++++++++++++++++++------
 esphome/external_files.py            |  75 +++++++++++++++++
 requirements.txt                     |   1 +
 tests/test2.yaml                     |  12 +++
 5 files changed, 184 insertions(+), 26 deletions(-)
 create mode 100644 esphome/external_files.py

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 1bf754464d..ee7c70bb0f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -38,6 +38,7 @@ RUN \
         openssh-client=1:9.2p1-2+deb12u1 \
         python3-cffi=1.15.1-5 \
         libcairo2=1.16.0-7 \
+        libmagic1=1:5.44-3 \
         patch=2.7.6-7; \
     if [ "$TARGETARCH$TARGETVARIANT" = "armv7" ]; then \
         apt-get install -y --no-install-recommends \
diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py
index 1b7c654b0b..c11021fc9c 100644
--- a/esphome/components/image/__init__.py
+++ b/esphome/components/image/__init__.py
@@ -1,15 +1,23 @@
+from __future__ import annotations
+
 import logging
 
+import hashlib
 import io
 from pathlib import Path
 import re
 import requests
+from magic import Magic
+
+from PIL import Image
 
 from esphome import core
 from esphome.components import font
+from esphome import external_files
 import esphome.config_validation as cv
 import esphome.codegen as cg
 from esphome.const import (
+    __version__,
     CONF_DITHER,
     CONF_FILE,
     CONF_ICON,
@@ -19,6 +27,7 @@ from esphome.const import (
     CONF_RESIZE,
     CONF_SOURCE,
     CONF_TYPE,
+    CONF_URL,
 )
 from esphome.core import CORE, HexInt
 
@@ -43,34 +52,74 @@ IMAGE_TYPE = {
 CONF_USE_TRANSPARENCY = "use_transparency"
 
 # If the MDI file cannot be downloaded within this time, abort.
-MDI_DOWNLOAD_TIMEOUT = 30  # seconds
+IMAGE_DOWNLOAD_TIMEOUT = 30  # seconds
 
 SOURCE_LOCAL = "local"
 SOURCE_MDI = "mdi"
+SOURCE_WEB = "web"
+
 
 Image_ = image_ns.class_("Image")
 
 
-def _compute_local_icon_path(value) -> Path:
-    base_dir = Path(CORE.data_dir) / DOMAIN / "mdi"
+def _compute_local_icon_path(value: dict) -> Path:
+    base_dir = external_files.compute_local_file_dir(DOMAIN) / "mdi"
     return base_dir / f"{value[CONF_ICON]}.svg"
 
 
-def download_mdi(value):
-    mdi_id = value[CONF_ICON]
-    path = _compute_local_icon_path(value)
-    if path.is_file():
-        return value
-    url = f"https://raw.githubusercontent.com/Templarian/MaterialDesign/master/svg/{mdi_id}.svg"
-    _LOGGER.debug("Downloading %s MDI image from %s", mdi_id, url)
+def _compute_local_image_path(value: dict) -> Path:
+    url = value[CONF_URL]
+    h = hashlib.new("sha256")
+    h.update(url.encode())
+    key = h.hexdigest()[:8]
+    base_dir = external_files.compute_local_file_dir(DOMAIN)
+    return base_dir / key
+
+
+def download_content(url: str, path: Path) -> None:
+    if not external_files.has_remote_file_changed(url, path):
+        _LOGGER.debug("Remote file has not changed %s", url)
+        return
+
+    _LOGGER.debug(
+        "Remote file has changed, downloading from %s to %s",
+        url,
+        path,
+    )
+
     try:
-        req = requests.get(url, timeout=MDI_DOWNLOAD_TIMEOUT)
+        req = requests.get(
+            url,
+            timeout=IMAGE_DOWNLOAD_TIMEOUT,
+            headers={"User-agent": f"ESPHome/{__version__} (https://esphome.io)"},
+        )
         req.raise_for_status()
     except requests.exceptions.RequestException as e:
-        raise cv.Invalid(f"Could not download MDI image {mdi_id} from {url}: {e}")
+        raise cv.Invalid(f"Could not download from {url}: {e}")
 
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_bytes(req.content)
+
+
+def download_mdi(value):
+    validate_cairosvg_installed(value)
+
+    mdi_id = value[CONF_ICON]
+    path = _compute_local_icon_path(value)
+
+    url = f"https://raw.githubusercontent.com/Templarian/MaterialDesign/master/svg/{mdi_id}.svg"
+
+    download_content(url, path)
+
+    return value
+
+
+def download_image(value):
+    url = value[CONF_URL]
+    path = _compute_local_image_path(value)
+
+    download_content(url, path)
+
     return value
 
 
@@ -139,6 +188,13 @@ def validate_file_shorthand(value):
                 CONF_ICON: icon,
             }
         )
+    if value.startswith("http://") or value.startswith("https://"):
+        return FILE_SCHEMA(
+            {
+                CONF_SOURCE: SOURCE_WEB,
+                CONF_URL: value,
+            }
+        )
     return FILE_SCHEMA(
         {
             CONF_SOURCE: SOURCE_LOCAL,
@@ -160,10 +216,18 @@ MDI_SCHEMA = cv.All(
     download_mdi,
 )
 
+WEB_SCHEMA = cv.All(
+    {
+        cv.Required(CONF_URL): cv.string,
+    },
+    download_image,
+)
+
 TYPED_FILE_SCHEMA = cv.typed_schema(
     {
         SOURCE_LOCAL: LOCAL_SCHEMA,
         SOURCE_MDI: MDI_SCHEMA,
+        SOURCE_WEB: WEB_SCHEMA,
     },
     key=CONF_SOURCE,
 )
@@ -201,9 +265,7 @@ IMAGE_SCHEMA = cv.Schema(
 CONFIG_SCHEMA = cv.All(font.validate_pillow_installed, IMAGE_SCHEMA)
 
 
-def load_svg_image(file: str, resize: tuple[int, int]):
-    from PIL import Image
-
+def load_svg_image(file: bytes, resize: tuple[int, int]):
     # This import is only needed in case of SVG images; adding it
     # to the top would force configurations not using SVG to also have it
     # installed for no reason.
@@ -212,19 +274,17 @@ def load_svg_image(file: str, resize: tuple[int, int]):
     if resize:
         req_width, req_height = resize
         svg_image = svg2png(
-            url=file,
+            file,
             output_width=req_width,
             output_height=req_height,
         )
     else:
-        svg_image = svg2png(url=file)
+        svg_image = svg2png(file)
 
     return Image.open(io.BytesIO(svg_image))
 
 
 async def to_code(config):
-    from PIL import Image
-
     conf_file = config[CONF_FILE]
 
     if conf_file[CONF_SOURCE] == SOURCE_LOCAL:
@@ -233,17 +293,26 @@ async def to_code(config):
     elif conf_file[CONF_SOURCE] == SOURCE_MDI:
         path = _compute_local_icon_path(conf_file).as_posix()
 
+    elif conf_file[CONF_SOURCE] == SOURCE_WEB:
+        path = _compute_local_image_path(conf_file).as_posix()
+
     try:
-        resize = config.get(CONF_RESIZE)
-        if path.lower().endswith(".svg"):
-            image = load_svg_image(path, resize)
-        else:
-            image = Image.open(path)
-            if resize:
-                image.thumbnail(resize)
+        with open(path, "rb") as f:
+            file_contents = f.read()
     except Exception as e:
         raise core.EsphomeError(f"Could not load image file {path}: {e}")
 
+    mime = Magic(mime=True)
+    file_type = mime.from_buffer(file_contents)
+
+    resize = config.get(CONF_RESIZE)
+    if "svg" in file_type:
+        image = load_svg_image(file_contents, resize)
+    else:
+        image = Image.open(io.BytesIO(file_contents))
+        if resize:
+            image.thumbnail(resize)
+
     width, height = image.size
 
     if CONF_RESIZE not in config and (width > 500 or height > 500):
diff --git a/esphome/external_files.py b/esphome/external_files.py
new file mode 100644
index 0000000000..5b476286f3
--- /dev/null
+++ b/esphome/external_files.py
@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+import os
+from datetime import datetime
+import requests
+import esphome.config_validation as cv
+from esphome.core import CORE, TimePeriodSeconds
+
+_LOGGER = logging.getLogger(__name__)
+CODEOWNERS = ["@landonr"]
+
+NETWORK_TIMEOUT = 30
+
+IF_MODIFIED_SINCE = "If-Modified-Since"
+CACHE_CONTROL = "Cache-Control"
+CACHE_CONTROL_MAX_AGE = "max-age="
+CONTENT_DISPOSITION = "content-disposition"
+TEMP_DIR = "temp"
+
+
+def has_remote_file_changed(url, local_file_path):
+    if os.path.exists(local_file_path):
+        _LOGGER.debug("has_remote_file_changed: File exists at %s", local_file_path)
+        try:
+            local_modification_time = os.path.getmtime(local_file_path)
+            local_modification_time_str = datetime.utcfromtimestamp(
+                local_modification_time
+            ).strftime("%a, %d %b %Y %H:%M:%S GMT")
+
+            headers = {
+                IF_MODIFIED_SINCE: local_modification_time_str,
+                CACHE_CONTROL: CACHE_CONTROL_MAX_AGE + "3600",
+            }
+            response = requests.head(url, headers=headers, timeout=NETWORK_TIMEOUT)
+
+            _LOGGER.debug(
+                "has_remote_file_changed: File %s, Local modified %s, response code %d",
+                local_file_path,
+                local_modification_time_str,
+                response.status_code,
+            )
+
+            if response.status_code == 304:
+                _LOGGER.debug(
+                    "has_remote_file_changed: File not modified since %s",
+                    local_modification_time_str,
+                )
+                return False
+            _LOGGER.debug("has_remote_file_changed: File modified")
+            return True
+        except requests.exceptions.RequestException as e:
+            raise cv.Invalid(
+                f"Could not check if {url} has changed, please check if file exists "
+                f"({e})"
+            )
+
+    _LOGGER.debug("has_remote_file_changed: File doesn't exists at %s", local_file_path)
+    return True
+
+
+def is_file_recent(file_path: str, refresh: TimePeriodSeconds) -> bool:
+    if os.path.exists(file_path):
+        creation_time = os.path.getctime(file_path)
+        current_time = datetime.now().timestamp()
+        return current_time - creation_time <= refresh.total_seconds
+    return False
+
+
+def compute_local_file_dir(domain: str) -> Path:
+    base_directory = Path(CORE.data_dir) / domain
+    base_directory.mkdir(parents=True, exist_ok=True)
+
+    return base_directory
diff --git a/requirements.txt b/requirements.txt
index 7a3bb421ed..2c2bf1ba19 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,7 @@ click==8.1.7
 esphome-dashboard==20231107.0
 aioesphomeapi==18.5.2
 zeroconf==0.123.0
+python-magic==0.4.27
 
 # esp-idf requires this, but doesn't bundle it by default
 # https://github.com/espressif/esp-idf/blob/220590d599e134d7a5e7f1e683cc4550349ffbf8/requirements.txt#L24
diff --git a/tests/test2.yaml b/tests/test2.yaml
index bfc886eaa4..91fb554146 100644
--- a/tests/test2.yaml
+++ b/tests/test2.yaml
@@ -752,6 +752,18 @@ image:
     file: pnglogo.png
     type: RGB565
     use_transparency: no
+  - id: web_svg_image
+    file: https://raw.githubusercontent.com/esphome/esphome-docs/a62d7ab193c1a464ed791670170c7d518189109b/images/logo.svg
+    resize: 256x48
+    type: TRANSPARENT_BINARY
+  - id: web_tiff_image
+    file: https://upload.wikimedia.org/wikipedia/commons/b/b6/SIPI_Jelly_Beans_4.1.07.tiff
+    type: RGB24
+    resize: 48x48
+  - id: web_redirect_image
+    file: https://avatars.githubusercontent.com/u/3060199?s=48&v=4
+    type: RGB24
+    resize: 48x48
 
   - id: mdi_alert
     file: mdi:alert-circle-outline

From 711faab329d5ff22bb0264def03461c3f1d77ca7 Mon Sep 17 00:00:00 2001
From: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
Date: Fri, 24 Nov 2023 10:24:25 +1300
Subject: [PATCH 16/25] Bump version to 2023.11.4

---
 esphome/const.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esphome/const.py b/esphome/const.py
index 5f6e9f5eb1..5596cbcd36 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.3"
+__version__ = "2023.11.4"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

From 1762204b0039386a561a489c37138e6197ffdd32 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Thu, 23 Nov 2023 23:49:55 +0100
Subject: [PATCH 17/25] dashboard: set nodelay on the websocket to avoid a
 delay seeing log messages (#5802)

---
 esphome/dashboard/web_server.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py
index 7c5f653b5b..9a9ccb462b 100644
--- a/esphome/dashboard/web_server.py
+++ b/esphome/dashboard/web_server.py
@@ -27,6 +27,7 @@ import tornado.process
 import tornado.queues
 import tornado.web
 import tornado.websocket
+import tornado.httputil
 import yaml
 from tornado.log import access_log
 
@@ -136,7 +137,15 @@ def websocket_method(name):
 
 @websocket_class
 class EsphomeCommandWebSocket(tornado.websocket.WebSocketHandler):
-    def __init__(self, application, request, **kwargs):
+    """Base class for ESPHome websocket commands."""
+
+    def __init__(
+        self,
+        application: tornado.web.Application,
+        request: tornado.httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the websocket."""
         super().__init__(application, request, **kwargs)
         self._proc = None
         self._queue = None
@@ -145,6 +154,12 @@ class EsphomeCommandWebSocket(tornado.websocket.WebSocketHandler):
         # use Popen() with a reading thread instead
         self._use_popen = os.name == "nt"
 
+    def open(self, *args: str, **kwargs: str) -> None:
+        """Handle new WebSocket connection."""
+        # Ensure messages from the subprocess are sent immediately
+        # to avoid a 200-500ms delay when nodelay is not set.
+        self.set_nodelay(True)
+
     @authenticated
     async def on_message(  # pylint: disable=invalid-overridden-method
         self, message: str

From 9f8a896e13136809ae0f7e585246db4be06d3d1d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 23 Nov 2023 17:02:44 -0600
Subject: [PATCH 18/25] Bump aioesphomeapi from 18.5.5 to 18.5.7 (#5822)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index abbdcf66d5..af53dc8e94 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.5.5
+aioesphomeapi==18.5.7
 zeroconf==0.127.0
 python-magic==0.4.27
 

From 5c31bec8c20626c0fb2558f08145ca1ab46e85be Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 24 Nov 2023 00:29:08 +0100
Subject: [PATCH 19/25] Ensure names containing characters other than `a-z`
 `A-Z` `0-9` or `_` are unique (#5810)

---
 esphome/core/helpers.cpp         | 11 +++++++----
 esphome/helpers.py               |  5 ++++-
 tests/unit_tests/test_helpers.py |  6 +++---
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp
index 714a1642f8..c95c0470de 100644
--- a/esphome/core/helpers.cpp
+++ b/esphome/core/helpers.cpp
@@ -278,10 +278,13 @@ std::string str_snake_case(const std::string &str) {
   return result;
 }
 std::string str_sanitize(const std::string &str) {
-  std::string out;
-  std::copy_if(str.begin(), str.end(), std::back_inserter(out), [](const char &c) {
-    return c == '-' || c == '_' || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
-  });
+  std::string out = str;
+  std::replace_if(
+      out.begin(), out.end(),
+      [](const char &c) {
+        return !(c == '-' || c == '_' || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'));
+      },
+      '_');
   return out;
 }
 std::string str_snprintf(const char *fmt, size_t len, ...) {
diff --git a/esphome/helpers.py b/esphome/helpers.py
index 4012b2067f..00416b591f 100644
--- a/esphome/helpers.py
+++ b/esphome/helpers.py
@@ -357,6 +357,9 @@ def snake_case(value):
     return value.replace(" ", "_").lower()
 
 
+_DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9_]")
+
+
 def sanitize(value):
     """Same behaviour as `helpers.cpp` method `str_sanitize`."""
-    return re.sub("[^-_0-9a-zA-Z]", r"", value)
+    return _DISALLOWED_CHARS.sub("_", value)
diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py
index 67fabd7af8..79d39901f0 100644
--- a/tests/unit_tests/test_helpers.py
+++ b/tests/unit_tests/test_helpers.py
@@ -258,9 +258,9 @@ def test_snake_case(text, expected):
     "text, expected",
     (
         ("foo_bar", "foo_bar"),
-        ('!"§$%&/()=?foo_bar', "foo_bar"),
-        ('foo_!"§$%&/()=?bar', "foo_bar"),
-        ('foo_bar!"§$%&/()=?', "foo_bar"),
+        ('!"§$%&/()=?foo_bar', "___________foo_bar"),
+        ('foo_!"§$%&/()=?bar', "foo____________bar"),
+        ('foo_bar!"§$%&/()=?', "foo_bar___________"),
     ),
 )
 def test_sanitize(text, expected):

From f456603c1b6e3428aa7204e7a58ca068b3a72353 Mon Sep 17 00:00:00 2001
From: Vincent Schmandt <vincentscode@gmail.com>
Date: Fri, 24 Nov 2023 05:31:07 +0100
Subject: [PATCH 20/25] Add ENS160 Sensor (#4243)

Co-authored-by: Keith Burzinski <kbx81x@gmail.com>
Co-authored-by: mrtoy-me <118446898+mrtoy-me@users.noreply.github.com>
---
 CODEOWNERS                            |   1 +
 esphome/components/ens160/__init__.py |   1 +
 esphome/components/ens160/ens160.cpp  | 321 ++++++++++++++++++++++++++
 esphome/components/ens160/ens160.h    |  60 +++++
 esphome/components/ens160/sensor.py   |  88 +++++++
 tests/test3.1.yaml                    |  11 +-
 tests/test3.yaml                      |   1 -
 7 files changed, 478 insertions(+), 5 deletions(-)
 create mode 100644 esphome/components/ens160/__init__.py
 create mode 100644 esphome/components/ens160/ens160.cpp
 create mode 100644 esphome/components/ens160/ens160.h
 create mode 100644 esphome/components/ens160/sensor.py

diff --git a/CODEOWNERS b/CODEOWNERS
index dd1586d039..af23f679c8 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -90,6 +90,7 @@ esphome/components/duty_time/* @dudanov
 esphome/components/ee895/* @Stock-M
 esphome/components/ektf2232/* @jesserockz
 esphome/components/emc2101/* @ellull
+esphome/components/ens160/* @vincentscode
 esphome/components/ens210/* @itn3rd77
 esphome/components/esp32/* @esphome/core
 esphome/components/esp32_ble/* @Rapsssito @jesserockz
diff --git a/esphome/components/ens160/__init__.py b/esphome/components/ens160/__init__.py
new file mode 100644
index 0000000000..d26770a89d
--- /dev/null
+++ b/esphome/components/ens160/__init__.py
@@ -0,0 +1 @@
+CODEOWNERS = ["@vincentscode"]
diff --git a/esphome/components/ens160/ens160.cpp b/esphome/components/ens160/ens160.cpp
new file mode 100644
index 0000000000..c7a6ccbb73
--- /dev/null
+++ b/esphome/components/ens160/ens160.cpp
@@ -0,0 +1,321 @@
+// ENS160 sensor with I2C interface from ScioSense
+//
+// Datasheet: https://www.sciosense.com/wp-content/uploads/documents/SC-001224-DS-7-ENS160-Datasheet.pdf
+//
+// Implementation based on:
+//   https://github.com/sciosense/ENS160_driver
+
+#include "ens160.h"
+#include "esphome/core/log.h"
+#include "esphome/core/hal.h"
+
+namespace esphome {
+namespace ens160 {
+
+static const char *const TAG = "ens160";
+
+static const uint8_t ENS160_BOOTING = 10;
+
+static const uint16_t ENS160_PART_ID = 0x0160;
+
+static const uint8_t ENS160_REG_PART_ID = 0x00;
+static const uint8_t ENS160_REG_OPMODE = 0x10;
+static const uint8_t ENS160_REG_CONFIG = 0x11;
+static const uint8_t ENS160_REG_COMMAND = 0x12;
+static const uint8_t ENS160_REG_TEMP_IN = 0x13;
+static const uint8_t ENS160_REG_DATA_STATUS = 0x20;
+static const uint8_t ENS160_REG_DATA_AQI = 0x21;
+static const uint8_t ENS160_REG_DATA_TVOC = 0x22;
+static const uint8_t ENS160_REG_DATA_ECO2 = 0x24;
+
+static const uint8_t ENS160_REG_GPR_READ_0 = 0x48;
+static const uint8_t ENS160_REG_GPR_READ_4 = ENS160_REG_GPR_READ_0 + 4;
+
+static const uint8_t ENS160_COMMAND_NOP = 0x00;
+static const uint8_t ENS160_COMMAND_CLRGPR = 0xCC;
+static const uint8_t ENS160_COMMAND_GET_APPVER = 0x0E;
+
+static const uint8_t ENS160_OPMODE_RESET = 0xF0;
+static const uint8_t ENS160_OPMODE_IDLE = 0x01;
+static const uint8_t ENS160_OPMODE_STD = 0x02;
+
+static const uint8_t ENS160_DATA_STATUS_STATAS = 0x80;
+static const uint8_t ENS160_DATA_STATUS_STATER = 0x40;
+static const uint8_t ENS160_DATA_STATUS_VALIDITY = 0x0C;
+static const uint8_t ENS160_DATA_STATUS_NEWDAT = 0x02;
+static const uint8_t ENS160_DATA_STATUS_NEWGPR = 0x01;
+
+// helps remove reserved bits in aqi data register
+static const uint8_t ENS160_DATA_AQI = 0x07;
+
+void ENS160Component::setup() {
+  ESP_LOGCONFIG(TAG, "Setting up ENS160...");
+
+  // check part_id
+  uint16_t part_id;
+  if (!this->read_bytes(ENS160_REG_PART_ID, reinterpret_cast<uint8_t *>(&part_id), 2)) {
+    this->error_code_ = COMMUNICATION_FAILED;
+    this->mark_failed();
+    return;
+  }
+  if (part_id != ENS160_PART_ID) {
+    this->error_code_ = INVALID_ID;
+    this->mark_failed();
+    return;
+  }
+
+  // set mode to reset
+  if (!this->write_byte(ENS160_REG_OPMODE, ENS160_OPMODE_RESET)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+  delay(ENS160_BOOTING);
+
+  // check status
+  uint8_t status_value;
+  if (!this->read_byte(ENS160_REG_DATA_STATUS, &status_value)) {
+    this->error_code_ = READ_FAILED;
+    this->mark_failed();
+    return;
+  }
+  this->validity_flag_ = static_cast<ValidityFlag>((ENS160_DATA_STATUS_VALIDITY & status_value) >> 2);
+
+  if (this->validity_flag_ == INVALID_OUTPUT) {
+    this->error_code_ = VALIDITY_INVALID;
+    this->mark_failed();
+    return;
+  }
+
+  // set mode to idle
+  if (!this->write_byte(ENS160_REG_OPMODE, ENS160_OPMODE_IDLE)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+  // clear command
+  if (!this->write_byte(ENS160_REG_COMMAND, ENS160_COMMAND_NOP)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+  if (!this->write_byte(ENS160_REG_COMMAND, ENS160_COMMAND_CLRGPR)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+
+  // read firmware version
+  if (!this->write_byte(ENS160_REG_COMMAND, ENS160_COMMAND_GET_APPVER)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+  uint8_t version_data[3];
+  if (!this->read_bytes(ENS160_REG_GPR_READ_4, version_data, 3)) {
+    this->error_code_ = READ_FAILED;
+    this->mark_failed();
+    return;
+  }
+  this->firmware_ver_major_ = version_data[0];
+  this->firmware_ver_minor_ = version_data[1];
+  this->firmware_ver_build_ = version_data[2];
+
+  // set mode to standard
+  if (!this->write_byte(ENS160_REG_OPMODE, ENS160_OPMODE_STD)) {
+    this->error_code_ = WRITE_FAILED;
+    this->mark_failed();
+    return;
+  }
+
+  // read opmode and check standard mode is achieved before finishing Setup
+  uint8_t op_mode;
+  if (!this->read_byte(ENS160_REG_OPMODE, &op_mode)) {
+    this->error_code_ = READ_FAILED;
+    this->mark_failed();
+    return;
+  }
+
+  if (op_mode != ENS160_OPMODE_STD) {
+    this->error_code_ = STD_OPMODE_FAILED;
+    this->mark_failed();
+    return;
+  }
+}
+
+void ENS160Component::update() {
+  uint8_t status_value, data_ready;
+
+  if (!this->read_byte(ENS160_REG_DATA_STATUS, &status_value)) {
+    ESP_LOGW(TAG, "Error reading status register");
+    this->status_set_warning();
+    return;
+  }
+
+  // verbose status logging
+  ESP_LOGV(TAG, "Status: ENS160 STATAS bit    0x%x",
+           (ENS160_DATA_STATUS_STATAS & (status_value)) == ENS160_DATA_STATUS_STATAS);
+  ESP_LOGV(TAG, "Status: ENS160 STATER bit    0x%x",
+           (ENS160_DATA_STATUS_STATER & (status_value)) == ENS160_DATA_STATUS_STATER);
+  ESP_LOGV(TAG, "Status: ENS160 VALIDITY FLAG 0x%02x", (ENS160_DATA_STATUS_VALIDITY & status_value) >> 2);
+  ESP_LOGV(TAG, "Status: ENS160 NEWDAT bit    0x%x",
+           (ENS160_DATA_STATUS_NEWDAT & (status_value)) == ENS160_DATA_STATUS_NEWDAT);
+  ESP_LOGV(TAG, "Status: ENS160 NEWGPR bit    0x%x",
+           (ENS160_DATA_STATUS_NEWGPR & (status_value)) == ENS160_DATA_STATUS_NEWGPR);
+
+  data_ready = ENS160_DATA_STATUS_NEWDAT & status_value;
+  this->validity_flag_ = static_cast<ValidityFlag>((ENS160_DATA_STATUS_VALIDITY & status_value) >> 2);
+
+  switch (validity_flag_) {
+    case NORMAL_OPERATION:
+      if (data_ready != ENS160_DATA_STATUS_NEWDAT) {
+        ESP_LOGD(TAG, "ENS160 readings unavailable - Normal Operation but readings not ready");
+        return;
+      }
+      break;
+    case INITIAL_STARTUP:
+      if (!this->initial_startup_) {
+        this->initial_startup_ = true;
+        ESP_LOGI(TAG, "ENS160 readings unavailable - 1 hour startup required after first power on");
+      }
+      return;
+    case WARMING_UP:
+      if (!this->warming_up_) {
+        this->warming_up_ = true;
+        ESP_LOGI(TAG, "ENS160 readings not available yet - Warming up requires 3 minutes");
+        this->send_env_data_();
+      }
+      return;
+    case INVALID_OUTPUT:
+      ESP_LOGE(TAG, "ENS160 Invalid Status - No Invalid Output");
+      this->status_set_warning();
+      return;
+  }
+
+  // read new data
+  uint16_t data_eco2;
+  if (!this->read_bytes(ENS160_REG_DATA_ECO2, reinterpret_cast<uint8_t *>(&data_eco2), 2)) {
+    ESP_LOGW(TAG, "Error reading eCO2 data register");
+    this->status_set_warning();
+    return;
+  }
+  if (this->co2_ != nullptr) {
+    this->co2_->publish_state(data_eco2);
+  }
+
+  uint16_t data_tvoc;
+  if (!this->read_bytes(ENS160_REG_DATA_TVOC, reinterpret_cast<uint8_t *>(&data_tvoc), 2)) {
+    ESP_LOGW(TAG, "Error reading TVOC data register");
+    this->status_set_warning();
+    return;
+  }
+  if (this->tvoc_ != nullptr) {
+    this->tvoc_->publish_state(data_tvoc);
+  }
+
+  uint8_t data_aqi;
+  if (!this->read_byte(ENS160_REG_DATA_AQI, &data_aqi)) {
+    ESP_LOGW(TAG, "Error reading AQI data register");
+    this->status_set_warning();
+    return;
+  }
+  if (this->aqi_ != nullptr) {
+    // remove reserved bits, just in case they are used in future
+    data_aqi = ENS160_DATA_AQI & data_aqi;
+
+    this->aqi_->publish_state(data_aqi);
+  }
+
+  this->status_clear_warning();
+
+  // set temperature and humidity compensation data
+  this->send_env_data_();
+}
+
+void ENS160Component::send_env_data_() {
+  if (this->temperature_ == nullptr && this->humidity_ == nullptr)
+    return;
+
+  float temperature = NAN;
+  if (this->temperature_ != nullptr)
+    temperature = this->temperature_->state;
+
+  if (std::isnan(temperature) || temperature < -40.0f || temperature > 85.0f) {
+    ESP_LOGW(TAG, "Invalid external temperature - compensation values not updated");
+    return;
+  } else {
+    ESP_LOGV(TAG, "External temperature compensation: %.1f°C", temperature);
+  }
+
+  float humidity = NAN;
+  if (this->humidity_ != nullptr)
+    humidity = this->humidity_->state;
+
+  if (std::isnan(humidity) || humidity < 0.0f || humidity > 100.0f) {
+    ESP_LOGW(TAG, "Invalid external humidity - compensation values not updated");
+    return;
+  } else {
+    ESP_LOGV(TAG, "External humidity compensation:    %.1f%%", humidity);
+  }
+
+  uint16_t t = (uint16_t) ((temperature + 273.15f) * 64.0f);
+  uint16_t h = (uint16_t) (humidity * 512.0f);
+
+  uint8_t data[4];
+  data[0] = t & 0xff;
+  data[1] = (t >> 8) & 0xff;
+  data[2] = h & 0xff;
+  data[3] = (h >> 8) & 0xff;
+
+  if (!this->write_bytes(ENS160_REG_TEMP_IN, data, 4)) {
+    ESP_LOGE(TAG, "Error writing compensation values");
+    this->status_set_warning();
+    return;
+  }
+}
+
+void ENS160Component::dump_config() {
+  ESP_LOGCONFIG(TAG, "ENS160:");
+
+  switch (this->error_code_) {
+    case COMMUNICATION_FAILED:
+      ESP_LOGE(TAG, "Communication failed! Is the sensor connected?");
+      break;
+    case READ_FAILED:
+      ESP_LOGE(TAG, "Error reading from register");
+      break;
+    case WRITE_FAILED:
+      ESP_LOGE(TAG, "Error writing to register");
+      break;
+    case INVALID_ID:
+      ESP_LOGE(TAG, "Sensor reported an invalid ID. Is this a ENS160?");
+      break;
+    case VALIDITY_INVALID:
+      ESP_LOGE(TAG, "Invalid Device Status - No valid output");
+      break;
+    case STD_OPMODE_FAILED:
+      ESP_LOGE(TAG, "Device failed to achieve Standard Operating Mode");
+      break;
+    case NONE:
+      ESP_LOGD(TAG, "Setup successful");
+      break;
+  }
+  ESP_LOGI(TAG, "Firmware Version: %d.%d.%d", this->firmware_ver_major_, this->firmware_ver_minor_,
+           this->firmware_ver_build_);
+
+  LOG_I2C_DEVICE(this);
+  LOG_UPDATE_INTERVAL(this);
+  LOG_SENSOR("  ", "CO2 Sensor:", this->co2_);
+  LOG_SENSOR("  ", "TVOC Sensor:", this->tvoc_);
+  LOG_SENSOR("  ", "AQI Sensor:", this->aqi_);
+
+  if (this->temperature_ != nullptr && this->humidity_ != nullptr) {
+    LOG_SENSOR("  ", "  Temperature Compensation:", this->temperature_);
+    LOG_SENSOR("  ", "  Humidity Compensation:", this->humidity_);
+  } else {
+    ESP_LOGCONFIG(TAG, "  Compensation: Not configured");
+  }
+}
+
+}  // namespace ens160
+}  // namespace esphome
diff --git a/esphome/components/ens160/ens160.h b/esphome/components/ens160/ens160.h
new file mode 100644
index 0000000000..88bc8e3501
--- /dev/null
+++ b/esphome/components/ens160/ens160.h
@@ -0,0 +1,60 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/components/sensor/sensor.h"
+#include "esphome/components/i2c/i2c.h"
+
+namespace esphome {
+namespace ens160 {
+
+class ENS160Component : public PollingComponent, public i2c::I2CDevice, public sensor::Sensor {
+ public:
+  void set_co2(sensor::Sensor *co2) { co2_ = co2; }
+  void set_tvoc(sensor::Sensor *tvoc) { tvoc_ = tvoc; }
+  void set_aqi(sensor::Sensor *aqi) { aqi_ = aqi; }
+
+  void set_humidity(sensor::Sensor *humidity) { humidity_ = humidity; }
+  void set_temperature(sensor::Sensor *temperature) { temperature_ = temperature; }
+
+  void setup() override;
+  void update() override;
+  void dump_config() override;
+  float get_setup_priority() const override { return setup_priority::DATA; }
+
+ protected:
+  void send_env_data_();
+
+  enum ErrorCode {
+    NONE = 0,
+    COMMUNICATION_FAILED,
+    INVALID_ID,
+    VALIDITY_INVALID,
+    READ_FAILED,
+    WRITE_FAILED,
+    STD_OPMODE_FAILED,
+  } error_code_{NONE};
+
+  enum ValidityFlag {
+    NORMAL_OPERATION = 0,
+    WARMING_UP,
+    INITIAL_STARTUP,
+    INVALID_OUTPUT,
+  } validity_flag_;
+
+  bool warming_up_{false};
+  bool initial_startup_{false};
+
+  uint8_t firmware_ver_major_{0};
+  uint8_t firmware_ver_minor_{0};
+  uint8_t firmware_ver_build_{0};
+
+  sensor::Sensor *co2_{nullptr};
+  sensor::Sensor *tvoc_{nullptr};
+  sensor::Sensor *aqi_{nullptr};
+
+  sensor::Sensor *humidity_{nullptr};
+  sensor::Sensor *temperature_{nullptr};
+};
+
+}  // namespace ens160
+}  // namespace esphome
diff --git a/esphome/components/ens160/sensor.py b/esphome/components/ens160/sensor.py
new file mode 100644
index 0000000000..55f0ff7b6f
--- /dev/null
+++ b/esphome/components/ens160/sensor.py
@@ -0,0 +1,88 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components import i2c, sensor
+from esphome.const import (
+    CONF_ECO2,
+    CONF_HUMIDITY,
+    CONF_ID,
+    CONF_TEMPERATURE,
+    CONF_TVOC,
+    DEVICE_CLASS_AQI,
+    DEVICE_CLASS_CARBON_DIOXIDE,
+    DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+    ICON_CHEMICAL_WEAPON,
+    ICON_MOLECULE_CO2,
+    ICON_RADIATOR,
+    STATE_CLASS_MEASUREMENT,
+    UNIT_PARTS_PER_BILLION,
+    UNIT_PARTS_PER_MILLION,
+)
+
+CODEOWNERS = ["@vincentscode"]
+DEPENDENCIES = ["i2c"]
+
+ens160_ns = cg.esphome_ns.namespace("ens160")
+ENS160Component = ens160_ns.class_(
+    "ENS160Component", cg.PollingComponent, i2c.I2CDevice, sensor.Sensor
+)
+
+CONF_AQI = "aqi"
+CONF_COMPENSATION = "compensation"
+UNIT_INDEX = "index"
+
+CONFIG_SCHEMA = (
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(ENS160Component),
+            cv.Required(CONF_ECO2): sensor.sensor_schema(
+                unit_of_measurement=UNIT_PARTS_PER_MILLION,
+                icon=ICON_MOLECULE_CO2,
+                accuracy_decimals=0,
+                device_class=DEVICE_CLASS_CARBON_DIOXIDE,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Required(CONF_TVOC): sensor.sensor_schema(
+                unit_of_measurement=UNIT_PARTS_PER_BILLION,
+                icon=ICON_RADIATOR,
+                accuracy_decimals=0,
+                device_class=DEVICE_CLASS_VOLATILE_ORGANIC_COMPOUNDS,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Required(CONF_AQI): sensor.sensor_schema(
+                unit_of_measurement=UNIT_INDEX,
+                icon=ICON_CHEMICAL_WEAPON,
+                accuracy_decimals=0,
+                device_class=DEVICE_CLASS_AQI,
+                state_class=STATE_CLASS_MEASUREMENT,
+            ),
+            cv.Optional(CONF_COMPENSATION): cv.Schema(
+                {
+                    cv.Required(CONF_TEMPERATURE): cv.use_id(sensor.Sensor),
+                    cv.Required(CONF_HUMIDITY): cv.use_id(sensor.Sensor),
+                }
+            ),
+        }
+    )
+    .extend(cv.polling_component_schema("60s"))
+    .extend(i2c.i2c_device_schema(0x53))
+)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+    await i2c.register_i2c_device(var, config)
+
+    sens = await sensor.new_sensor(config[CONF_ECO2])
+    cg.add(var.set_co2(sens))
+    sens = await sensor.new_sensor(config[CONF_TVOC])
+    cg.add(var.set_tvoc(sens))
+    sens = await sensor.new_sensor(config[CONF_AQI])
+    cg.add(var.set_aqi(sens))
+
+    if CONF_COMPENSATION in config:
+        compensation_config = config[CONF_COMPENSATION]
+        sens = await cg.get_variable(compensation_config[CONF_TEMPERATURE])
+        cg.add(var.set_temperature(sens))
+        sens = await cg.get_variable(compensation_config[CONF_HUMIDITY])
+        cg.add(var.set_humidity(sens))
diff --git a/tests/test3.1.yaml b/tests/test3.1.yaml
index 151e53fd62..9000636f63 100644
--- a/tests/test3.1.yaml
+++ b/tests/test3.1.yaml
@@ -225,6 +225,13 @@ sensor:
       name: "ADE7953 Reactive Power B"
     update_interval: 1s
 
+  - platform: ens160
+    eco2:
+      name: "ENS160 eCO2"
+    tvoc:
+      name: "ENS160 Total Volatile Organic Compounds"
+    aqi:
+      name: "ENS160 Air Quality Index"
   - platform: tmp102
     name: TMP102 Temperature
   - platform: hm3301
@@ -424,7 +431,6 @@ switch:
             direction: BACKWARD
             id: test_motor
 
-
 custom_component:
   lambda: |-
     auto s = new CustomComponent();
@@ -613,7 +619,6 @@ mcp23017:
 mcp23008:
   id: mcp23008_hub
 
-
 light:
   - platform: hbridge
     name: Icicle Lights
@@ -633,7 +638,6 @@ ttp229_bsf:
   sdo_pin: D2
   scl_pin: D1
 
-
 display:
   - platform: max7219digit
     cs_pin: GPIO15
@@ -645,7 +649,6 @@ display:
     lambda: |-
       it.printdigit("hello");
 
-
 http_request:
   useragent: esphome/device
   timeout: 10s
diff --git a/tests/test3.yaml b/tests/test3.yaml
index 41ded7ee39..0a405a2841 100644
--- a/tests/test3.yaml
+++ b/tests/test3.yaml
@@ -480,7 +480,6 @@ sensor:
       name: PZEMDC Power
     energy:
       name: PZEMDC Energy
-
   - platform: pmsx003
     uart_id: uart_9
     type: PMSX003

From 9f84b6390dd7808ac6f6a5e9c68f44ebf0c3c417 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 24 Nov 2023 09:27:56 +0100
Subject: [PATCH 21/25] Bump actions/github-script from 6.4.1 to 7.0.1 (#5803)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/needs-docs.yml | 2 +-
 .github/workflows/release.yml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/needs-docs.yml b/.github/workflows/needs-docs.yml
index 5019d64752..628b5cc5e3 100644
--- a/.github/workflows/needs-docs.yml
+++ b/.github/workflows/needs-docs.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check for needs-docs label
-        uses: actions/github-script@v6.4.1
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const { data: labels } = await github.rest.issues.listLabelsOnIssue({
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0e23db521a..69456619e9 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -203,7 +203,7 @@ jobs:
     needs: [deploy-manifest]
     steps:
       - name: Trigger Workflow
-        uses: actions/github-script@v6.4.1
+        uses: actions/github-script@v7.0.1
         with:
           github-token: ${{ secrets.DEPLOY_HA_ADDON_REPO_TOKEN }}
           script: |

From 91f1aa05ad31d94d79d92e516dab4719b70331fe Mon Sep 17 00:00:00 2001
From: Fabian <Fabian-Schmidt@users.noreply.github.com>
Date: Sat, 25 Nov 2023 00:18:36 +0100
Subject: [PATCH 22/25] Run all tests when local testing. (#5717)

---
 script/test | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/script/test b/script/test
index 36a58cd75a..e227c17f9f 100755
--- a/script/test
+++ b/script/test
@@ -6,12 +6,6 @@ cd "$(dirname "$0")/.."
 
 set -x
 
-esphome compile tests/test1.yaml
-esphome compile tests/test2.yaml
-esphome compile tests/test3.yaml
-esphome compile tests/test3.1.yaml
-esphome compile tests/test4.yaml
-esphome compile tests/test5.yaml
-esphome compile tests/test6.yaml
-esphome compile tests/test7.yaml
-esphome compile tests/test8.yaml
+for f in ./tests/test*.yaml; do
+  esphome compile $f
+done

From 636ee2b59713f0f0bad72565658d1f3a176ff0a0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 24 Nov 2023 18:06:57 -0600
Subject: [PATCH 23/25] Bump aioesphomeapi from 18.5.7 to 18.5.9 (#5830)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index af53dc8e94..3d096ffd52 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ platformio==6.1.11  # When updating platformio, also update Dockerfile
 esptool==4.6.2
 click==8.1.7
 esphome-dashboard==20231107.0
-aioesphomeapi==18.5.7
+aioesphomeapi==18.5.9
 zeroconf==0.127.0
 python-magic==0.4.27
 

From c43518c391120866d31163f6890773a475b0124d Mon Sep 17 00:00:00 2001
From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>
Date: Sat, 25 Nov 2023 14:56:32 +1100
Subject: [PATCH 24/25] Allow split uart pin inversion for ESP-IDF (#5831)

---
 esphome/components/uart/__init__.py | 3 ++-
 tests/test5.yaml                    | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py
index 36f2bb5851..9005422ce6 100644
--- a/esphome/components/uart/__init__.py
+++ b/esphome/components/uart/__init__.py
@@ -75,12 +75,13 @@ def validate_rx_pin(value):
 def validate_invert_esp32(config):
     if (
         CORE.is_esp32
+        and CORE.using_arduino
         and CONF_TX_PIN in config
         and CONF_RX_PIN in config
         and config[CONF_TX_PIN][CONF_INVERTED] != config[CONF_RX_PIN][CONF_INVERTED]
     ):
         raise cv.Invalid(
-            "Different invert values for TX and RX pin are not (yet) supported for ESP32."
+            "Different invert values for TX and RX pin are not supported for ESP32 when using Arduino."
         )
     return config
 
diff --git a/tests/test5.yaml b/tests/test5.yaml
index 82c201f017..46cedcabd2 100644
--- a/tests/test5.yaml
+++ b/tests/test5.yaml
@@ -41,7 +41,9 @@ uart:
     rx_pin: 3
     baud_rate: 9600
   - id: uart_2
-    tx_pin: 17
+    tx_pin:
+      number: 17
+      inverted: true
     rx_pin: 16
     baud_rate: 19200
 

From ccd7f0661c3fa3fa283dd59058a0779ea1360df2 Mon Sep 17 00:00:00 2001
From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com>
Date: Sat, 25 Nov 2023 09:38:45 +0100
Subject: [PATCH 25/25] Add `is_detected()` for Nextion displays (#5825)

---
 esphome/components/nextion/nextion.cpp    | 3 ++-
 esphome/components/nextion/nextion_base.h | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp
index 0134595050..92afdfe3dc 100644
--- a/esphome/components/nextion/nextion.cpp
+++ b/esphome/components/nextion/nextion.cpp
@@ -93,7 +93,8 @@ bool Nextion::check_connect_() {
     connect_info.push_back(response.substr(start, end - start));
   }
 
-  if (connect_info.size() == 7) {
+  this->is_detected_ = (connect_info.size() == 7);
+  if (this->is_detected_) {
     ESP_LOGN(TAG, "Received connect_info %zu", connect_info.size());
 
     this->device_model_ = connect_info[2];
diff --git a/esphome/components/nextion/nextion_base.h b/esphome/components/nextion/nextion_base.h
index a24fd74060..ffc431ed13 100644
--- a/esphome/components/nextion/nextion_base.h
+++ b/esphome/components/nextion/nextion_base.h
@@ -48,10 +48,12 @@ class NextionBase {
 
   bool is_sleeping() { return this->is_sleeping_; }
   bool is_setup() { return this->is_setup_; }
+  bool is_detected() { return this->is_detected_; }
 
  protected:
   bool is_setup_ = false;
   bool is_sleeping_ = false;
+  bool is_detected_ = false;
 };
 
 }  // namespace nextion