diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp
index d15d702d4b..fc5dd6e4e4 100644
--- a/esphome/components/voice_assistant/voice_assistant.cpp
+++ b/esphome/components/voice_assistant/voice_assistant.cpp
@@ -610,6 +610,11 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
       if (code == "wake-word-timeout" || code == "wake_word_detection_aborted") {
         // Don't change state here since either the "tts-end" or "run-end" events will do it.
         return;
+      } else if (code == "wake-provider-missing" || code == "wake-engine-missing") {
+        // Wake word is not set up or not ready on Home Assistant so stop and do not retry until user starts again.
+        this->request_stop();
+        this->error_trigger_->trigger(code, message);
+        return;
       }
       ESP_LOGE(TAG, "Error: %s - %s", code.c_str(), message.c_str());
       if (this->state_ != State::IDLE) {
diff --git a/esphome/const.py b/esphome/const.py
index 3b8b847de6..ddba48921e 100644
--- a/esphome/const.py
+++ b/esphome/const.py
@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.11.0b4"
+__version__ = "2023.11.0b5"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (