diff --git a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
index 2b38853528..0f45cf95c6 100644
--- a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
+++ b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
@@ -89,6 +89,10 @@ void I2SAudioMicrophone::start_() {
 void I2SAudioMicrophone::stop() {
   if (this->state_ == microphone::STATE_STOPPED || this->is_failed())
     return;
+  if (this->state_ == microphone::STATE_STARTING) {
+    this->state_ = microphone::STATE_STOPPED;
+    return;
+  }
   this->state_ = microphone::STATE_STOPPING;
 }
 
diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
index fa41a70277..5ae597dc7c 100644
--- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
+++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp
@@ -136,6 +136,10 @@ void I2SAudioSpeaker::player_task(void *params) {
 void I2SAudioSpeaker::stop() {
   if (this->state_ == speaker::STATE_STOPPED)
     return;
+  if (this->state_ == speaker::STATE_STARTING) {
+    this->state_ = speaker::STATE_STOPPED;
+    return;
+  }
   this->state_ = speaker::STATE_STOPPING;
   DataEvent data;
   data.stop = true;