diff --git a/esphome/components/esp32_ble_server/ble_service.cpp b/esphome/components/esp32_ble_server/ble_service.cpp
index 4fcd2e3e79..e5aaebc137 100644
--- a/esphome/components/esp32_ble_server/ble_service.cpp
+++ b/esphome/components/esp32_ble_server/ble_service.cpp
@@ -90,6 +90,8 @@ void BLEService::stop() {
     ESP_LOGE(TAG, "esp_ble_gatts_stop_service failed: %d", err);
     return;
   }
+  esp32_ble::global_ble->get_advertising()->remove_service_uuid(this->uuid_);
+  esp32_ble::global_ble->get_advertising()->start();
   this->running_state_ = STOPPING;
 }
 
diff --git a/esphome/components/esp32_ble_server/ble_service.h b/esphome/components/esp32_ble_server/ble_service.h
index 2766c931a7..93b4217517 100644
--- a/esphome/components/esp32_ble_server/ble_service.h
+++ b/esphome/components/esp32_ble_server/ble_service.h
@@ -7,11 +7,11 @@
 
 #ifdef USE_ESP32
 
+#include <esp_bt_defs.h>
 #include <esp_gap_ble_api.h>
 #include <esp_gatt_defs.h>
 #include <esp_gattc_api.h>
 #include <esp_gatts_api.h>
-#include <esp_bt_defs.h>
 
 namespace esphome {
 namespace esp32_ble_server {
diff --git a/esphome/components/esp32_improv/__init__.py b/esphome/components/esp32_improv/__init__.py
index ae7f0b6427..fba2e55ae8 100644
--- a/esphome/components/esp32_improv/__init__.py
+++ b/esphome/components/esp32_improv/__init__.py
@@ -4,7 +4,7 @@ from esphome.components import binary_sensor, output, esp32_ble_server
 from esphome.const import CONF_ID
 
 
-AUTO_LOAD = ["binary_sensor", "output", "esp32_ble_server"]
+AUTO_LOAD = ["esp32_ble_server"]
 CODEOWNERS = ["@jesserockz"]
 CONFLICTS_WITH = ["esp32_ble_beacon"]
 DEPENDENCIES = ["wifi", "esp32"]
diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp
index 85013c006b..5bdf7d19fe 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.cpp
+++ b/esphome/components/esp32_improv/esp32_improv_component.cpp
@@ -18,6 +18,17 @@ ESP32ImprovComponent::ESP32ImprovComponent() { global_improv_component = this; }
 void ESP32ImprovComponent::setup() {
   this->service_ = global_ble_server->create_service(improv::SERVICE_UUID, true);
   this->setup_characteristics();
+
+#ifdef USE_BINARY_SENSOR
+  if (this->authorizer_ != nullptr) {
+    this->authorizer_->add_on_state_callback([this](bool state) {
+      if (state) {
+        this->authorized_start_ = millis();
+        this->identify_start_ = 0;
+      }
+    });
+  }
+#endif
 }
 
 void ESP32ImprovComponent::setup_characteristics() {
@@ -50,8 +61,10 @@ void ESP32ImprovComponent::setup_characteristics() {
   BLEDescriptor *capabilities_descriptor = new BLE2902();
   this->capabilities_->add_descriptor(capabilities_descriptor);
   uint8_t capabilities = 0x00;
+#ifdef USE_OUTPUT
   if (this->status_indicator_ != nullptr)
     capabilities |= improv::CAPABILITY_IDENTIFY;
+#endif
   this->capabilities_->set_value(capabilities);
   this->setup_complete_ = true;
 }
@@ -63,8 +76,7 @@ void ESP32ImprovComponent::loop() {
 
   switch (this->state_) {
     case improv::STATE_STOPPED:
-      if (this->status_indicator_ != nullptr)
-        this->status_indicator_->turn_off();
+      this->set_status_indicator_state_(false);
 
       if (this->service_->is_created() && this->should_start_ && this->setup_complete_) {
         if (this->service_->is_running()) {
@@ -80,14 +92,17 @@ void ESP32ImprovComponent::loop() {
       }
       break;
     case improv::STATE_AWAITING_AUTHORIZATION: {
-      if (this->authorizer_ == nullptr || this->authorizer_->state) {
+#ifdef USE_BINARY_SENSOR
+      if (this->authorizer_ == nullptr ||
+          (this->authorized_start_ != 0 && ((now - this->authorized_start_) < this->authorized_duration_))) {
         this->set_state_(improv::STATE_AUTHORIZED);
-        this->authorized_start_ = now;
-      } else {
-        if (this->status_indicator_ != nullptr) {
-          if (!this->check_identify_())
-            this->status_indicator_->turn_on();
-        }
+      } else
+#else
+      this->set_state_(improv::STATE_AUTHORIZED);
+#endif
+      {
+        if (!this->check_identify_())
+          this->set_status_indicator_state_(true);
       }
       break;
     }
@@ -99,25 +114,13 @@ void ESP32ImprovComponent::loop() {
           return;
         }
       }
-      if (this->status_indicator_ != nullptr) {
-        if (!this->check_identify_()) {
-          if ((now % 1000) < 500) {
-            this->status_indicator_->turn_on();
-          } else {
-            this->status_indicator_->turn_off();
-          }
-        }
+      if (!this->check_identify_()) {
+        this->set_status_indicator_state_((now % 1000) < 500);
       }
       break;
     }
     case improv::STATE_PROVISIONING: {
-      if (this->status_indicator_ != nullptr) {
-        if ((now % 200) < 100) {
-          this->status_indicator_->turn_on();
-        } else {
-          this->status_indicator_->turn_off();
-        }
-      }
+      this->set_status_indicator_state_((now % 200) < 100);
       if (wifi::global_wifi_component->is_connected()) {
         wifi::global_wifi_component->save_wifi_sta(this->connecting_sta_.get_ssid(),
                                                    this->connecting_sta_.get_password());
@@ -142,13 +145,27 @@ void ESP32ImprovComponent::loop() {
     }
     case improv::STATE_PROVISIONED: {
       this->incoming_data_.clear();
-      if (this->status_indicator_ != nullptr)
-        this->status_indicator_->turn_off();
+      this->set_status_indicator_state_(false);
       break;
     }
   }
 }
 
+void ESP32ImprovComponent::set_status_indicator_state_(bool state) {
+#ifdef USE_OUTPUT
+  if (this->status_indicator_ == nullptr)
+    return;
+  if (this->status_indicator_state_ == state)
+    return;
+  this->status_indicator_state_ = state;
+  if (state) {
+    this->status_indicator_->turn_on();
+  } else {
+    this->status_indicator_->turn_off();
+  }
+#endif
+}
+
 bool ESP32ImprovComponent::check_identify_() {
   uint32_t now = millis();
 
@@ -156,11 +173,7 @@ bool ESP32ImprovComponent::check_identify_() {
 
   if (identify) {
     uint32_t time = now % 1000;
-    if (time < 600 && time % 200 < 100) {
-      this->status_indicator_->turn_on();
-    } else {
-      this->status_indicator_->turn_off();
-    }
+    this->set_status_indicator_state_(time < 600 && time % 200 < 100);
   }
   return identify;
 }
@@ -213,8 +226,12 @@ float ESP32ImprovComponent::get_setup_priority() const { return setup_priority::
 
 void ESP32ImprovComponent::dump_config() {
   ESP_LOGCONFIG(TAG, "ESP32 Improv:");
+#ifdef USE_BINARY_SENSOR
   LOG_BINARY_SENSOR("  ", "Authorizer", this->authorizer_);
+#endif
+#ifdef USE_OUTPUT
   ESP_LOGCONFIG(TAG, "  Status Indicator: '%s'", YESNO(this->status_indicator_ != nullptr));
+#endif
 }
 
 void ESP32ImprovComponent::process_incoming_data_() {
diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h
index 1a142c94b6..ba9892d6a5 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.h
+++ b/esphome/components/esp32_improv/esp32_improv_component.h
@@ -1,14 +1,22 @@
 #pragma once
 
-#include "esphome/components/binary_sensor/binary_sensor.h"
-#include "esphome/components/esp32_ble_server/ble_characteristic.h"
-#include "esphome/components/esp32_ble_server/ble_server.h"
-#include "esphome/components/output/binary_output.h"
-#include "esphome/components/wifi/wifi_component.h"
 #include "esphome/core/component.h"
+#include "esphome/core/defines.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
 
+#include "esphome/components/esp32_ble_server/ble_characteristic.h"
+#include "esphome/components/esp32_ble_server/ble_server.h"
+#include "esphome/components/wifi/wifi_component.h"
+
+#ifdef USE_BINARY_SENSOR
+#include "esphome/components/binary_sensor/binary_sensor.h"
+#endif
+
+#ifdef USE_OUTPUT
+#include "esphome/components/output/binary_output.h"
+#endif
+
 #include <vector>
 
 #ifdef USE_ESP32
@@ -34,8 +42,12 @@ class ESP32ImprovComponent : public Component, public BLEServiceComponent {
   void stop() override;
   bool is_active() const { return this->state_ != improv::STATE_STOPPED; }
 
+#ifdef USE_BINARY_SENSOR
   void set_authorizer(binary_sensor::BinarySensor *authorizer) { this->authorizer_ = authorizer; }
+#endif
+#ifdef USE_OUTPUT
   void set_status_indicator(output::BinaryOutput *status_indicator) { this->status_indicator_ = status_indicator; }
+#endif
   void set_identify_duration(uint32_t identify_duration) { this->identify_duration_ = identify_duration; }
   void set_authorized_duration(uint32_t authorized_duration) { this->authorized_duration_ = authorized_duration; }
 
@@ -58,12 +70,19 @@ class ESP32ImprovComponent : public Component, public BLEServiceComponent {
   BLECharacteristic *rpc_response_;
   BLECharacteristic *capabilities_;
 
+#ifdef USE_BINARY_SENSOR
   binary_sensor::BinarySensor *authorizer_{nullptr};
+#endif
+#ifdef USE_OUTPUT
   output::BinaryOutput *status_indicator_{nullptr};
+#endif
 
   improv::State state_{improv::STATE_STOPPED};
   improv::Error error_state_{improv::ERROR_NONE};
 
+  bool status_indicator_state_{false};
+  void set_status_indicator_state_(bool state);
+
   void set_state_(improv::State state);
   void set_error_(improv::Error error);
   void send_response_(std::vector<uint8_t> &response);
diff --git a/esphome/components/output/__init__.py b/esphome/components/output/__init__.py
index 4f1fb33fe7..726d1ac084 100644
--- a/esphome/components/output/__init__.py
+++ b/esphome/components/output/__init__.py
@@ -106,4 +106,5 @@ async def output_set_level_to_code(config, action_id, template_arg, args):
 
 
 async def to_code(config):
+    cg.add_define("USE_OUTPUT")
     cg.add_global(output_ns.using)
diff --git a/esphome/core/defines.h b/esphome/core/defines.h
index 1e0df74eec..71493119c0 100644
--- a/esphome/core/defines.h
+++ b/esphome/core/defines.h
@@ -37,6 +37,7 @@
 #define USE_OTA
 #define USE_OTA_PASSWORD
 #define USE_OTA_STATE_CALLBACK
+#define USE_OUTPUT
 #define USE_POWER_SUPPLY
 #define USE_QR_CODE
 #define USE_SELECT
@@ -117,6 +118,6 @@
 #endif
 
 // Disabled feature flags
-//#define USE_BSEC  // Requires a library with proprietary license.
+// #define USE_BSEC  // Requires a library with proprietary license.
 
 #define USE_DASHBOARD_IMPORT