diff --git a/CODEOWNERS b/CODEOWNERS
index 3f1abae5df..0594a60ef6 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -36,6 +36,7 @@ esphome/components/dfplayer/* @glmnet
 esphome/components/dht/* @OttoWinter
 esphome/components/ds1307/* @badbadc0ffee
 esphome/components/esp32_ble/* @jesserockz
+esphome/components/esp32_ble_server/* @jesserockz
 esphome/components/esp32_improv/* @jesserockz
 esphome/components/exposure_notifications/* @OttoWinter
 esphome/components/ezo/* @ssieb
diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py
index 6437216f69..ccf1f6cafe 100644
--- a/esphome/components/esp32_ble/__init__.py
+++ b/esphome/components/esp32_ble/__init__.py
@@ -1,30 +1,18 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
-from esphome.const import CONF_ID, CONF_MODEL, ESP_PLATFORM_ESP32
+from esphome.const import CONF_ID, ESP_PLATFORM_ESP32
 
 ESP_PLATFORMS = [ESP_PLATFORM_ESP32]
 CODEOWNERS = ["@jesserockz"]
-
-CONF_MANUFACTURER = "manufacturer"
-CONF_SERVER = "server"
+CONFLICTS_WITH = ["esp32_ble_tracker", "esp32_ble_beacon"]
 
 esp32_ble_ns = cg.esphome_ns.namespace("esp32_ble")
 ESP32BLE = esp32_ble_ns.class_("ESP32BLE", cg.Component)
-BLEServer = esp32_ble_ns.class_("BLEServer", cg.Component)
-
-BLEServiceComponent = esp32_ble_ns.class_("BLEServiceComponent")
 
 
 CONFIG_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.declare_id(ESP32BLE),
-        cv.Optional(CONF_SERVER): cv.Schema(
-            {
-                cv.GenerateID(): cv.declare_id(BLEServer),
-                cv.Optional(CONF_MANUFACTURER, default="ESPHome"): cv.string,
-                cv.Optional(CONF_MODEL): cv.string,
-            }
-        ),
     }
 ).extend(cv.COMPONENT_SCHEMA)
 
@@ -32,13 +20,3 @@ CONFIG_SCHEMA = cv.Schema(
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
-
-    if CONF_SERVER in config:
-        conf = config[CONF_SERVER]
-        server = cg.new_Pvariable(conf[CONF_ID])
-        await cg.register_component(server, conf)
-        cg.add(server.set_manufacturer(conf[CONF_MANUFACTURER]))
-        if CONF_MODEL in conf:
-            cg.add(server.set_model(conf[CONF_MODEL]))
-        cg.add_define("USE_ESP32_BLE_SERVER")
-        cg.add(var.set_server(server))
diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp
index 889c793017..18e80754df 100644
--- a/esphome/components/esp32_ble/ble.cpp
+++ b/esphome/components/esp32_ble/ble.cpp
@@ -1,4 +1,5 @@
 #include "ble.h"
+
 #include "esphome/core/application.h"
 #include "esphome/core/log.h"
 
@@ -26,14 +27,22 @@ void ESP32BLE::setup() {
     return;
   }
 
+  this->advertising_ = new BLEAdvertising();
+
+  this->advertising_->set_scan_response(true);
+  this->advertising_->set_min_preferred_interval(0x06);
+  this->advertising_->start();
+
   ESP_LOGD(TAG, "BLE setup complete");
 }
 
 void ESP32BLE::mark_failed() {
   Component::mark_failed();
+#ifdef USE_ESP32_BLE_SERVER
   if (this->server_ != nullptr) {
     this->server_->mark_failed();
   }
+#endif
 }
 
 bool ESP32BLE::ble_setup_() {
@@ -142,7 +151,9 @@ void ESP32BLE::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gat
 void ESP32BLE::real_gatts_event_handler_(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if,
                                          esp_ble_gatts_cb_param_t *param) {
   ESP_LOGV(TAG, "(BLE) gatts_event [esp_gatt_if: %d] - %d", gatts_if, event);
+#ifdef USE_ESP32_BLE_SERVER
   this->server_->gatts_event_handler(event, gatts_if, param);
+#endif
 }
 
 void ESP32BLE::real_gattc_event_handler_(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if,
diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h
index 27cb9a2092..149f0008a4 100644
--- a/esphome/components/esp32_ble/ble.h
+++ b/esphome/components/esp32_ble/ble.h
@@ -1,16 +1,21 @@
 #pragma once
 
+#include "ble_advertising.h"
+
 #include "esphome/core/component.h"
+#include "esphome/core/defines.h"
 #include "esphome/core/helpers.h"
-#include "ble_server.h"
 #include "queue.h"
 
+#ifdef USE_ESP32_BLE_SERVER
+#include "esphome/components/esp32_ble_server/ble_server.h"
+#endif
+
 #ifdef ARDUINO_ARCH_ESP32
 
 #include <esp_gap_ble_api.h>
 #include <esp_gatts_api.h>
 #include <esp_gattc_api.h>
-
 namespace esphome {
 namespace esp32_ble {
 
@@ -28,11 +33,20 @@ class ESP32BLE : public Component {
   float get_setup_priority() const override;
   void mark_failed() override;
 
-  bool has_server() { return this->server_ != nullptr; }
+  bool has_server() {
+#ifdef USE_ESP32_BLE_SERVER
+    return this->server_ != nullptr;
+#else
+    return false;
+#endif
+  }
   bool has_client() { return false; }
 
-  void set_server(BLEServer *server) { this->server_ = server; }
+  BLEAdvertising *get_advertising() { return this->advertising_; }
 
+#ifdef USE_ESP32_BLE_SERVER
+  void set_server(esp32_ble_server::BLEServer *server) { this->server_ = server; }
+#endif
  protected:
   static void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if, esp_ble_gatts_cb_param_t *param);
   static void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param);
@@ -44,8 +58,11 @@ class ESP32BLE : public Component {
 
   bool ble_setup_();
 
-  BLEServer *server_{nullptr};
+#ifdef USE_ESP32_BLE_SERVER
+  esp32_ble_server::BLEServer *server_{nullptr};
+#endif
   Queue<BLEEvent> ble_events_;
+  BLEAdvertising *advertising_;
 };
 
 extern ESP32BLE *global_ble;
diff --git a/esphome/components/esp32_ble/ble_advertising.cpp b/esphome/components/esp32_ble/ble_advertising.cpp
index 6162bf3a40..f215fb48a3 100644
--- a/esphome/components/esp32_ble/ble_advertising.cpp
+++ b/esphome/components/esp32_ble/ble_advertising.cpp
@@ -32,6 +32,10 @@ BLEAdvertising::BLEAdvertising() {
 }
 
 void BLEAdvertising::add_service_uuid(ESPBTUUID uuid) { this->advertising_uuids_.push_back(uuid); }
+void BLEAdvertising::remove_service_uuid(ESPBTUUID uuid) {
+  this->advertising_uuids_.erase(std::remove(this->advertising_uuids_.begin(), this->advertising_uuids_.end(), uuid),
+                                 this->advertising_uuids_.end());
+}
 
 void BLEAdvertising::start() {
   int num_services = this->advertising_uuids_.size();
diff --git a/esphome/components/esp32_ble/ble_advertising.h b/esphome/components/esp32_ble/ble_advertising.h
index 405edbf482..d86089f333 100644
--- a/esphome/components/esp32_ble/ble_advertising.h
+++ b/esphome/components/esp32_ble/ble_advertising.h
@@ -17,6 +17,7 @@ class BLEAdvertising {
   BLEAdvertising();
 
   void add_service_uuid(ESPBTUUID uuid);
+  void remove_service_uuid(ESPBTUUID uuid);
   void set_scan_response(bool scan_response) { this->scan_response_ = scan_response; }
   void set_min_preferred_interval(uint16_t interval) { this->advertising_data_.min_interval = interval; }
 
diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py
new file mode 100644
index 0000000000..0dcbcadc50
--- /dev/null
+++ b/esphome/components/esp32_ble_server/__init__.py
@@ -0,0 +1,39 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID, CONF_MODEL, ESP_PLATFORM_ESP32
+from esphome.components import esp32_ble
+
+AUTO_LOAD = ["esp32_ble"]
+CODEOWNERS = ["@jesserockz"]
+CONFLICTS_WITH = ["esp32_ble_tracker", "esp32_ble_beacon"]
+ESP_PLATFORMS = [ESP_PLATFORM_ESP32]
+
+CONF_MANUFACTURER = "manufacturer"
+CONF_BLE_ID = "ble_id"
+
+esp32_ble_server_ns = cg.esphome_ns.namespace("esp32_ble_server")
+BLEServer = esp32_ble_server_ns.class_("BLEServer", cg.Component)
+BLEServiceComponent = esp32_ble_server_ns.class_("BLEServiceComponent")
+
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(BLEServer),
+        cv.GenerateID(CONF_BLE_ID): cv.use_id(esp32_ble.ESP32BLE),
+        cv.Optional(CONF_MANUFACTURER, default="ESPHome"): cv.string,
+        cv.Optional(CONF_MODEL): cv.string,
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+async def to_code(config):
+    parent = await cg.get_variable(config[CONF_BLE_ID])
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+
+    cg.add(var.set_manufacturer(config[CONF_MANUFACTURER]))
+    if CONF_MODEL in config:
+        cg.add(var.set_model(config[CONF_MODEL]))
+    cg.add_define("USE_ESP32_BLE_SERVER")
+
+    cg.add(parent.set_server(var))
diff --git a/esphome/components/esp32_ble/ble_2901.cpp b/esphome/components/esp32_ble_server/ble_2901.cpp
similarity index 67%
rename from esphome/components/esp32_ble/ble_2901.cpp
rename to esphome/components/esp32_ble_server/ble_2901.cpp
index 952d82d5f0..962ba3ffbe 100644
--- a/esphome/components/esp32_ble/ble_2901.cpp
+++ b/esphome/components/esp32_ble_server/ble_2901.cpp
@@ -1,18 +1,18 @@
 #include "ble_2901.h"
-#include "ble_uuid.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
 BLE2901::BLE2901(const std::string &value) : BLE2901((uint8_t *) value.data(), value.length()) {}
-BLE2901::BLE2901(const uint8_t *data, size_t length) : BLEDescriptor(ESPBTUUID::from_uint16(0x2901)) {
+BLE2901::BLE2901(const uint8_t *data, size_t length) : BLEDescriptor(esp32_ble::ESPBTUUID::from_uint16(0x2901)) {
   this->set_value(data, length);
   this->permissions_ = ESP_GATT_PERM_READ;
 }
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_2901.h b/esphome/components/esp32_ble_server/ble_2901.h
similarity index 80%
rename from esphome/components/esp32_ble/ble_2901.h
rename to esphome/components/esp32_ble_server/ble_2901.h
index e606b6271a..3bb23ae69d 100644
--- a/esphome/components/esp32_ble/ble_2901.h
+++ b/esphome/components/esp32_ble_server/ble_2901.h
@@ -5,7 +5,7 @@
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
 class BLE2901 : public BLEDescriptor {
  public:
@@ -13,7 +13,7 @@ class BLE2901 : public BLEDescriptor {
   BLE2901(const uint8_t *data, size_t length);
 };
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_2902.cpp b/esphome/components/esp32_ble_server/ble_2902.cpp
similarity index 51%
rename from esphome/components/esp32_ble/ble_2902.cpp
rename to esphome/components/esp32_ble_server/ble_2902.cpp
index 164c8f40ff..0a87b239f9 100644
--- a/esphome/components/esp32_ble/ble_2902.cpp
+++ b/esphome/components/esp32_ble_server/ble_2902.cpp
@@ -1,18 +1,18 @@
 #include "ble_2902.h"
-#include "ble_uuid.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
-BLE2902::BLE2902() : BLEDescriptor(ESPBTUUID::from_uint16(0x2902)) {
+BLE2902::BLE2902() : BLEDescriptor(esp32_ble::ESPBTUUID::from_uint16(0x2902)) {
   this->value_.attr_len = 2;
   uint8_t data[2] = {0, 0};
   memcpy(this->value_.attr_value, data, 2);
 }
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_2902.h b/esphome/components/esp32_ble_server/ble_2902.h
similarity index 75%
rename from esphome/components/esp32_ble/ble_2902.h
rename to esphome/components/esp32_ble_server/ble_2902.h
index 356a0bb107..024eec755e 100644
--- a/esphome/components/esp32_ble/ble_2902.h
+++ b/esphome/components/esp32_ble_server/ble_2902.h
@@ -5,14 +5,14 @@
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
 class BLE2902 : public BLEDescriptor {
  public:
   BLE2902();
 };
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_characteristic.cpp b/esphome/components/esp32_ble_server/ble_characteristic.cpp
similarity index 86%
rename from esphome/components/esp32_ble/ble_characteristic.cpp
rename to esphome/components/esp32_ble_server/ble_characteristic.cpp
index 127f973146..23b2693839 100644
--- a/esphome/components/esp32_ble/ble_characteristic.cpp
+++ b/esphome/components/esp32_ble_server/ble_characteristic.cpp
@@ -7,9 +7,9 @@
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
-static const char *TAG = "esp32_ble.characteristic";
+static const char *const TAG = "esp32_ble_server.characteristic";
 
 BLECharacteristic::BLECharacteristic(const ESPBTUUID uuid, uint32_t properties) : uuid_(uuid) {
   this->set_value_lock_ = xSemaphoreCreateBinary();
@@ -148,39 +148,39 @@ bool BLECharacteristic::is_failed() {
 
 void BLECharacteristic::set_broadcast_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_BROADCAST);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_BROADCAST);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_BROADCAST);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_BROADCAST);
 }
 void BLECharacteristic::set_indicate_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_INDICATE);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_INDICATE);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_INDICATE);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_INDICATE);
 }
 void BLECharacteristic::set_notify_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_NOTIFY);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_NOTIFY);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_NOTIFY);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_NOTIFY);
 }
 void BLECharacteristic::set_read_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_READ);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_READ);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_READ);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_READ);
 }
 void BLECharacteristic::set_write_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_WRITE);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_WRITE);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_WRITE);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_WRITE);
 }
 void BLECharacteristic::set_write_no_response_property(bool value) {
   if (value)
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ | ESP_GATT_CHAR_PROP_BIT_WRITE_NR);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ | ESP_GATT_CHAR_PROP_BIT_WRITE_NR);
   else
-    this->properties_ = (esp_gatt_char_prop_t)(this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_WRITE_NR);
+    this->properties_ = (esp_gatt_char_prop_t) (this->properties_ & ~ESP_GATT_CHAR_PROP_BIT_WRITE_NR);
 }
 
 void BLECharacteristic::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if,
@@ -300,7 +300,7 @@ void BLECharacteristic::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt
   }
 }
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_characteristic.h b/esphome/components/esp32_ble_server/ble_characteristic.h
similarity index 94%
rename from esphome/components/esp32_ble/ble_characteristic.h
rename to esphome/components/esp32_ble_server/ble_characteristic.h
index 648cfb939d..bc5033f2ae 100644
--- a/esphome/components/esp32_ble/ble_characteristic.h
+++ b/esphome/components/esp32_ble_server/ble_characteristic.h
@@ -1,9 +1,9 @@
 #pragma once
 
-#include <vector>
-
-#include "ble_uuid.h"
 #include "ble_descriptor.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
+
+#include <vector>
 
 #ifdef ARDUINO_ARCH_ESP32
 
@@ -14,7 +14,9 @@
 #include <esp_bt_defs.h>
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
+
+using namespace esp32_ble;
 
 class BLEService;
 
@@ -89,7 +91,7 @@ class BLECharacteristic {
   } state_{INIT};
 };
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_descriptor.cpp b/esphome/components/esp32_ble_server/ble_descriptor.cpp
similarity index 95%
rename from esphome/components/esp32_ble/ble_descriptor.cpp
rename to esphome/components/esp32_ble_server/ble_descriptor.cpp
index 9738cc2fe7..a04343da3a 100644
--- a/esphome/components/esp32_ble/ble_descriptor.cpp
+++ b/esphome/components/esp32_ble_server/ble_descriptor.cpp
@@ -7,9 +7,9 @@
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
-static const char *TAG = "esp32_ble.descriptor";
+static const char *const TAG = "esp32_ble_server.descriptor";
 
 BLEDescriptor::BLEDescriptor(ESPBTUUID uuid, uint16_t max_len) {
   this->uuid_ = uuid;
@@ -71,7 +71,7 @@ void BLEDescriptor::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_
   }
 }
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_descriptor.h b/esphome/components/esp32_ble_server/ble_descriptor.h
similarity index 87%
rename from esphome/components/esp32_ble/ble_descriptor.h
rename to esphome/components/esp32_ble_server/ble_descriptor.h
index bc7b48e331..1a72cb2b54 100644
--- a/esphome/components/esp32_ble/ble_descriptor.h
+++ b/esphome/components/esp32_ble_server/ble_descriptor.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "ble_uuid.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
@@ -8,7 +8,9 @@
 #include <esp_gatts_api.h>
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
+
+using namespace esp32_ble;
 
 class BLECharacteristic;
 
@@ -43,7 +45,7 @@ class BLEDescriptor {
   } state_{INIT};
 };
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp
similarity index 84%
rename from esphome/components/esp32_ble/ble_server.cpp
rename to esphome/components/esp32_ble_server/ble_server.cpp
index 9d5be46113..db83eb6bee 100644
--- a/esphome/components/esp32_ble/ble_server.cpp
+++ b/esphome/components/esp32_ble_server/ble_server.cpp
@@ -1,5 +1,6 @@
 #include "ble_server.h"
-#include "ble.h"
+
+#include "esphome/components/esp32_ble/ble.h"
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
 #include "esphome/core/version.h"
@@ -14,11 +15,11 @@
 #include <esp_gap_ble_api.h>
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
-static const char *TAG = "esp32_ble.server";
+static const char *const TAG = "esp32_ble_server";
 
-static const uint16_t device_information_service__UUID = 0x180A;
+static const uint16_t DEVICE_INFORMATION_SERVICE_UUID = 0x180A;
 static const uint16_t MODEL_UUID = 0x2A24;
 static const uint16_t VERSION_UUID = 0x2A26;
 static const uint16_t MANUFACTURER_UUID = 0x2A29;
@@ -32,8 +33,6 @@ void BLEServer::setup() {
   ESP_LOGD(TAG, "Setting up BLE Server...");
 
   global_ble_server = this;
-
-  this->advertising_ = new BLEAdvertising();
 }
 
 void BLEServer::loop() {
@@ -53,35 +52,27 @@ void BLEServer::loop() {
     }
     case REGISTERING: {
       if (this->registered_) {
-        this->device_information_service_ = this->create_service(device_information_service__UUID);
+        this->device_information_service_ = this->create_service(DEVICE_INFORMATION_SERVICE_UUID);
 
         this->create_device_characteristics_();
 
-        this->advertising_->set_scan_response(true);
-        this->advertising_->set_min_preferred_interval(0x06);
-        this->advertising_->start();
-
         this->state_ = STARTING_SERVICE;
       }
       break;
     }
     case STARTING_SERVICE: {
+      if (!this->device_information_service_->is_created()) {
+        break;
+      }
       if (this->device_information_service_->is_running()) {
-        for (auto *component : this->service_components_) {
-          component->setup_service();
-        }
-        this->state_ = SETTING_UP_COMPONENT_SERVICES;
+        this->state_ = RUNNING;
+        this->can_proceed_ = true;
+        ESP_LOGD(TAG, "BLE server setup successfully");
       } else if (!this->device_information_service_->is_starting()) {
         this->device_information_service_->start();
       }
       break;
     }
-    case SETTING_UP_COMPONENT_SERVICES: {
-      this->state_ = RUNNING;
-      this->can_proceed_ = true;
-      ESP_LOGD(TAG, "BLE server setup successfully");
-      break;
-    }
   }
 }
 
@@ -123,7 +114,7 @@ BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t n
   BLEService *service = new BLEService(uuid, num_handles, inst_id);
   this->services_.push_back(service);
   if (advertise) {
-    this->advertising_->add_service_uuid(uuid);
+    esp32_ble::global_ble->get_advertising()->add_service_uuid(uuid);
   }
   service->do_create(this);
   return service;
@@ -145,7 +136,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga
       ESP_LOGD(TAG, "BLE Client disconnected");
       if (this->remove_client_(param->disconnect.conn_id))
         this->connected_clients_--;
-      this->advertising_->start();
+      esp32_ble::global_ble->get_advertising()->start();
       for (auto *component : this->service_components_) {
         component->on_client_disconnect();
       }
@@ -171,7 +162,7 @@ void BLEServer::dump_config() { ESP_LOGCONFIG(TAG, "ESP32 BLE Server:"); }
 
 BLEServer *global_ble_server = nullptr;
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h
similarity index 89%
rename from esphome/components/esp32_ble/ble_server.h
rename to esphome/components/esp32_ble_server/ble_server.h
index 5657773446..9d955dda79 100644
--- a/esphome/components/esp32_ble/ble_server.h
+++ b/esphome/components/esp32_ble_server/ble_server.h
@@ -1,15 +1,16 @@
 #pragma once
 
+#include "ble_service.h"
+#include "ble_characteristic.h"
+
+#include "esphome/components/esp32_ble/ble_advertising.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
+#include "esphome/components/esp32_ble/queue.h"
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
-#include "ble_service.h"
-#include "ble_characteristic.h"
-#include "ble_uuid.h"
-#include "ble_advertising.h"
-#include <map>
 
-#include "queue.h"
+#include <map>
 
 #ifdef ARDUINO_ARCH_ESP32
 
@@ -17,11 +18,12 @@
 #include <esp_gatts_api.h>
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
+
+using namespace esp32_ble;
 
 class BLEServiceComponent {
  public:
-  virtual void setup_service();
   virtual void on_client_connect(){};
   virtual void on_client_disconnect(){};
   virtual void start();
@@ -49,7 +51,6 @@ class BLEServer : public Component {
   esp_gatt_if_t get_gatts_if() { return this->gatts_if_; }
   uint32_t get_connected_client_count() { return this->connected_clients_; }
   const std::map<uint16_t, void *> &get_clients() { return this->clients_; }
-  BLEAdvertising *get_advertising() { return this->advertising_; }
 
   void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if, esp_ble_gatts_cb_param_t *param);
 
@@ -69,7 +70,6 @@ class BLEServer : public Component {
   optional<std::string> model_;
   esp_gatt_if_t gatts_if_{0};
   bool registered_{false};
-  BLEAdvertising *advertising_;
 
   uint32_t connected_clients_{0};
   std::map<uint16_t, void *> clients_;
@@ -83,14 +83,13 @@ class BLEServer : public Component {
     INIT = 0x00,
     REGISTERING,
     STARTING_SERVICE,
-    SETTING_UP_COMPONENT_SERVICES,
     RUNNING,
   } state_{INIT};
 };
 
 extern BLEServer *global_ble_server;
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_service.cpp b/esphome/components/esp32_ble_server/ble_service.cpp
similarity index 97%
rename from esphome/components/esp32_ble/ble_service.cpp
rename to esphome/components/esp32_ble_server/ble_service.cpp
index 4b85182696..b7c88a436c 100644
--- a/esphome/components/esp32_ble/ble_service.cpp
+++ b/esphome/components/esp32_ble_server/ble_service.cpp
@@ -5,9 +5,9 @@
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
-static const char *TAG = "esp32_ble.service";
+static const char *const TAG = "esp32_ble_server.service";
 
 BLEService::BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id)
     : uuid_(uuid), num_handles_(num_handles), inst_id_(inst_id) {}
@@ -136,7 +136,7 @@ void BLEService::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t g
   }
 }
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_ble/ble_service.h b/esphome/components/esp32_ble_server/ble_service.h
similarity index 93%
rename from esphome/components/esp32_ble/ble_service.h
rename to esphome/components/esp32_ble_server/ble_service.h
index a539631846..9fdb95bde5 100644
--- a/esphome/components/esp32_ble/ble_service.h
+++ b/esphome/components/esp32_ble_server/ble_service.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "ble_uuid.h"
 #include "ble_characteristic.h"
+#include "esphome/components/esp32_ble/ble_uuid.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
@@ -12,10 +12,12 @@
 #include <esp_bt_defs.h>
 
 namespace esphome {
-namespace esp32_ble {
+namespace esp32_ble_server {
 
 class BLEServer;
 
+using namespace esp32_ble;
+
 class BLEService {
  public:
   BLEService(ESPBTUUID uuid, uint16_t num_handles, uint8_t inst_id);
@@ -73,7 +75,7 @@ class BLEService {
   } running_state_{STOPPED};
 };
 
-}  // namespace esp32_ble
+}  // namespace esp32_ble_server
 }  // namespace esphome
 
 #endif
diff --git a/esphome/components/esp32_improv/__init__.py b/esphome/components/esp32_improv/__init__.py
index 8b91e9151d..4c337055cb 100644
--- a/esphome/components/esp32_improv/__init__.py
+++ b/esphome/components/esp32_improv/__init__.py
@@ -1,12 +1,13 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
-from esphome.components import binary_sensor, output, esp32_ble
+from esphome.components import binary_sensor, output, esp32_ble_server
 from esphome.const import CONF_ID, ESP_PLATFORM_ESP32
 
 
-AUTO_LOAD = ["binary_sensor", "output", "improv"]
+AUTO_LOAD = ["binary_sensor", "output", "improv", "esp32_ble_server"]
 CODEOWNERS = ["@jesserockz"]
-DEPENDENCIES = ["esp32_ble", "wifi"]
+CONFLICTS_WITH = ["esp32_ble_tracker", "esp32_ble_beacon"]
+DEPENDENCIES = ["wifi"]
 ESP_PLATFORMS = [ESP_PLATFORM_ESP32]
 
 CONF_AUTHORIZED_DURATION = "authorized_duration"
@@ -18,7 +19,7 @@ CONF_WIFI_TIMEOUT = "wifi_timeout"
 
 esp32_improv_ns = cg.esphome_ns.namespace("esp32_improv")
 ESP32ImprovComponent = esp32_improv_ns.class_(
-    "ESP32ImprovComponent", cg.Component, esp32_ble.BLEServiceComponent
+    "ESP32ImprovComponent", cg.Component, esp32_ble_server.BLEServiceComponent
 )
 
 
@@ -33,7 +34,7 @@ def validate_none_(value):
 CONFIG_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.declare_id(ESP32ImprovComponent),
-        cv.GenerateID(CONF_BLE_SERVER_ID): cv.use_id(esp32_ble.BLEServer),
+        cv.GenerateID(CONF_BLE_SERVER_ID): cv.use_id(esp32_ble_server.BLEServer),
         cv.Required(CONF_AUTHORIZER): cv.Any(
             validate_none_, cv.use_id(binary_sensor.BinarySensor)
         ),
diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp
index 491f26f46e..e076936771 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.cpp
+++ b/esphome/components/esp32_improv/esp32_improv_component.cpp
@@ -1,7 +1,9 @@
 #include "esp32_improv_component.h"
-#include "esphome/core/log.h"
+
+#include "esphome/components/esp32_ble/ble.h"
+#include "esphome/components/esp32_ble_server/ble_2902.h"
 #include "esphome/core/application.h"
-#include "esphome/components/esp32_ble/ble_2902.h"
+#include "esphome/core/log.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
@@ -12,40 +14,39 @@ static const char *TAG = "esp32_improv.component";
 
 ESP32ImprovComponent::ESP32ImprovComponent() { global_improv_component = this; }
 
-void ESP32ImprovComponent::setup_service() {
-  this->service_ = esp32_ble::global_ble_server->create_service(improv::SERVICE_UUID, true);
+void ESP32ImprovComponent::setup() {
+  this->service_ = global_ble_server->create_service(improv::SERVICE_UUID, true);
+  this->setup_characteristics();
 }
 
 void ESP32ImprovComponent::setup_characteristics() {
   this->status_ = this->service_->create_characteristic(
-      improv::STATUS_UUID, esp32_ble::BLECharacteristic::PROPERTY_READ | esp32_ble::BLECharacteristic::PROPERTY_NOTIFY);
-  esp32_ble::BLEDescriptor *status_descriptor = new esp32_ble::BLE2902();
+      improv::STATUS_UUID, BLECharacteristic::PROPERTY_READ | BLECharacteristic::PROPERTY_NOTIFY);
+  BLEDescriptor *status_descriptor = new BLE2902();
   this->status_->add_descriptor(status_descriptor);
 
   this->error_ = this->service_->create_characteristic(
-      improv::ERROR_UUID, esp32_ble::BLECharacteristic::PROPERTY_READ | esp32_ble::BLECharacteristic::PROPERTY_NOTIFY);
-  esp32_ble::BLEDescriptor *error_descriptor = new esp32_ble::BLE2902();
+      improv::ERROR_UUID, BLECharacteristic::PROPERTY_READ | BLECharacteristic::PROPERTY_NOTIFY);
+  BLEDescriptor *error_descriptor = new BLE2902();
   this->error_->add_descriptor(error_descriptor);
 
-  this->rpc_ =
-      this->service_->create_characteristic(improv::RPC_COMMAND_UUID, esp32_ble::BLECharacteristic::PROPERTY_WRITE);
+  this->rpc_ = this->service_->create_characteristic(improv::RPC_COMMAND_UUID, BLECharacteristic::PROPERTY_WRITE);
   this->rpc_->on_write([this](const std::vector<uint8_t> &data) {
     if (data.size() > 0) {
       this->incoming_data_.insert(this->incoming_data_.end(), data.begin(), data.end());
     }
   });
-  esp32_ble::BLEDescriptor *rpc_descriptor = new esp32_ble::BLE2902();
+  BLEDescriptor *rpc_descriptor = new BLE2902();
   this->rpc_->add_descriptor(rpc_descriptor);
 
-  this->rpc_response_ =
-      this->service_->create_characteristic(improv::RPC_RESULT_UUID, esp32_ble::BLECharacteristic::PROPERTY_READ |
-                                                                         esp32_ble::BLECharacteristic::PROPERTY_NOTIFY);
-  esp32_ble::BLEDescriptor *rpc_response_descriptor = new esp32_ble::BLE2902();
+  this->rpc_response_ = this->service_->create_characteristic(
+      improv::RPC_RESULT_UUID, BLECharacteristic::PROPERTY_READ | BLECharacteristic::PROPERTY_NOTIFY);
+  BLEDescriptor *rpc_response_descriptor = new BLE2902();
   this->rpc_response_->add_descriptor(rpc_response_descriptor);
 
   this->capabilities_ =
-      this->service_->create_characteristic(improv::CAPABILITIES_UUID, esp32_ble::BLECharacteristic::PROPERTY_READ);
-  esp32_ble::BLEDescriptor *capabilities_descriptor = new esp32_ble::BLE2902();
+      this->service_->create_characteristic(improv::CAPABILITIES_UUID, BLECharacteristic::PROPERTY_READ);
+  BLEDescriptor *capabilities_descriptor = new BLE2902();
   this->capabilities_->add_descriptor(capabilities_descriptor);
   uint8_t capabilities = 0x00;
   if (this->status_indicator_ != nullptr)
@@ -64,13 +65,9 @@ void ESP32ImprovComponent::loop() {
       if (this->status_indicator_ != nullptr)
         this->status_indicator_->turn_off();
 
-      if (this->service_->is_created() && !this->setup_complete_) {
-        this->setup_characteristics();
-      }
-
-      if (this->should_start_ && this->setup_complete_) {
+      if (this->service_->is_created() && this->should_start_ && this->setup_complete_) {
         if (this->service_->is_running()) {
-          this->service_->get_server()->get_advertising()->start();
+          esp32_ble::global_ble->get_advertising()->start();
 
           this->set_state_(improv::STATE_AWAITING_AUTHORIZATION);
           this->set_error_(improv::ERROR_NONE);
@@ -205,10 +202,7 @@ void ESP32ImprovComponent::stop() {
   });
 }
 
-float ESP32ImprovComponent::get_setup_priority() const {
-  // Before WiFi
-  return setup_priority::AFTER_BLUETOOTH;
-}
+float ESP32ImprovComponent::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH; }
 
 void ESP32ImprovComponent::dump_config() {
   ESP_LOGCONFIG(TAG, "ESP32 Improv:");
diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h
index e9b0c72ecd..262124f983 100644
--- a/esphome/components/esp32_improv/esp32_improv_component.h
+++ b/esphome/components/esp32_improv/esp32_improv_component.h
@@ -1,26 +1,28 @@
 #pragma once
 
+#include "esphome/components/binary_sensor/binary_sensor.h"
+#include "esphome/components/esp32_ble_server/ble_server.h"
+#include "esphome/components/esp32_ble_server/ble_characteristic.h"
+#include "esphome/components/improv/improv.h"
+#include "esphome/components/output/binary_output.h"
+#include "esphome/components/wifi/wifi_component.h"
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
-#include "esphome/components/binary_sensor/binary_sensor.h"
-#include "esphome/components/esp32_ble/ble_server.h"
-#include "esphome/components/esp32_ble/ble_characteristic.h"
-#include "esphome/components/output/binary_output.h"
-#include "esphome/components/wifi/wifi_component.h"
-#include "esphome/components/improv/improv.h"
 
 #ifdef ARDUINO_ARCH_ESP32
 
 namespace esphome {
 namespace esp32_improv {
 
-class ESP32ImprovComponent : public Component, public esp32_ble::BLEServiceComponent {
+using namespace esp32_ble_server;
+
+class ESP32ImprovComponent : public Component, public BLEServiceComponent {
  public:
   ESP32ImprovComponent();
   void dump_config() override;
   void loop() override;
-  void setup_service() override;
+  void setup() override;
   void setup_characteristics();
   void on_client_disconnect() override;
 
@@ -46,12 +48,12 @@ class ESP32ImprovComponent : public Component, public esp32_ble::BLEServiceCompo
   std::vector<uint8_t> incoming_data_;
   wifi::WiFiAP connecting_sta_;
 
-  esp32_ble::BLEService *service_;
-  esp32_ble::BLECharacteristic *status_;
-  esp32_ble::BLECharacteristic *error_;
-  esp32_ble::BLECharacteristic *rpc_;
-  esp32_ble::BLECharacteristic *rpc_response_;
-  esp32_ble::BLECharacteristic *capabilities_;
+  BLEService *service_;
+  BLECharacteristic *status_;
+  BLECharacteristic *error_;
+  BLECharacteristic *rpc_;
+  BLECharacteristic *rpc_response_;
+  BLECharacteristic *capabilities_;
 
   binary_sensor::BinarySensor *authorizer_{nullptr};
   output::BinaryOutput *status_indicator_{nullptr};
diff --git a/esphome/core/defines.h b/esphome/core/defines.h
index 24efd0fd14..90562510b9 100644
--- a/esphome/core/defines.h
+++ b/esphome/core/defines.h
@@ -19,6 +19,7 @@
 #define USE_JSON
 #ifdef ARDUINO_ARCH_ESP32
 #define USE_ESP32_CAMERA
+#define USE_ESP32_BLE_SERVER
 #define USE_IMPROV
 #endif
 #define USE_TIME
diff --git a/tests/test5.yaml b/tests/test5.yaml
index d4e19d985b..ba047721e2 100644
--- a/tests/test5.yaml
+++ b/tests/test5.yaml
@@ -29,9 +29,10 @@ output:
     id: built_in_led
 
 esp32_ble:
-  server:
-    manufacturer: "ESPHome"
-    model: "Test5"
+
+esp32_ble_server:
+  manufacturer: "ESPHome"
+  model: "Test5"
 
 esp32_improv:
   authorizer: io0_button