diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py
index 19e4046711..faf3cca280 100644
--- a/esphome/components/wifi/__init__.py
+++ b/esphome/components/wifi/__init__.py
@@ -159,8 +159,15 @@ def final_validate_power_esp32_ble(value):
         "esp32_ble_server",
         "esp32_ble_tracker",
     ]:
+        if conflicting not in fv.full_config.get():
+            continue
+
         try:
-            cv.require_framework_version(esp32_arduino=cv.Version(1, 0, 5))(None)
+            # Only arduino 1.0.5+ and esp-idf impacted
+            cv.require_framework_version(
+                esp32_arduino=cv.Version(1, 0, 5),
+                esp_idf=cv.Version(4, 0, 0),
+            )(None)
         except cv.Invalid:
             pass
         else: