diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index 61fbb53e3a..afce1421af 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -65,6 +65,8 @@ _LOGGER = logging.getLogger(__name__)
 CODEOWNERS = ["@esphome/core"]
 AUTO_LOAD = ["preferences"]
 
+CONF_RELEASE = "release"
+
 
 def set_core_data(config):
     CORE.data[KEY_ESP32] = {}
@@ -216,11 +218,17 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
     return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
 
 
-def _format_framework_espidf_version(ver: cv.Version) -> str:
+def _format_framework_espidf_version(
+    ver: cv.Version, release: str, for_platformio: bool
+) -> str:
     # format the given arduino (https://github.com/espressif/esp-idf/releases) version to
     # a PIO platformio/framework-espidf value
     # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-espidf
-    return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+    if for_platformio:
+        return f"platformio/framework-espidf@~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+    if release:
+        return f"pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v{str(ver)}.{release}/esp-idf-v{str(ver)}.zip"
+    return f"pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v{str(ver)}/esp-idf-v{str(ver)}.zip"
 
 
 # NOTE: Keep this in mind when updating the recommended version:
@@ -241,11 +249,25 @@ ARDUINO_PLATFORM_VERSION = cv.Version(5, 4, 0)
 # The default/recommended esp-idf framework version
 #  - https://github.com/espressif/esp-idf/releases
 #  - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-espidf
-RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION = cv.Version(4, 4, 8)
+RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION = cv.Version(5, 1, 5)
 # The platformio/espressif32 version to use for esp-idf frameworks
 #  - https://github.com/platformio/platform-espressif32/releases
 #  - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif32
-ESP_IDF_PLATFORM_VERSION = cv.Version(5, 4, 0)
+ESP_IDF_PLATFORM_VERSION = cv.Version(51, 3, 6)
+
+# List based on https://registry.platformio.org/tools/platformio/framework-espidf/versions
+SUPPORTED_PLATFORMIO_ESP_IDF_5X = [
+    cv.Version(5, 3, 1),
+    cv.Version(5, 3, 0),
+    cv.Version(5, 2, 2),
+    cv.Version(5, 2, 1),
+    cv.Version(5, 1, 2),
+    cv.Version(5, 1, 1),
+    cv.Version(5, 1, 0),
+    cv.Version(5, 0, 2),
+    cv.Version(5, 0, 1),
+    cv.Version(5, 0, 0),
+]
 
 
 def _arduino_check_versions(value):
@@ -286,8 +308,8 @@ def _arduino_check_versions(value):
 def _esp_idf_check_versions(value):
     value = value.copy()
     lookups = {
-        "dev": (cv.Version(5, 1, 2), "https://github.com/espressif/esp-idf.git"),
-        "latest": (cv.Version(5, 1, 2), None),
+        "dev": (cv.Version(5, 1, 5), "https://github.com/espressif/esp-idf.git"),
+        "latest": (cv.Version(5, 1, 5), None),
         "recommended": (RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION, None),
     }
 
@@ -305,13 +327,38 @@ def _esp_idf_check_versions(value):
     if version < cv.Version(4, 0, 0):
         raise cv.Invalid("Only ESP-IDF 4.0+ is supported.")
 
-    value[CONF_VERSION] = str(version)
-    value[CONF_SOURCE] = source or _format_framework_espidf_version(version)
+    # flag this for later *before* we set value[CONF_PLATFORM_VERSION] below
+    has_platform_ver = CONF_PLATFORM_VERSION in value
 
     value[CONF_PLATFORM_VERSION] = value.get(
         CONF_PLATFORM_VERSION, _parse_platform_version(str(ESP_IDF_PLATFORM_VERSION))
     )
 
+    if (
+        (is_platformio := _platform_is_platformio(value[CONF_PLATFORM_VERSION]))
+        and version.major >= 5
+        and version not in SUPPORTED_PLATFORMIO_ESP_IDF_5X
+    ):
+        raise cv.Invalid(
+            f"ESP-IDF {str(version)} not supported by platformio/espressif32"
+        )
+
+    if (
+        version in SUPPORTED_PLATFORMIO_ESP_IDF_5X or version.major < 5
+    ) and not has_platform_ver:
+        raise cv.Invalid(
+            f"ESP-IDF {value[CONF_VERSION]} may be supported by platformio/espressif32; please specify {CONF_PLATFORM_VERSION}"
+        )
+
+    value[CONF_VERSION] = str(version)
+    value[CONF_SOURCE] = source or _format_framework_espidf_version(
+        version, value.get(CONF_RELEASE, None), is_platformio
+    )
+
+    if value[CONF_SOURCE].startswith("http"):
+        # prefix is necessary or platformio will complain with a cryptic error
+        value[CONF_SOURCE] = f"framework-espidf@{value[CONF_SOURCE]}"
+
     if version != RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION:
         _LOGGER.warning(
             "The selected ESP-IDF framework version is not the recommended one. "
@@ -323,6 +370,9 @@ def _esp_idf_check_versions(value):
 
 def _parse_platform_version(value):
     try:
+        ver = cv.Version.parse(cv.version_number(value))
+        if ver.major >= 50:  # a pioarduino version
+            return f"https://github.com/pioarduino/platform-espressif32.git#{ver.major}.{ver.minor:02d}.{ver.patch:02d}"
         # if platform version is a valid version constraint, prefix the default package
         cv.platformio_version_constraint(value)
         return f"platformio/espressif32@{value}"
@@ -330,6 +380,14 @@ def _parse_platform_version(value):
         return value
 
 
+def _platform_is_platformio(value):
+    try:
+        ver = cv.Version.parse(cv.version_number(value))
+        return ver.major < 50
+    except cv.Invalid:
+        return "platformio" in value
+
+
 def _detect_variant(value):
     board = value[CONF_BOARD]
     if board in BOARDS:
@@ -412,6 +470,7 @@ ESP_IDF_FRAMEWORK_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Optional(CONF_VERSION, default="recommended"): cv.string_strict,
+            cv.Optional(CONF_RELEASE): cv.string_strict,
             cv.Optional(CONF_SOURCE): cv.string_strict,
             cv.Optional(CONF_PLATFORM_VERSION): _parse_platform_version,
             cv.Optional(CONF_SDKCONFIG_OPTIONS, default={}): {
@@ -515,10 +574,9 @@ async def to_code(config):
         cg.add_build_flag("-DUSE_ESP_IDF")
         cg.add_build_flag("-DUSE_ESP32_FRAMEWORK_ESP_IDF")
         cg.add_build_flag("-Wno-nonnull-compare")
-        cg.add_platformio_option(
-            "platform_packages",
-            [f"platformio/framework-espidf@{conf[CONF_SOURCE]}"],
-        )
+
+        cg.add_platformio_option("platform_packages", [conf[CONF_SOURCE]])
+
         # platformio/toolchain-esp32ulp does not support linux_aarch64 yet and has not been updated for over 2 years
         # This is espressif's own published version which is more up to date.
         cg.add_platformio_option(
diff --git a/platformio.ini b/platformio.ini
index 04afc059af..9a6af782ee 100644
--- a/platformio.ini
+++ b/platformio.ini
@@ -137,9 +137,9 @@ extra_scripts = post:esphome/components/esp32/post_build.py.script
 ; This are common settings for the ESP32 (all variants) using IDF.
 [common:esp32-idf]
 extends = common:idf
-platform = platformio/espressif32@5.4.0
+platform = https://github.com/pioarduino/platform-espressif32/releases/download/51.03.06/platform-espressif32.zip
 platform_packages =
-    platformio/framework-espidf@~3.40408.0
+    pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.1.4.241024/esp-idf-v5.1.4.zip
 
 framework = espidf
 lib_deps =
diff --git a/tests/test_build_components/build_components_base.esp32-c3-idf-51.yaml b/tests/test_build_components/build_components_base.esp32-c3-idf-51.yaml
deleted file mode 100644
index eb5b23a4ec..0000000000
--- a/tests/test_build_components/build_components_base.esp32-c3-idf-51.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-esphome:
-  name: componenttestesp32c3idf51
-  friendly_name: $component_name
-
-esp32:
-  board: lolin_c3_mini
-  framework:
-    type: esp-idf
-    version: 5.1.2
-    platform_version: 6.5.0
-
-logger:
-  level: VERY_VERBOSE
-
-packages:
-  component_under_test: !include
-    file: $component_test_file
-    vars:
-      component_test_file: $component_test_file
diff --git a/tests/test_build_components/build_components_base.esp32-c3-idf.yaml b/tests/test_build_components/build_components_base.esp32-c3-idf.yaml
index 18584497f4..72788a6f0f 100644
--- a/tests/test_build_components/build_components_base.esp32-c3-idf.yaml
+++ b/tests/test_build_components/build_components_base.esp32-c3-idf.yaml
@@ -6,6 +6,8 @@ esp32:
   board: lolin_c3_mini
   framework:
     type: esp-idf
+    version: 5.1.4
+    release: "241024"
 
 logger:
   level: VERY_VERBOSE
diff --git a/tests/test_build_components/build_components_base.esp32-idf-51.yaml b/tests/test_build_components/build_components_base.esp32-idf-51.yaml
deleted file mode 100644
index b5e3dd6d83..0000000000
--- a/tests/test_build_components/build_components_base.esp32-idf-51.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-esphome:
-  name: componenttestesp32idf51
-  friendly_name: $component_name
-
-esp32:
-  board: nodemcu-32s
-  framework:
-    type: esp-idf
-    version: 5.1.2
-    platform_version: 6.5.0
-
-logger:
-  level: VERY_VERBOSE
-
-packages:
-  component_under_test: !include
-    file: $component_test_file
-    vars:
-      component_test_file: $component_test_file
diff --git a/tests/test_build_components/build_components_base.esp32-idf.yaml b/tests/test_build_components/build_components_base.esp32-idf.yaml
index a62a995e68..0c574046a9 100644
--- a/tests/test_build_components/build_components_base.esp32-idf.yaml
+++ b/tests/test_build_components/build_components_base.esp32-idf.yaml
@@ -6,6 +6,8 @@ esp32:
   board: nodemcu-32s
   framework:
     type: esp-idf
+    version: 5.1.4
+    release: "241024"
 
 logger:
   level: VERY_VERBOSE
diff --git a/tests/test_build_components/build_components_base.esp32-s2-idf-51.yaml b/tests/test_build_components/build_components_base.esp32-s2-idf-51.yaml
deleted file mode 100644
index 11b077509e..0000000000
--- a/tests/test_build_components/build_components_base.esp32-s2-idf-51.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-esphome:
-  name: componenttestesp32s2idf51
-  friendly_name: $component_name
-
-esp32:
-  board: esp32-s2-saola-1
-  variant: ESP32S2
-  framework:
-    type: esp-idf
-    version: 5.1.2
-    platform_version: 6.5.0
-
-logger:
-  level: VERY_VERBOSE
-
-packages:
-  component_under_test: !include
-    file: $component_test_file
-    vars:
-      component_test_file: $component_test_file
diff --git a/tests/test_build_components/build_components_base.esp32-s2-idf.yaml b/tests/test_build_components/build_components_base.esp32-s2-idf.yaml
index 484906e8df..37d0da56be 100644
--- a/tests/test_build_components/build_components_base.esp32-s2-idf.yaml
+++ b/tests/test_build_components/build_components_base.esp32-s2-idf.yaml
@@ -7,6 +7,8 @@ esp32:
   variant: ESP32S2
   framework:
     type: esp-idf
+    version: 5.1.4
+    release: "241024"
 
 logger:
   level: VERY_VERBOSE
diff --git a/tests/test_build_components/build_components_base.esp32-s3-idf-51.yaml b/tests/test_build_components/build_components_base.esp32-s3-idf-51.yaml
deleted file mode 100644
index 4357b3581b..0000000000
--- a/tests/test_build_components/build_components_base.esp32-s3-idf-51.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-esphome:
-  name: componenttestesp32s3idf51
-  friendly_name: $component_name
-
-esp32:
-  board: esp32s3box
-  variant: ESP32S3
-  framework:
-    type: esp-idf
-    version: 5.1.2
-    platform_version: 6.5.0
-
-logger:
-  level: VERY_VERBOSE
-
-packages:
-  component_under_test: !include
-    file: $component_test_file
-    vars:
-      component_test_file: $component_test_file
diff --git a/tests/test_build_components/build_components_base.esp32-s3-idf.yaml b/tests/test_build_components/build_components_base.esp32-s3-idf.yaml
index ee209000e9..389176c8f1 100644
--- a/tests/test_build_components/build_components_base.esp32-s3-idf.yaml
+++ b/tests/test_build_components/build_components_base.esp32-s3-idf.yaml
@@ -7,6 +7,8 @@ esp32:
   variant: ESP32S3
   framework:
     type: esp-idf
+    version: 5.1.4
+    release: "241024"
 
 logger:
   level: VERY_VERBOSE