Expand the platformio dep installer to also install platforms and tools (#4716)

.github/workflows/ci-docker.yml

@@ -11,6 +11,7 @@ on:
       - ".github/workflows/**"
       - "requirements*.txt"
       - "platformio.ini"
+      - "script/platformio_install_deps.py"
 
   pull_request:
     paths:
@@ -18,6 +19,7 @@ on:
       - ".github/workflows/**"
       - "requirements*.txt"
       - "platformio.ini"
+      - "script/platformio_install_deps.py"
 
 permissions:
   contents: read

docker/Dockerfile

@@ -60,7 +60,7 @@ RUN \
 
 
 # First install requirements to leverage caching when requirements don't change
-COPY requirements.txt requirements_optional.txt docker/platformio_install_deps.py platformio.ini /
+COPY requirements.txt requirements_optional.txt script/platformio_install_deps.py platformio.ini /
 RUN \
     pip3 install --no-cache-dir -r /requirements.txt -r /requirements_optional.txt \
     && /platformio_install_deps.py /platformio.ini

docker/platformio_install_deps.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python3
-# This script is used in the docker containers to preinstall
-# all platformio libraries in the global storage
-
-import configparser
-import subprocess
-import sys
-
-config = configparser.ConfigParser(inline_comment_prefixes=(';', ))
-config.read(sys.argv[1])
-
-libs = []
-# Extract from every lib_deps key in all sections
-for section in config.sections():
-    conf = config[section]
-    if "lib_deps" not in conf:
-        continue
-    for lib_dep in conf["lib_deps"].splitlines():
-        if not lib_dep:
-            # Empty line or comment
-            continue
-        if lib_dep.startswith("${"):
-            # Extending from another section
-            continue
-        if "@" not in lib_dep:
-            # No version pinned, this is an internal lib
-            continue
-        libs.append(lib_dep)
-
-subprocess.check_call(['platformio', 'lib', '-g', 'install', *libs])

esphome/components/esp32/__init__.py

@@ -252,7 +252,7 @@ def _parse_platform_version(value):
     try:
         # if platform version is a valid version constraint, prefix the default package
         cv.platformio_version_constraint(value)
-        return f"platformio/espressif32 @ {value}"
+        return f"platformio/espressif32@{value}"
     except cv.Invalid:
         return value
 
@@ -367,12 +367,12 @@ async def to_code(config):
         cg.add_build_flag("-Wno-nonnull-compare")
         cg.add_platformio_option(
             "platform_packages",
-            [f"platformio/framework-espidf @ {conf[CONF_SOURCE]}"],
+            [f"platformio/framework-espidf@{conf[CONF_SOURCE]}"],
         )
         # platformio/toolchain-esp32ulp does not support linux_aarch64 yet and has not been updated for over 2 years
         # This is espressif's own published version which is more up to date.
         cg.add_platformio_option(
-            "platform_packages", ["espressif/toolchain-esp32ulp @ 2.35.0-20220830"]
+            "platform_packages", ["espressif/toolchain-esp32ulp@2.35.0-20220830"]
         )
         add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_SINGLE_APP", False)
         add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_CUSTOM", True)
@@ -433,7 +433,7 @@ async def to_code(config):
         cg.add_build_flag("-DUSE_ESP32_FRAMEWORK_ARDUINO")
         cg.add_platformio_option(
             "platform_packages",
-            [f"platformio/framework-arduinoespressif32 @ {conf[CONF_SOURCE]}"],
+            [f"platformio/framework-arduinoespressif32@{conf[CONF_SOURCE]}"],
         )
 
         cg.add_platformio_option("board_build.partitions", "partitions.csv")

esphome/components/esp8266/__init__.py

@@ -125,7 +125,7 @@ def _parse_platform_version(value):
     try:
         # if platform version is a valid version constraint, prefix the default package
         cv.platformio_version_constraint(value)
-        return f"platformio/espressif8266 @ {value}"
+        return f"platformio/espressif8266@{value}"
     except cv.Invalid:
         return value
 
@@ -181,7 +181,7 @@ async def to_code(config):
     cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
     cg.add_platformio_option(
         "platform_packages",
-        [f"platformio/framework-arduinoespressif8266 @ {conf[CONF_SOURCE]}"],
+        [f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"],
     )
 
     # Default for platformio is LWIP2_LOW_MEMORY with:

esphome/components/rp2040/__init__.py

@@ -102,7 +102,7 @@ def _parse_platform_version(value):
     try:
         # if platform version is a valid version constraint, prefix the default package
         cv.platformio_version_constraint(value)
-        return f"platformio/raspberrypi @ {value}"
+        return f"platformio/raspberrypi@{value}"
     except cv.Invalid:
         return value
 
@@ -148,7 +148,7 @@ async def to_code(config):
     cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
     cg.add_platformio_option(
         "platform_packages",
-        [f"earlephilhower/framework-arduinopico @ {conf[CONF_SOURCE]}"],
+        [f"earlephilhower/framework-arduinopico@{conf[CONF_SOURCE]}"],
     )
 
     cg.add_platformio_option("board_build.core", "earlephilhower")

platformio.ini

@@ -79,9 +79,9 @@ build_flags =
 ; This are common settings for the ESP8266 using Arduino.
 [common:esp8266-arduino]
 extends = common:arduino
-platform = platformio/espressif8266 @ 3.2.0
+platform = platformio/espressif8266@3.2.0
 platform_packages =
-    platformio/framework-arduinoespressif8266 @ ~3.30002.0
+    platformio/framework-arduinoespressif8266@~3.30002.0
 
 framework = arduino
 lib_deps =
@@ -103,9 +103,9 @@ extra_scripts = post:esphome/components/esp8266/post_build.py.script
 ; This are common settings for the ESP32 (all variants) using Arduino.
 [common:esp32-arduino]
 extends = common:arduino
-platform = platformio/espressif32 @ 5.3.0
+platform = platformio/espressif32@5.3.0
 platform_packages =
-    platformio/framework-arduinoespressif32 @ ~3.20005.0
+    platformio/framework-arduinoespressif32@~3.20005.0
 
 framework = arduino
 board = nodemcu-32s
@@ -133,9 +133,9 @@ extra_scripts = post:esphome/components/esp32/post_build.py.script
 ; This are common settings for the ESP32 (all variants) using IDF.
 [common:esp32-idf]
 extends = common:idf
-platform = platformio/espressif32 @ 5.3.0
+platform = platformio/espressif32@5.3.0
 platform_packages =
-    platformio/framework-espidf @ ~3.40404.0
+    platformio/framework-espidf@~3.40404.0
 
 framework = espidf
 lib_deps =
@@ -156,8 +156,8 @@ board_build.filesystem_size = 0.5m
 
 platform = https://github.com/maxgerhardt/platform-raspberrypi.git
 platform_packages =
-    ; earlephilhower/framework-arduinopico @ ~1.20602.0 ; Cannot use the platformio package until old releases stop getting deleted
+    ; earlephilhower/framework-arduinopico@~1.20602.0 ; Cannot use the platformio package until old releases stop getting deleted
-    earlephilhower/framework-arduinopico @ https://github.com/earlephilhower/arduino-pico/releases/download/2.6.2/rp2040-2.6.2.zip
+    earlephilhower/framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/2.6.2/rp2040-2.6.2.zip
 
 framework = arduino
 lib_deps =

script/platformio_install_deps.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# This script is used to preinstall
+# all platformio libraries in the global storage
+
+import configparser
+import subprocess
+import sys
+
+config = configparser.ConfigParser(inline_comment_prefixes=(";",))
+config.read(sys.argv[1])
+
+libs = []
+tools = []
+platforms = []
+# Extract from every lib_deps key in all sections
+for section in config.sections():
+    conf = config[section]
+    if "lib_deps" in conf:
+        for lib_dep in conf["lib_deps"].splitlines():
+            if not lib_dep:
+                # Empty line or comment
+                continue
+            if lib_dep.startswith("${"):
+                # Extending from another section
+                continue
+            if "@" not in lib_dep:
+                # No version pinned, this is an internal lib
+                continue
+            libs.append("-l")
+            libs.append(lib_dep)
+    if "platform" in conf:
+        platforms.append("-p")
+        platforms.append(conf["platform"])
+    if "platform_packages" in conf:
+        for tool in conf["platform_packages"].splitlines():
+            if not tool:
+                # Empty line or comment
+                continue
+            if tool.startswith("${"):
+                # Extending from another section
+                continue
+            if tool.find("https://github.com") != -1:
+                split = tool.find("@")
+                tool = tool[split + 1 :]
+            tools.append("-t")
+            tools.append(tool)
+
+subprocess.check_call(["platformio", "pkg", "install", "-g", *libs, *platforms, *tools])

script/setup

@@ -14,3 +14,5 @@ pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_te
 pip3 install --no-use-pep517 -e .
 
 pre-commit install
+
+script/platformio_install_deps.py platformio.ini