Bundle platformio lib_deps in docker images (#1625)

.github/workflows/ci-docker.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Set up env variables
         run: |
-          base_version="2.6.0"
+          base_version="2.8.0"
 
           if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
             build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"

.github/workflows/ci.yml

@@ -18,15 +18,6 @@ jobs:
     container: esphome/esphome-lint:latest
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment
@@ -49,15 +40,6 @@ jobs:
         split: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment

.github/workflows/release-dev.yml

@@ -15,15 +15,6 @@ jobs:
     container: esphome/esphome-lint:latest
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment
@@ -46,15 +37,6 @@ jobs:
         split: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment
@@ -192,7 +174,7 @@ jobs:
           echo "TAG=${TAG}" >> $GITHUB_ENV
       - name: Set up env variables
         run: |
-          base_version="2.6.0"
+          base_version="2.8.0"
 
           if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
             build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"

.github/workflows/release.yml

@@ -14,15 +14,6 @@ jobs:
     container: esphome/esphome-lint:latest
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment
@@ -45,15 +36,6 @@ jobs:
         split: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v2
-      # Cache platformio intermediary files (like libraries etc)
-      # Note: platformio platform versions should be cached via the esphome-lint image
-      - name: Cache Platformio
-        uses: actions/cache@v1
-        with:
-          path: .pio
-          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
-          restore-keys: |
-            lint-cpp-pio-
       # Set up the pio project so that the cpp checks know how files are compiled
       # (build flags, libraries etc)
       - name: Set up platformio environment
@@ -212,7 +194,7 @@ jobs:
           echo "TAG=${TAG}" >> $GITHUB_ENV
       - name: Set up env variables
         run: |
-          base_version="2.6.0"
+          base_version="2.8.0"
 
           if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
             build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"

docker/Dockerfile

@@ -1,9 +1,11 @@
-ARG BUILD_FROM=esphome/esphome-base-amd64:2.6.0
+ARG BUILD_FROM=esphome/esphome-base-amd64:2.8.0
 FROM ${BUILD_FROM}
 
 # First install requirements to leverage caching when requirements don't change
-COPY requirements.txt /
+COPY requirements.txt docker/platformio_install_deps.py platformio.ini /
-RUN pip3 install --no-cache-dir -r /requirements.txt
+RUN \
+    pip3 install --no-cache-dir -r /requirements.txt \
+    && /platformio_install_deps.py /platformio.ini
 
 # Then copy esphome and install
 COPY . .

docker/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM esphome/esphome-base-amd64:2.6.0
+FROM esphome/esphome-base-amd64:2.8.0
 
 COPY . .
 

docker/Dockerfile.hassio

@@ -2,8 +2,10 @@ ARG BUILD_FROM
 FROM ${BUILD_FROM}
 
 # First install requirements to leverage caching when requirements don't change
-COPY requirements.txt /
+COPY requirements.txt docker/platformio_install_deps.py platformio.ini /
-RUN pip3 install --no-cache-dir -r /requirements.txt
+RUN \
+    pip3 install --no-cache-dir -r /requirements.txt \
+    && /platformio_install_deps.py /platformio.ini
 
 # Copy root filesystem
 COPY docker/rootfs/ /

docker/Dockerfile.lint

@@ -1,7 +1,9 @@
-FROM esphome/esphome-lint-base:2.6.0
+FROM esphome/esphome-lint-base:2.8.0
 
-COPY requirements.txt requirements_test.txt /
+COPY requirements.txt requirements_test.txt docker/platformio_install_deps.py  platformio.ini /
-RUN pip3 install --no-cache-dir -r /requirements.txt -r /requirements_test.txt
+RUN \
+    pip3 install --no-cache-dir -r /requirements.txt -r /requirements_test.txt \
+    && /platformio_install_deps.py /platformio.ini
 
 VOLUME ["/esphome"]
 WORKDIR /esphome

docker/platformio_install_deps.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# This script is used in the docker containers to preinstall
+# all platformio libraries in the global storage
+
+import configparser
+import re
+import subprocess
+import sys
+
+config = configparser.ConfigParser()
+config.read(sys.argv[1])
+libs = []
+for line in config['common']['lib_deps'].splitlines():
+    # Format: '1655@1.0.2  ; TinyGPSPlus (has name conflict)' (includes comment)
+    m = re.search(r'([a-zA-Z0-9-_/]+@[0-9\.]+)', line)
+    if m is None:
+        continue
+    libs.append(m.group(1))
+
+subprocess.check_call(['platformio', 'lib', '-g', 'install', *libs])

esphome/const.py

@@ -16,30 +16,31 @@ ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_-"
 # See also https://github.com/platformio/platform-espressif32/releases
 ARDUINO_VERSION_ESP32 = {
     "dev": "https://github.com/platformio/platform-espressif32.git",
-    "1.0.4": "espressif32@1.12.4",
+    "1.0.5": "platformio/espressif32@3.1.1",
-    "1.0.3": "espressif32@1.10.0",
+    "1.0.4": "platformio/espressif32@3.0.0",
-    "1.0.2": "espressif32@1.9.0",
+    "1.0.3": "platformio/espressif32@1.10.0",
-    "1.0.1": "espressif32@1.7.0",
+    "1.0.2": "platformio/espressif32@1.9.0",
-    "1.0.0": "espressif32@1.5.0",
+    "1.0.1": "platformio/espressif32@1.7.0",
+    "1.0.0": "platformio/espressif32@1.5.0",
 }
 # See also https://github.com/platformio/platform-espressif8266/releases
 ARDUINO_VERSION_ESP8266 = {
     "dev": "https://github.com/platformio/platform-espressif8266.git",
-    "2.7.4": "espressif8266@2.6.2",
+    "2.7.4": "platformio/espressif8266@2.6.2",
-    "2.7.3": "espressif8266@2.6.1",
+    "2.7.3": "platformio/espressif8266@2.6.1",
-    "2.7.2": "espressif8266@2.6.0",
+    "2.7.2": "platformio/espressif8266@2.6.0",
-    "2.7.1": "espressif8266@2.5.3",
+    "2.7.1": "platformio/espressif8266@2.5.3",
-    "2.7.0": "espressif8266@2.5.0",
+    "2.7.0": "platformio/espressif8266@2.5.0",
-    "2.6.3": "espressif8266@2.4.0",
+    "2.6.3": "platformio/espressif8266@2.4.0",
-    "2.6.2": "espressif8266@2.3.1",
+    "2.6.2": "platformio/espressif8266@2.3.1",
-    "2.6.1": "espressif8266@2.3.0",
+    "2.6.1": "platformio/espressif8266@2.3.0",
-    "2.5.2": "espressif8266@2.2.3",
+    "2.5.2": "platformio/espressif8266@2.2.3",
-    "2.5.1": "espressif8266@2.1.1",
+    "2.5.1": "platformio/espressif8266@2.1.1",
-    "2.5.0": "espressif8266@2.0.4",
+    "2.5.0": "platformio/espressif8266@2.0.4",
-    "2.4.2": "espressif8266@1.8.0",
+    "2.4.2": "platformio/espressif8266@1.8.0",
-    "2.4.1": "espressif8266@1.7.3",
+    "2.4.1": "platformio/espressif8266@1.7.3",
-    "2.4.0": "espressif8266@1.6.0",
+    "2.4.0": "platformio/espressif8266@1.6.0",
-    "2.3.0": "espressif8266@1.5.0",
+    "2.3.0": "platformio/espressif8266@1.5.0",
 }
 SOURCE_FILE_EXTENSIONS = {".cpp", ".hpp", ".h", ".c", ".tcc", ".ino"}
 HEADER_FILE_EXTENSIONS = {".h", ".hpp", ".tcc"}

platformio.ini

@@ -14,7 +14,7 @@ lib_deps =
     AsyncMqttClient-esphome@0.8.4
     ArduinoJson-esphomelib@5.13.3
     ESPAsyncWebServer-esphome@1.2.7
-    FastLED@3.3.2
+    fastled/FastLED@3.3.2
     NeoPixelBus-esphome@2.5.7
     ESPAsyncTCP-esphome@1.2.3
     1655@1.0.2  ; TinyGPSPlus (has name conflict)

script/ci-custom.py

@@ -217,7 +217,7 @@ def lint_ext_check(fname):
     )
 
 
-@lint_file_check(exclude=["docker/rootfs/*", "script/*", "setup.py"])
+@lint_file_check(exclude=["docker/rootfs/*", "docker/*.py", "script/*", "setup.py"])
 def lint_executable_bit(fname):
     ex = EXECUTABLE_BIT[fname]
     if ex != 100644: