Refactor docker build system and workflows (#2023)

.github/workflows/ci-docker.yml

@@ -18,38 +18,23 @@ jobs:
     name: Build docker containers
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
         arch: [amd64, armv7, aarch64]
-        build_type: ["hassio", "docker"]
+        build_type: ["ha-addon", "docker", "lint"]
     steps:
-      - uses: actions/checkout@v2
+    - uses: actions/checkout@v2
-      - name: Set up env variables
+    - name: Set up Python
-        run: |
+      uses: actions/setup-python@v2
-          base_version="3.4.0"
+      with:
+        python-version: '3.9'
+    - name: Set TAG
+      run: |
+        echo "TAG=check" >> $GITHUB_ENV
 
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
+    - name: Run build
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
+      run: |
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
+        docker/build.py \
-            dockerfile="docker/Dockerfile.hassio"
+          --tag "${TAG}" \
-          else
+          --arch "${{ matrix.arch }}" \
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
+          --build-type "${{ matrix.build_type }}" \
-            build_to="esphome/esphome-${{ matrix.arch }}"
+          build
-            dockerfile="docker/Dockerfile"
-          fi
-
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "${BUILD_TO}:dev" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=ci" \
-            --cache-from "${BUILD_TO}:dev" \
-            --file "${DOCKERFILE}" \
-            .

.github/workflows/ci.yml

@@ -4,40 +4,36 @@ name: CI
 
 on:
   push:
-    # On dev branch release-dev already performs CI checks
+    branches: [dev, beta, release]
-    # On other branches the `pull_request` trigger will be used
-    branches: [beta, release]
 
   pull_request:
 
 jobs:
-  lint-clang-format:
+  ci-with-container:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
     strategy:
       fail-fast: false
       matrix:
-        split: [1, 2, 3, 4]
+        include:
+          - id: clang-format
+            name: Run script/clang-format
+          - id: clang-tidy
+            name: Run script/clang-tidy 1/4
+            split: 1
+          - id: clang-tidy
+            name: Run script/clang-tidy 2/4
+            split: 2
+          - id: clang-tidy
+            name: Run script/clang-tidy 3/4
+            split: 3
+          - id: clang-tidy
+            name: Run script/clang-tidy 4/4
+            split: 4
+
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:1.1
     steps:
       - uses: actions/checkout@v2
       # Set up the pio project so that the cpp checks know how files are compiled
@@ -45,26 +41,57 @@ jobs:
       - name: Set up platformio environment
         run: pio init --ide atom
 
-
       - name: Register problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
+
+      - name: Run clang-format
+        run: script/clang-format -i
+        if: ${{ matrix.id == 'clang-format' }}
+
       - name: Run clang-tidy
         run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+        if: ${{ matrix.id == 'clang-tidy' }}
+
       - name: Suggest changes
         run: script/ci-suggest-changes
 
-  lint-python:
+  ci:
     # Don't use the esphome-lint docker image because it may contain outdated requirements.
     # This way, all dependencies are cached via the cache action.
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - id: ci-custom
+            name: Run script/ci-custom
+          - id: lint-python
+            name: Run script/lint-python
+          - id: test
+            file: tests/test1.yaml
+            name: Test tests/test1.yaml
+          - id: test
+            file: tests/test2.yaml
+            name: Test tests/test2.yaml
+          - id: test
+            file: tests/test3.yaml
+            name: Test tests/test3.yaml
+          - id: test
+            file: tests/test4.yaml
+            name: Test tests/test4.yaml
+          - id: pytest
+            name: Run pytest
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: '3.7'
+
       - name: Cache pip modules
         uses: actions/cache@v1
         with:
@@ -72,6 +99,17 @@ jobs:
           key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
           restore-keys: |
             esphome-pip-3.7-
+
+      # Use per test platformio cache because tests have different platform versions
+      - name: Cache ~/.platformio
+        uses: actions/cache@v1
+        with:
+          path: ~/.platformio
+          key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }}
+          restore-keys: |
+            test-home-platformio-${{ matrix.file }}-
+        if: ${{ matrix.id == 'test' }}
+
       - name: Set up python environment
         run: script/setup
 
@@ -80,82 +118,22 @@ jobs:
           echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
           echo "::add-matcher::.github/workflows/matchers/lint-python.json"
           echo "::add-matcher::.github/workflows/matchers/python.json"
+          echo "::add-matcher::.github/workflows/matchers/pytest.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+
       - name: Lint Custom
-        run: script/ci-custom.py
+        run: |
+          script/ci-custom.py
+          script/build_codeowners.py --check
+        if: ${{ matrix.id == 'ci-custom' }}
       - name: Lint Python
         run: script/lint-python
-      - name: Lint CODEOWNERS
+        if: ${{ matrix.id == 'lint-python' }}
-        run: script/build_codeowners.py --check
 
-  test:
+      - run: esphome compile ${{ matrix.file }}
-    runs-on: ubuntu-latest
+        if: ${{ matrix.id == 'test' }}
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
 
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
       - name: Run pytest
         run: |
-          pytest \
+          pytest -vv --tb=native tests
-            -qq \
+        if: ${{ matrix.id == 'pytest' }}
-            --durations=10 \
-            -o console_output_style=count \
-            tests

.github/workflows/docker-lint-build.yml

@@ -13,30 +13,88 @@ on:
       - '.github/workflows/docker-lint-build.yml'
 
 jobs:
-  publish-docker-lint-iage:
+  deploy-docker:
-    name: Build docker containers
+    name: Build and publish docker containers
+    if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, armv7, aarch64]
+        build_type: ["lint"]
     steps:
-      - uses: actions/checkout@v2
+    - uses: actions/checkout@v2
-      - name: Set TAG
+    - name: Set up Python
-        run: |
+      uses: actions/setup-python@v2
-          echo "TAG=1.1" >> $GITHUB_ENV
+      with:
-      - name: Pull for cache
+        python-version: '3.9'
-        run: |
+    - name: Set TAG
-          docker pull "esphome/esphome-lint:latest" || true
+      run: |
-      - name: Build
+        echo "TAG=1.1" >> $GITHUB_ENV
-        run: |
+
-          docker build \
+    - name: Run build
-            --cache-from "esphome/esphome-lint:latest" \
+      run: |
-            --file "docker/Dockerfile.lint" \
+        docker/build.py \
-            --tag "esphome/esphome-lint:latest" \
+          --tag "${TAG}" \
-            --tag "esphome/esphome-lint:${TAG}" \
+          --arch "${{ matrix.arch }}" \
-            .
+          --build-type "${{ matrix.build_type }}" \
-      - name: Log in to docker hub
+          build
-        env:
+
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+    - name: Log in to docker hub
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      uses: docker/login-action@v1
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
+      with:
-      - run: |
+        username: ${{ secrets.DOCKER_USER }}
-          docker push "esphome/esphome-lint:${TAG}"
+        password: ${{ secrets.DOCKER_PASSWORD }}
-          docker push "esphome/esphome-lint:latest"
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run push
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          push
+
+  deploy-docker-manifest:
+    if: github.repository == 'esphome/esphome'
+    runs-on: ubuntu-latest
+    needs: [deploy-docker]
+    strategy:
+      matrix:
+        build_type: ["lint"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Set TAG
+      run: |
+        echo "TAG=1.1" >> $GITHUB_ENV
+    - name: Enable experimental manifest support
+      run: |
+        mkdir -p ~/.docker
+        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
+
+    - name: Log in to docker hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run manifest
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --build-type "${{ matrix.build_type }}" \
+          manifest

.github/workflows/matchers/pytest.json

@@ -0,0 +1,19 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "pytest",
+      "fileLocation": "absolute",
+      "pattern": [
+        {
+          "regexp": "^\\s+File \"(.*)\", line (\\d+), in (.*)$",
+          "file": 1,
+          "line": 2
+        },
+        {
+          "regexp": "^\\s+(.*)$",
+          "message": 1
+        }
+      ]
+    }
+  ]
+}

.github/workflows/release-dev.yml

@@ -1,247 +0,0 @@
-name: Publish dev releases to docker hub
-
-on:
-  push:
-    branches:
-    - dev
-
-jobs:
-  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
-
-  lint-clang-format:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
-    strategy:
-      fail-fast: false
-      matrix:
-        split: [1, 2, 3, 4]
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-      - name: Run clang-tidy
-        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-python:
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up python environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
-          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Lint Custom
-        run: script/ci-custom.py
-      - name: Lint Python
-        run: script/lint-python
-      - name: Lint CODEOWNERS
-        run: script/build_codeowners.py --check
-
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
-
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Run pytest
-        run: |
-          pytest \
-            -qq \
-            --durations=10 \
-            -o console_output_style=count \
-            tests
-
-  deploy-docker:
-    name: Build and publish docker containers
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
-    strategy:
-      matrix:
-        arch: [amd64, armv7, aarch64]
-        # Hassio dev image doesn't use esphome/esphome-hassio-$arch and uses base directly
-        build_type: ["docker"]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set TAG
-        run: |
-          TAG="${GITHUB_SHA:0:7}"
-          echo "TAG=${TAG}" >> $GITHUB_ENV
-      - name: Set up env variables
-        run: |
-          base_version="3.4.0"
-
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile.hassio"
-          else
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile"
-          fi
-
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "${BUILD_TO}:dev" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=${TAG}" \
-            --tag "${BUILD_TO}:${TAG}" \
-            --tag "${BUILD_TO}:dev" \
-            --cache-from "${BUILD_TO}:dev" \
-            --file "${DOCKERFILE}" \
-            .
-      - name: Log in to docker hub
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      - run: |
-          docker push "${BUILD_TO}:${TAG}"
-          docker push "${BUILD_TO}:dev"
-
-
-  deploy-docker-manifest:
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    needs: [deploy-docker]
-    steps:
-    - name: Enable experimental manifest support
-      run: |
-        mkdir -p ~/.docker
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - name: Set TAG
-      run: |
-        TAG="${GITHUB_SHA:0:7}"
-        echo "TAG=${TAG}" >> $GITHUB_ENV
-    - name: Log in to docker hub
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-    - name: "Create the manifest"
-      run: |
-        docker manifest create esphome/esphome:${TAG} \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:${TAG}
-
-        docker manifest create esphome/esphome:dev \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:dev

.github/workflows/release.yml

@@ -1,164 +1,35 @@
 name: Publish Release
 
 on:
+  workflow_dispatch:
   release:
     types: [published]
+  schedule:
+    - cron: "0 2 * * *"
 
 jobs:
-  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
+  init:
-
+    name: Initialize build
-  lint-clang-format:
     runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    outputs:
-    # doesn't have to be installed
+      tag: ${{ steps.tag.outputs.tag }}
-    container: esphome/esphome-lint:1.1
     steps:
       - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
+      - name: Get tag
-      # (build flags, libraries etc)
+        id: tag
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
-    strategy:
-      fail-fast: false
-      matrix:
-        split: [1, 2, 3, 4]
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-
-      - name: Register problem matchers
         run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
+          if [[ "$GITHUB_EVENT_NAME" = "release" ]]; then
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+            TAG="${GITHUB_REF#refs/tags/v}"
-      - name: Run clang-tidy
+          else
-        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+            TAG=$(cat esphome/const.py | sed -n -E "s/^__version__\s+=\s+\"(.+)\"$/\1/p")
-      - name: Suggest changes
+            today="$(date --utc '+%Y%m%d')"
-        run: script/ci-suggest-changes
+            TAG="${TAG}${today}"
-
+          fi
-  lint-python:
+          echo "::set-output name=tag::${TAG}"
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up python environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
-          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Lint Custom
-        run: script/ci-custom.py
-      - name: Lint Python
-        run: script/lint-python
-      - name: Lint CODEOWNERS
-        run: script/build_codeowners.py --check
-
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Run pytest
-        run: |
-          pytest \
-            -qq \
-            --durations=10 \
-            -o console_output_style=count \
-            tests
 
   deploy-pypi:
     name: Build and publish to PyPi
-    if: github.repository == 'esphome/esphome'
+    if: github.repository == 'esphome/esphome' && github.event_name == 'release'
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -182,119 +53,85 @@ jobs:
     name: Build and publish docker containers
     if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
+    needs: [init]
     strategy:
       matrix:
         arch: [amd64, armv7, aarch64]
-        build_type: ["hassio", "docker"]
+        build_type: ["ha-addon", "docker"]
     steps:
-      - uses: actions/checkout@v2
+    - uses: actions/checkout@v2
-      - name: Set TAG
+    - name: Set up Python
-        run: |
+      uses: actions/setup-python@v2
-          TAG="${GITHUB_REF#refs/tags/v}"
+      with:
-          echo "TAG=${TAG}" >> $GITHUB_ENV
+        python-version: '3.9'
-      - name: Set up env variables
-        run: |
-          base_version="3.4.0"
 
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
+    - name: Run build
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
+      run: |
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
+        docker/build.py \
-            dockerfile="docker/Dockerfile.hassio"
+          --tag "${{ needs.init.outputs.tag }}" \
-          else
+          --arch "${{ matrix.arch }}" \
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
+          --build-type "${{ matrix.build_type }}" \
-            build_to="esphome/esphome-${{ matrix.arch }}"
+          build
-            dockerfile="docker/Dockerfile"
-          fi
 
-          if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then
+    - name: Log in to docker hub
-            cache_tag="beta"
+      uses: docker/login-action@v1
-          else
+      with:
-            cache_tag="latest"
+        username: ${{ secrets.DOCKER_USER }}
-          fi
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-          # Set env variables so these values don't need to be calculated again
+    - name: Run push
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
+      run: |
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
+        docker/build.py \
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
+          --tag "${{ needs.init.outputs.tag }}" \
-          echo "CACHE_TAG=${cache_tag}" >> $GITHUB_ENV
+          --arch "${{ matrix.arch }}" \
-      - name: Pull for cache
+          --build-type "${{ matrix.build_type }}" \
-        run: |
+          push
-          docker pull "${BUILD_TO}:${CACHE_TAG}" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=${TAG}" \
-            --tag "${BUILD_TO}:${TAG}" \
-            --cache-from "${BUILD_TO}:${CACHE_TAG}" \
-            --file "${DOCKERFILE}" \
-            .
-      - name: Log in to docker hub
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      - run: docker push "${BUILD_TO}:${TAG}"
-
-      # Always publish to beta tag (also full releases)
-      - name: Publish docker beta tag
-        run: |
-          docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:beta"
-          docker push "${BUILD_TO}:beta"
-
-      - if: ${{ !github.event.release.prerelease }}
-        name: Publish docker latest tag
-        run: |
-          docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:latest"
-          docker push "${BUILD_TO}:latest"
 
   deploy-docker-manifest:
     if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
-    needs: [deploy-docker]
+    needs: [init, deploy-docker]
+    strategy:
+      matrix:
+        build_type: ["ha-addon", "docker"]
     steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
     - name: Enable experimental manifest support
       run: |
         mkdir -p ~/.docker
         echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - name: Set TAG
+
-      run: |
-        TAG="${GITHUB_REF#refs/tags/v}"
-        echo "TAG=${TAG}" >> $GITHUB_ENV
     - name: Log in to docker hub
-      env:
+      uses: docker/login-action@v1
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+      with:
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        username: ${{ secrets.DOCKER_USER }}
-      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
+        password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: "Create the manifest"
+    - name: Log in to the GitHub container registry
-      run: |
+      uses: docker/login-action@v1
-        docker manifest create esphome/esphome:${TAG} \
+      with:
-          esphome/esphome-aarch64:${TAG} \
+          registry: ghcr.io
-          esphome/esphome-amd64:${TAG} \
+          username: ${{ github.actor }}
-          esphome/esphome-armv7:${TAG}
+          password: ${{ secrets.GITHUB_TOKEN }}
-        docker manifest push esphome/esphome:${TAG}
 
-    - name: Publish docker beta tag
+    - name: Run manifest
       run: |
-        docker manifest create esphome/esphome:beta \
+        docker/build.py \
-          esphome/esphome-aarch64:${TAG} \
+          --tag "${{ needs.init.outputs.tag }}" \
-          esphome/esphome-amd64:${TAG} \
+          --build-type "${{ matrix.build_type }}" \
-          esphome/esphome-armv7:${TAG}
+          manifest
-        docker manifest push esphome/esphome:beta
-
-    - name: Publish docker latest tag
-      if: ${{ !github.event.release.prerelease }}
-      run: |
-        docker manifest create esphome/esphome:latest \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:latest
 
   deploy-hassio-repo:
-    if: github.repository == 'esphome/esphome'
+    if: github.repository == 'esphome/esphome' && github.event_name == 'release'
     runs-on: ubuntu-latest
     needs: [deploy-docker]
     steps:

docker/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_FROM=esphome/esphome-base-amd64:3.4.0
+ARG BUILD_FROM=esphome/esphome-base:latest
 FROM ${BUILD_FROM}
 
 # First install requirements to leverage caching when requirements don't change

docker/Dockerfile.hassio

@@ -1,4 +1,4 @@
-ARG BUILD_FROM
+ARG BUILD_FROM=esphome/esphome-hassio-base:latest
 FROM ${BUILD_FROM}
 
 # First install requirements to leverage caching when requirements don't change

docker/Dockerfile.lint

@@ -1,4 +1,5 @@
-FROM esphome/esphome-lint-base:3.4.0
+ARG BUILD_FROM=esphome/esphome-lint-base:latest
+FROM ${BUILD_FROM}
 
 COPY requirements.txt requirements_optional.txt requirements_test.txt docker/platformio_install_deps.py  platformio.ini /
 RUN \

docker/build.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+from dataclasses import dataclass
+import subprocess
+import argparse
+import platform
+import shlex
+import re
+import sys
+
+
+CHANNEL_DEV = 'dev'
+CHANNEL_BETA = 'beta'
+CHANNEL_RELEASE = 'release'
+CHANNELS = [CHANNEL_DEV, CHANNEL_BETA, CHANNEL_RELEASE]
+
+ARCH_AMD64 = 'amd64'
+ARCH_ARMV7 = 'armv7'
+ARCH_AARCH64 = 'aarch64'
+ARCHS = [ARCH_AMD64, ARCH_ARMV7, ARCH_AARCH64]
+
+TYPE_DOCKER = 'docker'
+TYPE_HA_ADDON = 'ha-addon'
+TYPE_LINT = 'lint'
+TYPES = [TYPE_DOCKER, TYPE_HA_ADDON, TYPE_LINT]
+
+
+BASE_VERSION = "3.6.0"
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--tag", type=str, required=True, help="The main docker tag to push to. If a version number also adds latest and/or beta tag")
+parser.add_argument("--arch", choices=ARCHS, required=False, help="The architecture to build for")
+parser.add_argument("--build-type", choices=TYPES, required=True, help="The type of build to run")
+parser.add_argument("--dry-run", action="store_true", help="Don't run any commands, just print them")
+subparsers = parser.add_subparsers(help="Action to perform", dest="command", required=True)
+build_parser = subparsers.add_parser("build", help="Build the image")
+push_parser = subparsers.add_parser("push", help="Tag the already built image and push it to docker hub")
+manifest_parser = subparsers.add_parser("manifest", help="Create a manifest from already pushed images")
+
+
+
+# only lists some possibilities, doesn't have to be perfect
+# https://stackoverflow.com/a/45125525
+UNAME_TO_ARCH = {
+    "x86_64": ARCH_AMD64,
+    "aarch64": ARCH_AARCH64,
+    "aarch64_be": ARCH_AARCH64,
+    "arm": ARCH_ARMV7,
+}
+
+
+@dataclass(frozen=True)
+class DockerParams:
+    build_from: str
+    build_to: str
+    manifest_to: str
+    dockerfile: str
+
+    @classmethod
+    def for_type_arch(cls, build_type, arch):
+        prefix = {
+            TYPE_DOCKER: "esphome/esphome",
+            TYPE_HA_ADDON: "esphome/esphome-hassio",
+            TYPE_LINT: "esphome/esphome-lint"
+        }[build_type]
+        build_from = f"ghcr.io/{prefix}-base-{arch}:{BASE_VERSION}"
+        build_to = f"{prefix}-{arch}"
+        dockerfile = {
+            TYPE_DOCKER: "docker/Dockerfile",
+            TYPE_HA_ADDON: "docker/Dockerfile.hassio",
+            TYPE_LINT: "docker/Dockerfile.lint",
+        }[build_type]
+        return cls(
+            build_from=build_from,
+            build_to=build_to,
+            manifest_to=prefix,
+            dockerfile=dockerfile
+        )
+
+
+def main():
+    args = parser.parse_args()
+
+    def run_command(*cmd, ignore_error: bool = False):
+        print(f"$ {shlex.join(list(cmd))}")
+        if not args.dry_run:
+            rc = subprocess.call(list(cmd))
+            if rc != 0 and not ignore_error:
+                print("Command failed")
+                sys.exit(1)
+
+    # detect channel from tag
+    match = re.match(r'\d+\.\d+(?:\.\d+)?(b\d+)?', args.tag)
+    if match is None:
+        channel = CHANNEL_DEV
+    elif match.group(1) is None:
+        channel = CHANNEL_RELEASE
+    else:
+        channel = CHANNEL_BETA
+
+    tags_to_push = [args.tag]
+    if channel == CHANNEL_DEV:
+        tags_to_push.append("dev")
+    elif channel == CHANNEL_BETA:
+        tags_to_push.append("beta")
+    elif channel == CHANNEL_RELEASE:
+        # Additionally push to beta
+        tags_to_push.append("beta")
+        tags_to_push.append("latest")
+
+    if args.command == "build":
+        # 1. pull cache image
+        params = DockerParams.for_type_arch(args.build_type, args.arch)
+        cache_tag = {
+            CHANNEL_DEV: "dev",
+            CHANNEL_BETA: "beta",
+            CHANNEL_RELEASE: "latest",
+        }[channel]
+        cache_img = f"ghcr.io/{params.build_to}:{cache_tag}"
+        run_command("docker", "pull", cache_img, ignore_error=True)
+
+        # 2. register QEMU binfmt (if not host arch)
+        is_native = UNAME_TO_ARCH.get(platform.machine()) == args.arch
+        if not is_native:
+            run_command(
+                "docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static:5.2.0-2",
+                "--reset", "-p", "yes"
+            )
+
+        # 3. build
+        run_command(
+            "docker", "build",
+            "--build-arg", f"BUILD_FROM={params.build_from}",
+            "--build-arg", f"BUILD_VERSION={args.tag}",
+            "--tag", f"{params.build_to}:{args.tag}",
+            "--cache-from", cache_img,
+            "--file", params.dockerfile,
+            "."
+        )
+    elif args.command == "push":
+        params = DockerParams.for_type_arch(args.build_type, args.arch)
+        imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push]
+        imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push]
+        src = imgs[0]
+        # 1. tag images
+        for img in imgs[1:]:
+            run_command(
+                "docker", "tag", src, img
+            )
+        # 2. push images
+        for img in imgs:
+            run_command(
+                "docker", "push", img
+            )
+    elif args.command == "manifest":
+        manifest = DockerParams.for_type_arch(args.build_type, ARCH_AMD64).manifest_to
+
+        targets = [f"{manifest}:{tag}" for tag in tags_to_push]
+        targets += [f"ghcr.io/{manifest}:{tag}" for tag in tags_to_push]
+        # 1. Create manifests
+        for target in targets:
+            cmd = ["docker", "manifest", "create", target]
+            for arch in ARCHS:
+                src = f"{DockerParams.for_type_arch(args.build_type, arch).build_to}:{args.tag}"
+                if target.startswith("ghcr.io"):
+                    src = f"ghcr.io/{src}"
+                cmd.append(src)
+            run_command(*cmd)
+        # 2. Push manifests
+        for target in targets:
+            run_command(
+                "docker", "manifest", "push", target
+            )
+
+
+if __name__ == "__main__":
+    main()

esphome/const.py

@@ -1,10 +1,6 @@
 """Constants used by esphome."""
 
-MAJOR_VERSION = 1
+__version__ = "1.21.0-dev"
-MINOR_VERSION = 21
-PATCH_VERSION = "0-dev"
-__short_version__ = f"{MAJOR_VERSION}.{MINOR_VERSION}"
-__version__ = f"{__short_version__}.{PATCH_VERSION}"
 
 ESP_PLATFORM_ESP32 = "ESP32"
 ESP_PLATFORM_ESP8266 = "ESP8266"

script/bump-docker-base-version.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import re
-import sys
-
-
-def sub(path, pattern, repl, expected_count=1):
-    with open(path) as fh:
-        content = fh.read()
-    content, count = re.subn(pattern, repl, content, flags=re.MULTILINE)
-    if expected_count is not None:
-        assert count == expected_count, f"Pattern {pattern} replacement failed!"
-    with open(path, "wt") as fh:
-        fh.write(content)
-
-
-def write_version(version: str):
-    for p in [
-        ".github/workflows/ci-docker.yml",
-        ".github/workflows/release-dev.yml",
-        ".github/workflows/release.yml",
-    ]:
-        sub(p, r'base_version=".*"', f'base_version="{version}"')
-
-    sub(
-        "docker/Dockerfile",
-        r"ARG BUILD_FROM=esphome/esphome-base-amd64:.*",
-        f"ARG BUILD_FROM=esphome/esphome-base-amd64:{version}",
-    )
-    sub(
-        "docker/Dockerfile.lint",
-        r"FROM esphome/esphome-lint-base:.*",
-        f"FROM esphome/esphome-lint-base:{version}",
-    )
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("new_version", type=str)
-    args = parser.parse_args()
-
-    version = args.new_version
-    print(f"Bumping to {version}")
-    write_version(version)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main() or 0)

script/bump-version.py

@@ -50,16 +50,10 @@ def sub(path, pattern, repl, expected_count=1):
 
 
 def write_version(version: Version):
-    sub(
-        "esphome/const.py", r"^MAJOR_VERSION = \d+$", f"MAJOR_VERSION = {version.major}"
-    )
-    sub(
-        "esphome/const.py", r"^MINOR_VERSION = \d+$", f"MINOR_VERSION = {version.minor}"
-    )
     sub(
         "esphome/const.py",
-        r"^PATCH_VERSION = .*$",
+        r"^__version__ = .*$",
-        f'PATCH_VERSION = "{version.full_patch}"',
+        f'__version__ = "{version}"',
     )