Split release workflow jobs per system arch (#5723)

.github/actions/build-image/action.yaml

@@ -0,0 +1,97 @@
+name: Build Image
+inputs:
+  platform:
+    description: "Platform to build for"
+    required: true
+    example: "linux/amd64"
+  target:
+    description: "Target to build"
+    required: true
+    example: "docker"
+  baseimg:
+    description: "Base image type"
+    required: true
+    example: "docker"
+  suffix:
+    description: "Suffix to add to tags"
+    required: true
+  version:
+    description: "Version to build"
+    required: true
+    example: "2023.12.0"
+runs:
+  using: "composite"
+  steps:
+    - name: Generate short tags
+      id: tags
+      shell: bash
+      run: |
+        output=$(docker/generate_tags.py \
+          --tag "${{ inputs.version }}" \
+          --suffix "${{ inputs.suffix }}")
+        echo $output
+        for l in $output; do
+          echo $l >> $GITHUB_OUTPUT
+        done
+
+    - name: Build and push to ghcr by digest
+      id: build-ghcr
+      uses: docker/build-push-action@v5.0.0
+      with:
+        context: .
+        file: ./docker/Dockerfile
+        platforms: ${{ inputs.platform }}
+        target: ${{ inputs.target }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          BASEIMGTYPE=${{ inputs.baseimg }}
+          BUILD_VERSION=${{ inputs.version }}
+        outputs: |
+          type=image,name=ghcr.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true
+
+    - name: Export ghcr digests
+      shell: bash
+      run: |
+        mkdir -p /tmp/digests/${{ inputs.target }}/ghcr
+        digest="${{ steps.build-ghcr.outputs.digest }}"
+        touch "/tmp/digests/${{ inputs.target }}/ghcr/${digest#sha256:}"
+
+    - name: Upload ghcr digest
+      uses: actions/upload-artifact@v3.1.3
+      with:
+        name: digests-${{ inputs.target }}-ghcr
+        path: /tmp/digests/${{ inputs.target }}/ghcr/*
+        if-no-files-found: error
+        retention-days: 1
+
+    - name: Build and push to dockerhub by digest
+      id: build-dockerhub
+      uses: docker/build-push-action@v5.0.0
+      with:
+        context: .
+        file: ./docker/Dockerfile
+        platforms: ${{ inputs.platform }}
+        target: ${{ inputs.target }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          BASEIMGTYPE=${{ inputs.baseimg }}
+          BUILD_VERSION=${{ inputs.version }}
+        outputs: |
+          type=image,name=docker.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true
+
+    - name: Export dockerhub digests
+      shell: bash
+      run: |
+        mkdir -p /tmp/digests/${{ inputs.target }}/dockerhub
+        digest="${{ steps.build-dockerhub.outputs.digest }}"
+        touch "/tmp/digests/${{ inputs.target }}/dockerhub/${digest#sha256:}"
+
+    - name: Upload dockerhub digest
+      uses: actions/upload-artifact@v3.1.3
+      with:
+        name: digests-${{ inputs.target }}-dockerhub
+        path: /tmp/digests/${{ inputs.target }}/dockerhub/*
+        if-no-files-found: error
+        retention-days: 1

.github/workflows/release.yml

@@ -63,30 +63,20 @@ jobs:
         run: twine upload dist/*
 
   deploy-docker:
-    name: Build and publish ESPHome ${{ matrix.image.title}}
+    name: Build ESPHome ${{ matrix.platform }}
     if: github.repository == 'esphome/esphome'
     permissions:
       contents: read
       packages: write
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.image.title == 'lint' }}
     needs: [init]
     strategy:
       fail-fast: false
       matrix:
-        image:
-          - title: "ha-addon"
-            suffix: "hassio"
-            target: "hassio"
-            baseimg: "hassio"
-          - title: "docker"
-            suffix: ""
-            target: "docker"
-            baseimg: "docker"
-          - title: "lint"
-            suffix: "lint"
-            target: "lint"
-            baseimg: "docker"
+        platform:
+          - linux/amd64
+          - linux/arm/v7
+          - linux/arm64
     steps:
       - uses: actions/checkout@v4.1.1
       - name: Set up Python
@@ -97,6 +87,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.0.0
       - name: Set up QEMU
+        if: matrix.platform != 'linux/amd64'
         uses: docker/setup-qemu-action@v3.0.0
 
       - name: Log in to docker hub
@@ -111,34 +102,105 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build docker
+        uses: ./.github/actions/build-image
+        with:
+          platform: ${{ matrix.platform }}
+          target: docker
+          baseimg: docker
+          suffix: ""
+          version: ${{ needs.init.outputs.tag }}
+
+      - name: Build ha-addon
+        uses: ./.github/actions/build-image
+        with:
+          platform: ${{ matrix.platform }}
+          target: hassio
+          baseimg: hassio
+          suffix: "hassio"
+          version: ${{ needs.init.outputs.tag }}
+
+      - name: Build lint
+        uses: ./.github/actions/build-image
+        with:
+          platform: ${{ matrix.platform }}
+          target: lint
+          baseimg: docker
+          suffix: lint
+          version: ${{ needs.init.outputs.tag }}
+
+  deploy-manifest:
+    name: Publish ESPHome ${{ matrix.image.title }} to ${{ matrix.registry }}
+    runs-on: ubuntu-latest
+    needs:
+      - init
+      - deploy-docker
+    if: github.repository == 'esphome/esphome'
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - title: "ha-addon"
+            target: "hassio"
+            suffix: "hassio"
+          - title: "docker"
+            target: "docker"
+            suffix: ""
+          - title: "lint"
+            target: "lint"
+            suffix: "lint"
+        registry:
+          - ghcr
+          - dockerhub
+    steps:
+      - uses: actions/checkout@v4.1.1
+      - name: Download digests
+        uses: actions/download-artifact@v3.0.2
+        with:
+          name: digests-${{ matrix.image.target }}-${{ matrix.registry }}
+          path: /tmp/digests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.0.0
+
+      - name: Log in to docker hub
+        if: matrix.registry == 'dockerhub'
+        uses: docker/login-action@v3.0.0
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Log in to the GitHub container registry
+        if: matrix.registry == 'ghcr'
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Generate short tags
         id: tags
         run: |
-          docker/generate_tags.py \
+          output=$(docker/generate_tags.py \
             --tag "${{ needs.init.outputs.tag }}" \
-            --suffix "${{ matrix.image.suffix }}"
+            --suffix "${{ matrix.image.suffix }}" \
+            --registry "${{ matrix.registry }}")
+          echo $output
+          for l in $output; do
+            echo $l >> $GITHUB_OUTPUT
+          done
 
-      - name: Build and push
-        uses: docker/build-push-action@v5.0.0
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
-          target: ${{ matrix.image.target }}
-          push: true
-          # yamllint disable rule:line-length
-          cache-from: type=registry,ref=ghcr.io/${{ steps.tags.outputs.image }}:cache-${{ steps.tags.outputs.channel }}
-          cache-to: type=registry,ref=ghcr.io/${{ steps.tags.outputs.image }}:cache-${{ steps.tags.outputs.channel }},mode=max
-          # yamllint enable rule:line-length
-          tags: ${{ steps.tags.outputs.tags }}
-          build-args: |
-            BASEIMGTYPE=${{ matrix.image.baseimg }}
-            BUILD_VERSION=${{ needs.init.outputs.tag }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -Rcnr 'inputs | . / "," | map("-t " + .) | join(" ")' <<< "${{ steps.tags.outputs.tags}}") \
+            $(printf '${{ steps.tags.outputs.image }}@sha256:%s ' *)
 
   deploy-ha-addon-repo:
     if: github.repository == 'esphome/esphome' && github.event_name == 'release'
     runs-on: ubuntu-latest
-    needs: [deploy-docker]
+    needs: [deploy-manifest]
     steps:
       - name: Trigger Workflow
         uses: actions/github-script@v6.4.1

docker/generate_tags.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
 import re
-import os
 import argparse
-import json
 
 CHANNEL_DEV = "dev"
 CHANNEL_BETA = "beta"
 CHANNEL_RELEASE = "release"
 
+GHCR = "ghcr"
+DOCKERHUB = "dockerhub"
+
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "--tag",
@@ -21,21 +22,31 @@ parser.add_argument(
     required=True,
     help="The suffix of the tag.",
 )
+parser.add_argument(
+    "--registry",
+    type=str,
+    choices=[GHCR, DOCKERHUB],
+    required=False,
+    action="append",
+    help="The registry to build tags for.",
+)
 
 
 def main():
     args = parser.parse_args()
 
     # detect channel from tag
-    match = re.match(r"^(\d+\.\d+)(?:\.\d+)?(b\d+)?$", args.tag)
+    match = re.match(r"^(\d+\.\d+)(?:\.\d+)(?:(b\d+)|(-dev\d+))?$", args.tag)
     major_minor_version = None
-    if match is None:
+    if match is None:  # eg 2023.12.0-dev20231109-testbranch
+        channel = None  # Ran with custom tag for a branch etc
+    elif match.group(3) is not None:  # eg 2023.12.0-dev20231109
         channel = CHANNEL_DEV
-    elif match.group(2) is None:
+    elif match.group(2) is not None:  # eg 2023.12.0b1
+        channel = CHANNEL_BETA
+    else:  # eg 2023.12.0
         major_minor_version = match.group(1)
         channel = CHANNEL_RELEASE
-    else:
-        channel = CHANNEL_BETA
 
     tags_to_push = [args.tag]
     if channel == CHANNEL_DEV:
@@ -53,15 +64,28 @@ def main():
 
     suffix = f"-{args.suffix}" if args.suffix else ""
 
-    with open(os.environ["GITHUB_OUTPUT"], "w") as f:
-        print(f"channel={channel}", file=f)
-        print(f"image=esphome/esphome{suffix}", file=f)
-        full_tags = []
+    image_name = f"esphome/esphome{suffix}"
 
-        for tag in tags_to_push:
-            full_tags += [f"ghcr.io/esphome/esphome{suffix}:{tag}"]
-            full_tags += [f"esphome/esphome{suffix}:{tag}"]
-        print(f"tags={','.join(full_tags)}", file=f)
+    print(f"channel={channel}")
+
+    if args.registry is None:
+        args.registry = [GHCR, DOCKERHUB]
+    elif len(args.registry) == 1:
+        if GHCR in args.registry:
+            print(f"image=ghcr.io/{image_name}")
+        if DOCKERHUB in args.registry:
+            print(f"image=docker.io/{image_name}")
+
+    print(f"image_name={image_name}")
+
+    full_tags = []
+
+    for tag in tags_to_push:
+        if GHCR in args.registry:
+            full_tags += [f"ghcr.io/{image_name}:{tag}"]
+        if DOCKERHUB in args.registry:
+            full_tags += [f"docker.io/{image_name}:{tag}"]
+    print(f"tags={','.join(full_tags)}")
 
 
 if __name__ == "__main__":