From 208edf89dc1dd458eb55ca6ca3f976bc231b7d4f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:06:16 +1300 Subject: [PATCH] Split release workflow jobs per system arch (#5723) --- .github/actions/build-image/action.yaml | 97 ++++++++++++++++++ .github/workflows/release.yml | 130 +++++++++++++++++------- docker/generate_tags.py | 54 +++++++--- 3 files changed, 232 insertions(+), 49 deletions(-) create mode 100644 .github/actions/build-image/action.yaml diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml new file mode 100644 index 0000000000..4c98a47ecd --- /dev/null +++ b/.github/actions/build-image/action.yaml @@ -0,0 +1,97 @@ +name: Build Image +inputs: + platform: + description: "Platform to build for" + required: true + example: "linux/amd64" + target: + description: "Target to build" + required: true + example: "docker" + baseimg: + description: "Base image type" + required: true + example: "docker" + suffix: + description: "Suffix to add to tags" + required: true + version: + description: "Version to build" + required: true + example: "2023.12.0" +runs: + using: "composite" + steps: + - name: Generate short tags + id: tags + shell: bash + run: | + output=$(docker/generate_tags.py \ + --tag "${{ inputs.version }}" \ + --suffix "${{ inputs.suffix }}") + echo $output + for l in $output; do + echo $l >> $GITHUB_OUTPUT + done + + - name: Build and push to ghcr by digest + id: build-ghcr + uses: docker/build-push-action@v5.0.0 + with: + context: . + file: ./docker/Dockerfile + platforms: ${{ inputs.platform }} + target: ${{ inputs.target }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BASEIMGTYPE=${{ inputs.baseimg }} + BUILD_VERSION=${{ inputs.version }} + outputs: | + type=image,name=ghcr.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true + + - name: Export ghcr digests + shell: bash + run: | + mkdir -p /tmp/digests/${{ inputs.target }}/ghcr + digest="${{ steps.build-ghcr.outputs.digest }}" + touch "/tmp/digests/${{ inputs.target }}/ghcr/${digest#sha256:}" + + - name: Upload ghcr digest + uses: actions/upload-artifact@v3.1.3 + with: + name: digests-${{ inputs.target }}-ghcr + path: /tmp/digests/${{ inputs.target }}/ghcr/* + if-no-files-found: error + retention-days: 1 + + - name: Build and push to dockerhub by digest + id: build-dockerhub + uses: docker/build-push-action@v5.0.0 + with: + context: . + file: ./docker/Dockerfile + platforms: ${{ inputs.platform }} + target: ${{ inputs.target }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BASEIMGTYPE=${{ inputs.baseimg }} + BUILD_VERSION=${{ inputs.version }} + outputs: | + type=image,name=docker.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true + + - name: Export dockerhub digests + shell: bash + run: | + mkdir -p /tmp/digests/${{ inputs.target }}/dockerhub + digest="${{ steps.build-dockerhub.outputs.digest }}" + touch "/tmp/digests/${{ inputs.target }}/dockerhub/${digest#sha256:}" + + - name: Upload dockerhub digest + uses: actions/upload-artifact@v3.1.3 + with: + name: digests-${{ inputs.target }}-dockerhub + path: /tmp/digests/${{ inputs.target }}/dockerhub/* + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14dbeee7b7..0e23db521a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,30 +63,20 @@ jobs: run: twine upload dist/* deploy-docker: - name: Build and publish ESPHome ${{ matrix.image.title}} + name: Build ESPHome ${{ matrix.platform }} if: github.repository == 'esphome/esphome' permissions: contents: read packages: write runs-on: ubuntu-latest - continue-on-error: ${{ matrix.image.title == 'lint' }} needs: [init] strategy: fail-fast: false matrix: - image: - - title: "ha-addon" - suffix: "hassio" - target: "hassio" - baseimg: "hassio" - - title: "docker" - suffix: "" - target: "docker" - baseimg: "docker" - - title: "lint" - suffix: "lint" - target: "lint" - baseimg: "docker" + platform: + - linux/amd64 + - linux/arm/v7 + - linux/arm64 steps: - uses: actions/checkout@v4.1.1 - name: Set up Python @@ -97,6 +87,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 - name: Set up QEMU + if: matrix.platform != 'linux/amd64' uses: docker/setup-qemu-action@v3.0.0 - name: Log in to docker hub @@ -111,34 +102,105 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build docker + uses: ./.github/actions/build-image + with: + platform: ${{ matrix.platform }} + target: docker + baseimg: docker + suffix: "" + version: ${{ needs.init.outputs.tag }} + + - name: Build ha-addon + uses: ./.github/actions/build-image + with: + platform: ${{ matrix.platform }} + target: hassio + baseimg: hassio + suffix: "hassio" + version: ${{ needs.init.outputs.tag }} + + - name: Build lint + uses: ./.github/actions/build-image + with: + platform: ${{ matrix.platform }} + target: lint + baseimg: docker + suffix: lint + version: ${{ needs.init.outputs.tag }} + + deploy-manifest: + name: Publish ESPHome ${{ matrix.image.title }} to ${{ matrix.registry }} + runs-on: ubuntu-latest + needs: + - init + - deploy-docker + if: github.repository == 'esphome/esphome' + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + image: + - title: "ha-addon" + target: "hassio" + suffix: "hassio" + - title: "docker" + target: "docker" + suffix: "" + - title: "lint" + target: "lint" + suffix: "lint" + registry: + - ghcr + - dockerhub + steps: + - uses: actions/checkout@v4.1.1 + - name: Download digests + uses: actions/download-artifact@v3.0.2 + with: + name: digests-${{ matrix.image.target }}-${{ matrix.registry }} + path: /tmp/digests + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Log in to docker hub + if: matrix.registry == 'dockerhub' + uses: docker/login-action@v3.0.0 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to the GitHub container registry + if: matrix.registry == 'ghcr' + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate short tags id: tags run: | - docker/generate_tags.py \ + output=$(docker/generate_tags.py \ --tag "${{ needs.init.outputs.tag }}" \ - --suffix "${{ matrix.image.suffix }}" + --suffix "${{ matrix.image.suffix }}" \ + --registry "${{ matrix.registry }}") + echo $output + for l in $output; do + echo $l >> $GITHUB_OUTPUT + done - - name: Build and push - uses: docker/build-push-action@v5.0.0 - with: - context: . - file: ./docker/Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 - target: ${{ matrix.image.target }} - push: true - # yamllint disable rule:line-length - cache-from: type=registry,ref=ghcr.io/${{ steps.tags.outputs.image }}:cache-${{ steps.tags.outputs.channel }} - cache-to: type=registry,ref=ghcr.io/${{ steps.tags.outputs.image }}:cache-${{ steps.tags.outputs.channel }},mode=max - # yamllint enable rule:line-length - tags: ${{ steps.tags.outputs.tags }} - build-args: | - BASEIMGTYPE=${{ matrix.image.baseimg }} - BUILD_VERSION=${{ needs.init.outputs.tag }} + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -Rcnr 'inputs | . / "," | map("-t " + .) | join(" ")' <<< "${{ steps.tags.outputs.tags}}") \ + $(printf '${{ steps.tags.outputs.image }}@sha256:%s ' *) deploy-ha-addon-repo: if: github.repository == 'esphome/esphome' && github.event_name == 'release' runs-on: ubuntu-latest - needs: [deploy-docker] + needs: [deploy-manifest] steps: - name: Trigger Workflow uses: actions/github-script@v6.4.1 diff --git a/docker/generate_tags.py b/docker/generate_tags.py index 71d0735526..3fc787d485 100755 --- a/docker/generate_tags.py +++ b/docker/generate_tags.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import re -import os import argparse -import json CHANNEL_DEV = "dev" CHANNEL_BETA = "beta" CHANNEL_RELEASE = "release" +GHCR = "ghcr" +DOCKERHUB = "dockerhub" + parser = argparse.ArgumentParser() parser.add_argument( "--tag", @@ -21,21 +22,31 @@ parser.add_argument( required=True, help="The suffix of the tag.", ) +parser.add_argument( + "--registry", + type=str, + choices=[GHCR, DOCKERHUB], + required=False, + action="append", + help="The registry to build tags for.", +) def main(): args = parser.parse_args() # detect channel from tag - match = re.match(r"^(\d+\.\d+)(?:\.\d+)?(b\d+)?$", args.tag) + match = re.match(r"^(\d+\.\d+)(?:\.\d+)(?:(b\d+)|(-dev\d+))?$", args.tag) major_minor_version = None - if match is None: + if match is None: # eg 2023.12.0-dev20231109-testbranch + channel = None # Ran with custom tag for a branch etc + elif match.group(3) is not None: # eg 2023.12.0-dev20231109 channel = CHANNEL_DEV - elif match.group(2) is None: + elif match.group(2) is not None: # eg 2023.12.0b1 + channel = CHANNEL_BETA + else: # eg 2023.12.0 major_minor_version = match.group(1) channel = CHANNEL_RELEASE - else: - channel = CHANNEL_BETA tags_to_push = [args.tag] if channel == CHANNEL_DEV: @@ -53,15 +64,28 @@ def main(): suffix = f"-{args.suffix}" if args.suffix else "" - with open(os.environ["GITHUB_OUTPUT"], "w") as f: - print(f"channel={channel}", file=f) - print(f"image=esphome/esphome{suffix}", file=f) - full_tags = [] + image_name = f"esphome/esphome{suffix}" - for tag in tags_to_push: - full_tags += [f"ghcr.io/esphome/esphome{suffix}:{tag}"] - full_tags += [f"esphome/esphome{suffix}:{tag}"] - print(f"tags={','.join(full_tags)}", file=f) + print(f"channel={channel}") + + if args.registry is None: + args.registry = [GHCR, DOCKERHUB] + elif len(args.registry) == 1: + if GHCR in args.registry: + print(f"image=ghcr.io/{image_name}") + if DOCKERHUB in args.registry: + print(f"image=docker.io/{image_name}") + + print(f"image_name={image_name}") + + full_tags = [] + + for tag in tags_to_push: + if GHCR in args.registry: + full_tags += [f"ghcr.io/{image_name}:{tag}"] + if DOCKERHUB in args.registry: + full_tags += [f"docker.io/{image_name}:{tag}"] + print(f"tags={','.join(full_tags)}") if __name__ == "__main__":