esphome/.github/workflows/ci.yml

---
name: CI

on:
  push:
    branches: [dev, beta, release]

  pull_request:
    paths:
      - "**"
      - "!.github/workflows/*.yml"
      - "!.github/actions/build-image/*"
      - ".github/workflows/ci.yml"
      - "!.yamllint"
      - "!.github/dependabot.yml"
  merge_group:

permissions:
  contents: read

env:
  DEFAULT_PYTHON: "3.9"
  PYUPGRADE_TARGET: "--py39-plus"

concurrency:
  # yamllint disable-line rule:line-length
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  common:
    name: Create common environment
    runs-on: ubuntu-latest
    outputs:
      cache-key: ${{ steps.cache-key.outputs.key }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Generate cache-key
        id: cache-key
        run: echo key="${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v5.1.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore Python virtual environment
        id: cache-venv
        uses: actions/cache@v4.0.2
        with:
          path: venv
          # yamllint disable-line rule:line-length
          key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }}
      - name: Create Python virtual environment
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          python -m venv venv
          . venv/bin/activate
          python --version
          pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
          pip install -e .          

  black:
    name: Check black
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Run black
        run: |
          . venv/bin/activate
          black --verbose esphome tests          
      - name: Suggested changes
        run: script/ci-suggest-changes
        if: always()

  flake8:
    name: Check flake8
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Run flake8
        run: |
          . venv/bin/activate
          flake8 esphome          
      - name: Suggested changes
        run: script/ci-suggest-changes
        if: always()

  pylint:
    name: Check pylint
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Run pylint
        run: |
          . venv/bin/activate
          pylint -f parseable --persistent=n esphome          
      - name: Suggested changes
        run: script/ci-suggest-changes
        if: always()

  pyupgrade:
    name: Check pyupgrade
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Run pyupgrade
        run: |
          . venv/bin/activate
          pyupgrade ${{ env.PYUPGRADE_TARGET }} `find esphome -name "*.py" -type f`          
      - name: Suggested changes
        run: script/ci-suggest-changes
        if: always()

  ci-custom:
    name: Run script/ci-custom
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Register matcher
        run: echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
      - name: Run script/ci-custom
        run: |
          . venv/bin/activate
          script/ci-custom.py
          script/build_codeowners.py --check          

  pytest:
    name: Run pytest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        os:
          - ubuntu-latest
          - macOS-latest
          - windows-latest
        exclude:
          # Minimize CI resource usage
          # by only running the Python version
          # version used for docker images on Windows and macOS
          - python-version: "3.12"
            os: windows-latest
          - python-version: "3.10"
            os: windows-latest
          - python-version: "3.9"
            os: windows-latest
          - python-version: "3.12"
            os: macOS-latest
          - python-version: "3.10"
            os: macOS-latest
          - python-version: "3.9"
            os: macOS-latest
    runs-on: ${{ matrix.os }}
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ matrix.python-version }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Register matcher
        run: echo "::add-matcher::.github/workflows/matchers/pytest.json"
      - name: Run pytest
        if: matrix.os == 'windows-latest'
        run: |
          ./venv/Scripts/activate
          pytest -vv --cov-report=xml --tb=native tests          
      - name: Run pytest
        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
        run: |
          . venv/bin/activate
          pytest -vv --cov-report=xml --tb=native tests          
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  clang-format:
    name: Check clang-format
    runs-on: ubuntu-latest
    needs:
      - common
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Install clang-format
        run: |
          . venv/bin/activate
          pip install clang-format -c requirements_dev.txt          
      - name: Run clang-format
        run: |
          . venv/bin/activate
          script/clang-format -i
          git diff-index --quiet HEAD --          
      - name: Suggested changes
        run: script/ci-suggest-changes
        if: always()

  clang-tidy:
    name: ${{ matrix.name }}
    runs-on: ubuntu-latest
    needs:
      - common
      - black
      - ci-custom
      - clang-format
      - flake8
      - pylint
      - pytest
      - pyupgrade
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        include:
          - id: clang-tidy
            name: Run script/clang-tidy for ESP8266
            options: --environment esp8266-arduino-tidy --grep USE_ESP8266
            pio_cache_key: tidyesp8266
          - id: clang-tidy
            name: Run script/clang-tidy for ESP32 Arduino 1/4
            options: --environment esp32-arduino-tidy --split-num 4 --split-at 1
            pio_cache_key: tidyesp32
          - id: clang-tidy
            name: Run script/clang-tidy for ESP32 Arduino 2/4
            options: --environment esp32-arduino-tidy --split-num 4 --split-at 2
            pio_cache_key: tidyesp32
          - id: clang-tidy
            name: Run script/clang-tidy for ESP32 Arduino 3/4
            options: --environment esp32-arduino-tidy --split-num 4 --split-at 3
            pio_cache_key: tidyesp32
          - id: clang-tidy
            name: Run script/clang-tidy for ESP32 Arduino 4/4
            options: --environment esp32-arduino-tidy --split-num 4 --split-at 4
            pio_cache_key: tidyesp32
          - id: clang-tidy
            name: Run script/clang-tidy for ESP32 IDF
            options: --environment esp32-idf-tidy --grep USE_ESP_IDF
            pio_cache_key: tidyesp32-idf
          - id: clang-tidy
            name: Run script/clang-tidy for ZEPHYR
            options: --environment nrf52-tidy --grep USE_ZEPHYR
            pio_cache_key: tidy-zephyr

    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}

      - name: Cache platformio
        if: github.ref == 'refs/heads/dev'
        uses: actions/cache@v4.0.2
        with:
          path: ~/.platformio
          key: platformio-${{ matrix.pio_cache_key }}

      - name: Cache platformio
        if: github.ref != 'refs/heads/dev'
        uses: actions/cache/restore@v4.0.2
        with:
          path: ~/.platformio
          key: platformio-${{ matrix.pio_cache_key }}

      - name: Install clang-tidy
        run: sudo apt-get install clang-tidy-14

      - name: Register problem matchers
        run: |
          echo "::add-matcher::.github/workflows/matchers/gcc.json"
          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"          

      - name: Run 'pio run --list-targets -e esp32-idf-tidy'
        if: matrix.name == 'Run script/clang-tidy for ESP32 IDF'
        run: |
          . venv/bin/activate
          mkdir -p .temp
          pio run --list-targets -e esp32-idf-tidy          

      - name: Run clang-tidy
        run: |
          . venv/bin/activate
          script/clang-tidy --all-headers --fix ${{ matrix.options }}          
        env:
          # Also cache libdeps, store them in a ~/.platformio subfolder
          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps

      - name: Suggested changes
        run: script/ci-suggest-changes
        # yamllint disable-line rule:line-length
        if: always()

  list-components:
    runs-on: ubuntu-latest
    needs:
      - common
    if: github.event_name == 'pull_request'
    outputs:
      components: ${{ steps.list-components.outputs.components }}
      count: ${{ steps.list-components.outputs.count }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
        with:
          # Fetch enough history so `git merge-base refs/remotes/origin/dev HEAD` works.
          fetch-depth: 500
      - name: Get target branch
        id: target-branch
        run: |
          echo "branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT          
      - name: Fetch ${{ steps.target-branch.outputs.branch }} branch
        run: |
          git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=1 origin +refs/heads/${{ steps.target-branch.outputs.branch }}:refs/remotes/origin/${{ steps.target-branch.outputs.branch }}
          git merge-base refs/remotes/origin/${{ steps.target-branch.outputs.branch }} HEAD          
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Find changed components
        id: list-components
        run: |
          . venv/bin/activate
          components=$(script/list-components.py --changed --branch ${{ steps.target-branch.outputs.branch }})
          output_components=$(echo "$components" | jq -R -s -c 'split("\n")[:-1] | map(select(length > 0))')
          count=$(echo "$output_components" | jq length)

          echo "components=$output_components" >> $GITHUB_OUTPUT
          echo "count=$count" >> $GITHUB_OUTPUT

          echo "$count Components:"
          echo "$output_components" | jq          

  test-build-components:
    name: Component test ${{ matrix.file }}
    runs-on: ubuntu-latest
    needs:
      - common
      - list-components
    if: github.event_name == 'pull_request' && fromJSON(needs.list-components.outputs.count) > 0 && fromJSON(needs.list-components.outputs.count) < 100
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        file: ${{ fromJson(needs.list-components.outputs.components) }}
    steps:
      - name: Install dependencies
        run: sudo apt-get install libsodium-dev libsdl2-dev

      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: test_build_components -e config -c ${{ matrix.file }}
        run: |
          . venv/bin/activate
          ./script/test_build_components -e config -c ${{ matrix.file }}          
      - name: test_build_components -e compile -c ${{ matrix.file }}
        run: |
          . venv/bin/activate
          ./script/test_build_components -e compile -c ${{ matrix.file }}          

  test-build-components-splitter:
    name: Split components for testing into 20 groups maximum
    runs-on: ubuntu-latest
    needs:
      - common
      - list-components
    if: github.event_name == 'pull_request' && fromJSON(needs.list-components.outputs.count) >= 100
    outputs:
      matrix: ${{ steps.split.outputs.components }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Split components into 20 groups
        id: split
        run: |
          components=$(echo '${{ needs.list-components.outputs.components }}' | jq -c '.[]' | shuf | jq -s -c '[_nwise(20) | join(" ")]')
          echo "components=$components" >> $GITHUB_OUTPUT          

  test-build-components-split:
    name: Test split components
    runs-on: ubuntu-latest
    needs:
      - common
      - list-components
      - test-build-components-splitter
    if: github.event_name == 'pull_request' && fromJSON(needs.list-components.outputs.count) >= 100
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        components: ${{ fromJson(needs.test-build-components-splitter.outputs.matrix) }}
    steps:
      - name: List components
        run: echo ${{ matrix.components }}

      - name: Install dependencies
        run: sudo apt-get install libsodium-dev libsdl2-dev

      - name: Check out code from GitHub
        uses: actions/checkout@v4.1.7
      - name: Restore Python
        uses: ./.github/actions/restore-python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          cache-key: ${{ needs.common.outputs.cache-key }}
      - name: Validate config
        run: |
          . venv/bin/activate
          for component in ${{ matrix.components }}; do
            ./script/test_build_components -e config -c $component
          done          
      - name: Compile config
        run: |
          . venv/bin/activate
          mkdir build_cache
          export PLATFORMIO_BUILD_CACHE_DIR=$PWD/build_cache
          for component in ${{ matrix.components }}; do
            ./script/test_build_components -e compile -c $component
          done          

  ci-status:
    name: CI Status
    runs-on: ubuntu-latest
    needs:
      - common
      - black
      - ci-custom
      - clang-format
      - flake8
      - pylint
      - pytest
      - pyupgrade
      - clang-tidy
      - list-components
      - test-build-components
      - test-build-components-splitter
      - test-build-components-split
    if: always()
    steps:
      - name: Success
        if: ${{ !(contains(needs.*.result, 'failure')) }}
        run: exit 0
      - name: Failure
        if: ${{ contains(needs.*.result, 'failure') }}
        env:
          JSON_DOC: ${{ toJSON(needs) }}
        run: |
          echo $JSON_DOC | jq
          exit 1