Rework CI into multiple dependent jobs (#4823)

This commit is contained in:
Jesse Hills 2023-05-17 16:28:09 +12:00 committed by GitHub
parent edfd82fd42
commit 90598d2405
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -12,60 +12,266 @@ on:
permissions: permissions:
contents: read contents: read
env:
DEFAULT_PYTHON: "3.9"
PYUPGRADE_TARGET: "--py39-plus"
CLANG_FORMAT_VERSION: "13.0.1"
concurrency: concurrency:
# yamllint disable-line rule:line-length # yamllint disable-line rule:line-length
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true cancel-in-progress: true
jobs: jobs:
ci: common:
name: ${{ matrix.name }} name: Create common environment
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/setup-python@v4.6.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv venv
. venv/bin/activate
python --version
pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
pip install -e .
yamllint:
name: yamllint
runs-on: ubuntu-latest
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Run yamllint
uses: frenck/action-yamllint@v1.4.0
black:
name: Check black
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Run black
run: |
. venv/bin/activate
black --verbose esphome tests
- name: Suggested changes
run: script/ci-suggest-changes
if: always()
flake8:
name: Check flake8
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Run flake8
run: |
. venv/bin/activate
flake8 esphome
- name: Suggested changes
run: script/ci-suggest-changes
if: always()
pylint:
name: Check pylint
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Run pylint
run: |
. venv/bin/activate
pylint -f parseable --persistent=n esphome
- name: Suggested changes
run: script/ci-suggest-changes
if: always()
pyupgrade:
name: Check pyupgrade
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Run pyupgrade
run: |
. venv/bin/activate
pyupgrade ${{ env.PYUPGRADE_TARGET }} `find esphome -name "*.py" -type f`
- name: Suggested changes
run: script/ci-suggest-changes
if: always()
ci-custom:
name: Run script/ci-custom
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Register matcher
run: echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
- name: Run script/ci-custom
run: |
. venv/bin/activate
script/ci-custom.py
script/build_codeowners.py --check
pytest:
name: Run pytest
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Register matcher
run: echo "::add-matcher::.github/workflows/matchers/pytest.json"
- name: Run pytest
run: |
. venv/bin/activate
pytest -vv --tb=native tests
clang-format:
name: Check clang-format
runs-on: ubuntu-latest
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Install clang-format
run: |
. venv/bin/activate
pip install clang-format==${{ env.CLANG_FORMAT_VERSION }}
- name: Run clang-format
run: |
. venv/bin/activate
script/clang-format -i
git diff-index --quiet HEAD --
- name: Suggested changes
run: script/ci-suggest-changes
if: always()
compile-tests:
name: Run YAML test ${{ matrix.file }}
runs-on: ubuntu-latest
needs:
- common
- black
- ci-custom
- clang-format
- flake8
- pylint
- pytest
- pyupgrade
- yamllint
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 5 max-parallel: 2
matrix:
file: [1, 2, 3, 3.1, 4, 5, 6, 7]
steps:
- name: Check out code from GitHub
uses: actions/checkout@v3.5.2
- name: Restore Python virtual environment
uses: actions/cache/restore@v3.3.1
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
- name: Cache platformio
uses: actions/cache@v3.3.1
with:
path: ~/.platformio
# yamllint disable-line rule:line-length
key: platformio-test${{ matrix.file }}-${{ hashFiles('platformio.ini') }}
- name: Run esphome compile tests/test${{ matrix.file }}.yaml
run: |
. venv/bin/activate
esphome compile tests/test${{ matrix.file }}.yaml
clang-tidy:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
needs:
- common
- black
- ci-custom
- clang-format
- flake8
- pylint
- pytest
- pyupgrade
- yamllint
strategy:
fail-fast: false
max-parallel: 2
matrix: matrix:
include: include:
- id: ci-custom
name: Run script/ci-custom
- id: lint-python
name: Run script/lint-python
- id: test
file: tests/test1.yaml
name: Test tests/test1.yaml
pio_cache_key: test1
- id: test
file: tests/test2.yaml
name: Test tests/test2.yaml
pio_cache_key: test2
- id: test
file: tests/test3.yaml
name: Test tests/test3.yaml
pio_cache_key: test3
- id: test
file: tests/test3.1.yaml
name: Test tests/test3.1.yaml
pio_cache_key: test3.1
- id: test
file: tests/test4.yaml
name: Test tests/test4.yaml
pio_cache_key: test4
- id: test
file: tests/test5.yaml
name: Test tests/test5.yaml
pio_cache_key: test5
- id: test
file: tests/test6.yaml
name: Test tests/test6.yaml
pio_cache_key: test6
- id: test
file: tests/test7.yaml
name: Test tests/test7.yaml
pio_cache_key: test7
- id: pytest
name: Run pytest
- id: clang-format
name: Run script/clang-format
- id: clang-tidy - id: clang-tidy
name: Run script/clang-tidy for ESP8266 name: Run script/clang-tidy for ESP8266
options: --environment esp8266-arduino-tidy --grep USE_ESP8266 options: --environment esp8266-arduino-tidy --grep USE_ESP8266
@ -90,119 +296,65 @@ jobs:
name: Run script/clang-tidy for ESP32 IDF name: Run script/clang-tidy for ESP32 IDF
options: --environment esp32-idf-tidy --grep USE_ESP_IDF options: --environment esp32-idf-tidy --grep USE_ESP_IDF
pio_cache_key: tidyesp32-idf pio_cache_key: tidyesp32-idf
- id: yamllint
name: Run yamllint
steps: steps:
- uses: actions/checkout@v3 - name: Check out code from GitHub
- name: Set up Python uses: actions/checkout@v3.5.2
uses: actions/setup-python@v4 - name: Restore Python virtual environment
id: python uses: actions/cache/restore@v3.3.1
with: with:
python-version: "3.9" path: venv
- name: Cache virtualenv
uses: actions/cache@v3
with:
path: .venv
# yamllint disable-line rule:line-length # yamllint disable-line rule:line-length
key: venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements*.txt') }} key: ${{ runner.os }}-${{ env.DEFAULT_PYTHON }}-venv-${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}
restore-keys: | # Use per check platformio cache because checks use different parts
venv-${{ steps.python.outputs.python-version }}-
- name: Set up virtualenv
# yamllint disable rule:line-length
run: |
python -m venv .venv
source .venv/bin/activate
pip install -U pip
pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
pip install -e .
echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
# yamllint enable rule:line-length
# Use per check platformio cache because checks use different parts
- name: Cache platformio - name: Cache platformio
uses: actions/cache@v3 uses: actions/cache@v3.3.1
with: with:
path: ~/.platformio path: ~/.platformio
# yamllint disable-line rule:line-length # yamllint disable-line rule:line-length
key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }}
if: matrix.id == 'test' || matrix.id == 'clang-tidy'
- name: Install clang tools - name: Install clang-tidy
run: | run: sudo apt-get install clang-tidy-11
sudo apt-get install \
clang-format-13 \
clang-tidy-11
if: matrix.id == 'clang-tidy' || matrix.id == 'clang-format'
- name: Register problem matchers - name: Register problem matchers
run: | run: |
echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
echo "::add-matcher::.github/workflows/matchers/lint-python.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
echo "::add-matcher::.github/workflows/matchers/pytest.json"
echo "::add-matcher::.github/workflows/matchers/gcc.json" echo "::add-matcher::.github/workflows/matchers/gcc.json"
echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
- name: Lint Custom
run: |
script/ci-custom.py
script/build_codeowners.py --check
if: matrix.id == 'ci-custom'
- name: Lint Python
run: script/lint-python -a
if: matrix.id == 'lint-python'
- run: esphome compile ${{ matrix.file }}
if: matrix.id == 'test'
env:
# Also cache libdeps, store them in a ~/.platformio subfolder
PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
- name: Run pytest
run: |
pytest -vv --tb=native tests
if: matrix.id == 'pytest'
# Also run git-diff-index so that the step is marked as failed on
# formatting errors, since clang-format doesn't do anything but
# change files if -i is passed.
- name: Run clang-format
run: |
script/clang-format -i
git diff-index --quiet HEAD --
if: matrix.id == 'clang-format'
- name: Run clang-tidy - name: Run clang-tidy
run: | run: |
. venv/bin/activate
script/clang-tidy --all-headers --fix ${{ matrix.options }} script/clang-tidy --all-headers --fix ${{ matrix.options }}
if: matrix.id == 'clang-tidy'
env: env:
# Also cache libdeps, store them in a ~/.platformio subfolder # Also cache libdeps, store them in a ~/.platformio subfolder
PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
- name: Run yamllint
if: matrix.id == 'yamllint'
uses: frenck/action-yamllint@v1.4.0
- name: Suggested changes - name: Suggested changes
run: script/ci-suggest-changes run: script/ci-suggest-changes
# yamllint disable-line rule:line-length # yamllint disable-line rule:line-length
if: always() && (matrix.id == 'clang-tidy' || matrix.id == 'clang-format' || matrix.id == 'lint-python') if: always()
ci-status: ci-status:
name: CI Status name: CI Status
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [ci] needs:
- common
- black
- ci-custom
- clang-format
- flake8
- pylint
- pytest
- pyupgrade
- yamllint
- compile-tests
- clang-tidy
if: always() if: always()
steps: steps:
- name: Successful deploy - name: Success
if: ${{ !(contains(needs.*.result, 'failure')) }} if: ${{ !(contains(needs.*.result, 'failure')) }}
run: exit 0 run: exit 0
- name: Failing deploy - name: Failure
if: ${{ contains(needs.*.result, 'failure') }} if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1 run: exit 1