Proposal: Test yaml for each component (#5398)

* Test for each component.

* When possible use commandline substitution.

* Add wildcard support.

* end file with new line.

* Move component tests into subfolder.

* Add component test to pipeline.

* Remove trailing whitespace.

* add restore python step.

* Add `. venv/bin/activate` to pipeline.

* step `changed-components` needs `common` step.

* start `list-components-changed.py` different.

* iterate on pipeline stage `list-components`.

* Update `checkout` action.

* Rename test folder from `tests` to `_test`.

* validate file exists.

* Move component test folder.

* extend list-components to include child components.

* File does not end with a newline

* Handle empty list-components matrix.

* list-components also check for changes in tests folder.

* Improve `list-components.py`.

* `*` is a forbidden character for filenames on windows.

---------

Co-authored-by: Your Name <you@example.com>
Co-authored-by: Keith Burzinski <kbx81x@gmail.com>

.github/workflows/ci.yml

@@ -392,6 +392,62 @@ jobs:
         # yamllint disable-line rule:line-length
         if: always()
 
+  list-components:
+    runs-on: ubuntu-latest
+    needs:
+      - common
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v4.1.1
+        with:
+          # Fetch enough history so `git merge-base refs/remotes/origin/dev HEAD` works.
+          fetch-depth: 500
+      - name: Fetch dev branch
+        run: |
+          git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=1 origin +refs/heads/dev*:refs/remotes/origin/dev* +refs/tags/dev*:refs/tags/dev*
+          git merge-base refs/remotes/origin/dev HEAD
+      - name: Restore Python
+        uses: ./.github/actions/restore-python
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+          cache-key: ${{ needs.common.outputs.cache-key }}
+      - name: Find changed components
+        id: set-matrix
+        run: |
+          . venv/bin/activate
+          echo "matrix=$(script/list-components.py --changed | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
+  test-build-components:
+    name: Component test ${{ matrix.file }}
+    runs-on: ubuntu-latest
+    needs:
+      - common
+      - list-components
+    if: ${{ needs.list-components.outputs.matrix != '[]' && needs.list-components.outputs.matrix != '' }}
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        file: ${{ fromJson(needs.list-components.outputs.matrix) }}
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v4.1.1
+      - name: Restore Python
+        uses: ./.github/actions/restore-python
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+          cache-key: ${{ needs.common.outputs.cache-key }}
+      - name: test_build_components -e config -c ${{ matrix.file }}
+        run: |
+          . venv/bin/activate
+          ./script/test_build_components -e config -c ${{ matrix.file }}
+      - name: test_build_components -e compile -c ${{ matrix.file }}
+        run: |
+          . venv/bin/activate
+          ./script/test_build_components -e compile -c ${{ matrix.file }}
+
   ci-status:
     name: CI Status
     runs-on: ubuntu-latest
@@ -406,6 +462,7 @@ jobs:
       - pyupgrade
       - compile-tests
       - clang-tidy
+      - test-build-components
     if: always()
     steps:
       - name: Success

script/fulltest

@@ -12,3 +12,4 @@ script/lint-cpp
 script/unit_test
 script/component_test
 script/test
+script/test_build_components

script/list-components.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import sys
+import argparse
+
+from helpers import git_ls_files, changed_files
+from esphome.loader import get_component, get_platform
+from esphome.core import CORE
+from esphome.const import (
+    KEY_CORE,
+    KEY_TARGET_FRAMEWORK,
+    KEY_TARGET_PLATFORM,
+    PLATFORM_ESP32,
+    PLATFORM_ESP8266,
+)
+
+
+def filter_component_files(str):
+    return str.startswith("esphome/components/") | str.startswith("tests/components/")
+
+
+def extract_component_names_array_from_files_array(files):
+    components = []
+    for file in files:
+        file_parts = file.split("/")
+        if len(file_parts) >= 4:
+            component_name = file_parts[2]
+            if component_name not in components:
+                components.append(component_name)
+    return components
+
+
+def add_item_to_components_graph(components_graph, parent, child):
+    if not parent.startswith("__") and parent != child:
+        if parent not in components_graph:
+            components_graph[parent] = []
+        if child not in components_graph[parent]:
+            components_graph[parent].append(child)
+
+
+def create_components_graph():
+    # The root directory of the repo
+    root = Path(__file__).parent.parent
+    components_dir = root / "esphome" / "components"
+    # Fake some directory so that get_component works
+    CORE.config_path = str(root)
+    # Various configuration to capture different outcomes used by `AUTO_LOAD` function.
+    TARGET_CONFIGURATIONS = [
+        {KEY_TARGET_FRAMEWORK: None, KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: "arduino", KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: "esp-idf", KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: None, KEY_TARGET_PLATFORM: PLATFORM_ESP32},
+    ]
+    CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+    components_graph = {}
+
+    for path in components_dir.iterdir():
+        if not path.is_dir():
+            continue
+        if not (path / "__init__.py").is_file():
+            continue
+        name = path.name
+        comp = get_component(name)
+        if comp is None:
+            print(
+                f"Cannot find component {name}. Make sure current path is pip installed ESPHome"
+            )
+            sys.exit(1)
+
+        for dependency in comp.dependencies:
+            add_item_to_components_graph(components_graph, dependency, name)
+
+        for target_config in TARGET_CONFIGURATIONS:
+            CORE.data[KEY_CORE] = target_config
+            for auto_load in comp.auto_load:
+                add_item_to_components_graph(components_graph, auto_load, name)
+        # restore config
+        CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+        for platform_path in path.iterdir():
+            platform_name = platform_path.stem
+            platform = get_platform(platform_name, name)
+            if platform is None:
+                continue
+
+            add_item_to_components_graph(components_graph, platform_name, name)
+
+            for dependency in platform.dependencies:
+                add_item_to_components_graph(components_graph, dependency, name)
+
+            for target_config in TARGET_CONFIGURATIONS:
+                CORE.data[KEY_CORE] = target_config
+                for auto_load in platform.auto_load:
+                    add_item_to_components_graph(components_graph, auto_load, name)
+            # restore config
+            CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+    return components_graph
+
+
+def find_children_of_component(components_graph, component_name, depth=0):
+    if component_name not in components_graph:
+        return []
+
+    children = []
+
+    for child in components_graph[component_name]:
+        children.append(child)
+        if depth < 10:
+            children.extend(
+                find_children_of_component(components_graph, child, depth + 1)
+            )
+    # Remove duplicate values
+    return list(set(children))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c", "--changed", action="store_true", help="Only run on changed files"
+    )
+    args = parser.parse_args()
+
+    files = git_ls_files()
+    files = filter(filter_component_files, files)
+
+    if args.changed:
+        changed = changed_files()
+        files = [f for f in files if f in changed]
+
+    components = extract_component_names_array_from_files_array(files)
+
+    if args.changed:
+        components_graph = create_components_graph()
+
+        all_changed_components = components.copy()
+        for c in components:
+            all_changed_components.extend(
+                find_children_of_component(components_graph, c)
+            )
+        # Remove duplicate values
+        all_changed_components = list(set(all_changed_components))
+
+        for c in sorted(all_changed_components):
+            print(c)
+    else:
+        for c in sorted(components):
+            print(c)
+
+
+if __name__ == "__main__":
+    main()

script/test_build_components

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Parse parameter:
+# - `e` - Parameter for `esphome` command. Default `compile`. Common alternative is `config`.
+# - `c` - Component folder name to test. Default `*`.
+esphome_command="compile"
+target_component="*"
+while getopts e:c: flag
+do
+    case $flag in
+        e) esphome_command=${OPTARG};;
+        c) target_component=${OPTARG};;
+        \?) echo "Usage: $0 [-e <config|compile|clean>] [-c <string>]" 1>&2; exit 1;;
+    esac
+done
+
+cd "$(dirname "$0")/.."
+
+if ! [ -d "./tests/test_build_components/build" ]; then
+  mkdir ./tests/test_build_components/build
+fi
+
+start_esphome() {
+  # create dynamic yaml file in `build` folder.
+  # `./tests/test_build_components/build/[target_component].[test_name].[target_platform].yaml`
+  component_test_file="./tests/test_build_components/build/$target_component.$test_name.$target_platform.yaml"
+
+  cp $target_platform_file $component_test_file
+  sed -i "s!\$component_test_file!../../.$f!g" $component_test_file
+
+  # Start esphome process
+  echo "> [$target_component] [$test_name] [$target_platform]"
+  echo "esphome -s component_name $target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file"
+  # TODO: Validate escape of Command line substitution value
+  esphome -s component_name $target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file
+}
+
+# Find all test yaml files.
+# - `./tests/components/[target_component]/[test_name].[target_platform].yaml`
+# - `./tests/components/[target_component]/[test_name].all.yaml`
+for f in ./tests/components/$target_component/*.*.yaml; do
+  [ -f "$f" ] || continue
+  IFS='/' read -r -a folder_name <<< "$f"
+  target_component="${folder_name[3]}"
+
+  IFS='.' read -r -a file_name <<< "${folder_name[4]}"
+  test_name="${file_name[0]}"
+  target_platform="${file_name[1]}"
+  file_name_parts=${#file_name[@]}
+
+  if [ "$target_platform" = "all" ] || [ $file_name_parts = 2 ]; then
+    # Test has *not* defined a specific target platform. Need to run tests for all possible target platforms.
+    
+    for target_platform_file in ./tests/test_build_components/build_components_base.*.yaml; do
+      IFS='/' read -r -a folder_name <<< "$target_platform_file"
+      IFS='.' read -r -a file_name <<< "${folder_name[3]}"
+      target_platform="${file_name[1]}"
+
+      start_esphome
+    done
+
+  else
+    # Test has defined a specific target platform.
+  
+    # Validate we have a base test yaml for selected platform.
+    # The target_platform is sourced from the following location.
+    # 1. `./tests/test_build_components/build_components_base.[target_platform].yaml`
+    # 2. `./tests/test_build_components/build_components_base.[target_platform]-ard.yaml`
+    target_platform_file="./tests/test_build_components/build_components_base.$target_platform.yaml"
+    if ! [ -f "$target_platform_file" ]; then
+      # Try find arduino test framework as platform.
+      target_platform_ard="$target_platform-ard"
+      target_platform_file="./tests/test_build_components/build_components_base.$target_platform_ard.yaml"
+      if ! [ -f "$target_platform_file" ]; then
+        echo "No base test file [./tests/test_build_components/build_components_base.$target_platform.yaml, ./tests/build_components_base.$target_platform_ard.yaml] for component test [$f] found."
+        exit 1
+      fi
+      target_platform=$target_platform_ard
+    fi
+
+    start_esphome
+  fi
+done

tests/components/adc/test.esp32-c3.yaml

@@ -0,0 +1,5 @@
+sensor:
+  - platform: adc
+    id: my_sensor
+    pin: 4
+    attenuation: 11db

tests/components/adc/test.esp32-idf.yaml

@@ -0,0 +1,11 @@
+sensor:
+  - platform: adc
+    pin: A0
+    name: Living Room Brightness
+    update_interval: "1:01"
+    attenuation: 2.5db
+    unit_of_measurement: "°C"
+    icon: "mdi:water-percent"
+    accuracy_decimals: 5
+    setup_priority: -100
+    force_update: true

tests/components/adc/test.esp32-s2.yaml

@@ -0,0 +1,5 @@
+sensor:
+  - platform: adc
+    id: my_sensor
+    pin: 1
+    attenuation: 11db

tests/components/adc/test.esp32-s3.yaml

@@ -0,0 +1,5 @@
+sensor:
+  - platform: adc
+    id: my_sensor
+    pin: 1
+    attenuation: 11db

tests/components/adc/test.esp32.yaml

@@ -0,0 +1,11 @@
+sensor:
+  - platform: adc
+    pin: A0
+    name: Living Room Brightness
+    update_interval: "1:01"
+    attenuation: 2.5db
+    unit_of_measurement: "°C"
+    icon: "mdi:water-percent"
+    accuracy_decimals: 5
+    setup_priority: -100
+    force_update: true

tests/components/adc/test.esp8266.yaml

@@ -0,0 +1,4 @@
+sensor:
+  - platform: adc
+    id: my_sensor
+    pin: VCC

tests/components/adc/test.rp2040.yaml

@@ -0,0 +1,4 @@
+sensor:
+  - platform: adc
+    pin: VCC
+    name: VSYS

tests/components/mopeka_std_check/test.esp32.yaml

@@ -0,0 +1,16 @@
+esp32_ble_tracker:
+
+sensor:
+  # Example using 11kg 100% propane tank.
+  - platform: mopeka_std_check
+    mac_address: D3:75:F2:DC:16:91
+    tank_type: Europe_11kg
+    temperature:
+        name: "Propane test temp"
+    level:
+        name: "Propane test level"
+    distance:
+        name: "Propane test distance"
+    battery_level:
+        name: "Propane test battery level"
+

tests/components/template/test.all.yaml

@@ -0,0 +1,127 @@
+sensor:
+  - platform: template
+    name: "Template Sensor"
+    id: template_sens
+    lambda: |-
+      if (id(some_binary_sensor).state) {
+        return 42.0;
+      } else {
+        return 0.0;
+      }
+    update_interval: 60s
+
+esphome:
+  on_boot:
+    - sensor.template.publish:
+        id: template_sens
+        state: 42.0
+
+    # Templated
+    - sensor.template.publish:
+        id: template_sens
+        state: !lambda 'return 42.0;'
+
+binary_sensor:
+  - platform: template
+    id: some_binary_sensor
+    name: "Garage Door Open"
+    lambda: |-
+      if (id(template_sens).state > 30) {
+        // Garage Door is open.
+        return true;
+      } else {
+        // Garage Door is closed.
+        return false;
+      }
+
+output:
+  - platform: template
+    id: outputsplit
+    type: float
+    write_action:
+      - logger.log: "write_action"
+
+switch:
+  - platform: template
+    name: "Template Switch"
+    lambda: |-
+      if (id(some_binary_sensor).state) {
+        return true;
+      } else {
+        return false;
+      }
+    turn_on_action:
+      - logger.log: "turn_on_action"
+    turn_off_action:
+      - logger.log: "turn_off_action"
+
+button:
+  - platform: template
+    name: "Template Button"
+    on_press:
+      - logger.log: Button Pressed
+
+cover:
+  - platform: template
+    name: "Template Cover"
+    lambda: |-
+      if (id(some_binary_sensor).state) {
+        return COVER_OPEN;
+      } else {
+        return COVER_CLOSED;
+      }
+    open_action:
+      - logger.log: open_action
+    close_action:
+      - logger.log: close_action
+    stop_action:
+      - logger.log: stop_action
+    optimistic: true
+
+number:
+  - platform: template
+    name: "Template number"
+    optimistic: true
+    min_value: 0
+    max_value: 100
+    step: 1
+
+select:
+  - platform: template
+    name: "Template select"
+    optimistic: true
+    options:
+      - one
+      - two
+      - three
+    initial_option: two
+
+lock:
+  - platform: template
+    name: "Template Lock"
+    lambda: |-
+      if (id(some_binary_sensor).state) {
+        return LOCK_STATE_LOCKED;
+      } else {
+        return LOCK_STATE_UNLOCKED;
+      }
+    lock_action:
+      - logger.log: lock_action
+    unlock_action:
+      - logger.log: unlock_action
+    open_action:
+      - logger.log: open_action
+
+text:
+  - platform: template
+    name: "Template text"
+    optimistic: true
+    min_length: 0
+    max_length: 100
+    mode: text
+
+alarm_control_panel:
+  - platform: template
+    name: Alarm Panel
+    codes:
+      - "1234"

tests/test_build_components/build_components_base.esp32-ard.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: componenttestesp32ard
+  friendly_name: $component_name
+
+esp32:
+  board: nodemcu-32s
+  framework:
+    type: arduino
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-c3-ard.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: componenttestesp32c3ard
+  friendly_name: $component_name
+
+esp32:
+  board: lolin_c3_mini
+  framework:
+    type: arduino
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-c3-idf.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: componenttestesp32c3idf
+  friendly_name: $component_name
+
+esp32:
+  board: lolin_c3_mini
+  framework:
+    type: esp-idf
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-idf.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: componenttestesp32idf
+  friendly_name: $component_name
+
+esp32:
+  board: nodemcu-32s
+  framework:
+    type: esp-idf
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-s2-ard.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: componenttestesp32s2ard
+  friendly_name: $component_name
+
+esp32:
+  board: esp32-s2-saola-1
+  variant: ESP32S2
+  framework:
+    type: arduino
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-s2-idf.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: componenttestesp32s2ard
+  friendly_name: $component_name
+
+esp32:
+  board: esp32-s2-saola-1
+  variant: ESP32S2
+  framework:
+    type: esp-idf
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-s3-ard.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: componenttestesp32s3ard
+  friendly_name: $component_name
+
+esp32:
+  board: esp32s3box
+  variant: ESP32S3
+  framework:
+    type: arduino
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp32-s3-idf.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: componenttestesp32s3ard
+  friendly_name: $component_name
+
+esp32:
+  board: esp32s3box
+  variant: ESP32S3
+  framework:
+    type: esp-idf
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.esp8266.yaml

@@ -0,0 +1,18 @@
+esphome:
+  name: componenttestesp8266
+  friendly_name: $component_name
+
+esp8266:
+  board: d1_mini
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file

tests/test_build_components/build_components_base.rp2040.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: componenttestrp2040
+  friendly_name: $component_name
+
+rp2040:
+  board: rpipicow
+  framework:
+    # Waiting for https://github.com/platformio/platform-raspberrypi/pull/36
+    platform_version: https://github.com/maxgerhardt/platform-raspberrypi.git
+
+logger:
+  level: VERY_VERBOSE
+
+packages:
+  component_under_test: !include
+    file: $component_test_file
+    vars:
+      component_name: $component_name
+      test_name: $test_name
+      target_platform: $target_platform
+      component_test_file: $component_test_file