diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 8392008222..98483e8ae9 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -165,7 +165,7 @@ def do_packages_pass(config: dict): f"Packages must be a key to value mapping, got {type(packages)} instead" ) - for package_name, package_config in packages.items(): + for package_name, package_config in reversed(packages.items()): with cv.prepend_path(package_name): recursive_package = package_config if CONF_URL in package_config: diff --git a/esphome/config.py b/esphome/config.py index e63cddf347..b04de020e0 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -14,6 +14,7 @@ from esphome import core, yaml_util, loader import esphome.core.config as core_config from esphome.const import ( CONF_ESPHOME, + CONF_ID, CONF_PLATFORM, CONF_PACKAGES, CONF_SUBSTITUTIONS, @@ -24,6 +25,7 @@ from esphome.core import CORE, EsphomeError from esphome.helpers import indent from esphome.util import safe_print, OrderedDict +from esphome.config_helpers import Extend from esphome.loader import get_component, get_platform, ComponentManifest from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue from esphome.voluptuous_schema import ExtraKeysInvalid @@ -334,6 +336,13 @@ class LoadValidationStep(ConfigValidationStep): continue p_name = p_config.get("platform") if p_name is None: + p_id = p_config.get(CONF_ID) + if isinstance(p_id, Extend): + result.add_str_error( + f"Source for extension of ID '{p_id.value}' was not found.", + path + [CONF_ID], + ) + continue result.add_str_error("No platform specified! See 'platform' key.", path) continue # Remove temp output path and construct new one diff --git a/esphome/config_helpers.py b/esphome/config_helpers.py index d36a2f1e7f..e1d63775bb 100644 --- a/esphome/config_helpers.py +++ b/esphome/config_helpers.py @@ -1,10 +1,27 @@ import json import os +from esphome.const import CONF_ID from esphome.core import CORE from esphome.helpers import read_file +class Extend: + def __init__(self, value): + self.value = value + + def __str__(self): + return f"!extend {self.value}" + + def __eq__(self, b): + """ + Check if two Extend objects contain the same ID. + + Only used in unit tests. + """ + return isinstance(b, Extend) and self.value == b.value + + def read_config_file(path: str) -> str: if CORE.vscode and ( not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path) @@ -36,7 +53,25 @@ def merge_config(full_old, full_new): if isinstance(new, list): if not isinstance(old, list): return new - return old + new + res = old.copy() + ids = { + v[CONF_ID]: i + for i, v in enumerate(res) + if CONF_ID in v and isinstance(v[CONF_ID], str) + } + for v in new: + if CONF_ID in v: + new_id = v[CONF_ID] + if isinstance(new_id, Extend): + new_id = new_id.value + if new_id in ids: + v[CONF_ID] = new_id + res[ids[new_id]] = merge(res[ids[new_id]], v) + continue + else: + ids[new_id] = len(res) + res.append(v) + return res if new is None: return old diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 359b9ba88e..7440f71790 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -13,6 +13,7 @@ import voluptuous as vol from esphome import core import esphome.codegen as cg +from esphome.config_helpers import Extend from esphome.const import ( ALLOWED_NAME_CHARS, CONF_AVAILABILITY, @@ -490,6 +491,8 @@ def declare_id(type): if value is None: return core.ID(None, is_declaration=True, type=type) + if isinstance(value, Extend): + raise Invalid(f"Source for extension of ID '{value.value}' was not found.") return core.ID(validate_id_name(value), is_declaration=True, type=type) return validator diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index b8aa8dd53f..8a03c431a7 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -10,7 +10,7 @@ import yaml import yaml.constructor from esphome import core -from esphome.config_helpers import read_config_file +from esphome.config_helpers import read_config_file, Extend from esphome.core import ( EsphomeError, IPAddress, @@ -338,6 +338,10 @@ class ESPHomeLoader(yaml.SafeLoader): obj = self.construct_scalar(node) return add_class_to_obj(obj, ESPForceValue) + @_add_data_ref + def construct_extend(self, node): + return Extend(str(node.value)) + ESPHomeLoader.add_constructor("tag:yaml.org,2002:int", ESPHomeLoader.construct_yaml_int) ESPHomeLoader.add_constructor( @@ -369,6 +373,7 @@ ESPHomeLoader.add_constructor( ) ESPHomeLoader.add_constructor("!lambda", ESPHomeLoader.construct_lambda) ESPHomeLoader.add_constructor("!force", ESPHomeLoader.construct_force) +ESPHomeLoader.add_constructor("!extend", ESPHomeLoader.construct_extend) def load_yaml(fname, clear_secrets=True): diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py new file mode 100644 index 0000000000..0e24d78f5c --- /dev/null +++ b/tests/component_tests/packages/test_packages.py @@ -0,0 +1,351 @@ +"""Tests for the packages component.""" + +import pytest + + +from esphome.const import ( + CONF_DOMAIN, + CONF_ESPHOME, + CONF_FILTERS, + CONF_ID, + CONF_MULTIPLY, + CONF_NAME, + CONF_OFFSET, + CONF_PACKAGES, + CONF_PASSWORD, + CONF_PLATFORM, + CONF_SENSOR, + CONF_SSID, + CONF_UPDATE_INTERVAL, + CONF_WIFI, +) +from esphome.components.packages import do_packages_pass +from esphome.config_helpers import Extend +import esphome.config_validation as cv + +# Test strings +TEST_DEVICE_NAME = "test_device_name" +TEST_PLATFORM = "test_platform" +TEST_WIFI_SSID = "test_wifi_ssid" +TEST_PACKAGE_WIFI_SSID = "test_package_wifi_ssid" +TEST_PACKAGE_WIFI_PASSWORD = "test_package_wifi_password" +TEST_DOMAIN = "test_domain_name" +TEST_SENSOR_PLATFORM_1 = "test_sensor_platform_1" +TEST_SENSOR_PLATFORM_2 = "test_sensor_platform_2" +TEST_SENSOR_NAME_1 = "test_sensor_name_1" +TEST_SENSOR_NAME_2 = "test_sensor_name_2" +TEST_SENSOR_ID_1 = "test_sensor_id_1" +TEST_SENSOR_ID_2 = "test_sensor_id_2" +TEST_SENSOR_UPDATE_INTERVAL = "test_sensor_update_interval" + + +@pytest.fixture(name="basic_wifi") +def fixture_basic_wifi(): + return { + CONF_SSID: TEST_PACKAGE_WIFI_SSID, + CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD, + } + + +@pytest.fixture(name="basic_esphome") +def fixture_basic_esphome(): + return {CONF_NAME: TEST_DEVICE_NAME, CONF_PLATFORM: TEST_PLATFORM} + + +def test_package_unused(basic_esphome, basic_wifi): + """ + Ensures do_package_pass does not change a config if packages aren't used. + """ + config = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi} + + actual = do_packages_pass(config) + assert actual == config + + +def test_package_invalid_dict(basic_esphome, basic_wifi): + """ + Ensures an error is raised if packages is not valid. + + """ + config = {CONF_ESPHOME: basic_esphome, CONF_PACKAGES: basic_wifi} + + with pytest.raises(cv.Invalid): + do_packages_pass(config) + + +def test_package_include(basic_wifi, basic_esphome): + """ + Tests the simple case where an independent config present in a package is added to the top-level config as is. + + In this test, the CONF_WIFI config is expected to be simply added to the top-level config. + """ + config = { + CONF_ESPHOME: basic_esphome, + CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}}, + } + + expected = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi} + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_append(basic_wifi, basic_esphome): + """ + Tests the case where a key is present in both a package and top-level config. + + In this test, CONF_WIFI is defined in a package, and CONF_DOMAIN is added to it at the top level. + """ + config = { + CONF_ESPHOME: basic_esphome, + CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}}, + CONF_WIFI: {CONF_DOMAIN: TEST_DOMAIN}, + } + + expected = { + CONF_ESPHOME: basic_esphome, + CONF_WIFI: { + CONF_SSID: TEST_PACKAGE_WIFI_SSID, + CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD, + CONF_DOMAIN: TEST_DOMAIN, + }, + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_override(basic_wifi, basic_esphome): + """ + Ensures that the top-level configuration takes precedence over duplicate keys defined in a package. + + In this test, CONF_SSID should be overwritten by that defined in the top-level config. + """ + config = { + CONF_ESPHOME: basic_esphome, + CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}}, + CONF_WIFI: {CONF_SSID: TEST_WIFI_SSID}, + } + + expected = { + CONF_ESPHOME: basic_esphome, + CONF_WIFI: { + CONF_SSID: TEST_WIFI_SSID, + CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD, + }, + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_multiple_package_order(): + """ + Ensures that mutiple packages are merged in order. + """ + config = { + CONF_PACKAGES: { + "package1": { + "logger": { + "level": "DEBUG", + }, + }, + "package2": { + "logger": { + "level": "VERBOSE", + }, + }, + }, + } + + expected = { + "logger": { + "level": "VERBOSE", + }, + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_list_merge(): + """ + Ensures lists defined in both a package and the top-level config are merged correctly + """ + config = { + CONF_PACKAGES: { + "package_sensors": { + CONF_SENSOR: [ + { + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_1, + }, + { + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_2, + }, + ] + } + }, + CONF_SENSOR: [ + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1}, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2}, + ], + } + + expected = { + CONF_SENSOR: [ + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_1}, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_2}, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1}, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2}, + ] + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_list_merge_by_id(): + """ + Ensures that components with matching IDs are merged correctly. + + In this test, a sensor is defined in a package, and a CONF_UPDATE_INTERVAL is added at the top level, + and a sensor name is overridden in another sensor. + """ + config = { + CONF_PACKAGES: { + "package_sensors": { + CONF_SENSOR: [ + { + CONF_ID: TEST_SENSOR_ID_1, + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_1, + }, + { + CONF_ID: TEST_SENSOR_ID_2, + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_2, + }, + ] + }, + "package2": { + CONF_SENSOR: [ + { + CONF_ID: Extend(TEST_SENSOR_ID_1), + CONF_DOMAIN: "2", + } + ], + }, + "package3": { + CONF_SENSOR: [ + { + CONF_ID: Extend(TEST_SENSOR_ID_1), + CONF_DOMAIN: "3", + } + ], + }, + }, + CONF_SENSOR: [ + { + CONF_ID: Extend(TEST_SENSOR_ID_1), + CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL, + }, + {CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_NAME: TEST_SENSOR_NAME_1}, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2}, + ], + } + + expected = { + CONF_SENSOR: [ + { + CONF_ID: TEST_SENSOR_ID_1, + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_1, + CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL, + CONF_DOMAIN: "3", + }, + { + CONF_ID: TEST_SENSOR_ID_2, + CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, + CONF_NAME: TEST_SENSOR_NAME_1, + }, + {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2}, + ] + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_merge_by_id_with_list(): + """ + Ensures that components with matching IDs are merged correctly when their configuration contains lists. + + For example, a sensor with filters defined in both a package and the top level config should be merged. + """ + + config = { + CONF_PACKAGES: { + "sensors": { + CONF_SENSOR: [ + {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]} + ] + } + }, + CONF_SENSOR: [ + {CONF_ID: Extend(TEST_SENSOR_ID_1), CONF_FILTERS: [{CONF_OFFSET: 146.0}]} + ], + } + + expected = { + CONF_SENSOR: [ + { + CONF_ID: TEST_SENSOR_ID_1, + CONF_FILTERS: [{CONF_MULTIPLY: 42.0}, {CONF_OFFSET: 146.0}], + } + ] + } + + actual = do_packages_pass(config) + assert actual == expected + + +def test_package_merge_by_missing_id(): + """ + Ensures that components with missing IDs are not merged. + """ + + config = { + CONF_PACKAGES: { + "sensors": { + CONF_SENSOR: [ + {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]}, + ] + } + }, + CONF_SENSOR: [ + {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 10.0}]}, + {CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_FILTERS: [{CONF_OFFSET: 146.0}]}, + ], + } + + expected = { + CONF_SENSOR: [ + { + CONF_ID: TEST_SENSOR_ID_1, + CONF_FILTERS: [{CONF_MULTIPLY: 42.0}], + }, + { + CONF_ID: TEST_SENSOR_ID_1, + CONF_FILTERS: [{CONF_MULTIPLY: 10.0}], + }, + { + CONF_ID: Extend(TEST_SENSOR_ID_2), + CONF_FILTERS: [{CONF_OFFSET: 146.0}], + }, + ] + } + + actual = do_packages_pass(config) + assert actual == expected diff --git a/tests/test1.yaml b/tests/test1.yaml index a567c41f2a..a77f8802b9 100644 --- a/tests/test1.yaml +++ b/tests/test1.yaml @@ -1076,6 +1076,8 @@ sensor: id: ultrasonic_sensor1 - platform: uptime name: Uptime Sensor + - id: !extend ${devicename}_uptime_pcg + unit_of_measurement: s - platform: wifi_signal name: WiFi Signal Sensor update_interval: 15s