Packages feature (#1052)

* Started to work on packages feature

* Added some more validation to packages config

* Fixed some linter warnings

* Updated tests

* Reordered consts to avoid linter error

* Reordered consts to avoid linter error

* Refactored test yaml files to integrate into existing test pipeline

Co-authored-by: Dmitry Berezovsky <dmitry.berezovsky@logicify.com>

esphome/components/packages/__init__.py

@@ -0,0 +1,51 @@
+from deepmerge import conservative_merger as package_merger
+
+import esphome.config_validation as cv
+
+from esphome.const import CONF_PACKAGES
+
+VALID_PACKAGE_NAME_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
+
+
+def _merge_package(config, package_name, package_config):
+    config = config.copy()
+    package_merger.merge(config, package_config)
+    return config
+
+
+def _is_valid_package_name(value: str) -> bool:
+    if not value:
+        return False
+    if value[0].isdigit():
+        return False
+    try:
+        cv.valid_name(value)
+    except cv.Invalid:
+        return False
+    return True
+
+
+def do_packages_pass(config: dict):
+    if CONF_PACKAGES not in config:
+        return
+    packages = config[CONF_PACKAGES]
+    temp_config = config.copy()
+    with cv.prepend_path(CONF_PACKAGES):
+        if packages is not None and not isinstance(packages, dict):
+            raise cv.Invalid("Packages must be a key to value mapping, got {} instead"
+                             "".format(type(packages)))
+        for package_name, package_config in packages.items():
+            with cv.prepend_path(package_name):
+                if not isinstance(package_config, dict):
+                    raise cv.Invalid("Package definition must be a dictionary containing valid "
+                                     "esphome configuration to be merged with the main "
+                                     "config, got {} instead"
+                                     .format(type(package_config)))
+                if not _is_valid_package_name(package_name):
+                    raise cv.Invalid("Package name is invalid. Valid name should consist of "
+                                     "letters, numbers and underscores. It shouldn't also "
+                                     "start with number")
+                temp_config = _merge_package(temp_config, package_name, package_config)
+        del temp_config[CONF_PACKAGES]
+        config.clear()
+        config.update(temp_config)

esphome/config.py

@@ -12,8 +12,9 @@ import voluptuous as vol
 
 from esphome import core, core_config, yaml_util
 from esphome.components import substitutions
+from esphome.components.packages import do_packages_pass
 from esphome.components.substitutions import CONF_SUBSTITUTIONS
-from esphome.const import CONF_ESPHOME, CONF_PLATFORM, ESP_PLATFORMS
+from esphome.const import CONF_ESPHOME, CONF_PLATFORM, ESP_PLATFORMS, CONF_PACKAGES
 from esphome.core import CORE, EsphomeError  # noqa
 from esphome.helpers import color, indent
 from esphome.util import safe_print, OrderedDict
@@ -390,6 +391,16 @@ def recursive_check_replaceme(value):
 def validate_config(config, command_line_substitutions):
     result = Config()
 
+    # 0. Load packages
+    if CONF_PACKAGES in config:
+        result.add_output_path([CONF_PACKAGES], CONF_PACKAGES)
+        try:
+            do_packages_pass(config)
+        except vol.Invalid as err:
+            result.update(config)
+            result.add_error(err)
+            return result
+
     # 1. Load substitutions
     if CONF_SUBSTITUTIONS in config:
         result[CONF_SUBSTITUTIONS] = {**config[CONF_SUBSTITUTIONS], **command_line_substitutions}

esphome/const.py

@@ -342,6 +342,7 @@ CONF_OUTPUT = 'output'
 CONF_OUTPUT_ID = 'output_id'
 CONF_OUTPUTS = 'outputs'
 CONF_OVERSAMPLING = 'oversampling'
+CONF_PACKAGES = 'packages'
 CONF_PAGE_ID = 'page_id'
 CONF_PAGES = 'pages'
 CONF_PANASONIC = 'panasonic'

requirements.txt

@@ -11,3 +11,4 @@ ifaddr==0.1.6
 platformio==4.3.3
 esptool==2.8
 click==7.1.2
+deepmerge==0.1.0

requirements_test.txt

@@ -10,6 +10,7 @@ pyserial==3.4
 ifaddr==0.1.6
 platformio==4.3.3
 esptool==2.8
+deepmerge==0.1.0
 
 pylint==2.5.0
 flake8==3.7.9

setup.py

@@ -33,6 +33,7 @@ REQUIRES = [
     'pytz==2020.1',
     'pyserial==3.4',
     'ifaddr==0.1.6',
+    'deepmerge==0.1.0'
 ]
 
 # If you have problems importing platformio and esptool as modules you can set

tests/test1.yaml

@@ -1,3 +1,6 @@
+substitutions:
+  devicename: test1
+
 esphome:
   name: test1
   platform: ESP32
@@ -25,6 +28,10 @@ esphome:
           white: 100%
   build_path: build/test1
 
+packages:
+  wifi: !include test_packages/test_packages_package_wifi.yaml
+  pkg_test: !include test_packages/test_packages_package1.yaml
+
 wifi:
   networks:
   - ssid: 'MySSID'

tests/test_packages/test_packages_package1.yaml

@@ -0,0 +1,2 @@
+sensor:
+  - <<: !include ./test_uptime_sensor.yaml

tests/test_packages/test_packages_package_wifi.yaml

@@ -0,0 +1,4 @@
+wifi:
+  networks:
+  - ssid: 'WiFiFromPackage'
+    password: 'password1'

tests/test_packages/test_uptime_sensor.yaml

@@ -0,0 +1,5 @@
+# Uptime sensor.
+platform: uptime
+id: ${devicename}_uptime_pcg
+name: Uptime From Package
+update_interval: 5min