Allow using a git source for a package (#2193)

esphome/components/external_components/__init__.py

@@ -1,13 +1,12 @@
 import re
 import logging
 from pathlib import Path
-import subprocess
-import hashlib
-import datetime
 
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_COMPONENTS,
+    CONF_REF,
+    CONF_REFRESH,
     CONF_SOURCE,
     CONF_URL,
     CONF_TYPE,
@@ -15,7 +14,7 @@ from esphome.const import (
     CONF_PATH,
 )
 from esphome.core import CORE
-from esphome import loader
+from esphome import git, loader
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -23,19 +22,11 @@ DOMAIN = CONF_EXTERNAL_COMPONENTS
 
 TYPE_GIT = "git"
 TYPE_LOCAL = "local"
-CONF_REFRESH = "refresh"
-CONF_REF = "ref"
-
-
-def validate_git_ref(value):
-    if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
-        raise cv.Invalid("Not a valid git ref")
-    return value
 
 
 GIT_SCHEMA = {
     cv.Required(CONF_URL): cv.url,
-    cv.Optional(CONF_REF): validate_git_ref,
+    cv.Optional(CONF_REF): cv.git_ref,
 }
 LOCAL_SCHEMA = {
     cv.Required(CONF_PATH): cv.directory,
@@ -68,14 +59,6 @@ def validate_source_shorthand(value):
     return SOURCE_SCHEMA(conf)
 
 
-def validate_refresh(value: str):
-    if value.lower() == "always":
-        return validate_refresh("0s")
-    if value.lower() == "never":
-        return validate_refresh("1000y")
-    return cv.positive_time_period_seconds(value)
-
-
 SOURCE_SCHEMA = cv.Any(
     validate_source_shorthand,
     cv.typed_schema(
@@ -90,7 +73,7 @@ SOURCE_SCHEMA = cv.Any(
 CONFIG_SCHEMA = cv.ensure_list(
     {
         cv.Required(CONF_SOURCE): SOURCE_SCHEMA,
-        cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh),
+        cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, cv.source_refresh),
         cv.Optional(CONF_COMPONENTS, default="all"): cv.Any(
             "all", cv.ensure_list(cv.string)
         ),
@@ -102,65 +85,13 @@ async def to_code(config):
     pass
 
 
-def _compute_destination_path(key: str) -> Path:
-    base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN
-    h = hashlib.new("sha256")
-    h.update(key.encode())
-    return base_dir / h.hexdigest()[:8]
-
-
-def _run_git_command(cmd, cwd=None):
-    try:
-        ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False)
-    except FileNotFoundError as err:
-        raise cv.Invalid(
-            "git is not installed but required for external_components.\n"
-            "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git"
-        ) from err
-
-    if ret.returncode != 0 and ret.stderr:
-        err_str = ret.stderr.decode("utf-8")
-        lines = [x.strip() for x in err_str.splitlines()]
-        if lines[-1].startswith("fatal:"):
-            raise cv.Invalid(lines[-1][len("fatal: ") :])
-        raise cv.Invalid(err_str)
-
-
 def _process_git_config(config: dict, refresh) -> str:
-    key = f"{config[CONF_URL]}@{config.get(CONF_REF)}"
+    repo_dir = git.clone_or_update(
-    repo_dir = _compute_destination_path(key)
+        url=config[CONF_URL],
-    if not repo_dir.is_dir():
+        ref=config.get(CONF_REF),
-        _LOGGER.info("Cloning %s", key)
+        refresh=refresh,
-        _LOGGER.debug("Location: %s", repo_dir)
+        domain=DOMAIN,
-        cmd = ["git", "clone", "--depth=1"]
+    )
-        if CONF_REF in config:
-            cmd += ["--branch", config[CONF_REF]]
-        cmd += ["--", config[CONF_URL], str(repo_dir)]
-        _run_git_command(cmd)
-
-    else:
-        # Check refresh needed
-        file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
-        # On first clone, FETCH_HEAD does not exists
-        if not file_timestamp.exists():
-            file_timestamp = Path(repo_dir / ".git" / "HEAD")
-        age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
-            file_timestamp.stat().st_mtime
-        )
-        if age.total_seconds() > refresh.total_seconds:
-            _LOGGER.info("Updating %s", key)
-            _LOGGER.debug("Location: %s", repo_dir)
-            # Stash local changes (if any)
-            _run_git_command(
-                ["git", "stash", "push", "--include-untracked"], str(repo_dir)
-            )
-            # Fetch remote ref
-            cmd = ["git", "fetch", "--", "origin"]
-            if CONF_REF in config:
-                cmd.append(config[CONF_REF])
-            _run_git_command(cmd, str(repo_dir))
-            # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
-            _run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
 
     if (repo_dir / "esphome" / "components").is_dir():
         components_dir = repo_dir / "esphome" / "components"

esphome/components/packages/__init__.py

@@ -1,6 +1,19 @@
+import re
+from pathlib import Path
+from esphome.core import EsphomeError
+
+from esphome import git, yaml_util
+from esphome.const import (
+    CONF_FILE,
+    CONF_FILES,
+    CONF_PACKAGES,
+    CONF_REF,
+    CONF_REFRESH,
+    CONF_URL,
+)
 import esphome.config_validation as cv
 
-from esphome.const import CONF_PACKAGES
+DOMAIN = CONF_PACKAGES
 
 
 def _merge_package(full_old, full_new):
@@ -23,11 +36,119 @@ def _merge_package(full_old, full_new):
     return merge(full_old, full_new)
 
 
+def validate_git_package(config: dict):
+    new_config = config
+    for key, conf in config.items():
+        if CONF_URL in conf:
+            try:
+                conf = BASE_SCHEMA(conf)
+                if CONF_FILE in conf:
+                    new_config[key][CONF_FILES] = [conf[CONF_FILE]]
+                    del new_config[key][CONF_FILE]
+            except cv.MultipleInvalid as e:
+                with cv.prepend_path([key]):
+                    raise e
+            except cv.Invalid as e:
+                raise cv.Invalid(
+                    "Extra keys not allowed in git based package",
+                    path=[key] + e.path,
+                ) from e
+    return new_config
+
+
+def validate_yaml_filename(value):
+    value = cv.string(value)
+
+    if not (value.endswith(".yaml") or value.endswith(".yml")):
+        raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.")
+
+    return value
+
+
+def validate_source_shorthand(value):
+    if not isinstance(value, str):
+        raise cv.Invalid("Shorthand only for strings")
+
+    m = re.match(
+        r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)/([a-zA-Z0-9\-_.\./]+)(?:@([a-zA-Z0-9\-_.\./]+))?",
+        value,
+    )
+    if m is None:
+        raise cv.Invalid(
+            "Source is not a file system path or in expected github://username/name/[sub-folder/]file-path.yml[@branch-or-tag] format!"
+        )
+
+    conf = {
+        CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
+        CONF_FILE: m.group(3),
+    }
+    if m.group(4):
+        conf[CONF_REF] = m.group(4)
+
+    # print(conf)
+    return BASE_SCHEMA(conf)
+
+
+BASE_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.Required(CONF_URL): cv.url,
+            cv.Exclusive(CONF_FILE, "files"): validate_yaml_filename,
+            cv.Exclusive(CONF_FILES, "files"): cv.All(
+                cv.ensure_list(validate_yaml_filename),
+                cv.Length(min=1),
+            ),
+            cv.Optional(CONF_REF): cv.git_ref,
+            cv.Optional(CONF_REFRESH, default="1d"): cv.All(
+                cv.string, cv.source_refresh
+            ),
+        }
+    ),
+    cv.has_at_least_one_key(CONF_FILE, CONF_FILES),
+)
+
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            str: cv.Any(validate_source_shorthand, BASE_SCHEMA, dict),
+        }
+    ),
+    validate_git_package,
+)
+
+
+def _process_base_package(config: dict) -> dict:
+    repo_dir = git.clone_or_update(
+        url=config[CONF_URL],
+        ref=config.get(CONF_REF),
+        refresh=config[CONF_REFRESH],
+        domain=DOMAIN,
+    )
+    files: str = config[CONF_FILES]
+
+    packages = {}
+    for file in files:
+        yaml_file: Path = repo_dir / file
+
+        if not yaml_file.is_file():
+            raise cv.Invalid(f"{file} does not exist in repository", path=[CONF_FILES])
+
+        try:
+            packages[file] = yaml_util.load_yaml(yaml_file)
+        except EsphomeError as e:
+            raise cv.Invalid(
+                f"{file} is not a valid YAML file. Please check the file contents."
+            ) from e
+    return {"packages": packages}
+
+
 def do_packages_pass(config: dict):
     if CONF_PACKAGES not in config:
         return config
     packages = config[CONF_PACKAGES]
     with cv.prepend_path(CONF_PACKAGES):
+        packages = CONFIG_SCHEMA(packages)
         if not isinstance(packages, dict):
             raise cv.Invalid(
                 "Packages must be a key to value mapping, got {} instead"
@@ -37,6 +158,8 @@ def do_packages_pass(config: dict):
         for package_name, package_config in packages.items():
             with cv.prepend_path(package_name):
                 recursive_package = package_config
+                if CONF_URL in package_config:
+                    package_config = _process_base_package(package_config)
                 if isinstance(package_config, dict):
                     recursive_package = do_packages_pass(package_config)
                 config = _merge_package(recursive_package, config)

esphome/config.py

@@ -333,6 +333,8 @@ def validate_config(config, command_line_substitutions):
             result.add_error(err)
             return result
 
+    CORE.raw_config = config
+
     # 1. Load substitutions
     if CONF_SUBSTITUTIONS in config:
         from esphome.components import substitutions
@@ -348,6 +350,8 @@ def validate_config(config, command_line_substitutions):
             result.add_error(err)
             return result
 
+    CORE.raw_config = config
+
     # 1.1. Check for REPLACEME special value
     try:
         recursive_check_replaceme(config)

esphome/config_validation.py

@@ -33,7 +33,6 @@ from esphome.const import (
     CONF_UPDATE_INTERVAL,
     CONF_TYPE_ID,
     CONF_TYPE,
-    CONF_PACKAGES,
 )
 from esphome.core import (
     CORE,
@@ -1455,15 +1454,7 @@ class OnlyWith(Optional):
     @property
     def default(self):
         # pylint: disable=unsupported-membership-test
-        if self._component in CORE.raw_config or (
+        if self._component in CORE.raw_config:
-            CONF_PACKAGES in CORE.raw_config
-            and self._component
-            in [
-                k
-                for package in CORE.raw_config[CONF_PACKAGES].values()
-                for k in package.keys()
-            ]
-        ):
             return self._default
         return vol.UNDEFINED
 
@@ -1633,3 +1624,17 @@ def url(value):
     if not parsed.scheme or not parsed.netloc:
         raise Invalid("Expected a URL scheme and host")
     return parsed.geturl()
+
+
+def git_ref(value):
+    if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
+        raise Invalid("Not a valid git ref")
+    return value
+
+
+def source_refresh(value: str):
+    if value.lower() == "always":
+        return source_refresh("0s")
+    if value.lower() == "never":
+        return source_refresh("1000y")
+    return positive_time_period_seconds(value)

esphome/const.py

@@ -235,6 +235,7 @@ CONF_FAN_WITH_COOLING = "fan_with_cooling"
 CONF_FAN_WITH_HEATING = "fan_with_heating"
 CONF_FAST_CONNECT = "fast_connect"
 CONF_FILE = "file"
+CONF_FILES = "files"
 CONF_FILTER = "filter"
 CONF_FILTER_OUT = "filter_out"
 CONF_FILTERS = "filters"
@@ -525,8 +526,10 @@ CONF_REACTIVE_POWER = "reactive_power"
 CONF_REBOOT_TIMEOUT = "reboot_timeout"
 CONF_RECEIVE_TIMEOUT = "receive_timeout"
 CONF_RED = "red"
+CONF_REF = "ref"
 CONF_REFERENCE_RESISTANCE = "reference_resistance"
 CONF_REFERENCE_TEMPERATURE = "reference_temperature"
+CONF_REFRESH = "refresh"
 CONF_REPEAT = "repeat"
 CONF_REPOSITORY = "repository"
 CONF_RESET_PIN = "reset_pin"

esphome/git.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+import subprocess
+import hashlib
+import logging
+
+from datetime import datetime
+
+from esphome.core import CORE, TimePeriodSeconds
+import esphome.config_validation as cv
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def run_git_command(cmd, cwd=None):
+    try:
+        ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False)
+    except FileNotFoundError as err:
+        raise cv.Invalid(
+            "git is not installed but required for external_components.\n"
+            "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git"
+        ) from err
+
+    if ret.returncode != 0 and ret.stderr:
+        err_str = ret.stderr.decode("utf-8")
+        lines = [x.strip() for x in err_str.splitlines()]
+        if lines[-1].startswith("fatal:"):
+            raise cv.Invalid(lines[-1][len("fatal: ") :])
+        raise cv.Invalid(err_str)
+
+
+def _compute_destination_path(key: str, domain: str) -> Path:
+    base_dir = Path(CORE.config_dir) / ".esphome" / domain
+    h = hashlib.new("sha256")
+    h.update(key.encode())
+    return base_dir / h.hexdigest()[:8]
+
+
+def clone_or_update(
+    *, url: str, ref: str = None, refresh: TimePeriodSeconds, domain: str
+) -> Path:
+    key = f"{url}@{ref}"
+    repo_dir = _compute_destination_path(key, domain)
+    if not repo_dir.is_dir():
+        _LOGGER.info("Cloning %s", key)
+        _LOGGER.debug("Location: %s", repo_dir)
+        cmd = ["git", "clone", "--depth=1"]
+        if ref is not None:
+            cmd += ["--branch", ref]
+        cmd += ["--", url, str(repo_dir)]
+        run_git_command(cmd)
+
+    else:
+        # Check refresh needed
+        file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
+        # On first clone, FETCH_HEAD does not exists
+        if not file_timestamp.exists():
+            file_timestamp = Path(repo_dir / ".git" / "HEAD")
+        age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime)
+        if age.total_seconds() > refresh.total_seconds:
+            _LOGGER.info("Updating %s", key)
+            _LOGGER.debug("Location: %s", repo_dir)
+            # Stash local changes (if any)
+            run_git_command(
+                ["git", "stash", "push", "--include-untracked"], str(repo_dir)
+            )
+            # Fetch remote ref
+            cmd = ["git", "fetch", "--", "origin"]
+            if ref is not None:
+                cmd.append(ref)
+            run_git_command(cmd, str(repo_dir))
+            # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
+            run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
+
+    return repo_dir