From f9b0666adf760591444301d60662ea2d302d96e5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 6 Sep 2021 08:23:06 +1200 Subject: [PATCH] Allow using a git source for a package (#2193) --- .../external_components/__init__.py | 91 ++----------- esphome/components/packages/__init__.py | 125 +++++++++++++++++- esphome/config.py | 4 + esphome/config_validation.py | 25 ++-- esphome/const.py | 3 + esphome/git.py | 74 +++++++++++ 6 files changed, 231 insertions(+), 91 deletions(-) create mode 100644 esphome/git.py diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index 8f6e2bece4..110a8d95ed 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -1,13 +1,12 @@ import re import logging from pathlib import Path -import subprocess -import hashlib -import datetime import esphome.config_validation as cv from esphome.const import ( CONF_COMPONENTS, + CONF_REF, + CONF_REFRESH, CONF_SOURCE, CONF_URL, CONF_TYPE, @@ -15,7 +14,7 @@ from esphome.const import ( CONF_PATH, ) from esphome.core import CORE -from esphome import loader +from esphome import git, loader _LOGGER = logging.getLogger(__name__) @@ -23,19 +22,11 @@ DOMAIN = CONF_EXTERNAL_COMPONENTS TYPE_GIT = "git" TYPE_LOCAL = "local" -CONF_REFRESH = "refresh" -CONF_REF = "ref" - - -def validate_git_ref(value): - if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None: - raise cv.Invalid("Not a valid git ref") - return value GIT_SCHEMA = { cv.Required(CONF_URL): cv.url, - cv.Optional(CONF_REF): validate_git_ref, + cv.Optional(CONF_REF): cv.git_ref, } LOCAL_SCHEMA = { cv.Required(CONF_PATH): cv.directory, @@ -68,14 +59,6 @@ def validate_source_shorthand(value): return SOURCE_SCHEMA(conf) -def validate_refresh(value: str): - if value.lower() == "always": - return validate_refresh("0s") - if value.lower() == "never": - return validate_refresh("1000y") - return cv.positive_time_period_seconds(value) - - SOURCE_SCHEMA = cv.Any( validate_source_shorthand, cv.typed_schema( @@ -90,7 +73,7 @@ SOURCE_SCHEMA = cv.Any( CONFIG_SCHEMA = cv.ensure_list( { cv.Required(CONF_SOURCE): SOURCE_SCHEMA, - cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh), + cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, cv.source_refresh), cv.Optional(CONF_COMPONENTS, default="all"): cv.Any( "all", cv.ensure_list(cv.string) ), @@ -102,65 +85,13 @@ async def to_code(config): pass -def _compute_destination_path(key: str) -> Path: - base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN - h = hashlib.new("sha256") - h.update(key.encode()) - return base_dir / h.hexdigest()[:8] - - -def _run_git_command(cmd, cwd=None): - try: - ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False) - except FileNotFoundError as err: - raise cv.Invalid( - "git is not installed but required for external_components.\n" - "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git" - ) from err - - if ret.returncode != 0 and ret.stderr: - err_str = ret.stderr.decode("utf-8") - lines = [x.strip() for x in err_str.splitlines()] - if lines[-1].startswith("fatal:"): - raise cv.Invalid(lines[-1][len("fatal: ") :]) - raise cv.Invalid(err_str) - - def _process_git_config(config: dict, refresh) -> str: - key = f"{config[CONF_URL]}@{config.get(CONF_REF)}" - repo_dir = _compute_destination_path(key) - if not repo_dir.is_dir(): - _LOGGER.info("Cloning %s", key) - _LOGGER.debug("Location: %s", repo_dir) - cmd = ["git", "clone", "--depth=1"] - if CONF_REF in config: - cmd += ["--branch", config[CONF_REF]] - cmd += ["--", config[CONF_URL], str(repo_dir)] - _run_git_command(cmd) - - else: - # Check refresh needed - file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD") - # On first clone, FETCH_HEAD does not exists - if not file_timestamp.exists(): - file_timestamp = Path(repo_dir / ".git" / "HEAD") - age = datetime.datetime.now() - datetime.datetime.fromtimestamp( - file_timestamp.stat().st_mtime - ) - if age.total_seconds() > refresh.total_seconds: - _LOGGER.info("Updating %s", key) - _LOGGER.debug("Location: %s", repo_dir) - # Stash local changes (if any) - _run_git_command( - ["git", "stash", "push", "--include-untracked"], str(repo_dir) - ) - # Fetch remote ref - cmd = ["git", "fetch", "--", "origin"] - if CONF_REF in config: - cmd.append(config[CONF_REF]) - _run_git_command(cmd, str(repo_dir)) - # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch) - _run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir)) + repo_dir = git.clone_or_update( + url=config[CONF_URL], + ref=config.get(CONF_REF), + refresh=refresh, + domain=DOMAIN, + ) if (repo_dir / "esphome" / "components").is_dir(): components_dir = repo_dir / "esphome" / "components" diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 8c5c9a0144..330ffc2bf2 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -1,6 +1,19 @@ +import re +from pathlib import Path +from esphome.core import EsphomeError + +from esphome import git, yaml_util +from esphome.const import ( + CONF_FILE, + CONF_FILES, + CONF_PACKAGES, + CONF_REF, + CONF_REFRESH, + CONF_URL, +) import esphome.config_validation as cv -from esphome.const import CONF_PACKAGES +DOMAIN = CONF_PACKAGES def _merge_package(full_old, full_new): @@ -23,11 +36,119 @@ def _merge_package(full_old, full_new): return merge(full_old, full_new) +def validate_git_package(config: dict): + new_config = config + for key, conf in config.items(): + if CONF_URL in conf: + try: + conf = BASE_SCHEMA(conf) + if CONF_FILE in conf: + new_config[key][CONF_FILES] = [conf[CONF_FILE]] + del new_config[key][CONF_FILE] + except cv.MultipleInvalid as e: + with cv.prepend_path([key]): + raise e + except cv.Invalid as e: + raise cv.Invalid( + "Extra keys not allowed in git based package", + path=[key] + e.path, + ) from e + return new_config + + +def validate_yaml_filename(value): + value = cv.string(value) + + if not (value.endswith(".yaml") or value.endswith(".yml")): + raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.") + + return value + + +def validate_source_shorthand(value): + if not isinstance(value, str): + raise cv.Invalid("Shorthand only for strings") + + m = re.match( + r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)/([a-zA-Z0-9\-_.\./]+)(?:@([a-zA-Z0-9\-_.\./]+))?", + value, + ) + if m is None: + raise cv.Invalid( + "Source is not a file system path or in expected github://username/name/[sub-folder/]file-path.yml[@branch-or-tag] format!" + ) + + conf = { + CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git", + CONF_FILE: m.group(3), + } + if m.group(4): + conf[CONF_REF] = m.group(4) + + # print(conf) + return BASE_SCHEMA(conf) + + +BASE_SCHEMA = cv.All( + cv.Schema( + { + cv.Required(CONF_URL): cv.url, + cv.Exclusive(CONF_FILE, "files"): validate_yaml_filename, + cv.Exclusive(CONF_FILES, "files"): cv.All( + cv.ensure_list(validate_yaml_filename), + cv.Length(min=1), + ), + cv.Optional(CONF_REF): cv.git_ref, + cv.Optional(CONF_REFRESH, default="1d"): cv.All( + cv.string, cv.source_refresh + ), + } + ), + cv.has_at_least_one_key(CONF_FILE, CONF_FILES), +) + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + str: cv.Any(validate_source_shorthand, BASE_SCHEMA, dict), + } + ), + validate_git_package, +) + + +def _process_base_package(config: dict) -> dict: + repo_dir = git.clone_or_update( + url=config[CONF_URL], + ref=config.get(CONF_REF), + refresh=config[CONF_REFRESH], + domain=DOMAIN, + ) + files: str = config[CONF_FILES] + + packages = {} + for file in files: + yaml_file: Path = repo_dir / file + + if not yaml_file.is_file(): + raise cv.Invalid(f"{file} does not exist in repository", path=[CONF_FILES]) + + try: + packages[file] = yaml_util.load_yaml(yaml_file) + except EsphomeError as e: + raise cv.Invalid( + f"{file} is not a valid YAML file. Please check the file contents." + ) from e + return {"packages": packages} + + def do_packages_pass(config: dict): if CONF_PACKAGES not in config: return config packages = config[CONF_PACKAGES] with cv.prepend_path(CONF_PACKAGES): + packages = CONFIG_SCHEMA(packages) if not isinstance(packages, dict): raise cv.Invalid( "Packages must be a key to value mapping, got {} instead" @@ -37,6 +158,8 @@ def do_packages_pass(config: dict): for package_name, package_config in packages.items(): with cv.prepend_path(package_name): recursive_package = package_config + if CONF_URL in package_config: + package_config = _process_base_package(package_config) if isinstance(package_config, dict): recursive_package = do_packages_pass(package_config) config = _merge_package(recursive_package, config) diff --git a/esphome/config.py b/esphome/config.py index 93413a009c..de261f7eba 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -333,6 +333,8 @@ def validate_config(config, command_line_substitutions): result.add_error(err) return result + CORE.raw_config = config + # 1. Load substitutions if CONF_SUBSTITUTIONS in config: from esphome.components import substitutions @@ -348,6 +350,8 @@ def validate_config(config, command_line_substitutions): result.add_error(err) return result + CORE.raw_config = config + # 1.1. Check for REPLACEME special value try: recursive_check_replaceme(config) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 4df65a38e3..fb659c41ea 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -33,7 +33,6 @@ from esphome.const import ( CONF_UPDATE_INTERVAL, CONF_TYPE_ID, CONF_TYPE, - CONF_PACKAGES, ) from esphome.core import ( CORE, @@ -1455,15 +1454,7 @@ class OnlyWith(Optional): @property def default(self): # pylint: disable=unsupported-membership-test - if self._component in CORE.raw_config or ( - CONF_PACKAGES in CORE.raw_config - and self._component - in [ - k - for package in CORE.raw_config[CONF_PACKAGES].values() - for k in package.keys() - ] - ): + if self._component in CORE.raw_config: return self._default return vol.UNDEFINED @@ -1633,3 +1624,17 @@ def url(value): if not parsed.scheme or not parsed.netloc: raise Invalid("Expected a URL scheme and host") return parsed.geturl() + + +def git_ref(value): + if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None: + raise Invalid("Not a valid git ref") + return value + + +def source_refresh(value: str): + if value.lower() == "always": + return source_refresh("0s") + if value.lower() == "never": + return source_refresh("1000y") + return positive_time_period_seconds(value) diff --git a/esphome/const.py b/esphome/const.py index 06ce736a85..7132cb1d1d 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -235,6 +235,7 @@ CONF_FAN_WITH_COOLING = "fan_with_cooling" CONF_FAN_WITH_HEATING = "fan_with_heating" CONF_FAST_CONNECT = "fast_connect" CONF_FILE = "file" +CONF_FILES = "files" CONF_FILTER = "filter" CONF_FILTER_OUT = "filter_out" CONF_FILTERS = "filters" @@ -525,8 +526,10 @@ CONF_REACTIVE_POWER = "reactive_power" CONF_REBOOT_TIMEOUT = "reboot_timeout" CONF_RECEIVE_TIMEOUT = "receive_timeout" CONF_RED = "red" +CONF_REF = "ref" CONF_REFERENCE_RESISTANCE = "reference_resistance" CONF_REFERENCE_TEMPERATURE = "reference_temperature" +CONF_REFRESH = "refresh" CONF_REPEAT = "repeat" CONF_REPOSITORY = "repository" CONF_RESET_PIN = "reset_pin" diff --git a/esphome/git.py b/esphome/git.py new file mode 100644 index 0000000000..12c6b41648 --- /dev/null +++ b/esphome/git.py @@ -0,0 +1,74 @@ +from pathlib import Path +import subprocess +import hashlib +import logging + +from datetime import datetime + +from esphome.core import CORE, TimePeriodSeconds +import esphome.config_validation as cv + +_LOGGER = logging.getLogger(__name__) + + +def run_git_command(cmd, cwd=None): + try: + ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False) + except FileNotFoundError as err: + raise cv.Invalid( + "git is not installed but required for external_components.\n" + "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git" + ) from err + + if ret.returncode != 0 and ret.stderr: + err_str = ret.stderr.decode("utf-8") + lines = [x.strip() for x in err_str.splitlines()] + if lines[-1].startswith("fatal:"): + raise cv.Invalid(lines[-1][len("fatal: ") :]) + raise cv.Invalid(err_str) + + +def _compute_destination_path(key: str, domain: str) -> Path: + base_dir = Path(CORE.config_dir) / ".esphome" / domain + h = hashlib.new("sha256") + h.update(key.encode()) + return base_dir / h.hexdigest()[:8] + + +def clone_or_update( + *, url: str, ref: str = None, refresh: TimePeriodSeconds, domain: str +) -> Path: + key = f"{url}@{ref}" + repo_dir = _compute_destination_path(key, domain) + if not repo_dir.is_dir(): + _LOGGER.info("Cloning %s", key) + _LOGGER.debug("Location: %s", repo_dir) + cmd = ["git", "clone", "--depth=1"] + if ref is not None: + cmd += ["--branch", ref] + cmd += ["--", url, str(repo_dir)] + run_git_command(cmd) + + else: + # Check refresh needed + file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD") + # On first clone, FETCH_HEAD does not exists + if not file_timestamp.exists(): + file_timestamp = Path(repo_dir / ".git" / "HEAD") + age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime) + if age.total_seconds() > refresh.total_seconds: + _LOGGER.info("Updating %s", key) + _LOGGER.debug("Location: %s", repo_dir) + # Stash local changes (if any) + run_git_command( + ["git", "stash", "push", "--include-untracked"], str(repo_dir) + ) + # Fetch remote ref + cmd = ["git", "fetch", "--", "origin"] + if ref is not None: + cmd.append(ref) + run_git_command(cmd, str(repo_dir)) + # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch) + run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir)) + + return repo_dir