Implement external custom components installing from YAML (#1630)

* Move components import loading to importlib MetaPathFinder and importlib.resources

* Add external_components component

* Fix

* Fix

* fix cv.url return

* fix validate shorthand git

* implement git refresh

* Use finders from sys.path_hooks instead of looking for __init__.py

* use github:// schema

* error handling

* add test

* fix handling git output

* revert file check handling

* fix test

* allow full component path be specified for local

* fix test

* fix path handling

* lint

Co-authored-by: Guillermo Ruffino <glm.net@gmail.com>

.github/workflows/ci.yml

@@ -115,7 +115,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

.github/workflows/release-dev.yml

@@ -112,7 +112,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

.github/workflows/release.yml

@@ -111,7 +111,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

esphome/components/external_components/__init__.py

@@ -0,0 +1,197 @@
+import re
+import logging
+from pathlib import Path
+import subprocess
+import hashlib
+import datetime
+
+import esphome.config_validation as cv
+from esphome.const import (
+    CONF_COMPONENTS,
+    CONF_SOURCE,
+    CONF_URL,
+    CONF_TYPE,
+    CONF_EXTERNAL_COMPONENTS,
+    CONF_PATH,
+)
+from esphome.core import CORE
+from esphome import loader
+
+_LOGGER = logging.getLogger(__name__)
+
+DOMAIN = CONF_EXTERNAL_COMPONENTS
+
+TYPE_GIT = "git"
+TYPE_LOCAL = "local"
+CONF_REFRESH = "refresh"
+CONF_REF = "ref"
+
+
+def validate_git_ref(value):
+    if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
+        raise cv.Invalid("Not a valid git ref")
+    return value
+
+
+GIT_SCHEMA = {
+    cv.Required(CONF_URL): cv.url,
+    cv.Optional(CONF_REF): validate_git_ref,
+}
+LOCAL_SCHEMA = {
+    cv.Required(CONF_PATH): cv.directory,
+}
+
+
+def validate_source_shorthand(value):
+    if not isinstance(value, str):
+        raise cv.Invalid("Shorthand only for strings")
+    try:
+        return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value})
+    except cv.Invalid:
+        pass
+    # Regex for GitHub repo name with optional branch/tag
+    # Note: git allows other branch/tag names as well, but never seen them used before
+    m = re.match(
+        r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?",
+        value,
+    )
+    if m is None:
+        raise cv.Invalid(
+            "Source is not a file system path or in expected github://username/name[@branch-or-tag] format!"
+        )
+    conf = {
+        CONF_TYPE: TYPE_GIT,
+        CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
+    }
+    if m.group(3):
+        conf[CONF_REF] = m.group(3)
+    return SOURCE_SCHEMA(conf)
+
+
+def validate_refresh(value: str):
+    if value.lower() == "always":
+        return validate_refresh("0s")
+    if value.lower() == "never":
+        return validate_refresh("1000y")
+    return cv.positive_time_period_seconds(value)
+
+
+SOURCE_SCHEMA = cv.Any(
+    validate_source_shorthand,
+    cv.typed_schema(
+        {
+            TYPE_GIT: cv.Schema(GIT_SCHEMA),
+            TYPE_LOCAL: cv.Schema(LOCAL_SCHEMA),
+        }
+    ),
+)
+
+
+CONFIG_SCHEMA = cv.ensure_list(
+    {
+        cv.Required(CONF_SOURCE): SOURCE_SCHEMA,
+        cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh),
+        cv.Optional(CONF_COMPONENTS, default="all"): cv.Any(
+            "all", cv.ensure_list(cv.string)
+        ),
+    }
+)
+
+
+def to_code(config):
+    pass
+
+
+def _compute_destination_path(key: str) -> Path:
+    base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN
+    h = hashlib.new("sha256")
+    h.update(key.encode())
+    return base_dir / h.hexdigest()[:8]
+
+
+def _handle_git_response(ret):
+    if ret.returncode != 0 and ret.stderr:
+        err_str = ret.stderr.decode("utf-8")
+        lines = [x.strip() for x in err_str.splitlines()]
+        if lines[-1].startswith("fatal:"):
+            raise cv.Invalid(lines[-1][len("fatal: ") :])
+        raise cv.Invalid(err_str)
+
+
+def _process_single_config(config: dict):
+    conf = config[CONF_SOURCE]
+    if conf[CONF_TYPE] == TYPE_GIT:
+        key = f"{conf[CONF_URL]}@{conf.get(CONF_REF)}"
+        repo_dir = _compute_destination_path(key)
+        if not repo_dir.is_dir():
+            cmd = ["git", "clone", "--depth=1"]
+            if CONF_REF in conf:
+                cmd += ["--branch", conf[CONF_REF]]
+            cmd += [conf[CONF_URL], str(repo_dir)]
+            ret = subprocess.run(cmd, capture_output=True, check=False)
+            _handle_git_response(ret)
+
+        else:
+            # Check refresh needed
+            file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
+            # On first clone, FETCH_HEAD does not exists
+            if not file_timestamp.exists():
+                file_timestamp = Path(repo_dir / ".git" / "HEAD")
+            age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
+                file_timestamp.stat().st_mtime
+            )
+            if age.seconds > config[CONF_REFRESH].total_seconds:
+                _LOGGER.info("Executing git pull %s", key)
+                cmd = ["git", "pull"]
+                ret = subprocess.run(
+                    cmd, cwd=repo_dir, capture_output=True, check=False
+                )
+                _handle_git_response(ret)
+
+        if (repo_dir / "esphome" / "components").is_dir():
+            components_dir = repo_dir / "esphome" / "components"
+        elif (repo_dir / "components").is_dir():
+            components_dir = repo_dir / "components"
+        else:
+            raise cv.Invalid(
+                "Could not find components folder for source. Please check the source contains a 'components' or 'esphome/components' folder",
+                [CONF_SOURCE],
+            )
+
+    elif conf[CONF_TYPE] == TYPE_LOCAL:
+        components_dir = Path(CORE.relative_config_path(conf[CONF_PATH]))
+    else:
+        raise NotImplementedError()
+
+    if config[CONF_COMPONENTS] == "all":
+        num_components = len(list(components_dir.glob("*/__init__.py")))
+        if num_components > 100:
+            # Prevent accidentally including all components from an esphome fork/branch
+            # In this case force the user to manually specify which components they want to include
+            raise cv.Invalid(
+                "This source is an ESPHome fork or branch. Please manually specify the components you want to import using the 'components' key",
+                [CONF_COMPONENTS],
+            )
+        allowed_components = None
+    else:
+        for i, name in enumerate(config[CONF_COMPONENTS]):
+            expected = components_dir / name / "__init__.py"
+            if not expected.is_file():
+                raise cv.Invalid(
+                    f"Could not find __init__.py file for component {name}. Please check the component is defined by this source (search path: {expected})",
+                    [CONF_COMPONENTS, i],
+                )
+        allowed_components = config[CONF_COMPONENTS]
+
+    loader.install_meta_finder(components_dir, allowed_components=allowed_components)
+
+
+def do_external_components_pass(config: dict) -> None:
+    conf = config.get(DOMAIN)
+    if conf is None:
+        return
+    with cv.prepend_path(DOMAIN):
+        conf = CONFIG_SCHEMA(conf)
+        for i, c in enumerate(conf):
+            with cv.prepend_path(i):
+                _process_single_config(c)

esphome/components/http_request/__init__.py

@@ -14,7 +14,7 @@ from esphome.const import (
     CONF_URL,
 )
 from esphome.core import CORE, Lambda
-from esphome.core_config import PLATFORMIO_ESP8266_LUT
+from esphome.core.config import PLATFORMIO_ESP8266_LUT
 
 DEPENDENCIES = ["network"]
 AUTO_LOAD = ["json"]

esphome/config.py

@@ -1,195 +1,34 @@
 import collections
-import importlib
 import logging
 import re
-import os.path
 
 # pylint: disable=unused-import, wrong-import-order
-import sys
 from contextlib import contextmanager
 
 import voluptuous as vol
 
-from esphome import core, core_config, yaml_util
+from esphome import core, yaml_util, loader
+import esphome.core.config as core_config
 from esphome.const import (
     CONF_ESPHOME,
     CONF_PLATFORM,
-    ESP_PLATFORMS,
     CONF_PACKAGES,
     CONF_SUBSTITUTIONS,
+    CONF_EXTERNAL_COMPONENTS,
 )
-from esphome.core import CORE, EsphomeError  # noqa
+from esphome.core import CORE, EsphomeError
 from esphome.helpers import indent
 from esphome.util import safe_print, OrderedDict
 
-from typing import List, Optional, Tuple, Union  # noqa
+from typing import List, Optional, Tuple, Union
-from esphome.core import ConfigType  # noqa
+from esphome.core import ConfigType
+from esphome.loader import get_component, get_platform, ComponentManifest
 from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue
 from esphome.voluptuous_schema import ExtraKeysInvalid
 from esphome.log import color, Fore
 
 _LOGGER = logging.getLogger(__name__)
 
-_COMPONENT_CACHE = {}
-
-
-class ComponentManifest:
-    def __init__(self, module, base_components_path, is_core=False, is_platform=False):
-        self.module = module
-        self._is_core = is_core
-        self.is_platform = is_platform
-        self.base_components_path = base_components_path
-
-    @property
-    def is_platform_component(self):
-        return getattr(self.module, "IS_PLATFORM_COMPONENT", False)
-
-    @property
-    def config_schema(self):
-        return getattr(self.module, "CONFIG_SCHEMA", None)
-
-    @property
-    def multi_conf(self):
-        return getattr(self.module, "MULTI_CONF", False)
-
-    @property
-    def to_code(self):
-        return getattr(self.module, "to_code", None)
-
-    @property
-    def esp_platforms(self):
-        return getattr(self.module, "ESP_PLATFORMS", ESP_PLATFORMS)
-
-    @property
-    def dependencies(self):
-        return getattr(self.module, "DEPENDENCIES", [])
-
-    @property
-    def conflicts_with(self):
-        return getattr(self.module, "CONFLICTS_WITH", [])
-
-    @property
-    def auto_load(self):
-        return getattr(self.module, "AUTO_LOAD", [])
-
-    @property
-    def codeowners(self) -> List[str]:
-        return getattr(self.module, "CODEOWNERS", [])
-
-    def _get_flags_set(self, name, config):
-        if not hasattr(self.module, name):
-            return set()
-        obj = getattr(self.module, name)
-        if callable(obj):
-            obj = obj(config)
-        if obj is None:
-            return set()
-        if not isinstance(obj, (list, tuple, set)):
-            obj = [obj]
-        return set(obj)
-
-    @property
-    def source_files(self):
-        if self._is_core:
-            core_p = os.path.abspath(os.path.join(os.path.dirname(__file__), "core"))
-            source_files = core.find_source_files(os.path.join(core_p, "dummy"))
-            ret = {}
-            for f in source_files:
-                ret[f"esphome/core/{f}"] = os.path.join(core_p, f)
-            return ret
-
-        source_files = core.find_source_files(self.module.__file__)
-        ret = {}
-        # Make paths absolute
-        directory = os.path.abspath(os.path.dirname(self.module.__file__))
-        for x in source_files:
-            full_file = os.path.join(directory, x)
-            rel = os.path.relpath(full_file, self.base_components_path)
-            # Always use / for C++ include names
-            rel = rel.replace(os.sep, "/")
-            target_file = f"esphome/components/{rel}"
-            ret[target_file] = full_file
-        return ret
-
-
-CORE_COMPONENTS_PATH = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "components")
-)
-_UNDEF = object()
-CUSTOM_COMPONENTS_PATH = _UNDEF
-
-
-def _mount_config_dir():
-    global CUSTOM_COMPONENTS_PATH
-    if CUSTOM_COMPONENTS_PATH is not _UNDEF:
-        return
-    custom_path = os.path.abspath(os.path.join(CORE.config_dir, "custom_components"))
-    if not os.path.isdir(custom_path):
-        CUSTOM_COMPONENTS_PATH = None
-        return
-    if CORE.config_dir not in sys.path:
-        sys.path.insert(0, CORE.config_dir)
-    CUSTOM_COMPONENTS_PATH = custom_path
-
-
-def _lookup_module(domain, is_platform):
-    if domain in _COMPONENT_CACHE:
-        return _COMPONENT_CACHE[domain]
-
-    _mount_config_dir()
-    # First look for custom_components
-    try:
-        module = importlib.import_module(f"custom_components.{domain}")
-    except ImportError as e:
-        # ImportError when no such module
-        if "No module named" not in str(e):
-            _LOGGER.warning(
-                "Unable to import custom component %s:", domain, exc_info=True
-            )
-    except Exception:  # pylint: disable=broad-except
-        # Other error means component has an issue
-        _LOGGER.error("Unable to load custom component %s:", domain, exc_info=True)
-        return None
-    else:
-        # Found in custom components
-        manif = ComponentManifest(
-            module, CUSTOM_COMPONENTS_PATH, is_platform=is_platform
-        )
-        _COMPONENT_CACHE[domain] = manif
-        return manif
-
-    try:
-        module = importlib.import_module(f"esphome.components.{domain}")
-    except ImportError as e:
-        if "No module named" not in str(e):
-            _LOGGER.error("Unable to import component %s:", domain, exc_info=True)
-        return None
-    except Exception:  # pylint: disable=broad-except
-        _LOGGER.error("Unable to load component %s:", domain, exc_info=True)
-        return None
-    else:
-        manif = ComponentManifest(module, CORE_COMPONENTS_PATH, is_platform=is_platform)
-        _COMPONENT_CACHE[domain] = manif
-        return manif
-
-
-def get_component(domain):
-    assert "." not in domain
-    return _lookup_module(domain, False)
-
-
-def get_platform(domain, platform):
-    full = f"{platform}.{domain}"
-    return _lookup_module(full, True)
-
-
-_COMPONENT_CACHE["esphome"] = ComponentManifest(
-    core_config,
-    CORE_COMPONENTS_PATH,
-    is_core=True,
-    is_platform=False,
-)
-
 
 def iter_components(config):
     for domain, conf in config.items():
@@ -453,6 +292,9 @@ def recursive_check_replaceme(value):
 def validate_config(config, command_line_substitutions):
     result = Config()
 
+    loader.clear_component_meta_finders()
+    loader.install_custom_components_meta_finder()
+
     # 0. Load packages
     if CONF_PACKAGES in config:
         from esphome.components.packages import do_packages_pass
@@ -486,6 +328,18 @@ def validate_config(config, command_line_substitutions):
     except vol.Invalid as err:
         result.add_error(err)
 
+    # 1.2. Load external_components
+    if CONF_EXTERNAL_COMPONENTS in config:
+        from esphome.components.external_components import do_external_components_pass
+
+        result.add_output_path([CONF_EXTERNAL_COMPONENTS], CONF_EXTERNAL_COMPONENTS)
+        try:
+            do_external_components_pass(config)
+        except vol.Invalid as err:
+            result.update(config)
+            result.add_error(err)
+            return result
+
     if "esphomeyaml" in config:
         _LOGGER.warning(
             "The esphomeyaml section has been renamed to esphome in 1.11.0. "

esphome/config_validation.py

@@ -1556,3 +1556,17 @@ def polling_component_schema(default_update_interval):
             ): update_interval,
         }
     )
+
+
+def url(value):
+    import urllib.parse
+
+    value = string_strict(value)
+    try:
+        parsed = urllib.parse.urlparse(value)
+    except ValueError as e:
+        raise Invalid("Not a valid URL") from e
+
+    if not parsed.scheme or not parsed.netloc:
+        raise Invalid("Expected a URL scheme and host")
+    return parsed.geturl()

esphome/const.py

@@ -193,6 +193,7 @@ CONF_ESPHOME = "esphome"
 CONF_ETHERNET = "ethernet"
 CONF_EVENT = "event"
 CONF_EXPIRE_AFTER = "expire_after"
+CONF_EXTERNAL_COMPONENTS = "external_components"
 CONF_EXTERNAL_VCC = "external_vcc"
 CONF_FALLING_EDGE = "falling_edge"
 CONF_FAMILY = "family"
@@ -405,6 +406,7 @@ CONF_PAGE_ID = "page_id"
 CONF_PAGES = "pages"
 CONF_PANASONIC = "panasonic"
 CONF_PASSWORD = "password"
+CONF_PATH = "path"
 CONF_PAYLOAD = "payload"
 CONF_PAYLOAD_AVAILABLE = "payload_available"
 CONF_PAYLOAD_NOT_AVAILABLE = "payload_not_available"
@@ -514,6 +516,7 @@ CONF_SLEEP_DURATION = "sleep_duration"
 CONF_SLEEP_PIN = "sleep_pin"
 CONF_SLEEP_WHEN_DONE = "sleep_when_done"
 CONF_SONY = "sony"
+CONF_SOURCE = "source"
 CONF_SPEED = "speed"
 CONF_SPEED_COMMAND_TOPIC = "speed_command_topic"
 CONF_SPEED_COUNT = "speed_count"

esphome/core/__init__.py

@@ -23,7 +23,7 @@ from esphome.helpers import ensure_unique_string, is_hassio
 from esphome.util import OrderedDict
 
 if TYPE_CHECKING:
-    from .cpp_generator import MockObj, MockObjClass, Statement
+    from ..cpp_generator import MockObj, MockObjClass, Statement
 
 _LOGGER = logging.getLogger(__name__)
 

esphome/helpers.py

@@ -222,7 +222,7 @@ def write_file_if_changed(path: Union[Path, str], text: str):
         write_file(path, text)
 
 
-def copy_file_if_changed(src, dst):
+def copy_file_if_changed(src: os.PathLike, dst: os.PathLike) -> None:
     import shutil
 
     if file_compare(src, dst):
@@ -240,7 +240,7 @@ def list_starts_with(list_, sub):
     return len(sub) <= len(list_) and all(list_[i] == x for i, x in enumerate(sub))
 
 
-def file_compare(path1, path2):
+def file_compare(path1: os.PathLike, path2: os.PathLike) -> bool:
     """Return True if the files path1 and path2 have the same contents."""
     import stat
 

esphome/loader.py

@@ -0,0 +1,179 @@
+import logging
+import typing
+from typing import Callable, List, Optional, Dict, Any, ContextManager
+from types import ModuleType
+import importlib
+import importlib.util
+import importlib.resources
+import importlib.abc
+import sys
+from pathlib import Path
+
+from esphome.const import ESP_PLATFORMS, SOURCE_FILE_EXTENSIONS
+import esphome.core.config
+from esphome.core import CORE
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class SourceFile:
+    def __init__(
+        self,
+        package: importlib.resources.Package,
+        resource: importlib.resources.Resource,
+    ) -> None:
+        self._package = package
+        self._resource = resource
+
+    def open_binary(self) -> typing.BinaryIO:
+        return importlib.resources.open_binary(self._package, self._resource)
+
+    def path(self) -> ContextManager[Path]:
+        return importlib.resources.path(self._package, self._resource)
+
+
+class ComponentManifest:
+    def __init__(self, module: ModuleType):
+        self.module = module
+
+    @property
+    def package(self) -> str:
+        return self.module.__package__
+
+    @property
+    def is_platform(self) -> bool:
+        return len(self.module.__name__.split(".")) == 4
+
+    @property
+    def is_platform_component(self) -> bool:
+        return getattr(self.module, "IS_PLATFORM_COMPONENT", False)
+
+    @property
+    def config_schema(self) -> Optional[Any]:
+        return getattr(self.module, "CONFIG_SCHEMA", None)
+
+    @property
+    def multi_conf(self) -> bool:
+        return getattr(self.module, "MULTI_CONF", False)
+
+    @property
+    def to_code(self) -> Optional[Callable[[Any], None]]:
+        return getattr(self.module, "to_code", None)
+
+    @property
+    def esp_platforms(self) -> List[str]:
+        return getattr(self.module, "ESP_PLATFORMS", ESP_PLATFORMS)
+
+    @property
+    def dependencies(self) -> List[str]:
+        return getattr(self.module, "DEPENDENCIES", [])
+
+    @property
+    def conflicts_with(self) -> List[str]:
+        return getattr(self.module, "CONFLICTS_WITH", [])
+
+    @property
+    def auto_load(self) -> List[str]:
+        return getattr(self.module, "AUTO_LOAD", [])
+
+    @property
+    def codeowners(self) -> List[str]:
+        return getattr(self.module, "CODEOWNERS", [])
+
+    @property
+    def source_files(self) -> Dict[Path, SourceFile]:
+        ret = {}
+        for resource in importlib.resources.contents(self.package):
+            if Path(resource).suffix not in SOURCE_FILE_EXTENSIONS:
+                continue
+            if not importlib.resources.is_resource(self.package, resource):
+                # Not a resource = this is a directory (yeah this is confusing)
+                continue
+            # Always use / for C++ include names
+            target_path = Path(*self.package.split(".")) / resource
+            ret[target_path] = SourceFile(self.package, resource)
+        return ret
+
+
+class ComponentMetaFinder(importlib.abc.MetaPathFinder):
+    def __init__(
+        self, components_path: Path, allowed_components: Optional[List[str]] = None
+    ) -> None:
+        self._allowed_components = allowed_components
+        self._finders = []
+        for hook in sys.path_hooks:
+            try:
+                finder = hook(str(components_path))
+            except ImportError:
+                continue
+            self._finders.append(finder)
+
+    def find_spec(self, fullname: str, path: Optional[List[str]], target=None):
+        if not fullname.startswith("esphome.components."):
+            return None
+        parts = fullname.split(".")
+        if len(parts) != 3:
+            # only handle direct components, not platforms
+            # platforms are handled automatically when parent is imported
+            return None
+        component = parts[2]
+        if (
+            self._allowed_components is not None
+            and component not in self._allowed_components
+        ):
+            return None
+
+        for finder in self._finders:
+            spec = finder.find_spec(fullname, target=target)
+            if spec is not None:
+                return spec
+        return None
+
+
+def clear_component_meta_finders():
+    sys.meta_path = [x for x in sys.meta_path if not isinstance(x, ComponentMetaFinder)]
+
+
+def install_meta_finder(
+    components_path: Path, allowed_components: Optional[List[str]] = None
+):
+    sys.meta_path.insert(0, ComponentMetaFinder(components_path, allowed_components))
+
+
+def install_custom_components_meta_finder():
+    custom_components_dir = (Path(CORE.config_dir) / "custom_components").resolve()
+    install_meta_finder(custom_components_dir)
+
+
+def _lookup_module(domain):
+    if domain in _COMPONENT_CACHE:
+        return _COMPONENT_CACHE[domain]
+
+    try:
+        module = importlib.import_module(f"esphome.components.{domain}")
+    except ImportError as e:
+        if "No module named" not in str(e):
+            _LOGGER.error("Unable to import component %s:", domain, exc_info=True)
+        return None
+    except Exception:  # pylint: disable=broad-except
+        _LOGGER.error("Unable to load component %s:", domain, exc_info=True)
+        return None
+    else:
+        manif = ComponentManifest(module)
+        _COMPONENT_CACHE[domain] = manif
+        return manif
+
+
+def get_component(domain):
+    assert "." not in domain
+    return _lookup_module(domain)
+
+
+def get_platform(domain, platform):
+    full = f"{platform}.{domain}"
+    return _lookup_module(full)
+
+
+_COMPONENT_CACHE = {}
+CORE_COMPONENTS_PATH = (Path(__file__).parent / "components").resolve()
+_COMPONENT_CACHE["esphome"] = ComponentManifest(esphome.core.config)

esphome/writer.py

@@ -1,6 +1,8 @@
 import logging
 import os
 import re
+from pathlib import Path
+from typing import Dict
 
 from esphome.config import iter_components
 from esphome.const import (
@@ -24,6 +26,7 @@ from esphome.helpers import (
 )
 from esphome.storage_json import StorageJSON, storage_path
 from esphome.pins import ESP8266_FLASH_SIZES, ESP8266_LD_SCRIPTS
+from esphome import loader
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -355,7 +358,7 @@ or use the custom_components folder.
 
 
 def copy_src_tree():
-    source_files = {}
+    source_files: Dict[Path, loader.SourceFile] = {}
     for _, component, _ in iter_components(CORE.config):
         source_files.update(component.source_files)
 
@@ -365,37 +368,40 @@ def copy_src_tree():
 
     # Build #include list for esphome.h
     include_l = []
-    for target, path in source_files_l:
+    for target, _ in source_files_l:
-        if os.path.splitext(path)[1] in HEADER_FILE_EXTENSIONS:
+        if target.suffix in HEADER_FILE_EXTENSIONS:
             include_l.append(f'#include "{target}"')
     include_l.append("")
     include_s = "\n".join(include_l)
 
     source_files_copy = source_files.copy()
-    source_files_copy.pop(DEFINES_H_TARGET)
+    ignore_targets = [Path(x) for x in (DEFINES_H_TARGET, VERSION_H_TARGET)]
+    for t in ignore_targets:
+        source_files_copy.pop(t)
 
-    for path in walk_files(CORE.relative_src_path("esphome")):
+    for fname in walk_files(CORE.relative_src_path("esphome")):
-        if os.path.splitext(path)[1] not in SOURCE_FILE_EXTENSIONS:
+        p = Path(fname)
+        if p.suffix not in SOURCE_FILE_EXTENSIONS:
             # Not a source file, ignore
             continue
         # Transform path to target path name
-        target = os.path.relpath(path, CORE.relative_src_path()).replace(
+        target = p.relative_to(CORE.relative_src_path())
-            os.path.sep, "/"
+        if target in ignore_targets:
-        )
-        if target in (DEFINES_H_TARGET, VERSION_H_TARGET):
             # Ignore defines.h, will be dealt with later
             continue
         if target not in source_files_copy:
             # Source file removed, delete target
-            os.remove(path)
+            p.unlink()
         else:
-            src_path = source_files_copy.pop(target)
+            src_file = source_files_copy.pop(target)
-            copy_file_if_changed(src_path, path)
+            with src_file.path() as src_path:
+                copy_file_if_changed(src_path, p)
 
     # Now copy new files
-    for target, src_path in source_files_copy.items():
+    for target, src_file in source_files_copy.items():
-        dst_path = CORE.relative_src_path(*target.split("/"))
+        dst_path = CORE.relative_src_path(*target.parts)
-        copy_file_if_changed(src_path, dst_path)
+        with src_file.path() as src_path:
+            copy_file_if_changed(src_path, dst_path)
 
     # Finally copy defines
     write_file_if_changed(

script/build_jsonschema.py

@@ -62,7 +62,7 @@ def add_definition_array_or_single_object(ref):
 
 
 def add_core():
-    from esphome.core_config import CONFIG_SCHEMA
+    from esphome.core.config import CONFIG_SCHEMA
 
     base_props["esphome"] = get_jschema("esphome", CONFIG_SCHEMA.schema)
 

tests/test4.yaml

@@ -160,3 +160,9 @@ display:
     lambda: |-
       it.rectangle(0, 0, it.get_width(), it.get_height());
 
+external_components:
+  - source: github://esphome/esphome@dev
+    refresh: 1d
+    components: ["bh1750"]
+  - source: ../esphome/components
+    components: ["sntp"]