Initial stab at importing idf components (#4000)

* Initial stab at importing idf components

* Handle repo with multiple components
Allow components directly from yaml

* Actually use the refresh config var

* Update esphome/components/esp32/__init__.py

esphome/components/esp32/__init__.py

@@ -4,29 +4,43 @@ from pathlib import Path
 import logging
 import os
 
-from esphome.helpers import copy_file_if_changed, write_file_if_changed
+from esphome.helpers import copy_file_if_changed, write_file_if_changed, mkdir_p
 from esphome.const import (
     CONF_BOARD,
+    CONF_COMPONENTS,
     CONF_FRAMEWORK,
+    CONF_NAME,
     CONF_SOURCE,
     CONF_TYPE,
     CONF_VARIANT,
     CONF_VERSION,
     CONF_ADVANCED,
+    CONF_REFRESH,
+    CONF_PATH,
+    CONF_URL,
+    CONF_REF,
     CONF_IGNORE_EFUSE_MAC_CRC,
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
+    TYPE_GIT,
+    TYPE_LOCAL,
     __version__,
 )
-from esphome.core import CORE, HexInt
+from esphome.core import CORE, HexInt, TimePeriod
 import esphome.config_validation as cv
 import esphome.codegen as cg
+from esphome import git
 
 from .const import (  # noqa
     KEY_BOARD,
+    KEY_COMPONENTS,
     KEY_ESP32,
+    KEY_PATH,
+    KEY_REF,
+    KEY_REFRESH,
+    KEY_REPO,
     KEY_SDKCONFIG_OPTIONS,
     KEY_VARIANT,
     VARIANT_ESP32C3,
@@ -51,6 +65,7 @@ def set_core_data(config):
     if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF:
         CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "esp-idf"
         CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] = {}
+        CORE.data[KEY_ESP32][KEY_COMPONENTS] = {}
     elif conf[CONF_TYPE] == FRAMEWORK_ARDUINO:
         CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino"
     CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse(
@@ -104,6 +119,21 @@ def add_idf_sdkconfig_option(name: str, value: SdkconfigValueType):
     CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS][name] = value
 
 
+def add_idf_component(
+    name: str, repo: str, ref: str = None, path: str = None, refresh: TimePeriod = None
+):
+    """Add an esp-idf component to the project."""
+    if not CORE.using_esp_idf:
+        raise ValueError("Not an esp-idf project")
+    if name not in CORE.data[KEY_ESP32][KEY_COMPONENTS]:
+        CORE.data[KEY_ESP32][KEY_COMPONENTS][name] = {
+            KEY_REPO: repo,
+            KEY_REF: ref,
+            KEY_PATH: path,
+            KEY_REFRESH: refresh,
+        }
+
+
 def _format_framework_arduino_version(ver: cv.Version) -> str:
     # format the given arduino (https://github.com/espressif/arduino-esp32/releases) version to
     # a PIO platformio/framework-arduinoespressif32 value
@@ -270,6 +300,18 @@ ESP_IDF_FRAMEWORK_SCHEMA = cv.All(
                     cv.Optional(CONF_IGNORE_EFUSE_MAC_CRC, default=False): cv.boolean,
                 }
             ),
+            cv.Optional(CONF_COMPONENTS, default=[]): cv.ensure_list(
+                cv.Schema(
+                    {
+                        cv.Required(CONF_NAME): cv.string_strict,
+                        cv.Required(CONF_SOURCE): cv.SOURCE_SCHEMA,
+                        cv.Optional(CONF_PATH): cv.string,
+                        cv.Optional(CONF_REFRESH, default="1d"): cv.All(
+                            cv.string, cv.source_refresh
+                        ),
+                    }
+                )
+            ),
         }
     ),
     _esp_idf_check_versions,
@@ -372,6 +414,19 @@ async def to_code(config):
             ),
         )
 
+        for component in conf[CONF_COMPONENTS]:
+            source = component[CONF_SOURCE]
+            if source[CONF_TYPE] == TYPE_GIT:
+                add_idf_component(
+                    name=component[CONF_NAME],
+                    repo=source[CONF_URL],
+                    ref=source.get(CONF_REF),
+                    path=component.get(CONF_PATH),
+                    refresh=component[CONF_REFRESH],
+                )
+            elif source[CONF_TYPE] == TYPE_LOCAL:
+                _LOGGER.warning("Local components are not implemented yet.")
+
     elif conf[CONF_TYPE] == FRAMEWORK_ARDUINO:
         cg.add_platformio_option("framework", "arduino")
         cg.add_build_flag("-DUSE_ARDUINO")
@@ -468,6 +523,33 @@ def copy_files():
             __version__,
         )
 
+        if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
+            import shutil
+
+            shutil.rmtree(CORE.relative_build_path("components"), ignore_errors=True)
+
+            components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]
+
+            for name, component in components.items():
+
+                repo_dir, _ = git.clone_or_update(
+                    url=component[KEY_REPO],
+                    ref=component[KEY_REF],
+                    refresh=component[KEY_REFRESH],
+                    domain="idf_components",
+                )
+                mkdir_p(CORE.relative_build_path("components"))
+                component_dir = repo_dir
+                if component[KEY_PATH] is not None:
+                    component_dir = component_dir / component[KEY_PATH]
+
+                shutil.copytree(
+                    component_dir,
+                    CORE.relative_build_path(f"components/{name}"),
+                    dirs_exist_ok=True,
+                    ignore=shutil.ignore_patterns(".git", ".github"),
+                )
+
     dir = os.path.dirname(__file__)
     post_build_file = os.path.join(dir, "post_build.py.script")
     copy_file_if_changed(

esphome/components/esp32/const.py

@@ -4,6 +4,11 @@ KEY_ESP32 = "esp32"
 KEY_BOARD = "board"
 KEY_VARIANT = "variant"
 KEY_SDKCONFIG_OPTIONS = "sdkconfig_options"
+KEY_COMPONENTS = "components"
+KEY_REPO = "repo"
+KEY_REF = "ref"
+KEY_REFRESH = "refresh"
+KEY_PATH = "path"
 
 VARIANT_ESP32 = "ESP32"
 VARIANT_ESP32S2 = "ESP32S2"

esphome/components/external_components/__init__.py

@@ -1,90 +1,32 @@
-import re
 import logging
 from pathlib import Path
 
 import esphome.config_validation as cv
+from esphome import git, loader
 from esphome.const import (
     CONF_COMPONENTS,
+    CONF_EXTERNAL_COMPONENTS,
+    CONF_PASSWORD,
+    CONF_PATH,
     CONF_REF,
     CONF_REFRESH,
     CONF_SOURCE,
-    CONF_URL,
     CONF_TYPE,
-    CONF_EXTERNAL_COMPONENTS,
+    CONF_URL,
-    CONF_PATH,
     CONF_USERNAME,
-    CONF_PASSWORD,
+    TYPE_GIT,
+    TYPE_LOCAL,
 )
 from esphome.core import CORE
-from esphome import git, loader
 
 _LOGGER = logging.getLogger(__name__)
 
 DOMAIN = CONF_EXTERNAL_COMPONENTS
 
-TYPE_GIT = "git"
-TYPE_LOCAL = "local"
-
-
-GIT_SCHEMA = {
-    cv.Required(CONF_URL): cv.url,
-    cv.Optional(CONF_REF): cv.git_ref,
-    cv.Optional(CONF_USERNAME): cv.string,
-    cv.Optional(CONF_PASSWORD): cv.string,
-}
-LOCAL_SCHEMA = {
-    cv.Required(CONF_PATH): cv.directory,
-}
-
-
-def validate_source_shorthand(value):
-    if not isinstance(value, str):
-        raise cv.Invalid("Shorthand only for strings")
-    try:
-        return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value})
-    except cv.Invalid:
-        pass
-    # Regex for GitHub repo name with optional branch/tag
-    # Note: git allows other branch/tag names as well, but never seen them used before
-    m = re.match(
-        r"github://(?:([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?|pr#([0-9]+))",
-        value,
-    )
-    if m is None:
-        raise cv.Invalid(
-            "Source is not a file system path, in expected github://username/name[@branch-or-tag] or github://pr#1234 format!"
-        )
-    if m.group(4):
-        conf = {
-            CONF_TYPE: TYPE_GIT,
-            CONF_URL: "https://github.com/esphome/esphome.git",
-            CONF_REF: f"pull/{m.group(4)}/head",
-        }
-    else:
-        conf = {
-            CONF_TYPE: TYPE_GIT,
-            CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
-        }
-        if m.group(3):
-            conf[CONF_REF] = m.group(3)
-
-    return SOURCE_SCHEMA(conf)
-
-
-SOURCE_SCHEMA = cv.Any(
-    validate_source_shorthand,
-    cv.typed_schema(
-        {
-            TYPE_GIT: cv.Schema(GIT_SCHEMA),
-            TYPE_LOCAL: cv.Schema(LOCAL_SCHEMA),
-        }
-    ),
-)
-
 
 CONFIG_SCHEMA = cv.ensure_list(
     {
-        cv.Required(CONF_SOURCE): SOURCE_SCHEMA,
+        cv.Required(CONF_SOURCE): cv.SOURCE_SCHEMA,
         cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, cv.source_refresh),
         cv.Optional(CONF_COMPONENTS, default="all"): cv.Any(
             "all", cv.ensure_list(cv.string)

esphome/config_validation.py

@@ -39,6 +39,11 @@ from esphome.const import (
     CONF_UPDATE_INTERVAL,
     CONF_TYPE_ID,
     CONF_TYPE,
+    CONF_REF,
+    CONF_URL,
+    CONF_PATH,
+    CONF_USERNAME,
+    CONF_PASSWORD,
     ENTITY_CATEGORY_CONFIG,
     ENTITY_CATEGORY_DIAGNOSTIC,
     ENTITY_CATEGORY_NONE,
@@ -46,6 +51,8 @@ from esphome.const import (
     KEY_FRAMEWORK_VERSION,
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
+    TYPE_GIT,
+    TYPE_LOCAL,
 )
 from esphome.core import (
     CORE,
@@ -1820,3 +1827,59 @@ def suppress_invalid():
         yield
     except vol.Invalid:
         pass
+
+
+GIT_SCHEMA = {
+    Required(CONF_URL): url,
+    Optional(CONF_REF): git_ref,
+    Optional(CONF_USERNAME): string,
+    Optional(CONF_PASSWORD): string,
+}
+LOCAL_SCHEMA = {
+    Required(CONF_PATH): directory,
+}
+
+
+def validate_source_shorthand(value):
+    if not isinstance(value, str):
+        raise Invalid("Shorthand only for strings")
+    try:
+        return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value})
+    except Invalid:
+        pass
+    # Regex for GitHub repo name with optional branch/tag
+    # Note: git allows other branch/tag names as well, but never seen them used before
+    m = re.match(
+        r"github://(?:([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?|pr#([0-9]+))",
+        value,
+    )
+    if m is None:
+        raise Invalid(
+            "Source is not a file system path, in expected github://username/name[@branch-or-tag] or github://pr#1234 format!"
+        )
+    if m.group(4):
+        conf = {
+            CONF_TYPE: TYPE_GIT,
+            CONF_URL: "https://github.com/esphome/esphome.git",
+            CONF_REF: f"pull/{m.group(4)}/head",
+        }
+    else:
+        conf = {
+            CONF_TYPE: TYPE_GIT,
+            CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
+        }
+        if m.group(3):
+            conf[CONF_REF] = m.group(3)
+
+    return SOURCE_SCHEMA(conf)
+
+
+SOURCE_SCHEMA = Any(
+    validate_source_shorthand,
+    typed_schema(
+        {
+            TYPE_GIT: Schema(GIT_SCHEMA),
+            TYPE_LOCAL: Schema(LOCAL_SCHEMA),
+        }
+    ),
+)

esphome/const.py

@@ -797,6 +797,9 @@ CONF_X_GRID = "x_grid"
 CONF_Y_GRID = "y_grid"
 CONF_ZERO = "zero"
 
+TYPE_GIT = "git"
+TYPE_LOCAL = "local"
+
 ENV_NOGITIGNORE = "ESPHOME_NOGITIGNORE"
 ENV_QUICKWIZARD = "ESPHOME_QUICKWIZARD"