Dashboard node import and render in browser (#2374)

CODEOWNERS

@@ -40,6 +40,7 @@ esphome/components/cover/* @esphome/core
 esphome/components/cs5460a/* @balrog-kun
 esphome/components/ct_clamp/* @jesserockz
 esphome/components/daly_bms/* @s1lvi0
+esphome/components/dashboard_import/* @esphome/core
 esphome/components/debug/* @OttoWinter
 esphome/components/dfplayer/* @glmnet
 esphome/components/dht/* @OttoWinter

esphome/components/dashboard_import/__init__.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.components.packages import validate_source_shorthand
+from esphome.yaml_util import dump
+
+
+dashboard_import_ns = cg.esphome_ns.namespace("dashboard_import")
+
+# payload is in `esphomelib` mdns record, which only exists if api
+# is enabled
+DEPENDENCIES = ["api"]
+CODEOWNERS = ["@esphome/core"]
+
+
+def validate_import_url(value):
+    value = cv.string_strict(value)
+    value = cv.Length(max=255)(value)
+    # ignore result, only check if it's a valid shorthand
+    validate_source_shorthand(value)
+    return value
+
+
+CONF_PACKAGE_IMPORT_URL = "package_import_url"
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.Required(CONF_PACKAGE_IMPORT_URL): validate_import_url,
+    }
+)
+
+
+async def to_code(config):
+    cg.add_define("USE_DASHBOARD_IMPORT")
+    cg.add(dashboard_import_ns.set_package_import_url(config[CONF_PACKAGE_IMPORT_URL]))
+
+
+def import_config(path: str, name: str, project_name: str, import_url: str) -> None:
+    p = Path(path)
+
+    if p.exists():
+        raise FileExistsError
+
+    config = {"substitutions": {"name": name}, "packages": {project_name: import_url}}
+    p.write_text(dump(config), encoding="utf8")

esphome/components/dashboard_import/dashboard_import.cpp

@@ -0,0 +1,12 @@
+#include "dashboard_import.h"
+
+namespace esphome {
+namespace dashboard_import {
+
+static std::string g_package_import_url;  // NOLINT
+
+std::string get_package_import_url() { return g_package_import_url; }
+void set_package_import_url(std::string url) { g_package_import_url = std::move(url); }
+
+}  // namespace dashboard_import
+}  // namespace esphome

esphome/components/dashboard_import/dashboard_import.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+
+namespace esphome {
+namespace dashboard_import {
+
+std::string get_package_import_url();
+void set_package_import_url(std::string url);
+
+}  // namespace dashboard_import
+}  // namespace esphome

esphome/components/mdns/mdns_component.cpp

@@ -6,6 +6,9 @@
 #ifdef USE_API
 #include "esphome/components/api/api_server.h"
 #endif
+#ifdef USE_DASHBOARD_IMPORT
+#include "esphome/components/dashboard_import/dashboard_import.h"
+#endif
 
 namespace esphome {
 namespace mdns {
@@ -42,6 +45,11 @@ std::vector<MDNSService> MDNSComponent::compile_services_() {
     service.txt_records.push_back({"project_name", ESPHOME_PROJECT_NAME});
     service.txt_records.push_back({"project_version", ESPHOME_PROJECT_VERSION});
 #endif  // ESPHOME_PROJECT_NAME
+
+#ifdef USE_DASHBOARD_IMPORT
+    service.txt_records.push_back({"package_import_url", dashboard_import::get_package_import_url()});
+#endif
+
     res.push_back(service);
   }
 #endif  // USE_API

esphome/core/defines.h

@@ -67,3 +67,5 @@
 
 // Disabled feature flags
 //#define USE_BSEC  // Requires a library with proprietary license.
+
+#define USE_DASHBOARD_IMPORT

esphome/dashboard/dashboard.py

@@ -41,7 +41,7 @@ from .util import password_hash
 # pylint: disable=unused-import, wrong-import-order
 from typing import Optional  # noqa
 
-from esphome.zeroconf import DashboardStatus, EsphomeZeroconf
+from esphome.zeroconf import DashboardImportDiscovery, DashboardStatus, EsphomeZeroconf
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -154,9 +154,6 @@ def is_authenticated(request_handler):
 def bind_config(func):
     def decorator(self, *args, **kwargs):
         configuration = self.get_argument("configuration")
-        if not is_allowed(configuration):
-            self.set_status(500)
-            return None
         kwargs = kwargs.copy()
         kwargs["configuration"] = configuration
         return func(self, *args, **kwargs)
@@ -363,8 +360,8 @@ class WizardRequestHandler(BaseHandler):
         from esphome import wizard
 
         kwargs = {
-            k: "".join(x.decode() for x in v)
+            k: v
-            for k, v in self.request.arguments.items()
+            for k, v in json.loads(self.request.body.decode()).items()
             if k in ("name", "platform", "board", "ssid", "psk", "password")
         }
         kwargs["ota_password"] = secrets.token_hex(16)
@@ -374,6 +371,29 @@ class WizardRequestHandler(BaseHandler):
         self.finish()
 
 
+class ImportRequestHandler(BaseHandler):
+    @authenticated
+    def post(self):
+        from esphome.components.dashboard_import import import_config
+
+        args = json.loads(self.request.body.decode())
+        try:
+            name = args["name"]
+            import_config(
+                settings.rel_path(f"{name}.yaml"),
+                name,
+                args["project_name"],
+                args["package_import_url"],
+            )
+        except FileExistsError:
+            self.set_status(500)
+            self.write("File already exists")
+            return
+
+        self.set_status(200)
+        self.finish()
+
+
 class DownloadBinaryRequestHandler(BaseHandler):
     @authenticated
     @bind_config
@@ -469,15 +489,51 @@ class DashboardEntry:
         return self.storage.loaded_integrations
 
 
+class ListDevicesHandler(BaseHandler):
+    @authenticated
+    def get(self):
+        entries = _list_dashboard_entries()
+        self.set_header("content-type", "application/json")
+        configured = {entry.name for entry in entries}
+        self.write(
+            json.dumps(
+                {
+                    "configured": [
+                        {
+                            "name": entry.name,
+                            "configuration": entry.filename,
+                            "loaded_integrations": entry.loaded_integrations,
+                            "deployed_version": entry.update_old,
+                            "current_version": entry.update_new,
+                            "path": entry.path,
+                            "comment": entry.comment,
+                            "address": entry.address,
+                            "target_platform": entry.target_platform,
+                        }
+                        for entry in entries
+                    ],
+                    "importable": [
+                        {
+                            "name": res.device_name,
+                            "package_import_url": res.package_import_url,
+                            "project_name": res.project_name,
+                            "project_version": res.project_version,
+                        }
+                        for res in IMPORT_RESULT.values()
+                        if res.device_name not in configured
+                    ],
+                }
+            )
+        )
+
+
 class MainRequestHandler(BaseHandler):
     @authenticated
     def get(self):
         begin = bool(self.get_argument("begin", False))
-        entries = _list_dashboard_entries()
 
         self.render(
             get_template_path("index"),
-            entries=entries,
             begin=begin,
             **template_args(),
             login_enabled=settings.using_auth,
@@ -495,6 +551,8 @@ def _ping_func(filename, address):
 
 class MDNSStatusThread(threading.Thread):
     def run(self):
+        global IMPORT_RESULT
+
         zc = EsphomeZeroconf()
 
         def on_update(dat):
@@ -502,17 +560,22 @@ class MDNSStatusThread(threading.Thread):
                 PING_RESULT[key] = b
 
         stat = DashboardStatus(zc, on_update)
+        imports = DashboardImportDiscovery(zc)
+
         stat.start()
         while not STOP_EVENT.is_set():
             entries = _list_dashboard_entries()
             stat.request_query(
                 {entry.filename: f"{entry.name}.local." for entry in entries}
             )
+            IMPORT_RESULT = imports.import_state
 
             PING_REQUEST.wait()
             PING_REQUEST.clear()
+
         stat.stop()
         stat.join()
+        imports.cancel()
         zc.close()
 
 
@@ -567,10 +630,6 @@ class PingRequestHandler(BaseHandler):
         self.write(json.dumps(PING_RESULT))
 
 
-def is_allowed(configuration):
-    return os.path.sep not in configuration
-
-
 class InfoRequestHandler(BaseHandler):
     @authenticated
     @bind_config
@@ -613,20 +672,18 @@ class DeleteRequestHandler(BaseHandler):
     def post(self, configuration=None):
         config_file = settings.rel_path(configuration)
         storage_path = ext_storage_path(settings.config_dir, configuration)
-        storage_json = StorageJSON.load(storage_path)
-        if storage_json is None:
-            self.set_status(500)
-            return
 
-        name = storage_json.name
         trash_path = trash_storage_path(settings.config_dir)
         mkdir_p(trash_path)
         shutil.move(config_file, os.path.join(trash_path, configuration))
 
-        # Delete build folder (if exists)
+        storage_json = StorageJSON.load(storage_path)
-        build_folder = os.path.join(settings.config_dir, name)
+        if storage_json is not None:
-        if build_folder is not None:
+            # Delete build folder (if exists)
-            shutil.rmtree(build_folder, os.path.join(trash_path, name))
+            name = storage_json.name
+            build_folder = os.path.join(settings.config_dir, name)
+            if build_folder is not None:
+                shutil.rmtree(build_folder, os.path.join(trash_path, name))
 
 
 class UndoDeleteRequestHandler(BaseHandler):
@@ -639,6 +696,7 @@ class UndoDeleteRequestHandler(BaseHandler):
 
 
 PING_RESULT = {}  # type: dict
+IMPORT_RESULT = {}
 STOP_EVENT = threading.Event()
 PING_REQUEST = threading.Event()
 
@@ -808,8 +866,10 @@ def make_app(debug=get_bool_env(ENV_DEV)):
             (f"{rel}ping", PingRequestHandler),
             (f"{rel}delete", DeleteRequestHandler),
             (f"{rel}undo-delete", UndoDeleteRequestHandler),
-            (f"{rel}wizard.html", WizardRequestHandler),
+            (f"{rel}wizard", WizardRequestHandler),
             (f"{rel}static/(.*)", StaticFileHandler, {"path": get_static_path()}),
+            (f"{rel}devices", ListDevicesHandler),
+            (f"{rel}import", ImportRequestHandler),
         ],
         **app_settings,
     )

esphome/zeroconf.py

@@ -2,6 +2,8 @@ import socket
 import threading
 import time
 from typing import Dict, Optional
+import logging
+from dataclasses import dataclass
 
 from zeroconf import (
     DNSAddress,
@@ -10,11 +12,14 @@ from zeroconf import (
     DNSQuestion,
     RecordUpdateListener,
     Zeroconf,
+    ServiceBrowser,
 )
+from zeroconf._services import ServiceStateChange
 
 _CLASS_IN = 1
 _FLAGS_QR_QUERY = 0x0000  # query
 _TYPE_A = 1
+_LOGGER = logging.getLogger(__name__)
 
 
 class HostResolver(RecordUpdateListener):
@@ -57,7 +62,7 @@ class HostResolver(RecordUpdateListener):
         return True
 
 
-class DashboardStatus(RecordUpdateListener, threading.Thread):
+class DashboardStatus(threading.Thread):
     PING_AFTER = 15 * 1000  # Send new mDNS request after 15 seconds
     OFFLINE_AFTER = PING_AFTER * 2  # Offline if no mDNS response after 30 seconds
 
@@ -70,9 +75,6 @@ class DashboardStatus(RecordUpdateListener, threading.Thread):
         self.query_event = threading.Event()
         self.on_update = on_update
 
-    def update_record(self, zc: Zeroconf, now: float, record: DNSRecord) -> None:
-        pass
-
     def request_query(self, hosts: Dict[str, str]) -> None:
         self.query_hosts = set(hosts.values())
         self.key_to_host = hosts
@@ -93,7 +95,6 @@ class DashboardStatus(RecordUpdateListener, threading.Thread):
         )
 
     def run(self) -> None:
-        self.zc.add_listener(self, None)
         while not self.stop_event.is_set():
             self.on_update(
                 {key: self.host_status(host) for key, host in self.key_to_host.items()}
@@ -110,7 +111,75 @@ class DashboardStatus(RecordUpdateListener, threading.Thread):
                     self.zc.send(out)
             self.query_event.wait()
             self.query_event.clear()
-        self.zc.remove_listener(self)
+
+
+ESPHOME_SERVICE_TYPE = "_esphomelib._tcp.local."
+TXT_RECORD_PACKAGE_IMPORT_URL = b"package_import_url"
+TXT_RECORD_PROJECT_NAME = b"project_name"
+TXT_RECORD_PROJECT_VERSION = b"project_version"
+
+
+@dataclass
+class DiscoveredImport:
+    device_name: str
+    package_import_url: str
+    project_name: str
+    project_version: str
+
+
+class DashboardImportDiscovery:
+    def __init__(self, zc: Zeroconf) -> None:
+        self.zc = zc
+        self.service_browser = ServiceBrowser(
+            self.zc, ESPHOME_SERVICE_TYPE, [self._on_update]
+        )
+        self.import_state = {}
+
+    def _on_update(
+        self,
+        zeroconf: Zeroconf,
+        service_type: str,
+        name: str,
+        state_change: ServiceStateChange,
+    ) -> None:
+        _LOGGER.debug(
+            "service_update: type=%s name=%s state_change=%s",
+            service_type,
+            name,
+            state_change,
+        )
+        if service_type != ESPHOME_SERVICE_TYPE:
+            return
+        if state_change == ServiceStateChange.Removed:
+            self.import_state.pop(name, None)
+
+        info = zeroconf.get_service_info(service_type, name)
+        _LOGGER.debug("-> resolved info: %s", info)
+        if info is None:
+            return
+        node_name = name[: -len(ESPHOME_SERVICE_TYPE) - 1]
+        required_keys = [
+            TXT_RECORD_PACKAGE_IMPORT_URL,
+            TXT_RECORD_PROJECT_NAME,
+            TXT_RECORD_PROJECT_VERSION,
+        ]
+        if any(key not in info.properties for key in required_keys):
+            # Not a dashboard import device
+            return
+
+        import_url = info.properties[TXT_RECORD_PACKAGE_IMPORT_URL].decode()
+        project_name = info.properties[TXT_RECORD_PROJECT_NAME].decode()
+        project_version = info.properties[TXT_RECORD_PROJECT_VERSION].decode()
+
+        self.import_state[name] = DiscoveredImport(
+            device_name=node_name,
+            package_import_url=import_url,
+            project_name=project_name,
+            project_version=project_version,
+        )
+
+    def cancel(self) -> None:
+        self.service_browser.cancel()
 
 
 class EsphomeZeroconf(Zeroconf):