From 45940b051439cb0053889597643d3a5a62ab73ad Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Mon, 27 Sep 2021 19:10:53 +0200 Subject: [PATCH] Dashboard node import and render in browser (#2374) --- CODEOWNERS | 1 + .../components/dashboard_import/__init__.py | 45 ++++++++ .../dashboard_import/dashboard_import.cpp | 12 ++ .../dashboard_import/dashboard_import.h | 12 ++ esphome/components/mdns/mdns_component.cpp | 8 ++ esphome/core/defines.h | 2 + esphome/dashboard/dashboard.py | 104 ++++++++++++++---- esphome/zeroconf.py | 81 +++++++++++++- 8 files changed, 237 insertions(+), 28 deletions(-) create mode 100644 esphome/components/dashboard_import/__init__.py create mode 100644 esphome/components/dashboard_import/dashboard_import.cpp create mode 100644 esphome/components/dashboard_import/dashboard_import.h diff --git a/CODEOWNERS b/CODEOWNERS index 17825a9205..0b974b95e6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -40,6 +40,7 @@ esphome/components/cover/* @esphome/core esphome/components/cs5460a/* @balrog-kun esphome/components/ct_clamp/* @jesserockz esphome/components/daly_bms/* @s1lvi0 +esphome/components/dashboard_import/* @esphome/core esphome/components/debug/* @OttoWinter esphome/components/dfplayer/* @glmnet esphome/components/dht/* @OttoWinter diff --git a/esphome/components/dashboard_import/__init__.py b/esphome/components/dashboard_import/__init__.py new file mode 100644 index 0000000000..2b884d3b9a --- /dev/null +++ b/esphome/components/dashboard_import/__init__.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.components.packages import validate_source_shorthand +from esphome.yaml_util import dump + + +dashboard_import_ns = cg.esphome_ns.namespace("dashboard_import") + +# payload is in `esphomelib` mdns record, which only exists if api +# is enabled +DEPENDENCIES = ["api"] +CODEOWNERS = ["@esphome/core"] + + +def validate_import_url(value): + value = cv.string_strict(value) + value = cv.Length(max=255)(value) + # ignore result, only check if it's a valid shorthand + validate_source_shorthand(value) + return value + + +CONF_PACKAGE_IMPORT_URL = "package_import_url" +CONFIG_SCHEMA = cv.Schema( + { + cv.Required(CONF_PACKAGE_IMPORT_URL): validate_import_url, + } +) + + +async def to_code(config): + cg.add_define("USE_DASHBOARD_IMPORT") + cg.add(dashboard_import_ns.set_package_import_url(config[CONF_PACKAGE_IMPORT_URL])) + + +def import_config(path: str, name: str, project_name: str, import_url: str) -> None: + p = Path(path) + + if p.exists(): + raise FileExistsError + + config = {"substitutions": {"name": name}, "packages": {project_name: import_url}} + p.write_text(dump(config), encoding="utf8") diff --git a/esphome/components/dashboard_import/dashboard_import.cpp b/esphome/components/dashboard_import/dashboard_import.cpp new file mode 100644 index 0000000000..6875fd61a5 --- /dev/null +++ b/esphome/components/dashboard_import/dashboard_import.cpp @@ -0,0 +1,12 @@ +#include "dashboard_import.h" + +namespace esphome { +namespace dashboard_import { + +static std::string g_package_import_url; // NOLINT + +std::string get_package_import_url() { return g_package_import_url; } +void set_package_import_url(std::string url) { g_package_import_url = std::move(url); } + +} // namespace dashboard_import +} // namespace esphome diff --git a/esphome/components/dashboard_import/dashboard_import.h b/esphome/components/dashboard_import/dashboard_import.h new file mode 100644 index 0000000000..0ca2994aab --- /dev/null +++ b/esphome/components/dashboard_import/dashboard_import.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace esphome { +namespace dashboard_import { + +std::string get_package_import_url(); +void set_package_import_url(std::string url); + +} // namespace dashboard_import +} // namespace esphome diff --git a/esphome/components/mdns/mdns_component.cpp b/esphome/components/mdns/mdns_component.cpp index c742fe3948..372d980eb0 100644 --- a/esphome/components/mdns/mdns_component.cpp +++ b/esphome/components/mdns/mdns_component.cpp @@ -6,6 +6,9 @@ #ifdef USE_API #include "esphome/components/api/api_server.h" #endif +#ifdef USE_DASHBOARD_IMPORT +#include "esphome/components/dashboard_import/dashboard_import.h" +#endif namespace esphome { namespace mdns { @@ -42,6 +45,11 @@ std::vector MDNSComponent::compile_services_() { service.txt_records.push_back({"project_name", ESPHOME_PROJECT_NAME}); service.txt_records.push_back({"project_version", ESPHOME_PROJECT_VERSION}); #endif // ESPHOME_PROJECT_NAME + +#ifdef USE_DASHBOARD_IMPORT + service.txt_records.push_back({"package_import_url", dashboard_import::get_package_import_url()}); +#endif + res.push_back(service); } #endif // USE_API diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 62468dfbfe..1c3b17d071 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -67,3 +67,5 @@ // Disabled feature flags //#define USE_BSEC // Requires a library with proprietary license. + +#define USE_DASHBOARD_IMPORT diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py index bfe6808729..492b86384d 100644 --- a/esphome/dashboard/dashboard.py +++ b/esphome/dashboard/dashboard.py @@ -41,7 +41,7 @@ from .util import password_hash # pylint: disable=unused-import, wrong-import-order from typing import Optional # noqa -from esphome.zeroconf import DashboardStatus, EsphomeZeroconf +from esphome.zeroconf import DashboardImportDiscovery, DashboardStatus, EsphomeZeroconf _LOGGER = logging.getLogger(__name__) @@ -154,9 +154,6 @@ def is_authenticated(request_handler): def bind_config(func): def decorator(self, *args, **kwargs): configuration = self.get_argument("configuration") - if not is_allowed(configuration): - self.set_status(500) - return None kwargs = kwargs.copy() kwargs["configuration"] = configuration return func(self, *args, **kwargs) @@ -363,8 +360,8 @@ class WizardRequestHandler(BaseHandler): from esphome import wizard kwargs = { - k: "".join(x.decode() for x in v) - for k, v in self.request.arguments.items() + k: v + for k, v in json.loads(self.request.body.decode()).items() if k in ("name", "platform", "board", "ssid", "psk", "password") } kwargs["ota_password"] = secrets.token_hex(16) @@ -374,6 +371,29 @@ class WizardRequestHandler(BaseHandler): self.finish() +class ImportRequestHandler(BaseHandler): + @authenticated + def post(self): + from esphome.components.dashboard_import import import_config + + args = json.loads(self.request.body.decode()) + try: + name = args["name"] + import_config( + settings.rel_path(f"{name}.yaml"), + name, + args["project_name"], + args["package_import_url"], + ) + except FileExistsError: + self.set_status(500) + self.write("File already exists") + return + + self.set_status(200) + self.finish() + + class DownloadBinaryRequestHandler(BaseHandler): @authenticated @bind_config @@ -469,15 +489,51 @@ class DashboardEntry: return self.storage.loaded_integrations +class ListDevicesHandler(BaseHandler): + @authenticated + def get(self): + entries = _list_dashboard_entries() + self.set_header("content-type", "application/json") + configured = {entry.name for entry in entries} + self.write( + json.dumps( + { + "configured": [ + { + "name": entry.name, + "configuration": entry.filename, + "loaded_integrations": entry.loaded_integrations, + "deployed_version": entry.update_old, + "current_version": entry.update_new, + "path": entry.path, + "comment": entry.comment, + "address": entry.address, + "target_platform": entry.target_platform, + } + for entry in entries + ], + "importable": [ + { + "name": res.device_name, + "package_import_url": res.package_import_url, + "project_name": res.project_name, + "project_version": res.project_version, + } + for res in IMPORT_RESULT.values() + if res.device_name not in configured + ], + } + ) + ) + + class MainRequestHandler(BaseHandler): @authenticated def get(self): begin = bool(self.get_argument("begin", False)) - entries = _list_dashboard_entries() self.render( get_template_path("index"), - entries=entries, begin=begin, **template_args(), login_enabled=settings.using_auth, @@ -495,6 +551,8 @@ def _ping_func(filename, address): class MDNSStatusThread(threading.Thread): def run(self): + global IMPORT_RESULT + zc = EsphomeZeroconf() def on_update(dat): @@ -502,17 +560,22 @@ class MDNSStatusThread(threading.Thread): PING_RESULT[key] = b stat = DashboardStatus(zc, on_update) + imports = DashboardImportDiscovery(zc) + stat.start() while not STOP_EVENT.is_set(): entries = _list_dashboard_entries() stat.request_query( {entry.filename: f"{entry.name}.local." for entry in entries} ) + IMPORT_RESULT = imports.import_state PING_REQUEST.wait() PING_REQUEST.clear() + stat.stop() stat.join() + imports.cancel() zc.close() @@ -567,10 +630,6 @@ class PingRequestHandler(BaseHandler): self.write(json.dumps(PING_RESULT)) -def is_allowed(configuration): - return os.path.sep not in configuration - - class InfoRequestHandler(BaseHandler): @authenticated @bind_config @@ -613,20 +672,18 @@ class DeleteRequestHandler(BaseHandler): def post(self, configuration=None): config_file = settings.rel_path(configuration) storage_path = ext_storage_path(settings.config_dir, configuration) - storage_json = StorageJSON.load(storage_path) - if storage_json is None: - self.set_status(500) - return - name = storage_json.name trash_path = trash_storage_path(settings.config_dir) mkdir_p(trash_path) shutil.move(config_file, os.path.join(trash_path, configuration)) - # Delete build folder (if exists) - build_folder = os.path.join(settings.config_dir, name) - if build_folder is not None: - shutil.rmtree(build_folder, os.path.join(trash_path, name)) + storage_json = StorageJSON.load(storage_path) + if storage_json is not None: + # Delete build folder (if exists) + name = storage_json.name + build_folder = os.path.join(settings.config_dir, name) + if build_folder is not None: + shutil.rmtree(build_folder, os.path.join(trash_path, name)) class UndoDeleteRequestHandler(BaseHandler): @@ -639,6 +696,7 @@ class UndoDeleteRequestHandler(BaseHandler): PING_RESULT = {} # type: dict +IMPORT_RESULT = {} STOP_EVENT = threading.Event() PING_REQUEST = threading.Event() @@ -808,8 +866,10 @@ def make_app(debug=get_bool_env(ENV_DEV)): (f"{rel}ping", PingRequestHandler), (f"{rel}delete", DeleteRequestHandler), (f"{rel}undo-delete", UndoDeleteRequestHandler), - (f"{rel}wizard.html", WizardRequestHandler), + (f"{rel}wizard", WizardRequestHandler), (f"{rel}static/(.*)", StaticFileHandler, {"path": get_static_path()}), + (f"{rel}devices", ListDevicesHandler), + (f"{rel}import", ImportRequestHandler), ], **app_settings, ) diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py index e6853531f2..a19fc143ec 100644 --- a/esphome/zeroconf.py +++ b/esphome/zeroconf.py @@ -2,6 +2,8 @@ import socket import threading import time from typing import Dict, Optional +import logging +from dataclasses import dataclass from zeroconf import ( DNSAddress, @@ -10,11 +12,14 @@ from zeroconf import ( DNSQuestion, RecordUpdateListener, Zeroconf, + ServiceBrowser, ) +from zeroconf._services import ServiceStateChange _CLASS_IN = 1 _FLAGS_QR_QUERY = 0x0000 # query _TYPE_A = 1 +_LOGGER = logging.getLogger(__name__) class HostResolver(RecordUpdateListener): @@ -57,7 +62,7 @@ class HostResolver(RecordUpdateListener): return True -class DashboardStatus(RecordUpdateListener, threading.Thread): +class DashboardStatus(threading.Thread): PING_AFTER = 15 * 1000 # Send new mDNS request after 15 seconds OFFLINE_AFTER = PING_AFTER * 2 # Offline if no mDNS response after 30 seconds @@ -70,9 +75,6 @@ class DashboardStatus(RecordUpdateListener, threading.Thread): self.query_event = threading.Event() self.on_update = on_update - def update_record(self, zc: Zeroconf, now: float, record: DNSRecord) -> None: - pass - def request_query(self, hosts: Dict[str, str]) -> None: self.query_hosts = set(hosts.values()) self.key_to_host = hosts @@ -93,7 +95,6 @@ class DashboardStatus(RecordUpdateListener, threading.Thread): ) def run(self) -> None: - self.zc.add_listener(self, None) while not self.stop_event.is_set(): self.on_update( {key: self.host_status(host) for key, host in self.key_to_host.items()} @@ -110,7 +111,75 @@ class DashboardStatus(RecordUpdateListener, threading.Thread): self.zc.send(out) self.query_event.wait() self.query_event.clear() - self.zc.remove_listener(self) + + +ESPHOME_SERVICE_TYPE = "_esphomelib._tcp.local." +TXT_RECORD_PACKAGE_IMPORT_URL = b"package_import_url" +TXT_RECORD_PROJECT_NAME = b"project_name" +TXT_RECORD_PROJECT_VERSION = b"project_version" + + +@dataclass +class DiscoveredImport: + device_name: str + package_import_url: str + project_name: str + project_version: str + + +class DashboardImportDiscovery: + def __init__(self, zc: Zeroconf) -> None: + self.zc = zc + self.service_browser = ServiceBrowser( + self.zc, ESPHOME_SERVICE_TYPE, [self._on_update] + ) + self.import_state = {} + + def _on_update( + self, + zeroconf: Zeroconf, + service_type: str, + name: str, + state_change: ServiceStateChange, + ) -> None: + _LOGGER.debug( + "service_update: type=%s name=%s state_change=%s", + service_type, + name, + state_change, + ) + if service_type != ESPHOME_SERVICE_TYPE: + return + if state_change == ServiceStateChange.Removed: + self.import_state.pop(name, None) + + info = zeroconf.get_service_info(service_type, name) + _LOGGER.debug("-> resolved info: %s", info) + if info is None: + return + node_name = name[: -len(ESPHOME_SERVICE_TYPE) - 1] + required_keys = [ + TXT_RECORD_PACKAGE_IMPORT_URL, + TXT_RECORD_PROJECT_NAME, + TXT_RECORD_PROJECT_VERSION, + ] + if any(key not in info.properties for key in required_keys): + # Not a dashboard import device + return + + import_url = info.properties[TXT_RECORD_PACKAGE_IMPORT_URL].decode() + project_name = info.properties[TXT_RECORD_PROJECT_NAME].decode() + project_version = info.properties[TXT_RECORD_PROJECT_VERSION].decode() + + self.import_state[name] = DiscoveredImport( + device_name=node_name, + package_import_url=import_url, + project_name=project_name, + project_version=project_version, + ) + + def cancel(self) -> None: + self.service_browser.cancel() class EsphomeZeroconf(Zeroconf):