From 74e9196dad2b451af4563d5e4c7fb840eb7ed48e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Nov 2023 15:50:52 -0600 Subject: [PATCH 1/2] dashboard: fix subprocesses blocking the event loop - break apart the util module - adds a new util to run subprocesses with asyncio --- esphome/dashboard/settings.py | 2 +- esphome/dashboard/status/ping.py | 15 +++----- esphome/dashboard/util.py | 52 ---------------------------- esphome/dashboard/util/__init__.py | 0 esphome/dashboard/util/itertools.py | 22 ++++++++++++ esphome/dashboard/util/password.py | 11 ++++++ esphome/dashboard/util/subprocess.py | 31 +++++++++++++++++ esphome/dashboard/util/text.py | 25 +++++++++++++ esphome/dashboard/web_server.py | 13 +++---- 9 files changed, 101 insertions(+), 70 deletions(-) delete mode 100644 esphome/dashboard/util.py create mode 100644 esphome/dashboard/util/__init__.py create mode 100644 esphome/dashboard/util/itertools.py create mode 100644 esphome/dashboard/util/password.py create mode 100644 esphome/dashboard/util/subprocess.py create mode 100644 esphome/dashboard/util/text.py diff --git a/esphome/dashboard/settings.py b/esphome/dashboard/settings.py index 1ddb6f652d..888616f6f7 100644 --- a/esphome/dashboard/settings.py +++ b/esphome/dashboard/settings.py @@ -10,7 +10,7 @@ from esphome.helpers import get_bool_env from esphome.storage_json import ext_storage_path from .entries import DashboardEntry -from .util import password_hash +from .util.password import password_hash class DashboardSettings: diff --git a/esphome/dashboard/status/ping.py b/esphome/dashboard/status/ping.py index 17c1254c9d..678d7844ae 100644 --- a/esphome/dashboard/status/ping.py +++ b/esphome/dashboard/status/ping.py @@ -7,22 +7,15 @@ from typing import cast from ..core import DASHBOARD from ..entries import DashboardEntry from ..core import list_dashboard_entries -from ..util import chunked +from ..util.itertools import chunked +from ..util.subprocess import async_system_command_status async def _async_ping_host(host: str) -> bool: """Ping a host.""" - ping_command = ["ping", "-n" if os.name == "nt" else "-c", "1"] - process = await asyncio.create_subprocess_exec( - *ping_command, - host, - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, - close_fds=False, + return await async_system_command_status( + ["ping", "-n" if os.name == "nt" else "-c", "1", host] ) - await process.wait() - return process.returncode == 0 class PingStatus: diff --git a/esphome/dashboard/util.py b/esphome/dashboard/util.py deleted file mode 100644 index 7b6572b989..0000000000 --- a/esphome/dashboard/util.py +++ /dev/null @@ -1,52 +0,0 @@ -import hashlib -import unicodedata -from collections.abc import Iterable -from functools import partial -from itertools import islice -from typing import Any - -from esphome.const import ALLOWED_NAME_CHARS - - -def password_hash(password: str) -> bytes: - """Create a hash of a password to transform it to a fixed-length digest. - - Note this is not meant for secure storage, but for securely comparing passwords. - """ - return hashlib.sha256(password.encode()).digest() - - -def strip_accents(value): - return "".join( - c - for c in unicodedata.normalize("NFD", str(value)) - if unicodedata.category(c) != "Mn" - ) - - -def friendly_name_slugify(value): - value = ( - strip_accents(value) - .lower() - .replace(" ", "-") - .replace("_", "-") - .replace("--", "-") - .strip("-") - ) - return "".join(c for c in value if c in ALLOWED_NAME_CHARS) - - -def take(take_num: int, iterable: Iterable) -> list[Any]: - """Return first n items of the iterable as a list. - - From itertools recipes - """ - return list(islice(iterable, take_num)) - - -def chunked(iterable: Iterable, chunked_num: int) -> Iterable[Any]: - """Break *iterable* into lists of length *n*. - - From more-itertools - """ - return iter(partial(take, chunked_num, iter(iterable)), []) diff --git a/esphome/dashboard/util/__init__.py b/esphome/dashboard/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/dashboard/util/itertools.py b/esphome/dashboard/util/itertools.py new file mode 100644 index 0000000000..54e95ef802 --- /dev/null +++ b/esphome/dashboard/util/itertools.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from collections.abc import Iterable +from functools import partial +from itertools import islice +from typing import Any + + +def take(take_num: int, iterable: Iterable) -> list[Any]: + """Return first n items of the iterable as a list. + + From itertools recipes + """ + return list(islice(iterable, take_num)) + + +def chunked(iterable: Iterable, chunked_num: int) -> Iterable[Any]: + """Break *iterable* into lists of length *n*. + + From more-itertools + """ + return iter(partial(take, chunked_num, iter(iterable)), []) diff --git a/esphome/dashboard/util/password.py b/esphome/dashboard/util/password.py new file mode 100644 index 0000000000..e7ea28c25d --- /dev/null +++ b/esphome/dashboard/util/password.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import hashlib + + +def password_hash(password: str) -> bytes: + """Create a hash of a password to transform it to a fixed-length digest. + + Note this is not meant for secure storage, but for securely comparing passwords. + """ + return hashlib.sha256(password.encode()).digest() diff --git a/esphome/dashboard/util/subprocess.py b/esphome/dashboard/util/subprocess.py new file mode 100644 index 0000000000..583dd116e3 --- /dev/null +++ b/esphome/dashboard/util/subprocess.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Iterable + + +async def async_system_command_status(command: Iterable[str]) -> bool: + """Run a system command checking only the status.""" + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + close_fds=False, + ) + await process.wait() + return process.returncode == 0 + + +async def async_run_system_command(command: Iterable[str]) -> tuple[bool, bytes, bytes]: + """Run a system command and return a tuple of returncode, stdout, stderr.""" + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await process.communicate() + await process.wait() + return process.returncode, stdout, stderr diff --git a/esphome/dashboard/util/text.py b/esphome/dashboard/util/text.py new file mode 100644 index 0000000000..08d2df6abf --- /dev/null +++ b/esphome/dashboard/util/text.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import unicodedata + +from esphome.const import ALLOWED_NAME_CHARS + + +def strip_accents(value): + return "".join( + c + for c in unicodedata.normalize("NFD", str(value)) + if unicodedata.category(c) != "Mn" + ) + + +def friendly_name_slugify(value): + value = ( + strip_accents(value) + .lower() + .replace(" ", "-") + .replace("_", "-") + .replace("--", "-") + .strip("-") + ) + return "".join(c for c in value if c in ALLOWED_NAME_CHARS) diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 086a28cbb2..35dcd33bce 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -31,13 +31,14 @@ import yaml from tornado.log import access_log from esphome import const, platformio_api, yaml_util -from esphome.helpers import get_bool_env, mkdir_p, run_system_command +from esphome.helpers import get_bool_env, mkdir_p from esphome.storage_json import StorageJSON, ext_storage_path, trash_storage_path from esphome.util import get_serial_ports, shlex_quote from .core import DASHBOARD, list_dashboard_entries from .entries import DashboardEntry -from .util import friendly_name_slugify +from .util.text import friendly_name_slugify +from .util.subprocess import async_run_system_command _LOGGER = logging.getLogger(__name__) @@ -522,7 +523,7 @@ class DownloadListRequestHandler(BaseHandler): class DownloadBinaryRequestHandler(BaseHandler): @authenticated @bind_config - def get(self, configuration=None): + async def get(self, configuration=None): compressed = self.get_argument("compressed", "0") == "1" storage_path = ext_storage_path(configuration) @@ -548,7 +549,7 @@ class DownloadBinaryRequestHandler(BaseHandler): if not Path(path).is_file(): args = ["esphome", "idedata", settings.rel_path(configuration)] - rc, stdout, _ = run_system_command(*args) + rc, stdout, _ = await async_run_system_command(*args) if rc != 0: self.send_error(404 if rc == 2 else 500) @@ -902,7 +903,7 @@ SafeLoaderIgnoreUnknown.add_constructor( class JsonConfigRequestHandler(BaseHandler): @authenticated @bind_config - def get(self, configuration=None): + async def get(self, configuration=None): filename = settings.rel_path(configuration) if not os.path.isfile(filename): self.send_error(404) @@ -910,7 +911,7 @@ class JsonConfigRequestHandler(BaseHandler): args = ["esphome", "config", filename, "--show-secrets"] - rc, stdout, _ = run_system_command(*args) + rc, stdout, _ = await async_run_system_command(*args) if rc != 0: self.send_error(422) From 86555b33987e14864267d3f3f2672eecca8293e3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Nov 2023 15:52:50 -0600 Subject: [PATCH 2/2] take a list --- esphome/dashboard/web_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 35dcd33bce..c0cc00b66c 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -549,7 +549,7 @@ class DownloadBinaryRequestHandler(BaseHandler): if not Path(path).is_file(): args = ["esphome", "idedata", settings.rel_path(configuration)] - rc, stdout, _ = await async_run_system_command(*args) + rc, stdout, _ = await async_run_system_command(args) if rc != 0: self.send_error(404 if rc == 2 else 500) @@ -911,7 +911,7 @@ class JsonConfigRequestHandler(BaseHandler): args = ["esphome", "config", filename, "--show-secrets"] - rc, stdout, _ = await async_run_system_command(*args) + rc, stdout, _ = await async_run_system_command(args) if rc != 0: self.send_error(422)