From f002a23d2d7cbdc5820caaec846c71dd003d6ed1 Mon Sep 17 00:00:00 2001 From: Guillermo Ruffino Date: Thu, 16 Jun 2022 22:46:20 -0300 Subject: [PATCH] Language schema 202204 (#3492) --- esphome/automation.py | 15 +- esphome/components/light/effects.py | 8 +- esphome/components/remote_base/__init__.py | 8 +- esphome/config_validation.py | 46 +- .../{jsonschema.py => schema_extractors.py} | 27 +- esphome/voluptuous_schema.py | 4 +- script/build_jsonschema.py | 828 ------------------ script/build_language_schema.py | 156 +++- 8 files changed, 188 insertions(+), 904 deletions(-) rename esphome/{jsonschema.py => schema_extractors.py} (75%) delete mode 100644 script/build_jsonschema.py diff --git a/esphome/automation.py b/esphome/automation.py index 4007dc4c51..4aede00c5e 100644 --- a/esphome/automation.py +++ b/esphome/automation.py @@ -12,7 +12,7 @@ from esphome.const import ( CONF_TYPE_ID, CONF_TIME, ) -from esphome.jsonschema import jschema_extractor +from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor from esphome.util import Registry @@ -23,11 +23,10 @@ def maybe_simple_id(*validators): def maybe_conf(conf, *validators): validator = cv.All(*validators) - @jschema_extractor("maybe") + @schema_extractor("maybe") def validate(value): - # pylint: disable=comparison-with-callable - if value == jschema_extractor: - return validator + if value == SCHEMA_EXTRACT: + return (validator, conf) if isinstance(value, dict): return validator(value) @@ -111,11 +110,9 @@ def validate_automation(extra_schema=None, extra_validators=None, single=False): # This should only happen with invalid configs, but let's have a nice error message. return [schema(value)] - @jschema_extractor("automation") + @schema_extractor("automation") def validator(value): - # hack to get the schema - # pylint: disable=comparison-with-callable - if value == jschema_extractor: + if value == SCHEMA_EXTRACT: return schema value = validator_(value) diff --git a/esphome/components/light/effects.py b/esphome/components/light/effects.py index a987e0fc96..cef7cd7f3a 100644 --- a/esphome/components/light/effects.py +++ b/esphome/components/light/effects.py @@ -1,4 +1,4 @@ -from esphome.jsonschema import jschema_extractor +from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor import esphome.codegen as cg import esphome.config_validation as cv from esphome import automation @@ -479,11 +479,11 @@ async def addressable_flicker_effect_to_code(config, effect_id): def validate_effects(allowed_effects): - @jschema_extractor("effects") + @schema_extractor("effects") def validator(value): - # pylint: disable=comparison-with-callable - if value == jschema_extractor: + if value == SCHEMA_EXTRACT: return (allowed_effects, EFFECTS_REGISTRY) + value = cv.validate_registry("effect", EFFECTS_REGISTRY)(value) errors = [] names = set() diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py index 165da95d67..b4d538c096 100644 --- a/esphome/components/remote_base/__init__.py +++ b/esphome/components/remote_base/__init__.py @@ -32,7 +32,7 @@ from esphome.const import ( CONF_LEVEL, ) from esphome.core import coroutine -from esphome.jsonschema import jschema_extractor +from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor from esphome.util import Registry, SimpleRegistry AUTO_LOAD = ["binary_sensor"] @@ -195,14 +195,14 @@ def validate_dumpers(value): def validate_triggers(base_schema): assert isinstance(base_schema, cv.Schema) - @jschema_extractor("triggers") + @schema_extractor("triggers") def validator(config): added_keys = {} for key, (_, valid) in TRIGGER_REGISTRY.items(): added_keys[cv.Optional(key)] = valid new_schema = base_schema.extend(added_keys) - # pylint: disable=comparison-with-callable - if config == jschema_extractor: + + if config == SCHEMA_EXTRACT: return new_schema return new_schema(config) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index dacbcf72ba..c4cb916e57 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -57,11 +57,12 @@ from esphome.core import ( TimePeriodMinutes, ) from esphome.helpers import list_starts_with, add_class_to_obj -from esphome.jsonschema import ( - jschema_list, - jschema_extractor, - jschema_registry, - jschema_typed, +from esphome.schema_extractors import ( + SCHEMA_EXTRACT, + schema_extractor_list, + schema_extractor, + schema_extractor_registry, + schema_extractor_typed, ) from esphome.util import parse_esphome_version from esphome.voluptuous_schema import _Schema @@ -327,7 +328,7 @@ def boolean(value): ) -@jschema_list +@schema_extractor_list def ensure_list(*validators): """Validate this configuration option to be a list. @@ -452,7 +453,11 @@ def validate_id_name(value): def use_id(type): """Declare that this configuration option should point to an ID with the given type.""" + @schema_extractor("use_id") def validator(value): + if value == SCHEMA_EXTRACT: + return type + check_not_templatable(value) if value is None: return core.ID(None, is_declaration=False, type=type) @@ -475,7 +480,11 @@ def declare_id(type): If two IDs with the same name exist, a validation error is thrown. """ + @schema_extractor("declare_id") def validator(value): + if value == SCHEMA_EXTRACT: + return type + check_not_templatable(value) if value is None: return core.ID(None, is_declaration=True, type=type) @@ -494,11 +503,11 @@ def templatable(other_validators): """ schema = Schema(other_validators) - @jschema_extractor("templatable") + @schema_extractor("templatable") def validator(value): - # pylint: disable=comparison-with-callable - if value == jschema_extractor: + if value == SCHEMA_EXTRACT: return other_validators + if isinstance(value, Lambda): return returning_lambda(value) if isinstance(other_validators, dict): @@ -1177,10 +1186,9 @@ def one_of(*values, **kwargs): if kwargs: raise ValueError - @jschema_extractor("one_of") + @schema_extractor("one_of") def validator(value): - # pylint: disable=comparison-with-callable - if value == jschema_extractor: + if value == SCHEMA_EXTRACT: return values if string_: @@ -1220,10 +1228,9 @@ def enum(mapping, **kwargs): assert isinstance(mapping, dict) one_of_validator = one_of(*mapping, **kwargs) - @jschema_extractor("enum") + @schema_extractor("enum") def validator(value): - # pylint: disable=comparison-with-callable - if value == jschema_extractor: + if value == SCHEMA_EXTRACT: return mapping value = one_of_validator(value) @@ -1396,7 +1403,7 @@ def extract_keys(schema): return keys -@jschema_typed +@schema_extractor_typed def typed_schema(schemas, **kwargs): """Create a schema that has a key to distinguish between schemas""" key = kwargs.pop("key", CONF_TYPE) @@ -1510,7 +1517,7 @@ def validate_registry_entry(name, registry): ) ignore_keys = extract_keys(base_schema) - @jschema_registry(registry) + @schema_extractor_registry(registry) def validator(value): if isinstance(value, str): value = {value: {}} @@ -1555,12 +1562,15 @@ def validate_registry(name, registry): return ensure_list(validate_registry_entry(name, registry)) -@jschema_list def maybe_simple_value(*validators, **kwargs): key = kwargs.pop("key", CONF_VALUE) validator = All(*validators) + @schema_extractor("maybe") def validate(value): + if value == SCHEMA_EXTRACT: + return (validator, key) + if isinstance(value, dict) and key in value: return validator(value) return validator({key: value}) diff --git a/esphome/jsonschema.py b/esphome/schema_extractors.py similarity index 75% rename from esphome/jsonschema.py rename to esphome/schema_extractors.py index 94325f4abc..2280a84849 100644 --- a/esphome/jsonschema.py +++ b/esphome/schema_extractors.py @@ -9,9 +9,9 @@ However there is a property to further disable decorator impact.""" -# This is set to true by script/build_jsonschema.py +# This is set to true by script/build_language_schema.py # only, so data is collected (again functionality is not modified) -EnableJsonSchemaCollect = False +EnableSchemaExtraction = False extended_schemas = {} list_schemas = {} @@ -19,9 +19,12 @@ registry_schemas = {} hidden_schemas = {} typed_schemas = {} +# This key is used to generate schema files of Esphome configuration. +SCHEMA_EXTRACT = object() -def jschema_extractor(validator_name): - if EnableJsonSchemaCollect: + +def schema_extractor(validator_name): + if EnableSchemaExtraction: def decorator(func): hidden_schemas[repr(func)] = validator_name @@ -35,8 +38,8 @@ def jschema_extractor(validator_name): return dummy -def jschema_extended(func): - if EnableJsonSchemaCollect: +def schema_extractor_extended(func): + if EnableSchemaExtraction: def decorate(*args, **kwargs): ret = func(*args, **kwargs) @@ -49,8 +52,8 @@ def jschema_extended(func): return func -def jschema_list(func): - if EnableJsonSchemaCollect: +def schema_extractor_list(func): + if EnableSchemaExtraction: def decorate(*args, **kwargs): ret = func(*args, **kwargs) @@ -63,8 +66,8 @@ def jschema_list(func): return func -def jschema_registry(registry): - if EnableJsonSchemaCollect: +def schema_extractor_registry(registry): + if EnableSchemaExtraction: def decorator(func): registry_schemas[repr(func)] = registry @@ -78,8 +81,8 @@ def jschema_registry(registry): return dummy -def jschema_typed(func): - if EnableJsonSchemaCollect: +def schema_extractor_typed(func): + if EnableSchemaExtraction: def decorate(*args, **kwargs): ret = func(*args, **kwargs) diff --git a/esphome/voluptuous_schema.py b/esphome/voluptuous_schema.py index 0fdae423cf..f4ed2fe03b 100644 --- a/esphome/voluptuous_schema.py +++ b/esphome/voluptuous_schema.py @@ -2,7 +2,7 @@ import difflib import itertools import voluptuous as vol -from esphome.jsonschema import jschema_extended +from esphome.schema_extractors import schema_extractor_extended class ExtraKeysInvalid(vol.Invalid): @@ -203,7 +203,7 @@ class _Schema(vol.Schema): self._extra_schemas.append(validator) return self - @jschema_extended + @schema_extractor_extended # pylint: disable=signature-differs def extend(self, *schemas, **kwargs): extra = kwargs.pop("extra", None) diff --git a/script/build_jsonschema.py b/script/build_jsonschema.py deleted file mode 100644 index 5373d404a7..0000000000 --- a/script/build_jsonschema.py +++ /dev/null @@ -1,828 +0,0 @@ -#!/usr/bin/env python3 - -from esphome.cpp_generator import MockObj -import json -import argparse -import os -import re -from pathlib import Path -import voluptuous as vol - -# NOTE: Cannot import other esphome components globally as a modification in jsonschema -# is needed before modules are loaded -import esphome.jsonschema as ejs - -ejs.EnableJsonSchemaCollect = True - -DUMP_COMMENTS = False - -JSC_ACTION = "automation.ACTION_REGISTRY" -JSC_ALLOF = "allOf" -JSC_ANYOF = "anyOf" -JSC_COMMENT = "$comment" -JSC_CONDITION = "automation.CONDITION_REGISTRY" -JSC_DESCRIPTION = "description" -JSC_ONEOF = "oneOf" -JSC_PROPERTIES = "properties" -JSC_REF = "$ref" - -# this should be required, but YAML Language server completion does not work properly if required are specified. -# still needed for other features / checks -JSC_REQUIRED = "required_" - -SIMPLE_AUTOMATION = "simple_automation" - -schema_names = {} -schema_registry = {} -components = {} -modules = {} -registries = [] -pending_refs = [] - -definitions = {} -base_props = {} - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--output", default="esphome.json", help="Output filename", type=os.path.abspath -) - -args = parser.parse_args() - - -def get_ref(definition): - return {JSC_REF: "#/definitions/" + definition} - - -def is_ref(jschema): - return isinstance(jschema, dict) and JSC_REF in jschema - - -def unref(jschema): - return definitions.get(jschema[JSC_REF][len("#/definitions/") :]) - - -def add_definition_array_or_single_object(ref): - return {JSC_ANYOF: [{"type": "array", "items": ref}, ref]} - - -def add_core(): - from esphome.core.config import CONFIG_SCHEMA - - base_props["esphome"] = get_jschema("esphome", CONFIG_SCHEMA) - - -def add_buses(): - # uart - from esphome.components.uart import UART_DEVICE_SCHEMA - - get_jschema("uart_bus", UART_DEVICE_SCHEMA) - - # spi - from esphome.components.spi import spi_device_schema - - get_jschema("spi_bus", spi_device_schema(False)) - - # i2c - from esphome.components.i2c import i2c_device_schema - - get_jschema("i2c_bus", i2c_device_schema(None)) - - -def add_registries(): - for domain, module in modules.items(): - add_module_registries(domain, module) - - -def add_module_registries(domain, module): - from esphome.util import Registry - - for c in dir(module): - m = getattr(module, c) - if isinstance(m, Registry): - add_registry(domain + "." + c, m) - - -def add_registry(registry_name, registry): - validators = [] - registries.append((registry, registry_name)) - for name in registry.keys(): - schema = get_jschema(str(name), registry[name].schema, create_return_ref=False) - if not schema: - schema = {"type": "null"} - o_schema = {"type": "object", JSC_PROPERTIES: {name: schema}} - o_schema = create_ref( - registry_name + "-" + name, str(registry[name].schema) + "x", o_schema - ) - validators.append(o_schema) - definitions[registry_name] = {JSC_ANYOF: validators} - - -def get_registry_ref(registry): - # we don't know yet - ref = {JSC_REF: "pending"} - pending_refs.append((ref, registry)) - return ref - - -def solve_pending_refs(): - for ref, registry in pending_refs: - for registry_match, name in registries: - if registry == registry_match: - ref[JSC_REF] = "#/definitions/" + name - - -def add_module_schemas(name, module): - import esphome.config_validation as cv - - for c in dir(module): - v = getattr(module, c) - if isinstance(v, cv.Schema): - get_jschema(name + "." + c, v) - - -def get_dirs(): - from esphome.loader import CORE_COMPONENTS_PATH - - dir_names = [ - d - for d in os.listdir(CORE_COMPONENTS_PATH) - if not d.startswith("__") - and os.path.isdir(os.path.join(CORE_COMPONENTS_PATH, d)) - ] - return dir_names - - -def get_logger_tags(): - from esphome.loader import CORE_COMPONENTS_PATH - import glob - - pattern = re.compile(r'^static const char(\*\s|\s\*)TAG = "(\w.*)";', re.MULTILINE) - tags = [ - "app", - "component", - "esphal", - "helpers", - "preferences", - "scheduler", - "api.service", - ] - for x in os.walk(CORE_COMPONENTS_PATH): - for y in glob.glob(os.path.join(x[0], "*.cpp")): - with open(y) as file: - data = file.read() - match = pattern.search(data) - if match: - tags.append(match.group(2)) - return tags - - -def load_components(): - import esphome.config_validation as cv - from esphome.config import get_component - - modules["cv"] = cv - from esphome import automation - - modules["automation"] = automation - - for domain in get_dirs(): - components[domain] = get_component(domain) - modules[domain] = components[domain].module - - -def add_components(): - from esphome.config import get_platform - - for domain, c in components.items(): - if c.is_platform_component: - # this is a platform_component, e.g. binary_sensor - platform_schema = [ - { - "type": "object", - "properties": {"platform": {"type": "string"}}, - } - ] - if domain not in ("output", "display"): - # output bases are either FLOAT or BINARY so don't add common base for this - # display bases are either simple or FULL so don't add common base for this - platform_schema = [ - {"$ref": f"#/definitions/{domain}.{domain.upper()}_SCHEMA"} - ] + platform_schema - - base_props[domain] = {"type": "array", "items": {"allOf": platform_schema}} - - add_module_registries(domain, c.module) - add_module_schemas(domain, c.module) - - # need first to iterate all platforms then iterate components - # a platform component can have other components as properties, - # e.g. climate components usually have a temperature sensor - - for domain, c in components.items(): - if (c.config_schema is not None) or c.is_platform_component: - if c.is_platform_component: - platform_schema = base_props[domain]["items"]["allOf"] - for platform in get_dirs(): - p = get_platform(domain, platform) - if p is not None: - # this is a platform element, e.g. - # - platform: gpio - schema = get_jschema( - domain + "-" + platform, - p.config_schema, - create_return_ref=False, - ) - if ( - schema - ): # for invalid schemas, None is returned thus is deprecated - platform_schema.append( - { - "if": { - JSC_PROPERTIES: { - "platform": {"const": platform} - } - }, - "then": schema, - } - ) - - elif c.config_schema is not None: - # adds root components which are not platforms, e.g. api: logger: - if c.multi_conf: - schema = get_jschema(domain, c.config_schema) - schema = add_definition_array_or_single_object(schema) - else: - schema = get_jschema(domain, c.config_schema, False) - base_props[domain] = schema - - -def get_automation_schema(name, vschema): - from esphome.automation import AUTOMATION_SCHEMA - - # ensure SIMPLE_AUTOMATION - if SIMPLE_AUTOMATION not in definitions: - simple_automation = add_definition_array_or_single_object(get_ref(JSC_ACTION)) - simple_automation[JSC_ANYOF].append( - get_jschema(AUTOMATION_SCHEMA.__module__, AUTOMATION_SCHEMA) - ) - - definitions[schema_names[str(AUTOMATION_SCHEMA)]][JSC_PROPERTIES][ - "then" - ] = add_definition_array_or_single_object(get_ref(JSC_ACTION)) - definitions[SIMPLE_AUTOMATION] = simple_automation - - extra_vschema = None - if AUTOMATION_SCHEMA == ejs.extended_schemas[str(vschema)][0]: - extra_vschema = ejs.extended_schemas[str(vschema)][1] - - if not extra_vschema: - return get_ref(SIMPLE_AUTOMATION) - - # add then property - extra_jschema = get_jschema(name, extra_vschema, False) - - if is_ref(extra_jschema): - return extra_jschema - - if not JSC_PROPERTIES in extra_jschema: - # these are interval: and exposure_notifications, featuring automations a component - extra_jschema[JSC_ALLOF][0][JSC_PROPERTIES][ - "then" - ] = add_definition_array_or_single_object(get_ref(JSC_ACTION)) - ref = create_ref(name, extra_vschema, extra_jschema) - return add_definition_array_or_single_object(ref) - - # automations can be either - # * a single action, - # * an array of action, - # * an object with automation's schema and a then key - # with again a single action or an array of actions - - if len(extra_jschema[JSC_PROPERTIES]) == 0: - return get_ref(SIMPLE_AUTOMATION) - - extra_jschema[JSC_PROPERTIES]["then"] = add_definition_array_or_single_object( - get_ref(JSC_ACTION) - ) - # if there is a required element in extra_jschema then this automation does not support - # directly a list of actions - if JSC_REQUIRED in extra_jschema: - return create_ref(name, extra_vschema, extra_jschema) - - jschema = add_definition_array_or_single_object(get_ref(JSC_ACTION)) - jschema[JSC_ANYOF].append(extra_jschema) - - return create_ref(name, extra_vschema, jschema) - - -def get_entry(parent_key, vschema): - from esphome.voluptuous_schema import _Schema as schema_type - - entry = {} - # annotate schema validator info - if DUMP_COMMENTS: - entry[JSC_COMMENT] = "entry: " + parent_key + "/" + str(vschema) - - if isinstance(vschema, dict): - entry = {"what": "is_this"} - elif isinstance(vschema, list): - ref = get_jschema(parent_key + "[]", vschema[0]) - entry = {"type": "array", "items": ref} - elif isinstance(vschema, schema_type) and hasattr(vschema, "schema"): - entry = get_jschema(parent_key, vschema, False) - elif hasattr(vschema, "validators"): - entry = get_jschema(parent_key, vschema, False) - elif vschema in schema_registry: - entry = schema_registry[vschema].copy() - elif str(vschema) in ejs.registry_schemas: - entry = get_registry_ref(ejs.registry_schemas[str(vschema)]) - elif str(vschema) in ejs.list_schemas: - ref = get_jschema(parent_key, ejs.list_schemas[str(vschema)][0]) - entry = {JSC_ANYOF: [ref, {"type": "array", "items": ref}]} - elif str(vschema) in ejs.typed_schemas: - schema_types = [{"type": "object", "properties": {"type": {"type": "string"}}}] - entry = {"allOf": schema_types} - for schema_key, vschema_type in ejs.typed_schemas[str(vschema)][0][0].items(): - schema_types.append( - { - "if": {"properties": {"type": {"const": schema_key}}}, - "then": get_jschema(f"{parent_key}-{schema_key}", vschema_type), - } - ) - - elif str(vschema) in ejs.hidden_schemas: - # get the schema from the automation schema - type = ejs.hidden_schemas[str(vschema)] - inner_vschema = vschema(ejs.jschema_extractor) - if type == "automation": - entry = get_automation_schema(parent_key, inner_vschema) - elif type == "maybe": - entry = get_jschema(parent_key, inner_vschema) - elif type == "one_of": - entry = {"enum": list(inner_vschema)} - elif type == "enum": - entry = {"enum": list(inner_vschema.keys())} - elif type == "effects": - # Like list schema but subset from list. - subset_list = inner_vschema[0] - # get_jschema('strobex', registry['strobe'].schema) - registry_schemas = [] - for name in subset_list: - registry_schemas.append(get_ref("light.EFFECTS_REGISTRY-" + name)) - - entry = { - JSC_ANYOF: [{"type": "array", "items": {JSC_ANYOF: registry_schemas}}] - } - - else: - raise ValueError("Unknown extracted schema type") - elif str(vschema).startswith(" 0: - output[JSC_REQUIRED] = required - return output - - -def add_pin_schema(): - from esphome import pins - - add_module_schemas("PIN", pins) - - -def add_pin_registry(): - from esphome import pins - - pin_registry = pins.PIN_SCHEMA_REGISTRY - assert len(pin_registry) > 0 - # Here are schemas for pcf8574, mcp23xxx and other port expanders which add - # gpio registers - # ESPHome validates pins schemas if it founds a key in the pin configuration. - # This key is added to a required in jsonschema, and all options are part of a - # oneOf section, so only one is selected. Also internal schema adds number as required. - - for mode in ("INPUT", "OUTPUT"): - schema_name = f"PIN.GPIO_FULL_{mode}_PIN_SCHEMA" - - # TODO: get pin definitions properly - if schema_name not in definitions: - definitions[schema_name] = {"type": ["object", "null"], JSC_PROPERTIES: {}} - - internal = definitions[schema_name] - definitions[schema_name]["additionalItems"] = False - definitions[f"PIN.{mode}_INTERNAL"] = internal - internal[JSC_PROPERTIES]["number"] = {"type": ["number", "string"]} - schemas = [get_ref(f"PIN.{mode}_INTERNAL")] - schemas[0]["required"] = ["number"] - # accept string and object, for internal shorthand pin IO: - definitions[schema_name] = {"oneOf": schemas, "type": ["string", "object"]} - - for k, v in pin_registry.items(): - if isinstance(v[1], vol.validators.All): - pin_jschema = get_jschema(f"PIN.{mode}_" + k, v[1]) - if unref(pin_jschema): - pin_jschema["required"] = [k] - schemas.append(pin_jschema) - - -def dump_schema(): - import esphome.config_validation as cv - - from esphome import automation - from esphome.automation import validate_potentially_and_condition - from esphome import pins - from esphome.core import CORE - from esphome.helpers import write_file_if_changed - from esphome.components import remote_base - - # The root directory of the repo - root = Path(__file__).parent.parent - - # Fake some directory so that get_component works - CORE.config_path = str(root) - - file_path = args.output - - schema_registry[cv.boolean] = {"type": "boolean"} - - for v in [ - cv.int_, - cv.int_range, - cv.positive_int, - cv.float_, - cv.positive_float, - cv.positive_float, - cv.positive_not_null_int, - cv.negative_one_to_one_float, - cv.port, - ]: - schema_registry[v] = {"type": "number"} - - for v in [ - cv.string, - cv.string_strict, - cv.valid_name, - cv.hex_int, - cv.hex_int_range, - pins.gpio_output_pin_schema, - pins.gpio_input_pin_schema, - pins.gpio_input_pullup_pin_schema, - cv.float_with_unit, - cv.subscribe_topic, - cv.publish_topic, - cv.mqtt_payload, - cv.ssid, - cv.percentage_int, - cv.percentage, - cv.possibly_negative_percentage, - cv.positive_time_period, - cv.positive_time_period_microseconds, - cv.positive_time_period_milliseconds, - cv.positive_time_period_minutes, - cv.positive_time_period_seconds, - ]: - schema_registry[v] = {"type": "string"} - - schema_registry[validate_potentially_and_condition] = get_ref("condition_list") - - for v in [pins.gpio_input_pin_schema, pins.gpio_input_pullup_pin_schema]: - schema_registry[v] = get_ref("PIN.GPIO_FULL_INPUT_PIN_SCHEMA") - for v in [pins.internal_gpio_input_pin_schema, pins.gpio_input_pin_schema]: - schema_registry[v] = get_ref("PIN.INPUT_INTERNAL") - - for v in [pins.gpio_output_pin_schema, pins.internal_gpio_output_pin_schema]: - schema_registry[v] = get_ref("PIN.GPIO_FULL_OUTPUT_PIN_SCHEMA") - for v in [pins.internal_gpio_output_pin_schema, pins.gpio_output_pin_schema]: - schema_registry[v] = get_ref("PIN.OUTPUT_INTERNAL") - - add_module_schemas("CONFIG", cv) - get_jschema("POLLING_COMPONENT", cv.polling_component_schema("60s")) - - add_pin_schema() - - add_module_schemas("REMOTE_BASE", remote_base) - add_module_schemas("AUTOMATION", automation) - - load_components() - add_registries() - - definitions["condition_list"] = { - JSC_ONEOF: [ - {"type": "array", "items": get_ref(JSC_CONDITION)}, - get_ref(JSC_CONDITION), - ] - } - - output = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "definitions": definitions, - JSC_PROPERTIES: base_props, - } - - add_core() - add_buses() - add_components() - - add_registries() # need second pass, e.g. climate.pid.autotune - add_pin_registry() - solve_pending_refs() - - write_file_if_changed(file_path, json.dumps(output)) - print(f"Wrote {file_path}") - - -dump_schema() diff --git a/script/build_language_schema.py b/script/build_language_schema.py index ec0912e9e6..89f8d52ea9 100644 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -1,15 +1,16 @@ import inspect import json import argparse -from operator import truediv import os +import glob +import re import voluptuous as vol -# NOTE: Cannot import other esphome components globally as a modification in jsonschema +# NOTE: Cannot import other esphome components globally as a modification in vol_schema # is needed before modules are loaded -import esphome.jsonschema as ejs +import esphome.schema_extractors as ejs -ejs.EnableJsonSchemaCollect = True +ejs.EnableSchemaExtraction = True # schema format: # Schemas are splitted in several files in json format, one for core stuff, one for each platform (sensor, binary_sensor, etc) and @@ -60,15 +61,6 @@ solve_registry = [] def get_component_names(): - # return [ - # "esphome", - # "esp32", - # "esp8266", - # "logger", - # "sensor", - # "remote_receiver", - # "binary_sensor", - # ] from esphome.loader import CORE_COMPONENTS_PATH component_names = ["esphome", "sensor"] @@ -100,7 +92,7 @@ from esphome import automation from esphome import pins from esphome.components import remote_base from esphome.const import CONF_TYPE -from esphome.loader import get_platform +from esphome.loader import get_platform, CORE_COMPONENTS_PATH from esphome.helpers import write_file_if_changed from esphome.util import Registry @@ -120,10 +112,12 @@ def write_file(name, obj): def register_module_schemas(key, module, manifest=None): for name, schema in module_schemas(module): register_known_schema(key, name, schema) - if ( - manifest and manifest.multi_conf and S_CONFIG_SCHEMA in output[key][S_SCHEMAS] - ): # not sure about 2nd part of the if, might be useless config (e.g. as3935) - output[key][S_SCHEMAS][S_CONFIG_SCHEMA]["is_list"] = True + + if manifest: + # Multi conf should allow list of components + # not sure about 2nd part of the if, might be useless config (e.g. as3935) + if manifest.multi_conf and S_CONFIG_SCHEMA in output[key][S_SCHEMAS]: + output[key][S_SCHEMAS][S_CONFIG_SCHEMA]["is_list"] = True def register_known_schema(module, name, schema): @@ -265,13 +259,58 @@ def do_esp8266(): def fix_remote_receiver(): - output["remote_receiver.binary_sensor"]["schemas"]["CONFIG_SCHEMA"] = { + remote_receiver_schema = output["remote_receiver.binary_sensor"]["schemas"] + remote_receiver_schema["CONFIG_SCHEMA"] = { "type": "schema", "schema": { "extends": ["binary_sensor.BINARY_SENSOR_SCHEMA", "core.COMPONENT_SCHEMA"], - "config_vars": output["remote_base"]["binary"], + "config_vars": output["remote_base"].pop("binary"), }, } + remote_receiver_schema["CONFIG_SCHEMA"]["schema"]["config_vars"]["receiver_id"] = { + "key": "GeneratedID", + "use_id_type": "remote_base::RemoteReceiverBase", + "type": "use_id", + } + + +def fix_script(): + output["script"][S_SCHEMAS][S_CONFIG_SCHEMA][S_TYPE] = S_SCHEMA + config_schema = output["script"][S_SCHEMAS][S_CONFIG_SCHEMA] + config_schema[S_SCHEMA][S_CONFIG_VARS]["id"]["id_type"] = { + "class": "script::Script" + } + config_schema["is_list"] = True + + +def get_logger_tags(): + pattern = re.compile(r'^static const char \*const TAG = "(\w.*)";', re.MULTILINE) + # tags not in components dir + tags = [ + "app", + "component", + "entity_base", + "scheduler", + "api.service", + ] + for x in os.walk(CORE_COMPONENTS_PATH): + for y in glob.glob(os.path.join(x[0], "*.cpp")): + with open(y, encoding="utf-8") as file: + data = file.read() + match = pattern.search(data) + if match: + tags.append(match.group(1)) + return tags + + +def add_logger_tags(): + tags = get_logger_tags() + logs = output["logger"]["schemas"]["CONFIG_SCHEMA"]["schema"]["config_vars"][ + "logs" + ]["schema"]["config_vars"] + for t in tags: + logs[t] = logs["string"].copy() + logs.pop("string") def add_referenced_recursive(referenced_schemas, config_var, path, eat_schema=False): @@ -401,7 +440,7 @@ def shrink(): else: print("expected extends here!" + x) arr_s = merge(key_s, arr_s) - if arr_s[S_TYPE] == "enum": + if arr_s[S_TYPE] in ["enum", "typed"]: arr_s.pop(S_SCHEMA) else: arr_s.pop(S_EXTENDS) @@ -491,14 +530,20 @@ def build_schema(): if domain not in platforms: if manifest.config_schema is not None: core_components[domain] = {} + if len(manifest.dependencies) > 0: + core_components[domain]["dependencies"] = manifest.dependencies register_module_schemas(domain, manifest.module, manifest) for platform in platforms: platform_manifest = get_platform(domain=platform, platform=domain) if platform_manifest is not None: output[platform][S_COMPONENTS][domain] = {} + if len(platform_manifest.dependencies) > 0: + output[platform][S_COMPONENTS][domain][ + "dependencies" + ] = platform_manifest.dependencies register_module_schemas( - f"{domain}.{platform}", platform_manifest.module + f"{domain}.{platform}", platform_manifest.module, platform_manifest ) # Do registries @@ -517,6 +562,8 @@ def build_schema(): do_esp8266() do_esp32() fix_remote_receiver() + fix_script() + add_logger_tags() shrink() # aggregate components, so all component info is in same file, otherwise we have dallas.json, dallas.sensor.json, etc. @@ -585,7 +632,7 @@ def convert_1(schema, config_var, path): assert S_EXTENDS not in config_var if not S_TYPE in config_var: config_var[S_TYPE] = S_SCHEMA - assert config_var[S_TYPE] == S_SCHEMA + # assert config_var[S_TYPE] == S_SCHEMA if S_SCHEMA not in config_var: config_var[S_SCHEMA] = {} @@ -662,7 +709,7 @@ def convert_1(schema, config_var, path): elif repr_schema in ejs.hidden_schemas: schema_type = ejs.hidden_schemas[repr_schema] - data = schema(ejs.jschema_extractor) + data = schema(ejs.SCHEMA_EXTRACT) # enums, e.g. esp32/variant if schema_type == "one_of": @@ -672,8 +719,9 @@ def convert_1(schema, config_var, path): config_var[S_TYPE] = "enum" config_var["values"] = list(data.keys()) elif schema_type == "maybe": - config_var[S_TYPE] = "maybe" - config_var["schema"] = convert_config(data, path + "/maybe")["schema"] + config_var[S_TYPE] = S_SCHEMA + config_var["maybe"] = data[1] + config_var["schema"] = convert_config(data[0], path + "/maybe")["schema"] # esphome/on_boot elif schema_type == "automation": extra_schema = None @@ -717,8 +765,50 @@ def convert_1(schema, config_var, path): elif schema_type == "sensor": schema = data convert_1(data, config_var, path + "/trigger") + elif schema_type == "declare_id": + # pylint: disable=protected-access + parents = data._parents + + config_var["id_type"] = { + "class": str(data.base), + "parents": [str(x.base) for x in parents] + if isinstance(parents, list) + else None, + } + elif schema_type == "use_id": + if inspect.ismodule(data): + m_attr_obj = getattr(data, "CONFIG_SCHEMA") + use_schema = known_schemas.get(repr(m_attr_obj)) + if use_schema: + [output_module, output_name] = use_schema[0][1].split(".") + use_id_config = output[output_module][S_SCHEMAS][output_name] + config_var["use_id_type"] = use_id_config["schema"]["config_vars"][ + "id" + ]["id_type"]["class"] + config_var[S_TYPE] = "use_id" + else: + print("TODO deferred?") + else: + if isinstance(data, str): + # TODO: Figure out why pipsolar does this + config_var["use_id_type"] = data + else: + config_var["use_id_type"] = str(data.base) + config_var[S_TYPE] = "use_id" else: raise Exception("Unknown extracted schema type") + elif config_var.get("key") == "GeneratedID": + if path == "i2c/CONFIG_SCHEMA/extL/all/id": + config_var["id_type"] = {"class": "i2c::I2CBus", "parents": ["Component"]} + elif path == "uart/CONFIG_SCHEMA/val 1/extL/all/id": + config_var["id_type"] = { + "class": "uart::UARTComponent", + "parents": ["Component"], + } + elif path == "pins/esp32/val 1/id": + config_var["id_type"] = "pin" + else: + raise Exception("Cannot determine id_type for " + path) elif repr_schema in ejs.registry_schemas: solve_registry.append((ejs.registry_schemas[repr_schema], config_var)) @@ -787,7 +877,13 @@ def convert_keys(converted, schema, path): result["key"] = "Optional" else: converted["key"] = "String" - converted["key_dump"] = str(k) + key_string_match = re.search( + r"", str(k), re.IGNORECASE + ) + if key_string_match: + converted["key_type"] = key_string_match.group(1) + else: + converted["key_type"] = str(k) esphome_core.CORE.data = { esphome_core.KEY_CORE: {esphome_core.KEY_TARGET_PLATFORM: "esp8266"} @@ -808,6 +904,12 @@ def convert_keys(converted, schema, path): if base_k in result and base_v == result[base_k]: result.pop(base_k) converted["schema"][S_CONFIG_VARS][str(k)] = result + if "key" in converted and converted["key"] == "String": + config_vars = converted["schema"]["config_vars"] + assert len(config_vars) == 1 + key = list(config_vars.keys())[0] + assert key.startswith("<") + config_vars["string"] = config_vars.pop(key) build_schema()