String manipulation filters for text sensors (#2393)

* initial text sensor filter POC

* fixed verbose logging

* add append, prepend, substitute filters

* add to lower, get to upper working without dummy

* clang lint

* more linting...

* std::move append and prepend filters

* fix verbose filter::input logging

* value.c_str() in input print

* lambda filter verbose log fix

* correct log tag, neaten to upper and to lower

* add on_raw_value automation/trigger

esphome/components/text_sensor/__init__.py

@@ -4,16 +4,22 @@ from esphome import automation
 from esphome.components import mqtt
 from esphome.const import (
     CONF_DISABLED_BY_DEFAULT,
+    CONF_FILTERS,
     CONF_ICON,
     CONF_ID,
     CONF_INTERNAL,
     CONF_ON_VALUE,
+    CONF_ON_RAW_VALUE,
     CONF_TRIGGER_ID,
     CONF_MQTT_ID,
     CONF_NAME,
     CONF_STATE,
+    CONF_FROM,
+    CONF_TO,
 )
 from esphome.core import CORE, coroutine_with_priority
+from esphome.util import Registry
+
 
 IS_PLATFORM_COMPONENT = True
 
@@ -25,6 +31,9 @@ TextSensorPtr = TextSensor.operator("ptr")
 TextSensorStateTrigger = text_sensor_ns.class_(
     "TextSensorStateTrigger", automation.Trigger.template(cg.std_string)
 )
+TextSensorStateRawTrigger = text_sensor_ns.class_(
+    "TextSensorStateRawTrigger", automation.Trigger.template(cg.std_string)
+)
 TextSensorPublishAction = text_sensor_ns.class_(
     "TextSensorPublishAction", automation.Action
 )
@@ -32,21 +41,101 @@ TextSensorStateCondition = text_sensor_ns.class_(
     "TextSensorStateCondition", automation.Condition
 )
 
+FILTER_REGISTRY = Registry()
+validate_filters = cv.validate_registry("filter", FILTER_REGISTRY)
+
+# Filters
+Filter = text_sensor_ns.class_("Filter")
+LambdaFilter = text_sensor_ns.class_("LambdaFilter", Filter)
+ToUpperFilter = text_sensor_ns.class_("ToUpperFilter", Filter)
+ToLowerFilter = text_sensor_ns.class_("ToLowerFilter", Filter)
+AppendFilter = text_sensor_ns.class_("AppendFilter", Filter)
+PrependFilter = text_sensor_ns.class_("PrependFilter", Filter)
+SubstituteFilter = text_sensor_ns.class_("SubstituteFilter", Filter)
+
+
+@FILTER_REGISTRY.register("lambda", LambdaFilter, cv.returning_lambda)
+async def lambda_filter_to_code(config, filter_id):
+    lambda_ = await cg.process_lambda(
+        config, [(cg.std_string, "x")], return_type=cg.optional.template(cg.std_string)
+    )
+    return cg.new_Pvariable(filter_id, lambda_)
+
+
+@FILTER_REGISTRY.register("to_upper", ToUpperFilter, {})
+async def to_upper_filter_to_code(config, filter_id):
+    return cg.new_Pvariable(filter_id)
+
+
+@FILTER_REGISTRY.register("to_lower", ToLowerFilter, {})
+async def to_lower_filter_to_code(config, filter_id):
+    return cg.new_Pvariable(filter_id)
+
+
+@FILTER_REGISTRY.register("append", AppendFilter, cv.string)
+async def append_filter_to_code(config, filter_id):
+    return cg.new_Pvariable(filter_id, config)
+
+
+@FILTER_REGISTRY.register("prepend", PrependFilter, cv.string)
+async def prepend_filter_to_code(config, filter_id):
+    return cg.new_Pvariable(filter_id, config)
+
+
+def validate_substitute(value):
+    if isinstance(value, dict):
+        return cv.Schema(
+            {
+                cv.Required(CONF_FROM): cv.string,
+                cv.Required(CONF_TO): cv.string,
+            }
+        )(value)
+    value = cv.string(value)
+    if "->" not in value:
+        raise cv.Invalid("Substitute mapping must contain '->'")
+    a, b = value.split("->", 1)
+    a, b = a.strip(), b.strip()
+    return validate_substitute({CONF_FROM: cv.string(a), CONF_TO: cv.string(b)})
+
+
+@FILTER_REGISTRY.register(
+    "substitute",
+    SubstituteFilter,
+    cv.All(cv.ensure_list(validate_substitute), cv.Length(min=2)),
+)
+async def substitute_filter_to_code(config, filter_id):
+    from_strings = [conf[CONF_FROM] for conf in config]
+    to_strings = [conf[CONF_TO] for conf in config]
+    return cg.new_Pvariable(filter_id, from_strings, to_strings)
+
+
 icon = cv.icon
 
 TEXT_SENSOR_SCHEMA = cv.NAMEABLE_SCHEMA.extend(cv.MQTT_COMPONENT_SCHEMA).extend(
     {
         cv.OnlyWith(CONF_MQTT_ID, "mqtt"): cv.declare_id(mqtt.MQTTTextSensor),
         cv.Optional(CONF_ICON): icon,
+        cv.Optional(CONF_FILTERS): validate_filters,
         cv.Optional(CONF_ON_VALUE): automation.validate_automation(
             {
                 cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(TextSensorStateTrigger),
             }
         ),
+        cv.Optional(CONF_ON_RAW_VALUE): automation.validate_automation(
+            {
+                cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(
+                    TextSensorStateRawTrigger
+                ),
+            }
+        ),
     }
 )
 
 
+async def build_filters(config):
+    return await cg.build_registry_list(FILTER_REGISTRY, config)
+
+
 async def setup_text_sensor_core_(var, config):
     cg.add(var.set_name(config[CONF_NAME]))
     cg.add(var.set_disabled_by_default(config[CONF_DISABLED_BY_DEFAULT]))
@@ -55,10 +144,18 @@ async def setup_text_sensor_core_(var, config):
     if CONF_ICON in config:
         cg.add(var.set_icon(config[CONF_ICON]))
 
+    if config.get(CONF_FILTERS):  # must exist and not be empty
+        filters = await build_filters(config[CONF_FILTERS])
+        cg.add(var.set_filters(filters))
+
     for conf in config.get(CONF_ON_VALUE, []):
         trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
         await automation.build_automation(trigger, [(cg.std_string, "x")], conf)
 
+    for conf in config.get(CONF_ON_RAW_VALUE, []):
+        trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var)
+        await automation.build_automation(trigger, [(cg.std_string, "x")], conf)
+
     if CONF_MQTT_ID in config:
         mqtt_ = cg.new_Pvariable(config[CONF_MQTT_ID], var)
         await mqtt.register_mqtt_component(mqtt_, config)

esphome/components/text_sensor/automation.h

@@ -16,6 +16,13 @@ class TextSensorStateTrigger : public Trigger<std::string> {
   }
 };
 
+class TextSensorStateRawTrigger : public Trigger<std::string> {
+ public:
+  explicit TextSensorStateRawTrigger(TextSensor *parent) {
+    parent->add_on_raw_state_callback([this](std::string value) { this->trigger(std::move(value)); });
+  }
+};
+
 template<typename... Ts> class TextSensorStateCondition : public Condition<Ts...> {
  public:
   explicit TextSensorStateCondition(TextSensor *parent) : parent_(parent) {}

esphome/components/text_sensor/filter.cpp

@@ -0,0 +1,74 @@
+#include "filter.h"
+#include "text_sensor.h"
+#include "esphome/core/log.h"
+#include "esphome/core/hal.h"
+
+namespace esphome {
+namespace text_sensor {
+
+static const char *const TAG = "text_sensor.filter";
+
+// Filter
+void Filter::input(const std::string &value) {
+  ESP_LOGVV(TAG, "Filter(%p)::input(%s)", this, value.c_str());
+  optional<std::string> out = this->new_value(value);
+  if (out.has_value())
+    this->output(*out);
+}
+void Filter::output(const std::string &value) {
+  if (this->next_ == nullptr) {
+    ESP_LOGVV(TAG, "Filter(%p)::output(%s) -> SENSOR", this, value.c_str());
+    this->parent_->internal_send_state_to_frontend(value);
+  } else {
+    ESP_LOGVV(TAG, "Filter(%p)::output(%s) -> %p", this, value.c_str(), this->next_);
+    this->next_->input(value);
+  }
+}
+void Filter::initialize(TextSensor *parent, Filter *next) {
+  ESP_LOGVV(TAG, "Filter(%p)::initialize(parent=%p next=%p)", this, parent, next);
+  this->parent_ = parent;
+  this->next_ = next;
+}
+
+// LambdaFilter
+LambdaFilter::LambdaFilter(lambda_filter_t lambda_filter) : lambda_filter_(std::move(lambda_filter)) {}
+const lambda_filter_t &LambdaFilter::get_lambda_filter() const { return this->lambda_filter_; }
+void LambdaFilter::set_lambda_filter(const lambda_filter_t &lambda_filter) { this->lambda_filter_ = lambda_filter; }
+
+optional<std::string> LambdaFilter::new_value(std::string value) {
+  auto it = this->lambda_filter_(value);
+  ESP_LOGVV(TAG, "LambdaFilter(%p)::new_value(%s) -> %s", this, value.c_str(), it.value_or("").c_str());
+  return it;
+}
+
+// ToUpperFilter
+optional<std::string> ToUpperFilter::new_value(std::string value) {
+  for (char &c : value)
+    c = ::toupper(c);
+  return value;
+}
+
+// ToLowerFilter
+optional<std::string> ToLowerFilter::new_value(std::string value) {
+  for (char &c : value)
+    c = ::toupper(c);
+  return value;
+}
+
+// Append
+optional<std::string> AppendFilter::new_value(std::string value) { return value + this->suffix_; }
+
+// Prepend
+optional<std::string> PrependFilter::new_value(std::string value) { return this->prefix_ + value; }
+
+// Substitute
+optional<std::string> SubstituteFilter::new_value(std::string value) {
+  std::size_t pos;
+  for (int i = 0; i < this->from_strings_.size(); i++)
+    while ((pos = value.find(this->from_strings_[i])) != std::string::npos)
+      value.replace(pos, this->from_strings_[i].size(), this->to_strings_[i]);
+  return value;
+}
+
+}  // namespace text_sensor
+}  // namespace esphome

esphome/components/text_sensor/filter.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/helpers.h"
+#include <queue>
+#include <utility>
+
+namespace esphome {
+namespace text_sensor {
+
+class TextSensor;
+
+/** Apply a filter to text sensor values such as to_upper.
+ *
+ * This class is purposefully kept quite simple, since more complicated
+ * filters should really be done with the filter sensor in Home Assistant.
+ */
+class Filter {
+ public:
+  /** This will be called every time the filter receives a new value.
+   *
+   * It can return an empty optional to indicate that the filter chain
+   * should stop, otherwise the value in the filter will be passed down
+   * the chain.
+   *
+   * @param value The new value.
+   * @return An optional string, the new value that should be pushed out.
+   */
+  virtual optional<std::string> new_value(std::string value);
+
+  /// Initialize this filter, please note this can be called more than once.
+  virtual void initialize(TextSensor *parent, Filter *next);
+
+  void input(const std::string &value);
+
+  void output(const std::string &value);
+
+ protected:
+  friend TextSensor;
+
+  Filter *next_{nullptr};
+  TextSensor *parent_{nullptr};
+};
+
+using lambda_filter_t = std::function<optional<std::string>(std::string)>;
+
+/** This class allows for creation of simple template filters.
+ *
+ * The constructor accepts a lambda of the form std::string -> optional<std::string>.
+ * It will be called with each new value in the filter chain and returns the modified
+ * value that shall be passed down the filter chain. Returning an empty Optional
+ * means that the value shall be discarded.
+ */
+class LambdaFilter : public Filter {
+ public:
+  explicit LambdaFilter(lambda_filter_t lambda_filter);
+
+  optional<std::string> new_value(std::string value) override;
+
+  const lambda_filter_t &get_lambda_filter() const;
+  void set_lambda_filter(const lambda_filter_t &lambda_filter);
+
+ protected:
+  lambda_filter_t lambda_filter_;
+};
+
+/// A simple filter that converts all text to uppercase
+class ToUpperFilter : public Filter {
+ public:
+  optional<std::string> new_value(std::string value) override;
+};
+
+/// A simple filter that converts all text to lowercase
+class ToLowerFilter : public Filter {
+ public:
+  optional<std::string> new_value(std::string value) override;
+};
+
+/// A simple filter that adds a string to the end of another string
+class AppendFilter : public Filter {
+ public:
+  AppendFilter(std::string suffix) : suffix_(std::move(suffix)) {}
+  optional<std::string> new_value(std::string value) override;
+
+ protected:
+  std::string suffix_;
+};
+
+/// A simple filter that adds a string to the start of another string
+class PrependFilter : public Filter {
+ public:
+  PrependFilter(std::string prefix) : prefix_(std::move(prefix)) {}
+  optional<std::string> new_value(std::string value) override;
+
+ protected:
+  std::string prefix_;
+};
+
+/// A simple filter that replaces a substring with another substring
+class SubstituteFilter : public Filter {
+ public:
+  SubstituteFilter(std::vector<std::string> from_strings, std::vector<std::string> to_strings)
+      : from_strings_(std::move(from_strings)), to_strings_(std::move(to_strings)) {}
+  optional<std::string> new_value(std::string value) override;
+
+ protected:
+  std::vector<std::string> from_strings_;
+  std::vector<std::string> to_strings_;
+};
+
+}  // namespace text_sensor
+}  // namespace esphome

esphome/components/text_sensor/text_sensor.cpp

@@ -10,21 +10,72 @@ TextSensor::TextSensor() : TextSensor("") {}
 TextSensor::TextSensor(const std::string &name) : Nameable(name) {}
 
 void TextSensor::publish_state(const std::string &state) {
-  this->state = state;
+  this->raw_state = state;
+  this->raw_callback_.call(state);
+
+  ESP_LOGV(TAG, "'%s': Received new state %s", this->name_.c_str(), state.c_str());
+
+  if (this->filter_list_ == nullptr) {
+    this->internal_send_state_to_frontend(state);
+  } else {
+    this->filter_list_->input(state);
+  }
+}
+
+void TextSensor::add_filter(Filter *filter) {
+  // inefficient, but only happens once on every sensor setup and nobody's going to have massive amounts of
+  // filters
+  ESP_LOGVV(TAG, "TextSensor(%p)::add_filter(%p)", this, filter);
+  if (this->filter_list_ == nullptr) {
+    this->filter_list_ = filter;
+  } else {
+    Filter *last_filter = this->filter_list_;
+    while (last_filter->next_ != nullptr)
+      last_filter = last_filter->next_;
+    last_filter->initialize(this, filter);
+  }
+  filter->initialize(this, nullptr);
+}
+void TextSensor::add_filters(const std::vector<Filter *> &filters) {
+  for (Filter *filter : filters) {
+    this->add_filter(filter);
+  }
+}
+void TextSensor::set_filters(const std::vector<Filter *> &filters) {
+  this->clear_filters();
+  this->add_filters(filters);
+}
+void TextSensor::clear_filters() {
+  if (this->filter_list_ != nullptr) {
+    ESP_LOGVV(TAG, "TextSensor(%p)::clear_filters()", this);
+  }
+  this->filter_list_ = nullptr;
+}
+
+void TextSensor::add_on_state_callback(std::function<void(std::string)> callback) {
+  this->callback_.add(std::move(callback));
+}
+void TextSensor::add_on_raw_state_callback(std::function<void(std::string)> callback) {
+  this->raw_callback_.add(std::move(callback));
+}
+
+std::string TextSensor::get_state() const { return this->state; }
+std::string TextSensor::get_raw_state() const { return this->raw_state; }
+void TextSensor::internal_send_state_to_frontend(const std::string &state) {
+  this->state = this->raw_state;
   this->has_state_ = true;
   ESP_LOGD(TAG, "'%s': Sending state '%s'", this->name_.c_str(), state.c_str());
   this->callback_.call(state);
 }
+
 void TextSensor::set_icon(const std::string &icon) { this->icon_ = icon; }
-void TextSensor::add_on_state_callback(std::function<void(std::string)> callback) {
-  this->callback_.add(std::move(callback));
-}
 std::string TextSensor::get_icon() {
   if (this->icon_.has_value())
     return *this->icon_;
   return this->icon();
 }
 std::string TextSensor::icon() { return ""; }
+
 std::string TextSensor::unique_id() { return ""; }
 bool TextSensor::has_state() { return this->has_state_; }
 uint32_t TextSensor::hash_base() { return 334300109UL; }

esphome/components/text_sensor/text_sensor.h

@@ -2,6 +2,7 @@
 
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
+#include "esphome/components/text_sensor/filter.h"
 
 namespace esphome {
 namespace text_sensor {
@@ -22,13 +23,33 @@ class TextSensor : public Nameable {
   explicit TextSensor();
   explicit TextSensor(const std::string &name);
 
+  /// Getter-syntax for .state.
+  std::string get_state() const;
+  /// Getter-syntax for .raw_state
+  std::string get_raw_state() const;
+
   void publish_state(const std::string &state);
 
   void set_icon(const std::string &icon);
 
+  /// Add a filter to the filter chain. Will be appended to the back.
+  void add_filter(Filter *filter);
+
+  /// Add a list of vectors to the back of the filter chain.
+  void add_filters(const std::vector<Filter *> &filters);
+
+  /// Clear the filters and replace them by filters.
+  void set_filters(const std::vector<Filter *> &filters);
+
+  /// Clear the entire filter chain.
+  void clear_filters();
+
   void add_on_state_callback(std::function<void(std::string)> callback);
+  /// Add a callback that will be called every time the sensor sends a raw value.
+  void add_on_raw_state_callback(std::function<void(std::string)> callback);
 
   std::string state;
+  std::string raw_state;
 
   // ========== INTERNAL METHODS ==========
   // (In most use cases you won't need these)
@@ -40,10 +61,16 @@ class TextSensor : public Nameable {
 
   bool has_state();
 
+  void internal_send_state_to_frontend(const std::string &state);
+
  protected:
   uint32_t hash_base() override;
 
-  CallbackManager<void(std::string)> callback_;
+  CallbackManager<void(std::string)> raw_callback_;  ///< Storage for raw state callbacks.
+  CallbackManager<void(std::string)> callback_;      ///< Storage for filtered state callbacks.
+
+  Filter *filter_list_{nullptr};  ///< Store all active filters.
+
   optional<std::string> icon_;
   bool has_state_{false};
 };