Add API message for setting noise psk

Allow no encryption key in yaml
Alter mdns and mqtt device info to show supported encryption when not currently in use
Add test

esphome/components/api/__init__.py

@@ -82,6 +82,19 @@ ACTIONS_SCHEMA = automation.validate_automation(
     ),
 )
 
+ENCRYPTION_SCHEMA = cv.Schema(
+    {
+        cv.Optional(CONF_KEY): validate_encryption_key,
+    }
+)
+
+
+def _encryption_schema(config):
+    if config is None:
+        config = {}
+    return ENCRYPTION_SCHEMA(config)
+
+
 CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
@@ -95,11 +108,7 @@ CONFIG_SCHEMA = cv.All(
                 CONF_SERVICES, group_of_exclusion=CONF_ACTIONS
             ): ACTIONS_SCHEMA,
             cv.Exclusive(CONF_ACTIONS, group_of_exclusion=CONF_ACTIONS): ACTIONS_SCHEMA,
-            cv.Optional(CONF_ENCRYPTION): cv.Schema(
+            cv.Optional(CONF_ENCRYPTION): _encryption_schema,
-                {
-                    cv.Required(CONF_KEY): validate_encryption_key,
-                }
-            ),
             cv.Optional(CONF_ON_CLIENT_CONNECTED): automation.validate_automation(
                 single=True
             ),
@@ -151,9 +160,17 @@ async def to_code(config):
             config[CONF_ON_CLIENT_DISCONNECTED],
         )
 
-    if encryption_config := config.get(CONF_ENCRYPTION):
+    if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None:
-        decoded = base64.b64decode(encryption_config[CONF_KEY])
+        if key := encryption_config.get(CONF_KEY):
-        cg.add(var.set_noise_psk(list(decoded)))
+            decoded = base64.b64decode(key)
+            cg.add(var.set_noise_psk(list(decoded)))
+        else:
+            # No key provided, but encryption desired
+            # This will allow a plaintext client to provide a noise key,
+            # send it to the device, and then switch to noise.
+            # The key will be saved in flash and used for future connections
+            # and plaintext disabled. Only a factory reset can remove it.
+            cg.add_define("USE_API_PLAINTEXT")
         cg.add_define("USE_API_NOISE")
         cg.add_library("esphome/noise-c", "0.1.6")
     else:

esphome/components/api/api.proto

@@ -31,6 +31,7 @@ service APIConnection {
     option (needs_authentication) = false;
   }
   rpc execute_service (ExecuteServiceRequest) returns (void) {}
+  rpc noise_encryption_set_key (NoiseEncryptionSetKeyRequest) returns (NoiseEncryptionSetKeyResponse) {}
 
   rpc cover_command (CoverCommandRequest) returns (void) {}
   rpc fan_command (FanCommandRequest) returns (void) {}
@@ -225,6 +226,9 @@ message DeviceInfoResponse {
   uint32 voice_assistant_feature_flags = 17;
 
   string suggested_area = 16;
+
+  // Supports receiving and saving api encryption key
+  bool api_encryption_supported = 18;
 }
 
 message ListEntitiesRequest {
@@ -649,6 +653,23 @@ message SubscribeLogsResponse {
   bool send_failed = 4;
 }
 
+// ==================== NOISE ENCRYPTION ====================
+message NoiseEncryptionSetKeyRequest {
+  option (id) = 119;
+  option (source) = SOURCE_CLIENT;
+  option (ifdef) = "USE_API_NOISE";
+
+  bytes key = 1;
+}
+
+message NoiseEncryptionSetKeyResponse {
+  option (id) = 120;
+  option (source) = SOURCE_SERVER;
+  option (ifdef) = "USE_API_NOISE";
+
+  bool success = 1;
+}
+
 // ==================== HOMEASSISTANT.SERVICE ====================
 message SubscribeHomeassistantServicesRequest {
   option (id) = 34;

esphome/components/api/api_connection.cpp

@@ -31,7 +31,14 @@ APIConnection::APIConnection(std::unique_ptr<socket::Socket> sock, APIServer *pa
     : parent_(parent), initial_state_iterator_(this), list_entities_iterator_(this) {
   this->proto_write_buffer_.reserve(64);
 
-#if defined(USE_API_PLAINTEXT)
+#if defined(USE_API_PLAINTEXT) && defined(USE_API_NOISE)
+  auto noise_ctx = parent->get_noise_ctx();
+  if (noise_ctx->has_psk()) {
+    this->helper_ = std::unique_ptr<APIFrameHelper>{new APINoiseFrameHelper(std::move(sock), noise_ctx)};
+  } else {
+    this->helper_ = std::unique_ptr<APIFrameHelper>{new APIPlaintextFrameHelper(std::move(sock))};
+  }
+#elif defined(USE_API_PLAINTEXT)
   this->helper_ = std::unique_ptr<APIFrameHelper>{new APIPlaintextFrameHelper(std::move(sock))};
 #elif defined(USE_API_NOISE)
   this->helper_ = std::unique_ptr<APIFrameHelper>{new APINoiseFrameHelper(std::move(sock), parent->get_noise_ctx())};
@@ -1434,6 +1441,9 @@ DeviceInfoResponse APIConnection::device_info(const DeviceInfoRequest &msg) {
 #ifdef USE_VOICE_ASSISTANT
   resp.legacy_voice_assistant_version = voice_assistant::global_voice_assistant->get_legacy_version();
   resp.voice_assistant_feature_flags = voice_assistant::global_voice_assistant->get_feature_flags();
+#endif
+#ifdef USE_API_NOISE
+  resp.api_encryption_supported = true;
 #endif
   return resp;
 }
@@ -1455,6 +1465,25 @@ void APIConnection::execute_service(const ExecuteServiceRequest &msg) {
     ESP_LOGV(TAG, "Could not find matching service!");
   }
 }
+NoiseEncryptionSetKeyResponse APIConnection::noise_encryption_set_key(const NoiseEncryptionSetKeyRequest &msg) {
+  psk_t psk{};
+  NoiseEncryptionSetKeyResponse resp;
+  if (base64_decode(msg.key.data(), psk.data(), msg.key.size()) != psk.size()) {
+    ESP_LOGW(TAG, "Invalid encryption key length");
+    resp.success = false;
+    return resp;
+  }
+
+  if (!this->parent_->save_noise_psk(psk, true)) {
+    ESP_LOGW(TAG, "Failed to save encryption key");
+    resp.success = false;
+    return resp;
+  }
+
+  resp.success = true;
+  return resp;
+}
+
 void APIConnection::subscribe_home_assistant_states(const SubscribeHomeAssistantStatesRequest &msg) {
   state_subs_at_ = 0;
 }

esphome/components/api/api_connection.h

@@ -204,6 +204,9 @@ class APIConnection : public APIServerConnection {
     return {};
   }
   void execute_service(const ExecuteServiceRequest &msg) override;
+#ifdef USE_API_NOISE
+  NoiseEncryptionSetKeyResponse noise_encryption_set_key(const NoiseEncryptionSetKeyRequest &msg) override;
+#endif
 
   bool is_authenticated() override { return this->connection_state_ == ConnectionState::AUTHENTICATED; }
   bool is_connection_setup() override {

esphome/components/api/api_noise_context.h

@@ -1,6 +1,6 @@
 #pragma once
-#include <cstdint>
 #include <array>
+#include <cstdint>
 #include "esphome/core/defines.h"
 
 namespace esphome {
@@ -11,11 +11,20 @@ using psk_t = std::array<uint8_t, 32>;
 
 class APINoiseContext {
  public:
-  void set_psk(psk_t psk) { psk_ = psk; }
+  void set_psk(psk_t psk) {
-  const psk_t &get_psk() const { return psk_; }
+    this->psk_ = psk;
+    bool has_psk = false;
+    for (auto i : psk) {
+      has_psk |= i;
+    }
+    this->has_psk_ = has_psk;
+  }
+  const psk_t &get_psk() const { return this->psk_; }
+  const bool has_psk() const { return this->has_psk_; }
 
  protected:
-  psk_t psk_;
+  psk_t psk_{};
+  bool has_psk_{false};
 };
 #endif  // USE_API_NOISE
 

esphome/components/api/api_pb2.cpp

@@ -780,6 +780,10 @@ bool DeviceInfoResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
       this->voice_assistant_feature_flags = value.as_uint32();
       return true;
     }
+    case 18: {
+      this->api_encryption_supported = value.as_bool();
+      return true;
+    }
     default:
       return false;
   }
@@ -848,6 +852,7 @@ void DeviceInfoResponse::encode(ProtoWriteBuffer buffer) const {
   buffer.encode_uint32(14, this->legacy_voice_assistant_version);
   buffer.encode_uint32(17, this->voice_assistant_feature_flags);
   buffer.encode_string(16, this->suggested_area);
+  buffer.encode_bool(18, this->api_encryption_supported);
 }
 #ifdef HAS_PROTO_MESSAGE_DUMP
 void DeviceInfoResponse::dump_to(std::string &out) const {
@@ -925,6 +930,10 @@ void DeviceInfoResponse::dump_to(std::string &out) const {
   out.append("  suggested_area: ");
   out.append("'").append(this->suggested_area).append("'");
   out.append("\n");
+
+  out.append("  api_encryption_supported: ");
+  out.append(YESNO(this->api_encryption_supported));
+  out.append("\n");
   out.append("}");
 }
 #endif
@@ -2988,6 +2997,48 @@ void SubscribeLogsResponse::dump_to(std::string &out) const {
   out.append("}");
 }
 #endif
+bool NoiseEncryptionSetKeyRequest::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
+  switch (field_id) {
+    case 1: {
+      this->key = value.as_string();
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+void NoiseEncryptionSetKeyRequest::encode(ProtoWriteBuffer buffer) const { buffer.encode_string(1, this->key); }
+#ifdef HAS_PROTO_MESSAGE_DUMP
+void NoiseEncryptionSetKeyRequest::dump_to(std::string &out) const {
+  __attribute__((unused)) char buffer[64];
+  out.append("NoiseEncryptionSetKeyRequest {\n");
+  out.append("  key: ");
+  out.append("'").append(this->key).append("'");
+  out.append("\n");
+  out.append("}");
+}
+#endif
+bool NoiseEncryptionSetKeyResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
+  switch (field_id) {
+    case 1: {
+      this->success = value.as_bool();
+      return true;
+    }
+    default:
+      return false;
+  }
+}
+void NoiseEncryptionSetKeyResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_bool(1, this->success); }
+#ifdef HAS_PROTO_MESSAGE_DUMP
+void NoiseEncryptionSetKeyResponse::dump_to(std::string &out) const {
+  __attribute__((unused)) char buffer[64];
+  out.append("NoiseEncryptionSetKeyResponse {\n");
+  out.append("  success: ");
+  out.append(YESNO(this->success));
+  out.append("\n");
+  out.append("}");
+}
+#endif
 void SubscribeHomeassistantServicesRequest::encode(ProtoWriteBuffer buffer) const {}
 #ifdef HAS_PROTO_MESSAGE_DUMP
 void SubscribeHomeassistantServicesRequest::dump_to(std::string &out) const {

esphome/components/api/api_pb2.h

@@ -350,6 +350,7 @@ class DeviceInfoResponse : public ProtoMessage {
   uint32_t legacy_voice_assistant_version{0};
   uint32_t voice_assistant_feature_flags{0};
   std::string suggested_area{};
+  bool api_encryption_supported{false};
   void encode(ProtoWriteBuffer buffer) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   void dump_to(std::string &out) const override;
@@ -786,6 +787,28 @@ class SubscribeLogsResponse : public ProtoMessage {
   bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override;
   bool decode_varint(uint32_t field_id, ProtoVarInt value) override;
 };
+class NoiseEncryptionSetKeyRequest : public ProtoMessage {
+ public:
+  std::string key{};
+  void encode(ProtoWriteBuffer buffer) const override;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  void dump_to(std::string &out) const override;
+#endif
+
+ protected:
+  bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override;
+};
+class NoiseEncryptionSetKeyResponse : public ProtoMessage {
+ public:
+  bool success{false};
+  void encode(ProtoWriteBuffer buffer) const override;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  void dump_to(std::string &out) const override;
+#endif
+
+ protected:
+  bool decode_varint(uint32_t field_id, ProtoVarInt value) override;
+};
 class SubscribeHomeassistantServicesRequest : public ProtoMessage {
  public:
   void encode(ProtoWriteBuffer buffer) const override;

esphome/components/api/api_pb2_service.cpp

@@ -179,6 +179,16 @@ bool APIServerConnectionBase::send_text_sensor_state_response(const TextSensorSt
 bool APIServerConnectionBase::send_subscribe_logs_response(const SubscribeLogsResponse &msg) {
   return this->send_message_<SubscribeLogsResponse>(msg, 29);
 }
+#ifdef USE_API_NOISE
+#endif
+#ifdef USE_API_NOISE
+bool APIServerConnectionBase::send_noise_encryption_set_key_response(const NoiseEncryptionSetKeyResponse &msg) {
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  ESP_LOGVV(TAG, "send_noise_encryption_set_key_response: %s", msg.dump().c_str());
+#endif
+  return this->send_message_<NoiseEncryptionSetKeyResponse>(msg, 120);
+}
+#endif
 bool APIServerConnectionBase::send_homeassistant_service_response(const HomeassistantServiceResponse &msg) {
 #ifdef HAS_PROTO_MESSAGE_DUMP
   ESP_LOGVV(TAG, "send_homeassistant_service_response: %s", msg.dump().c_str());
@@ -1135,6 +1145,17 @@ bool APIServerConnectionBase::read_message(uint32_t msg_size, uint32_t msg_type,
       ESP_LOGVV(TAG, "on_update_command_request: %s", msg.dump().c_str());
 #endif
       this->on_update_command_request(msg);
+#endif
+      break;
+    }
+    case 119: {
+#ifdef USE_API_NOISE
+      NoiseEncryptionSetKeyRequest msg;
+      msg.decode(msg_data, msg_size);
+#ifdef HAS_PROTO_MESSAGE_DUMP
+      ESP_LOGVV(TAG, "on_noise_encryption_set_key_request: %s", msg.dump().c_str());
+#endif
+      this->on_noise_encryption_set_key_request(msg);
 #endif
       break;
     }
@@ -1255,6 +1276,22 @@ void APIServerConnection::on_execute_service_request(const ExecuteServiceRequest
   }
   this->execute_service(msg);
 }
+#ifdef USE_API_NOISE
+void APIServerConnection::on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &msg) {
+  if (!this->is_connection_setup()) {
+    this->on_no_setup_connection();
+    return;
+  }
+  if (!this->is_authenticated()) {
+    this->on_unauthenticated_access();
+    return;
+  }
+  NoiseEncryptionSetKeyResponse ret = this->noise_encryption_set_key(msg);
+  if (!this->send_noise_encryption_set_key_response(ret)) {
+    this->on_fatal_error();
+  }
+}
+#endif
 #ifdef USE_COVER
 void APIServerConnection::on_cover_command_request(const CoverCommandRequest &msg) {
   if (!this->is_connection_setup()) {

esphome/components/api/api_pb2_service.h

@@ -83,6 +83,12 @@ class APIServerConnectionBase : public ProtoService {
 #endif
   virtual void on_subscribe_logs_request(const SubscribeLogsRequest &value){};
   bool send_subscribe_logs_response(const SubscribeLogsResponse &msg);
+#ifdef USE_API_NOISE
+  virtual void on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &value){};
+#endif
+#ifdef USE_API_NOISE
+  bool send_noise_encryption_set_key_response(const NoiseEncryptionSetKeyResponse &msg);
+#endif
   virtual void on_subscribe_homeassistant_services_request(const SubscribeHomeassistantServicesRequest &value){};
   bool send_homeassistant_service_response(const HomeassistantServiceResponse &msg);
   virtual void on_subscribe_home_assistant_states_request(const SubscribeHomeAssistantStatesRequest &value){};
@@ -334,6 +340,9 @@ class APIServerConnection : public APIServerConnectionBase {
   virtual void subscribe_home_assistant_states(const SubscribeHomeAssistantStatesRequest &msg) = 0;
   virtual GetTimeResponse get_time(const GetTimeRequest &msg) = 0;
   virtual void execute_service(const ExecuteServiceRequest &msg) = 0;
+#ifdef USE_API_NOISE
+  virtual NoiseEncryptionSetKeyResponse noise_encryption_set_key(const NoiseEncryptionSetKeyRequest &msg) = 0;
+#endif
 #ifdef USE_COVER
   virtual void cover_command(const CoverCommandRequest &msg) = 0;
 #endif
@@ -435,6 +444,9 @@ class APIServerConnection : public APIServerConnectionBase {
   void on_subscribe_home_assistant_states_request(const SubscribeHomeAssistantStatesRequest &msg) override;
   void on_get_time_request(const GetTimeRequest &msg) override;
   void on_execute_service_request(const ExecuteServiceRequest &msg) override;
+#ifdef USE_API_NOISE
+  void on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &msg) override;
+#endif
 #ifdef USE_COVER
   void on_cover_command_request(const CoverCommandRequest &msg) override;
 #endif

esphome/components/api/api_server.cpp

@@ -159,7 +159,12 @@ void APIServer::dump_config() {
   ESP_LOGCONFIG(TAG, "API Server:");
   ESP_LOGCONFIG(TAG, "  Address: %s: %u", network::get_use_address().c_str(), this->port_);
 #ifdef USE_API_NOISE
-  ESP_LOGCONFIG(TAG, "  Using noise encryption: YES");
+  if (this->noise_ctx_->has_psk()) {
+    ESP_LOGCONFIG(TAG, "  Using noise encryption: YES");
+  } else {
+    ESP_LOGCONFIG(TAG, "  Using noise encryption: NO");
+    ESP_LOGCONFIG(TAG, "  Supports noise encryption: YES");
+  }
 #else
   ESP_LOGCONFIG(TAG, "  Using noise encryption: NO");
 #endif
@@ -415,24 +420,34 @@ uint16_t APIServer::get_port() const { return this->port_; }
 void APIServer::set_reboot_timeout(uint32_t reboot_timeout) { this->reboot_timeout_ = reboot_timeout; }
 
 #ifdef USE_API_NOISE
-void APIServer::save_noise_psk(psk_t psk, bool make_active) {
+bool APIServer::save_noise_psk(psk_t psk, bool make_active) {
-  if (psk == this->noise_ctx_->get_psk()) {
+  auto &old_psk = this->noise_ctx_->get_psk();
-    ESP_LOGW(TAG, "New PSK matches old; doing nothing");
+  if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) {
-    return;
+    ESP_LOGW(TAG, "New PSK matches old");
+    return true;
   }
 
   SavedNoisePsk new_saved_psk{psk};
-  this->noise_pref_.save(&new_saved_psk);
+  if (!this->noise_pref_.save(&new_saved_psk)) {
-  // ensure it's written immediately
+    ESP_LOGW(TAG, "Failed to save Noise PSK");
-  global_preferences->sync();
+    return false;
-  ESP_LOGD(TAG, "Noise PSK updated");
-  if (make_active) {
-    ESP_LOGW(TAG, "Disconnecting all clients to reset connections");
-    this->set_noise_psk(psk);
-    for (auto &c : this->clients_) {
-      c->send_disconnect_request(DisconnectRequest());
-    }
   }
+  // ensure it's written immediately
+  if (!global_preferences->sync()) {
+    ESP_LOGW(TAG, "Failed to sync preferences");
+    return false;
+  }
+  ESP_LOGD(TAG, "Noise PSK saved");
+  if (make_active) {
+    this->set_timeout(100, [this, psk]() {
+      ESP_LOGW(TAG, "Disconnecting all clients to reset connections");
+      this->set_noise_psk(psk);
+      for (auto &c : this->clients_) {
+        c->send_disconnect_request(DisconnectRequest());
+      }
+    });
+  }
+  return true;
 }
 #endif
 

esphome/components/api/api_server.h

@@ -40,7 +40,7 @@ class APIServer : public Component, public Controller {
   void set_reboot_timeout(uint32_t reboot_timeout);
 
 #ifdef USE_API_NOISE
-  void save_noise_psk(psk_t psk, bool make_active = true);
+  bool save_noise_psk(psk_t psk, bool make_active = true);
   void set_noise_psk(psk_t psk) { noise_ctx_->set_psk(psk); }
   std::shared_ptr<APINoiseContext> get_noise_ctx() { return noise_ctx_; }
 #endif  // USE_API_NOISE

esphome/components/mdns/mdns_component.cpp

@@ -1,9 +1,9 @@
 #include "esphome/core/defines.h"
 #ifdef USE_MDNS
-#include "mdns_component.h"
-#include "esphome/core/version.h"
 #include "esphome/core/application.h"
 #include "esphome/core/log.h"
+#include "esphome/core/version.h"
+#include "mdns_component.h"
 
 #ifdef USE_API
 #include "esphome/components/api/api_server.h"
@@ -62,7 +62,11 @@ void MDNSComponent::compile_records_() {
 #endif
 
 #ifdef USE_API_NOISE
-    service.txt_records.push_back({"api_encryption", "Noise_NNpsk0_25519_ChaChaPoly_SHA256"});
+    if (api::global_api_server->get_noise_ctx()->has_psk()) {
+      service.txt_records.push_back({"api_encryption", "Noise_NNpsk0_25519_ChaChaPoly_SHA256"});
+    } else {
+      service.txt_records.push_back({"api_encryption_supported", "Noise_NNpsk0_25519_ChaChaPoly_SHA256"});
+    }
 #endif
 
 #ifdef ESPHOME_PROJECT_NAME

esphome/components/mqtt/mqtt_client.cpp

@@ -135,7 +135,11 @@ void MQTTClientComponent::send_device_info_() {
 #endif
 
 #ifdef USE_API_NOISE
-        root["api_encryption"] = "Noise_NNpsk0_25519_ChaChaPoly_SHA256";
+        if (api::global_api_server->get_noise_ctx()->has_psk()) {
+          root["api_encryption"] = "Noise_NNpsk0_25519_ChaChaPoly_SHA256";
+        } else {
+          root["api_encryption_supported"] = "Noise_NNpsk0_25519_ChaChaPoly_SHA256";
+        }
 #endif
       },
       2, this->discovery_info_.retain);

tests/components/api/test-dynamic-encryption.esp32-idf.yaml

@@ -0,0 +1,10 @@
+packages:
+  common: !include common.yaml
+
+wifi:
+  ssid: MySSID
+  password: password1
+
+api:
+  encryption:
+    key: !remove