diff --git a/esphome/components/remote_receiver/__init__.py b/esphome/components/remote_receiver/__init__.py
index e5085bb33c..704538e8c5 100644
--- a/esphome/components/remote_receiver/__init__.py
+++ b/esphome/components/remote_receiver/__init__.py
@@ -1,18 +1,18 @@
-import esphome.codegen as cg
-import esphome.config_validation as cv
 from esphome import pins
-from esphome.components import remote_base, esp32_rmt
+import esphome.codegen as cg
+from esphome.components import esp32_rmt, remote_base
+import esphome.config_validation as cv
 from esphome.const import (
     CONF_BUFFER_SIZE,
     CONF_DUMP,
     CONF_FILTER,
     CONF_ID,
     CONF_IDLE,
+    CONF_MEMORY_BLOCKS,
     CONF_PIN,
+    CONF_RMT_CHANNEL,
     CONF_TOLERANCE,
     CONF_TYPE,
-    CONF_MEMORY_BLOCKS,
-    CONF_RMT_CHANNEL,
     CONF_VALUE,
 )
 from esphome.core import CORE, TimePeriod
@@ -115,12 +115,7 @@ CONFIG_SCHEMA = remote_base.validate_triggers(
 async def to_code(config):
     pin = await cg.gpio_pin_expression(config[CONF_PIN])
     if CORE.is_esp32:
-        if (rmt_channel := config.get(CONF_RMT_CHANNEL, None)) is not None:
-            var = cg.new_Pvariable(
-                config[CONF_ID], pin, rmt_channel, config[CONF_MEMORY_BLOCKS]
-            )
-        else:
-            var = cg.new_Pvariable(config[CONF_ID], pin, config[CONF_MEMORY_BLOCKS])
+        var = cg.new_Pvariable(config[CONF_ID], pin, config[CONF_MEMORY_BLOCKS])
         cg.add(var.set_clock_divider(config[CONF_CLOCK_DIVIDER]))
     else:
         var = cg.new_Pvariable(config[CONF_ID], pin)
diff --git a/esphome/components/remote_receiver/remote_receiver.h b/esphome/components/remote_receiver/remote_receiver.h
index 28742792d7..dc89ef6ae7 100644
--- a/esphome/components/remote_receiver/remote_receiver.h
+++ b/esphome/components/remote_receiver/remote_receiver.h
@@ -29,15 +29,17 @@ struct RemoteReceiverComponentStore {
   uint32_t filter_us{10};
   ISRInternalGPIOPin pin;
 #elif defined(USE_ESP32)
-  uint8_t *buffer{nullptr};
+  /// Stores RMT symbols and rx done event data
+  volatile uint8_t *buffer{nullptr};
+  /// The position last written to
   volatile uint32_t buffer_write_at{0};
+  /// The position last read from
   uint32_t buffer_read_at{0};
   uint32_t buffer_size{1000};
-  uint32_t max_size{0};
-  rmt_rx_done_event_data_t data;
+  uint32_t receive_size{0};
+  esp_err_t error{ESP_OK};
   rmt_receive_config_t config;
   rmt_channel_handle_t channel;
-  esp_err_t error{ESP_OK};
 #endif
 };
 
diff --git a/esphome/components/remote_receiver/remote_receiver_esp32.cpp b/esphome/components/remote_receiver/remote_receiver_esp32.cpp
index 8272a72260..6b111ab598 100644
--- a/esphome/components/remote_receiver/remote_receiver_esp32.cpp
+++ b/esphome/components/remote_receiver/remote_receiver_esp32.cpp
@@ -7,17 +7,17 @@ namespace esphome {
 namespace remote_receiver {
 
 static const char *const TAG = "remote_receiver.esp32";
+static const uint32_t MEM_BLOCK_SIZE = 64u;
 
-static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_rx_done_event_data_t *data,
-                                       void *user_data) {
-  RemoteReceiverComponentStore *store = (RemoteReceiverComponentStore *) user_data;
+static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_rx_done_event_data_t *data, void *arg) {
+  RemoteReceiverComponentStore *store = (RemoteReceiverComponentStore *) arg;
   uint32_t next =
       store->buffer_write_at + sizeof(rmt_rx_done_event_data_t) + data->num_symbols * sizeof(rmt_symbol_word_t);
-  if ((next + store->max_size) > store->buffer_size) {
+  if ((next + sizeof(rmt_rx_done_event_data_t) + store->receive_size) > store->buffer_size) {
     next = 0;
   }
-  store->error = rmt_receive(store->channel, store->buffer + next + sizeof(rmt_rx_done_event_data_t),
-                             sizeof(rmt_symbol_word_t) * 64 * 4, &store->config);
+  store->error = rmt_receive(store->channel, (void *) (store->buffer + next + sizeof(rmt_rx_done_event_data_t)),
+                             store->receive_size, &store->config);
   *(rmt_rx_done_event_data_t *) (store->buffer + store->buffer_write_at) = *data;
   store->buffer_write_at = next;
   return false;
@@ -25,20 +25,12 @@ static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_r
 
 void RemoteReceiverComponent::setup() {
   ESP_LOGCONFIG(TAG, "Setting up Remote Receiver...");
-  this->pin_->setup();
-  if (this->filter_us_ == 0) {
-    this->store_.config.signal_range_min_ns = 0;
-  } else {
-    this->store_.config.signal_range_min_ns = 500;  // std::min(this->filter_us_, (uint32_t) 255) * 1000;
-  }
-  this->store_.config.signal_range_max_ns = std::min(this->idle_us_, (uint32_t) 65535) * 1000;
-
-  rmt_rx_channel_config_t rmt{};
-  rmt.clk_src = RMT_CLK_SRC_DEFAULT;
-  rmt.resolution_hz = 1 * 1000 * 1000;
-  rmt.mem_block_symbols = 64 * this->mem_block_num_;
-  rmt.gpio_num = gpio_num_t(this->pin_->get_pin());
-  esp_err_t error = rmt_new_rx_channel(&rmt, &this->channel_);
+  rmt_rx_channel_config_t channel{};
+  channel.clk_src = RMT_CLK_SRC_DEFAULT;
+  channel.resolution_hz = 1 * 1000 * 1000;
+  channel.mem_block_symbols = MEM_BLOCK_SIZE * this->mem_block_num_;
+  channel.gpio_num = gpio_num_t(this->pin_->get_pin());
+  esp_err_t error = rmt_new_rx_channel(&channel, &this->channel_);
   if (error != ESP_OK) {
     this->error_code_ = error;
     this->error_string_ = "in rmt_new_rx_channel";
@@ -63,14 +55,16 @@ void RemoteReceiverComponent::setup() {
     return;
   }
 
+  uint32_t max_filter_ns = 255u * 1000 / this->clock_divider_;
+  uint32_t max_idle_ns = 65535u * 1000;
+  this->store_.config.signal_range_min_ns = std::min(this->filter_us_ * 1000, max_filter_ns);
+  this->store_.config.signal_range_max_ns = std::min(this->idle_us_ * 1000, max_idle_ns);
   this->store_.channel = this->channel_;
-  this->store_.buffer = (uint8_t *) new uint32_t[this->buffer_size_ / 4];
-  this->store_.buffer_write_at = 0;
-  this->store_.buffer_read_at = 0;
-  this->store_.buffer_size = this->buffer_size_;
-  this->store_.max_size = 64 * this->mem_block_num_ * sizeof(rmt_symbol_word_t) + sizeof(rmt_rx_done_event_data_t);
-  error = rmt_receive(this->channel_, this->store_.buffer, 64 * this->mem_block_num_ * sizeof(rmt_symbol_word_t),
-                      &this->store_.config);
+  this->store_.receive_size = MEM_BLOCK_SIZE * this->mem_block_num_ * sizeof(rmt_symbol_word_t);
+  this->store_.buffer_size = std::max(sizeof(rmt_rx_done_event_data_t) + this->store_.receive_size, this->buffer_size_);
+  this->store_.buffer = new uint8_t[this->buffer_size_];
+  error = rmt_receive(this->channel_, (uint8_t *) this->store_.buffer + sizeof(rmt_rx_done_event_data_t),
+                      this->store_.receive_size, &this->store_.config);
   if (error != ESP_OK) {
     this->error_code_ = error;
     this->error_string_ = "in rmt_receive";
@@ -103,13 +97,13 @@ void RemoteReceiverComponent::loop() {
     this->error_code_ = this->store_.error;
     this->error_string_ = "in rmt_callback";
     this->mark_failed();
-    return;
   }
   if (this->store_.buffer_write_at != this->store_.buffer_read_at) {
     rmt_rx_done_event_data_t *data = (rmt_rx_done_event_data_t *) (this->store_.buffer + this->store_.buffer_read_at);
     this->decode_rmt_(data->received_symbols, data->num_symbols);
     this->store_.buffer_read_at += sizeof(rmt_rx_done_event_data_t) + data->num_symbols * sizeof(rmt_symbol_word_t);
-    if ((this->store_.buffer_read_at + this->store_.max_size) > this->store_.buffer_size) {
+    if ((this->store_.buffer_read_at + sizeof(rmt_rx_done_event_data_t) + this->store_.receive_size) >
+        this->store_.buffer_size) {
       this->store_.buffer_read_at = 0;
     }
     if (!this->temp_.empty()) {
diff --git a/esphome/components/remote_transmitter/remote_transmitter_esp32.cpp b/esphome/components/remote_transmitter/remote_transmitter_esp32.cpp
index 459f87f353..4dda792b2d 100644
--- a/esphome/components/remote_transmitter/remote_transmitter_esp32.cpp
+++ b/esphome/components/remote_transmitter/remote_transmitter_esp32.cpp
@@ -9,7 +9,10 @@ namespace remote_transmitter {
 
 static const char *const TAG = "remote_transmitter";
 
-void RemoteTransmitterComponent::setup() { this->configure_rmt_(); }
+void RemoteTransmitterComponent::setup() {
+  ESP_LOGCONFIG(TAG, "Setting up Remote Transmitter...");
+  this->configure_rmt_();
+}
 
 void RemoteTransmitterComponent::dump_config() {
   ESP_LOGCONFIG(TAG, "Remote Transmitter...");