diff --git a/esphome/core/component.h b/esphome/core/component.h
index 1d8499e262..3b873cbc9f 100644
--- a/esphome/core/component.h
+++ b/esphome/core/component.h
@@ -185,19 +185,29 @@ class Component {
 
   /** Set an retry function with a unique name. Empty name means no cancelling possible.
    *
-   * This will call f. If f returns RetryResult::RETRY f is called again after initial_wait_time ms.
-   * f should return RetryResult::DONE if no repeat is required. The initial wait time will be increased
-   * by backoff_increase_factor for each iteration. Default is doubling the time between iterations
-   * Can be cancelled via cancel_retry().
+   * This will call the retry function f on the next scheduler loop. f should return RetryResult::DONE if
+   * it is successful and no repeat is required. Otherwise, returning RetryResult::RETRY will call f
+   * again in the future.
+   *
+   * The first retry of f happens after `initial_wait_time` milliseconds. The delay between retries is
+   * increased by multipling by `backoff_increase_factor` each time. If no backoff_increase_factor is
+   * supplied (default = 1.0), the wait time will stay constant.
+   *
+   * This retry function can also be cancelled by name via cancel_retry().
    *
    * IMPORTANT: Do not rely on this having correct timing. This is only called from
    * loop() and therefore can be significantly delayed.
    *
+   * REMARK: It is an error to supply a negative or zero `backoff_increase_factor`, and 1.0 will be used instead.
+   *
+   * REMARK: The interval between retries is stored into a `uint32_t`, so this doesn't behave correctly
+   * if `initial_wait_time * (backoff_increase_factor ** (max_attempts - 2))` overflows.
+   *
    * @param name The identifier for this retry function.
    * @param initial_wait_time The time in ms before f is called again
-   * @param max_attempts The maximum number of retries
+   * @param max_attempts The maximum number of executions
    * @param f The function (or lambda) that should be called
-   * @param backoff_increase_factor time between retries is increased by this factor on every retry
+   * @param backoff_increase_factor time between retries is multiplied by this factor on every retry after the first
    * @see cancel_retry()
    */
   void set_retry(const std::string &name, uint32_t initial_wait_time, uint8_t max_attempts,  // NOLINT
diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp
index cc4074b94d..dfec4fcaa2 100644
--- a/esphome/core/scheduler.cpp
+++ b/esphome/core/scheduler.cpp
@@ -87,8 +87,10 @@ static void retry_handler(const std::shared_ptr<RetryArgs> &args) {
   RetryResult retry_result = args->func();
   if (retry_result == RetryResult::DONE || --args->retry_countdown <= 0)
     return;
-  args->current_interval *= args->backoff_increase_factor;
+  // second execution of `func` hapens after `initial_wait_time`
   args->scheduler->set_timeout(args->component, args->name, args->current_interval, [args]() { retry_handler(args); });
+  // backoff_increase_factor applied to third & later executions
+  args->current_interval *= args->backoff_increase_factor;
 }
 
 void HOT Scheduler::set_retry(Component *component, const std::string &name, uint32_t initial_wait_time,
@@ -102,6 +104,13 @@ void HOT Scheduler::set_retry(Component *component, const std::string &name, uin
   ESP_LOGVV(TAG, "set_retry(name='%s', initial_wait_time=%u, max_attempts=%u, backoff_factor=%0.1f)", name.c_str(),
             initial_wait_time, max_attempts, backoff_increase_factor);
 
+  if (backoff_increase_factor < 0.0001) {
+    ESP_LOGE(TAG,
+             "set_retry(name='%s'): backoff_factor cannot be close to zero nor negative (%0.1f). Using 1.0 instead",
+             name.c_str(), backoff_increase_factor);
+    backoff_increase_factor = 1;
+  }
+
   auto args = std::make_shared<RetryArgs>();
   args->func = std::move(func);
   args->retry_countdown = max_attempts;
@@ -111,7 +120,8 @@ void HOT Scheduler::set_retry(Component *component, const std::string &name, uin
   args->backoff_increase_factor = backoff_increase_factor;
   args->scheduler = this;
 
-  this->set_timeout(component, args->name, initial_wait_time, [args]() { retry_handler(args); });
+  // First exectuion of `func` immediately
+  this->set_timeout(component, args->name, 0, [args]() { retry_handler(args); });
 }
 bool HOT Scheduler::cancel_retry(Component *component, const std::string &name) {
   return this->cancel_timeout(component, "retry$" + name);