diff --git a/esphome/components/usb_device/__init__.py b/esphome/components/usb_device/__init__.py
index 3ffedbbf0d..36e8235488 100644
--- a/esphome/components/usb_device/__init__.py
+++ b/esphome/components/usb_device/__init__.py
@@ -7,7 +7,6 @@ from esphome.const import (
 from esphome.components.esp32.const import (
     KEY_ESP32,
     KEY_VARIANT,
-    VARIANT_ESP32S2,
     VARIANT_ESP32S3,
 )
 
@@ -17,7 +16,7 @@ CONF_USB_DEVICE_ID = "usb_device_id"
 
 def _validate_variant(value):
     variant = CORE.data[KEY_ESP32][KEY_VARIANT]
-    if variant not in [VARIANT_ESP32S2, VARIANT_ESP32S3]:
+    if variant not in [VARIANT_ESP32S3]:
         raise cv.Invalid(f"USB device is unsupported by ESP32 variant {variant}")
     return value
 
@@ -33,7 +32,6 @@ CONFIG_SCHEMA = cv.All(
             cv.GenerateID(): cv.declare_id(UsbDevice),
         }
     ).extend(cv.polling_component_schema("10s")),
-    cv.only_with_arduino,
     cv.only_on_esp32,
     _validate_variant,
 )
diff --git a/esphome/components/usb_device/usb_device.cpp b/esphome/components/usb_device/usb_device.cpp
index fa61bf4f59..88390b6d13 100644
--- a/esphome/components/usb_device/usb_device.cpp
+++ b/esphome/components/usb_device/usb_device.cpp
@@ -1,12 +1,13 @@
-#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if defined(USE_ESP32_VARIANT_ESP32S3)
 #include "usb_device.h"
 #include "esphome/core/log.h"
-#include "USB.h"
-// based on defines in HWCDC.cpp
-#if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+
+#ifdef USE_ARDUINO
 #if ARDUINO_USB_MODE
 #include "HWCDC.h"
 #endif
+#else
+#include "esp32s3/rom/usb/usb_dc.h"
 #endif
 
 namespace esphome {
@@ -14,7 +15,13 @@ namespace usb_device {
 
 static const char *const TAG = "usb_device";
 
+usb_dc_status_code g_cb_status;
+void status_callback(enum usb_dc_status_code cb_status, uint8_t *param) {
+  g_cb_status = cb_status;
+}
+
 void UsbDevice::update() {
+  ESP_LOGD(TAG, "update %d", g_cb_status);
 #ifdef USE_BINARY_SENSOR
   if (configured_ != nullptr) {
     configured_->publish_state(get_configured_());
@@ -22,6 +29,12 @@ void UsbDevice::update() {
 #endif
 }
 
+void UsbDevice::setup() {
+#ifndef USE_ARDUINO
+  usb_dc_set_status_callback(status_callback);
+#endif
+}
+
 void UsbDevice::dump_config() { ESP_LOGCONFIG(TAG, "USB device - configured: %s", YESNO(get_configured_())); }
 
 #ifdef USE_BINARY_SENSOR
@@ -29,8 +42,13 @@ void UsbDevice::set_configured_binary_sensor(binary_sensor::BinarySensor *sensor
 #endif
 
 bool UsbDevice::get_configured_() {
+// ESP32:
+// - Arduino framework does not support USB_SERIAL_JTAG.
+// ESP32-S3
+// - Arduino CDC logger is based on HWCDC.
+
+#ifdef USE_ARDUINO
 // based on defines in HWCDC.cpp
-#if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
 #if ARDUINO_USB_MODE
 #if ARDUINO_USB_CDC_ON_BOOT  // Serial used for USB CDC
   return Serial;
@@ -41,11 +59,8 @@ bool UsbDevice::get_configured_() {
 // this is subject of change by other components so make sure that we won't fail to report silently
 #error Not implemented
 #endif
-#elif USE_ESP32_VARIANT_ESP32S2
-  return USB;
 #else
-// this is subject of change by other components so make sure that we won't fail to report silently
-#error Not implemented
+
 #endif
   return false;
 }
diff --git a/esphome/components/usb_device/usb_device.h b/esphome/components/usb_device/usb_device.h
index 55b2490e44..22d5b4fa23 100644
--- a/esphome/components/usb_device/usb_device.h
+++ b/esphome/components/usb_device/usb_device.h
@@ -1,7 +1,7 @@
 #pragma once
-#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
-#include "esphome/core/component.h"
+#if defined(USE_ESP32_VARIANT_ESP32S3)
 #include "esphome/core/defines.h"
+#include "esphome/core/component.h"
 #ifdef USE_BINARY_SENSOR
 #include "esphome/components/binary_sensor/binary_sensor.h"
 #endif
@@ -11,6 +11,7 @@ namespace usb_device {
 class UsbDevice : public PollingComponent {
  public:
   void update() override;
+  void setup() override;
   void dump_config() override;
 #ifdef USE_BINARY_SENSOR
   void set_configured_binary_sensor(binary_sensor::BinarySensor *sensor);
diff --git a/tests/components/usb_device/test.esp32-s2-ard.yaml b/tests/components/usb_device/test.esp32-s3-idf.yaml
similarity index 100%
rename from tests/components/usb_device/test.esp32-s2-ard.yaml
rename to tests/components/usb_device/test.esp32-s3-idf.yaml