diff --git a/esphome/components/color/__init__.py b/esphome/components/color/__init__.py
index 4a55beef38..609d416a0b 100644
--- a/esphome/components/color/__init__.py
+++ b/esphome/components/color/__init__.py
@@ -14,15 +14,41 @@ CONF_HEX = "hex"
 
 
 def hex_color(value):
+    if isinstance(value, int):
+        value = str(value)
+    if not isinstance(value, str):
+        raise cv.Invalid("Invalid value for hex color")
     if len(value) != 6:
-        raise cv.Invalid("Color must have six digits")
+        raise cv.Invalid("Hex color must have six digits")
     try:
-        return (int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16))
+        return int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16)
     except ValueError as exc:
         raise cv.Invalid("Color must be hexadecimal") from exc
 
 
-CONFIG_SCHEMA = cv.Any(
+components = {
+    CONF_RED,
+    CONF_RED_INT,
+    CONF_GREEN,
+    CONF_GREEN_INT,
+    CONF_BLUE,
+    CONF_BLUE_INT,
+    CONF_WHITE,
+    CONF_WHITE_INT,
+}
+
+
+def validate_color(config):
+    has_components = set(config) & components
+    has_hex = CONF_HEX in config
+    if has_hex and has_components:
+        raise cv.Invalid("Hex color value may not be combined with component values")
+    if not has_hex and not has_components:
+        raise cv.Invalid("Must provide at least one color option")
+    return config
+
+
+CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Required(CONF_ID): cv.declare_id(ColorStruct),
@@ -34,14 +60,10 @@ CONFIG_SCHEMA = cv.Any(
             cv.Exclusive(CONF_BLUE_INT, "blue"): cv.uint8_t,
             cv.Exclusive(CONF_WHITE, "white"): cv.percentage,
             cv.Exclusive(CONF_WHITE_INT, "white"): cv.uint8_t,
+            cv.Optional(CONF_HEX): hex_color,
         }
     ).extend(cv.COMPONENT_SCHEMA),
-    cv.Schema(
-        {
-            cv.Required(CONF_ID): cv.declare_id(ColorStruct),
-            cv.Required(CONF_HEX): hex_color,
-        }
-    ).extend(cv.COMPONENT_SCHEMA),
+    validate_color,
 )
 
 
diff --git a/tests/components/color/common.yaml b/tests/components/color/common.yaml
index 7aa308bb63..88524e6a5f 100644
--- a/tests/components/color/common.yaml
+++ b/tests/components/color/common.yaml
@@ -9,3 +9,12 @@ color:
     blue: 100%
   - id: kbx_green
     hex: "3DEC55"
+  - id: kbx_green_1
+    hex: 3DEC55
+  - id: cps_red
+    hex: 800000
+  - id: cps_green
+    hex: 008000
+  - id: cps_blue
+    hex: 000080
+