Allow compilation against IDF from repository (#2355)

* Fix src_filter in platformio.ini after src_dir change

* Add -Wno-nonnull-compare to platformio.ini as well

* Create default sdkconfig for static analysis

* Add more compiler flags to clang ignore list

* Clean-up platformio.ini

* Remove unnecessary blank line

* Fix accidentally dropped library

* Don't gitignore sdkconfig.defaults

Co-authored-by: Otto winter <otto@otto-winter.com>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

.gitignore

@@ -126,3 +126,4 @@ tests/.esphome/
 .pio/
 
 sdkconfig.*
+!sdkconfig.defaults

platformio.ini

@@ -4,7 +4,7 @@
 ; It's *not* used during runtime.
 
 [platformio]
-default_envs = esp8266, esp32
+default_envs = esp8266, esp32, esp32-idf
 src_dir = esphome
 include_dir =
 
@@ -26,18 +26,8 @@ build_flags =
 
 [common]
 lib_deps =
-    ottowinter/AsyncMqttClient-esphome@0.8.4              ; mqtt
+    esphome/noise-c@0.1.1     ; api
-    ottowinter/ArduinoJson-esphomelib@5.13.3              ; json
+    makuna/NeoPixelBus@2.6.7  ; neopixelbus
-    esphome/ESPAsyncWebServer-esphome@1.3.0               ; web_server_base
-    fastled/FastLED@3.3.2                                 ; fastled_base
-    makuna/NeoPixelBus@2.6.7                              ; neopixelbus
-    mikalhart/TinyGPSPlus@1.0.2                           ; gps
-    freekode/TM1651@1.0.1                                 ; tm1651
-    seeed-studio/Grove - Laser PM2.5 Sensor HM3301@1.0.3  ; hm3301
-    glmnet/Dsmr@0.5                                       ; dsmr
-    rweather/Crypto@0.2.0                                 ; dsmr
-    esphome/noise-c@0.1.1                                 ; api
-    dudanov/MideaUART@1.1.0                               ; midea
 build_flags =
     -DESPHOME_LOG_LEVEL=ESPHOME_LOG_LEVEL_VERY_VERBOSE
 src_filter =
@@ -45,8 +35,32 @@ src_filter =
     +<../tests/dummy_main.cpp>
     +<../.temp/all-include.cpp>
 
-[common:esp8266]
+[common:arduino]
 extends = common
+lib_deps =
+    ${common.lib_deps}
+    ottowinter/AsyncMqttClient-esphome@0.8.4              ; mqtt
+    ottowinter/ArduinoJson-esphomelib@5.13.3              ; json
+    esphome/ESPAsyncWebServer-esphome@1.3.0               ; web_server_base
+    fastled/FastLED@3.3.2                                 ; fastled_base
+    mikalhart/TinyGPSPlus@1.0.2                           ; gps
+    freekode/TM1651@1.0.1                                 ; tm1651
+    seeed-studio/Grove - Laser PM2.5 Sensor HM3301@1.0.3  ; hm3301
+    glmnet/Dsmr@0.5                                       ; dsmr
+    rweather/Crypto@0.2.0                                 ; dsmr
+    dudanov/MideaUART@1.1.0                               ; midea
+build_flags =
+    ${common.build_flags}
+    -DUSE_ARDUINO
+
+[common:idf]
+extends = common
+build_flags =
+    ${common.build_flags}
+    -DUSE_ESP_IDF
+
+[common:esp8266]
+extends = common:arduino
 ; when changing this also copy it to esphome-docker-base images
 platform = platformio/espressif8266 @ 3.2.0
 platform_packages =
@@ -55,16 +69,17 @@ platform_packages =
 framework = arduino
 board = nodemcuv2
 lib_deps =
-    ${common.lib_deps}
+    ${common:arduino.lib_deps}
     ESP8266WiFi                           ; wifi (Arduino built-in)
     Update                                ; ota (Arduino built-in)
     ottowinter/ESPAsyncTCP-esphome@1.2.3  ; async_tcp
 build_flags =
-    ${common.build_flags}
+    ${common:arduino.build_flags}
     -DUSE_ESP8266
+    -DUSE_ESP8266_FRAMEWORK_ARDUINO
 
-[common:esp32]
+[common:esp32-arduino]
-extends = common
+extends = common:arduino
 ; when changing this also copy it to esphome-docker-base images
 platform = platformio/espressif32 @ 3.3.2
 platform_packages =
@@ -73,15 +88,16 @@ platform_packages =
 framework = arduino
 board = nodemcu-32s
 lib_deps =
-    ${common.lib_deps}
+    ${common:arduino.lib_deps}
     Hash                            ; ota (Arduino built-in)
     esphome/AsyncTCP-esphome@1.2.2  ; async_tcp
 build_flags =
-    ${common.build_flags}
+    ${common:arduino.build_flags}
     -DUSE_ESP32
-src_filter = ${common.src_filter}
+    -DUSE_ESP32_FRAMEWORK_ARDUINO
 
-[common:espidf]
+[common:esp32-idf]
+extends = common:idf
 ; when changing this also copy it to esphome-docker-base images
 platform = platformio/espressif32 @ 3.3.2
 platform_packages =
@@ -90,57 +106,48 @@ platform_packages =
 framework = espidf
 board = nodemcu-32s
 lib_deps =
-    esphome/noise-c@0.1.1  ; used by api
+    ${common:idf.lib_deps}
-    espressif/esp32-camera@1.0.0  ; used by esp32_camera
+    espressif/esp32-camera@1.0.0  ; esp32_camera
-    NeoPixelBus@2.6.7
 build_flags =
-    ${common.build_flags}
+    ${common:idf.build_flags}
-    ${common:esp32.build_flags}
+    -Wno-nonnull-compare
-    -DUSE_ESP_IDF
+    -DUSE_ESP32
     -DUSE_ESP32_FRAMEWORK_ESP_IDF
-src_filter = ${common:esp32.src_filter}
 
 [env:esp8266]
 extends = common:esp8266
 build_flags =
     ${common:esp8266.build_flags}
     ${runtime.build_flags}
-    -DUSE_ARDUINO
-    -DUSE_ESP8266_FRAMEWORK_ARDUINO
 
 [env:esp8266-tidy]
 extends = common:esp8266
 build_flags =
     ${common:esp8266.build_flags}
     ${clangtidy.build_flags}
-    -DUSE_ARDUINO
-    -DUSE_ESP8266_FRAMEWORK_ARDUINO
 
 [env:esp32]
-extends = common:esp32
+extends = common:esp32-arduino
 build_flags =
-    ${common:esp32.build_flags}
+    ${common:esp32-arduino.build_flags}
     ${runtime.build_flags}
-    -DUSE_ARDUINO
-    -DUSE_ESP32_FRAMEWORK_ARDUINO
 
 [env:esp32-tidy]
-extends = common:esp32
+extends = common:esp32-arduino
 build_flags =
-    ${common:esp32.build_flags}
+    ${common:esp32-arduino.build_flags}
     ${clangtidy.build_flags}
-    -DUSE_ARDUINO
-    -DUSE_ESP32_FRAMEWORK_ARDUINO
 
 [env:esp32-idf]
-extends = common:espidf
+extends = common:esp32-idf
+board_build.esp-idf.sdkconfig_path = .temp/sdkconfig-esp32-idf
 build_flags =
-    ${common:espidf.build_flags}
+    ${common:esp32-idf.build_flags}
     ${runtime.build_flags}
 
-
 [env:esp32-idf-tidy]
-extends = common:espidf
+extends = common:esp32-idf
+board_build.esp-idf.sdkconfig_path = .temp/sdkconfig-esp32-idf-tidy
 build_flags =
-    ${common:espidf.build_flags}
+    ${common:esp32-idf.build_flags}
     ${clangtidy.build_flags}

script/clang-tidy

@@ -54,7 +54,9 @@ def clang_options(idedata):
 
     # copy compiler flags, except those clang doesn't understand.
     cmd.extend(flag for flag in idedata['cxx_flags'].split(' ')
-               if flag not in ('-free', '-fipa-pta', '-fstrict-volatile-bitfields', '-mlongcalls', '-mtext-section-literals'))
+               if flag not in ('-free', '-fipa-pta', '-fstrict-volatile-bitfields',
+                               '-mlongcalls', '-mtext-section-literals',
+                               '-mfix-esp32-psram-cache-issue', '-mfix-esp32-psram-cache-strategy=memw'))
 
     # defines
     cmd.extend(f'-D{define}' for define in idedata['defines'])

script/helpers.py

@@ -112,11 +112,6 @@ def git_ls_files(patterns=None):
     return {s[3].strip(): int(s[0]) for s in lines}
 
 
-IDF_TIDY_SDKCONFIG = """\
-CONFIG_BT_ENABLED=y
-"""
-
-
 def load_idedata(environment):
     platformio_ini = Path(root_path) / "platformio.ini"
     temp_idedata = Path(temp_folder) / f"idedata-{environment}.json"
@@ -126,30 +121,26 @@ def load_idedata(environment):
     elif platformio_ini.stat().st_mtime >= temp_idedata.stat().st_mtime:
         changed = True
 
-    if environment == "esp32-idf-tidy":
+    if "idf" in environment:
-        # sdkconfig needs to be written before idedata is run
+        # remove full sdkconfig when the defaults have changed so that it is regenerated
-        # but the file is also modified by the build process, so
+        default_sdkconfig = Path(root_path) / "sdkconfig.defaults"
-        # store a temp file to keep track of the
+        temp_sdkconfig = Path(temp_folder) / f"sdkconfig-{environment}"
 
-        sdk_internal = Path(temp_folder) / f"{environment}-internal-sdkconfig"
+        if not temp_sdkconfig.is_file():
-        sdkconfig = Path(root_path) / f"sdkconfig.{environment}"
+            changed = True
-        if (
+        elif default_sdkconfig.stat().st_mtime >= temp_sdkconfig.stat().st_mtime:
-            changed
+            temp_sdkconfig.unlink()
-            or not sdk_internal.is_file()
-            or sdk_internal.read_text() != IDF_TIDY_SDKCONFIG
-        ):
             changed = True
-            sdkconfig.write_text(IDF_TIDY_SDKCONFIG)
-            sdk_internal.parent.mkdir(exist_ok=True)
-            sdk_internal.write_text(IDF_TIDY_SDKCONFIG)
 
     if not changed:
         return json.loads(temp_idedata.read_text())
 
+    # ensure temp directory exists before running pio, as it writes sdkconfig to it
+    Path(temp_folder).mkdir(exist_ok=True)
+
     stdout = subprocess.check_output(["pio", "run", "-t", "idedata", "-e", environment])
     match = re.search(r'{\s*".*}', stdout.decode("utf-8"))
     data = json.loads(match.group())
 
-    temp_idedata.parent.mkdir(exist_ok=True)
     temp_idedata.write_text(json.dumps(data, indent=2) + "\n")
     return data

sdkconfig.defaults

@@ -0,0 +1,17 @@
+# ESP-IDF sdkconfig defaults used for development purposes only, not used during runtime. Used when PlatformIO is ran
+# directly from the source directory, e.g. by IDEs or for static analysis (clang-tidy). This should enable all flags
+# that are set by any component.
+
+# esp32
+CONFIG_COMPILER_OPTIMIZATION_DEFAULT=n
+CONFIG_COMPILER_OPTIMIZATION_SIZE=y
+CONFIG_PARTITION_TABLE_CUSTOM=y
+#CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+CONFIG_PARTITION_TABLE_SINGLE_APP=n
+
+# esp32_ble
+CONFIG_BT_ENABLED=y
+
+# esp32_camera
+CONFIG_RTCIO_SUPPORT_RTC_GPIO_DESC=y
+CONFIG_ESP32_SPIRAM_SUPPORT=y