add clang-tidy for zephyr

script/clang-tidy

@@ -1,21 +1,6 @@
 #!/usr/bin/env python3
 
-from helpers import (
-    print_error_for_file,
-    get_output,
-    filter_grep,
-    build_all_include,
-    temp_header_file,
-    git_ls_files,
-    filter_changed,
-    load_idedata,
-    root_path,
-    basepath,
-    get_binary,
-)
 import argparse
-import click
-import colorama
 import multiprocessing
 import os
 import queue
@@ -26,6 +11,20 @@ import sys
 import tempfile
 import threading
 
+import click
+import colorama
+from helpers import (
+    basepath,
+    build_all_include,
+    filter_changed,
+    filter_grep,
+    get_binary,
+    git_ls_files,
+    load_idedata,
+    print_error_for_file,
+    root_path,
+    temp_header_file,
+)
 
 
 def clang_options(idedata):
@@ -40,12 +39,40 @@ def clang_options(idedata):
     else:
         cmd.append(f"--target={triplet}")
 
+    omit_flags = (
+        "-free",
+        "-fipa-pta",
+        "-fstrict-volatile-bitfields",
+        "-mlongcalls",
+        "-mtext-section-literals",
+        "-mfix-esp32-psram-cache-issue",
+        "-mfix-esp32-psram-cache-strategy=memw",
+        "-fno-tree-switch-conversion",
+    )
+
+    if "zephyr" in triplet:
+        omit_flags += (
+            "-fno-printf-return-value",
+            "-fno-reorder-functions",
+            "-format-zero-length",
+            "-mfp16-format=ieee",
+            "-std=c99",
+            "-fno-defer-pop",
+            "--param=min-pagesize=0",
+            "--specs=picolibc.specs",
+        )
+    else:
+        cmd.extend(
+            [
+                "-nostdinc++",
+            ]
+        )
+
     # set flags
     cmd.extend(
         [
             # disable built-in include directories from the host
             "-nostdinc",
-            "-nostdinc++",
             # replace pgmspace.h, as it uses GNU extensions clang doesn't support
             # https://github.com/earlephilhower/newlib-xtensa/pull/18
             "-D_PGMSPACE_H_",
@@ -72,21 +99,7 @@ def clang_options(idedata):
     )
 
     # copy compiler flags, except those clang doesn't understand.
-    cmd.extend(
+    cmd.extend(flag for flag in idedata["cxx_flags"] if flag not in omit_flags)
-        flag
-        for flag in idedata["cxx_flags"]
-        if flag
-        not in (
-            "-free",
-            "-fipa-pta",
-            "-fstrict-volatile-bitfields",
-            "-mlongcalls",
-            "-mtext-section-literals",
-            "-mfix-esp32-psram-cache-issue",
-            "-mfix-esp32-psram-cache-strategy=memw",
-            "-fno-tree-switch-conversion",
-        )
-    )
 
     # defines
     cmd.extend(f"-D{define}" for define in idedata["defines"])
@@ -105,6 +118,7 @@ def clang_options(idedata):
             not directory.startswith(f"{root_path}/")
             or directory.startswith(f"{root_path}/.pio/")
             or directory.startswith(f"{root_path}/managed_components/")
+            or "zephyr/include/generated" in directory
         ):
             cmd.extend(["-isystem", directory])
 
@@ -116,9 +130,10 @@ def clang_options(idedata):
 
 pids = set()
 
-def run_tidy(executable, args, options, tmpdir, queue, lock, failed_files):
+
+def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files):
     while True:
-        path = queue.get()
+        path = path_queue.get()
         invocation = [executable]
 
         if tmpdir is not None:
@@ -140,17 +155,20 @@ def run_tidy(executable, args, options, tmpdir, queue, lock, failed_files):
         invocation.append("--")
         invocation.extend(options)
 
-        proc = subprocess.run(invocation, capture_output=True, encoding="utf-8")
+        proc = subprocess.run(
+            invocation, capture_output=True, encoding="utf-8", check=False
+        )
         if proc.returncode != 0:
             with lock:
                 print_error_for_file(path, proc.stdout)
                 failed_files.append(path)
-        queue.task_done()
+        path_queue.task_done()
 
 
 def progress_bar_show(value):
     if value is None:
         return ""
+    return None
 
 
 def split_list(a, n):
@@ -238,7 +256,15 @@ def main():
         for _ in range(args.jobs):
             t = threading.Thread(
                 target=run_tidy,
-                args=(executable, args, options, tmpdir, task_queue, lock, failed_files),
+                args=(
+                    executable,
+                    args,
+                    options,
+                    tmpdir,
+                    task_queue,
+                    lock,
+                    failed_files,
+                ),
             )
             t.daemon = True
             t.start()
@@ -246,14 +272,14 @@ def main():
         # Fill the queue with files.
         with click.progressbar(
             files, width=30, file=sys.stderr, item_show_func=progress_bar_show
-        ) as bar:
+        ) as progress_bar:
-            for name in bar:
+            for name in progress_bar:
                 task_queue.put(name)
 
         # Wait for all threads to be done.
         task_queue.join()
 
-    except FileNotFoundError as ex:
+    except FileNotFoundError:
         return 1
     except KeyboardInterrupt:
         print()
@@ -263,7 +289,7 @@ def main():
         # Kill subprocesses (and ourselves!)
         # No simple, clean alternative appears to be available.
         os.kill(0, 9)
-        return 2    # Will not execute.
+        return 2  # Will not execute.
 
     if args.fix and failed_files:
         print("Applying fixes ...")
@@ -273,7 +299,10 @@ def main():
             except FileNotFoundError:
                 subprocess.call(["clang-apply-replacements", tmpdir])
         except FileNotFoundError:
-                print("Error please install clang-apply-replacements-14 or clang-apply-replacements.\n", file=sys.stderr)
+            print(
+                "Error please install clang-apply-replacements-14 or clang-apply-replacements.\n",
+                file=sys.stderr,
+            )
         except:
             print("Error applying fixes.\n", file=sys.stderr)
             raise

script/helpers.py

@@ -1,8 +1,8 @@
 import json
 import os.path
+from pathlib import Path
 import re
 import subprocess
-from pathlib import Path
 
 import colorama
 
@@ -147,10 +147,129 @@ def load_idedata(environment):
     # ensure temp directory exists before running pio, as it writes sdkconfig to it
     Path(temp_folder).mkdir(exist_ok=True)
 
-    stdout = subprocess.check_output(["pio", "run", "-t", "idedata", "-e", environment])
+    if "nrf" in environment:
-    match = re.search(r'{\s*".*}', stdout.decode("utf-8"))
+        build_environment = environment.replace("-tidy", "")
-    data = json.loads(match.group())
+        build_dir = Path(temp_folder) / f"build-{build_environment}"
+        Path(build_dir).mkdir(exist_ok=True)
+        Path(build_dir / "platformio.ini").write_text(
+            Path(platformio_ini).read_text(encoding="utf-8"), encoding="utf-8"
+        )
+        esphome_dir = Path(build_dir / "esphome")
+        esphome_dir.mkdir(exist_ok=True)
+        Path(esphome_dir / "main.cpp").write_text(
+            """
+#include <zephyr/kernel.h>
+int main() { return 0;}
+""",
+            encoding="utf-8",
+        )
+        zephyr_dir = Path(build_dir / "zephyr")
+        zephyr_dir.mkdir(exist_ok=True)
+        Path(zephyr_dir / "prj.conf").write_text("", encoding="utf-8")
+        result = subprocess.run(
+            ["pio", "run", "-e", build_environment, "-d", build_dir], check=False
+        )
+        if result.returncode != 0:
+            print("Unable to compile empty main to build env")
 
+        def extract_include_paths(command):
+            include_paths = []
+            include_pattern = re.compile(r"(-I|-isystem)\s*([^\s]+)")
+            for match in include_pattern.findall(command):
+                include_paths.append(match[1])
+            return include_paths
+
+        def extract_defines(command):
+            """
+            Extracts defined macros from the command string.
+            """
+            defines = []
+            define_pattern = re.compile(r"-D\s*([^\s]+)")
+            for match in define_pattern.findall(command):
+                defines.append(match)
+            return defines
+
+        def find_cxx_path(commands):
+            for entry in commands:
+                command = entry["command"]
+                cxx_path = command.split()[0]
+                return cxx_path
+            raise ValueError("No valid compiler path found in the compile commands")
+
+        def get_builtin_include_paths(compiler):
+            result = subprocess.run(
+                [compiler, "-E", "-x", "c++", "-", "-v"],
+                input="",
+                text=True,
+                stderr=subprocess.PIPE,
+                check=False,
+            )
+            include_paths = []
+            start_collecting = False
+            for line in result.stderr.splitlines():
+                if start_collecting:
+                    if line.startswith(" "):
+                        include_paths.append(line.strip())
+                    else:
+                        break
+                if "#include <...> search starts here:" in line:
+                    start_collecting = True
+            return include_paths
+
+        def extract_cxx_flags(command):
+            """
+            Extracts CXXFLAGS from the command string, excluding includes and defines.
+            """
+            flags = []
+            flag_pattern = re.compile(
+                r"(-O[0-3s]|-g|-std=[^\s]+|-Wall|-Wextra|-Werror|--[^\s]+|-f[^\s]+|-m[^\s]+|-imacros\s*[^\s]+)"
+            )
+            for match in flag_pattern.findall(command):
+                flags.append(match)
+            return flags
+
+        def transform_to_idedata_format(compile_commands):
+            cxx_path = find_cxx_path(compile_commands)
+            idedata = {
+                "includes": {
+                    "toolchain": get_builtin_include_paths(cxx_path),
+                    "build": set(),
+                },
+                "defines": set(),
+                "cxx_path": cxx_path,
+                "cxx_flags": set(),
+            }
+
+            for entry in compile_commands:
+                command = entry["command"]
+
+                idedata["includes"]["build"].update(extract_include_paths(command))
+                idedata["defines"].update(extract_defines(command))
+                idedata["cxx_flags"].update(extract_cxx_flags(command))
+
+            # Convert sets to lists for JSON serialization
+            idedata["includes"]["build"] = list(idedata["includes"]["build"])
+            idedata["defines"] = list(idedata["defines"])
+            idedata["cxx_flags"] = list(idedata["cxx_flags"])
+
+            return idedata
+
+        compile_commands = json.loads(
+            Path(
+                build_dir
+                / ".pio"
+                / "build"
+                / build_environment
+                / "compile_commands.json"
+            ).read_text(encoding="utf-8")
+        )
+        data = transform_to_idedata_format(compile_commands)
+    else:
+        stdout = subprocess.check_output(
+            ["pio", "run", "-t", "idedata", "-e", environment]
+        )
+        match = re.search(r'{\s*".*}', stdout.decode("utf-8"))
+        data = json.loads(match.group())
     temp_idedata.write_text(json.dumps(data, indent=2) + "\n")
     return data
 
@@ -158,21 +277,23 @@ def load_idedata(environment):
 def get_binary(name: str, version: str) -> str:
     binary_file = f"{name}-{version}"
     try:
-        result = subprocess.check_output([binary_file, "-version"])
+        # If no exception was raised, the command was successful
-        if result.returncode == 0:
+        result = subprocess.check_output(
-            return binary_file
+            [binary_file, "-version"], stderr=subprocess.STDOUT
-    except Exception:
+        )
+        return binary_file
+    except FileNotFoundError:
         pass
     binary_file = name
     try:
         result = subprocess.run(
-            [binary_file, "-version"], text=True, capture_output=True
+            [binary_file, "-version"], text=True, capture_output=True, check=False
         )
         if result.returncode == 0 and (f"version {version}") in result.stdout:
             return binary_file
         raise FileNotFoundError(f"{name} not found")
 
-    except FileNotFoundError as ex:
+    except FileNotFoundError:
         print(
             f"""
             Oops. It looks like {name} is not installed. It should be available under venv/bin