diff --git a/script/clang-tidy b/script/clang-tidy index bd919825fd..2fd01241db 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -1,21 +1,6 @@ #!/usr/bin/env python3 -from helpers import ( - print_error_for_file, - get_output, - filter_grep, - build_all_include, - temp_header_file, - git_ls_files, - filter_changed, - load_idedata, - root_path, - basepath, - get_binary, -) import argparse -import click -import colorama import multiprocessing import os import queue @@ -26,6 +11,20 @@ import sys import tempfile import threading +import click +import colorama +from helpers import ( + basepath, + build_all_include, + filter_changed, + filter_grep, + get_binary, + git_ls_files, + load_idedata, + print_error_for_file, + root_path, + temp_header_file, +) def clang_options(idedata): @@ -40,12 +39,40 @@ def clang_options(idedata): else: cmd.append(f"--target={triplet}") + omit_flags = ( + "-free", + "-fipa-pta", + "-fstrict-volatile-bitfields", + "-mlongcalls", + "-mtext-section-literals", + "-mfix-esp32-psram-cache-issue", + "-mfix-esp32-psram-cache-strategy=memw", + "-fno-tree-switch-conversion", + ) + + if "zephyr" in triplet: + omit_flags += ( + "-fno-printf-return-value", + "-fno-reorder-functions", + "-format-zero-length", + "-mfp16-format=ieee", + "-std=c99", + "-fno-defer-pop", + "--param=min-pagesize=0", + "--specs=picolibc.specs", + ) + else: + cmd.extend( + [ + "-nostdinc++", + ] + ) + # set flags cmd.extend( [ # disable built-in include directories from the host "-nostdinc", - "-nostdinc++", # replace pgmspace.h, as it uses GNU extensions clang doesn't support # https://github.com/earlephilhower/newlib-xtensa/pull/18 "-D_PGMSPACE_H_", @@ -72,21 +99,7 @@ def clang_options(idedata): ) # copy compiler flags, except those clang doesn't understand. - cmd.extend( - flag - for flag in idedata["cxx_flags"] - if flag - not in ( - "-free", - "-fipa-pta", - "-fstrict-volatile-bitfields", - "-mlongcalls", - "-mtext-section-literals", - "-mfix-esp32-psram-cache-issue", - "-mfix-esp32-psram-cache-strategy=memw", - "-fno-tree-switch-conversion", - ) - ) + cmd.extend(flag for flag in idedata["cxx_flags"] if flag not in omit_flags) # defines cmd.extend(f"-D{define}" for define in idedata["defines"]) @@ -105,6 +118,7 @@ def clang_options(idedata): not directory.startswith(f"{root_path}/") or directory.startswith(f"{root_path}/.pio/") or directory.startswith(f"{root_path}/managed_components/") + or "zephyr/include/generated" in directory ): cmd.extend(["-isystem", directory]) @@ -116,9 +130,10 @@ def clang_options(idedata): pids = set() -def run_tidy(executable, args, options, tmpdir, queue, lock, failed_files): + +def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): while True: - path = queue.get() + path = path_queue.get() invocation = [executable] if tmpdir is not None: @@ -140,17 +155,20 @@ def run_tidy(executable, args, options, tmpdir, queue, lock, failed_files): invocation.append("--") invocation.extend(options) - proc = subprocess.run(invocation, capture_output=True, encoding="utf-8") + proc = subprocess.run( + invocation, capture_output=True, encoding="utf-8", check=False + ) if proc.returncode != 0: with lock: print_error_for_file(path, proc.stdout) failed_files.append(path) - queue.task_done() + path_queue.task_done() def progress_bar_show(value): if value is None: return "" + return None def split_list(a, n): @@ -238,7 +256,15 @@ def main(): for _ in range(args.jobs): t = threading.Thread( target=run_tidy, - args=(executable, args, options, tmpdir, task_queue, lock, failed_files), + args=( + executable, + args, + options, + tmpdir, + task_queue, + lock, + failed_files, + ), ) t.daemon = True t.start() @@ -246,14 +272,14 @@ def main(): # Fill the queue with files. with click.progressbar( files, width=30, file=sys.stderr, item_show_func=progress_bar_show - ) as bar: - for name in bar: + ) as progress_bar: + for name in progress_bar: task_queue.put(name) # Wait for all threads to be done. task_queue.join() - except FileNotFoundError as ex: + except FileNotFoundError: return 1 except KeyboardInterrupt: print() @@ -263,7 +289,7 @@ def main(): # Kill subprocesses (and ourselves!) # No simple, clean alternative appears to be available. os.kill(0, 9) - return 2 # Will not execute. + return 2 # Will not execute. if args.fix and failed_files: print("Applying fixes ...") @@ -273,7 +299,10 @@ def main(): except FileNotFoundError: subprocess.call(["clang-apply-replacements", tmpdir]) except FileNotFoundError: - print("Error please install clang-apply-replacements-14 or clang-apply-replacements.\n", file=sys.stderr) + print( + "Error please install clang-apply-replacements-14 or clang-apply-replacements.\n", + file=sys.stderr, + ) except: print("Error applying fixes.\n", file=sys.stderr) raise diff --git a/script/helpers.py b/script/helpers.py index 52b0658fb6..8690edee45 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1,8 +1,8 @@ import json import os.path +from pathlib import Path import re import subprocess -from pathlib import Path import colorama @@ -147,10 +147,129 @@ def load_idedata(environment): # ensure temp directory exists before running pio, as it writes sdkconfig to it Path(temp_folder).mkdir(exist_ok=True) - stdout = subprocess.check_output(["pio", "run", "-t", "idedata", "-e", environment]) - match = re.search(r'{\s*".*}', stdout.decode("utf-8")) - data = json.loads(match.group()) + if "nrf" in environment: + build_environment = environment.replace("-tidy", "") + build_dir = Path(temp_folder) / f"build-{build_environment}" + Path(build_dir).mkdir(exist_ok=True) + Path(build_dir / "platformio.ini").write_text( + Path(platformio_ini).read_text(encoding="utf-8"), encoding="utf-8" + ) + esphome_dir = Path(build_dir / "esphome") + esphome_dir.mkdir(exist_ok=True) + Path(esphome_dir / "main.cpp").write_text( + """ +#include +int main() { return 0;} +""", + encoding="utf-8", + ) + zephyr_dir = Path(build_dir / "zephyr") + zephyr_dir.mkdir(exist_ok=True) + Path(zephyr_dir / "prj.conf").write_text("", encoding="utf-8") + result = subprocess.run( + ["pio", "run", "-e", build_environment, "-d", build_dir], check=False + ) + if result.returncode != 0: + print("Unable to compile empty main to build env") + def extract_include_paths(command): + include_paths = [] + include_pattern = re.compile(r"(-I|-isystem)\s*([^\s]+)") + for match in include_pattern.findall(command): + include_paths.append(match[1]) + return include_paths + + def extract_defines(command): + """ + Extracts defined macros from the command string. + """ + defines = [] + define_pattern = re.compile(r"-D\s*([^\s]+)") + for match in define_pattern.findall(command): + defines.append(match) + return defines + + def find_cxx_path(commands): + for entry in commands: + command = entry["command"] + cxx_path = command.split()[0] + return cxx_path + raise ValueError("No valid compiler path found in the compile commands") + + def get_builtin_include_paths(compiler): + result = subprocess.run( + [compiler, "-E", "-x", "c++", "-", "-v"], + input="", + text=True, + stderr=subprocess.PIPE, + check=False, + ) + include_paths = [] + start_collecting = False + for line in result.stderr.splitlines(): + if start_collecting: + if line.startswith(" "): + include_paths.append(line.strip()) + else: + break + if "#include <...> search starts here:" in line: + start_collecting = True + return include_paths + + def extract_cxx_flags(command): + """ + Extracts CXXFLAGS from the command string, excluding includes and defines. + """ + flags = [] + flag_pattern = re.compile( + r"(-O[0-3s]|-g|-std=[^\s]+|-Wall|-Wextra|-Werror|--[^\s]+|-f[^\s]+|-m[^\s]+|-imacros\s*[^\s]+)" + ) + for match in flag_pattern.findall(command): + flags.append(match) + return flags + + def transform_to_idedata_format(compile_commands): + cxx_path = find_cxx_path(compile_commands) + idedata = { + "includes": { + "toolchain": get_builtin_include_paths(cxx_path), + "build": set(), + }, + "defines": set(), + "cxx_path": cxx_path, + "cxx_flags": set(), + } + + for entry in compile_commands: + command = entry["command"] + + idedata["includes"]["build"].update(extract_include_paths(command)) + idedata["defines"].update(extract_defines(command)) + idedata["cxx_flags"].update(extract_cxx_flags(command)) + + # Convert sets to lists for JSON serialization + idedata["includes"]["build"] = list(idedata["includes"]["build"]) + idedata["defines"] = list(idedata["defines"]) + idedata["cxx_flags"] = list(idedata["cxx_flags"]) + + return idedata + + compile_commands = json.loads( + Path( + build_dir + / ".pio" + / "build" + / build_environment + / "compile_commands.json" + ).read_text(encoding="utf-8") + ) + data = transform_to_idedata_format(compile_commands) + else: + stdout = subprocess.check_output( + ["pio", "run", "-t", "idedata", "-e", environment] + ) + match = re.search(r'{\s*".*}', stdout.decode("utf-8")) + data = json.loads(match.group()) temp_idedata.write_text(json.dumps(data, indent=2) + "\n") return data @@ -158,21 +277,23 @@ def load_idedata(environment): def get_binary(name: str, version: str) -> str: binary_file = f"{name}-{version}" try: - result = subprocess.check_output([binary_file, "-version"]) - if result.returncode == 0: - return binary_file - except Exception: + # If no exception was raised, the command was successful + result = subprocess.check_output( + [binary_file, "-version"], stderr=subprocess.STDOUT + ) + return binary_file + except FileNotFoundError: pass binary_file = name try: result = subprocess.run( - [binary_file, "-version"], text=True, capture_output=True + [binary_file, "-version"], text=True, capture_output=True, check=False ) if result.returncode == 0 and (f"version {version}") in result.stdout: return binary_file raise FileNotFoundError(f"{name} not found") - except FileNotFoundError as ex: + except FileNotFoundError: print( f""" Oops. It looks like {name} is not installed. It should be available under venv/bin