From 00965fe19e857a83cb7f500da640c0c80694e79e Mon Sep 17 00:00:00 2001
From: Oxan van Leeuwen <oxan@oxanvanleeuwen.nl>
Date: Thu, 25 Nov 2021 21:54:11 +0100
Subject: [PATCH] Consistently format errors in CI scripts (#2762)

---
 .github/workflows/matchers/ci-custom.json   |  2 +-
 .github/workflows/matchers/lint-python.json |  4 +--
 requirements_test.txt                       |  1 -
 script/ci-custom.py                         | 16 +++++-----
 script/clang-format                         | 16 ++++------
 script/clang-tidy                           | 33 +++++++++------------
 script/helpers.py                           | 19 +++++++-----
 script/lint-python                          | 21 ++++++++-----
 8 files changed, 57 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/matchers/ci-custom.json b/.github/workflows/matchers/ci-custom.json
index 4e1eafff5e..9888dbb440 100644
--- a/.github/workflows/matchers/ci-custom.json
+++ b/.github/workflows/matchers/ci-custom.json
@@ -4,7 +4,7 @@
             "owner": "ci-custom",
             "pattern": [
                 {
-                    "regexp": "^ERROR (.*):(\\d+):(\\d+) - (.*)$",
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(.*)$",
                     "file": 1,
                     "line": 2,
                     "column": 3,
diff --git a/.github/workflows/matchers/lint-python.json b/.github/workflows/matchers/lint-python.json
index decbe36c4a..e88f74f6f6 100644
--- a/.github/workflows/matchers/lint-python.json
+++ b/.github/workflows/matchers/lint-python.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
           {
-          "regexp": "^(.*):(\\d+) - ([EFCDNW]\\d{3}.*)$",
+          "regexp": "^(.*):(\\d+): ([EFCDNW]\\d{3}.*)$",
           "file": 1,
           "line": 2,
           "message": 3
@@ -17,7 +17,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^(.*):(\\d+) - (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
+          "regexp": "^(.*):(\\d+): (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
           "file": 1,
           "line": 2,
           "message": 3
diff --git a/requirements_test.txt b/requirements_test.txt
index ad98ec2a4f..b916e8bb1b 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -1,7 +1,6 @@
 pylint==2.12.1
 flake8==4.0.1
 black==21.11b1
-pexpect==4.8.0
 pre-commit
 
 # Unit tests
diff --git a/script/ci-custom.py b/script/ci-custom.py
index 89550afd3d..3f01fb81bf 100755
--- a/script/ci-custom.py
+++ b/script/ci-custom.py
@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 
-from helpers import git_ls_files, filter_changed
+from helpers import styled, print_error_for_file, git_ls_files, filter_changed
+import argparse
 import codecs
 import collections
+import colorama
 import fnmatch
+import functools
 import os.path
 import re
-import subprocess
 import sys
 import time
-import functools
-import argparse
 
 sys.path.append(os.path.dirname(__file__))
 
@@ -30,6 +30,8 @@ def find_all(a_str, sub):
             column += len(sub)
 
 
+colorama.init()
+
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "files", nargs="*", default=[], help="files to be processed (regex on path)"
@@ -657,10 +659,8 @@ for fname in files:
 run_checks(LINT_POST_CHECKS, "POST")
 
 for f, errs in sorted(errors.items()):
-    print(f"\033[0;32m************* File \033[1;32m{f}\033[0m")
-    for lineno, col, msg in errs:
-        print(f"ERROR {f}:{lineno}:{col} - {msg}")
-    print()
+    err_str = (f"{styled(colorama.Style.BRIGHT, f'{f}:{lineno}:{col}:')} {msg}\n" for lineno, col, msg in errs)
+    print_error_for_file(f, "\n".join(err_str))
 
 if args.print_slowest:
     lint_times = []
diff --git a/script/clang-format b/script/clang-format
index d6588f1ccb..515df4c027 100755
--- a/script/clang-format
+++ b/script/clang-format
@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+from helpers import print_error_for_file, get_output, git_ls_files, filter_changed
 import argparse
+import click
+import colorama
 import multiprocessing
 import os
 import queue
@@ -9,11 +12,6 @@ import subprocess
 import sys
 import threading
 
-import click
-
-sys.path.append(os.path.dirname(__file__))
-from helpers import get_output, git_ls_files, filter_changed
-
 
 def run_format(args, queue, lock, failed_files):
     """Takes filenames out of queue and runs clang-format on them."""
@@ -29,11 +27,7 @@ def run_format(args, queue, lock, failed_files):
         proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
         if proc.returncode != 0:
             with lock:
-                print()
-                print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
-                print(proc.stdout)
-                print(proc.stderr)
-                print()
+                print_error_for_file(path, proc.stderr)
                 failed_files.append(path)
         queue.task_done()
 
@@ -43,6 +37,8 @@ def progress_bar_show(value):
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
                         default=multiprocessing.cpu_count(),
diff --git a/script/clang-tidy b/script/clang-tidy
index ad5fdfeb04..7450084634 100755
--- a/script/clang-tidy
+++ b/script/clang-tidy
@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 
+from helpers import print_error_for_file, get_output, filter_grep, \
+    build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
 import argparse
-import json
+import click
+import colorama
 import multiprocessing
 import os
 import queue
@@ -12,13 +15,6 @@ import sys
 import tempfile
 import threading
 
-import click
-import pexpect
-
-sys.path.append(os.path.dirname(__file__))
-from helpers import shlex_quote, get_output, filter_grep, \
-    build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
-
 
 def clang_options(idedata):
     cmd = [
@@ -87,23 +83,20 @@ def run_tidy(args, options, tmpdir, queue, lock, failed_files):
             invocation.append(name)
 
         if args.quiet:
-            invocation.append('-quiet')
+            invocation.append('--quiet')
+
+        if sys.stdout.isatty():
+            invocation.append('--use-color')
 
-        invocation.append(os.path.abspath(path))
         invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*")
+        invocation.append(os.path.abspath(path))
         invocation.append('--')
         invocation.extend(options)
-        invocation_s = ' '.join(shlex_quote(x) for x in invocation)
 
-        # Use pexpect for a pseudy-TTY with colored output
-        output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
-                                 timeout=15 * 60)
-        if rc != 0:
+        proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
+        if proc.returncode != 0:
             with lock:
-                print()
-                print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
-                print(output)
-                print()
+                print_error_for_file(path, proc.stdout)
                 failed_files.append(path)
         queue.task_done()
 
@@ -119,6 +112,8 @@ def split_list(a, n):
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
                         default=multiprocessing.cpu_count(),
diff --git a/script/helpers.py b/script/helpers.py
index 430d8a8e7f..abf970b8a2 100644
--- a/script/helpers.py
+++ b/script/helpers.py
@@ -1,4 +1,4 @@
-import codecs
+import colorama
 import os.path
 import re
 import subprocess
@@ -11,13 +11,18 @@ temp_folder = os.path.join(root_path, ".temp")
 temp_header_file = os.path.join(temp_folder, "all-include.cpp")
 
 
-def shlex_quote(s):
-    if not s:
-        return "''"
-    if re.search(r"[^\w@%+=:,./-]", s) is None:
-        return s
+def styled(color, msg, reset=True):
+    prefix = ''.join(color) if isinstance(color, tuple) else color
+    suffix = colorama.Style.RESET_ALL if reset else ''
+    return prefix + msg + suffix
 
-    return "'" + s.replace("'", "'\"'\"'") + "'"
+
+def print_error_for_file(file, body):
+    print(styled(colorama.Fore.GREEN, "### File ") + styled((colorama.Fore.GREEN, colorama.Style.BRIGHT), file))
+    print()
+    if body is not None:
+        print(body)
+        print()
 
 
 def build_all_include():
diff --git a/script/lint-python b/script/lint-python
index 41885b9672..8ee038a661 100755
--- a/script/lint-python
+++ b/script/lint-python
@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
 
 from __future__ import print_function
-from helpers import get_output, get_err, git_ls_files, filter_changed
-
+from helpers import styled, print_error_for_file, get_output, get_err, git_ls_files, filter_changed
 import argparse
+import colorama
 import os
 import re
 import sys
 
-sys.path.append(os.path.dirname(__file__))
-
 curfile = None
 
 
@@ -17,14 +15,18 @@ def print_error(file, lineno, msg):
     global curfile
 
     if curfile != file:
-        print()
-        print("\033[0;32m************* File \033[1;32m{}\033[0m".format(file))
+        print_error_for_file(file, None)
         curfile = file
 
-    print("{}:{} - {}".format(file, lineno, msg))
+    if lineno is not None:
+        print(f"{styled(colorama.Style.BRIGHT, f'{file}:{lineno}:')} {msg}")
+    else:
+        print(f"{styled(colorama.Style.BRIGHT, f'{file}:')} {msg}")
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "files", nargs="*", default=[], help="files to be processed (regex on path)"
@@ -56,6 +58,7 @@ def main():
 
     cmd = ["black", "--verbose", "--check"] + files
     print("Running black...")
+    print()
     log = get_err(*cmd)
     for line in log.splitlines():
         WOULD_REFORMAT = "would reformat"
@@ -65,7 +68,9 @@ def main():
             errors += 1
 
     cmd = ["flake8"] + files
+    print()
     print("Running flake8...")
+    print()
     log = get_output(*cmd)
     for line in log.splitlines():
         line = line.split(":", 4)
@@ -78,7 +83,9 @@ def main():
         errors += 1
 
     cmd = ["pylint", "-f", "parseable", "--persistent=n"] + files
+    print()
     print("Running pylint...")
+    print()
     log = get_output(*cmd)
     for line in log.splitlines():
         line = line.split(":", 3)