Consistently format errors in CI scripts (#2762)

.github/workflows/matchers/ci-custom.json

@@ -4,7 +4,7 @@
             "owner": "ci-custom",
             "pattern": [
                 {
-                    "regexp": "^ERROR (.*):(\\d+):(\\d+) - (.*)$",
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(.*)$",
                     "file": 1,
                     "line": 2,
                     "column": 3,

.github/workflows/matchers/lint-python.json

@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
           {
-          "regexp": "^(.*):(\\d+) - ([EFCDNW]\\d{3}.*)$",
+          "regexp": "^(.*):(\\d+): ([EFCDNW]\\d{3}.*)$",
           "file": 1,
           "line": 2,
           "message": 3
@@ -17,7 +17,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^(.*):(\\d+) - (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
+          "regexp": "^(.*):(\\d+): (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
           "file": 1,
           "line": 2,
           "message": 3

requirements_test.txt

@@ -1,7 +1,6 @@
 pylint==2.12.1
 flake8==4.0.1
 black==21.11b1
-pexpect==4.8.0
 pre-commit
 
 # Unit tests

script/ci-custom.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 
-from helpers import git_ls_files, filter_changed
+from helpers import styled, print_error_for_file, git_ls_files, filter_changed
+import argparse
 import codecs
 import collections
+import colorama
 import fnmatch
+import functools
 import os.path
 import re
-import subprocess
 import sys
 import time
-import functools
-import argparse
 
 sys.path.append(os.path.dirname(__file__))
 
@@ -30,6 +30,8 @@ def find_all(a_str, sub):
             column += len(sub)
 
 
+colorama.init()
+
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "files", nargs="*", default=[], help="files to be processed (regex on path)"
@@ -657,10 +659,8 @@ for fname in files:
 run_checks(LINT_POST_CHECKS, "POST")
 
 for f, errs in sorted(errors.items()):
-    print(f"\033[0;32m************* File \033[1;32m{f}\033[0m")
+    err_str = (f"{styled(colorama.Style.BRIGHT, f'{f}:{lineno}:{col}:')} {msg}\n" for lineno, col, msg in errs)
-    for lineno, col, msg in errs:
+    print_error_for_file(f, "\n".join(err_str))
-        print(f"ERROR {f}:{lineno}:{col} - {msg}")
-    print()
 
 if args.print_slowest:
     lint_times = []

script/clang-format

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+from helpers import print_error_for_file, get_output, git_ls_files, filter_changed
 import argparse
+import click
+import colorama
 import multiprocessing
 import os
 import queue
@@ -9,11 +12,6 @@ import subprocess
 import sys
 import threading
 
-import click
-
-sys.path.append(os.path.dirname(__file__))
-from helpers import get_output, git_ls_files, filter_changed
-
 
 def run_format(args, queue, lock, failed_files):
     """Takes filenames out of queue and runs clang-format on them."""
@@ -29,11 +27,7 @@ def run_format(args, queue, lock, failed_files):
         proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
         if proc.returncode != 0:
             with lock:
-                print()
+                print_error_for_file(path, proc.stderr)
-                print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
-                print(proc.stdout)
-                print(proc.stderr)
-                print()
                 failed_files.append(path)
         queue.task_done()
 
@@ -43,6 +37,8 @@ def progress_bar_show(value):
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
                         default=multiprocessing.cpu_count(),

script/clang-tidy

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 
+from helpers import print_error_for_file, get_output, filter_grep, \
+    build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
 import argparse
-import json
+import click
+import colorama
 import multiprocessing
 import os
 import queue
@@ -12,13 +15,6 @@ import sys
 import tempfile
 import threading
 
-import click
-import pexpect
-
-sys.path.append(os.path.dirname(__file__))
-from helpers import shlex_quote, get_output, filter_grep, \
-    build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
-
 
 def clang_options(idedata):
     cmd = [
@@ -87,23 +83,20 @@ def run_tidy(args, options, tmpdir, queue, lock, failed_files):
             invocation.append(name)
 
         if args.quiet:
-            invocation.append('-quiet')
+            invocation.append('--quiet')
+
+        if sys.stdout.isatty():
+            invocation.append('--use-color')
 
-        invocation.append(os.path.abspath(path))
         invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*")
+        invocation.append(os.path.abspath(path))
         invocation.append('--')
         invocation.extend(options)
-        invocation_s = ' '.join(shlex_quote(x) for x in invocation)
 
-        # Use pexpect for a pseudy-TTY with colored output
+        proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
-        output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
+        if proc.returncode != 0:
-                                 timeout=15 * 60)
-        if rc != 0:
             with lock:
-                print()
+                print_error_for_file(path, proc.stdout)
-                print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
-                print(output)
-                print()
                 failed_files.append(path)
         queue.task_done()
 
@@ -119,6 +112,8 @@ def split_list(a, n):
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
                         default=multiprocessing.cpu_count(),

script/helpers.py

@@ -1,4 +1,4 @@
-import codecs
+import colorama
 import os.path
 import re
 import subprocess
@@ -11,13 +11,18 @@ temp_folder = os.path.join(root_path, ".temp")
 temp_header_file = os.path.join(temp_folder, "all-include.cpp")
 
 
-def shlex_quote(s):
+def styled(color, msg, reset=True):
-    if not s:
+    prefix = ''.join(color) if isinstance(color, tuple) else color
-        return "''"
+    suffix = colorama.Style.RESET_ALL if reset else ''
-    if re.search(r"[^\w@%+=:,./-]", s) is None:
+    return prefix + msg + suffix
-        return s
 
-    return "'" + s.replace("'", "'\"'\"'") + "'"
+
+def print_error_for_file(file, body):
+    print(styled(colorama.Fore.GREEN, "### File ") + styled((colorama.Fore.GREEN, colorama.Style.BRIGHT), file))
+    print()
+    if body is not None:
+        print(body)
+        print()
 
 
 def build_all_include():

script/lint-python

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
 
 from __future__ import print_function
-from helpers import get_output, get_err, git_ls_files, filter_changed
+from helpers import styled, print_error_for_file, get_output, get_err, git_ls_files, filter_changed
-
 import argparse
+import colorama
 import os
 import re
 import sys
 
-sys.path.append(os.path.dirname(__file__))
-
 curfile = None
 
 
@@ -17,14 +15,18 @@ def print_error(file, lineno, msg):
     global curfile
 
     if curfile != file:
-        print()
+        print_error_for_file(file, None)
-        print("\033[0;32m************* File \033[1;32m{}\033[0m".format(file))
         curfile = file
 
-    print("{}:{} - {}".format(file, lineno, msg))
+    if lineno is not None:
+        print(f"{styled(colorama.Style.BRIGHT, f'{file}:{lineno}:')} {msg}")
+    else:
+        print(f"{styled(colorama.Style.BRIGHT, f'{file}:')} {msg}")
 
 
 def main():
+    colorama.init()
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "files", nargs="*", default=[], help="files to be processed (regex on path)"
@@ -56,6 +58,7 @@ def main():
 
     cmd = ["black", "--verbose", "--check"] + files
     print("Running black...")
+    print()
     log = get_err(*cmd)
     for line in log.splitlines():
         WOULD_REFORMAT = "would reformat"
@@ -65,7 +68,9 @@ def main():
             errors += 1
 
     cmd = ["flake8"] + files
+    print()
     print("Running flake8...")
+    print()
     log = get_output(*cmd)
     for line in log.splitlines():
         line = line.split(":", 4)
@@ -78,7 +83,9 @@ def main():
         errors += 1
 
     cmd = ["pylint", "-f", "parseable", "--persistent=n"] + files
+    print()
     print("Running pylint...")
+    print()
     log = get_output(*cmd)
     for line in log.splitlines():
         line = line.split(":", 3)