Consistently format errors in CI scripts (#2762)

This commit is contained in:
Oxan van Leeuwen 2021-11-25 21:54:11 +01:00 committed by GitHub
parent 9681dfb458
commit 00965fe19e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 57 additions and 55 deletions

View file

@ -4,7 +4,7 @@
"owner": "ci-custom",
"pattern": [
{
"regexp": "^ERROR (.*):(\\d+):(\\d+) - (.*)$",
"regexp": "^(.*):(\\d+):(\\d+):\\s+(.*)$",
"file": 1,
"line": 2,
"column": 3,

View file

@ -5,7 +5,7 @@
"severity": "error",
"pattern": [
{
"regexp": "^(.*):(\\d+) - ([EFCDNW]\\d{3}.*)$",
"regexp": "^(.*):(\\d+): ([EFCDNW]\\d{3}.*)$",
"file": 1,
"line": 2,
"message": 3
@ -17,7 +17,7 @@
"severity": "error",
"pattern": [
{
"regexp": "^(.*):(\\d+) - (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
"regexp": "^(.*):(\\d+): (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
"file": 1,
"line": 2,
"message": 3

View file

@ -1,7 +1,6 @@
pylint==2.12.1
flake8==4.0.1
black==21.11b1
pexpect==4.8.0
pre-commit
# Unit tests

View file

@ -1,16 +1,16 @@
#!/usr/bin/env python3
from helpers import git_ls_files, filter_changed
from helpers import styled, print_error_for_file, git_ls_files, filter_changed
import argparse
import codecs
import collections
import colorama
import fnmatch
import functools
import os.path
import re
import subprocess
import sys
import time
import functools
import argparse
sys.path.append(os.path.dirname(__file__))
@ -30,6 +30,8 @@ def find_all(a_str, sub):
column += len(sub)
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument(
"files", nargs="*", default=[], help="files to be processed (regex on path)"
@ -657,10 +659,8 @@ for fname in files:
run_checks(LINT_POST_CHECKS, "POST")
for f, errs in sorted(errors.items()):
print(f"\033[0;32m************* File \033[1;32m{f}\033[0m")
for lineno, col, msg in errs:
print(f"ERROR {f}:{lineno}:{col} - {msg}")
print()
err_str = (f"{styled(colorama.Style.BRIGHT, f'{f}:{lineno}:{col}:')} {msg}\n" for lineno, col, msg in errs)
print_error_for_file(f, "\n".join(err_str))
if args.print_slowest:
lint_times = []

View file

@ -1,6 +1,9 @@
#!/usr/bin/env python3
from helpers import print_error_for_file, get_output, git_ls_files, filter_changed
import argparse
import click
import colorama
import multiprocessing
import os
import queue
@ -9,11 +12,6 @@ import subprocess
import sys
import threading
import click
sys.path.append(os.path.dirname(__file__))
from helpers import get_output, git_ls_files, filter_changed
def run_format(args, queue, lock, failed_files):
"""Takes filenames out of queue and runs clang-format on them."""
@ -29,11 +27,7 @@ def run_format(args, queue, lock, failed_files):
proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
if proc.returncode != 0:
with lock:
print()
print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
print(proc.stdout)
print(proc.stderr)
print()
print_error_for_file(path, proc.stderr)
failed_files.append(path)
queue.task_done()
@ -43,6 +37,8 @@ def progress_bar_show(value):
def main():
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--jobs', type=int,
default=multiprocessing.cpu_count(),

View file

@ -1,7 +1,10 @@
#!/usr/bin/env python3
from helpers import print_error_for_file, get_output, filter_grep, \
build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
import argparse
import json
import click
import colorama
import multiprocessing
import os
import queue
@ -12,13 +15,6 @@ import sys
import tempfile
import threading
import click
import pexpect
sys.path.append(os.path.dirname(__file__))
from helpers import shlex_quote, get_output, filter_grep, \
build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata, basepath
def clang_options(idedata):
cmd = [
@ -87,23 +83,20 @@ def run_tidy(args, options, tmpdir, queue, lock, failed_files):
invocation.append(name)
if args.quiet:
invocation.append('-quiet')
invocation.append('--quiet')
if sys.stdout.isatty():
invocation.append('--use-color')
invocation.append(os.path.abspath(path))
invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*")
invocation.append(os.path.abspath(path))
invocation.append('--')
invocation.extend(options)
invocation_s = ' '.join(shlex_quote(x) for x in invocation)
# Use pexpect for a pseudy-TTY with colored output
output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
timeout=15 * 60)
if rc != 0:
proc = subprocess.run(invocation, capture_output=True, encoding='utf-8')
if proc.returncode != 0:
with lock:
print()
print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
print(output)
print()
print_error_for_file(path, proc.stdout)
failed_files.append(path)
queue.task_done()
@ -119,6 +112,8 @@ def split_list(a, n):
def main():
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--jobs', type=int,
default=multiprocessing.cpu_count(),

View file

@ -1,4 +1,4 @@
import codecs
import colorama
import os.path
import re
import subprocess
@ -11,13 +11,18 @@ temp_folder = os.path.join(root_path, ".temp")
temp_header_file = os.path.join(temp_folder, "all-include.cpp")
def shlex_quote(s):
if not s:
return "''"
if re.search(r"[^\w@%+=:,./-]", s) is None:
return s
def styled(color, msg, reset=True):
prefix = ''.join(color) if isinstance(color, tuple) else color
suffix = colorama.Style.RESET_ALL if reset else ''
return prefix + msg + suffix
return "'" + s.replace("'", "'\"'\"'") + "'"
def print_error_for_file(file, body):
print(styled(colorama.Fore.GREEN, "### File ") + styled((colorama.Fore.GREEN, colorama.Style.BRIGHT), file))
print()
if body is not None:
print(body)
print()
def build_all_include():

View file

@ -1,15 +1,13 @@
#!/usr/bin/env python3
from __future__ import print_function
from helpers import get_output, get_err, git_ls_files, filter_changed
from helpers import styled, print_error_for_file, get_output, get_err, git_ls_files, filter_changed
import argparse
import colorama
import os
import re
import sys
sys.path.append(os.path.dirname(__file__))
curfile = None
@ -17,14 +15,18 @@ def print_error(file, lineno, msg):
global curfile
if curfile != file:
print()
print("\033[0;32m************* File \033[1;32m{}\033[0m".format(file))
print_error_for_file(file, None)
curfile = file
print("{}:{} - {}".format(file, lineno, msg))
if lineno is not None:
print(f"{styled(colorama.Style.BRIGHT, f'{file}:{lineno}:')} {msg}")
else:
print(f"{styled(colorama.Style.BRIGHT, f'{file}:')} {msg}")
def main():
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument(
"files", nargs="*", default=[], help="files to be processed (regex on path)"
@ -56,6 +58,7 @@ def main():
cmd = ["black", "--verbose", "--check"] + files
print("Running black...")
print()
log = get_err(*cmd)
for line in log.splitlines():
WOULD_REFORMAT = "would reformat"
@ -65,7 +68,9 @@ def main():
errors += 1
cmd = ["flake8"] + files
print()
print("Running flake8...")
print()
log = get_output(*cmd)
for line in log.splitlines():
line = line.split(":", 4)
@ -78,7 +83,9 @@ def main():
errors += 1
cmd = ["pylint", "-f", "parseable", "--persistent=n"] + files
print()
print("Running pylint...")
print()
log = get_output(*cmd)
for line in log.splitlines():
line = line.split(":", 3)