Fix ci-custom.py const.py ordered check and improve code (#1222)

This commit is contained in:
Otto Winter 2020-07-29 18:19:48 +02:00 committed by GitHub
parent 2697c9465b
commit f9fceb7ffc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 61 additions and 18 deletions

View file

@ -7,9 +7,18 @@ import os.path
import re import re
import subprocess import subprocess
import sys import sys
import time
import functools
import argparse
sys.path.append(os.path.dirname(__file__))
from helpers import git_ls_files, filter_changed
def find_all(a_str, sub): def find_all(a_str, sub):
if not a_str.find(sub):
# Optimization: If str is not in whole text, then do not try
# on each line
return
for i, line in enumerate(a_str.splitlines()): for i, line in enumerate(a_str.splitlines()):
column = 0 column = 0
while True: while True:
@ -20,15 +29,24 @@ def find_all(a_str, sub):
column += len(sub) column += len(sub)
command = ['git', 'ls-files', '-s'] parser = argparse.ArgumentParser()
proc = subprocess.Popen(command, stdout=subprocess.PIPE) parser.add_argument('files', nargs='*', default=[],
output, err = proc.communicate() help='files to be processed (regex on path)')
lines = [x.split() for x in output.decode('utf-8').splitlines()] parser.add_argument('-c', '--changed', action='store_true',
EXECUTABLE_BIT = { help='Only run on changed files')
s[3].strip(): int(s[0]) for s in lines parser.add_argument('--print-slowest', action='store_true',
} help='Print the slowest checks')
files = [s[3].strip() for s in lines] args = parser.parse_args()
files = list(filter(os.path.exists, files))
EXECUTABLE_BIT = git_ls_files()
files = list(EXECUTABLE_BIT.keys())
# Match against re
file_name_re = re.compile('|'.join(args.files))
files = [p for p in files if file_name_re.search(p)]
if args.changed:
files = filter_changed(files)
files.sort() files.sort()
file_types = ('.h', '.c', '.cpp', '.tcc', '.yaml', '.yml', '.ini', '.txt', '.ico', '.svg', file_types = ('.h', '.c', '.cpp', '.tcc', '.yaml', '.yml', '.ini', '.txt', '.ico', '.svg',
@ -60,7 +78,14 @@ def run_check(lint_obj, fname, *args):
def run_checks(lints, fname, *args): def run_checks(lints, fname, *args):
for lint in lints: for lint in lints:
add_errors(fname, run_check(lint, fname, *args)) start = time.process_time()
try:
add_errors(fname, run_check(lint, fname, *args))
except Exception:
print(f"Check {lint['func'].__name__} on file {fname} failed:")
raise
duration = time.process_time() - start
lint.setdefault('durations', []).append(duration)
def _add_check(checks, func, include=None, exclude=None): def _add_check(checks, func, include=None, exclude=None):
@ -96,6 +121,7 @@ def lint_re_check(regex, **kwargs):
decor = lint_content_check(**kwargs) decor = lint_content_check(**kwargs)
def decorator(func): def decorator(func):
@functools.wraps(func)
def new_func(fname, content): def new_func(fname, content):
errors = [] errors = []
for match in prog.finditer(content): for match in prog.finditer(content):
@ -109,6 +135,7 @@ def lint_re_check(regex, **kwargs):
continue continue
errors.append((lineno, col+1, err)) errors.append((lineno, col+1, err))
return errors return errors
return decor(new_func) return decor(new_func)
return decorator return decorator
@ -117,6 +144,7 @@ def lint_content_find_check(find, **kwargs):
decor = lint_content_check(**kwargs) decor = lint_content_check(**kwargs)
def decorator(func): def decorator(func):
@functools.wraps(func)
def new_func(fname, content): def new_func(fname, content):
find_ = find find_ = find
if callable(find): if callable(find):
@ -206,6 +234,10 @@ def lint_no_long_delays(fname, match):
@lint_content_check(include=['esphome/const.py']) @lint_content_check(include=['esphome/const.py'])
def lint_const_ordered(fname, content): def lint_const_ordered(fname, content):
"""Lint that value in const.py are ordered.
Reason: Otherwise people add it to the end, and then that results in merge conflicts.
"""
lines = content.splitlines() lines = content.splitlines()
errors = [] errors = []
for start in ['CONF_', 'ICON_', 'UNIT_']: for start in ['CONF_', 'ICON_', 'UNIT_']:
@ -217,10 +249,10 @@ def lint_const_ordered(fname, content):
continue continue
target = next(i for i, l in ordered if l == ml) target = next(i for i, l in ordered if l == ml)
target_text = next(l for i, l in matching if target == i) target_text = next(l for i, l in matching if target == i)
errors.append((ml, None, errors.append((mi, 1,
"Constant {} is not ordered, please make sure all constants are ordered. " f"Constant {highlight(ml)} is not ordered, please make sure all "
"See line {} (should go to line {}, {})" f"constants are ordered. See line {mi} (should go to line {target}, "
"".format(highlight(ml), mi, target, target_text))) f"{target_text})"))
return errors return errors
@ -302,7 +334,7 @@ def lint_no_arduino_framework_functions(fname, match):
) )
@lint_re_check(r'[^\w\d]byte\s+[\w\d]+\s*=.*', include=cpp_include, exclude={ @lint_re_check(r'[^\w\d]byte\s+[\w\d]+\s*=', include=cpp_include, exclude={
'esphome/components/tuya/tuya.h', 'esphome/components/tuya/tuya.h',
}) })
def lint_no_byte_datatype(fname, match): def lint_no_byte_datatype(fname, match):
@ -385,8 +417,8 @@ def lint_pragma_once(fname, content):
return None return None
@lint_re_check(r'(whitelist|blacklist|slave)', exclude=['script/ci-custom.py'], @lint_re_check(r'(whitelist|blacklist|slave)',
flags=re.IGNORECASE | re.MULTILINE) exclude=['script/ci-custom.py'], flags=re.IGNORECASE | re.MULTILINE)
def lint_inclusive_language(fname, match): def lint_inclusive_language(fname, match):
# From https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=49decddd39e5f6132ccd7d9fdc3d7c470b0061bb # From https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=49decddd39e5f6132ccd7d9fdc3d7c470b0061bb
return ("Avoid the use of whitelist/blacklist/slave.\n" return ("Avoid the use of whitelist/blacklist/slave.\n"
@ -471,4 +503,15 @@ for f, errs in sorted(errors.items()):
print(f"ERROR {f}:{lineno}:{col} - {msg}") print(f"ERROR {f}:{lineno}:{col} - {msg}")
print() print()
if args.print_slowest:
lint_times = []
for lint in LINT_FILE_CHECKS + LINT_CONTENT_CHECKS + LINT_POST_CHECKS:
durations = lint.get('durations', [])
lint_times.append((sum(durations), len(durations), lint['func'].__name__))
lint_times.sort(key=lambda x: -x[0])
for i in range(min(len(lint_times), 10)):
dur, invocations, name = lint_times[i]
print(f" - '{name}' took {dur:.2f}s total (ran on {invocations} files)")
print(f"Total time measured: {sum(x[0] for x in lint_times):.2f}s")
sys.exit(len(errors)) sys.exit(len(errors))

View file

@ -6,6 +6,6 @@ cd "$(dirname "$0")/.."
set -x set -x
script/ci-custom.py script/ci-custom.py -c
script/lint-python -c script/lint-python -c
script/lint-cpp -c script/lint-cpp -c