From 6f0f83660e222cfdd05b05ad134763a7ab26f097 Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Mon, 17 May 2021 11:11:09 +0300 Subject: [PATCH] test: aggregate-results updates notmuch-test will now call aggregate-results.sh with file list that it compiles based on the test ran, and aggregate-results will report failure is any of the test files are missing. With this notmuch-test no longer has to exit in non-parallel run if some test fail to write its report file -- so it works as parallel tests in this sense. Changed test_done() in test-lib.sh write report file in one write(2), so there is (even) less chance it being partially written. Also, now it writes 'total' last and aggregate-results.sh expects this line to exist in all report files for reporting to be successful. Added 'set -eu' to notmuch-test and modified code to work with these settings. That makes it harder to get mistakes slipped into committed code. --- test/aggregate-results.sh | 22 ++++++++++++++++-- test/notmuch-test | 47 ++++++++++++++++++++++----------------- test/test-lib.sh | 15 +++++++------ 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/test/aggregate-results.sh b/test/aggregate-results.sh index 75400e6e..6845fcf0 100755 --- a/test/aggregate-results.sh +++ b/test/aggregate-results.sh @@ -8,9 +8,16 @@ failed=0 broken=0 total=0 all_skipped=0 +rep_failed=0 for file do + if [ ! -f "$file" ]; then + echo "'$file' does not exist!" + rep_failed=$((rep_failed + 1)) + continue + fi + has_total=0 while read type value do case $type in @@ -24,18 +31,23 @@ do broken=$((broken + value)) ;; total) total=$((total + value)) + has_total=1 if [ "$value" -eq 0 ]; then all_skipped=$((all_skipped + 1)) fi esac done <"$file" + if [ "$has_total" -eq 0 ]; then + echo "'$file' lacks 'total ...'; results may be inconsistent." + failed=$((failed + 1)) + fi done pluralize_s () { [ "$1" -eq 1 ] && s='' || s='s'; } echo "Notmuch test suite complete." -if [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ]; then +if [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ] && [ "$rep_failed" -eq 0 ]; then pluralize_s "$total" printf "All $total test$s " if [ "$broken" -eq 0 ]; then @@ -70,10 +82,16 @@ if [ "$all_skipped" -ne 0 ]; then echo "All tests in $all_skipped file$s skipped." fi +if [ "$rep_failed" -ne 0 ]; then + pluralize_s "$rep_failed" + echo "$rep_failed test$s failed to report results." +fi + # Note that we currently do not consider skipped tests as failing the # build. -if [ "$success" -gt 0 ] && [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ] +if [ "$success" -gt 0 ] && [ "$fixed" -eq 0 ] && + [ "$failed" -eq 0 ] && [ "$rep_failed" -eq 0 ] then exit 0 else diff --git a/test/notmuch-test b/test/notmuch-test index cbd33f93..ce142f7c 100755 --- a/test/notmuch-test +++ b/test/notmuch-test @@ -18,12 +18,14 @@ fi # Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set. . $(dirname "$0")/export-dirs.sh || exit 1 +set -eu + TESTS= -for test in $NOTMUCH_TESTS; do +for test in ${NOTMUCH_TESTS-}; do TESTS="$TESTS $NOTMUCH_SRCDIR/test/$test" done -if [[ -z "$TESTS" ]]; then +if [ -z "$TESTS" ]; then TESTS="$NOTMUCH_SRCDIR/test/T[0-9][0-9][0-9]-*.sh" fi @@ -44,43 +46,46 @@ else TEST_TIMEOUT_CMD="" fi -trap 'e=$?; kill $!; exit $e' HUP INT TERM - META_FAILURE= +RES=0 # Run the tests -if test -z "$NOTMUCH_TEST_SERIALIZE" && command -v parallel >/dev/null ; then +if test -z "${NOTMUCH_TEST_SERIALIZE-}" && command -v parallel >/dev/null ; then test -t 1 && export COLORS_WITHOUT_TTY=t || : if parallel --version 2>&1 | grep -q GNU ; then echo "INFO: running tests with GNU parallel" - printf '%s\n' $TESTS | $TEST_TIMEOUT_CMD parallel + printf '%s\n' $TESTS | $TEST_TIMEOUT_CMD parallel || RES=$? else echo "INFO: running tests with moreutils parallel" - $TEST_TIMEOUT_CMD parallel -- $TESTS + $TEST_TIMEOUT_CMD parallel -- $TESTS || RES=$? fi - RES=$? - if [[ $RES != 0 ]]; then + if [ $RES != 0 ]; then META_FAILURE="parallel test suite returned error code $RES" fi else + trap 'e=$?; trap - 0; kill ${!-}; exit $e' 0 HUP INT TERM for test in $TESTS; do $TEST_TIMEOUT_CMD $test "$@" & - wait $! - # If the test failed without producing results, then it aborted, - # so we should abort, too. - RES=$? - testname=$(basename $test .sh) - if [[ $RES != 0 && ! -e "$NOTMUCH_BUILDDIR/test/test-results/$testname" ]]; then - META_FAILURE="Aborting on $testname (returned $RES)" - break - fi + wait $! && ev=0 || ev=$? + test $ev = 0 || RES=$ev done + trap - 0 HUP INT TERM + if [ $RES != 0 ]; then + META_FAILURE="some tests failed; first failed returned error code $RES" + fi fi -trap - HUP INT TERM # Report results +RESULT_FILES= +for file in $TESTS +do + file=${file##*/} # drop leading path components + file=${file%.sh} # drop trailing '.sh' + RESULT_FILES="$RESULT_FILES $NOTMUCH_BUILDDIR/test/test-results/$file" +done + echo -$NOTMUCH_SRCDIR/test/aggregate-results.sh $NOTMUCH_BUILDDIR/test/test-results/* -ev=$? +$NOTMUCH_SRCDIR/test/aggregate-results.sh $RESULT_FILES && ev=0 || ev=$? + if [ -n "$META_FAILURE" ]; then printf 'ERROR: %s\n' "$META_FAILURE" if [ $ev = 0 ]; then diff --git a/test/test-lib.sh b/test/test-lib.sh index 0bca76df..23f7c8f8 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -858,15 +858,16 @@ test_when_finished () { test_done () { GIT_EXIT_OK=t test_results_dir="$TEST_DIRECTORY/test-results" - mkdir -p "$test_results_dir" + test -d "$test_results_dir" || mkdir "$test_results_dir" test_results_path="$test_results_dir/$this_test" - echo "total $test_count" >> $test_results_path - echo "success $test_success" >> $test_results_path - echo "fixed $test_fixed" >> $test_results_path - echo "broken $test_broken" >> $test_results_path - echo "failed $test_failure" >> $test_results_path - echo "" >> $test_results_path + printf %s\\n \ + "success $test_success" \ + "fixed $test_fixed" \ + "broken $test_broken" \ + "failed $test_failure" \ + "total $test_count" \ + > $test_results_path [ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)'