test: require test_begin_subtest before test_expect_success

Unify the subtests by requiring test_begin_subtest before
test_expect_success. (Similar change for test_expect_code will
follow.)

This increases clarity in the test scripts by having a separate line
for the start of the subtest with the heading, and makes it possible
to simplify the test infrastructure by making all subtests similar.
This commit is contained in:
Jani Nikula 2017-02-26 15:43:00 +02:00 committed by David Bremner
parent b8f12bd371
commit d0cd253b37
18 changed files with 124 additions and 103 deletions

View file

@ -189,17 +189,17 @@ Test harness library
There are a handful helper functions defined in the test harness
library for your script to use.
test_expect_success <message> <script>
This takes two strings as parameter, and evaluates the
<script>. If it yields success, test is considered
successful. <message> should state what it is testing.
test_begin_subtest <message>
Set the test description message for a subsequent test_expect_equal
Set the test description message for a subsequent test_expect_*
invocation (see below).
test_expect_success <script>
This takes a string as parameter, and evaluates the
<script>. If it yields success, test is considered
successful.
test_subtest_known_broken
Mark the current test as broken. Such tests are expected to fail.

View file

@ -18,20 +18,17 @@ fi
################################################################
# Test harness
test_expect_success 'success is reported like this' '
:
'
test_begin_subtest 'success is reported like this'
test_expect_success ':'
test_begin_subtest 'test runs if prerequisite is satisfied'
test_set_prereq HAVEIT
haveit=no
test_expect_success 'test runs if prerequisite is satisfied' '
test_have_prereq HAVEIT &&
haveit=yes
'
test_expect_success 'test_have_prereq HAVEIT && haveit=yes'
test_begin_subtest 'tests clean up after themselves'
clean=no
test_expect_success 'tests clean up after themselves' '
test_when_finished clean=yes
'
test_expect_success 'test_when_finished clean=yes'
cleaner=no
test_expect_code 1 'tests clean up even after a failure' '
@ -72,19 +69,15 @@ test_expect_equal "$output" "$expected"
################################################################
# Test mail store prepared in test-lib.sh
test_expect_success \
'test that mail store was created' \
'test -d "${MAIL_DIR}"'
test_begin_subtest 'test that mail store was created'
test_expect_success 'test -d "${MAIL_DIR}"'
test_begin_subtest 'mail store should be empty'
find "${MAIL_DIR}" -type f -print >should-be-empty
test_expect_success \
'mail store should be empty' \
'cmp -s /dev/null should-be-empty'
test_expect_success 'cmp -s /dev/null should-be-empty'
test_expect_success \
'NOTMUCH_CONFIG is set and points to an existing file' \
'test -f "${NOTMUCH_CONFIG}"'
test_begin_subtest 'NOTMUCH_CONFIG is set and points to an existing file'
test_expect_success 'test -f "${NOTMUCH_CONFIG}"'
test_begin_subtest 'PATH is set to build directory'
test_expect_equal \

View file

@ -3,18 +3,27 @@
test_description="online help"
. ./test-lib.sh || exit 1
test_expect_success 'notmuch --help' 'notmuch --help'
test_expect_success 'notmuch help' 'notmuch help'
test_expect_success 'notmuch --version' 'notmuch --version'
test_begin_subtest 'notmuch --help'
test_expect_success 'notmuch --help'
test_begin_subtest 'notmuch help'
test_expect_success 'notmuch help'
test_begin_subtest 'notmuch --version'
test_expect_success 'notmuch --version'
if [ $NOTMUCH_HAVE_MAN -eq 1 ]; then
test_expect_success 'notmuch --help tag' 'notmuch --help tag'
test_expect_success 'notmuch help tag' 'notmuch help tag'
test_begin_subtest 'notmuch --help tag'
test_expect_success 'notmuch --help tag'
test_begin_subtest 'notmuch help tag'
test_expect_success 'notmuch help tag'
else
test_expect_success 'notmuch --help tag (man pages not available)' \
'test_must_fail notmuch --help tag >/dev/null'
test_expect_success 'notmuch help tag (man pages not available)' \
'test_must_fail notmuch help tag >/dev/null'
test_begin_subtest 'notmuch --help tag (man pages not available)'
test_expect_success 'test_must_fail notmuch --help tag >/dev/null'
test_begin_subtest 'notmuch help tag (man pages not available)'
test_expect_success 'test_must_fail notmuch help tag >/dev/null'
fi
test_done

View file

@ -21,7 +21,8 @@ Compaction failed: Unsupported operation"
test_done
fi
test_expect_success "Running compact" "notmuch compact --backup=${TEST_DIRECTORY}/xapian.old"
test_begin_subtest "Running compact"
test_expect_success "notmuch compact --backup=${TEST_DIRECTORY}/xapian.old"
test_begin_subtest "Compact preserves database"
output=$(notmuch search \* | notmuch_search_sanitize)
@ -30,8 +31,8 @@ thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag1 unread)
thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag2 unread)
thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Three (inbox tag3 unread)"
test_expect_success 'Restoring Backup' \
'rm -Rf ${MAIL_DIR}/.notmuch/xapian &&
test_begin_subtest "Restoring Backup"
test_expect_success 'rm -Rf ${MAIL_DIR}/.notmuch/xapian &&
mv ${TEST_DIRECTORY}/xapian.old ${MAIL_DIR}/.notmuch/xapian'
test_begin_subtest "Checking restored backup"

View file

@ -338,9 +338,8 @@ Non-text part: application/pgp-signature
EOF
test_expect_equal_file OUTPUT EXPECTED
test_expect_success \
"--format=text --part=8, no part, expect error" \
"notmuch show --format=text --part=8 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=text --part=8, no part, expect error"
test_expect_success "notmuch show --format=text --part=8 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=json --part=0, full message"
notmuch show --format=json --part=0 'id:87liy5ap00.fsf@yoom.home.cworth.org' >OUTPUT
@ -444,9 +443,8 @@ cat <<EOF >EXPECTED
EOF
test_expect_equal_json "$(cat OUTPUT)" "$(cat EXPECTED)"
test_expect_success \
"--format=json --part=10, no part, expect error" \
"notmuch show --format=json --part=10 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=json --part=10, no part, expect error"
test_expect_success "notmuch show --format=json --part=10 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=raw"
notmuch show --format=raw 'id:87liy5ap00.fsf@yoom.home.cworth.org' >OUTPUT
@ -580,9 +578,8 @@ W6cAmQE4dcYrx/LPLtYLZm1jsGauE5hE
EOF
test_expect_equal_file OUTPUT EXPECTED
test_expect_success \
"--format=raw --part=10, no part, expect error" \
"notmuch show --format=raw --part=8 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=raw --part=10, no part, expect error"
test_expect_success "notmuch show --format=raw --part=8 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=mbox"
notmuch show --format=mbox 'id:87liy5ap00.fsf@yoom.home.cworth.org' >OUTPUT
@ -592,9 +589,8 @@ cat "${MAIL_DIR}"/multipart >>EXPECTED
echo >>EXPECTED
test_expect_equal_file OUTPUT EXPECTED
test_expect_success \
"--format=mbox --part=1, incompatible, expect error" \
"! notmuch show --format=mbox --part=1 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "--format=mbox --part=1, incompatible, expect error"
test_expect_success "! notmuch show --format=mbox --part=1 'id:87liy5ap00.fsf@yoom.home.cworth.org'"
test_begin_subtest "'notmuch reply' to a multipart message"
notmuch reply 'id:87liy5ap00.fsf@yoom.home.cworth.org' >OUTPUT

View file

@ -4,52 +4,58 @@ test_description="\"notmuch dump\" and \"notmuch restore\""
add_email_corpus
test_expect_success 'Dumping all tags' \
'generate_message &&
notmuch new &&
notmuch dump > dump.expected'
test_begin_subtest "Dumping all tags"
test_expect_success 'generate_message && notmuch new && notmuch dump > dump.expected'
# The use of from:cworth is rather arbitrary: it matches some of the
# email corpus' messages, but not all of them.
test_expect_success 'Dumping all tags II' \
test_begin_subtest "Dumping all tags II"
test_expect_success \
'notmuch tag +ABC +DEF -- from:cworth &&
notmuch dump > dump-ABC_DEF.expected &&
! cmp dump.expected dump-ABC_DEF.expected'
test_expect_success 'Clearing all tags' \
test_begin_subtest "Clearing all tags"
test_expect_success \
'sed -e "s/(\([^(]*\))$/()/" < dump.expected > clear.expected &&
notmuch restore --input=clear.expected &&
notmuch dump > clear.actual &&
test_cmp clear.expected clear.actual'
test_expect_success 'Accumulate original tags' \
test_begin_subtest "Clearing all tags"
test_expect_success \
'notmuch tag +ABC +DEF -- from:cworth &&
notmuch restore --accumulate < dump.expected &&
notmuch dump > dump.actual &&
test_cmp dump-ABC_DEF.expected dump.actual'
test_expect_success 'Restoring original tags' \
test_begin_subtest "Restoring original tags"
test_expect_success \
'notmuch restore --input=dump.expected &&
notmuch dump > dump.actual &&
test_cmp dump.expected dump.actual'
test_expect_success 'Restore with nothing to do' \
test_begin_subtest "Restore with nothing to do"
test_expect_success \
'notmuch restore < dump.expected &&
notmuch dump > dump.actual &&
test_cmp dump.expected dump.actual'
test_expect_success 'Accumulate with existing tags' \
test_begin_subtest "Accumulate with existing tags"
test_expect_success \
'notmuch restore --accumulate --input=dump.expected &&
notmuch dump > dump.actual &&
test_cmp dump.expected dump.actual'
test_expect_success 'Accumulate with no tags' \
test_begin_subtest "Accumulate with no tags"
test_expect_success \
'notmuch restore --accumulate < clear.expected &&
notmuch dump > dump.actual &&
test_cmp dump.expected dump.actual'
test_expect_success 'Accumulate with new tags' \
test_begin_subtest "Accumulate with new tags"
test_expect_success \
'notmuch restore --input=dump.expected &&
notmuch restore --accumulate --input=dump-ABC_DEF.expected &&
notmuch dump > OUTPUT.$test_count &&
@ -57,7 +63,8 @@ test_expect_success 'Accumulate with new tags' \
test_cmp dump-ABC_DEF.expected OUTPUT.$test_count'
# notmuch restore currently only considers the first argument.
test_expect_success 'Invalid restore invocation' \
test_begin_subtest "Invalid restore invocation"
test_expect_success \
'test_must_fail notmuch restore --input=dump.expected another_one'
test_begin_subtest "dump --output=outfile"

View file

@ -8,8 +8,8 @@ EXPECTED=$TEST_DIRECTORY/emacs.expected-output
add_email_corpus
# syntax errors in test-lib.el cause mysterious failures
test_expect_success 'Syntax of emacs test library' \
"${TEST_EMACS} -Q --batch --load $TEST_DIRECTORY/test-lib.el"
test_begin_subtest "Syntax of emacs test library"
test_expect_success "${TEST_EMACS} -Q --batch --load $TEST_DIRECTORY/test-lib.el"
test_begin_subtest "Basic notmuch-hello view in emacs"
test_emacs '(notmuch-hello)
@ -983,9 +983,8 @@ test_emacs "(let ((mm-text-html-renderer
# Different Emacs versions and renderers give very different results,
# so just check that something reasonable showed up. We first cat the
# output so the test framework will print it if the test fails.
test_expect_success "Rendering HTML mail with images" \
'cat OUTPUT && grep -q smiley OUTPUT'
test_begin_subtest "Rendering HTML mail with images"
test_expect_success 'cat OUTPUT && grep -q smiley OUTPUT'
test_begin_subtest "Search handles subprocess error exit codes"
cat > notmuch_fail <<EOF

View file

@ -52,7 +52,8 @@ test_expect_equal_json "$output" '[[[{"id": "XXXXX",
"content": "This is just a test message (#3)\n"}]},
[]]]]'
test_expect_success 'notmuch reply works with renamed file (without notmuch new)' 'notmuch reply id:${gen_msg_id}'
test_begin_subtest "notmuch reply works with renamed file (without notmuch new)"
test_expect_success 'notmuch reply id:${gen_msg_id}'
test_begin_subtest "notmuch new detects no file rename after tag->flag synchronization"
output=$(NOTMUCH_NEW)
@ -123,9 +124,10 @@ output+=$(notmuch search subject:"Message to lose maildir info" | notmuch_search
test_expect_equal "$output" "No new mail. Detected 1 file rename.
thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Message to lose maildir info (inbox)"
test_begin_subtest "Can remove unread tag from message in non-maildir directory"
add_message [subject]='"Non-maildir message"' [dir]=notmaildir [filename]='non-maildir-message'
expected=$(notmuch search --output=files subject:"Non-maildir message")
test_expect_success "Can remove unread tag from message in non-maildir directory" 'notmuch tag -unread subject:"Non-maildir message"'
test_expect_success 'notmuch tag -unread subject:"Non-maildir message"'
test_begin_subtest "Message in non-maildir directory does not get renamed"
output=$(notmuch search --output=files subject:"Non-maildir message")

View file

@ -28,7 +28,8 @@ add_gnupg_home
# Change this if we ship a new test key
FINGERPRINT="5AEAB11F5E33DCE875DDB75B6D92612D94E46381"
test_expect_success 'emacs delivery of signed message' \
test_begin_subtest "emacs delivery of signed message"
test_expect_success \
'emacs_fcc_message \
"test signed message 001" \
"This is a test signed message." \
@ -134,11 +135,12 @@ test_expect_equal_json \
"$expected"
mv "${GNUPGHOME}"{.bak,}
test_begin_subtest "emacs delivery of encrypted message with attachment"
# create a test encrypted message with attachment
cat <<EOF >TESTATTACHMENT
This is a test file.
EOF
test_expect_success 'emacs delivery of encrypted message with attachment' \
test_expect_success \
'emacs_fcc_message \
"test encrypted message 001" \
"This is a test encrypted message.\n" \
@ -264,7 +266,8 @@ test_expect_equal_json \
"$expected"
mv "${GNUPGHOME}"{.bak,}
test_expect_success 'emacs delivery of encrypted + signed message' \
test_begin_subtest "emacs delivery of encrypted + signed message"
test_expect_success \
'emacs_fcc_message \
"test encrypted message 002" \
"This is another test encrypted message.\n" \

View file

@ -23,14 +23,16 @@ FINGERPRINT=$(openssl x509 -fingerprint -in test_suite.pem -noout | sed -e 's/^.
add_gpgsm_home
test_expect_success 'emacs delivery of S/MIME signed message' \
test_begin_subtest "emacs delivery of S/MIME signed message"
test_expect_success \
'emacs_fcc_message \
"test signed message 001" \
"This is a test signed message." \
"(mml-secure-message-sign \"smime\")"'
test_begin_subtest "emacs delivery of S/MIME encrypted + signed message"
# Hard code the MML to avoid several interactive questions
test_expect_success 'emacs delivery of S/MIME encrypted + signed message' \
test_expect_success \
'emacs_fcc_message \
"test encrypted message 001" \
"<#secure method=smime mode=signencrypt keyfile=\\\"test_suite.pem\\\" certfile=\\\"test_suite.pem\\\">\nThis is a test encrypted message.\n"'

View file

@ -95,6 +95,7 @@ fi
test_begin_subtest '"notmuch new" is idempotent under arbitrary aborts'
test_expect_equal_file searchall expectall
test_expect_success "detected $outcount>10 abort points" "test $outcount -gt 10"
test_begin_subtest "detected $outcount>10 abort points"
test_expect_success "test $outcount -gt 10"
test_done

View file

@ -91,11 +91,11 @@ test_expect_equal_file expected output
# depends on the previous subtest leaving broken hook behind
test_expect_code 1 "post-new non-zero exit status (notmuch status)" "notmuch new"
test_begin_subtest "post-insert hook does not affect insert status"
rm_hooks
generate_message
create_failing_hook "post-insert"
test_expect_success "post-insert hook does not affect insert status" \
"notmuch insert < \"$gen_msg_filename\" > /dev/null"
test_expect_success "notmuch insert < \"$gen_msg_filename\" > /dev/null"
# test_begin_subtest "hook without executable permissions"
rm_hooks

View file

@ -10,8 +10,8 @@ if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
fi
test_begin_subtest "database checksum"
test_expect_success \
'database checksum' \
'( cd $TEST_DIRECTORY/test-databases &&
sha256sum --quiet --check --status ${dbtarball}.sha256 )'
@ -25,7 +25,8 @@ test_begin_subtest "path: search does not work with old database version"
output=$(notmuch search path:foo)
test_expect_equal "$output" ""
test_expect_success 'pre upgrade dump' 'notmuch dump | sort > pre-upgrade-dump'
test_begin_subtest "pre upgrade dump"
test_expect_success 'notmuch dump | sort > pre-upgrade-dump'
test_begin_subtest "database upgrade from format version 1"
output=$(notmuch new | sed -e 's/^Backing up tags to .*$/Backing up tags to FILENAME/')

View file

@ -5,7 +5,8 @@ test_description="error reporting for library"
add_email_corpus
test_expect_success "building database" "NOTMUCH_NEW"
test_begin_subtest "building database"
test_expect_success "NOTMUCH_NEW"
test_begin_subtest "Open null pointer"
test_C <<'EOF'

View file

@ -49,11 +49,11 @@ test_expect_equal 1 ${result}
notmuch count --lastmod '*' | cut -f2 > UUID
test_expect_success 'search succeeds with correct uuid' \
"notmuch search --uuid=$(cat UUID) '*'"
test_begin_subtest "search succeeds with correct uuid"
test_expect_success "notmuch search --uuid=$(cat UUID) '*'"
test_expect_success 'uuid works as global option ' \
"notmuch --uuid=$(cat UUID) search '*'"
test_begin_subtest "uuid works as global option"
test_expect_success "notmuch --uuid=$(cat UUID) search '*'"
test_expect_code 1 'uuid works as global option II' \
"notmuch --uuid=this-is-no-uuid search '*'"
@ -61,14 +61,14 @@ test_expect_code 1 'uuid works as global option II' \
test_expect_code 1 'search fails with incorrect uuid' \
"notmuch search --uuid=this-is-no-uuid '*'"
test_expect_success 'show succeeds with correct uuid' \
"notmuch show --uuid=$(cat UUID) '*'"
test_begin_subtest "show succeeds with correct uuid"
test_expect_success "notmuch show --uuid=$(cat UUID) '*'"
test_expect_code 1 'show fails with incorrect uuid' \
"notmuch show --uuid=this-is-no-uuid '*'"
test_expect_success 'tag succeeds with correct uuid' \
"notmuch tag --uuid=$(cat UUID) +test '*'"
test_begin_subtest "tag succeeds with correct uuid"
test_expect_success "notmuch tag --uuid=$(cat UUID) +test '*'"
test_expect_code 1 'tag fails with incorrect uuid' \
"notmuch tag --uuid=this-is-no-uuid '*' +test2"

View file

@ -9,12 +9,12 @@ test_expect_code 1 "error adding named query before initializing DB" \
add_email_corpus
test_expect_success "adding named query" \
"notmuch config set query.test \"$QUERYSTR\""
test_begin_subtest "adding named query"
test_expect_success "notmuch config set query.test \"$QUERYSTR\""
test_begin_subtest "adding nested named query"
QUERYSTR2="query:test and subject:Maildir"
test_expect_success "adding nested named query" \
"notmuch config set query.test2 \"$QUERYSTR2\""
test_expect_success "notmuch config set query.test2 \"$QUERYSTR2\""
test_begin_subtest "retrieve named query"
output=$(notmuch config get query.test)

View file

@ -916,13 +916,17 @@ test_subtest_known_broken () {
}
test_expect_success () {
test "$#" = 2 ||
error "bug in the test script: not 2 parameters to test_expect_success"
test_subtest_name="$1"
test_reset_state_
if ! test_skip "$@"
exec 1>&6 2>&7 # Restore stdout and stderr
if [ -z "$inside_subtest" ]; then
error "bug in the test script: test_expect_success without test_begin_subtest"
fi
inside_subtest=
test "$#" = 1 ||
error "bug in the test script: not 1 parameters to test_expect_success"
if ! test_skip "$test_subtest_name"
then
test_run_ "$2"
test_run_ "$1"
run_ret="$?"
# test_run_ may update missing external prerequisites
test_check_missing_external_prereqs_ "$@" ||
@ -930,7 +934,7 @@ test_expect_success () {
then
test_ok_
else
test_failure_ "$2"
test_failure_ "$1"
fi
fi
}

View file

@ -4,12 +4,14 @@ test_description='the verbosity options of the test framework itself.'
. ./test-lib.sh || exit 1
test_expect_success 'print something in test_expect_success and pass' '
test_begin_subtest 'print something in test_expect_success and pass'
test_expect_success '
echo "hello stdout" &&
echo "hello stderr" >&2 &&
true
'
test_expect_success 'print something in test_expect_success and fail' '
test_begin_subtest 'print something in test_expect_success and fail'
test_expect_success '
echo "hello stdout" &&
echo "hello stderr" >&2 &&
false