test: support testing notmuch as installed

We put some effort into testing the built copy rather than some
installed copy. On the other hand for people like packagers, testing
the installed copy is also of interest.

When NOTMUCH_TEST_INSTALLED is set to a nonempty value, tests do not
require a built notmuch tree or running configure.

Some of the tests marked as broken when running against installed
notmuch are probably fixable.
This commit is contained in:
David Bremner 2023-04-09 11:26:26 -03:00
parent 73f3081160
commit ec26eeaeec
26 changed files with 139 additions and 16 deletions

View file

@ -137,6 +137,23 @@ detection of missing prerequisites. In the future we may treat tests
unable to run because of missing prerequisites, but not explicitly
skipped by the user, as failures.
Testing installed notmuch
-------------------------
Systems integrators (e.g. Linux distros) may wish to test an installed
version of notmuch. This can be done be running
$ NOTMUCH_TEST_INSTALLED=1 ./test/notmuch-test
In this scenario the test suite does not assume a built tree, and in
particular cannot rely on the output of 'configure'. You may want to
set certain feature environment variables ('NOTMUCH_HAVE_*') directly
if you know those apply to your installed notmuch). Consider also
setting TERM=dumb if the value of TERM cannot be used (e.g. in a
chroot with missing terminfo). Note that having a built tree may cause
surprising/broken results for NOTMUCH_TEST_INSTALLED, so consider
cleaning first.
Writing Tests
-------------
The test script is written as a shell script. It is to be named as

View file

@ -66,6 +66,7 @@ test_begin_subtest 'NOTMUCH_CONFIG is set and points to an existing file'
test_expect_success 'test -f "${NOTMUCH_CONFIG}"'
test_begin_subtest 'PATH is set to build directory'
test_subtest_broken_for_installed
test_expect_equal \
"$(dirname ${TEST_DIRECTORY})" \
"$(echo $PATH|cut -f1 -d: | sed -e 's,/test/valgrind/bin$,,')"

View file

@ -19,6 +19,9 @@ if [ "${NOTMUCH_HAVE_MAN-0}" = "1" ]; then
test_begin_subtest 'notmuch help tag'
test_expect_success 'notmuch help tag'
else
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest 'notmuch --help tag (man pages not available)'
test_expect_success 'test_must_fail notmuch --help tag >/dev/null'

View file

@ -65,6 +65,7 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\",
\"tags\": [\"inbox\",
\"unread\"]}]"
if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_begin_subtest "Search message: json, 64-bit timestamp"
if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then
test_subtest_known_broken
@ -81,6 +82,7 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\",
\"query\": [\"id:$gen_msg_id\", null],
\"tags\": [\"inbox\",
\"unread\"]}]"
fi # NOTMUCH_TEST_INSTALLED undefined / empty
test_begin_subtest "Format version: too low"
test_expect_code 20 "notmuch search --format-version=0 \\*"

View file

@ -139,6 +139,7 @@ notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
test_begin_subtest "Check for a safe set of message-ids"
test_subtest_broken_for_installed
notmuch search --output=messages from:cworth | sed s/^id:// > EXPECTED
notmuch search --output=messages from:cworth | sed s/^id:// |\
$TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT
@ -246,6 +247,7 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count
test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
test_begin_subtest 'format=batch-tag, checking encoded output'
test_subtest_broken_for_installed
NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\
awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count
NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count

View file

@ -258,6 +258,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "Verify that sent messages are saved/searchable (via FCC)"
test_subtest_broken_for_installed
notmuch new > /dev/null
output=$(notmuch search 'subject:"testing message sent via SMTP"' | notmuch_search_sanitize)
test_expect_equal "$output" "thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; Testing message sent via SMTP (inbox)"
@ -350,6 +351,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "Reply within emacs"
test_subtest_broken_for_installed
test_emacs '(let ((message-hidden-headers ''()))
(notmuch-search "subject:\"testing message sent via SMTP\"")
(notmuch-test-wait)

View file

@ -30,6 +30,7 @@ msg_file=$(notmuch search --output=files subject:signed-message-sent-via-SMTP)
test_expect_equal_message_body sent_message "$msg_file"
test_begin_subtest "signed part content-type indexing"
test_subtest_broken_for_installed
notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize > OUTPUT
cat <<EOF >EXPECTED
thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed)

View file

@ -183,6 +183,7 @@ test_begin_subtest "show PKCS#7 SignedData outputs valid JSON"
output=$(notmuch show --format=json id:smime-onepart-signed@protected-headers.example)
test_valid_json "$output"
if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_begin_subtest "Verify signature on PKCS#7 SignedData message"
if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then
test_subtest_known_broken
@ -194,6 +195,7 @@ test_json_nodes <<<"$output" \
'expires:[0][0][0]["crypto"]["signed"]["status"][0]["expires"]=2611032858' \
'fingerprint:[0][0][0]["crypto"]["signed"]["status"][0]["fingerprint"]="702BA4B157F1E2B7D16B0C6A5FFC8A7DE2057DEB"' \
'status:[0][0][0]["crypto"]["signed"]["status"][0]["status"]="good"'
fi # NOTMUCH_TEST_INSTALLED undefined / empty
test_begin_subtest "Verify signature on PKCS#7 SignedData message signer User ID"
if [ $NOTMUCH_GMIME_X509_CERT_VALIDITY -ne 1 ]; then

View file

@ -11,6 +11,10 @@ test_description='exception symbol hiding'
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest 'running test' run_test
mkdir -p ${PWD}/fakedb/.notmuch
$TEST_DIRECTORY/symbol-test ${PWD}/fakedb ${PWD}/nonexistent 2>&1 \

View file

@ -4,6 +4,10 @@ test_description="python bindings"
test_require_external_prereq ${NOTMUCH_PYTHON}
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
add_email_corpus
add_gnupg_home

View file

@ -2,6 +2,10 @@
test_description="python bindings (pytest)"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
if [ "${NOTMUCH_HAVE_PYTHON3_CFFI-0}" = "0" -o "${NOTMUCH_HAVE_PYTHON3_PYTEST-0}" = "0" ]; then
test_done
fi

View file

@ -2,7 +2,7 @@
test_description="ruby bindings"
. $(dirname "$0")/test-lib.sh || exit 1
if [ "${NOTMUCH_HAVE_RUBY_DEV}" = "0" ]; then
if [ -z "${NOTMUCH_TEST_INSTALLED-}" -a "${NOTMUCH_HAVE_RUBY_DEV-0}" = "0" ]; then
test_subtest_missing_external_prereq_["ruby development files"]=t
fi
@ -15,7 +15,11 @@ test_ruby() {
db = Notmuch::Database.new()
EOF
cat
) | $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"> OUTPUT
) | if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
ruby
else
$NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"
fi> OUTPUT
test_expect_equal_file EXPECTED OUTPUT
}

View file

@ -2,6 +2,10 @@
test_description="argument parsing"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest "sanity check"
$TEST_DIRECTORY/arg-test pos1 --keyword=one --boolean --string=foo pos2 --int=7 --flag=one --flag=three > OUTPUT
cat <<EOF > EXPECTED

View file

@ -2,6 +2,10 @@
test_description="hex encoding and decoding"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest "round trip"
find $NOTMUCH_SRCDIR/test/corpora/default -type f -print | sort | xargs cat > EXPECTED
$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT

View file

@ -2,6 +2,10 @@
test_description="date/time parser module"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
# Sanity/smoke tests for the date/time parser independent of notmuch
_date () {

View file

@ -3,6 +3,10 @@ test_description="database version and feature compatibility"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest "future database versions abort open"
${TEST_DIRECTORY}/make-db-version ${MAIL_DIR} 9999 ""
output=$(notmuch search x 2>&1 | sed 's/\(database at\) .*/\1 FILENAME/')

View file

@ -3,6 +3,10 @@ test_description="API tests for notmuch_message_*"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED}" ]; then
test_done
fi
add_email_corpus
test_begin_subtest "building database"

View file

@ -21,6 +21,10 @@ test_description='thread breakage during reindexing'
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
message_a () {
mkdir -p ${MAIL_DIR}/cur
cat > ${MAIL_DIR}/cur/a <<EOF

View file

@ -3,6 +3,10 @@ test_description="message id parsing"
. $(dirname "$0")/test-lib.sh || exit 1
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_done
fi
test_begin_subtest "good message ids"
${TEST_DIRECTORY}/message-id-parse <<EOF >OUTPUT
<018b1a8f2d1df62e804ce88b65401304832dfbbf.1346614915.git.jani@nikula.org>

View file

@ -7,6 +7,11 @@ if [ "${NOTMUCH_HAVE_ASAN-0}" != "1" ]; then
test_done
fi
if [ -n "${LD_PRELOAD-}" ]; then
printf "Skipping due to ASAN LD_PRELOAD restrictions\n"
test_done
fi
add_email_corpus
TEST_CFLAGS="${TEST_CFLAGS:-} -fsanitize=address"

View file

@ -233,6 +233,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "invoke as nmbug sets defaults"
test_subtest_broken_for_installed
"$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^\(prefix\|repository\)' | notmuch_dir_sanitize > OUTPUT
cat <<EOF > EXPECTED
prefix = notmuch::
@ -241,6 +242,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "env variable NOTMUCH_GIT_DIR works when invoked as nmbug"
test_subtest_broken_for_installed
NOTMUCH_GIT_DIR=`pwd`/foo "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
cat <<EOF > EXPECTED
repository = CWD/foo
@ -256,6 +258,7 @@ test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "env variable NOTMUCH_GIT_DIR overrides config when invoked as 'nmbug'"
test_subtest_broken_for_installed
notmuch config set git.path `pwd`/bar
NOTMUCH_GIT_DIR=`pwd`/remote.git "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
notmuch config set git.path
@ -274,6 +277,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as 'nmbug'"
test_subtest_broken_for_installed
NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
cat <<EOF > EXPECTED
prefix = env::
@ -281,6 +285,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as nmbug"
test_subtest_broken_for_installed
NOTMUCH_GIT_PREFIX=foo:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
cat <<EOF > EXPECTED
prefix = foo::
@ -288,6 +293,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "env variable NOTMUCH_GIT_PREFIX overrides config when invoked as 'nmbug'"
test_subtest_broken_for_installed
notmuch config set git.tag_prefix config::
NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
notmuch config set git.path

View file

@ -24,7 +24,7 @@ find_builddir () {
if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then
export NOTMUCH_BUILDDIR="$(find_builddir "$(pwd)")"
if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then
if [ -z "${NOTMUCH_BUILDDIR}" -a "${NOTMUCH_TEST_INSTALLED-0}" = "0" ]; then
echo "Run tests in a subdir of built notmuch tree." >&2
exit 1
fi

View file

@ -20,6 +20,14 @@ fi
set -eu
# Where to run the tests
# XXX FIXME this code is duplicated with test-lib.sh
if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then
TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
else
TEST_DIRECTORY=$NOTMUCH_SRCDIR/test
fi
TESTS=
for test in ${NOTMUCH_TESTS-}; do
TESTS="$TESTS $NOTMUCH_SRCDIR/test/$test"
@ -80,7 +88,7 @@ for file in $TESTS
do
file=${file##*/} # drop leading path components
file=${file%.sh} # drop trailing '.sh'
RESULT_FILES="$RESULT_FILES $NOTMUCH_BUILDDIR/test/test-results/$file"
RESULT_FILES="$RESULT_FILES $TEST_DIRECTORY/test-results/$file"
done
echo
@ -94,6 +102,6 @@ if [ -n "$META_FAILURE" ]; then
fi
# Clean up
rm -rf $NOTMUCH_BUILDDIR/test/test-results
rm -rf $TEST_DIRECTORY/test-results
exit $ev

View file

@ -24,7 +24,7 @@
#
type die >/dev/null 2>&1 || die () { echo "$@" >&2; exit 1; }
if [[ -z "$NOTMUCH_SRCDIR" ]] || [[ -z "$NOTMUCH_BUILDDIR" ]]; then
if [[ -z "$NOTMUCH_SRCDIR" ]] || [ -z "${NOTMUCH_TEST_INSTALLED-}" -a -z "$NOTMUCH_BUILDDIR" ]; then
echo "internal: srcdir or builddir not set" >&2
exit 1
fi
@ -61,7 +61,9 @@ LD_LIBRARY_PATH=${TEST_DIRECTORY%/*}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
export LD_LIBRARY_PATH
# configure output
. "$NOTMUCH_BUILDDIR/sh.config" || exit 1
if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then
. "$NOTMUCH_BUILDDIR/sh.config" || exit 1
fi
# load OS specifics
if [[ -e "$NOTMUCH_SRCDIR/test/test-lib-$PLATFORM.sh" ]]; then
@ -315,7 +317,12 @@ export PATH MANPATH
# Test repository
test="tmp.$(basename "$0" .sh)"
TMP_DIRECTORY="$TEST_DIRECTORY/$test"
if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then
TMP_DIRECTORY="$TEST_DIRECTORY/$test"
else
TMP_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-$test.XXXXXX")
fi
test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
rm -rf "$TMP_DIRECTORY" || {
GIT_EXIT_OK=t

View file

@ -30,6 +30,7 @@ test_require_emacs () {
# to the message and encrypting/signing.
emacs_deliver_message () {
local subject body smtp_dummy_pid smtp_dummy_port
test_subtest_broken_for_installed
subject="$1"
body="$2"
shift 2
@ -144,6 +145,13 @@ emacs_generate_script () {
# Construct a little test script here for the benefit of the user,
# (who can easily run "run_emacs" to get the same emacs environment
# for investigating any failures).
if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then
find_notmuch_el='--directory "$NOTMUCH_BUILDDIR/emacs"'
else
### XXX FIXME: this should really use the installed emacs lisp files
find_notmuch_el='--directory "$NOTMUCH_SRCDIR/emacs"'
fi
cat <<EOF >"$TMP_DIRECTORY/run_emacs"
#!/bin/sh
export PATH=$PATH
@ -158,8 +166,8 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG
#
# --load Force loading of notmuch.el and test-lib.el
exec ${TEST_EMACS} --quick \
--directory "$NOTMUCH_BUILDDIR/emacs" --load notmuch.el \
exec ${TEST_EMACS} ${find_notmuch_el} --quick \
${EXTRA_DIR} --load notmuch.el \
--directory "$NOTMUCH_SRCDIR/test" --load test-lib.el \
"\$@"
EOF

View file

@ -29,8 +29,8 @@ shopt -u xpg_echo
# Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set.
. $(dirname "$0")/export-dirs.sh || exit 1
# It appears that people try to run tests without building...
if [[ ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then
# We need either a built tree, or a promise of an installed notmuch
if [ -z "${NOTMUCH_TEST_INSTALLED-}" -a ! -x "$NOTMUCH_BUILDDIR/notmuch" ]; then
echo >&2 'You do not seem to have built notmuch yet.'
exit 1
fi
@ -748,6 +748,12 @@ test_subtest_known_broken () {
test_subtest_known_broken_=t
}
test_subtest_broken_for_installed () {
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
test_subtest_known_broken_=t
fi
}
test_subtest_broken_for_root () {
if [ "$EUID" = "0" ]; then
test_subtest_known_broken_=t
@ -929,11 +935,16 @@ make_shim () {
}
notmuch_with_shim () {
local base_name shim_file
base_name="$1"
local base_name shim_file notmuch_cmd
if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then
notmuch_cmd="notmuch"
else
notmuch_cmd="notmuch-shared"
fi
base_name=$1
shift
shim_file="${base_name}.so"
LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} notmuch-shared "$@"
LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} $notmuch_cmd "$@"
}
# Creates a script that counts how much time it is executed and calls
@ -985,7 +996,11 @@ test_init_ () {
# Where to run the tests
TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then
TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
else
TEST_DIRECTORY=$NOTMUCH_SRCDIR/test
fi
. "$NOTMUCH_SRCDIR/test/test-lib-common.sh" || exit 1