From 5197d3e11f5774a5eb9989cdbea07112a7f18fb8 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 19 May 2021 00:40:44 +0300 Subject: [PATCH 1/5] doc: fix man page build for Sphinx 4.x Sphinx 4.0 changed the default value of man_make_section_directory from False to True. We create the section directories and move the files manually, so fix the immediate man build failure by disabling the feature. The Sphinx documentation on this [1] is confusing, and has the change backwards. Git history says the default changed from False to True. [1] https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-man_make_section_directory --- doc/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 1a5c217b..d0f7f66c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -87,6 +87,8 @@ html_use_smartypants = False notmuch_authors = u'Carl Worth and many others' +man_make_section_directory = False + man_pages = [ ('man1/notmuch', 'notmuch', u'thread-based email index, search, and tagging', From e715ec9371058ec74992a2623b791af48751b54f Mon Sep 17 00:00:00 2001 From: Luis Henriques Date: Wed, 19 May 2021 12:08:18 +0100 Subject: [PATCH 2/5] doc: fix variable name in documentation Variable 'notmuch-saved-searches-sort-function' does not exist; 'notmuch-saved-search-sort-function' is the correct name. Signed-off-by: Luis Henriques --- doc/notmuch-emacs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index de47b726..d9b497a3 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -100,7 +100,7 @@ unread mail, but there are several options for customization: The list of saved searches, including names, queries, and additional per-query options. -:index:`notmuch-saved-searches-sort-function` +:index:`notmuch-saved-search-sort-function` This variable controls how saved searches should be sorted. A value of ``nil`` displays the saved searches in the order they are stored in ‘notmuch-saved-searches’. From 892f4e65ae45f86992777a6803f3bbda4c820188 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 8 May 2021 10:57:35 -0300 Subject: [PATCH 3/5] perf-test/T00-new: fix backup creation. "xargs tar cf backup.tar < $manifest" recreates the tar file with each "batch" execed by xargs. In general this results in only a fraction of the desired files being backed up. --- performance-test/T00-new.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/performance-test/T00-new.sh b/performance-test/T00-new.sh index 53de1c27..de260b2d 100755 --- a/performance-test/T00-new.sh +++ b/performance-test/T00-new.sh @@ -5,16 +5,16 @@ test_description='notmuch new' . $(dirname "$0")/perf-test-lib.sh || exit 1 uncache_database - time_start +manifest=$(mktemp manifestXXXXXX) +find mail -type f ! -path 'mail/.notmuch/*' | sed -n '1~4 p' > $manifest +xargs tar uf backup.tar < $manifest + for i in $(seq 2 6); do time_run "notmuch new #$i" 'notmuch new' done -manifest=$(mktemp manifestXXXXXX) - -find mail -type f ! -path 'mail/.notmuch/*' | sed -n '1~4 p' > $manifest # arithmetic context is to eat extra whitespace on e.g. some BSDs count=$((`wc -l < $manifest`)) @@ -26,8 +26,6 @@ perl -nle 'rename "$_.renamed", $_' $manifest time_run "new ($count mv back)" 'notmuch new' -xargs tar cf backup.tar < $manifest - perl -nle 'unlink $_; unlink $_.copy' $manifest time_run "new ($count rm)" 'notmuch new' From 349fc2980346db2dba619cda5d9cdf0d943d3053 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 15 May 2021 10:05:07 -0300 Subject: [PATCH 4/5] test: add known broken test for duplicate thread-id terms According to my bijection, this bug has been present since commit 411675a6ce in 2017. It is not completely clear what harm it causes in regulary use, but it (at least) makes notmuch crash when compiled with -DDEBUG_DATABASE_SANITY. --- test/T670-duplicate-mid.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh index 4e5672ab..cf0041b0 100755 --- a/test/T670-duplicate-mid.sh +++ b/test/T670-duplicate-mid.sh @@ -2,10 +2,26 @@ test_description="duplicate message ids" . $(dirname "$0")/test-lib.sh || exit 1 +test_require_external_prereq xapian-delve + add_message '[id]="duplicate"' '[subject]="message 1" [filename]=copy1' add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2' add_message '[id]="duplicate"' '[subject]="message 0" [filename]=copy0' + +test_begin_subtest 'at most 1 thread-id per xapian document' +test_subtest_known_broken +db=${MAIL_DIR}/.notmuch/xapian +for doc in $(xapian-delve -1 -t '' "$db" | grep '^[1-9]'); do + xapian-delve -1 -r "$doc" "$db" | grep -c '^G' +done > OUTPUT.raw +sort -u < OUTPUT.raw > OUTPUT +cat < EXPECTED +0 +1 +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest 'search: first indexed subject preserved' cat < EXPECTED thread:XXX 2001-01-05 [1/1(3)] Notmuch Test Suite; message 1 (inbox unread) From 3f4de98e7c8c70f9a86a4f899147126f79907bd9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 15 May 2021 15:40:22 -0300 Subject: [PATCH 5/5] lib/n_d_index_file: re-use thread-id of existing message This prevents the message document getting multiple thread-id terms when there are multiple files with the same message-id. This change shifts some thread ids, requiring adjustments to other tests. --- lib/add-message.cc | 9 ++++++--- test/T357-index-decryption.sh | 26 ++++++++++++-------------- test/T670-duplicate-mid.sh | 1 - 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 0c34d318..d6e5e73d 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -407,14 +407,17 @@ static notmuch_status_t _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_message_t *message, notmuch_message_file_t *message_file, - bool is_ghost) + bool is_ghost, + bool is_new) { void *local = talloc_new (NULL); notmuch_status_t status; const char *thread_id = NULL; /* Check if the message already had a thread ID */ - if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { + if (! is_new) { + thread_id = notmuch_message_get_thread_id (message); + } else if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { if (is_ghost) thread_id = notmuch_message_get_thread_id (message); } else { @@ -541,7 +544,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } ret = _notmuch_database_link_message (notmuch, message, - message_file, is_ghost); + message_file, is_ghost, is_new); if (ret) goto DONE; diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh index 1ed5f28c..1951ef78 100755 --- a/test/T357-index-decryption.sh +++ b/test/T357-index-decryption.sh @@ -112,12 +112,10 @@ test_expect_equal \ "$expected" # try inserting it with decryption, should appear as a single copy -# (note: i think thread id skips 4 because of duplicate message-id -# insertion, above) test_begin_subtest "message cleartext is present with insert --decrypt=true" notmuch insert --folder=sent --decrypt=true <<<"$contents" -output=$(notmuch search wumpus) -expected='thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' +output=$(notmuch search wumpus | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -127,9 +125,9 @@ test_expect_equal \ test_begin_subtest 'tagging all messages' test_expect_success 'notmuch tag +blarney "encrypted message"' test_begin_subtest "verify that tags have not changed" -output=$(notmuch search tag:blarney) -expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) -thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' +output=$(notmuch search tag:blarney | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -138,14 +136,14 @@ test_expect_equal \ test_begin_subtest 'reindex old messages' test_expect_success 'notmuch reindex --decrypt=true tag:encrypted and not property:index.decryption=success' test_begin_subtest "reindexed encrypted message, including cleartext" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" # and the same search, but by property ($expected is untouched): test_begin_subtest "emacs search by property for both messages" -output=$(notmuch search property:index.decryption=success) +output=$(notmuch search property:index.decryption=success | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" @@ -154,7 +152,7 @@ test_expect_equal \ test_begin_subtest 'reindex in auto mode' test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success' test_begin_subtest "reindexed encrypted messages, should not have changed" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" @@ -188,9 +186,9 @@ test_expect_equal \ # ensure that the tags remain even when we are dropping the cleartext. test_begin_subtest "verify that tags remain without cleartext" -output=$(notmuch search tag:blarney) -expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) -thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' +output=$(notmuch search tag:blarney | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -199,7 +197,7 @@ test_begin_subtest "index cleartext without keeping session keys" test_expect_success "notmuch reindex --decrypt=nostash tag:blarney" test_begin_subtest "Ensure that the indexed terms are present" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh index cf0041b0..8fec291e 100755 --- a/test/T670-duplicate-mid.sh +++ b/test/T670-duplicate-mid.sh @@ -10,7 +10,6 @@ add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2' add_message '[id]="duplicate"' '[subject]="message 0" [filename]=copy0' test_begin_subtest 'at most 1 thread-id per xapian document' -test_subtest_known_broken db=${MAIL_DIR}/.notmuch/xapian for doc in $(xapian-delve -1 -t '' "$db" | grep '^[1-9]'); do xapian-delve -1 -r "$doc" "$db" | grep -c '^G'