From dab32dc70c15789b188001b78b7225c61bed4784 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 15 May 2021 15:40:22 -0300 Subject: [PATCH 1/6] lib/n_d_index_file: re-use thread-id of existing message This prevents the message document getting multiple thread-id terms when there are multiple files with the same message-id. This change shifts some thread ids, requiring adjustments to other tests. (cherry picked from commit 3f4de98e7c8c70f9a86a4f899147126f79907bd9) --- lib/add-message.cc | 9 ++++++--- test/T357-index-decryption.sh | 26 ++++++++++++-------------- test/T670-duplicate-mid.sh | 13 +++++++++++++ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 485debad..b47aa501 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -407,14 +407,17 @@ static notmuch_status_t _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_message_t *message, notmuch_message_file_t *message_file, - bool is_ghost) + bool is_ghost, + bool is_new) { void *local = talloc_new (NULL); notmuch_status_t status; const char *thread_id = NULL; /* Check if the message already had a thread ID */ - if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { + if (! is_new) { + thread_id = notmuch_message_get_thread_id (message); + } else if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { if (is_ghost) thread_id = notmuch_message_get_thread_id (message); } else { @@ -539,7 +542,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } ret = _notmuch_database_link_message (notmuch, message, - message_file, is_ghost); + message_file, is_ghost, is_new); if (ret) goto DONE; diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh index 1ed5f28c..1951ef78 100755 --- a/test/T357-index-decryption.sh +++ b/test/T357-index-decryption.sh @@ -112,12 +112,10 @@ test_expect_equal \ "$expected" # try inserting it with decryption, should appear as a single copy -# (note: i think thread id skips 4 because of duplicate message-id -# insertion, above) test_begin_subtest "message cleartext is present with insert --decrypt=true" notmuch insert --folder=sent --decrypt=true <<<"$contents" -output=$(notmuch search wumpus) -expected='thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' +output=$(notmuch search wumpus | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -127,9 +125,9 @@ test_expect_equal \ test_begin_subtest 'tagging all messages' test_expect_success 'notmuch tag +blarney "encrypted message"' test_begin_subtest "verify that tags have not changed" -output=$(notmuch search tag:blarney) -expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) -thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' +output=$(notmuch search tag:blarney | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -138,14 +136,14 @@ test_expect_equal \ test_begin_subtest 'reindex old messages' test_expect_success 'notmuch reindex --decrypt=true tag:encrypted and not property:index.decryption=success' test_begin_subtest "reindexed encrypted message, including cleartext" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" # and the same search, but by property ($expected is untouched): test_begin_subtest "emacs search by property for both messages" -output=$(notmuch search property:index.decryption=success) +output=$(notmuch search property:index.decryption=success | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" @@ -154,7 +152,7 @@ test_expect_equal \ test_begin_subtest 'reindex in auto mode' test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success' test_begin_subtest "reindexed encrypted messages, should not have changed" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" @@ -188,9 +186,9 @@ test_expect_equal \ # ensure that the tags remain even when we are dropping the cleartext. test_begin_subtest "verify that tags remain without cleartext" -output=$(notmuch search tag:blarney) -expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) -thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' +output=$(notmuch search tag:blarney | notmuch_search_sanitize) +expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' test_expect_equal \ "$output" \ "$expected" @@ -199,7 +197,7 @@ test_begin_subtest "index cleartext without keeping session keys" test_expect_success "notmuch reindex --decrypt=nostash tag:blarney" test_begin_subtest "Ensure that the indexed terms are present" -output=$(notmuch search wumpus) +output=$(notmuch search wumpus | notmuch_search_sanitize) test_expect_equal \ "$output" \ "$expected" diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh index 4e5672ab..7ae60595 100755 --- a/test/T670-duplicate-mid.sh +++ b/test/T670-duplicate-mid.sh @@ -6,6 +6,19 @@ add_message '[id]="duplicate"' '[subject]="message 1" [filename]=copy1' add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2' add_message '[id]="duplicate"' '[subject]="message 0" [filename]=copy0' + +test_begin_subtest 'at most 1 thread-id per xapian document' +db=${MAIL_DIR}/.notmuch/xapian +for doc in $(xapian-delve -1 -t '' "$db" | grep '^[1-9]'); do + xapian-delve -1 -r "$doc" "$db" | grep -c '^G' +done > OUTPUT.raw +sort -u < OUTPUT.raw > OUTPUT +cat < EXPECTED +0 +1 +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest 'search: first indexed subject preserved' cat < EXPECTED thread:XXX 2001-01-05 [1/1(3)] Notmuch Test Suite; message 1 (inbox unread) From dd5cd144766dffc8a36fa3b3f00f8e194c2e4d14 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Jun 2021 22:44:36 -0300 Subject: [PATCH 2/6] changelog for 0.31.4-2 --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index ec3de63d..382c8184 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +notmuch (0.31.4-2) unstable; urgency=medium + + * Cherry pick upstream commit 3f4de98e7c8, which fixes a bug where + duplicate message-ids can cause multiple thread-ids for some message + documents. + + -- David Bremner Sat, 26 Jun 2021 22:44:08 -0300 + notmuch (0.31.4-1) unstable; urgency=medium * New upstream bugfix release From 3f102654c6e51c68cbc26bfd2195d2a19b72de3c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 28 Jun 2021 22:09:54 -0300 Subject: [PATCH 3/6] debian/changelog: refinalize for upload --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 382c8184..d767fb4b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ notmuch (0.31.4-2) unstable; urgency=medium duplicate message-ids can cause multiple thread-ids for some message documents. - -- David Bremner Sat, 26 Jun 2021 22:44:08 -0300 + -- David Bremner Mon, 28 Jun 2021 22:09:09 -0300 notmuch (0.31.4-1) unstable; urgency=medium From fef433f30700445ce04ff702681e81e979cc307a Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 24 Apr 2021 12:13:37 -0300 Subject: [PATCH 4/6] debian: add xapian-tools for tests xapian-metadata is used by some of the new tests. (cherry picked from commit 9f8458a998a0a20f435b5d47ac085acb9490ddae) --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 8b34d9ed..633d0be8 100644 --- a/debian/control +++ b/debian/control @@ -37,6 +37,7 @@ Build-Depends: ruby, ruby-dev (>>1:1.9.3~), texinfo, + xapian-tools , Standards-Version: 4.4.1 Homepage: https://notmuchmail.org/ Vcs-Git: https://git.notmuchmail.org/git/notmuch -b release From 7b6f1b0f08230b51c8b90b169716aaa463402cd5 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 28 Jun 2021 22:44:24 -0300 Subject: [PATCH 5/6] Commit Debian 3.0 (quilt) metadata [dgit (9.13) quilt-fixup] --- debian/patches/debian-changes | 136 ++++++++++++++++++++++++++++++++++ debian/patches/series | 1 + 2 files changed, 137 insertions(+) create mode 100644 debian/patches/debian-changes create mode 100644 debian/patches/series diff --git a/debian/patches/debian-changes b/debian/patches/debian-changes new file mode 100644 index 00000000..d6927fd0 --- /dev/null +++ b/debian/patches/debian-changes @@ -0,0 +1,136 @@ +This is an autogenerated patch header for a single-debian-patch file. The +delta against upstream is either kept as a single patch, or maintained +in some VCS, and exported as a single patch instead of more manageable +atomic patches. + +--- notmuch-0.31.4.orig/lib/add-message.cc ++++ notmuch-0.31.4/lib/add-message.cc +@@ -407,14 +407,17 @@ static notmuch_status_t + _notmuch_database_link_message (notmuch_database_t *notmuch, + notmuch_message_t *message, + notmuch_message_file_t *message_file, +- bool is_ghost) ++ bool is_ghost, ++ bool is_new) + { + void *local = talloc_new (NULL); + notmuch_status_t status; + const char *thread_id = NULL; + + /* Check if the message already had a thread ID */ +- if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { ++ if (! is_new) { ++ thread_id = notmuch_message_get_thread_id (message); ++ } else if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { + if (is_ghost) + thread_id = notmuch_message_get_thread_id (message); + } else { +@@ -539,7 +542,7 @@ notmuch_database_index_file (notmuch_dat + } + + ret = _notmuch_database_link_message (notmuch, message, +- message_file, is_ghost); ++ message_file, is_ghost, is_new); + if (ret) + goto DONE; + +--- notmuch-0.31.4.orig/test/T357-index-decryption.sh ++++ notmuch-0.31.4/test/T357-index-decryption.sh +@@ -112,12 +112,10 @@ test_expect_equal \ + "$expected" + + # try inserting it with decryption, should appear as a single copy +-# (note: i think thread id skips 4 because of duplicate message-id +-# insertion, above) + test_begin_subtest "message cleartext is present with insert --decrypt=true" + notmuch insert --folder=sent --decrypt=true <<<"$contents" +-output=$(notmuch search wumpus) +-expected='thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' ++output=$(notmuch search wumpus | notmuch_search_sanitize) ++expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)' + test_expect_equal \ + "$output" \ + "$expected" +@@ -127,9 +125,9 @@ test_expect_equal \ + test_begin_subtest 'tagging all messages' + test_expect_success 'notmuch tag +blarney "encrypted message"' + test_begin_subtest "verify that tags have not changed" +-output=$(notmuch search tag:blarney) +-expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +-thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' ++output=$(notmuch search tag:blarney | notmuch_search_sanitize) ++expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) ++thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' + test_expect_equal \ + "$output" \ + "$expected" +@@ -138,14 +136,14 @@ test_expect_equal \ + test_begin_subtest 'reindex old messages' + test_expect_success 'notmuch reindex --decrypt=true tag:encrypted and not property:index.decryption=success' + test_begin_subtest "reindexed encrypted message, including cleartext" +-output=$(notmuch search wumpus) ++output=$(notmuch search wumpus | notmuch_search_sanitize) + test_expect_equal \ + "$output" \ + "$expected" + + # and the same search, but by property ($expected is untouched): + test_begin_subtest "emacs search by property for both messages" +-output=$(notmuch search property:index.decryption=success) ++output=$(notmuch search property:index.decryption=success | notmuch_search_sanitize) + test_expect_equal \ + "$output" \ + "$expected" +@@ -154,7 +152,7 @@ test_expect_equal \ + test_begin_subtest 'reindex in auto mode' + test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success' + test_begin_subtest "reindexed encrypted messages, should not have changed" +-output=$(notmuch search wumpus) ++output=$(notmuch search wumpus | notmuch_search_sanitize) + test_expect_equal \ + "$output" \ + "$expected" +@@ -188,9 +186,9 @@ test_expect_equal \ + + # ensure that the tags remain even when we are dropping the cleartext. + test_begin_subtest "verify that tags remain without cleartext" +-output=$(notmuch search tag:blarney) +-expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +-thread:0000000000000005 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' ++output=$(notmuch search tag:blarney | notmuch_search_sanitize) ++expected='thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) ++thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)' + test_expect_equal \ + "$output" \ + "$expected" +@@ -199,7 +197,7 @@ test_begin_subtest "index cleartext with + test_expect_success "notmuch reindex --decrypt=nostash tag:blarney" + + test_begin_subtest "Ensure that the indexed terms are present" +-output=$(notmuch search wumpus) ++output=$(notmuch search wumpus | notmuch_search_sanitize) + test_expect_equal \ + "$output" \ + "$expected" +--- notmuch-0.31.4.orig/test/T670-duplicate-mid.sh ++++ notmuch-0.31.4/test/T670-duplicate-mid.sh +@@ -6,6 +6,19 @@ add_message '[id]="duplicate"' '[subject + add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2' + + add_message '[id]="duplicate"' '[subject]="message 0" [filename]=copy0' ++ ++test_begin_subtest 'at most 1 thread-id per xapian document' ++db=${MAIL_DIR}/.notmuch/xapian ++for doc in $(xapian-delve -1 -t '' "$db" | grep '^[1-9]'); do ++ xapian-delve -1 -r "$doc" "$db" | grep -c '^G' ++done > OUTPUT.raw ++sort -u < OUTPUT.raw > OUTPUT ++cat < EXPECTED ++0 ++1 ++EOF ++test_expect_equal_file EXPECTED OUTPUT ++ + test_begin_subtest 'search: first indexed subject preserved' + cat < EXPECTED + thread:XXX 2001-01-05 [1/1(3)] Notmuch Test Suite; message 1 (inbox unread) diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 00000000..7bb82529 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +debian-changes From b6cfc8a61c20aec1e54185829718f87c7f7dea79 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 28 Jun 2021 22:48:29 -0300 Subject: [PATCH 6/6] debian: document new build-dependency --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d767fb4b..ebfd9209 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,9 @@ notmuch (0.31.4-2) unstable; urgency=medium * Cherry pick upstream commit 3f4de98e7c8, which fixes a bug where duplicate message-ids can cause multiple thread-ids for some message documents. + * Add build-dependency on xapian-tools, for new test - -- David Bremner Mon, 28 Jun 2021 22:09:09 -0300 + -- David Bremner Mon, 28 Jun 2021 22:48:02 -0300 notmuch (0.31.4-1) unstable; urgency=medium