From b6565c1c54e35563843e7ddece601680170bb84a Mon Sep 17 00:00:00 2001 From: Matt Armstrong Date: Wed, 12 Oct 2022 20:20:38 -0700 Subject: [PATCH 01/90] emacs: fix dangling overlays in notmuch-search notmuch-search-insert-authors now sets the evaporate property on the ellipsis overlays. Emacs will delete them when the buffer contents are zeroed out, which happens with `notmuch-refresh-buffer`. This prevents them from being collapsed to zero-width overlays in position 1. See Emacs bug#58479. An upcoming change in Emacs will make these dangling overlays visible to the user. --- emacs/notmuch.el | 1 + 1 file changed, 1 insertion(+) diff --git a/emacs/notmuch.el b/emacs/notmuch.el index 26181758..6eef4af1 100644 --- a/emacs/notmuch.el +++ b/emacs/notmuch.el @@ -841,6 +841,7 @@ non-authors is found, assume that all of the authors match." overlay) (insert invisible-string) (setq overlay (make-overlay start (point))) + (overlay-put overlay 'evaporate t) (overlay-put overlay 'invisible 'ellipsis) (overlay-put overlay 'isearch-open-invisible #'delete-overlay))) (insert padding)))) From 793f2980910f612fa33806ef71fa7ee35d093657 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Tue, 18 Oct 2022 21:41:58 +0200 Subject: [PATCH 02/90] cli: add options --offset and --limit to notmuch show notmuch search does not output header values. However, when browsing through a large email corpus, it can be time saving to be able to paginate without running notmuch show for each message/thread. Add --offset and --limit options to notmuch show. This is inspired from commit 796b629c3b82 ("cli: add options --offset and --limit to notmuch search"). Update man page, shell completion and add a test case to ensure it works as expected. Cc: Tim Culverhouse Cc: Tomi Ollila Signed-off-by: Robin Jarry --- completion/notmuch-completion.bash | 2 +- completion/zsh/_notmuch | 2 + doc/man1/notmuch-show.rst | 9 ++++ notmuch-client.h | 2 + notmuch-show.c | 49 +++++++++++++++--- test/T131-show-limiting.sh | 81 ++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+), 7 deletions(-) create mode 100755 test/T131-show-limiting.sh diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 0022b54b..3748846e 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -530,7 +530,7 @@ _notmuch_show() ! $split && case "${cur}" in -*) - local options="--entire-thread= --format= --exclude= --body= --format-version= --part= --verify --decrypt= --include-html ${_notmuch_shared_options}" + local options="--entire-thread= --format= --exclude= --body= --format-version= --part= --verify --decrypt= --include-html --limit= --offset= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) ;; diff --git a/completion/zsh/_notmuch b/completion/zsh/_notmuch index e207d90b..0bdd7f77 100644 --- a/completion/zsh/_notmuch +++ b/completion/zsh/_notmuch @@ -245,6 +245,8 @@ _notmuch_show() { '--exclude=[respect excluded tags setting]:exclude tags:(true false)' \ '--body=[output body]:output body content:(true false)' \ '--include-html[include text/html parts in the output]' \ + '--limit=[limit the number of displayed results]:limit: ' \ + '--offset=[skip displaying the first N results]:offset: ' \ '*::search term:_notmuch_search_term' } diff --git a/doc/man1/notmuch-show.rst b/doc/man1/notmuch-show.rst index 2c0a0de6..c13d94de 100644 --- a/doc/man1/notmuch-show.rst +++ b/doc/man1/notmuch-show.rst @@ -130,6 +130,15 @@ Supported options for **show** include By default, results will be displayed in reverse chronological order, (that is, the newest results will be displayed first). +.. option:: --offset=[-]N + + Skip displaying the first N results. With the leading '-', start + at the Nth result from the end. + +.. option:: --limit=N + + Limit the number of displayed results to N. + .. option:: --verify Compute and report the validity of any MIME cryptographic diff --git a/notmuch-client.h b/notmuch-client.h index 21b49908..1a87240d 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -77,6 +77,8 @@ typedef struct notmuch_show_params { bool output_body; int duplicate; int part; + int offset; + int limit; _notmuch_crypto_t crypto; bool include_html; GMimeStream *out_stream; diff --git a/notmuch-show.c b/notmuch-show.c index ee9efa74..7fb40ce9 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -1159,6 +1159,18 @@ do_show_threaded (void *ctx, notmuch_thread_t *thread; notmuch_messages_t *messages; notmuch_status_t status, res = NOTMUCH_STATUS_SUCCESS; + int i; + + if (params->offset < 0) { + unsigned count; + notmuch_status_t s = notmuch_query_count_threads (query, &count); + if (print_status_query ("notmuch show", query, s)) + return 1; + + params->offset += count; + if (params->offset < 0) + params->offset = 0; + } status = notmuch_query_search_threads (query, &threads); if (print_status_query ("notmuch show", query, status)) @@ -1166,11 +1178,16 @@ do_show_threaded (void *ctx, sp->begin_list (sp); - for (; - notmuch_threads_valid (threads); - notmuch_threads_move_to_next (threads)) { + for (i = 0; + notmuch_threads_valid (threads) && (params->limit < 0 || i < params->offset + params->limit); + notmuch_threads_move_to_next (threads), i++) { thread = notmuch_threads_get (threads); + if (i < params->offset) { + notmuch_thread_destroy (thread); + continue; + } + messages = notmuch_thread_get_toplevel_messages (thread); if (messages == NULL) @@ -1201,6 +1218,18 @@ do_show_unthreaded (void *ctx, notmuch_message_t *message; notmuch_status_t status, res = NOTMUCH_STATUS_SUCCESS; notmuch_bool_t excluded; + int i; + + if (params->offset < 0) { + unsigned count; + notmuch_status_t s = notmuch_query_count_messages (query, &count); + if (print_status_query ("notmuch show", query, s)) + return 1; + + params->offset += count; + if (params->offset < 0) + params->offset = 0; + } status = notmuch_query_search_messages (query, &messages); if (print_status_query ("notmuch show", query, status)) @@ -1208,9 +1237,13 @@ do_show_unthreaded (void *ctx, sp->begin_list (sp); - for (; - notmuch_messages_valid (messages); - notmuch_messages_move_to_next (messages)) { + for (i = 0; + notmuch_messages_valid (messages) && (params->limit < 0 || i < params->offset + params->limit); + notmuch_messages_move_to_next (messages), i++) { + if (i < params->offset) { + continue; + } + sp->begin_list (sp); sp->begin_list (sp); @@ -1287,6 +1320,8 @@ notmuch_show_command (notmuch_database_t *notmuch, int argc, char *argv[]) notmuch_show_params_t params = { .part = -1, .duplicate = 0, + .offset = 0, + .limit = -1, /* unlimited */ .omit_excluded = true, .output_body = true, .crypto = { .decrypt = NOTMUCH_DECRYPT_AUTO }, @@ -1328,6 +1363,8 @@ notmuch_show_command (notmuch_database_t *notmuch, int argc, char *argv[]) { .opt_bool = ¶ms.output_body, .name = "body" }, { .opt_bool = ¶ms.include_html, .name = "include-html" }, { .opt_int = ¶ms.duplicate, .name = "duplicate" }, + { .opt_int = ¶ms.limit, .name = "limit" }, + { .opt_int = ¶ms.offset, .name = "offset" }, { .opt_inherit = notmuch_shared_options }, { } }; diff --git a/test/T131-show-limiting.sh b/test/T131-show-limiting.sh new file mode 100755 index 00000000..30d1f254 --- /dev/null +++ b/test/T131-show-limiting.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +test_description='"notmuch show" --offset and --limit parameters' +. $(dirname "$0")/test-lib.sh || exit 1 + +add_email_corpus + +show () { + local kind="$1" + shift + if [ "$kind" = messages ]; then + set -- --unthreaded "$@" + fi + notmuch show --body=false --format=text --entire-thread=false "$@" "*" | + sed -nre 's/^.message\{.*\.*/&/p' +} + +for outp in messages threads; do + test_begin_subtest "$outp: limit does the right thing" + show $outp | head -n 20 >expected + show $outp --limit=20 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: concatenation of limited shows" + show $outp | head -n 20 >expected + show $outp --limit=10 >output + show $outp --limit=10 --offset=10 >>output + test_expect_equal_file expected output + + test_begin_subtest "$outp: limit larger than result set" + N=$(notmuch count --output=$outp "*") + show $outp >expected + show $outp --limit=$((1 + N)) >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: limit = 0" + test_expect_equal "$(show $outp --limit=0)" "" + + test_begin_subtest "$outp: offset does the right thing" + # note: tail -n +N is 1-based + show $outp | tail -n +21 >expected + show $outp --offset=20 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: offset = 0" + show $outp >expected + show $outp --offset=0 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset" + show $outp | tail -n 20 >expected + show $outp --offset=-20 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset" + show $outp | tail -n 1 >expected + show $outp --offset=-1 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset combined with limit" + show $outp | tail -n 20 | head -n 10 >expected + show $outp --offset=-20 --limit=10 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset combined with equal limit" + show $outp | tail -n 20 >expected + show $outp --offset=-20 --limit=20 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset combined with large limit" + show $outp | tail -n 20 >expected + show $outp --offset=-20 --limit=50 >output + test_expect_equal_file expected output + + test_begin_subtest "$outp: negative offset larger than results" + N=$(notmuch count --output=$outp "*") + show $outp >expected + show $outp --offset=-$((1 + N)) >output + test_expect_equal_file expected output +done + +test_done From 82aa1acc0c6a66eb3b771357e513eb4d16f9f276 Mon Sep 17 00:00:00 2001 From: Russell Sim Date: Tue, 20 Sep 2022 22:18:03 +0200 Subject: [PATCH 03/90] emacs: move From header addition to after header intern OTHER-HEADERS are expected to be passed as strings, to match the implementation of `compose-mail'. But the "From" header is currently expected to be passed as a symbol. Instead the "From" header can be safely added after converting all the headers to symbols. --- emacs/notmuch-mua.el | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el index ac878a61..3679d7d7 100644 --- a/emacs/notmuch-mua.el +++ b/emacs/notmuch-mua.el @@ -416,11 +416,6 @@ moved to the \"To:\" header." (let ((user-agent (funcall notmuch-mua-user-agent-function))) (unless (string-empty-p user-agent) (push (cons 'User-Agent user-agent) other-headers)))) - (unless (assq 'From other-headers) - (push (cons 'From (message-make-from - (notmuch-user-name) - (notmuch-user-primary-email))) - other-headers)) (notmuch-mua-pop-to-buffer (message-buffer-name "mail" to) (or switch-function (notmuch-mua-get-switch-function))) @@ -439,6 +434,11 @@ moved to the \"To:\" header." ;; Cause `message-setup-1' to do things relevant for mail, ;; such as observe `message-default-mail-headers'. (message-this-is-mail t)) + (unless (assq 'From headers) + (push (cons 'From (message-make-from + (notmuch-user-name) + (notmuch-user-primary-email))) + headers)) (message-setup-1 headers yank-action send-actions return-action)) (notmuch-fcc-header-setup) (notmuch-mua--remove-dont-reply-to-names) From 2b842a1d8cb25981c19d9adb33fe962f4ebd2e9f Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 11 Nov 2022 16:48:29 -0500 Subject: [PATCH 04/90] emacs/show: use plist to pass message info to n-s-insert-headerline This should allow calling notmuch-show-insert-headerline from other places without duplicating the set of plist accesses. --- emacs/notmuch-show.el | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el index ec998ede..765b88b9 100644 --- a/emacs/notmuch-show.el +++ b/emacs/notmuch-show.el @@ -530,11 +530,17 @@ Return unchanged ADDRESS if parsing fails." (plist-put msg :height height) height)))) -(defun notmuch-show-insert-headerline (headers date tags depth duplicate file-count) +(defun notmuch-show-insert-headerline (msg-plist depth tags) "Insert a notmuch style headerline based on HEADERS for a message at DEPTH in the current thread." - (let ((start (point)) - (from (notmuch-sanitize + (let* ((start (point)) + (headers (plist-get msg-plist :headers)) + (duplicate (or (plist-get msg-plist :duplicate) 0)) + (file-count (length (plist-get msg-plist :filename))) + (date (or (and notmuch-show-relative-dates + (plist-get msg-plist :date_relative)) + (plist-get headers :Date))) + (from (notmuch-sanitize (notmuch-show-clean-address (plist-get headers :From))))) (when (string-match "\\cR" from) ;; If the From header has a right-to-left character add @@ -1171,8 +1177,6 @@ is out of range." (defun notmuch-show-insert-msg (msg depth) "Insert the message MSG at depth DEPTH in the current thread." (let* ((headers (plist-get msg :headers)) - (duplicate (or (plist-get msg :duplicate) 0)) - (files (length (plist-get msg :filename))) ;; Indentation causes the buffer offset of the start/end ;; points to move, so we must use markers. message-start message-end @@ -1180,11 +1184,7 @@ is out of range." headers-start headers-end (bare-subject (notmuch-show-strip-re (plist-get headers :Subject)))) (setq message-start (point-marker)) - (notmuch-show-insert-headerline headers - (or (and notmuch-show-relative-dates - (plist-get msg :date_relative)) - (plist-get headers :Date)) - (plist-get msg :tags) depth duplicate files) + (notmuch-show-insert-headerline msg depth (plist-get msg :tags)) (setq content-start (point-marker)) ;; Set `headers-start' to point after the 'Subject:' header to be ;; compatible with the existing implementation. This just sets it From 056249627ac68823e5246b2586db9fbb8002c6b0 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 11 Nov 2022 16:48:30 -0500 Subject: [PATCH 05/90] emacs/show: add optional orig-tags argument to n-s-i-headerline This will support use of this function in notmuch-show-update-tags. --- emacs/notmuch-show.el | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el index 765b88b9..be68fc3b 100644 --- a/emacs/notmuch-show.el +++ b/emacs/notmuch-show.el @@ -530,7 +530,7 @@ Return unchanged ADDRESS if parsing fails." (plist-put msg :height height) height)))) -(defun notmuch-show-insert-headerline (msg-plist depth tags) +(defun notmuch-show-insert-headerline (msg-plist depth tags &optional orig-tags) "Insert a notmuch style headerline based on HEADERS for a message at DEPTH in the current thread." (let* ((start (point)) @@ -555,7 +555,7 @@ message at DEPTH in the current thread." " (" date ") (" - (notmuch-tag-format-tags tags tags) + (notmuch-tag-format-tags tags (or orig-tags tags)) ")") (insert (if (> file-count 1) From 5ba7684445b9338ab289d502d4688d9aff3d8803 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 11 Nov 2022 16:48:31 -0500 Subject: [PATCH 06/90] emacs/show: use n-s-i-headerline to update tags Although this has more steps than the previous regular expression search and replace, it should be more robust against changes in the headerline format, such as the inclusion of duplicate numbers (which broke the previous version). --- emacs/notmuch-show.el | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el index be68fc3b..36cce619 100644 --- a/emacs/notmuch-show.el +++ b/emacs/notmuch-show.el @@ -452,14 +452,19 @@ operation on the contents of the current buffer." (defun notmuch-show-update-tags (tags) "Update the displayed tags of the current message." (save-excursion - (goto-char (notmuch-show-message-top)) - (when (re-search-forward "(\\([^()]*\\))$" (line-end-position) t) - (let ((inhibit-read-only t)) - (replace-match (concat "(" - (notmuch-tag-format-tags - tags - (notmuch-show-get-prop :orig-tags)) - ")")))))) + (let ((inhibit-read-only t) + (start (notmuch-show-message-top)) + (depth (notmuch-show-get-prop :depth)) + (orig-tags (notmuch-show-get-prop :orig-tags)) + (props (notmuch-show-get-message-properties)) + (extent (notmuch-show-message-extent))) + (goto-char start) + (notmuch-show-insert-headerline props depth tags orig-tags) + (put-text-property start (1+ start) + :notmuch-message-properties props) + (put-text-property (car extent) (cdr extent) :notmuch-message-extent extent) + ;; delete original headerline, but do not save to kill ring + (delete-region (point) (1+ (line-end-position)))))) (defun notmuch-clean-address (address) "Try to clean a single email ADDRESS for display. Return a cons From a5f7efd7221f338982dbf1b72692bf9095a9a96a Mon Sep 17 00:00:00 2001 From: Michael J Gruber Date: Wed, 30 Nov 2022 14:58:34 +0100 Subject: [PATCH 07/90] doc: mark `--output=summary` as default `notmuch search` behaves differently depending on the output option: It either outputs information pertaining to all threads with matching messages (summary, threads) or to all matching messages (messages, files, tags). The man page refres solely to the former in the main description. Help the user by clearly marking `summary` as the default output option. Signed-off-by: Michael J Gruber --- doc/man1/notmuch-search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst index ad305efd..b87737ea 100644 --- a/doc/man1/notmuch-search.rst +++ b/doc/man1/notmuch-search.rst @@ -43,7 +43,7 @@ Supported options for **search** include .. option:: --output=(summary|threads|messages|files|tags) - summary + summary (default) Output a summary of each thread with any message matching the search terms. The summary includes the thread ID, date, the number of messages in the thread (both the number matched and From 891af1d457a174e12943baf111175af14bb4bb53 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 16 Oct 2022 09:49:00 -0300 Subject: [PATCH 08/90] CLI/git: use --exclude=false when calling notmuch-search We use notmuch search in two places in notmuch-git.py: to find which tags have a given prefix, and to see if message with given id exists locally. In both cases we do not want the presence of exclude tags (e.g. deleted) to change the results. --- notmuch-git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notmuch-git.py b/notmuch-git.py index ceb86fbc..57098aae 100644 --- a/notmuch-git.py +++ b/notmuch-git.py @@ -254,7 +254,7 @@ def count_messages(prefix=None): def get_tags(prefix=None): "Get a list of tags with a given prefix." (status, stdout, stderr) = _spawn( - args=['notmuch', 'search', '--query=sexp', '--output=tags', _tag_query(prefix)], + args=['notmuch', 'search', '--exclude=false', '--query=sexp', '--output=tags', _tag_query(prefix)], stdout=_subprocess.PIPE, wait=True) return [tag for tag in stdout.splitlines()] @@ -719,7 +719,7 @@ class DatabaseCache: self._known[id] = False else: (_, stdout, stderr) = _spawn( - args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)], + args=['notmuch', 'search', '--exclude=false', '--output=files', 'id:{0}'.format(id)], stdout=_subprocess.PIPE, wait=True) self._known[id] = stdout != None From 16d92abf9f3326da1ec0c6f84c2c8876efc77ecb Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 3 Dec 2022 22:28:55 -0400 Subject: [PATCH 09/90] lib/database: propagate status code from _notmuch_message_delete _notmuch_message_delete can return (at least) NOTMUCH_STATUS_XAPIAN_EXCEPTION, which we should not ignore. --- lib/database.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/database.cc b/lib/database.cc index c05d70d3..d1e5f1af 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -1456,7 +1456,7 @@ notmuch_database_remove_message (notmuch_database_t *notmuch, if (status == NOTMUCH_STATUS_SUCCESS && message) { status = _notmuch_message_remove_filename (message, filename); if (status == NOTMUCH_STATUS_SUCCESS) - _notmuch_message_delete (message); + status = _notmuch_message_delete (message); else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) _notmuch_message_sync (message); From 966f40086f36a5bf26d8c180cd34a01636971a84 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 3 Dec 2022 22:28:56 -0400 Subject: [PATCH 10/90] test: add known broken test for exception handling in _n_m_delete In [1], Thomas Schneider reported an uncaught Xapian exception when running out of disk space. We generate the same exception via database corruption. [1]: id:wwuk039sk2p.fsf@chaotikum.eu --- test/T566-lib-message.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/T566-lib-message.sh b/test/T566-lib-message.sh index 511d56ca..562ab05a 100755 --- a/test/T566-lib-message.sh +++ b/test/T566-lib-message.sh @@ -516,4 +516,32 @@ cat < EXPECTED EOF test_expect_equal_file EXPECTED OUTPUT +TERMLIST_PATH=(${MAIL_DIR}/.notmuch/xapian/termlist.*) +test_begin_subtest "remove message with corrupted db" +test_subtest_known_broken +backup_database +cat c_head0 - c_tail <<'EOF' | test_private_C ${MAIL_DIR} ${TERMLIST_PATH} + { + notmuch_status_t status; + + int fd = open(argv[2],O_WRONLY|O_TRUNC); + if (fd < 0) { + fprintf (stderr, "error opening %s\n", argv[1]); + exit (1); + } + + stat = _notmuch_message_delete (message); + printf ("%d\n", stat == NOTMUCH_STATUS_XAPIAN_EXCEPTION); + } +EOF +cat < EXPECTED +== stdout == +1 +== stderr == +A Xapian exception occurred at message.cc:XXX: EOF reading block YYY +EOF +sed 's/EOF reading block [0-9]*/EOF reading block YYY/' < OUTPUT > OUTPUT.clean +test_expect_equal_file EXPECTED OUTPUT.clean +restore_database + test_done From 1d5d0ae68689e7b1d6b974e275f19371250c4b25 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 3 Dec 2022 22:28:57 -0400 Subject: [PATCH 11/90] lib/message: move xapian call inside try/catch block in _n_m_delete The call to delete_document can throw exceptions (and can happen in practice [1]), so catch the exception and extract the error message. As a side effect, also move the call to _n_m_has_term inside the try/catch. This should not change anything as that function already traps any Xapian exceptions. [1]: id:wwuk039sk2p.fsf@chaotikum.eu --- lib/message.cc | 22 +++++++++++----------- test/T566-lib-message.sh | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/message.cc b/lib/message.cc index 1c87f8c0..5ccca95a 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -1383,21 +1383,21 @@ _notmuch_message_delete (notmuch_message_t *message) if (status) return status; - message->notmuch->writable_xapian_db->delete_document (message->doc_id); - - /* if this was a ghost to begin with, we are done */ - private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost); - if (private_status) - return COERCE_STATUS (private_status, - "Error trying to determine whether message was a ghost"); - if (is_ghost) - return NOTMUCH_STATUS_SUCCESS; - - /* look for a non-ghost message in the same thread */ try { Xapian::PostingIterator thread_doc, thread_doc_end; Xapian::PostingIterator mail_doc, mail_doc_end; + message->notmuch->writable_xapian_db->delete_document (message->doc_id); + + /* look for a non-ghost message in the same thread */ + /* if this was a ghost to begin with, we are done */ + private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost); + if (private_status) + return COERCE_STATUS (private_status, + "Error trying to determine whether message was a ghost"); + if (is_ghost) + return NOTMUCH_STATUS_SUCCESS; + _notmuch_database_find_doc_ids (message->notmuch, "thread", tid, &thread_doc, &thread_doc_end); _notmuch_database_find_doc_ids (message->notmuch, "type", "mail", &mail_doc, &mail_doc_end); diff --git a/test/T566-lib-message.sh b/test/T566-lib-message.sh index 562ab05a..7f0e8eb0 100755 --- a/test/T566-lib-message.sh +++ b/test/T566-lib-message.sh @@ -518,7 +518,6 @@ test_expect_equal_file EXPECTED OUTPUT TERMLIST_PATH=(${MAIL_DIR}/.notmuch/xapian/termlist.*) test_begin_subtest "remove message with corrupted db" -test_subtest_known_broken backup_database cat c_head0 - c_tail <<'EOF' | test_private_C ${MAIL_DIR} ${TERMLIST_PATH} { From f4dc32e71b6fbaca933e0c7edd6d08b65feed0ec Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 4 Jan 2023 08:41:42 -0400 Subject: [PATCH 12/90] test: mark some tests as broken when run as root. File permission errors e.g., are hard to trigger as root. --- test/T050-new.sh | 1 + test/T150-tagging.sh | 1 + test/test-lib.sh | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/test/T050-new.sh b/test/T050-new.sh index cb67889c..240c1810 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -384,6 +384,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "Xapian exception: read only files" +test_subtest_broken_for_root chmod u-w ${MAIL_DIR}/.notmuch/xapian/*.* output=$(NOTMUCH_NEW --debug 2>&1 | sed 's/: .*$//' ) chmod u+w ${MAIL_DIR}/.notmuch/xapian/*.* diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh index 1a2fd77e..ac3f2539 100755 --- a/test/T150-tagging.sh +++ b/test/T150-tagging.sh @@ -320,6 +320,7 @@ test_begin_subtest "Tag name beginning with -" test_expect_code 1 'notmuch tag +- One' test_begin_subtest "Xapian exception: read only files" +test_subtest_broken_for_root chmod u-w ${MAIL_DIR}/.notmuch/xapian/*.* output=$(notmuch tag +something '*' 2>&1 | sed 's/: .*$//' ) chmod u+w ${MAIL_DIR}/.notmuch/xapian/*.* diff --git a/test/test-lib.sh b/test/test-lib.sh index eec5c5b4..1a6525df 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -740,6 +740,12 @@ test_subtest_known_broken () { test_subtest_known_broken_=t } +test_subtest_broken_for_root () { + if [ "$EUID" = "0" ]; then + test_subtest_known_broken_=t + fi +} + test_expect_success () { exec 1>&6 2>&7 # Restore stdout and stderr if [ -z "$inside_subtest" ]; then From db4b48f6cc033255d0214c76bc72a05d61bff118 Mon Sep 17 00:00:00 2001 From: Daniel Kahn Gillmor Date: Wed, 2 Nov 2022 12:15:12 -0400 Subject: [PATCH 13/90] configure: use pkg-config for gpgme GnuPG upstream has supported pkg-config since gpgme version 1.13 and gpg-error 1.33, and now prefers the use of pkg-config by default, instead of relying on gpg-error-config and gpgme-config. As of libgpg-error 1.46, upstream deliberately does not ship gpg-error-config by default. As of gpgme 1.18.0, upstream does not ship gpgme-config if gpg-error-config is also not present. Both of these versions of upstream libraries are in debian unstable now. To the extent that notmuch is dependent on GnuPG, it should follow GnuPG upstream's lead. Signed-off-by: Daniel Kahn Gillmor --- configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index be69b34d..c3629a73 100755 --- a/configure +++ b/configure @@ -541,8 +541,8 @@ version of GPGME. Please try to rebuild your version of GMime against a more recent version of GPGME (at least GPGME 1.8.0). EOF - if command -v gpgme-config >/dev/null; then - printf 'Your current GPGME development version is: %s\n' "$(gpgme-config --version)" + if GPGME_VERS="$(pkg-config --modversion gpgme || gpgme-config --version)"; then + printf 'Your current GPGME development version is: %s\n' "$GPGME_VERS" else printf 'You do not have the GPGME development libraries installed.\n' fi From f63d14a8c12ad76024d2865c0223a06f6f4bb372 Mon Sep 17 00:00:00 2001 From: jao Date: Tue, 13 Dec 2022 02:15:42 +0000 Subject: [PATCH 14/90] emacs: notmuch-tree-outline-mode With this mode, one can fold trees in the notmuch-tree buffer as if they were outlines, using all the commands provided by outline-minor-mode. We also define a couple of movement commands that, optional, will ensure that only the thread around point is unfolded. The implementation is based on registering a :level property in the messages p-list, that is then used by outline-minor-mode to to recognise headers. Amended by db: Copy docstring to manual and edit for presentation. Add two tests. Fix typo "wether". --- doc/notmuch-emacs.rst | 39 ++++ emacs/notmuch-tree.el | 183 +++++++++++++++++- test/T460-emacs-tree.sh | 24 +++ test/emacs-tree.expected-output/inbox-outline | 25 +++ 4 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 test/emacs-tree.expected-output/inbox-outline diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index 846f5e67..71f10e20 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -606,6 +606,45 @@ can be controlled by the variable ``notmuch-search-oldest-first``. See also :el:defcustom:`notmuch-search-result-format` and :el:defcustom:`notmuch-unthreaded-result-format`. +.. _notmuch-tree-outline: + +notmuch-tree-outline +-------------------- + +When this mode is set, each thread and subthread in the results +list is treated as a foldable section, with its first message as +its header. + +The mode just makes available in the tree buffer all the +keybindings in info:emacs#Outline_Mode, and binds the following +additional keys: + +.. el:define-key:: + + Cycle visibility state of the current message's tree. + +.. el:define-key:: + + Cycle visibility state of all trees in the buffer. + +The behaviour of this minor mode is affected by the following +customizable variables: + +.. el:defcustom:: notmuch-tree-outline-enabled + + |docstring::notmuch-tree-outline-enabled| + +.. el:defcustom:: notmuch-tree-outline-visibility + + |docstring::notmuch-tree-outline-visibility| + +.. el:defcustom:: notmuch-tree-outline-auto-close + + |docstring::notmuch-tree-outline-auto-close| + +.. el:defcustom:: notmuch-tree-outline-open-on-next + + |docstring::notmuch-tree-outline-open-on-next| .. _notmuch-unthreaded: diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el index b3c2c992..14775d59 100644 --- a/emacs/notmuch-tree.el +++ b/emacs/notmuch-tree.el @@ -1014,7 +1014,10 @@ unchanged ADDRESS if parsing fails." A message tree is another name for a single sub-thread: i.e., a message together with all its descendents." (let ((msg (car tree)) - (replies (cadr tree))) + (replies (cadr tree)) + ;; outline level, computed from the message's depth and + ;; whether or not it's the first message in the tree. + (level (1+ (if (and (eq 0 depth) (not first)) 1 depth)))) (cond ((and (< 0 depth) (not last)) (push (alist-get 'vertical-tee notmuch-tree-thread-symbols) tree-status)) @@ -1034,6 +1037,7 @@ message together with all its descendents." (setq msg (plist-put msg :first (and first (eq 0 depth)))) (setq msg (plist-put msg :tree-status tree-status)) (setq msg (plist-put msg :orig-tags (plist-get msg :tags))) + (setq msg (plist-put msg :level level)) (notmuch-tree-goto-and-insert-msg msg) (pop tree-status) (pop tree-status) @@ -1080,7 +1084,8 @@ Complete list of currently available key bindings: (setq notmuch-buffer-refresh-function #'notmuch-tree-refresh-view) (hl-line-mode 1) (setq buffer-read-only t) - (setq truncate-lines t)) + (setq truncate-lines t) + (when notmuch-tree-outline-enabled (notmuch-tree-outline-mode 1))) (defvar notmuch-tree-process-exit-functions nil "Functions called when the process inserting a tree of results finishes. @@ -1278,6 +1283,180 @@ search results and that are also tagged with the given TAG." nil notmuch-search-oldest-first))) +;;; Tree outline mode +;;;; Custom variables +(defcustom notmuch-tree-outline-enabled nil + "Whether to automatically activate `notmuch-tree-outline-mode' in tree views." + :type 'boolean) + +(defcustom notmuch-tree-outline-visibility 'hide-others + "Default state of the forest outline for `notmuch-tree-outline-mode'. + +This variable controls the state of a forest initially and after +a movement command. If set to nil, all trees are displayed while +the symbol hide-all indicates that all trees in the forest should +be folded and hide-other that only the first one should be +unfolded." + :type '(choice (const :tag "Show all" nil) + (const :tag "Hide others" hide-others) + (const :tag "Hide all" hide-all))) + +(defcustom notmuch-tree-outline-auto-close nil + "Close message and tree windows when moving past the last message." + :type 'boolean) + +(defcustom notmuch-tree-outline-open-on-next nil + "Open new messages under point if they are closed when moving to next one. + +When this flag is set, using the command +`notmuch-tree-outline-next' with point on a header for a new +message that is not shown will open its `notmuch-show' buffer +instead of moving point to next matching message." + :type 'boolean) + +;;;; Helper functions +(defsubst notmuch-tree-outline--pop-at-end (pop-at-end) + (if notmuch-tree-outline-auto-close (not pop-at-end) pop-at-end)) + +(defun notmuch-tree-outline--set-visibility () + (when (and notmuch-tree-outline-mode (> (point-max) (point-min))) + (cl-case notmuch-tree-outline-visibility + (hide-others (notmuch-tree-outline-hide-others)) + (hide-all (outline-hide-body))))) + +(defun notmuch-tree-outline--on-exit (proc) + (when (eq (process-status proc) 'exit) + (notmuch-tree-outline--set-visibility))) + +(add-hook 'notmuch-tree-process-exit-functions #'notmuch-tree-outline--on-exit) + +(defsubst notmuch-tree-outline--level (&optional props) + (or (plist-get (or props (notmuch-tree-get-message-properties)) :level) 0)) + +(defsubst notmuch-tree-outline--message-open-p () + (and (buffer-live-p notmuch-tree-message-buffer) + (get-buffer-window notmuch-tree-message-buffer) + (let ((id (notmuch-tree-get-message-id))) + (and id + (with-current-buffer notmuch-tree-message-buffer + (string= (notmuch-show-get-message-id) id)))))) + +(defsubst notmuch-tree-outline--at-original-match-p () + (and (notmuch-tree-get-prop :match) + (equal (notmuch-tree-get-prop :orig-tags) + (notmuch-tree-get-prop :tags)))) + +(defun notmuch-tree-outline--next (prev thread pop-at-end &optional open-new) + (cond (thread + (notmuch-tree-thread-top) + (if prev + (outline-backward-same-level 1) + (outline-forward-same-level 1)) + (when (> (notmuch-tree-outline--level) 0) (outline-show-branches)) + (notmuch-tree-outline--next nil nil pop-at-end t)) + ((and (or open-new notmuch-tree-outline-open-on-next) + (notmuch-tree-outline--at-original-match-p) + (not (notmuch-tree-outline--message-open-p))) + (notmuch-tree-outline-hide-others t)) + (t (outline-next-visible-heading (if prev -1 1)) + (unless (notmuch-tree-get-prop :match) + (notmuch-tree-matching-message prev pop-at-end)) + (notmuch-tree-outline-hide-others t)))) + +;;;; User commands +(defun notmuch-tree-outline-hide-others (&optional and-show) + "Fold all threads except the one around point. +If AND-SHOW is t, make the current message visible if it's not." + (interactive) + (save-excursion + (while (and (not (bobp)) (> (notmuch-tree-outline--level) 1)) + (outline-previous-heading)) + (outline-hide-sublevels 1)) + (when (> (notmuch-tree-outline--level) 0) + (outline-show-subtree) + (when and-show (notmuch-tree-show-message nil)))) + +(defun notmuch-tree-outline-next (&optional pop-at-end) + "Next matching message in a forest, taking care of thread visibility. +A prefix argument reverses the meaning of `notmuch-tree-outline-auto-close'." + (interactive "P") + (let ((pop (notmuch-tree-outline--pop-at-end pop-at-end))) + (if (null notmuch-tree-outline-visibility) + (notmuch-tree-matching-message nil pop) + (notmuch-tree-outline--next nil nil pop)))) + +(defun notmuch-tree-outline-previous (&optional pop-at-end) + "Previous matching message in forest, taking care of thread visibility. +With prefix, quit the tree view if there is no previous message." + (interactive "P") + (if (null notmuch-tree-outline-visibility) + (notmuch-tree-prev-matching-message pop-at-end) + (notmuch-tree-outline--next t nil pop-at-end))) + +(defun notmuch-tree-outline-next-thread () + "Next matching thread in forest, taking care of thread visibility." + (interactive) + (if (null notmuch-tree-outline-visibility) + (notmuch-tree-next-thread) + (notmuch-tree-outline--next nil t nil))) + +(defun notmuch-tree-outline-previous-thread () + "Previous matching thread in forest, taking care of thread visibility." + (interactive) + (if (null notmuch-tree-outline-visibility) + (notmuch-tree-prev-thread) + (notmuch-tree-outline--next t t nil))) + +;;;; Mode definition +(defvar notmuch-tree-outline-mode-lighter nil + "The lighter mark for notmuch-tree-outline mode. +Usually empty since outline-minor-mode's lighter will be active.") + +(define-minor-mode notmuch-tree-outline-mode + "Minor mode allowing message trees to be folded as outlines. + +When this mode is set, each thread and subthread in the results +list is treated as a foldable section, with its first message as +its header. + +The mode just makes available in the tree buffer all the +keybindings in `outline-minor-mode', and binds the following +additional keys: + +\\{notmuch-tree-outline-mode-map} + +The customizable variable `notmuch-tree-outline-visibility' +controls how navigation in the buffer is affected by this mode: + + - If it is set to nil, `notmuch-tree-outline-previous', + `notmuch-tree-outline-next', and their thread counterparts + behave just as the corresponding notmuch-tree navigation keys + when this mode is not enabled. + + - If, on the other hand, `notmuch-tree-outline-visibility' is + set to a non-nil value, these commands hiding the outlines of + the trees you are not reading as you move to new messages. + +To enable notmuch-tree-outline-mode by default in all +notmuch-tree buffers, just set +`notmuch-tree-outline-mode-enabled' to t." + :lighter notmuch-tree-outline-mode-lighter + :keymap `((,(kbd "TAB") . outline-cycle) + (,(kbd "M-TAB") . outline-cycle-buffer) + ("n" . notmuch-tree-outline-next) + ("p" . notmuch-tree-outline-previous) + (,(kbd "M-n") . notmuch-tree-outline-next-thread) + (,(kbd "M-p") . notmuch-tree-outline-previous-thread)) + (outline-minor-mode notmuch-tree-outline-mode) + (unless (derived-mode-p 'notmuch-tree-mode) + (user-error "notmuch-tree-outline-mode is only meaningful for notmuch trees!")) + (if notmuch-tree-outline-mode + (progn (setq-local outline-regexp "^[^\n]+" + outline-level #'notmuch-tree-outline--level) + (notmuch-tree-outline--set-visibility)) + (setq-local outline-regexp (default-value 'outline-regexp) + outline-level (default-value 'outline-level)))) + ;;; _ (provide 'notmuch-tree) diff --git a/test/T460-emacs-tree.sh b/test/T460-emacs-tree.sh index 3a1c449e..8e071443 100755 --- a/test/T460-emacs-tree.sh +++ b/test/T460-emacs-tree.sh @@ -200,6 +200,30 @@ test_emacs '(test-log-error (notmuch-tree "*")))' test_expect_equal "$(cat MESSAGES)" "COMPLETE" +# reinitialize database for outline tests +add_email_corpus + +test_begin_subtest "start in outline mode" +test_emacs '(let ((notmuch-tree-outline-enabled t)) + (notmuch-tree "tag:inbox") + (notmuch-test-wait) + (test-visible-output))' +# folding all messages by height or depth should look the same +test_expect_equal_file $EXPECTED/inbox-outline OUTPUT + +test_begin_subtest "outline-cycle-buffer" +test_emacs '(let ((notmuch-tree-outline-enabled t)) + (notmuch-tree "tag:inbox") + (notmuch-test-wait) + (outline-cycle-buffer) + (outline-cycle-buffer) + (notmuch-test-wait) + (test-visible-output))' +# folding all messages by height or depth should look the same +test_expect_equal_file $EXPECTED/notmuch-tree-tag-inbox OUTPUT + +test_done + add_email_corpus duplicate ID3=87r2ecrr6x.fsf@zephyr.silentflame.com diff --git a/test/emacs-tree.expected-output/inbox-outline b/test/emacs-tree.expected-output/inbox-outline new file mode 100644 index 00000000..9119a916 --- /dev/null +++ b/test/emacs-tree.expected-output/inbox-outline @@ -0,0 +1,25 @@ + 2010-12-29 François Boulogne ─►[aur-general] Guidelines: cp, mkdir vs install (inbox unread) + 2010-12-16 Olivier Berger ─►Essai accentué (inbox unread) + 2009-11-18 Chris Wilson ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox unread) + 2009-11-18 Alex Botero-Lowry ┬►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (attachment inbox unread) + 2009-11-17 Ingmar Vanhassel ┬►[notmuch] [PATCH] Typsos (inbox unread) + 2009-11-17 Adrian Perez de Cast ┬►[notmuch] Introducing myself (inbox signed unread) + 2009-11-17 Israel Herraiz ┬►[notmuch] New to the list (inbox unread) + 2009-11-17 Jan Janak ┬►[notmuch] What a great idea! (inbox unread) + 2009-11-17 Jan Janak ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox unread) + 2009-11-17 Aron Griffis ┬►[notmuch] archive (inbox unread) + 2009-11-17 Keith Packard ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox unread) + 2009-11-17 Lars Kellogg-Stedman ┬►[notmuch] Working with Maildir storage? (inbox signed unread) + 2009-11-17 Mikhail Gusarov ┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox unread) + 2009-11-18 Keith Packard ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox unread) + 2009-11-18 Alexander Botero-Low ─►[notmuch] request for pull (inbox unread) + 2009-11-18 Jjgod Jiang ┬►[notmuch] Mac OS X/Darwin compatibility issues (inbox unread) + 2009-11-18 Rolland Santimano ─►[notmuch] Link to mailing list archives ? (inbox unread) + 2009-11-18 Jan Janak ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool subdirectories into tags (inbox unread) + 2009-11-18 Stewart Smith ─►[notmuch] [PATCH] count_files: sort directory in inode order before statting (inbox unread) + 2009-11-18 Stewart Smith ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox unread) + 2009-11-18 Stewart Smith ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++ libs. (inbox unread) + 2009-11-18 Lars Kellogg-Stedman ┬►[notmuch] "notmuch help" outputs to stderr? (attachment inbox signed unread) + 2009-11-17 Mikhail Gusarov ─►[notmuch] [PATCH] Handle rename of message file (inbox unread) + 2009-11-17 Alex Botero-Lowry ┬►[notmuch] preliminary FreeBSD support (attachment inbox unread) +End of search results. From 4e6c6c8aac7f581448df525abd09b85c08358dd3 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 27 Dec 2022 13:08:46 -0400 Subject: [PATCH 15/90] test: add known broken test for diagnostics from over long filenames. Previously we tested over long directory names, add similar testing for over long filenames. --- test/T050-new.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/T050-new.sh b/test/T050-new.sh index 240c1810..c4a51a05 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -383,6 +383,21 @@ No new mail. Removed 1 message. EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "Long file names have reasonable diagnostics" +test_subtest_known_broken +printf -v name 'f%.0s' {1..234} +generate_message "[filename]=$name" +notmuch new 2>&1 | notmuch_dir_sanitize >OUTPUT +rm ${MAIL_DIR}/${name} +cat < EXPECTED +Note: Ignoring non-indexable path: MAIL_DIR/$name +add_file: Path supplied is illegal for this function +filename too long for file-direntry term: MAIL_DIR/$name +Processed 1 file in almost no time. +No new mail. +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "Xapian exception: read only files" test_subtest_broken_for_root chmod u-w ${MAIL_DIR}/.notmuch/xapian/*.* From 09f2ad8e853375930c63bca847f623bc722b9cc0 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 27 Dec 2022 13:08:47 -0400 Subject: [PATCH 16/90] lib: add better diagnostics for over long filenames. Previously we just crashed with an internal error. With this change, the caller can handle it better. Update notmuch-new so that it doesn't crash with "unknown error code" because of this change. --- lib/message.cc | 16 ++++++++++++---- lib/notmuch.h | 2 ++ notmuch-new.c | 4 ++++ test/T050-new.sh | 1 - 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/message.cc b/lib/message.cc index 5ccca95a..1b1a071a 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -941,6 +941,7 @@ _notmuch_message_add_filename (notmuch_message_t *message, { const char *relative, *directory; notmuch_status_t status; + notmuch_private_status_t private_status; void *local = talloc_new (message); char *direntry; @@ -964,10 +965,17 @@ _notmuch_message_add_filename (notmuch_message_t *message, /* New file-direntry allows navigating to this message with * notmuch_directory_get_child_files() . */ - status = COERCE_STATUS (_notmuch_message_add_term (message, "file-direntry", direntry), - "adding file-direntry term"); - if (status) - return status; + private_status = _notmuch_message_add_term (message, "file-direntry", direntry); + switch (private_status) { + case NOTMUCH_PRIVATE_STATUS_SUCCESS: + break; + case NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG: + _notmuch_database_log (message->notmuch, "filename too long for file-direntry term: %s\n", + filename); + return NOTMUCH_STATUS_PATH_ERROR; + default: + return COERCE_STATUS (private_status, "adding file-direntry term"); + } status = _notmuch_message_add_folder_terms (message, directory); if (status) diff --git a/lib/notmuch.h b/lib/notmuch.h index 0b0540b1..ce375c04 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -293,6 +293,8 @@ typedef struct _notmuch_indexopts notmuch_indexopts_t; * * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory. * + * NOTMUCH_STATUS_PATH_ERROR: filename is too long + * * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to create the * database file (such as permission denied, or file not found, * etc.), or the database already exists. diff --git a/notmuch-new.c b/notmuch-new.c index 346e6469..4a53e3eb 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -413,6 +413,10 @@ add_file (notmuch_database_t *notmuch, const char *filename, case NOTMUCH_STATUS_FILE_NOT_EMAIL: fprintf (stderr, "Note: Ignoring non-mail file: %s\n", filename); break; + case NOTMUCH_STATUS_PATH_ERROR: + fprintf (stderr, "Note: Ignoring non-indexable path: %s\n", filename); + (void) print_status_database ("add_file", notmuch, status); + break; case NOTMUCH_STATUS_FILE_ERROR: /* Someone renamed/removed the file between scandir and now. */ state->vanished_files++; diff --git a/test/T050-new.sh b/test/T050-new.sh index c4a51a05..09c2bfc6 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -384,7 +384,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "Long file names have reasonable diagnostics" -test_subtest_known_broken printf -v name 'f%.0s' {1..234} generate_message "[filename]=$name" notmuch new 2>&1 | notmuch_dir_sanitize >OUTPUT From 48d774bbf4dc6e442e1be96aab712947b408fc5a Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 21 Feb 2023 07:49:15 -0400 Subject: [PATCH 17/90] emacs/tree: use two argument form of setq-local Apparently the macro setq-local only takes two arguments in Emacs 26.1 --- emacs/notmuch-tree.el | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el index 14775d59..b58fa6a6 100644 --- a/emacs/notmuch-tree.el +++ b/emacs/notmuch-tree.el @@ -1451,11 +1451,11 @@ notmuch-tree buffers, just set (unless (derived-mode-p 'notmuch-tree-mode) (user-error "notmuch-tree-outline-mode is only meaningful for notmuch trees!")) (if notmuch-tree-outline-mode - (progn (setq-local outline-regexp "^[^\n]+" - outline-level #'notmuch-tree-outline--level) + (progn (setq-local outline-regexp "^[^\n]+") + (setq-local outline-level #'notmuch-tree-outline--level) (notmuch-tree-outline--set-visibility)) - (setq-local outline-regexp (default-value 'outline-regexp) - outline-level (default-value 'outline-level)))) + (setq-local outline-regexp (default-value 'outline-regexp)) + (setq-local outline-level (default-value 'outline-level)))) ;;; _ From d86e03c786ec51e2ca4af4e7c756cd19adbe17a8 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Mon, 27 Feb 2023 13:06:29 +0100 Subject: [PATCH 18/90] lib/notmuch: update example Likely missed in 86cbd215e, when notmuch_query_search_messages_st was renamed to notmuch_query_search_messages. --- lib/notmuch.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/notmuch.h b/lib/notmuch.h index ce375c04..89018392 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -1172,7 +1172,10 @@ notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out * * query = notmuch_query_create (database, query_string); * - * for (messages = notmuch_query_search_messages (query); + * if (notmuch_query_search_messages (query, &messages) != NOTMUCH_STATUS_SUCCESS) + * return EXIT_FAILURE; + * + * for (; * notmuch_messages_valid (messages); * notmuch_messages_move_to_next (messages)) * { From c810312e24067a7176a26b35893cf64f754886c5 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:19:57 +0200 Subject: [PATCH 19/90] test: uncaught exception when editing properties of a removed message These two functions don't fail gracefully when editing a removed message: BROKEN edit property on removed message without uncaught exception --- T610-message-property.20.EXPECTED 2023-02-27 11:33:25.792764376 +0000 +++ T610-message-property.20.OUTPUT 2023-02-27 11:33:25.793764381 +0000 @@ -1,2 +1,3 @@ == stdout == == stderr == +terminate called after throwing an instance of 'Xapian::DocNotFoundError' The other functions appear to be safe. --- test/T610-message-property.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 2685f3b5..402b4e9a 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -362,4 +362,36 @@ for (key,val) in msg.get_properties("testkey",True): EOF test_expect_equal_file /dev/null OUTPUT +test_begin_subtest "edit property on removed message without uncaught exception" +test_subtest_known_broken +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} +EXPECT0(notmuch_database_remove_message (db, notmuch_message_get_filename (message))); +stat = notmuch_message_remove_property (message, "example", "example"); +if (stat == NOTMUCH_STATUS_XAPIAN_EXCEPTION) + fprintf (stderr, "unable to remove properties on message"); +EOF +cat <<'EOF' >EXPECTED +== stdout == +== stderr == +unable to remove properties on message +EOF +test_expect_equal_file EXPECTED OUTPUT + +add_email_corpus + +test_begin_subtest "remove all properties on removed message without uncaught exception" +test_subtest_known_broken +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} +EXPECT0(notmuch_database_remove_message (db, notmuch_message_get_filename (message))); +stat = notmuch_message_remove_all_properties_with_prefix (message, ""); +if (stat == NOTMUCH_STATUS_XAPIAN_EXCEPTION) + fprintf (stderr, "unable to remove properties on message"); +EOF +cat <<'EOF' >EXPECTED +== stdout == +== stderr == +unable to remove properties on message +EOF +test_expect_equal_file EXPECTED OUTPUT + test_done From 568f6bc3c2fd2396c05d254e2649750fb82b00b6 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:19:58 +0200 Subject: [PATCH 20/90] lib/message-property: catch xapian exceptions Since libnotmuch exposes a C interface there's no way for clients to catch this. Inspired by what's done for tags (see notmuch_message_remove_tag). --- lib/message-property.cc | 36 +++++++++++++++++++++++++++++------ test/T610-message-property.sh | 2 -- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lib/message-property.cc b/lib/message-property.cc index d5afa30c..0d444bb8 100644 --- a/lib/message-property.cc +++ b/lib/message-property.cc @@ -25,6 +25,20 @@ #include "database-private.h" #include "message-private.h" +#define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error) + +static void +_log_xapian_exception (const char *where, notmuch_message_t *message, const Xapian::Error error) +{ + notmuch_database_t *notmuch = notmuch_message_get_database (message); + + _notmuch_database_log (notmuch, + "A Xapian exception occurred at %s: %s\n", + where, + error.get_msg ().c_str ()); + notmuch->exception_reported = true; +} + notmuch_status_t notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value) { @@ -83,10 +97,15 @@ _notmuch_message_modify_property (notmuch_message_t *message, const char *key, c term = talloc_asprintf (message, "%s=%s", key, value); - if (delete_it) - private_status = _notmuch_message_remove_term (message, "property", term); - else - private_status = _notmuch_message_add_term (message, "property", term); + try { + if (delete_it) + private_status = _notmuch_message_remove_term (message, "property", term); + else + private_status = _notmuch_message_add_term (message, "property", term); + } catch (Xapian::Error &error) { + LOG_XAPIAN_EXCEPTION (message, error); + return NOTMUCH_STATUS_XAPIAN_EXCEPTION; + } if (private_status) return COERCE_STATUS (private_status, @@ -130,8 +149,13 @@ _notmuch_message_remove_all_properties (notmuch_message_t *message, const char * else term_prefix = _find_prefix ("property"); - /* XXX better error reporting ? */ - _notmuch_message_remove_terms (message, term_prefix); + try { + /* XXX better error reporting ? */ + _notmuch_message_remove_terms (message, term_prefix); + } catch (Xapian::Error &error) { + LOG_XAPIAN_EXCEPTION (message, error); + return NOTMUCH_STATUS_XAPIAN_EXCEPTION; + } return NOTMUCH_STATUS_SUCCESS; } diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 402b4e9a..13f8a3f9 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -363,7 +363,6 @@ EOF test_expect_equal_file /dev/null OUTPUT test_begin_subtest "edit property on removed message without uncaught exception" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_database_remove_message (db, notmuch_message_get_filename (message))); stat = notmuch_message_remove_property (message, "example", "example"); @@ -380,7 +379,6 @@ test_expect_equal_file EXPECTED OUTPUT add_email_corpus test_begin_subtest "remove all properties on removed message without uncaught exception" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_database_remove_message (db, notmuch_message_get_filename (message))); stat = notmuch_message_remove_all_properties_with_prefix (message, ""); From ce59df52968d661bc5a173f75a37f27d290d955b Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:13:29 +0200 Subject: [PATCH 21/90] test: display key name in property tests To make the tests a bit easier to understand. --- test/T610-message-property.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 13f8a3f9..07a48053 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -12,7 +12,7 @@ void print_properties (notmuch_message_t *message, const char *prefix, notmuch_b notmuch_message_properties_t *list; for (list = notmuch_message_get_properties (message, prefix, exact); notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) { - printf("%s\n", notmuch_message_properties_value(list)); + printf("%s = %s\n", notmuch_message_properties_key(list), notmuch_message_properties_value(list)); } notmuch_message_properties_destroy (list); } @@ -157,7 +157,7 @@ print_properties (message, "testkey1", TRUE); EOF cat <<'EOF' >EXPECTED == stdout == -testvalue1 +testkey1 = testvalue1 == stderr == EOF test_expect_equal_file EXPECTED OUTPUT @@ -171,10 +171,10 @@ print_properties (message, "testkey1", TRUE); EOF cat <<'EOF' >EXPECTED == stdout == -alice -bob -testvalue1 -testvalue2 +testkey1 = alice +testkey1 = bob +testkey1 = testvalue1 +testkey1 = testvalue2 == stderr == EOF test_expect_equal_file EXPECTED OUTPUT @@ -200,13 +200,13 @@ awk ' NR == 1 { print; next } \ rm unsorted_OUTPUT cat <<'EOF' >EXPECTED == stdout == -alice -bob -testvalue1 -testvalue2 -alice3 -bob3 -testvalue3 +testkey1 = alice +testkey1 = bob +testkey1 = testvalue1 +testkey1 = testvalue2 +testkey3 = alice3 +testkey3 = bob3 +testkey3 = testvalue3 == stderr == EOF test_expect_equal_file EXPECTED OUTPUT From 552d9ec9f766762ffdd1f565fc88b0a980f15b39 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:13:30 +0200 Subject: [PATCH 22/90] test: remove unnecessary sorting The other tests rely on a stable output. --- test/T610-message-property.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 07a48053..a4814d64 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -186,18 +186,6 @@ EXPECT0(notmuch_message_add_property (message, "testkey3", "testvalue3")); EXPECT0(notmuch_message_add_property (message, "testkey3", "alice3")); print_properties (message, "testkey", FALSE); EOF -# expected: 4 values for testkey1, 3 values for testkey3 -# they are not guaranteed to be sorted, so sort them, leaving the first -# line '== stdout ==' and the end ('== stderr ==' and whatever error -# may have been printed) alone -mv OUTPUT unsorted_OUTPUT -awk ' NR == 1 { print; next } \ - NR < 6 { print | "sort"; next } \ - NR == 6 { close("sort") } \ - NR < 9 { print | "sort"; next } \ - NR == 9 { close("sort") } \ - { print }' unsorted_OUTPUT > OUTPUT -rm unsorted_OUTPUT cat <<'EOF' >EXPECTED == stdout == testkey1 = alice From e191d3c574e91199e524acbf6d7199bd58fd73d5 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:13:31 +0200 Subject: [PATCH 23/90] test: reorganize tests and mark a few of them as broken notmuch_message_remove_all_properties should have removed the testkey1 = testvalue1 property but hasn't. Delay the execution of the corresponding test to avoid updating a few tests that actually relied on the broken behavior. --- test/T610-message-property.sh | 39 ++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index a4814d64..1bc423b5 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -89,17 +89,6 @@ testkey2 = NULL EOF test_expect_equal_file EXPECTED OUTPUT -test_begin_subtest "notmuch_message_remove_all_properties" -cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} -EXPECT0(notmuch_message_remove_all_properties (message, NULL)); -print_properties (message, "", FALSE); -EOF -cat <<'EOF' >EXPECTED -== stdout == -== stderr == -EOF -test_expect_equal_file EXPECTED OUTPUT - test_begin_subtest "testing string map binary search (via message properties)" cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} { @@ -162,7 +151,19 @@ testkey1 = testvalue1 EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "notmuch_message_remove_all_properties" +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} +EXPECT0(notmuch_message_remove_all_properties (message, NULL)); +print_properties (message, "", FALSE); +EOF +cat <<'EOF' >EXPECTED +== stdout == +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "notmuch_message_properties: multiple values" +test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey1", "bob")); EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue2")); @@ -173,13 +174,13 @@ cat <<'EOF' >EXPECTED == stdout == testkey1 = alice testkey1 = bob -testkey1 = testvalue1 testkey1 = testvalue2 == stderr == EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch_message_properties: prefix" +test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey3", "bob3")); EXPECT0(notmuch_message_add_property (message, "testkey3", "testvalue3")); @@ -190,7 +191,6 @@ cat <<'EOF' >EXPECTED == stdout == testkey1 = alice testkey1 = bob -testkey1 = testvalue1 testkey1 = testvalue2 testkey3 = alice3 testkey3 = bob3 @@ -235,8 +235,9 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "dump message properties" +test_subtest_known_broken cat < PROPERTIES -#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 +#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 EOF cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "fancy key with áccènts", "import value with =")); @@ -245,15 +246,17 @@ notmuch dump | grep '^#=' > OUTPUT test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "dump _only_ message properties" +test_subtest_known_broken cat < EXPECTED #notmuch-dump batch-tag:3 properties -#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 +#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 EOF notmuch dump --include=properties > OUTPUT test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "restore missing message property (single line)" +test_subtest_known_broken notmuch dump | grep '^#=' > BEFORE1 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob")); @@ -264,6 +267,7 @@ test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "restore missing message property (full dump)" +test_subtest_known_broken notmuch dump > BEFORE2 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob")); @@ -273,6 +277,7 @@ notmuch dump | grep '^#=' > OUTPUT test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "restore clear extra message property" +test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey1", "charles")); EOF @@ -306,6 +311,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "msg.get_properties (python)" +test_subtest_known_broken test_python <<'EOF' import notmuch db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY) @@ -316,12 +322,12 @@ EOF cat <<'EOF' > EXPECTED testkey1 = alice testkey1 = bob -testkey1 = testvalue1 testkey1 = testvalue2 EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "msg.get_properties (python, prefix)" +test_subtest_known_broken test_python <<'EOF' import notmuch db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY) @@ -332,7 +338,6 @@ EOF cat <<'EOF' > EXPECTED testkey1 = alice testkey1 = bob -testkey1 = testvalue1 testkey1 = testvalue2 testkey3 = alice3 testkey3 = bob3 From 336334996750240608d5f29ed5dd8e40a69c4d79 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 30 Mar 2023 07:56:17 -0300 Subject: [PATCH 24/90] test: reveal notmuch_message_remove_all_properties as broken Close and re-open the database to show that the removal is not committed to the database. --- test/T610-message-property.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 1bc423b5..afcda4a8 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -152,8 +152,19 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch_message_remove_all_properties" +test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_all_properties (message, NULL)); +EXPECT0(notmuch_database_destroy(db)); +EXPECT0(notmuch_database_open_with_config (argv[1], + NOTMUCH_DATABASE_MODE_READ_WRITE, + "", NULL, &db, &msg)); +if (msg) fputs (msg, stderr); +EXPECT0(notmuch_database_find_message(db, "4EFC743A.3060609@april.org", &message)); +if (message == NULL) { + fprintf (stderr, "unable to find message"); + exit (1); +} print_properties (message, "", FALSE); EOF cat <<'EOF' >EXPECTED From fb55ff28a2fdaa9c218af5ca10b1cae674869edd Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:13:32 +0200 Subject: [PATCH 25/90] lib/message-property: sync removed properties to the database _notmuch_message_remove_all_properties wasn't syncing the message back to the database but was still invalidating the metadata, giving the impression the properties had actually been removed. Also move the metadata invalidation to _notmuch_message_remove_terms to be closer to what's done in _notmuch_message_modify_property and _notmuch_message_remove_term. --- lib/message-property.cc | 4 +++- lib/message.cc | 2 ++ test/T610-message-property.sh | 10 ---------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/message-property.cc b/lib/message-property.cc index 0d444bb8..7f520340 100644 --- a/lib/message-property.cc +++ b/lib/message-property.cc @@ -142,7 +142,6 @@ _notmuch_message_remove_all_properties (notmuch_message_t *message, const char * if (status) return status; - _notmuch_message_invalidate_metadata (message, "property"); if (key) term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key, prefix ? "" : "="); @@ -157,6 +156,9 @@ _notmuch_message_remove_all_properties (notmuch_message_t *message, const char * return NOTMUCH_STATUS_XAPIAN_EXCEPTION; } + if (! _notmuch_message_frozen (message)) + _notmuch_message_sync (message); + return NOTMUCH_STATUS_SUCCESS; } diff --git a/lib/message.cc b/lib/message.cc index 1b1a071a..53f35dd1 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -719,6 +719,8 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix) /* Ignore failure to remove non-existent term. */ } } + + _notmuch_message_invalidate_metadata (message, "property"); } diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index afcda4a8..05cfd3b5 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -152,7 +152,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch_message_remove_all_properties" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_all_properties (message, NULL)); EXPECT0(notmuch_database_destroy(db)); @@ -174,7 +173,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch_message_properties: multiple values" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey1", "bob")); EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue2")); @@ -191,7 +189,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch_message_properties: prefix" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey3", "bob3")); EXPECT0(notmuch_message_add_property (message, "testkey3", "testvalue3")); @@ -246,7 +243,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "dump message properties" -test_subtest_known_broken cat < PROPERTIES #= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 EOF @@ -257,7 +253,6 @@ notmuch dump | grep '^#=' > OUTPUT test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "dump _only_ message properties" -test_subtest_known_broken cat < EXPECTED #notmuch-dump batch-tag:3 properties #= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 @@ -267,7 +262,6 @@ test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "restore missing message property (single line)" -test_subtest_known_broken notmuch dump | grep '^#=' > BEFORE1 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob")); @@ -278,7 +272,6 @@ test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "restore missing message property (full dump)" -test_subtest_known_broken notmuch dump > BEFORE2 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob")); @@ -288,7 +281,6 @@ notmuch dump | grep '^#=' > OUTPUT test_expect_equal_file PROPERTIES OUTPUT test_begin_subtest "restore clear extra message property" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_message_add_property (message, "testkey1", "charles")); EOF @@ -322,7 +314,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "msg.get_properties (python)" -test_subtest_known_broken test_python <<'EOF' import notmuch db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY) @@ -338,7 +329,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "msg.get_properties (python, prefix)" -test_subtest_known_broken test_python <<'EOF' import notmuch db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY) From a95959c491cc9ed7ca1493272b3fa91313f75041 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 29 Mar 2023 18:13:33 +0200 Subject: [PATCH 26/90] test: add test for notmuch_message_remove_all_properties_with_prefix It wasn't covered, though it shares most of its implementation with notmuch_message_remove_all_properties. --- test/T610-message-property.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index 05cfd3b5..a7cbe048 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -356,6 +356,21 @@ for (key,val) in msg.get_properties("testkey",True): EOF test_expect_equal_file /dev/null OUTPUT +test_begin_subtest "notmuch_message_remove_all_properties_with_prefix" +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} +EXPECT0(notmuch_message_remove_all_properties_with_prefix (message, "testkey3")); +print_properties (message, "", FALSE); +EOF +cat <<'EOF' >EXPECTED +== stdout == +fancy key with áccènts = import value with = +testkey1 = alice +testkey1 = bob +testkey1 = testvalue2 +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "edit property on removed message without uncaught exception" cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} EXPECT0(notmuch_database_remove_message (db, notmuch_message_get_filename (message))); From 83ea220178f771bf5a4f6e2f4fe3a3dd2e96676e Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 22 Mar 2023 20:05:22 -0600 Subject: [PATCH 27/90] ruby: use database_open_with_config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes warning: warning: ‘notmuch_database_open’ is deprecated: function deprecated as of libnotmuch 5.4 Signed-off-by: Felipe Contreras --- bindings/ruby/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/ruby/database.c b/bindings/ruby/database.c index 9c3dbd96..a78d508b 100644 --- a/bindings/ruby/database.c +++ b/bindings/ruby/database.c @@ -91,7 +91,7 @@ notmuch_rb_database_initialize (int argc, VALUE *argv, VALUE self) if (create) ret = notmuch_database_create (path, &database); else - ret = notmuch_database_open (path, mode, &database); + ret = notmuch_database_open_with_config (path, mode, NULL, NULL, &database, NULL); notmuch_rb_status_raise (ret); DATA_PTR (self) = notmuch_rb_object_create (database, "notmuch_rb_database"); From 4152e1bc20fa2803186740c76579b495f4c77fb6 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 22 Mar 2023 20:05:23 -0600 Subject: [PATCH 28/90] ruby: database: make path arg optional It can be automatically loaded from the configuration now. Signed-off-by: Felipe Contreras --- bindings/ruby/database.c | 10 +++++++--- test/T395-ruby.sh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bindings/ruby/database.c b/bindings/ruby/database.c index a78d508b..b6de1254 100644 --- a/bindings/ruby/database.c +++ b/bindings/ruby/database.c @@ -59,10 +59,14 @@ notmuch_rb_database_initialize (int argc, VALUE *argv, VALUE self) notmuch_status_t ret; /* Check arguments */ - rb_scan_args (argc, argv, "11", &pathv, &hashv); + rb_scan_args (argc, argv, "02", &pathv, &hashv); - SafeStringValue (pathv); - path = RSTRING_PTR (pathv); + if (!NIL_P (pathv)) { + SafeStringValue (pathv); + path = RSTRING_PTR (pathv); + } else { + path = NULL; + } if (!NIL_P (hashv)) { Check_Type (hashv, T_HASH); diff --git a/test/T395-ruby.sh b/test/T395-ruby.sh index e828efed..c066c842 100755 --- a/test/T395-ruby.sh +++ b/test/T395-ruby.sh @@ -12,7 +12,7 @@ test_ruby() { ( cat <<-EOF require 'notmuch' - db = Notmuch::Database.new('$MAIL_DIR') + db = Notmuch::Database.new() EOF cat ) | $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"> OUTPUT From 6273966d0b50541a37a652ccf6113f184eff5300 Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Thu, 2 Mar 2023 18:59:15 +0100 Subject: [PATCH 29/90] lib: replace some uses of Query::MatchAll with a thread-safe alternative This replaces two instances of Xapian::Query::MatchAll with the equivalent but thread-safe alternative Xapian::Query(std::string()). Xapian::Query::MatchAll maintains an internal pointer to a refcounted Xapian::Internal::QueryTerm. None of this is thread-safe but that wouldn't be an issue if Xapian::Query::MatchAll wasn't static. Because it's static, the refcounting goes awry when Notmuch is called from multiple threads. This is actually documented by Xapian: https://github.com/xapian/xapian/blob/4715de3a9fcee741587439dc3cc1d2ff01ffeaf2/xapian-core/include/xapian/query.h#L65 While static, Xapian::Query::MatchNothing is safe because it doesn't maintain an internal object and as such, doesn't use references. Two best-effort tests making use of TSan were added to showcase the issue (I couldn't figure out a way to deterministically reproduce it without making an unmaintainable mess). First, when two databases are created in parallel, a query that uses Xapian::Query::MatchAll is made (lib/query.cc), resulting in the following backtrace on a segfault: #0 0x00007ffff76822af in Xapian::Query::get_terms_begin (this=0x7fffe80137f0) at api/query.cc:141 #1 0x00007ffff7f933f5 in _notmuch_query_cache_terms (query=0x7fffe80137c0) at lib/query.cc:176 #2 0x00007ffff7f93784 in _notmuch_query_ensure_parsed_xapian (query=0x7fffe80137c0) at lib/query.cc:225 #3 0x00007ffff7f9381a in _notmuch_query_ensure_parsed (query=0x7fffe80137c0) at lib/query.cc:260 #4 0x00007ffff7f93bfe in _notmuch_query_search_documents (query=0x7fffe80137c0, type=0x7ffff7fa9b1e "mail", out=0x7ffff666da18) at lib/query.cc:361 #5 0x00007ffff7f93ba4 in notmuch_query_search_messages (query=0x7fffe80137c0, out=0x7ffff666da18) at lib/query.cc:349 #6 0x00007ffff7f83d98 in notmuch_database_upgrade (notmuch=0x7fffe8000bd0, progress_notify=0x0, closure=0x0) at lib/database.cc:934 #7 0x00007ffff7fa110f in notmuch_database_create_with_config (database_path=0x7ffff666dcb0 "/tmp/notmuch.MZ2AGr", config_path=0x7ffff7faab3c "", profile=0x0, database=0x0, status_string=0x7ffff666dc90) at lib/open.cc:754 #8 0x00007ffff7fa0d6f in notmuch_database_create_verbose (path=0x7ffff666dcb0 "/tmp/notmuch.MZ2AGr", database=0x0, status_string=0x7ffff666dc90) at lib/open.cc:653 #9 0x00007ffff7fa0ceb in notmuch_database_create (path=0x7ffff666dcb0 "/tmp/notmuch.MZ2AGr", database=0x0) at lib/open.cc:637 ... Second, some queries would make use of Xapian::Query::MatchAll (lib/regexp-fields.cc), resulting in the following backtrace on a segfault: #0 0x00007f629828b690 in Xapian::Internal::QueryBranch::gather_terms (this=0x7f628800def0, void_terms=0x7f629726d5a0) at api/queryinternal.cc:1245 #1 0x00007f629828c260 in Xapian::Internal::QueryScaleWeight::gather_terms (this=0x7f628800df70, void_terms=0x7f629726d5a0) at api/queryinternal.cc:1434 #2 0x00007f629828b69f in Xapian::Internal::QueryBranch::gather_terms (this=0x7f628800dd90, void_terms=0x7f629726d5a0) at api/queryinternal.cc:1245 #3 0x00007f6298282571 in Xapian::Query::get_unique_terms_begin (this=0x7f628800dcd8) at api/query.cc:166 #4 0x00007f629841a59b in Xapian::Weight::Internal::accumulate_stats (this=0x7f628800dca0, subdb=..., rset=...) at weight/weightinternal.cc:86 #5 0x00007f62983c15ba in LocalSubMatch::prepare_match (this=0x7f628800df20, nowait=true, total_stats=...) at matcher/localsubmatch.cc:172 #6 0x00007f62983c8fcc in prepare_sub_matches (leaves=std::vector of length 1, capacity 1 = {...}, stats=...) at matcher/multimatch.cc:237 #7 0x00007f62983c98a3 in MultiMatch::MultiMatch (this=0x7f629726d9a0, db_=..., query_=..., qlen=3, omrset=0x0, collapse_max_=0, collapse_key_=4294967295, percent_cutoff_=0, weight_cutoff_=0, order_=Xapian::Enquire::ASCENDING, sort_key_=0, sort_by_=Xapian::Enquire::Internal::VAL, sort_value_forward_=true, time_limit_=0, stats=..., weight_=0x7f6288008d50, matchspies_=std::vector of length 0, capacity 0, have_sorter=false, have_mdecider=false) at matcher/multimatch.cc:353 #8 0x00007f629826fcba in Xapian::Enquire::Internal::get_mset (this=0x7f628800e0b0, first=0, maxitems=0, check_at_least=0, rset=0x0, mdecider=0x0) at api/omenquire.cc:569 #9 0x00007f629827181c in Xapian::Enquire::get_mset (this=0x7f629726db80, first=0, maxitems=0, check_at_least=0, rset=0x0, mdecider=0x0) at api/omenquire.cc:937 #10 0x00007f6298be529a in _notmuch_query_search_documents (query=0x7f6288009750, type=0x7f6298bfaafe "mail", out=0x7f629726dcc0) at lib/query.cc:447 #11 0x00007f6298be4ae8 in notmuch_query_search_messages (query=0x7f6288009750, out=0x7f629726dcc0) at lib/query.cc:349 ... Printing Xapian::Query::MatchAll->internal.px->_refs in these circumstances can help quickly identifying this scenario. This is motivated by some test frameworks (like Rust's Cargo) that runs unit tests in parallel and would easily encounter this issue, unless client code gates every call to Notmuch behind a lock. This is what can be expected from the tests when they fail: == stderr == +================== +WARNING: ThreadSanitizer: data race (pid=207931) + Read of size 1 at 0x7b10000001a0 by thread T2: + #0 memcpy (libtsan.so.2+0x62506) + #1 void std::__cxx11::basic_string, std::allocator >::_M_construct(char*, char*, std::forward_iterator_tag) [clone .isra.0] (libxapian.so.30+0x872b3) + + Previous write of size 8 at 0x7b10000001a0 by thread T1: + #0 operator new(unsigned long) (libtsan.so.2+0x8ba83) + #1 Xapian::Query::Query(std::__cxx11::basic_string, std::allocator > const&, unsigned int, unsigned int) (libxapian.so.30+0x855cd) ... --- configure | 15 +++++- lib/query.cc | 3 +- lib/regexp-fields.cc | 3 +- test/T800-asan.sh | 2 +- test/T810-tsan.sh | 92 +++++++++++++++++++++++++++++++++++++ test/T810-tsan.suppressions | 5 ++ util/xapian-extra.h | 15 ++++++ 7 files changed, 131 insertions(+), 4 deletions(-) create mode 100755 test/T810-tsan.sh create mode 100644 test/T810-tsan.suppressions create mode 100644 util/xapian-extra.h diff --git a/configure b/configure index c3629a73..7afd08c7 100755 --- a/configure +++ b/configure @@ -422,6 +422,18 @@ else fi unset test_cmdline +printf "C compiler supports thread sanitizer... " +test_cmdline="${CC} ${CFLAGS} ${CPPFLAGS} -fsanitize=thread minimal.c ${LDFLAGS} -o minimal" +if ${test_cmdline} >/dev/null 2>&1 && ./minimal +then + printf "Yes.\n" + have_tsan=1 +else + printf "Nope, skipping those tests.\n" + have_tsan=0 +fi +unset test_cmdline + printf "Reading libnotmuch version from source... " cat > _libversion.c < @@ -1590,8 +1602,9 @@ NOTMUCH_GMIME_VERIFY_WITH_SESSION_KEY=${gmime_verify_with_session_key} NOTMUCH_ZLIB_CFLAGS="${zlib_cflags}" NOTMUCH_ZLIB_LDFLAGS="${zlib_ldflags}" -# Does the C compiler support the address sanitizer +# Does the C compiler support the sanitizers NOTMUCH_HAVE_ASAN=${have_asan} +NOTMUCH_HAVE_TSAN=${have_tsan} # do we have man pages? NOTMUCH_HAVE_MAN=$((have_sphinx)) diff --git a/lib/query.cc b/lib/query.cc index 707f6222..1c60c122 100644 --- a/lib/query.cc +++ b/lib/query.cc @@ -20,6 +20,7 @@ #include "notmuch-private.h" #include "database-private.h" +#include "xapian-extra.h" #include /* GHashTable, GPtrArray */ @@ -186,7 +187,7 @@ _notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch, { try { if (query_string == "" || query_string == "*") { - output = Xapian::Query::MatchAll; + output = xapian_query_match_all (); } else { output = notmuch->query_parser-> diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc index 539915d8..3a775261 100644 --- a/lib/regexp-fields.cc +++ b/lib/regexp-fields.cc @@ -25,6 +25,7 @@ #include "regexp-fields.h" #include "notmuch-private.h" #include "database-private.h" +#include "xapian-extra.h" notmuch_status_t compile_regex (regex_t ®exp, const char *str, std::string &msg) @@ -200,7 +201,7 @@ RegexpFieldProcessor::operator() (const std::string & str) if (str.empty ()) { if (options & NOTMUCH_FIELD_PROBABILISTIC) { return Xapian::Query (Xapian::Query::OP_AND_NOT, - Xapian::Query::MatchAll, + xapian_query_match_all (), Xapian::Query (Xapian::Query::OP_WILDCARD, term_prefix)); } else { return Xapian::Query (term_prefix); diff --git a/test/T800-asan.sh b/test/T800-asan.sh index 8607732e..5055c93e 100755 --- a/test/T800-asan.sh +++ b/test/T800-asan.sh @@ -9,7 +9,7 @@ fi add_email_corpus -TEST_CFLAGS="-fsanitize=address" +TEST_CFLAGS="${TEST_CFLAGS:-} -fsanitize=address" test_begin_subtest "open and destroy" test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} < /dev/null && pwd) + +test_description='run code with TSan enabled against the library' +# Note it is hard to ensure race conditions are deterministic so this +# only provides best effort detection. + +. "$test_directory"/test-lib.sh || exit 1 + +if [ $NOTMUCH_HAVE_TSAN -ne 1 ]; then + printf "Skipping due to missing TSan support\n" + test_done +fi + +export TSAN_OPTIONS="suppressions=$test_directory/T810-tsan.suppressions" +TEST_CFLAGS="${TEST_CFLAGS:-} -fsanitize=thread" + +cp -r ${MAIL_DIR} ${MAIL_DIR}-2 + +test_begin_subtest "create" +test_C ${MAIL_DIR} ${MAIL_DIR}-2 < +#include + +void *thread (void *arg) { + char *mail_dir = arg; + /* + * Calls into notmuch_query_search_messages which was using the thread-unsafe + * Xapian::Query::MatchAll. + */ + EXPECT0(notmuch_database_create (mail_dir, NULL)); + return NULL; +} + +int main (int argc, char **argv) { + pthread_t t1, t2; + EXPECT0(pthread_create (&t1, NULL, thread, argv[1])); + EXPECT0(pthread_create (&t2, NULL, thread, argv[2])); + EXPECT0(pthread_join (t1, NULL)); + EXPECT0(pthread_join (t2, NULL)); + return 0; +} +EOF +cat < EXPECTED +== stdout == +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT + +add_email_corpus +rm -r ${MAIL_DIR}-2 +cp -r ${MAIL_DIR} ${MAIL_DIR}-2 + +test_begin_subtest "query" +test_C ${MAIL_DIR} ${MAIL_DIR}-2 < +#include + +void *thread (void *arg) { + char *mail_dir = arg; + notmuch_database_t *db; + /* + * 'from' is NOTMUCH_FIELD_PROBABILISTIC | NOTMUCH_FIELD_PROCESSOR and an + * empty string gets us to RegexpFieldProcessor::operator which was using + * the tread-unsafe Xapian::Query::MatchAll. + */ + EXPECT0(notmuch_database_open_with_config (mail_dir, + NOTMUCH_DATABASE_MODE_READ_ONLY, + NULL, NULL, &db, NULL)); + notmuch_query_t *query = notmuch_query_create (db, "from:\"\""); + notmuch_messages_t *messages; + EXPECT0(notmuch_query_search_messages (query, &messages)); + return NULL; +} + +int main (int argc, char **argv) { + pthread_t t1, t2; + EXPECT0(pthread_create (&t1, NULL, thread, argv[1])); + EXPECT0(pthread_create (&t2, NULL, thread, argv[2])); + EXPECT0(pthread_join (t1, NULL)); + EXPECT0(pthread_join (t2, NULL)); + return 0; +} +EOF +cat < EXPECTED +== stdout == +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_done diff --git a/test/T810-tsan.suppressions b/test/T810-tsan.suppressions new file mode 100644 index 00000000..dbd16a94 --- /dev/null +++ b/test/T810-tsan.suppressions @@ -0,0 +1,5 @@ +# It's unclear how TSan-friendly GLib is: +# https://gitlab.gnome.org/GNOME/glib/-/issues/1672 +race:g_rw_lock_reader_lock +# https://gitlab.gnome.org/GNOME/glib/-/issues/1952 +race:g_slice_alloc0 diff --git a/util/xapian-extra.h b/util/xapian-extra.h new file mode 100644 index 00000000..39c7f48f --- /dev/null +++ b/util/xapian-extra.h @@ -0,0 +1,15 @@ +#ifndef _XAPIAN_EXTRA_H +#define _XAPIAN_EXTRA_H + +#include +#include + +inline Xapian::Query +xapian_query_match_all (void) +{ + // Xapian::Query::MatchAll isn't thread safe (a static object with reference + // counting) so instead reconstruct the equivalent on demand. + return Xapian::Query (std::string ()); +} + +#endif From ae1336dea5cc6a221699fff260d8b44cee677f77 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 22 Mar 2023 17:43:44 -0600 Subject: [PATCH 30/90] ruby: add tags helper Right now it doesn't do much, but it will help for further reorganization. Signed-off-by: Felipe Contreras --- bindings/ruby/database.c | 2 +- bindings/ruby/defs.h | 3 +++ bindings/ruby/message.c | 2 +- bindings/ruby/messages.c | 2 +- bindings/ruby/tags.c | 6 ++++++ bindings/ruby/thread.c | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bindings/ruby/database.c b/bindings/ruby/database.c index b6de1254..e6387f59 100644 --- a/bindings/ruby/database.c +++ b/bindings/ruby/database.c @@ -412,7 +412,7 @@ notmuch_rb_database_get_all_tags (VALUE self) rb_raise (notmuch_rb_eBaseError, "%s", msg); } - return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); + return notmuch_rb_tags_get (tags); } /* diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h index e2541e8f..9454658b 100644 --- a/bindings/ruby/defs.h +++ b/bindings/ruby/defs.h @@ -369,6 +369,9 @@ VALUE notmuch_rb_message_thaw (VALUE self); /* tags.c */ +VALUE +notmuch_rb_tags_get (notmuch_tags_t *tags); + VALUE notmuch_rb_tags_destroy (VALUE self); diff --git a/bindings/ruby/message.c b/bindings/ruby/message.c index f45c95cc..81085f75 100644 --- a/bindings/ruby/message.c +++ b/bindings/ruby/message.c @@ -221,7 +221,7 @@ notmuch_rb_message_get_tags (VALUE self) if (!tags) rb_raise (notmuch_rb_eMemoryError, "Out of memory"); - return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); + return notmuch_rb_tags_get (tags); } /* diff --git a/bindings/ruby/messages.c b/bindings/ruby/messages.c index ca5b10d0..6369d052 100644 --- a/bindings/ruby/messages.c +++ b/bindings/ruby/messages.c @@ -71,5 +71,5 @@ notmuch_rb_messages_collect_tags (VALUE self) if (!tags) rb_raise (notmuch_rb_eMemoryError, "Out of memory"); - return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); + return notmuch_rb_tags_get (tags); } diff --git a/bindings/ruby/tags.c b/bindings/ruby/tags.c index 2af85e36..cc6ea59e 100644 --- a/bindings/ruby/tags.c +++ b/bindings/ruby/tags.c @@ -20,6 +20,12 @@ #include "defs.h" +VALUE +notmuch_rb_tags_get (notmuch_tags_t *tags) +{ + return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); +} + /* * call-seq: TAGS.destroy! => nil * diff --git a/bindings/ruby/thread.c b/bindings/ruby/thread.c index 7cb2a3dc..b20ed893 100644 --- a/bindings/ruby/thread.c +++ b/bindings/ruby/thread.c @@ -204,5 +204,5 @@ notmuch_rb_thread_get_tags (VALUE self) if (!tags) rb_raise (notmuch_rb_eMemoryError, "Out of memory"); - return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); + return notmuch_rb_tags_get (tags); } From 3a52290609b07b51b16b527d1c14ac9836760300 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 22 Mar 2023 17:43:45 -0600 Subject: [PATCH 31/90] ruby: tags: return string array directly Signed-off-by: Felipe Contreras --- bindings/ruby/tags.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bindings/ruby/tags.c b/bindings/ruby/tags.c index cc6ea59e..955d790f 100644 --- a/bindings/ruby/tags.c +++ b/bindings/ruby/tags.c @@ -23,7 +23,13 @@ VALUE notmuch_rb_tags_get (notmuch_tags_t *tags) { - return Data_Wrap_Notmuch_Object (notmuch_rb_cTags, ¬much_rb_tags_type, tags); + VALUE rb_array = rb_ary_new (); + + for (; notmuch_tags_valid (tags); notmuch_tags_move_to_next (tags)) { + const char *tag = notmuch_tags_get (tags); + rb_ary_push (rb_array, rb_str_new2 (tag)); + } + return rb_array; } /* From 44924a6a09659c97d6cde383bf7deac0413d07c3 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 22 Mar 2023 17:43:46 -0600 Subject: [PATCH 32/90] ruby: remove Tags object Not used anymore now that we return an array of strings directly. Signed-off-by: Felipe Contreras --- bindings/ruby/defs.h | 7 ------ bindings/ruby/init.c | 14 ----------- bindings/ruby/tags.c | 55 -------------------------------------------- 3 files changed, 76 deletions(-) diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h index 9454658b..027408a1 100644 --- a/bindings/ruby/defs.h +++ b/bindings/ruby/defs.h @@ -33,7 +33,6 @@ extern VALUE notmuch_rb_cThreads; extern VALUE notmuch_rb_cThread; extern VALUE notmuch_rb_cMessages; extern VALUE notmuch_rb_cMessage; -extern VALUE notmuch_rb_cTags; extern VALUE notmuch_rb_eBaseError; extern VALUE notmuch_rb_eDatabaseError; @@ -372,12 +371,6 @@ notmuch_rb_message_thaw (VALUE self); VALUE notmuch_rb_tags_get (notmuch_tags_t *tags); -VALUE -notmuch_rb_tags_destroy (VALUE self); - -VALUE -notmuch_rb_tags_each (VALUE self); - /* init.c */ void Init_notmuch (void); diff --git a/bindings/ruby/init.c b/bindings/ruby/init.c index cd9f04cd..db6e7e5a 100644 --- a/bindings/ruby/init.c +++ b/bindings/ruby/init.c @@ -28,7 +28,6 @@ VALUE notmuch_rb_cThreads; VALUE notmuch_rb_cThread; VALUE notmuch_rb_cMessages; VALUE notmuch_rb_cMessage; -VALUE notmuch_rb_cTags; VALUE notmuch_rb_eBaseError; VALUE notmuch_rb_eDatabaseError; @@ -71,7 +70,6 @@ define_type (threads); define_type (thread); define_type (messages); define_type (message); -define_type (tags); /* * Document-module: Notmuch @@ -92,7 +90,6 @@ define_type (tags); * - Notmuch::Messages * - Notmuch::Thread * - Notmuch::Message - * - Notmuch::Tags */ void @@ -395,15 +392,4 @@ Init_notmuch (void) rb_define_method (notmuch_rb_cMessage, "tags_to_maildir_flags", notmuch_rb_message_tags_to_maildir_flags, 0); /* in message.c */ rb_define_method (notmuch_rb_cMessage, "freeze", notmuch_rb_message_freeze, 0); /* in message.c */ rb_define_method (notmuch_rb_cMessage, "thaw", notmuch_rb_message_thaw, 0); /* in message.c */ - - /* - * Document-class: Notmuch::Tags - * - * Notmuch tags - */ - notmuch_rb_cTags = rb_define_class_under (mod, "Tags", rb_cObject); - rb_undef_method (notmuch_rb_cTags, "initialize"); - rb_define_method (notmuch_rb_cTags, "destroy!", notmuch_rb_tags_destroy, 0); /* in tags.c */ - rb_define_method (notmuch_rb_cTags, "each", notmuch_rb_tags_each, 0); /* in tags.c */ - rb_include_module (notmuch_rb_cTags, rb_mEnumerable); } diff --git a/bindings/ruby/tags.c b/bindings/ruby/tags.c index 955d790f..b64874d1 100644 --- a/bindings/ruby/tags.c +++ b/bindings/ruby/tags.c @@ -1,23 +1,3 @@ -/* The Ruby interface to the notmuch mail library - * - * Copyright © 2010, 2011 Ali Polatel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see https://www.gnu.org/licenses/ . - * - * Author: Ali Polatel - */ - #include "defs.h" VALUE @@ -31,38 +11,3 @@ notmuch_rb_tags_get (notmuch_tags_t *tags) } return rb_array; } - -/* - * call-seq: TAGS.destroy! => nil - * - * Destroys the tags, freeing all resources allocated for it. - */ -VALUE -notmuch_rb_tags_destroy (VALUE self) -{ - notmuch_rb_object_destroy (self, ¬much_rb_tags_type); - - return Qnil; -} - -/* - * call-seq: TAGS.each {|item| block } => TAGS - * - * Calls +block+ once for each element in +self+, passing that element as a - * parameter. - */ -VALUE -notmuch_rb_tags_each (VALUE self) -{ - const char *tag; - notmuch_tags_t *tags; - - Data_Get_Notmuch_Tags (self, tags); - - for (; notmuch_tags_valid (tags); notmuch_tags_move_to_next (tags)) { - tag = notmuch_tags_get (tags); - rb_yield (rb_str_new2 (tag)); - } - - return self; -} From c6733a45c8ff698505ff330d2edce92c90cbc946 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 5 Jan 2023 20:02:04 -0400 Subject: [PATCH 33/90] lib: add config key INDEX_AS_TEXT Higher level processing as a list of regular expressions and documentation will follow. --- lib/config.cc | 3 +++ lib/notmuch.h | 1 + test/T030-config.sh | 1 + test/T055-path-config.sh | 1 + test/T590-libconfig.sh | 5 +++++ 5 files changed, 11 insertions(+) diff --git a/lib/config.cc b/lib/config.cc index 503a0c8b..2323860d 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -599,6 +599,8 @@ _notmuch_config_key_to_string (notmuch_config_key_t key) return "database.autocommit"; case NOTMUCH_CONFIG_EXTRA_HEADERS: return "show.extra_headers"; + case NOTMUCH_CONFIG_INDEX_AS_TEXT: + return "index.as_text"; default: return NULL; } @@ -642,6 +644,7 @@ _notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key) else email = _get_email_from_passwd_file (notmuch); return email; + case NOTMUCH_CONFIG_INDEX_AS_TEXT: case NOTMUCH_CONFIG_NEW_IGNORE: return ""; case NOTMUCH_CONFIG_AUTOCOMMIT: diff --git a/lib/notmuch.h b/lib/notmuch.h index 89018392..76156178 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -2563,6 +2563,7 @@ typedef enum { NOTMUCH_CONFIG_USER_NAME, NOTMUCH_CONFIG_AUTOCOMMIT, NOTMUCH_CONFIG_EXTRA_HEADERS, + NOTMUCH_CONFIG_INDEX_AS_TEXT, NOTMUCH_CONFIG_LAST } notmuch_config_key_t; diff --git a/test/T030-config.sh b/test/T030-config.sh index 43bbce31..ea0b4012 100755 --- a/test/T030-config.sh +++ b/test/T030-config.sh @@ -57,6 +57,7 @@ database.mail_root=MAIL_DIR database.path=MAIL_DIR foo.list=this;is another;list value; foo.string=this is another string value +index.as_text= maildir.synchronize_flags=true new.ignore= new.tags=unread;inbox diff --git a/test/T055-path-config.sh b/test/T055-path-config.sh index fe295324..efc79e8b 100755 --- a/test/T055-path-config.sh +++ b/test/T055-path-config.sh @@ -299,6 +299,7 @@ database.backup_dir database.hook_dir database.mail_root=MAIL_DIR database.path +index.as_text= maildir.synchronize_flags=true new.ignore= new.tags=unread;inbox diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 26a1f033..9326ba3e 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -440,6 +440,7 @@ cat <<'EOF' >EXPECTED 10: 'USER_FULL_NAME' 11: '8000' 12: 'NULL' +13: '' == stderr == EOF unset MAILDIR @@ -725,6 +726,7 @@ test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "list by keys (ndlc)" notmuch config set search.exclude_tags "foo;bar;fub" notmuch config set new.ignore "sekrit_junk" +notmuch config set index.as_text "text/" cat c_head2 - c_tail <<'EOF' | test_C ${MAIL_DIR} %NULL% %NULL% { notmuch_config_key_t key; @@ -751,6 +753,7 @@ cat <<'EOF' >EXPECTED 10: 'Notmuch Test Suite' 11: '8000' 12: 'NULL' +13: 'text/' == stderr == EOF test_expect_equal_file EXPECTED OUTPUT @@ -785,6 +788,7 @@ cat <<'EOF' >EXPECTED 10: 'USER_FULL_NAME' 11: '8000' 12: 'NULL' +13: '' == stderr == EOF test_expect_equal_file EXPECTED OUTPUT.clean @@ -856,6 +860,7 @@ database.backup_dir MAIL_DIR/.notmuch/backups database.hook_dir MAIL_DIR/.notmuch/hooks database.mail_root MAIL_DIR database.path MAIL_DIR +index.as_text text/ key with spaces value, with, spaces! maildir.synchronize_flags true new.ignore sekrit_junk From 3f5809bf28becbddfed9ff33d6f1242346904c23 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 5 Jan 2023 20:02:05 -0400 Subject: [PATCH 34/90] lib: parse index.as_text We pre-parse into a list of compiled regular expressions to avoid calling regexc on the hot (indexing) path. As explained in the code comment, this cannot be done lazily with reasonable error reporting, at least not without touching a lot of the code in index.cc. --- lib/database-private.h | 4 ++++ lib/open.cc | 53 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/lib/database-private.h b/lib/database-private.h index b9be4e22..61232f1a 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -291,6 +291,10 @@ struct _notmuch_database { /* Track what parameters were specified when opening */ notmuch_open_param_t params; + + /* list of regular expressions to check for text indexing */ + regex_t *index_as_text; + size_t index_as_text_length; }; /* Prior to database version 3, features were implied by the database diff --git a/lib/open.cc b/lib/open.cc index 67ff868c..54d1faf3 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -320,6 +320,8 @@ _alloc_notmuch (const char *database_path, const char *config_path, const char * notmuch->transaction_count = 0; notmuch->transaction_threshold = 0; notmuch->view = 1; + notmuch->index_as_text = NULL; + notmuch->index_as_text_length = 0; notmuch->params = NOTMUCH_PARAM_NONE; if (database_path) @@ -427,6 +429,53 @@ _load_database_state (notmuch_database_t *notmuch) notmuch, notmuch->xapian_db->get_uuid ().c_str ()); } +/* XXX This should really be done lazily, but the error reporting path in the indexing code + * would need to be redone to report any errors. + */ +notmuch_status_t +_ensure_index_as_text (notmuch_database_t *notmuch, char **message) +{ + int nregex = 0; + regex_t *regexv = NULL; + + if (notmuch->index_as_text) + return NOTMUCH_STATUS_SUCCESS; + + for (notmuch_config_values_t *list = notmuch_config_get_values (notmuch, + NOTMUCH_CONFIG_INDEX_AS_TEXT); + notmuch_config_values_valid (list); + notmuch_config_values_move_to_next (list)) { + regex_t *new_regex; + int rerr; + const char *str = notmuch_config_values_get (list); + size_t len = strlen (str); + + /* str must be non-empty, because n_c_get_values skips empty + * strings */ + assert (len > 0); + + regexv = talloc_realloc (notmuch, regexv, regex_t, nregex + 1); + new_regex = ®exv[nregex]; + + rerr = regcomp (new_regex, str, REG_EXTENDED | REG_NOSUB); + if (rerr) { + size_t error_size = regerror (rerr, new_regex, NULL, 0); + char *error = (char *) talloc_size (str, error_size); + + regerror (rerr, new_regex, error, error_size); + IGNORE_RESULT (asprintf (message, "Error in index.as_text: %s: %s\n", error, str)); + + return NOTMUCH_STATUS_ILLEGAL_ARGUMENT; + } + nregex++; + } + + notmuch->index_as_text = regexv; + notmuch->index_as_text_length = nregex; + + return NOTMUCH_STATUS_SUCCESS; +} + static notmuch_status_t _finish_open (notmuch_database_t *notmuch, const char *profile, @@ -531,6 +580,10 @@ _finish_open (notmuch_database_t *notmuch, if (status) goto DONE; + status = _ensure_index_as_text (notmuch, &message); + if (status) + goto DONE; + autocommit_str = notmuch_config_get (notmuch, NOTMUCH_CONFIG_AUTOCOMMIT); if (unlikely (! autocommit_str)) { INTERNAL_ERROR ("missing configuration for autocommit"); From a554690d6af0ac8cb55166a20efd0f449abde389 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 5 Jan 2023 20:02:06 -0400 Subject: [PATCH 35/90] lib: index attachments with mime types matching index.as_text Instead of skipping indexing all attachments, we check of a (user configured) mime type that is indexable as text. --- doc/man1/notmuch-config.rst | 10 ++++ lib/database.cc | 12 +++++ lib/index.cc | 25 ++++++++-- lib/notmuch-private.h | 4 ++ test/T050-new.sh | 8 ---- test/T760-as-text.sh | 77 ++++++++++++++++++++++++++++++ test/corpora/indexing/fake-pdf:2,S | 11 +++++ 7 files changed, 136 insertions(+), 11 deletions(-) create mode 100755 test/T760-as-text.sh create mode 100644 test/corpora/indexing/fake-pdf:2,S diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 388315f6..988dc5a3 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -122,6 +122,16 @@ paths are presumed relative to `$HOME` for items in section Default tag prefix (filter) for :any:`notmuch-git`. +.. nmconfig:: index.as_text + + List of regular expressions (without delimiters) for MIME types to + be indexed as text. Currently this applies only to attachments. By + default the regex matches anywhere in the content type; if they + user wants an anchored match, they should include anchors in their + regexes. + + History: This configuration value was introduced in notmuch 0.38. + .. nmconfig:: index.decrypt Policy for decrypting encrypted messages during indexing. Must be diff --git a/lib/database.cc b/lib/database.cc index d1e5f1af..6987e2f4 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -1573,3 +1573,15 @@ notmuch_database_status_string (const notmuch_database_t *notmuch) { return notmuch->status_string; } + +bool +_notmuch_database_indexable_as_text (notmuch_database_t *notmuch, const char *mime_string) +{ + for (size_t i = 0; i < notmuch->index_as_text_length; i++) { + if (regexec (¬much->index_as_text[i], mime_string, 0, NULL, 0) == 0) { + return true; + } + } + + return false; +} diff --git a/lib/index.cc b/lib/index.cc index 728bfb22..629dcb22 100644 --- a/lib/index.cc +++ b/lib/index.cc @@ -380,6 +380,23 @@ _index_pkcs7_part (notmuch_message_t *message, GMimeObject *part, _notmuch_message_crypto_t *msg_crypto); +static bool +_indexable_as_text (notmuch_message_t *message, GMimeObject *part) +{ + GMimeContentType *content_type = g_mime_object_get_content_type (part); + notmuch_database_t *notmuch = notmuch_message_get_database (message); + + if (content_type) { + char *mime_string = g_mime_content_type_get_mime_type (content_type); + if (mime_string) { + bool ret = _notmuch_database_indexable_as_text (notmuch, mime_string); + g_free (mime_string); + return ret; + } + } + return false; +} + /* Callback to generate terms for each mime part of a message. */ static void _index_mime_part (notmuch_message_t *message, @@ -497,9 +514,11 @@ _index_mime_part (notmuch_message_t *message, _notmuch_message_add_term (message, "tag", "attachment"); _notmuch_message_gen_terms (message, "attachment", filename); - /* XXX: Would be nice to call out to something here to parse - * the attachment into text and then index that. */ - goto DONE; + if (! _indexable_as_text (message, part)) { + /* XXX: Would be nice to call out to something here to parse + * the attachment into text and then index that. */ + goto DONE; + } } byte_array = g_byte_array_new (); diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index 1d3d2b0c..c19ee8e2 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -259,6 +259,10 @@ _notmuch_database_filename_to_direntry (void *ctx, notmuch_find_flags_t flags, char **direntry); +bool +_notmuch_database_indexable_as_text (notmuch_database_t *notmuch, + const char *mime_string); + /* directory.cc */ notmuch_directory_t * diff --git a/test/T050-new.sh b/test/T050-new.sh index 09c2bfc6..52888be2 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -470,12 +470,4 @@ Date: Fri, 17 Jun 2016 22:14:41 -0400 EOF test_expect_equal_file EXPECTED OUTPUT -add_email_corpus indexing - -test_begin_subtest "index text/* attachments" -test_subtest_known_broken -notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain > EXPECTED -notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz > OUTPUT -test_expect_equal_file_nonempty EXPECTED OUTPUT - test_done diff --git a/test/T760-as-text.sh b/test/T760-as-text.sh new file mode 100755 index 00000000..744567f2 --- /dev/null +++ b/test/T760-as-text.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +test_description='index attachments as text' +. $(dirname "$0")/test-lib.sh || exit 1 + +add_email_corpus indexing +test_begin_subtest "empty as_text; skip text/x-diff" +messages=$(notmuch count id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain) +count=$(notmuch count id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz) +test_expect_equal "$messages,$count" "1,0" + +notmuch config set index.as_text "^text/" +add_email_corpus indexing + +test_begin_subtest "as_index is text/; find text/x-diff" +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain > EXPECTED +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "reindex with empty as_text, skips text/x-diff" +notmuch config set index.as_text +notmuch reindex '*' +messages=$(notmuch count id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain) +count=$(notmuch count id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz) +test_expect_equal "$messages,$count" "1,0" + +test_begin_subtest "reindex with empty as_text; skips application/pdf" +notmuch config set index.as_text +notmuch reindex '*' +gmessages=$(notmuch count id:871qo9p4tf.fsf@tethera.net) +count=$(notmuch count id:871qo9p4tf.fsf@tethera.net and body:not-really-PDF) +test_expect_equal "$messages,$count" "1,0" + +test_begin_subtest "reindex with as_text as text/; finds text/x-diff" +notmuch config set index.as_text "^text/" +notmuch reindex '*' +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain > EXPECTED +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "reindex with as_text as text/; skips application/pdf" +notmuch config set index.as_text "^text/" +notmuch config set index.as_text +notmuch reindex '*' +messages=$(notmuch count id:871qo9p4tf.fsf@tethera.net) +count=$(notmuch count id:871qo9p4tf.fsf@tethera.net and body:not-really-PDF) +test_expect_equal "$messages,$count" "1,0" + +test_begin_subtest "as_text has multiple regexes" +notmuch config set index.as_text "blahblah;^text/" +notmuch reindex '*' +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain > EXPECTED +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "as_text is non-anchored regex" +notmuch config set index.as_text "e.t/" +notmuch reindex '*' +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain > EXPECTED +notmuch search id:20200930101213.2m2pt3jrspvcrxfx@localhost.localdomain and ersatz > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "as_text is 'application/pdf'" +notmuch config set index.as_text "^application/pdf$" +notmuch reindex '*' +notmuch search id:871qo9p4tf.fsf@tethera.net > EXPECTED +notmuch search id:871qo9p4tf.fsf@tethera.net and '"not really PDF"' > OUTPUT +test_expect_equal_file_nonempty EXPECTED OUTPUT + +test_begin_subtest "as_text is bad regex" +notmuch config set index.as_text '[' +notmuch reindex '*' >& OUTPUT +cat< EXPECTED +Error in index.as_text: Invalid regular expression: [ +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_done diff --git a/test/corpora/indexing/fake-pdf:2,S b/test/corpora/indexing/fake-pdf:2,S new file mode 100644 index 00000000..60a7a47f --- /dev/null +++ b/test/corpora/indexing/fake-pdf:2,S @@ -0,0 +1,11 @@ +From: David Bremner +To: example@example.com +Subject: attachment content type +Date: Thu, 05 Jan 2023 08:02:36 -0400 +Message-ID: <871qo9p4tf.fsf@tethera.net> +MIME-Version: 1.0 +Content-Type: application/pdf +Content-Disposition: attachment; filename=fake.pdf +Content-Transfer-Encoding: base64 + +dGhpcyBpcyBub3QgcmVhbGx5IFBERgo= \ No newline at end of file From 777b02a7d7b922bcae08af1c16e475051ec7d8f3 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 27 Mar 2023 15:59:40 -0600 Subject: [PATCH 36/90] ruby: add filenames helper Right now it doesn't do much, but it will help for further reorganization. Signed-off-by: Felipe Contreras --- bindings/ruby/defs.h | 3 +++ bindings/ruby/directory.c | 4 ++-- bindings/ruby/filenames.c | 6 ++++++ bindings/ruby/message.c | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h index 027408a1..97aea25d 100644 --- a/bindings/ruby/defs.h +++ b/bindings/ruby/defs.h @@ -224,6 +224,9 @@ VALUE notmuch_rb_directory_get_child_directories (VALUE self); /* filenames.c */ +VALUE +notmuch_rb_filenames_get (notmuch_filenames_t *fnames); + VALUE notmuch_rb_filenames_destroy (VALUE self); diff --git a/bindings/ruby/directory.c b/bindings/ruby/directory.c index 910f0a99..f267d82f 100644 --- a/bindings/ruby/directory.c +++ b/bindings/ruby/directory.c @@ -87,7 +87,7 @@ notmuch_rb_directory_get_child_files (VALUE self) fnames = notmuch_directory_get_child_files (dir); - return Data_Wrap_Notmuch_Object (notmuch_rb_cFileNames, ¬much_rb_filenames_type, fnames); + return notmuch_rb_filenames_get (fnames); } /* @@ -106,5 +106,5 @@ notmuch_rb_directory_get_child_directories (VALUE self) fnames = notmuch_directory_get_child_directories (dir); - return Data_Wrap_Notmuch_Object (notmuch_rb_cFileNames, ¬much_rb_filenames_type, fnames); + return notmuch_rb_filenames_get (fnames); } diff --git a/bindings/ruby/filenames.c b/bindings/ruby/filenames.c index 0dec1952..17873393 100644 --- a/bindings/ruby/filenames.c +++ b/bindings/ruby/filenames.c @@ -20,6 +20,12 @@ #include "defs.h" +VALUE +notmuch_rb_filenames_get (notmuch_filenames_t *fnames) +{ + return Data_Wrap_Notmuch_Object (notmuch_rb_cFileNames, ¬much_rb_filenames_type, fnames); +} + /* * call-seq: FILENAMES.destroy! => nil * diff --git a/bindings/ruby/message.c b/bindings/ruby/message.c index 81085f75..13c182f6 100644 --- a/bindings/ruby/message.c +++ b/bindings/ruby/message.c @@ -120,7 +120,7 @@ notmuch_rb_message_get_filenames (VALUE self) fnames = notmuch_message_get_filenames (message); - return Data_Wrap_Notmuch_Object (notmuch_rb_cFileNames, ¬much_rb_filenames_type, fnames); + return notmuch_rb_filenames_get (fnames); } /* From 837426d7be63cf11d51b74464f174724935c5d4f Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 27 Mar 2023 15:59:41 -0600 Subject: [PATCH 37/90] ruby: filenames: return string array directly Signed-off-by: Felipe Contreras --- bindings/ruby/filenames.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bindings/ruby/filenames.c b/bindings/ruby/filenames.c index 17873393..20541402 100644 --- a/bindings/ruby/filenames.c +++ b/bindings/ruby/filenames.c @@ -23,7 +23,11 @@ VALUE notmuch_rb_filenames_get (notmuch_filenames_t *fnames) { - return Data_Wrap_Notmuch_Object (notmuch_rb_cFileNames, ¬much_rb_filenames_type, fnames); + VALUE rb_array = rb_ary_new (); + + for (; notmuch_filenames_valid (fnames); notmuch_filenames_move_to_next (fnames)) + rb_ary_push (rb_array, rb_str_new2 (notmuch_filenames_get (fnames))); + return rb_array; } /* From e4d75fcc8349ae95ec22d0e6679880d23bed37f8 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 27 Mar 2023 15:59:42 -0600 Subject: [PATCH 38/90] ruby: remove FileNames object Not used anymore now that we return an array of strings directly. Signed-off-by: Felipe Contreras --- bindings/ruby/defs.h | 10 -------- bindings/ruby/filenames.c | 52 --------------------------------------- bindings/ruby/init.c | 14 ----------- 3 files changed, 76 deletions(-) diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h index 97aea25d..f2a7e484 100644 --- a/bindings/ruby/defs.h +++ b/bindings/ruby/defs.h @@ -58,7 +58,6 @@ extern ID ID_db_mode; extern const rb_data_type_t notmuch_rb_object_type; extern const rb_data_type_t notmuch_rb_database_type; extern const rb_data_type_t notmuch_rb_directory_type; -extern const rb_data_type_t notmuch_rb_filenames_type; extern const rb_data_type_t notmuch_rb_query_type; extern const rb_data_type_t notmuch_rb_threads_type; extern const rb_data_type_t notmuch_rb_thread_type; @@ -91,9 +90,6 @@ extern const rb_data_type_t notmuch_rb_tags_type; #define Data_Get_Notmuch_Directory(obj, ptr) \ Data_Get_Notmuch_Object ((obj), ¬much_rb_directory_type, (ptr)) -#define Data_Get_Notmuch_FileNames(obj, ptr) \ - Data_Get_Notmuch_Object ((obj), ¬much_rb_filenames_type, (ptr)) - #define Data_Get_Notmuch_Query(obj, ptr) \ Data_Get_Notmuch_Object ((obj), ¬much_rb_query_type, (ptr)) @@ -227,12 +223,6 @@ notmuch_rb_directory_get_child_directories (VALUE self); VALUE notmuch_rb_filenames_get (notmuch_filenames_t *fnames); -VALUE -notmuch_rb_filenames_destroy (VALUE self); - -VALUE -notmuch_rb_filenames_each (VALUE self); - /* query.c */ VALUE notmuch_rb_query_destroy (VALUE self); diff --git a/bindings/ruby/filenames.c b/bindings/ruby/filenames.c index 20541402..60c3fb8b 100644 --- a/bindings/ruby/filenames.c +++ b/bindings/ruby/filenames.c @@ -1,23 +1,3 @@ -/* The Ruby interface to the notmuch mail library - * - * Copyright © 2010 Ali Polatel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see https://www.gnu.org/licenses/ . - * - * Author: Ali Polatel - */ - #include "defs.h" VALUE @@ -29,35 +9,3 @@ notmuch_rb_filenames_get (notmuch_filenames_t *fnames) rb_ary_push (rb_array, rb_str_new2 (notmuch_filenames_get (fnames))); return rb_array; } - -/* - * call-seq: FILENAMES.destroy! => nil - * - * Destroys the filenames, freeing all resources allocated for it. - */ -VALUE -notmuch_rb_filenames_destroy (VALUE self) -{ - notmuch_rb_object_destroy (self, ¬much_rb_filenames_type); - - return Qnil; -} - -/* - * call-seq: FILENAMES.each {|item| block } => FILENAMES - * - * Calls +block+ once for each element in +self+, passing that element as a - * parameter. - */ -VALUE -notmuch_rb_filenames_each (VALUE self) -{ - notmuch_filenames_t *fnames; - - Data_Get_Notmuch_FileNames (self, fnames); - - for (; notmuch_filenames_valid (fnames); notmuch_filenames_move_to_next (fnames)) - rb_yield (rb_str_new2 (notmuch_filenames_get (fnames))); - - return self; -} diff --git a/bindings/ruby/init.c b/bindings/ruby/init.c index db6e7e5a..e0c9936a 100644 --- a/bindings/ruby/init.c +++ b/bindings/ruby/init.c @@ -22,7 +22,6 @@ VALUE notmuch_rb_cDatabase; VALUE notmuch_rb_cDirectory; -VALUE notmuch_rb_cFileNames; VALUE notmuch_rb_cQuery; VALUE notmuch_rb_cThreads; VALUE notmuch_rb_cThread; @@ -64,7 +63,6 @@ const rb_data_type_t notmuch_rb_object_type = { define_type (database); define_type (directory); -define_type (filenames); define_type (query); define_type (threads); define_type (thread); @@ -84,7 +82,6 @@ define_type (message); * the user: * * - Notmuch::Database - * - Notmuch::FileNames * - Notmuch::Query * - Notmuch::Threads * - Notmuch::Messages @@ -294,17 +291,6 @@ Init_notmuch (void) rb_define_method (notmuch_rb_cDirectory, "child_files", notmuch_rb_directory_get_child_files, 0); /* in directory.c */ rb_define_method (notmuch_rb_cDirectory, "child_directories", notmuch_rb_directory_get_child_directories, 0); /* in directory.c */ - /* - * Document-class: Notmuch::FileNames - * - * Notmuch file names - */ - notmuch_rb_cFileNames = rb_define_class_under (mod, "FileNames", rb_cObject); - rb_undef_method (notmuch_rb_cFileNames, "initialize"); - rb_define_method (notmuch_rb_cFileNames, "destroy!", notmuch_rb_filenames_destroy, 0); /* in filenames.c */ - rb_define_method (notmuch_rb_cFileNames, "each", notmuch_rb_filenames_each, 0); /* in filenames.c */ - rb_include_module (notmuch_rb_cFileNames, rb_mEnumerable); - /* * Document-class: Notmuch::Query * From 1b878877d928a5279c3c0003d067d8b875f7e42a Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Thu, 13 Apr 2023 15:52:11 +0200 Subject: [PATCH 39/90] doc: fix typos --- doc/man1/notmuch-config.rst | 2 +- doc/man1/notmuch-git.rst | 4 ++-- doc/man7/notmuch-sexp-queries.rst | 8 ++++---- doc/notmuch-emacs.rst | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 988dc5a3..bd34afa4 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -126,7 +126,7 @@ paths are presumed relative to `$HOME` for items in section List of regular expressions (without delimiters) for MIME types to be indexed as text. Currently this applies only to attachments. By - default the regex matches anywhere in the content type; if they + default the regex matches anywhere in the content type; if the user wants an anchored match, they should include anchors in their regexes. diff --git a/doc/man1/notmuch-git.rst b/doc/man1/notmuch-git.rst index ac1908b6..33a46f84 100644 --- a/doc/man1/notmuch-git.rst +++ b/doc/man1/notmuch-git.rst @@ -325,11 +325,11 @@ If it is unset, 'default' is assumed. .. envvar:: NOTMUCH_GIT_DIR - Default location of git repository. Overriden by :option:`--git-dir`. + Default location of git repository. Overridden by :option:`--git-dir`. .. envvar:: NOTMUCH_GIT_PREFIX - Default tag prefix (filter). Overriden by :option:`--tag-prefix`. + Default tag prefix (filter). Overridden by :option:`--tag-prefix`. SEE ALSO ======== diff --git a/doc/man7/notmuch-sexp-queries.rst b/doc/man7/notmuch-sexp-queries.rst index 422154c7..858ff685 100644 --- a/doc/man7/notmuch-sexp-queries.rst +++ b/doc/man7/notmuch-sexp-queries.rst @@ -274,7 +274,7 @@ EXAMPLES Matches any word starting with "prelim", inside a message subject. -``(subject (starts-wih quick) "brown fox")`` +``(subject (starts-with quick) "brown fox")`` Match messages whose subject contains "quick brown fox", but also "brown fox quicksand". @@ -336,7 +336,7 @@ user defined fields is permitted within a macro. NOTES ===== -.. [#macro-details] Technically macros impliment lazy evaluation and +.. [#macro-details] Technically macros implement lazy evaluation and lexical scope. There is one top level scope containing all macro definitions, but all parameter definitions are local to a given macro. @@ -347,10 +347,10 @@ NOTES .. [#aka-bool] a.k.a. boolean prefixes -.. [#not-phrase] Due to the implemention of phrase fields in Xapian, +.. [#not-phrase] Due to the implementation of phrase fields in Xapian, regex queries could only match individual words. -.. [#not-body] Due the the way ``body`` is implemented in notmuch, +.. [#not-body] Due to the way ``body`` is implemented in notmuch, this modifier is not supported in the ``body`` field. .. [#not-path] Due to the way recursive ``path`` queries are implemented diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index 71f10e20..7dff7d64 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -14,7 +14,7 @@ manual to refer to the Emacs interface to Notmuch. When this distinction is important, we’ll refer to the Emacs interface as *notmuch-emacs*. -Notmuch-emacs is highly customizable via the the Emacs customization +Notmuch-emacs is highly customizable via the Emacs customization framework (or just by setting the appropriate variables). We try to point out relevant variables in this manual, but in order to avoid duplication of information, you can usually find the most detailed @@ -493,7 +493,7 @@ in :ref:`notmuch-search`. Dealing with duplicates ----------------------- -If there are are multiple files with the same :mailheader:`Message-ID` +If there are multiple files with the same :mailheader:`Message-ID` (see :any:`duplicate-files`), then :any:`notmuch-show` displays the number of duplicates and identifies the current duplicate. In the following example duplicate 3 of 5 is displayed. @@ -717,7 +717,7 @@ operations specified in ``notmuch-tagging-keys``; i.e. each notmuch-tag-undo ---------------- -Each notmuch buffer supporting tagging operations (i.e buffers in +Each notmuch buffer supporting tagging operations (i.e. buffers in :any:`notmuch-show`, :any:`notmuch-search`, :any:`notmuch-tree`, and :any:`notmuch-unthreaded` mode) keeps a local stack of tagging operations. These can be undone via :any:`notmuch-tag-undo`. By default From 3f4c61592f5a8667a11e5b90369572f60e58b388 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Wed, 3 May 2023 16:12:19 -0600 Subject: [PATCH 40/90] vim: doc: small fixes Signed-off-by: Felipe Contreras --- vim/README | 4 ++-- vim/notmuch.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vim/README b/vim/README index c137bacd..777c20c0 100644 --- a/vim/README +++ b/vim/README @@ -53,10 +53,10 @@ Enjoy ;) As an example to configure a key mapping to add the tag 'to-do' and archive, this is what I use: -let g:notmuch_rb_custom_search_maps = { +let g:notmuch_custom_search_maps = { \ 't': 'search_tag("+to-do -inbox")', \ } -let g:notmuch_rb_custom_show_maps = { +let g:notmuch_custom_show_maps = { \ 't': 'show_tag("+to-do -inbox")', \ } diff --git a/vim/notmuch.txt b/vim/notmuch.txt index 43741022..4a038102 100644 --- a/vim/notmuch.txt +++ b/vim/notmuch.txt @@ -89,7 +89,7 @@ s Send CONFIGURATION *notmuch-config* You can add the following configurations to your `.vimrc`, or -`~/.vim/plugin/notmuch.vim`. +`~/.vim/after/plugin/notmuch.vim`. *g:notmuch_folders* @@ -138,7 +138,7 @@ You can do the same for the thread view: If you want to count the threads instead of the messages in the folder view: > - let g:notmuch_folders_count_threads = 0 + let g:notmuch_folders_count_threads = 1 < *g:notmuch_reader* From 74c2c86769a1e195fe7f465213c2dc360519d43f Mon Sep 17 00:00:00 2001 From: John Gliksberg Date: Wed, 3 May 2023 16:12:20 -0600 Subject: [PATCH 41/90] vim: doc: minor spelling fix Signed-off-by: Felipe Contreras --- vim/notmuch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vim/notmuch.txt b/vim/notmuch.txt index 4a038102..c98f2b53 100644 --- a/vim/notmuch.txt +++ b/vim/notmuch.txt @@ -144,7 +144,7 @@ If you want to count the threads instead of the messages in the folder view: *g:notmuch_reader* *g:notmuch_sendmail* -You can also configure your externail mail reader and sendemail program: +You can also configure your external mail reader and sendmail program: > let g:notmuch_reader = 'mutt -f %s' let g:notmuch_sendmail = 'sendmail' From afa45bd6b886838b760884a1870f274fc33b5f89 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 3 Apr 2023 14:27:21 -0600 Subject: [PATCH 42/90] ruby: query: fix get sort The order was wrong, right now `query.sort` doesn't return a number. Signed-off-by: Felipe Contreras --- bindings/ruby/query.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/ruby/query.c b/bindings/ruby/query.c index 8a2b4d3d..077def02 100644 --- a/bindings/ruby/query.c +++ b/bindings/ruby/query.c @@ -45,7 +45,7 @@ notmuch_rb_query_get_sort (VALUE self) Data_Get_Notmuch_Query (self, query); - return FIX2INT (notmuch_query_get_sort (query)); + return INT2FIX (notmuch_query_get_sort (query)); } /* From d155f29eca0ab9afc35292e447fb67b1c73e9df9 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Wed, 24 May 2023 06:30:21 +0800 Subject: [PATCH 43/90] notmuch-mutt: convert ISO-8859-15 copyright statement to UTF-8 Suggested-by: isutf8 & check-all-the-things --- contrib/notmuch-mutt/notmuch-mutt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index d1e2c084..1e12038c 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -2,7 +2,7 @@ # # notmuch-mutt - notmuch (of a) helper for Mutt # -# Copyright: © 2011-2015 Stefano Zacchiroli +# Copyright: © 2011-2015 Stefano Zacchiroli # License: GNU General Public License (GPL), version 3 or above # # See the bottom of this file for more documentation. From 6d383d404982c6e12dd68dcdf94b3490e3de4645 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sun, 9 Apr 2023 12:41:42 +0800 Subject: [PATCH 44/90] notmuch-mutt: fix Xapian query construction Spaces need to be stripped when querying the Message-Id, because notmuch stores them in Xapian with spaces stripped. All double-quote characters need to be doubled to escape them, otherwise they will be added as extra query terms outside the id. --- contrib/notmuch-mutt/notmuch-mutt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index 1e12038c..b38258f5 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -124,7 +124,11 @@ sub thread_action($$@) { empty_maildir($results_dir); die "notmuch-mutt: cannot find Message-Id, abort.\n"; } - my $search_cmd = 'notmuch search --output=threads ' . shell_quote("id:$mid"); + + $mid =~ s/ //g; # notmuch strips spaces before storing Message-Id + $mid =~ s/"/""/g; # escape all double quote characters + + my $search_cmd = 'notmuch search --output=threads ' . shell_quote(qq{id:"$mid"}); my $tid = `$search_cmd`; # get thread id chomp($tid); @@ -135,7 +139,10 @@ sub tag_action(@) { my $mid = get_message_id(); defined $mid or die "notmuch-mutt: cannot find Message-Id, abort.\n"; - system("notmuch", "tag", @_, "--", "id:$mid"); + $mid =~ s/ //g; # notmuch strips spaces before storing Message-Id + $mid =~ s/"/""/g; # escape all double quote characters + + system("notmuch", "tag", @_, "--", qq{id:"$mid"}); } sub die_usage() { From 93d936c5ae2c694d7fcc310503a182b6bbd603ee Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sun, 9 Apr 2023 12:41:43 +0800 Subject: [PATCH 45/90] notmuch-mutt: replace extra command with notmuch thread search feature This should be be slightly faster since it avoids forking a shell and is less code in and less dependencies for the script. Since String::ShellQuote isn't used elsewhere, drop mention of it. --- contrib/notmuch-mutt/README | 2 -- contrib/notmuch-mutt/notmuch-mutt | 9 ++------- debian/control | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/contrib/notmuch-mutt/README b/contrib/notmuch-mutt/README index 26996c4a..c7520228 100644 --- a/contrib/notmuch-mutt/README +++ b/contrib/notmuch-mutt/README @@ -39,8 +39,6 @@ To *run* notmuch-mutt you will need Perl with the following libraries: (Debian package: libmail-box-perl) - Mail::Header (Debian package: libmailtools-perl) -- String::ShellQuote - (Debian package: libstring-shellquote-perl) - Term::ReadLine::Gnu (Debian package: libterm-readline-gnu-perl) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index b38258f5..01db6908 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -17,7 +17,6 @@ use Getopt::Long qw(:config no_getopt_compat); use Mail::Header; use Mail::Box::Maildir; use Pod::Usage; -use String::ShellQuote; use Term::ReadLine; use Digest::SHA; @@ -126,13 +125,9 @@ sub thread_action($$@) { } $mid =~ s/ //g; # notmuch strips spaces before storing Message-Id - $mid =~ s/"/""/g; # escape all double quote characters + $mid =~ s/"/""""/g; # escape all double quote characters twice - my $search_cmd = 'notmuch search --output=threads ' . shell_quote(qq{id:"$mid"}); - my $tid = `$search_cmd`; # get thread id - chomp($tid); - - search($results_dir, $remove_dups, $tid); + search($results_dir, $remove_dups, qq{thread:"{id:""$mid""}"}); } sub tag_action(@) { diff --git a/debian/control b/debian/control index 2dcb8cc7..135eb7ce 100644 --- a/debian/control +++ b/debian/control @@ -227,7 +227,6 @@ Architecture: all Depends: libmail-box-perl, libmailtools-perl, - libstring-shellquote-perl, libterm-readline-gnu-perl, notmuch (>= 0.4), ${misc:Depends}, From 85916d8a9636645e8b15061e59df2e28691aa012 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sun, 28 May 2023 10:29:44 +0800 Subject: [PATCH 46/90] notmuch-mutt: clarify the empty Maildir function operates on search caches Rename the function and adjust the documentation comment. --- contrib/notmuch-mutt/notmuch-mutt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index 01db6908..0f7379ac 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -26,8 +26,8 @@ $xdg_cache_dir = $ENV{XDG_CACHE_HOME} if $ENV{XDG_CACHE_HOME}; my $cache_dir = "$xdg_cache_dir/notmuch/mutt"; -# create an empty maildir (if missing) or empty an existing maildir" -sub empty_maildir($) { +# create an empty search cache maildir (if missing) or empty existing one +sub empty_search_cache_maildir($) { my ($maildir) = (@_); rmtree($maildir) if (-d $maildir); my $folder = new Mail::Box::Maildir(folder => $maildir, @@ -45,7 +45,7 @@ sub search($$$) { push @args, "--duplicate=1" if $remove_dups; push @args, $query; - empty_maildir($maildir); + empty_search_cache_maildir($maildir); open my $pipe, '-|', @args or die "Running @args failed: $!\n"; while (<$pipe>) { chomp; @@ -120,7 +120,7 @@ sub thread_action($$@) { my $mid = get_message_id(); if (! defined $mid) { - empty_maildir($results_dir); + empty_search_cache_maildir($results_dir); die "notmuch-mutt: cannot find Message-Id, abort.\n"; } From 18e35950dad8daa485d86e244ffd5d5e6a63d03a Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sun, 28 May 2023 10:29:45 +0800 Subject: [PATCH 47/90] notmuch-mutt: do not clear search cache Maildir when nothing is found The previous results might be useful to the user but an empty directory definitely isn't useful. --- contrib/notmuch-mutt/notmuch-mutt | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index 0f7379ac..875fd032 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -120,7 +120,6 @@ sub thread_action($$@) { my $mid = get_message_id(); if (! defined $mid) { - empty_search_cache_maildir($results_dir); die "notmuch-mutt: cannot find Message-Id, abort.\n"; } From 239fdbbbf0cbd6cd6ebafb87e88cdb3cded75364 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sun, 28 May 2023 10:29:46 +0800 Subject: [PATCH 48/90] notmuch-mutt: check that the search cache Maildir is not a real Maildir This prevents data loss when users configure the search cache Maildir to be a real Maildir containing their real mail data, since the search cache Maildir is expected to contain only symlinks to the real mail data. Prevents: --- contrib/notmuch-mutt/notmuch-mutt | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/contrib/notmuch-mutt/notmuch-mutt b/contrib/notmuch-mutt/notmuch-mutt index 875fd032..1ac68065 100755 --- a/contrib/notmuch-mutt/notmuch-mutt +++ b/contrib/notmuch-mutt/notmuch-mutt @@ -13,6 +13,7 @@ use warnings; use File::Path; use File::Basename; +use File::Find; use Getopt::Long qw(:config no_getopt_compat); use Mail::Header; use Mail::Box::Maildir; @@ -25,6 +26,50 @@ my $xdg_cache_dir = "$ENV{HOME}/.cache"; $xdg_cache_dir = $ENV{XDG_CACHE_HOME} if $ENV{XDG_CACHE_HOME}; my $cache_dir = "$xdg_cache_dir/notmuch/mutt"; +sub die_dir($$) { + my ($maildir, $error) = @_; + die "notmuch-mutt: search cache maildir $maildir $error\n". + "Please ensure that the notmuch-mutt search cache Maildir\n". + "contains no subfolders or real mail data, only symlinks to mail\n"; +} + +sub die_subdir($$$) { + my ($maildir, $subdir, $error) = @_; + die_dir($maildir, "subdir $subdir $error"); +} + +# check that the search cache maildir is that and not a real maildir +# otherwise there could be data loss when the search cache is emptied +sub check_search_cache_maildir($) { + my ($maildir) = (@_); + + return unless -e $maildir; + + -d $maildir or die_dir($maildir, 'is not a directory'); + + opendir(my $mdh, $maildir) or die_dir($maildir, "cannot be opened: $!"); + my @contents = grep { !/^\.\.?$/ } readdir $mdh; + closedir $mdh; + + my @required = ('cur', 'new', 'tmp'); + foreach my $d (@required) { + -l "$maildir/$d" and die_dir($maildir, "contains symlink $d"); + -e "$maildir/$d" or die_subdir($maildir, $d, 'is missing'); + -d "$maildir/$d" or die_subdir($maildir, $d, 'is not a directory'); + find(sub { + $_ eq '.' and return; + $_ eq '..' and return; + -l $_ or die_subdir($maildir, $d, "contains non-symlink $_"); + }, "$maildir/$d"); + } + + my %required = map { $_ => 1 } @required; + foreach my $d (@contents) { + -l "$maildir/$d" and die_dir( $maildir, "contains symlink $d"); + -d "$maildir/$d" or die_dir( $maildir, "contains non-directory $d"); + exists($required[$d]) or die_dir( $maildir, "contains directory $d"); + } +} # create an empty search cache maildir (if missing) or empty existing one sub empty_search_cache_maildir($) { @@ -45,6 +90,7 @@ sub search($$$) { push @args, "--duplicate=1" if $remove_dups; push @args, $query; + check_search_cache_maildir($maildir); empty_search_cache_maildir($maildir); open my $pipe, '-|', @args or die "Running @args failed: $!\n"; while (<$pipe>) { From 95a4bf38173311597bcc4fa7f597d58bb11ed119 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 31 Mar 2023 14:40:50 -0600 Subject: [PATCH 49/90] ruby: db: reorganize initializer In order to make it more extensible. No functional changes. Signed-off-by: Felipe Contreras --- bindings/ruby/database.c | 52 ++++++++++++++++++++++++---------------- bindings/ruby/defs.h | 2 -- bindings/ruby/init.c | 4 ---- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/bindings/ruby/database.c b/bindings/ruby/database.c index e6387f59..ed224ef7 100644 --- a/bindings/ruby/database.c +++ b/bindings/ruby/database.c @@ -58,37 +58,49 @@ notmuch_rb_database_initialize (int argc, VALUE *argv, VALUE self) notmuch_database_t *database; notmuch_status_t ret; + path = NULL; + create = 0; + mode = NOTMUCH_DATABASE_MODE_READ_ONLY; + /* Check arguments */ rb_scan_args (argc, argv, "02", &pathv, &hashv); if (!NIL_P (pathv)) { SafeStringValue (pathv); path = RSTRING_PTR (pathv); - } else { - path = NULL; } if (!NIL_P (hashv)) { - Check_Type (hashv, T_HASH); - create = RTEST (rb_hash_aref (hashv, ID2SYM (ID_db_create))); - modev = rb_hash_aref (hashv, ID2SYM (ID_db_mode)); - if (NIL_P (modev)) - mode = NOTMUCH_DATABASE_MODE_READ_ONLY; - else if (!FIXNUM_P (modev)) - rb_raise (rb_eTypeError, ":mode isn't a Fixnum"); - else { - mode = FIX2INT (modev); - switch (mode) { - case NOTMUCH_DATABASE_MODE_READ_ONLY: - case NOTMUCH_DATABASE_MODE_READ_WRITE: - break; - default: - rb_raise ( rb_eTypeError, "Invalid mode"); + VALUE rmode, rcreate; + VALUE kwargs[2]; + static ID keyword_ids[2]; + + if (!keyword_ids[0]) { + keyword_ids[0] = rb_intern_const ("mode"); + keyword_ids[1] = rb_intern_const ("create"); + } + + rb_get_kwargs (hashv, keyword_ids, 0, 2, kwargs); + + rmode = kwargs[0]; + rcreate = kwargs[1]; + + if (rmode != Qundef) { + if (!FIXNUM_P (rmode)) + rb_raise (rb_eTypeError, ":mode isn't a Fixnum"); + else { + mode = FIX2INT (rmode); + switch (mode) { + case NOTMUCH_DATABASE_MODE_READ_ONLY: + case NOTMUCH_DATABASE_MODE_READ_WRITE: + break; + default: + rb_raise ( rb_eTypeError, "Invalid mode"); + } } } - } else { - create = 0; - mode = NOTMUCH_DATABASE_MODE_READ_ONLY; + if (rcreate != Qundef) + create = RTEST (rcreate); } rb_check_typeddata (self, ¬much_rb_database_type); diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h index f2a7e484..a2cb38c8 100644 --- a/bindings/ruby/defs.h +++ b/bindings/ruby/defs.h @@ -47,8 +47,6 @@ extern VALUE notmuch_rb_eUnbalancedFreezeThawError; extern VALUE notmuch_rb_eUnbalancedAtomicError; extern ID ID_call; -extern ID ID_db_create; -extern ID ID_db_mode; /* RSTRING_PTR() is new in ruby-1.9 */ #if !defined(RSTRING_PTR) diff --git a/bindings/ruby/init.c b/bindings/ruby/init.c index e0c9936a..2d1994af 100644 --- a/bindings/ruby/init.c +++ b/bindings/ruby/init.c @@ -41,8 +41,6 @@ VALUE notmuch_rb_eUnbalancedFreezeThawError; VALUE notmuch_rb_eUnbalancedAtomicError; ID ID_call; -ID ID_db_create; -ID ID_db_mode; const rb_data_type_t notmuch_rb_object_type = { .wrap_struct_name = "notmuch_object", @@ -95,8 +93,6 @@ Init_notmuch (void) VALUE mod; ID_call = rb_intern ("call"); - ID_db_create = rb_intern ("create"); - ID_db_mode = rb_intern ("mode"); mod = rb_define_module ("Notmuch"); From 3771832b013012e408696897215fd0edde7dce37 Mon Sep 17 00:00:00 2001 From: Michael J Gruber Date: Fri, 16 Jun 2023 13:19:26 +0200 Subject: [PATCH 50/90] python: adjust legacy bindings to py 3.12 Py 3.12 finally pulled the plug on the `SafeConfigParser` class which has been deprecated since py 3.2. We use it in the legacy bindings only, so take the easy route of importing `ConfigParser` as `SafeConfigParser` and monkey-patching so that the class has the expected interface. --- bindings/python/notmuch/compat.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bindings/python/notmuch/compat.py b/bindings/python/notmuch/compat.py index c931329e..4a94e05c 100644 --- a/bindings/python/notmuch/compat.py +++ b/bindings/python/notmuch/compat.py @@ -47,7 +47,10 @@ if sys.version_info[0] == 2: return value else: - from configparser import SafeConfigParser + from configparser import ConfigParser as SafeConfigParser + + if not hasattr(SafeConfigParser, 'readfp'): # py >= 3.12 + SafeConfigParser.readfp = SafeConfigParser.read_file class Python3StringMixIn(object): def __str__(self): From 808fd6f258e6f74f4cdb5b6f61fe102bdba59e6e Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 27 May 2023 14:20:51 -0300 Subject: [PATCH 51/90] test: add known broken test for message-id with embedded spaces. According to my reading of RFC5322, there is an obsolete syntax for Message-Id which permits folding whitespace (i.e. to be removed / ignored by parsers). In [1] Paul Wise observed that notmuch removed whitespace on indexing, but does not do any corresponding normalization of queries. Mark the latter as a bug by adding a failing test. [1]: id:20230409044143.4041560-1-pabs3@bonedaddy.net --- test/T080-search.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/T080-search.sh b/test/T080-search.sh index a3f0dead..515eb88f 100755 --- a/test/T080-search.sh +++ b/test/T080-search.sh @@ -34,6 +34,12 @@ add_message '[subject]="search by id"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000" output=$(notmuch search id:${gen_msg_id} | notmuch_search_sanitize) test_expect_equal "$output" "thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by id (inbox unread)" +test_begin_subtest "Message-Id with spaces" +test_subtest_known_broken +add_message '[id]="message id@example.com"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' +output=$(notmuch search --output=messages id:"message id@example.com") +test_expect_equal "$output" "messageid@example.com" + test_begin_subtest "Search by mid:" add_message '[subject]="search by mid"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' output=$(notmuch search mid:${gen_msg_id} | notmuch_search_sanitize) @@ -132,6 +138,7 @@ thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by to (inbox unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; subjectsearchtest (inbox unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; utf8-sübjéct (inbox unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by id (inbox unread) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; Message-Id with spaces (inbox unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by mid (inbox unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by tag (inbox searchbytag unread) thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; search by thread (inbox unread) From a62b8a95c0e890f11cb39cc2aaea3a4893c8268d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 29 May 2023 08:01:40 -0300 Subject: [PATCH 52/90] doc/lib: clarify ownership for notmuch_database_get_revision The ownership is implicit in the const declaration (I think!), but that does not show up in the doxygen generated API docs. --- lib/notmuch.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/notmuch.h b/lib/notmuch.h index 76156178..4e2b0fa4 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -722,7 +722,8 @@ notmuch_database_end_atomic (notmuch_database_t *notmuch); * * The UUID is a NUL-terminated opaque string that uniquely identifies * this database. Two revision numbers are only comparable if they - * have the same database UUID. + * have the same database UUID. The string 'uuid' is owned by notmuch + * and should not be freed or modified by the user. */ unsigned long notmuch_database_get_revision (notmuch_database_t *notmuch, From f873790b6fb89ad4e733e6d59833d612ce265996 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 31 May 2023 08:19:18 -0300 Subject: [PATCH 53/90] perf-test: update corpus signature In the decade (!) since this corpus was last updated, the keyserver network is essentially dead, and I have migrated gpg keys. Bump the version number as a clean way of switching signatures. Also update the instructions to suggest using "--locate-external-key" to download the public key. By default this uses WKD, which is now supported for my UID. --- performance-test/README | 4 ++-- .../download/notmuch-email-corpus-0.5.tar.xz.asc | 16 ++++++++++++++++ performance-test/version.sh | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 performance-test/download/notmuch-email-corpus-0.5.tar.xz.asc diff --git a/performance-test/README b/performance-test/README index 59b37b1b..1ca0df2f 100644 --- a/performance-test/README +++ b/performance-test/README @@ -24,11 +24,11 @@ Getting set up to run tests: First, you need to get the corpus. If you don't already have the gpg key for David Bremner, run - % gpg --search 'david@tethera.net' + % gpg --locate-external-key 'david@tethera.net' This should get you a key with fingerprint - 815B 6398 2A79 F8E7 C727 86C4 762B 57BB 7842 06AD + 7A18 807F 100A 4570 C596 8420 7E4E 65C8 720B 706B (the last 8 digits are printed as the "key id"). diff --git a/performance-test/download/notmuch-email-corpus-0.5.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.5.tar.xz.asc new file mode 100644 index 00000000..2318c2f6 --- /dev/null +++ b/performance-test/download/notmuch-email-corpus-0.5.tar.xz.asc @@ -0,0 +1,16 @@ +-----BEGIN PGP SIGNATURE----- + +iQIzBAABCAAdFiEEkiyHYXwaY0SiY6fqA0U5G1WqFSEFAmR119gACgkQA0U5G1Wq +FSHlSQ/+NSRj27PEZjaP2I+3j+rsMG3pnVckNcuOQyfgjJ+zEagMZyRu3vaIA/pX +xtBrNIX4l4CQIkqwyNjsuqJdzh6S3DeCWSEr1Q+GSBki+wQCBiRuDYY2HQoDezEK +4bMfniEWZpKJD8PfIabz0OOqMUsfXEYMd9kefew5/J4OGnDIv8E5pKfqvDNNO4rW +MhZ9w9uR9wkvmfmpO66kAgTfLllwiyNHWoWnzQfNmqM8eULFn7XxM1PEZShUEqXf +pTWCqqm5OyUcy8f+gy9Mb7DRRvnwLpHHRQlCzzH2c+ENQRpt1ErsgVKpHTVk4UsB +EML+zwyWEaQg7xVKWXRJDuGCF47S1GCQNUtvtx57HJl6Ds6N2mlr2KEGaI7qtiz5 +5hdaTc0L/TVN0WS+uCdfdDDozFErf1kwhA6Jnpi0YNNdK+wpFzj7ISvA+DNHwJ75 +TLBuJIU/h3QfX3NDC5xDbsWAgpv7a84e7ePO6+kAVkHsNYDbFjiunr5fRbqDsJcJ +B+aVGhKvFZbziC84Dn5Ae9Lpa40fQlxbdb+So2nDIiuR3P33vt7wr/e2ptVfrqkn +a1DM96n03VWexwEDMye3b3rOTXsN5Ul87zucg9xWm5JT75NGuqJ1WDJN/wwNPDro +ZXS1OHh7UKsU1tP2J9+gLiKYNBP4m4BQjEgYXpiYEoge9A1QplQ= +=5/Ep +-----END PGP SIGNATURE----- diff --git a/performance-test/version.sh b/performance-test/version.sh index f02527a7..357b9dad 100644 --- a/performance-test/version.sh +++ b/performance-test/version.sh @@ -1,3 +1,3 @@ # this should be both a valid Makefile fragment and valid POSIX(ish) shell. -PERFTEST_VERSION=0.4 +PERFTEST_VERSION=0.5 From e9ff896f8491e8780fcf7322436fc0c0575fe8f9 Mon Sep 17 00:00:00 2001 From: Tim Culverhouse Date: Sat, 6 May 2023 19:59:27 -0500 Subject: [PATCH 54/90] schemata: document 'excluded' field in structured output Include the 'excluded' field in the structured output schema. Signed-off-by: Tim Culverhouse --- devel/schemata | 1 + 1 file changed, 1 insertion(+) diff --git a/devel/schemata b/devel/schemata index 66bcdbed..4e05cdac 100644 --- a/devel/schemata +++ b/devel/schemata @@ -76,6 +76,7 @@ message = { # (format_message_sprinter) id: messageid, match: bool, + excluded: bool, filename: [string*], timestamp: unix_time, # date header as unix time date_relative: string, # user-friendly timestamp From dfa43a19218ed46f677034ab7bf8b8907a327935 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Apr 2023 11:26:23 -0300 Subject: [PATCH 55/90] test: treat undefined feature variables as 0 When running the test suite without building first, it is desirable to have the tests consider these variables being undefined as equivalent to the feature not being present, and in particular for the tests not to generate errors. --- test/T010-help-test.sh | 2 +- test/T060-count.sh | 2 +- test/T081-sexpr-search.sh | 2 +- test/T095-address.sh | 2 +- test/T150-tagging.sh | 2 +- test/T160-json.sh | 2 +- test/T220-reply.sh | 2 +- test/T240-dump-restore.sh | 2 +- test/T355-smime.sh | 2 +- test/T391-python-cffi.sh | 3 +-- test/T392-python-cffi-notmuch.sh | 2 +- test/T520-show.sh | 2 +- test/T570-revision-tracking.sh | 2 +- test/T700-reindex.sh | 2 +- test/T800-asan.sh | 2 +- test/T810-tsan.sh | 2 +- test/T850-git.sh | 2 +- 17 files changed, 17 insertions(+), 18 deletions(-) diff --git a/test/T010-help-test.sh b/test/T010-help-test.sh index da45d3ae..1fbc46a2 100755 --- a/test/T010-help-test.sh +++ b/test/T010-help-test.sh @@ -12,7 +12,7 @@ test_expect_success 'notmuch help' test_begin_subtest 'notmuch --version' test_expect_success 'notmuch --version' -if [ $NOTMUCH_HAVE_MAN -eq 1 ]; then +if [ "${NOTMUCH_HAVE_MAN-0}" = "1" ]; then test_begin_subtest 'notmuch --help tag' test_expect_success 'notmuch --help tag' diff --git a/test/T060-count.sh b/test/T060-count.sh index 48146706..6e855b59 100755 --- a/test/T060-count.sh +++ b/test/T060-count.sh @@ -157,7 +157,7 @@ print("4: {} messages".format(query.count_messages())) EOF test_expect_equal_file EXPECTED OUTPUT -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_begin_subtest "and of exact terms (query=sexp)" output=$(notmuch count --query=sexp '(and "wonderful" "wizard")') diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 0c7db9c2..8800b545 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -2,7 +2,7 @@ test_description='"notmuch search" in several variations' . $(dirname "$0")/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_SFSEXP -ne 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" != "1" ]; then printf "Skipping due to missing sfsexp library\n" test_done fi diff --git a/test/T095-address.sh b/test/T095-address.sh index 8bb3627a..0cafbe20 100755 --- a/test/T095-address.sh +++ b/test/T095-address.sh @@ -325,7 +325,7 @@ cat <EXPECTED EOF test_expect_equal_file EXPECTED OUTPUT -if [[ NOTMUCH_HAVE_SFSEXP = 1 ]]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_begin_subtest "sexpr query: all messages" notmuch address '*' > EXPECTED notmuch address --query=sexp '()' > OUTPUT diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh index ac3f2539..273c0af3 100755 --- a/test/T150-tagging.sh +++ b/test/T150-tagging.sh @@ -328,7 +328,7 @@ test_expect_equal "$output" "A Xapian exception occurred opening database" add_email_corpus -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_query_syntax '(and "wonderful" "wizard")' 'wonderful and wizard' test_query_syntax '(or "php" "wizard")' 'php or wizard' diff --git a/test/T160-json.sh b/test/T160-json.sh index 4a797f6a..f61c3f2d 100755 --- a/test/T160-json.sh +++ b/test/T160-json.sh @@ -66,7 +66,7 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\", \"unread\"]}]" test_begin_subtest "Search message: json, 64-bit timestamp" -if [ $NOTMUCH_HAVE_64BIT_TIME_T -ne 1 ]; then +if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then test_subtest_known_broken fi add_message "[subject]=\"json-search-64bit-timestamp-subject\"" "[date]=\"Tue, 01 Jan 2999 12:00:00 -0000\"" "[body]=\"json-search-64bit-timestamp-message\"" diff --git a/test/T220-reply.sh b/test/T220-reply.sh index 207f5788..120d7138 100755 --- a/test/T220-reply.sh +++ b/test/T220-reply.sh @@ -24,7 +24,7 @@ test_begin_subtest "Basic reply" notmuch reply id:${gen_msg_id} >OUTPUT 2>&1 && echo OK >> OUTPUT test_expect_equal_file basic.expected OUTPUT -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_begin_subtest "Basic reply (query=sexp)" notmuch reply --query=sexp "(id ${gen_msg_id})" >OUTPUT 2>&1 && echo OK >> OUTPUT diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh index a86f0fb7..80939030 100755 --- a/test/T240-dump-restore.sh +++ b/test/T240-dump-restore.sh @@ -118,7 +118,7 @@ notmuch dump -- from:cworth > dump-dash-cworth.actual test_expect_equal_file dump-cworth.expected dump-dash-cworth.actual -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_begin_subtest "dump --query=sexp -- '(from cworth)'" notmuch dump --query=sexp -- '(from cworth)' > dump-dash-cworth.actual2 diff --git a/test/T355-smime.sh b/test/T355-smime.sh index 809274ce..3bd5a17e 100755 --- a/test/T355-smime.sh +++ b/test/T355-smime.sh @@ -184,7 +184,7 @@ output=$(notmuch show --format=json id:smime-onepart-signed@protected-headers.ex test_valid_json "$output" test_begin_subtest "Verify signature on PKCS#7 SignedData message" -if [ $NOTMUCH_HAVE_64BIT_TIME_T -ne 1 ]; then +if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then test_subtest_known_broken fi output=$(notmuch show --format=json id:smime-onepart-signed@protected-headers.example) diff --git a/test/T391-python-cffi.sh b/test/T391-python-cffi.sh index 30872af0..0ef9e0d3 100755 --- a/test/T391-python-cffi.sh +++ b/test/T391-python-cffi.sh @@ -2,11 +2,10 @@ test_description="python bindings (pytest)" . $(dirname "$0")/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_PYTHON3_CFFI -eq 0 -o $NOTMUCH_HAVE_PYTHON3_PYTEST -eq 0 ]; then +if [ "${NOTMUCH_HAVE_PYTHON3_CFFI-0}" = "0" -o "${NOTMUCH_HAVE_PYTHON3_PYTEST-0}" = "0" ]; then test_done fi - test_begin_subtest "python cffi tests (NOTMUCH_CONFIG set)" pytest_dir=$NOTMUCH_BUILDDIR/bindings/python-cffi/build/stage printf "[pytest]\nminversion = 3.0\naddopts = -ra\n" > $pytest_dir/pytest.ini diff --git a/test/T392-python-cffi-notmuch.sh b/test/T392-python-cffi-notmuch.sh index 15c8fc6b..06161219 100755 --- a/test/T392-python-cffi-notmuch.sh +++ b/test/T392-python-cffi-notmuch.sh @@ -2,7 +2,7 @@ test_description="python bindings (notmuch test suite)" . $(dirname "$0")/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_PYTHON3_CFFI -eq 0 -o $NOTMUCH_HAVE_PYTHON3_PYTEST -eq 0 ]; then +if [ "${NOTMUCH_HAVE_PYTHON3_CFFI-0}" = "0" -o "${NOTMUCH_HAVE_PYTHON3_PYTEST-0}" = "0" ]; then test_done fi diff --git a/test/T520-show.sh b/test/T520-show.sh index c7b73a6d..6bcf109c 100755 --- a/test/T520-show.sh +++ b/test/T520-show.sh @@ -35,7 +35,7 @@ notmuch show --entire-thread=true --sort=oldest-first $QUERY > OUTPUT test_expect_equal_file EXPECTED OUTPUT -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_query_syntax '(and "wonderful" "wizard")' 'wonderful and wizard' test_query_syntax '(or "php" "wizard")' 'php or wizard' diff --git a/test/T570-revision-tracking.sh b/test/T570-revision-tracking.sh index a7480050..bcc97dd9 100755 --- a/test/T570-revision-tracking.sh +++ b/test/T570-revision-tracking.sh @@ -95,7 +95,7 @@ subtotal=$(notmuch count lastmod:..$lastmod) result=$(($subtotal == $total-1)) test_expect_equal 1 "$result" -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then test_begin_subtest 'exclude one message using negative lastmod (sexp)' total=$(notmuch count '*') notmuch tag +${RANDOM} id:4EFC743A.3060609@april.org diff --git a/test/T700-reindex.sh b/test/T700-reindex.sh index 347f8483..af34ad7c 100755 --- a/test/T700-reindex.sh +++ b/test/T700-reindex.sh @@ -5,7 +5,7 @@ test_description='reindexing messages' add_email_corpus -if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then count=$(notmuch count --lastmod '*' | cut -f 3) for query in '()' '(not)' '(and)' '(or ())' '(or (not))' '(or (and))' \ diff --git a/test/T800-asan.sh b/test/T800-asan.sh index 5055c93e..7c28dc7c 100755 --- a/test/T800-asan.sh +++ b/test/T800-asan.sh @@ -2,7 +2,7 @@ test_description='run code with ASAN enabled against the library' . $(dirname "$0")/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_ASAN -ne 1 ]; then +if [ "${NOTMUCH_HAVE_ASAN-0}" != "1" ]; then printf "Skipping due to missing ASAN support\n" test_done fi diff --git a/test/T810-tsan.sh b/test/T810-tsan.sh index 7e877b27..4071e296 100755 --- a/test/T810-tsan.sh +++ b/test/T810-tsan.sh @@ -8,7 +8,7 @@ test_description='run code with TSan enabled against the library' . "$test_directory"/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_TSAN -ne 1 ]; then +if [ "${NOTMUCH_HAVE_TSAN-0}" != "1" ]; then printf "Skipping due to missing TSan support\n" test_done fi diff --git a/test/T850-git.sh b/test/T850-git.sh index 55cec78a..be6e0cd4 100755 --- a/test/T850-git.sh +++ b/test/T850-git.sh @@ -2,7 +2,7 @@ test_description='"notmuch git" to save and restore tags' . $(dirname "$0")/test-lib.sh || exit 1 -if [ $NOTMUCH_HAVE_SFSEXP -ne 1 ]; then +if [ "${NOTMUCH_HAVE_SFSEXP-0}" != "1" ]; then printf "Skipping due to missing sfsexp library\n" test_done fi From f6fcdf12da455d82a9ed0a0a33eddac60253a6e8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Apr 2023 11:26:24 -0300 Subject: [PATCH 56/90] test: check for empty/missing files in test_expect_equal_message_body Messages can have empty bodies, but empty files are not messages. --- test/test-lib.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test-lib.sh b/test/test-lib.sh index 1a6525df..f218fa03 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -388,6 +388,14 @@ test_expect_equal_message_body () { test "$#" = 2 || error "bug in the test script: not 2 parameters to test_expect_equal_file" + for file in "$1" "$2"; do + if [ ! -s "$file" ]; then + test_failure_ "Missing or zero length file: $file" + inside_subtest= + return 1 + fi + done + expected=$(sed '1,/^$/d' "$1") output=$(sed '1,/^$/d' "$2") test_expect_equal "$expected" "$output" From 73f3081160fb80345f3953cbdeba340975375325 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Apr 2023 11:26:25 -0300 Subject: [PATCH 57/90] test: Guess a value for NOTMUCH_PYTHON python3 will work for many people, and reduce the friction to running the tests without running configure first. --- test/test-lib-common.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh index 18fa29c0..4d14e0b3 100644 --- a/test/test-lib-common.sh +++ b/test/test-lib-common.sh @@ -323,6 +323,10 @@ rm -rf "$TMP_DIRECTORY" || { exit 1 } +# Provide a guess at a usable Python, to support running tests without +# running configure first. +NOTMUCH_PYTHON=${NOTMUCH_PYTHON-python3} + # A temporary home directory is needed by at least: # - emacs/"Sending a message via (fake) SMTP" # - emacs/"Reply within emacs" From ec26eeaeec87781dee7dbf720103a5bc9b6bba5d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Apr 2023 11:26:26 -0300 Subject: [PATCH 58/90] test: support testing notmuch as installed We put some effort into testing the built copy rather than some installed copy. On the other hand for people like packagers, testing the installed copy is also of interest. When NOTMUCH_TEST_INSTALLED is set to a nonempty value, tests do not require a built notmuch tree or running configure. Some of the tests marked as broken when running against installed notmuch are probably fixable. --- test/README | 17 +++++++++++++++++ test/T000-basic.sh | 1 + test/T010-help-test.sh | 3 +++ test/T160-json.sh | 2 ++ test/T240-dump-restore.sh | 2 ++ test/T310-emacs.sh | 2 ++ test/T350-crypto.sh | 1 + test/T355-smime.sh | 2 ++ test/T360-symbol-hiding.sh | 4 ++++ test/T390-python.sh | 4 ++++ test/T391-python-cffi.sh | 4 ++++ test/T395-ruby.sh | 8 ++++++-- test/T410-argument-parsing.sh | 4 ++++ test/T480-hex-escaping.sh | 4 ++++ test/T490-parse-time-string.sh | 4 ++++ test/T550-db-features.sh | 4 ++++ test/T566-lib-message.sh | 4 ++++ test/T592-thread-breakage.sh | 4 ++++ test/T710-message-id.sh | 4 ++++ test/T800-asan.sh | 5 +++++ test/T850-git.sh | 6 ++++++ test/export-dirs.sh | 2 +- test/notmuch-test | 12 ++++++++++-- test/test-lib-common.sh | 13 ++++++++++--- test/test-lib-emacs.sh | 12 ++++++++++-- test/test-lib.sh | 27 +++++++++++++++++++++------ 26 files changed, 139 insertions(+), 16 deletions(-) diff --git a/test/README b/test/README index 10f127cb..a81808b1 100644 --- a/test/README +++ b/test/README @@ -137,6 +137,23 @@ detection of missing prerequisites. In the future we may treat tests unable to run because of missing prerequisites, but not explicitly skipped by the user, as failures. +Testing installed notmuch +------------------------- + +Systems integrators (e.g. Linux distros) may wish to test an installed +version of notmuch. This can be done be running + + $ NOTMUCH_TEST_INSTALLED=1 ./test/notmuch-test + +In this scenario the test suite does not assume a built tree, and in +particular cannot rely on the output of 'configure'. You may want to +set certain feature environment variables ('NOTMUCH_HAVE_*') directly +if you know those apply to your installed notmuch). Consider also +setting TERM=dumb if the value of TERM cannot be used (e.g. in a +chroot with missing terminfo). Note that having a built tree may cause +surprising/broken results for NOTMUCH_TEST_INSTALLED, so consider +cleaning first. + Writing Tests ------------- The test script is written as a shell script. It is to be named as diff --git a/test/T000-basic.sh b/test/T000-basic.sh index a2f4d93f..642f918d 100755 --- a/test/T000-basic.sh +++ b/test/T000-basic.sh @@ -66,6 +66,7 @@ test_begin_subtest 'NOTMUCH_CONFIG is set and points to an existing file' test_expect_success 'test -f "${NOTMUCH_CONFIG}"' test_begin_subtest 'PATH is set to build directory' +test_subtest_broken_for_installed test_expect_equal \ "$(dirname ${TEST_DIRECTORY})" \ "$(echo $PATH|cut -f1 -d: | sed -e 's,/test/valgrind/bin$,,')" diff --git a/test/T010-help-test.sh b/test/T010-help-test.sh index 1fbc46a2..827edc14 100755 --- a/test/T010-help-test.sh +++ b/test/T010-help-test.sh @@ -19,6 +19,9 @@ if [ "${NOTMUCH_HAVE_MAN-0}" = "1" ]; then test_begin_subtest 'notmuch help tag' test_expect_success 'notmuch help tag' else + if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done + fi test_begin_subtest 'notmuch --help tag (man pages not available)' test_expect_success 'test_must_fail notmuch --help tag >/dev/null' diff --git a/test/T160-json.sh b/test/T160-json.sh index f61c3f2d..318c9788 100755 --- a/test/T160-json.sh +++ b/test/T160-json.sh @@ -65,6 +65,7 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\", \"tags\": [\"inbox\", \"unread\"]}]" +if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then test_begin_subtest "Search message: json, 64-bit timestamp" if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then test_subtest_known_broken @@ -81,6 +82,7 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\", \"query\": [\"id:$gen_msg_id\", null], \"tags\": [\"inbox\", \"unread\"]}]" +fi # NOTMUCH_TEST_INSTALLED undefined / empty test_begin_subtest "Format version: too low" test_expect_code 20 "notmuch search --format-version=0 \\*" diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh index 80939030..c3f18839 100755 --- a/test/T240-dump-restore.sh +++ b/test/T240-dump-restore.sh @@ -139,6 +139,7 @@ notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual test_begin_subtest "Check for a safe set of message-ids" +test_subtest_broken_for_installed notmuch search --output=messages from:cworth | sed s/^id:// > EXPECTED notmuch search --output=messages from:cworth | sed s/^id:// |\ $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT @@ -246,6 +247,7 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count test_begin_subtest 'format=batch-tag, checking encoded output' +test_subtest_broken_for_installed NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\ awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count diff --git a/test/T310-emacs.sh b/test/T310-emacs.sh index 9d0df187..d3aa2e7d 100755 --- a/test/T310-emacs.sh +++ b/test/T310-emacs.sh @@ -258,6 +258,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "Verify that sent messages are saved/searchable (via FCC)" +test_subtest_broken_for_installed notmuch new > /dev/null output=$(notmuch search 'subject:"testing message sent via SMTP"' | notmuch_search_sanitize) test_expect_equal "$output" "thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; Testing message sent via SMTP (inbox)" @@ -350,6 +351,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "Reply within emacs" +test_subtest_broken_for_installed test_emacs '(let ((message-hidden-headers ''())) (notmuch-search "subject:\"testing message sent via SMTP\"") (notmuch-test-wait) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 721cbfdd..27c0e86d 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -30,6 +30,7 @@ msg_file=$(notmuch search --output=files subject:signed-message-sent-via-SMTP) test_expect_equal_message_body sent_message "$msg_file" test_begin_subtest "signed part content-type indexing" +test_subtest_broken_for_installed notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize > OUTPUT cat <EXPECTED thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed) diff --git a/test/T355-smime.sh b/test/T355-smime.sh index 3bd5a17e..d2118c04 100755 --- a/test/T355-smime.sh +++ b/test/T355-smime.sh @@ -183,6 +183,7 @@ test_begin_subtest "show PKCS#7 SignedData outputs valid JSON" output=$(notmuch show --format=json id:smime-onepart-signed@protected-headers.example) test_valid_json "$output" +if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then test_begin_subtest "Verify signature on PKCS#7 SignedData message" if [ "${NOTMUCH_HAVE_64BIT_TIME_T-0}" != "1" ]; then test_subtest_known_broken @@ -194,6 +195,7 @@ test_json_nodes <<<"$output" \ 'expires:[0][0][0]["crypto"]["signed"]["status"][0]["expires"]=2611032858' \ 'fingerprint:[0][0][0]["crypto"]["signed"]["status"][0]["fingerprint"]="702BA4B157F1E2B7D16B0C6A5FFC8A7DE2057DEB"' \ 'status:[0][0][0]["crypto"]["signed"]["status"][0]["status"]="good"' +fi # NOTMUCH_TEST_INSTALLED undefined / empty test_begin_subtest "Verify signature on PKCS#7 SignedData message signer User ID" if [ $NOTMUCH_GMIME_X509_CERT_VALIDITY -ne 1 ]; then diff --git a/test/T360-symbol-hiding.sh b/test/T360-symbol-hiding.sh index 642457bf..ff06ff69 100755 --- a/test/T360-symbol-hiding.sh +++ b/test/T360-symbol-hiding.sh @@ -11,6 +11,10 @@ test_description='exception symbol hiding' . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + test_begin_subtest 'running test' run_test mkdir -p ${PWD}/fakedb/.notmuch $TEST_DIRECTORY/symbol-test ${PWD}/fakedb ${PWD}/nonexistent 2>&1 \ diff --git a/test/T390-python.sh b/test/T390-python.sh index 9f71ce3c..21912431 100755 --- a/test/T390-python.sh +++ b/test/T390-python.sh @@ -4,6 +4,10 @@ test_description="python bindings" test_require_external_prereq ${NOTMUCH_PYTHON} +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + add_email_corpus add_gnupg_home diff --git a/test/T391-python-cffi.sh b/test/T391-python-cffi.sh index 0ef9e0d3..0059b050 100755 --- a/test/T391-python-cffi.sh +++ b/test/T391-python-cffi.sh @@ -2,6 +2,10 @@ test_description="python bindings (pytest)" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + if [ "${NOTMUCH_HAVE_PYTHON3_CFFI-0}" = "0" -o "${NOTMUCH_HAVE_PYTHON3_PYTEST-0}" = "0" ]; then test_done fi diff --git a/test/T395-ruby.sh b/test/T395-ruby.sh index c066c842..d0c6bb17 100755 --- a/test/T395-ruby.sh +++ b/test/T395-ruby.sh @@ -2,7 +2,7 @@ test_description="ruby bindings" . $(dirname "$0")/test-lib.sh || exit 1 -if [ "${NOTMUCH_HAVE_RUBY_DEV}" = "0" ]; then +if [ -z "${NOTMUCH_TEST_INSTALLED-}" -a "${NOTMUCH_HAVE_RUBY_DEV-0}" = "0" ]; then test_subtest_missing_external_prereq_["ruby development files"]=t fi @@ -15,7 +15,11 @@ test_ruby() { db = Notmuch::Database.new() EOF cat - ) | $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"> OUTPUT + ) | if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + ruby + else + $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby" + fi> OUTPUT test_expect_equal_file EXPECTED OUTPUT } diff --git a/test/T410-argument-parsing.sh b/test/T410-argument-parsing.sh index d9aa8e2d..40b625fe 100755 --- a/test/T410-argument-parsing.sh +++ b/test/T410-argument-parsing.sh @@ -2,6 +2,10 @@ test_description="argument parsing" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + test_begin_subtest "sanity check" $TEST_DIRECTORY/arg-test pos1 --keyword=one --boolean --string=foo pos2 --int=7 --flag=one --flag=three > OUTPUT cat < EXPECTED diff --git a/test/T480-hex-escaping.sh b/test/T480-hex-escaping.sh index b2eb80b9..8bddf3e7 100755 --- a/test/T480-hex-escaping.sh +++ b/test/T480-hex-escaping.sh @@ -2,6 +2,10 @@ test_description="hex encoding and decoding" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + test_begin_subtest "round trip" find $NOTMUCH_SRCDIR/test/corpora/default -type f -print | sort | xargs cat > EXPECTED $TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT diff --git a/test/T490-parse-time-string.sh b/test/T490-parse-time-string.sh index f89755ed..3b6e48c4 100755 --- a/test/T490-parse-time-string.sh +++ b/test/T490-parse-time-string.sh @@ -2,6 +2,10 @@ test_description="date/time parser module" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + # Sanity/smoke tests for the date/time parser independent of notmuch _date () { diff --git a/test/T550-db-features.sh b/test/T550-db-features.sh index 9d5a9e70..3048c7c4 100755 --- a/test/T550-db-features.sh +++ b/test/T550-db-features.sh @@ -3,6 +3,10 @@ test_description="database version and feature compatibility" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + test_begin_subtest "future database versions abort open" ${TEST_DIRECTORY}/make-db-version ${MAIL_DIR} 9999 "" output=$(notmuch search x 2>&1 | sed 's/\(database at\) .*/\1 FILENAME/') diff --git a/test/T566-lib-message.sh b/test/T566-lib-message.sh index 7f0e8eb0..69051937 100755 --- a/test/T566-lib-message.sh +++ b/test/T566-lib-message.sh @@ -3,6 +3,10 @@ test_description="API tests for notmuch_message_*" . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED}" ]; then + test_done +fi + add_email_corpus test_begin_subtest "building database" diff --git a/test/T592-thread-breakage.sh b/test/T592-thread-breakage.sh index 92a70e3e..2334fcaf 100755 --- a/test/T592-thread-breakage.sh +++ b/test/T592-thread-breakage.sh @@ -21,6 +21,10 @@ test_description='thread breakage during reindexing' . $(dirname "$0")/test-lib.sh || exit 1 +if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_done +fi + message_a () { mkdir -p ${MAIL_DIR}/cur cat > ${MAIL_DIR}/cur/a <OUTPUT <018b1a8f2d1df62e804ce88b65401304832dfbbf.1346614915.git.jani@nikula.org> diff --git a/test/T800-asan.sh b/test/T800-asan.sh index 7c28dc7c..9ce6baa7 100755 --- a/test/T800-asan.sh +++ b/test/T800-asan.sh @@ -7,6 +7,11 @@ if [ "${NOTMUCH_HAVE_ASAN-0}" != "1" ]; then test_done fi +if [ -n "${LD_PRELOAD-}" ]; then + printf "Skipping due to ASAN LD_PRELOAD restrictions\n" + test_done +fi + add_email_corpus TEST_CFLAGS="${TEST_CFLAGS:-} -fsanitize=address" diff --git a/test/T850-git.sh b/test/T850-git.sh index be6e0cd4..831e4678 100755 --- a/test/T850-git.sh +++ b/test/T850-git.sh @@ -233,6 +233,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "invoke as nmbug sets defaults" +test_subtest_broken_for_installed "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^\(prefix\|repository\)' | notmuch_dir_sanitize > OUTPUT cat < EXPECTED prefix = notmuch:: @@ -241,6 +242,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "env variable NOTMUCH_GIT_DIR works when invoked as nmbug" +test_subtest_broken_for_installed NOTMUCH_GIT_DIR=`pwd`/foo "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT cat < EXPECTED repository = CWD/foo @@ -256,6 +258,7 @@ test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "env variable NOTMUCH_GIT_DIR overrides config when invoked as 'nmbug'" +test_subtest_broken_for_installed notmuch config set git.path `pwd`/bar NOTMUCH_GIT_DIR=`pwd`/remote.git "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT notmuch config set git.path @@ -274,6 +277,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as 'nmbug'" +test_subtest_broken_for_installed NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT cat < EXPECTED prefix = env:: @@ -281,6 +285,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as nmbug" +test_subtest_broken_for_installed NOTMUCH_GIT_PREFIX=foo:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT cat < EXPECTED prefix = foo:: @@ -288,6 +293,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "env variable NOTMUCH_GIT_PREFIX overrides config when invoked as 'nmbug'" +test_subtest_broken_for_installed notmuch config set git.tag_prefix config:: NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT notmuch config set git.path diff --git a/test/export-dirs.sh b/test/export-dirs.sh index 0a048e1f..9bc99a2c 100644 --- a/test/export-dirs.sh +++ b/test/export-dirs.sh @@ -24,7 +24,7 @@ find_builddir () { if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then export NOTMUCH_BUILDDIR="$(find_builddir "$(pwd)")" - if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then + if [ -z "${NOTMUCH_BUILDDIR}" -a "${NOTMUCH_TEST_INSTALLED-0}" = "0" ]; then echo "Run tests in a subdir of built notmuch tree." >&2 exit 1 fi diff --git a/test/notmuch-test b/test/notmuch-test index 14747bdb..5d27e4d1 100755 --- a/test/notmuch-test +++ b/test/notmuch-test @@ -20,6 +20,14 @@ fi set -eu +# Where to run the tests +# XXX FIXME this code is duplicated with test-lib.sh +if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then + TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test +else + TEST_DIRECTORY=$NOTMUCH_SRCDIR/test +fi + TESTS= for test in ${NOTMUCH_TESTS-}; do TESTS="$TESTS $NOTMUCH_SRCDIR/test/$test" @@ -80,7 +88,7 @@ for file in $TESTS do file=${file##*/} # drop leading path components file=${file%.sh} # drop trailing '.sh' - RESULT_FILES="$RESULT_FILES $NOTMUCH_BUILDDIR/test/test-results/$file" + RESULT_FILES="$RESULT_FILES $TEST_DIRECTORY/test-results/$file" done echo @@ -94,6 +102,6 @@ if [ -n "$META_FAILURE" ]; then fi # Clean up -rm -rf $NOTMUCH_BUILDDIR/test/test-results +rm -rf $TEST_DIRECTORY/test-results exit $ev diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh index 4d14e0b3..f5d72e12 100644 --- a/test/test-lib-common.sh +++ b/test/test-lib-common.sh @@ -24,7 +24,7 @@ # type die >/dev/null 2>&1 || die () { echo "$@" >&2; exit 1; } -if [[ -z "$NOTMUCH_SRCDIR" ]] || [[ -z "$NOTMUCH_BUILDDIR" ]]; then +if [[ -z "$NOTMUCH_SRCDIR" ]] || [ -z "${NOTMUCH_TEST_INSTALLED-}" -a -z "$NOTMUCH_BUILDDIR" ]; then echo "internal: srcdir or builddir not set" >&2 exit 1 fi @@ -61,7 +61,9 @@ LD_LIBRARY_PATH=${TEST_DIRECTORY%/*}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} export LD_LIBRARY_PATH # configure output -. "$NOTMUCH_BUILDDIR/sh.config" || exit 1 +if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then + . "$NOTMUCH_BUILDDIR/sh.config" || exit 1 +fi # load OS specifics if [[ -e "$NOTMUCH_SRCDIR/test/test-lib-$PLATFORM.sh" ]]; then @@ -315,7 +317,12 @@ export PATH MANPATH # Test repository test="tmp.$(basename "$0" .sh)" -TMP_DIRECTORY="$TEST_DIRECTORY/$test" +if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then + TMP_DIRECTORY="$TEST_DIRECTORY/$test" +else + TMP_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-$test.XXXXXX") +fi + test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY rm -rf "$TMP_DIRECTORY" || { GIT_EXIT_OK=t diff --git a/test/test-lib-emacs.sh b/test/test-lib-emacs.sh index ad4c4aeb..0ab58fc2 100644 --- a/test/test-lib-emacs.sh +++ b/test/test-lib-emacs.sh @@ -30,6 +30,7 @@ test_require_emacs () { # to the message and encrypting/signing. emacs_deliver_message () { local subject body smtp_dummy_pid smtp_dummy_port + test_subtest_broken_for_installed subject="$1" body="$2" shift 2 @@ -144,6 +145,13 @@ emacs_generate_script () { # Construct a little test script here for the benefit of the user, # (who can easily run "run_emacs" to get the same emacs environment # for investigating any failures). + if [ -z "${NOTMUCH_TEST_INSTALLED-}" ]; then + find_notmuch_el='--directory "$NOTMUCH_BUILDDIR/emacs"' + else + ### XXX FIXME: this should really use the installed emacs lisp files + find_notmuch_el='--directory "$NOTMUCH_SRCDIR/emacs"' + fi + cat <"$TMP_DIRECTORY/run_emacs" #!/bin/sh export PATH=$PATH @@ -158,8 +166,8 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG # # --load Force loading of notmuch.el and test-lib.el -exec ${TEST_EMACS} --quick \ - --directory "$NOTMUCH_BUILDDIR/emacs" --load notmuch.el \ +exec ${TEST_EMACS} ${find_notmuch_el} --quick \ + ${EXTRA_DIR} --load notmuch.el \ --directory "$NOTMUCH_SRCDIR/test" --load test-lib.el \ "\$@" EOF diff --git a/test/test-lib.sh b/test/test-lib.sh index f218fa03..059e110c 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -29,8 +29,8 @@ shopt -u xpg_echo # Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set. . $(dirname "$0")/export-dirs.sh || exit 1 -# It appears that people try to run tests without building... -if [[ ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then +# We need either a built tree, or a promise of an installed notmuch +if [ -z "${NOTMUCH_TEST_INSTALLED-}" -a ! -x "$NOTMUCH_BUILDDIR/notmuch" ]; then echo >&2 'You do not seem to have built notmuch yet.' exit 1 fi @@ -748,6 +748,12 @@ test_subtest_known_broken () { test_subtest_known_broken_=t } +test_subtest_broken_for_installed () { + if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + test_subtest_known_broken_=t + fi +} + test_subtest_broken_for_root () { if [ "$EUID" = "0" ]; then test_subtest_known_broken_=t @@ -929,11 +935,16 @@ make_shim () { } notmuch_with_shim () { - local base_name shim_file - base_name="$1" + local base_name shim_file notmuch_cmd + if [ -n "${NOTMUCH_TEST_INSTALLED-}" ]; then + notmuch_cmd="notmuch" + else + notmuch_cmd="notmuch-shared" + fi + base_name=$1 shift shim_file="${base_name}.so" - LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} notmuch-shared "$@" + LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} $notmuch_cmd "$@" } # Creates a script that counts how much time it is executed and calls @@ -985,7 +996,11 @@ test_init_ () { # Where to run the tests -TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test +if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then + TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test +else + TEST_DIRECTORY=$NOTMUCH_SRCDIR/test +fi . "$NOTMUCH_SRCDIR/test/test-lib-common.sh" || exit 1 From 08ca74d71531f7907d3db820c55d59259bb1cb5b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Apr 2023 11:26:27 -0300 Subject: [PATCH 59/90] debian: add autopkgtests We generate output to stderr for BROKEN tests, which are not failures, so tell the test runner not to fail because of output on stderr. --- debian/tests/control | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 debian/tests/control diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 00000000..87e00dc8 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,17 @@ +Test-command: env NOTMUCH_TEST_INSTALLED=1 TERM=dumb + NOTMUCH_HAVE_MAN=1 NOTMUCH_HAVE_SFSEXP=1 NOTMUCH_HAVE_XAPIAN_DB_RETRY_LOCK=1 + NOTMUCH_HAVE_PYTHON3_CFFI=1 NOTMUCH_HAVE_PYTHON3_PYTEST=1 + NOTMUCH_HAVE_ASAN=1 NOTMUCH_HAVE_TSAN=1 + ./test/notmuch-test +Restrictions: allow-stderr +Depends: @, + build-essential, + dtach, + emacs-nox, + gdb, + git, + gnupg, + gpgsm, + libtalloc-dev, + man, + xapian-tools From d93d49b6aed5b3f71651ffe79225da08c7d8f1aa Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jul 2023 09:08:00 -0300 Subject: [PATCH 60/90] lib/message: check message type before deleting document It isn't really clear how this worked before. Traversing the terms of a document after deleting it from the database seems likely to be undefined behaviour at best --- lib/message.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/message.cc b/lib/message.cc index 53f35dd1..46638f80 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -1397,14 +1397,15 @@ _notmuch_message_delete (notmuch_message_t *message) Xapian::PostingIterator thread_doc, thread_doc_end; Xapian::PostingIterator mail_doc, mail_doc_end; - message->notmuch->writable_xapian_db->delete_document (message->doc_id); - /* look for a non-ghost message in the same thread */ /* if this was a ghost to begin with, we are done */ private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost); if (private_status) return COERCE_STATUS (private_status, "Error trying to determine whether message was a ghost"); + + message->notmuch->writable_xapian_db->delete_document (message->doc_id); + if (is_ghost) return NOTMUCH_STATUS_SUCCESS; From b6f144abe1f5aa3519240cf52f4cb9907fefcd0e Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jul 2023 09:08:01 -0300 Subject: [PATCH 61/90] lib/n_d_remove_message: do not remove unique filename It is wasteful to remove a filename term when the whole message document is about to be removed from the database. Profiling with perf shows this takes a significant portion of the time when cleaning up removed files in the database. The logic of n_d_remove_message becomes a bit more convoluted here in order to make the change minimal. It is possible that this function can be further optimized, since the expansion of filename terms into filenames is probably not needed here. --- lib/database.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/database.cc b/lib/database.cc index 6987e2f4..737a3f30 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -1454,7 +1454,9 @@ notmuch_database_remove_message (notmuch_database_t *notmuch, &message); if (status == NOTMUCH_STATUS_SUCCESS && message) { - status = _notmuch_message_remove_filename (message, filename); + if (notmuch_message_count_files (message) > 1) { + status = _notmuch_message_remove_filename (message, filename); + } if (status == NOTMUCH_STATUS_SUCCESS) status = _notmuch_message_delete (message); else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) From 1719b9e568fb944ffb91126b164a18a30a415d0a Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 20 Aug 2023 14:32:02 -0300 Subject: [PATCH 62/90] test/emacs: adapt to breaking change in Gnus defaults As of Emacs 29.1, In-Reply-To is in the default value for message-hidden-headers. We actually want to see that in the test suite, so remove it again. To future proof the tests, fix a default value for message-hidden-headers specifically for the test suite. --- test/test-lib.el | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test-lib.el b/test/test-lib.el index 79a9d4d6..236dd99e 100644 --- a/test/test-lib.el +++ b/test/test-lib.el @@ -205,3 +205,8 @@ running, quit if it terminated." ;; environments (setq mm-text-html-renderer 'html2text) + +;; Set our own default for message-hidden-headers, to avoid tests +;; breaking when the Emacs default changes. +(setq message-hidden-headers + '("^References:" "^Face:" "^X-Face:" "^X-Draft-From:")) From df45194d5ffc110d69811eecd50f744c2b8c3805 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 20 Aug 2023 12:10:56 -0300 Subject: [PATCH 63/90] CLI/config: simulate top level comments when creating config According to discussion on https://gitlab.gnome.org/GNOME/glib/-/issues/3078 it looks like upstream will stop supporting top of file comments. It is questionable whether we really need this feature, but for now update notmuch-config to simulate it. --- notmuch-config.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/notmuch-config.c b/notmuch-config.c index e9456d79..8123e438 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -278,18 +278,24 @@ notmuch_conffile_open (notmuch_database_t *notmuch, return NULL; } - if (config->is_new) - g_key_file_set_comment (config->key_file, NULL, NULL, - toplevel_config_comment, NULL); - for (size_t i = 0; i < ARRAY_SIZE (group_comment_table); i++) { const char *name = group_comment_table[i].group_name; if (! g_key_file_has_group (config->key_file, name)) { /* Force group to exist before adding comment */ g_key_file_set_value (config->key_file, name, "dummy_key", "dummy_val"); g_key_file_remove_key (config->key_file, name, "dummy_key", NULL); - g_key_file_set_comment (config->key_file, name, NULL, - group_comment_table[i].comment, NULL); + if (config->is_new && (i == 0) ) { + const char *comment; + + comment = talloc_asprintf (config, "%s\n%s", + toplevel_config_comment, + group_comment_table[i].comment); + g_key_file_set_comment (config->key_file, name, NULL, comment, + NULL); + } else { + g_key_file_set_comment (config->key_file, name, NULL, + group_comment_table[i].comment, NULL); + } } } return config; From 368e0f61d5498ceae180e8349e848a4904008b0b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 20 Aug 2023 12:23:19 -0300 Subject: [PATCH 64/90] test/setup: ignore blank lines in generated config The presense of the blank lines between sections depends on the version of glib. Strip them before comparison. --- test/T040-setup.sh | 3 ++- test/setup.expected-output/config-with-comments | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/test/T040-setup.sh b/test/T040-setup.sh index 10b29ec3..39846d34 100755 --- a/test/T040-setup.sh +++ b/test/T040-setup.sh @@ -21,7 +21,8 @@ baz EOF expected_dir=$NOTMUCH_SRCDIR/test/setup.expected-output -test_expect_equal_file ${expected_dir}/config-with-comments new-notmuch-config +sed '/^$/d' < new-notmuch-config > filtered-config +test_expect_equal_file ${expected_dir}/config-with-comments filtered-config test_begin_subtest "setup consistent with config-set for single items" # note this relies on the config state from the previous test. diff --git a/test/setup.expected-output/config-with-comments b/test/setup.expected-output/config-with-comments index d8397714..d925acea 100644 --- a/test/setup.expected-output/config-with-comments +++ b/test/setup.expected-output/config-with-comments @@ -1,7 +1,6 @@ # .notmuch-config - Configuration file for the notmuch mail system # # For more information about notmuch, see https://notmuchmail.org - # Database configuration # # The only value supported here is 'path' which should be the top-level @@ -12,7 +11,6 @@ # [database] path=/path/to/maildir - # User configuration # # Here is where you can let notmuch know how you would like to be @@ -32,7 +30,6 @@ path=/path/to/maildir name=Test Suite primary_email=test.suite@example.com other_email=another.suite@example.com - # Configuration for "notmuch new" # # The following options are supported here: @@ -49,7 +46,6 @@ other_email=another.suite@example.com # [new] tags=foo;bar; - # Search configuration # # The following option is supported here: @@ -61,7 +57,6 @@ tags=foo;bar; # [search] exclude_tags=baz - # Maildir compatibility configuration # # The following option is supported here: From 2f874fb2cc65e3486a47238f039aa80b990f8150 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 24 Aug 2023 07:58:19 -0300 Subject: [PATCH 65/90] version: set up 0.38~rc0 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 943f335c..298a533f 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.37' +__VERSION__ = '0.38~rc0' SOVERSION = '5' diff --git a/version.txt b/version.txt index c128d4d9..bce5c958 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.37 +0.38~rc0 From aa5674fce027d55850f49cedcc52005b1566cf6b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 24 Aug 2023 07:58:51 -0300 Subject: [PATCH 66/90] NEWS: start NEWS for 0.38 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index b83551f1..644453f2 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +Notmuch 0.38 (UNRELEASED) +========================= + Notmuch 0.37 (2022-08-21) ========================= From 695f663f810ec1586aca8bfb09883c4d772d1b3b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 24 Aug 2023 08:00:29 -0300 Subject: [PATCH 67/90] debian: changelog for 0.38~rc0-1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 846ccd2e..300064d4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +notmuch (0.38~rc0-1) unstable; urgency=medium + + * New upstream release candidate + + -- David Bremner Thu, 24 Aug 2023 07:59:59 -0300 + notmuch (0.37-1) unstable; urgency=medium * New upstream release. From 5ed121bed92601e5512258f5a5f7bf9dbb705fd8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 24 Aug 2023 08:03:46 -0300 Subject: [PATCH 68/90] doc: update copyright date --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index e23cb7d7..7ac13a5d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -16,7 +16,7 @@ master_doc = 'index' # General information about the project. project = u'notmuch' -copyright = u'2009-2022, Carl Worth and many others' +copyright = u'2009-2023, Carl Worth and many others' location = os.path.dirname(__file__) From 8c8fda965f17e90046de51743c8d545653085697 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 24 Aug 2023 10:56:33 -0300 Subject: [PATCH 69/90] debian: set suite to experimental --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 300064d4..864b0423 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ -notmuch (0.38~rc0-1) unstable; urgency=medium +notmuch (0.38~rc0-1) experimental; urgency=medium * New upstream release candidate - -- David Bremner Thu, 24 Aug 2023 07:59:59 -0300 + -- David Bremner Thu, 24 Aug 2023 10:56:06 -0300 notmuch (0.37-1) unstable; urgency=medium From 115d4d69ebbab213baae672ac4e81947877f364b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Aug 2023 07:45:36 -0300 Subject: [PATCH 70/90] test: minimize impact of native compilation. Native compilation is kindof useless in the test suite because we throw away the cache after every subtest. The test suite could in principle share an eln cache within a given test file; for now try to minimize the amount of native-compilation. There is an intermittent bug where emacs loses track of its default-directory; I suspect (but have no proof) that bug is related to native compilation and/or race conditions. This patch seems to prevent that bug (or at least reduce its frequency). --- test/test-lib.el | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test-lib.el b/test/test-lib.el index 236dd99e..4cfb8ef1 100644 --- a/test/test-lib.el +++ b/test/test-lib.el @@ -22,6 +22,15 @@ ;;; Code: +;; minimize impact of native compilation on the test suite. +;; These are the Emacs 29.1 version of the variables. +;; Leave trampolines enabled per Emacs upstream recommendations. +;; It is important to set these variables before loading any +;; .elc files. +(setq native-comp-jit-compilation nil) +(setq native-comp-speed -1) +(setq native-comp-async-jobs-number 1) + (require 'cl-lib) ;; Ensure that the dynamic variables that are defined by this library From 25c933d7e6aeff5267112ce926335d6fd712da4b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Aug 2023 08:22:33 -0300 Subject: [PATCH 71/90] debian: use architecture.mk I plan to add further architecture dependent checks, which makes this preferable to directly calling dpkg-architecture. --- debian/rules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index f9196b82..dfcf2cc8 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,5 @@ #!/usr/bin/make -f +include /usr/share/dpkg/architecture.mk export DEB_BUILD_MAINT_OPTIONS = hardening=+all @@ -7,7 +8,7 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all override_dh_auto_configure: BASHCMD=/bin/bash ./configure --prefix=/usr \ - --libdir=/usr/lib/$$(dpkg-architecture -q DEB_TARGET_MULTIARCH) \ + --libdir=/usr/lib/${DEB_TARGET_MULTIARCH} \ --includedir=/usr/include \ --mandir=/usr/share/man \ --infodir=/usr/share/info \ From 90c61828256bb4592cf3473599add36e9f43d5e5 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Aug 2023 08:28:33 -0300 Subject: [PATCH 72/90] debian: skip T810-tsan on ppc64el Hopefully just a temporary measure. --- debian/rules | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian/rules b/debian/rules index dfcf2cc8..8bf97439 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,10 @@ #!/usr/bin/make -f include /usr/share/dpkg/architecture.mk +ifeq ($(DEB_HOST_ARCH),ppc64el) + export NOTMUCH_SKIP_TESTS = T810-tsan +endif + export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: From 240e9fff90d30efb3c53f1c08917740939a6666d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Aug 2023 08:29:45 -0300 Subject: [PATCH 73/90] version: bump to 0.38~rc1 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 298a533f..5b6b5da7 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.38~rc0' +__VERSION__ = '0.38~rc1' SOVERSION = '5' diff --git a/version.txt b/version.txt index bce5c958..158d0032 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.38~rc0 +0.38~rc1 From 9a1126241d6c4a12610b5de61044606352ca4712 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 26 Aug 2023 08:31:49 -0300 Subject: [PATCH 74/90] debian: changelog for 0.38~rc1-1 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index 864b0423..bcbf63eb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +notmuch (0.38~rc1-1) experimental; urgency=medium + + * New upstream release candidate + * Hopefully reduce/eliminate intermittent failures of T460 by + controlling Emacs native compilation. + * Disable T810-tsan on ppc64el + + -- David Bremner Sat, 26 Aug 2023 08:31:21 -0300 + notmuch (0.38~rc0-1) experimental; urgency=medium * New upstream release candidate From a84dc2f7e64e70bd9696efe933671b2cbdfcd329 Mon Sep 17 00:00:00 2001 From: Michael J Gruber Date: Sat, 26 Aug 2023 16:53:12 +0200 Subject: [PATCH 75/90] compat: probe for strcasestr more thoroughly Depending on compiler (gcc, g++, clang) and standard options (c99, c11), string.h may or may not include strings.h, leading to possibly missing or conflicting declarations of strcasestr. Include both so that both detection and compilation phases use the same (possibly optimised) implementations. Suggested-by: Thomas Schneider Suggested-by: Florian Weimer Suggested-by: Tomi Ollila --- compat/have_strcasestr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compat/have_strcasestr.c b/compat/have_strcasestr.c index 3cd1838d..8e004572 100644 --- a/compat/have_strcasestr.c +++ b/compat/have_strcasestr.c @@ -1,5 +1,6 @@ #define _GNU_SOURCE -#include +#include /* strcasecmp() in POSIX */ +#include /* strcasecmp() in *BSD */ int main () From c1a23a64ae06decdffea6caeaba5270854fa0ddd Mon Sep 17 00:00:00 2001 From: Kevin Boulain Date: Wed, 30 Aug 2023 14:52:55 +0200 Subject: [PATCH 76/90] test: suppress all interceptors in glib On ppc64el, races are detected by TSan: WARNING: ThreadSanitizer: data race (pid=4520) Read of size 8 at 0x7ffff20016c0 by thread T1: #0 strlen ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:386 (libtsan.so.2+0x77c0c) #1 strlen ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:378 (libtsan.so.2+0x77c0c) #2 g_strdup ../../../glib/gstrfuncs.c:362 (libglib-2.0.so.0+0xa4ac4) Previous write of size 8 at 0x7ffff20016c0 by thread T2: #0 malloc ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:647 (libtsan.so.2+0x471f0) #1 g_malloc ../../../glib/gmem.c:130 (libglib-2.0.so.0+0x7bb68) Location is heap block of size 20 at 0x7ffff20016c0 allocated by thread T2: #0 malloc ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:647 (libtsan.so.2+0x471f0) #1 g_malloc ../../../glib/gmem.c:130 (libglib-2.0.so.0+0x7bb68) This appears to be a false positive in GLib, as explained at https://gitlab.gnome.org/GNOME/glib/-/issues/1672#note_1831968 In short, a call to fstat fails under TSan and GLib's g_sterror will intern the error message, which will be reused by other threads. Since upstream appears to be aware that GLib doesn't play nicely with TSan, suppress everything coming from the library instead of maintaining a fine grained list. Reported at https://buildd.debian.org/status/fetch.php?pkg=notmuch&arch=ppc64el&ver=0.38%7Erc0-1&stamp=1692959868&raw=0 --- test/T810-tsan.suppressions | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/T810-tsan.suppressions b/test/T810-tsan.suppressions index dbd16a94..80dc062f 100644 --- a/test/T810-tsan.suppressions +++ b/test/T810-tsan.suppressions @@ -1,5 +1,3 @@ # It's unclear how TSan-friendly GLib is: # https://gitlab.gnome.org/GNOME/glib/-/issues/1672 -race:g_rw_lock_reader_lock -# https://gitlab.gnome.org/GNOME/glib/-/issues/1952 -race:g_slice_alloc0 +called_from_lib:libglib*.so From ed5b8f65fc4546fbe9520b3d4d68abb92d3ee92d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 3 Sep 2023 09:09:22 -0300 Subject: [PATCH 77/90] version: bump to 0.31~rc2 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 5b6b5da7..8e62314e 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.38~rc1' +__VERSION__ = '0.38~rc2' SOVERSION = '5' diff --git a/version.txt b/version.txt index 158d0032..2139400c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.38~rc1 +0.38~rc2 From 2c81de8f5f43952dc40aeb5b12cfc0b3ae25a14c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 3 Sep 2023 09:11:53 -0300 Subject: [PATCH 78/90] debian: changelog for 0.38~rc2-1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index bcbf63eb..a3b0dcb2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +notmuch (0.38~rc2-1) experimental; urgency=medium + + * New upstream release candidate + + -- David Bremner Sun, 03 Sep 2023 09:10:24 -0300 + notmuch (0.38~rc1-1) experimental; urgency=medium * New upstream release candidate From 8dc8cedb3b39ebb5753415ae559a03189eebe558 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 9 Sep 2023 10:59:20 -0300 Subject: [PATCH 79/90] NEWS: NEWS for 0.38 --- NEWS | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/NEWS b/NEWS index 644453f2..b63ffbde 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,75 @@ Notmuch 0.38 (UNRELEASED) ========================= +General +------- + +Support relative lastmod queries (see notmuch-sexp-queries(7) and +notmuch-search-terms(7) for details). + +Support indexing of designated attachments as text (see +notmuch-config(1) for details). + +CLI +--- + +Add options --offset and --limit to notmuch-show(1). + +Emacs +----- + +New commands notmuch-search-edit-search and notmuch-tree-edit-search. + +Introduce notmuch-tree-outline-mode. + +Some compatibility fixes for Emacs 29. At least one issue (hiding +images) remains in 0.38. + +Support completion when piping to external command. + +Fix regression in updating tag display introduced by 0.37. + +Library +------- + +Fix bug creating database when database.path is not set. + +Incremental performance improvements for message deletion. + +Catch Xapian exceptions when deleting messages. + +Sync removed message properties to the database. + +Replace use of thread-unsafe Query::MatchAll in the infix query +parser. + +Notmuch-Mutt +------------ + +Be more careful when clearing the results directory. + +Ruby +---- + +Use database_open_with_config, and provide compatible path search +semantics. + +Bugfix for query.get_sort + +Test Suite +---------- + +Support testing installed version of notmuch. + +Adapt to some breaking changes in glib handling of init files. + +Replace OpenPGP key used in test suite. + +Performance Tests +----------------- + +Update signatures for performance test corpus. + Notmuch 0.37 (2022-08-21) ========================= From 139ea43f3d14cc88151040c172efb01814984e73 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 11 Sep 2023 19:56:21 -0300 Subject: [PATCH 80/90] version: bump to 0.38 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 8e62314e..c847a1a3 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.38~rc2' +__VERSION__ = '0.38' SOVERSION = '5' diff --git a/version.txt b/version.txt index 2139400c..1cbc8125 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.38~rc2 +0.38 From db8bf6f7adcc5dcfa29b27f8584c785c495f001a Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 11 Sep 2023 20:19:25 -0300 Subject: [PATCH 81/90] debian: changelog for 0.38-1 --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index a3b0dcb2..9c975599 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +notmuch (0.38-1) unstable; urgency=medium + + * New upstream release + * Bug fix: "FTBFS: 6 tests failed.", thanks to Aurelien Jarno (Closes: + #1051111). + + -- David Bremner Mon, 11 Sep 2023 20:02:39 -0300 + notmuch (0.38~rc2-1) experimental; urgency=medium * New upstream release candidate From 32f24b97d053bbe9fee31dcc3d6582b6d87105e4 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 82/90] bindings/python-cffi: clean up _notmuch_config.py _notmuch_config.py is generated by configure, and cannot be cleaned up by the current python build system, since it is imported as a module by that same build system. Use DISTCLEAN rather than CLEAN for consistency with other configure related things. --- bindings/Makefile.local | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bindings/Makefile.local b/bindings/Makefile.local index 7b10af08..25ba1920 100644 --- a/bindings/Makefile.local +++ b/bindings/Makefile.local @@ -34,3 +34,6 @@ CLEAN += $(patsubst %,$(dir)/ruby/%, \ CLEAN += bindings/ruby/.vendorarchdir.time $(dir)/ruby.stamp CLEAN += bindings/python-cffi/build $(dir)/python-cffi.stamp + +DISTCLEAN += bindings/python-cffi/_notmuch_config.py + From 14e05a3eeeb1ceabe7eb9389e161334bfebb48db Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 83/90] debian: rely on main notmuch (dist)clean. The python setuputils clean relys on including _notmuch_config.py, which is cleaned up. Rather than relying on careful ordering, just do all the cleaning from the GNU Make based build system. --- debian/rules | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/rules b/debian/rules index 8bf97439..a77ffa15 100755 --- a/debian/rules +++ b/debian/rules @@ -29,7 +29,6 @@ override_dh_auto_build: override_dh_auto_clean: dh_auto_clean PYBUILD_NAME=notmuch dh_auto_clean --buildsystem=pybuild --sourcedirectory bindings/python - PYBUILD_NAME=notmuch2 dh_auto_clean --buildsystem=pybuild --sourcedirectory bindings/python-cffi dh_auto_clean --sourcedirectory bindings/ruby $(MAKE) -C contrib/notmuch-mutt clean From b21747e0c3b7e2a779f5fad10cef4c8fcf233ae4 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 84/90] build: clean up __pycache__ directories These are generated indirectly by certain uses of python in the build. --- bindings/Makefile.local | 1 + doc/Makefile.local | 2 ++ 2 files changed, 3 insertions(+) diff --git a/bindings/Makefile.local b/bindings/Makefile.local index 25ba1920..c82a91c6 100644 --- a/bindings/Makefile.local +++ b/bindings/Makefile.local @@ -34,6 +34,7 @@ CLEAN += $(patsubst %,$(dir)/ruby/%, \ CLEAN += bindings/ruby/.vendorarchdir.time $(dir)/ruby.stamp CLEAN += bindings/python-cffi/build $(dir)/python-cffi.stamp +CLEAN += bindings/python-cffi/__pycache__ DISTCLEAN += bindings/python-cffi/_notmuch_config.py diff --git a/doc/Makefile.local b/doc/Makefile.local index 51c729cf..aafa77a0 100644 --- a/doc/Makefile.local +++ b/doc/Makefile.local @@ -159,3 +159,5 @@ $(dir)/config.dox: version.stamp CLEAN := $(CLEAN) $(DOCBUILDDIR) $(DOCBUILDDIR)/.roff.stamp $(DOCBUILDDIR)/.texi.stamp CLEAN := $(CLEAN) $(DOCBUILDDIR)/.html.stamp $(DOCBUILDDIR)/.info.stamp CLEAN := $(CLEAN) $(MAN_GZIP_FILES) $(MAN_ROFF_FILES) $(dir)/conf.pyc $(dir)/config.dox + +CLEAN := $(CLEAN) $(dir)/__pycache__ From 972283777a166faba24abf6dffbc95e3b0146cf2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 85/90] NEWS: set release date for 0.38 --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index b63ffbde..621b5929 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -Notmuch 0.38 (UNRELEASED) +Notmuch 0.38 (2023-09-12) ========================= General From a76c16b75eae1d1e4aa2dd2e8ab46117c92bf7c2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 86/90] NEWS: quote function name This avoids spurious underline/italics when exporting to markdown. --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 621b5929..00f4fe47 100644 --- a/NEWS +++ b/NEWS @@ -51,7 +51,7 @@ Be more careful when clearing the results directory. Ruby ---- -Use database_open_with_config, and provide compatible path search +Use `database_open_with_config`, and provide compatible path search semantics. Bugfix for query.get_sort From c31f8c0b3beb98be6a9ba8a969496a075a331162 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 12 Sep 2023 08:33:35 -0300 Subject: [PATCH 87/90] debian: note introduction of autopkgtests Having autopkgtests changes Debian package migration (for better and worse), so make a note when they were introduced to unstable. --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 9c975599..a4363281 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,9 @@ notmuch (0.38-1) unstable; urgency=medium * New upstream release * Bug fix: "FTBFS: 6 tests failed.", thanks to Aurelien Jarno (Closes: #1051111). + * Run most of upstream test suite as autopkgtests - -- David Bremner Mon, 11 Sep 2023 20:02:39 -0300 + -- David Bremner Tue, 12 Sep 2023 08:33:24 -0300 notmuch (0.38~rc2-1) experimental; urgency=medium From 60b5ea319a45900b1e610715481faaa339ea3a4c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 10 Sep 2023 20:13:13 -0300 Subject: [PATCH 88/90] bindings/python-cffi: clean up notmuch2.egg-info This is created (at least) by the Debian build, but there seems no harm in cleaning it for everyone. --- bindings/Makefile.local | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/Makefile.local b/bindings/Makefile.local index c82a91c6..9875123a 100644 --- a/bindings/Makefile.local +++ b/bindings/Makefile.local @@ -36,5 +36,5 @@ CLEAN += bindings/ruby/.vendorarchdir.time $(dir)/ruby.stamp CLEAN += bindings/python-cffi/build $(dir)/python-cffi.stamp CLEAN += bindings/python-cffi/__pycache__ -DISTCLEAN += bindings/python-cffi/_notmuch_config.py - +DISTCLEAN += bindings/python-cffi/_notmuch_config.py \ + bindings/python-cffi/notmuch2.egg-info From 40a44bfe6101da7bc4285c84166dd0f90a9b2baf Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 13 Sep 2023 19:58:39 -0300 Subject: [PATCH 89/90] debian: upload 0.38-2: disable most autopkgtests --- debian/changelog | 9 +++++++++ debian/tests/control | 1 + 2 files changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index a4363281..c2c2f6ab 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +notmuch (0.38-2) unstable; urgency=medium + + * Restrict autopkgtests to amd64 and aarch64. There are failures in + remaining architectures, but the same tests pass at build time, so any + bugs are probably related to either the autopkgtest environment, or + the (new) upstream test runner for installed notmuch. + + -- David Bremner Wed, 13 Sep 2023 19:55:00 -0300 + notmuch (0.38-1) unstable; urgency=medium * New upstream release diff --git a/debian/tests/control b/debian/tests/control index 87e00dc8..80be1deb 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -4,6 +4,7 @@ Test-command: env NOTMUCH_TEST_INSTALLED=1 TERM=dumb NOTMUCH_HAVE_ASAN=1 NOTMUCH_HAVE_TSAN=1 ./test/notmuch-test Restrictions: allow-stderr +Architecture: amd64, arm64 Depends: @, build-essential, dtach, From d4e0aaa76bd9e7a9e36abf47dc9ad3ea8bc10334 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 16 Sep 2023 11:15:46 -0300 Subject: [PATCH 90/90] devel/nmweb: read mail files in binary mode. "ju" reported on IRC that browsing https://nmbug.notmuchmail.org/nmweb/show/20160719094205.qmf5sjnja6crt5t3%40gotlib crashed. The underlying issue is that python3 defaults to utf8 decoding files unless they are opened in binary mode. The file in question (in the nmbug archive; it depends a bit on the routing the message took) has Content-Type: text/plain; charset=iso-8859-1 Content-Transfer-Encoding: 8bit and some of it is not valid utf8. --- devel/notmuch-web/nmweb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devel/notmuch-web/nmweb.py b/devel/notmuch-web/nmweb.py index b0d4d5cd..e0e87b49 100755 --- a/devel/notmuch-web/nmweb.py +++ b/devel/notmuch-web/nmweb.py @@ -207,7 +207,7 @@ env.globals['thread_nav'] = thread_nav def format_message(nm_msg, mid): fn = list(nm_msg.filenames())[0] - msg = MaildirMessage(open(fn)) + msg = MaildirMessage(open(fn, 'rb')) return format_message_walk(msg, mid) def decodeAnyway(txt, charset='ascii'):