From e0834e376a1078fc3a79978cf36a94785cdb2d30 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 13 Oct 2021 17:03:10 +0300 Subject: [PATCH 001/113] lib: fix commented out NOTMUCH_DEPRECATED() Remove the comment markers from the placeholder NOTMUCH_DEPRECATED(), added in commit e5f3c3ed5024 ("lib: add stub for notmuch_database_open_with_config"). --- lib/notmuch.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/notmuch.h b/lib/notmuch.h index 074fc682..cb721b27 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -323,7 +323,7 @@ typedef enum { * config_path="" and error_message=NULL * @deprecated Deprecated as of libnotmuch 5.4 (notmuch 0.32) */ -/* NOTMUCH_DEPRECATED(5, 4) */ +NOTMUCH_DEPRECATED(5, 4) notmuch_status_t notmuch_database_open (const char *path, notmuch_database_mode_t mode, @@ -335,7 +335,7 @@ notmuch_database_open (const char *path, * @deprecated Deprecated as of libnotmuch 5.4 (notmuch 0.32) * */ -/* NOTMUCH_DEPRECATED(5, 4) */ +NOTMUCH_DEPRECATED(5, 4) notmuch_status_t notmuch_database_open_verbose (const char *path, notmuch_database_mode_t mode, From 6987286a5b562709c1de583db66673c202fd926c Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 13 Oct 2021 17:02:16 +0300 Subject: [PATCH 002/113] lib: remove enum names from typedefs There are some enum typedefs with the enum name: typedef enum _name_t { ... } name_t; We don't need or use the enum names _name_t for anything, and not all of the enum typedefs have them. We have the typedefs specifically to use the typedef name. Use the anonymous enum in the typedefs: typedef enum { ... } name_t; --- lib/database-private.h | 2 +- lib/notmuch-private.h | 4 ++-- lib/notmuch.h | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/database-private.h b/lib/database-private.h index 8b9d67fe..7a045051 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -160,7 +160,7 @@ operator&= (_notmuch_features &a, _notmuch_features b) /* * Configuration options for xapian database fields */ -typedef enum notmuch_field_flags { +typedef enum { NOTMUCH_FIELD_NO_FLAGS = 0, NOTMUCH_FIELD_EXTERNAL = 1 << 0, NOTMUCH_FIELD_PROBABILISTIC = 1 << 1, diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index 093c29b1..e9ce74a4 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -121,7 +121,7 @@ typedef enum { */ #define NOTMUCH_MESSAGE_ID_MAX (200 - sizeof (NOTMUCH_METADATA_THREAD_ID_PREFIX)) -typedef enum _notmuch_private_status { +typedef enum { /* First, copy all the public status values. */ NOTMUCH_PRIVATE_STATUS_SUCCESS = NOTMUCH_STATUS_SUCCESS, NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY = NOTMUCH_STATUS_OUT_OF_MEMORY, @@ -173,7 +173,7 @@ typedef enum _notmuch_private_status { (notmuch_status_t) private_status) /* Flags shared by various lookup functions. */ -typedef enum _notmuch_find_flags { +typedef enum { /* Lookup without creating any documents. This is the default * behavior. */ NOTMUCH_FIND_LOOKUP = 0, diff --git a/lib/notmuch.h b/lib/notmuch.h index cb721b27..19391614 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -112,7 +112,7 @@ typedef int notmuch_bool_t; * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function * completed without error. Any other value indicates an error. */ -typedef enum _notmuch_status { +typedef enum { /** * No error occurred. */ @@ -1678,7 +1678,7 @@ notmuch_message_reindex (notmuch_message_t *message, /** * Message flags. */ -typedef enum _notmuch_message_flag { +typedef enum { NOTMUCH_MESSAGE_FLAG_MATCH, NOTMUCH_MESSAGE_FLAG_EXCLUDED, @@ -2524,7 +2524,7 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); /** * Configuration keys known to libnotmuch */ -typedef enum _notmuch_config_key { +typedef enum { NOTMUCH_CONFIG_FIRST, NOTMUCH_CONFIG_DATABASE_PATH = NOTMUCH_CONFIG_FIRST, NOTMUCH_CONFIG_MAIL_ROOT, From f316f7ef6af98b8f89f094dde24e36b837b3c5f2 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 13 Oct 2021 17:02:17 +0300 Subject: [PATCH 003/113] cli: remove enum names from typedefs There are some enum typedefs with the enum name: typedef enum _name_t { ... } name_t; We don't need or use the enum names _name_t for anything, and not all of the enum typedefs have them. We have the typedefs specifically to use the typedef name. Use the anonymous enum in the typedefs: typedef enum { ... } name_t; --- notmuch-client.h | 4 ++-- util/hex-escape.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notmuch-client.h b/notmuch-client.h index 96d81166..82ae44e4 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -426,13 +426,13 @@ mime_node_seek_dfs (mime_node_t *node, int n); const _notmuch_message_crypto_t * mime_node_get_message_crypto_status (mime_node_t *node); -typedef enum dump_formats { +typedef enum { DUMP_FORMAT_AUTO, DUMP_FORMAT_BATCH_TAG, DUMP_FORMAT_SUP } dump_format_t; -typedef enum dump_includes { +typedef enum { DUMP_INCLUDE_TAGS = 1, DUMP_INCLUDE_CONFIG = 2, DUMP_INCLUDE_PROPERTIES = 4 diff --git a/util/hex-escape.h b/util/hex-escape.h index 8703334c..83a4c6f1 100644 --- a/util/hex-escape.h +++ b/util/hex-escape.h @@ -5,7 +5,7 @@ extern "C" { #endif -typedef enum hex_status { +typedef enum { HEX_SUCCESS = 0, HEX_SYNTAX_ERROR, HEX_OUT_OF_MEMORY From c128c995bc3e7474d4086af7e2066f60f54329b2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 12 Jun 2021 10:26:16 -0300 Subject: [PATCH 004/113] lib: make indexopts pointers opaque There is no reason for anything outside the indexopts.c compilation unit to have access to structure members. --- lib/indexopts.c | 4 ++++ lib/notmuch-private.h | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/indexopts.c b/lib/indexopts.c index 4a860858..2ffd1942 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -20,6 +20,10 @@ #include "notmuch-private.h" +struct _notmuch_indexopts { + _notmuch_crypto_t crypto; +}; + notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db) { diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index e9ce74a4..3cc79bc4 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -711,9 +711,7 @@ _notmuch_thread_create (void *ctx, /* indexopts.c */ -struct _notmuch_indexopts { - _notmuch_crypto_t crypto; -}; +struct _notmuch_indexopts; #define CONFIG_HEADER_PREFIX "index.header." From 2ce6c76a61c907b5d1d508e12255e83d9256937c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 12 Jun 2021 10:26:17 -0300 Subject: [PATCH 005/113] CLI: move indexopts variable out of shared options block This reduces the amount of global state. Furthermore, index options can be set (in principle) in several ways, not just in the one function for processing indexing command line options. --- notmuch-client.h | 3 +-- notmuch-insert.c | 6 ++++-- notmuch-new.c | 7 +++++-- notmuch-reindex.c | 5 +++-- notmuch.c | 13 +++++-------- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/notmuch-client.h b/notmuch-client.h index 82ae44e4..de318e1f 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -499,11 +499,10 @@ int notmuch_minimal_options (const char *subcommand_name, struct _notmuch_client_indexing_cli_choices { int decrypt_policy; bool decrypt_policy_set; - notmuch_indexopts_t *opts; }; extern struct _notmuch_client_indexing_cli_choices indexing_cli_choices; extern const notmuch_opt_desc_t notmuch_shared_indexing_options []; notmuch_status_t -notmuch_process_shared_indexing_options (notmuch_database_t *notmuch); +notmuch_process_shared_indexing_options (notmuch_indexopts_t *opts); #endif diff --git a/notmuch-insert.c b/notmuch-insert.c index 72e2e35f..214d4d03 100644 --- a/notmuch-insert.c +++ b/notmuch-insert.c @@ -461,6 +461,8 @@ notmuch_insert_command (notmuch_database_t *notmuch, int argc, char *argv[]) char *maildir; char *newpath; int opt_index; + notmuch_indexopts_t *indexopts = notmuch_database_get_default_indexopts (notmuch); + void *local = talloc_new (NULL); notmuch_opt_desc_t options[] = { @@ -550,7 +552,7 @@ notmuch_insert_command (notmuch_database_t *notmuch, int argc, char *argv[]) return EXIT_FAILURE; } - status = notmuch_process_shared_indexing_options (notmuch); + status = notmuch_process_shared_indexing_options (indexopts); if (status != NOTMUCH_STATUS_SUCCESS) { fprintf (stderr, "Error: Failed to process index options. (%s)\n", notmuch_status_to_string (status)); @@ -558,7 +560,7 @@ notmuch_insert_command (notmuch_database_t *notmuch, int argc, char *argv[]) } /* Index the message. */ - status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexing_cli_choices.opts); + status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexopts); /* Commit changes. */ close_status = notmuch_database_close (notmuch); diff --git a/notmuch-new.c b/notmuch-new.c index b7a5f2ea..5b8fa340 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -45,6 +45,7 @@ typedef struct { const char *db_path; const char *mail_root; + notmuch_indexopts_t *indexopts; int output_is_a_tty; enum verbosity verbosity; bool debug; @@ -376,7 +377,7 @@ add_file (notmuch_database_t *notmuch, const char *filename, if (status) goto DONE; - status = notmuch_database_index_file (notmuch, filename, indexing_cli_choices.opts, &message); + status = notmuch_database_index_file (notmuch, filename, state->indexopts, &message); switch (status) { /* Success. */ case NOTMUCH_STATUS_SUCCESS: @@ -1150,6 +1151,8 @@ notmuch_new_command (notmuch_database_t *notmuch, int argc, char *argv[]) else if (verbose) add_files_state.verbosity = VERBOSITY_VERBOSE; + add_files_state.indexopts = notmuch_database_get_default_indexopts (notmuch); + add_files_state.new_tags = notmuch_config_get_values (notmuch, NOTMUCH_CONFIG_NEW_TAGS); if (print_status_database ( @@ -1217,7 +1220,7 @@ notmuch_new_command (notmuch_database_t *notmuch, int argc, char *argv[]) if (notmuch == NULL) return EXIT_FAILURE; - status = notmuch_process_shared_indexing_options (notmuch); + status = notmuch_process_shared_indexing_options (add_files_state.indexopts); if (status != NOTMUCH_STATUS_SUCCESS) { fprintf (stderr, "Error: Failed to process index options. (%s)\n", notmuch_status_to_string (status)); diff --git a/notmuch-reindex.c b/notmuch-reindex.c index 49eacd47..e9a65456 100644 --- a/notmuch-reindex.c +++ b/notmuch-reindex.c @@ -90,6 +90,7 @@ notmuch_reindex_command (notmuch_database_t *notmuch, int argc, char *argv[]) int opt_index; int ret; notmuch_status_t status; + notmuch_indexopts_t *indexopts = notmuch_database_get_default_indexopts (notmuch); /* Set up our handler for SIGINT */ memset (&action, 0, sizeof (struct sigaction)); @@ -110,7 +111,7 @@ notmuch_reindex_command (notmuch_database_t *notmuch, int argc, char *argv[]) notmuch_process_shared_options (notmuch, argv[0]); - status = notmuch_process_shared_indexing_options (notmuch); + status = notmuch_process_shared_indexing_options (indexopts); if (status != NOTMUCH_STATUS_SUCCESS) { fprintf (stderr, "Error: Failed to process index options. (%s)\n", notmuch_status_to_string (status)); @@ -128,7 +129,7 @@ notmuch_reindex_command (notmuch_database_t *notmuch, int argc, char *argv[]) return EXIT_FAILURE; } - ret = reindex_query (notmuch, query_string, indexing_cli_choices.opts); + ret = reindex_query (notmuch, query_string, indexopts); notmuch_database_destroy (notmuch); diff --git a/notmuch.c b/notmuch.c index 3fb58bf2..ac25ae18 100644 --- a/notmuch.c +++ b/notmuch.c @@ -141,21 +141,18 @@ const notmuch_opt_desc_t notmuch_shared_indexing_options [] = { notmuch_status_t -notmuch_process_shared_indexing_options (notmuch_database_t *notmuch) +notmuch_process_shared_indexing_options (notmuch_indexopts_t *opts) { - if (indexing_cli_choices.opts == NULL) - indexing_cli_choices.opts = notmuch_database_get_default_indexopts (notmuch); + if (opts == NULL) + return NOTMUCH_STATUS_NULL_POINTER; + if (indexing_cli_choices.decrypt_policy_set) { notmuch_status_t status; - if (indexing_cli_choices.opts == NULL) - return NOTMUCH_STATUS_OUT_OF_MEMORY; - status = notmuch_indexopts_set_decrypt_policy (indexing_cli_choices.opts, + status = notmuch_indexopts_set_decrypt_policy (opts, indexing_cli_choices.decrypt_policy); if (status != NOTMUCH_STATUS_SUCCESS) { fprintf (stderr, "Error: Failed to set index decryption policy to %d. (%s)\n", indexing_cli_choices.decrypt_policy, notmuch_status_to_string (status)); - notmuch_indexopts_destroy (indexing_cli_choices.opts); - indexing_cli_choices.opts = NULL; return status; } } From 00fdf10937a3ef0292424fb67208b74c753b35fd Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 11 Oct 2021 09:56:14 -0300 Subject: [PATCH 006/113] doc: remove explicit formatting of terms in definition lists Sphinx-doc already formats the terms appropriately for a given backend (bold in html and man). `makeinfo` complains noisily about formatting inside a @item if we add our own explicit formatting. This change may change the formatting in the info output. On the other hand, the existing use of quotes for bold is not that great anyway. In some places blank lines were removed to preserve the logical structure of a definition list. --- doc/man1/notmuch-address.rst | 14 +++++++------- doc/man1/notmuch-config.rst | 34 ++++++++++++++++----------------- doc/man1/notmuch-count.rst | 6 +++--- doc/man1/notmuch-dump.rst | 10 +++++----- doc/man1/notmuch-reply.rst | 12 ++++++------ doc/man1/notmuch-restore.rst | 12 ++++++------ doc/man1/notmuch-search.rst | 18 ++++++++--------- doc/man1/notmuch-show.rst | 10 +++++----- doc/man5/notmuch-hooks.rst | 6 +++--- doc/man7/notmuch-properties.rst | 8 +++----- 10 files changed, 64 insertions(+), 66 deletions(-) diff --git a/doc/man1/notmuch-address.rst b/doc/man1/notmuch-address.rst index 7423b629..c70dde35 100644 --- a/doc/man1/notmuch-address.rst +++ b/doc/man1/notmuch-address.rst @@ -42,7 +42,7 @@ Supported options for **address** include neither ``--output=sender`` nor ``--output=recipients`` is given, ``--output=sender`` is implied. - **sender** + sender Output all addresses from the *From* header. Note: Searching for **sender** should be much faster than @@ -50,17 +50,17 @@ Supported options for **address** include cached directly in the database whereas other addresses need to be fetched from message files. - **recipients** + recipients Output all addresses from the *To*, *Cc* and *Bcc* headers. - **count** + count Print the count of how many times was the address encountered during search. Note: With this option, addresses are printed only after the whole search is finished. This may take long time. - **address** + address Output only the email addresses instead of the full mailboxes with names and email addresses. This option has no effect on the JSON or S-Expression output formats. @@ -69,17 +69,17 @@ Supported options for **address** include Control the deduplication of results. - **no** + no Output all occurrences of addresses in the matching messages. This is not applicable with ``--output=count``. - **mailbox** + mailbox Deduplicate addresses based on the full, case sensitive name and email address, or mailbox. This is effectively the same as piping the ``--deduplicate=no`` output to **sort | uniq**, except for the order of results. This is the default. - **address** + address Deduplicate addresses based on the case insensitive address part of the mailbox. Of all the variants (with different name or case), print the one occurring most frequently among the diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 7d901758..6f12c773 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -55,14 +55,14 @@ The available configuration items are described below. Non-absolute paths are presumed relative to `$HOME` for items in section **database**. -**database.path** +database.path Notmuch will store its database here, (in sub-directory named ``.notmuch`` if **database.mail\_root** is unset). Default: see :ref:`database` -**database.mail_root** +database.mail_root The top-level directory where your mail currently exists and to where mail will be delivered in the future. Files should be individual email messages. @@ -72,7 +72,7 @@ paths are presumed relative to `$HOME` for items in section Default: For compatibility with older configurations, the value of database.path is used if **database.mail\_root** is unset. -**database.backup_dir** +database.backup_dir Directory to store tag dumps when upgrading database. History: this configuration value was introduced in notmuch 0.32. @@ -80,7 +80,7 @@ paths are presumed relative to `$HOME` for items in section Default: A sibling directory of the Xapian database called `backups`. -**database.hook_dir** +database.hook_dir Directory containing hooks run by notmuch commands. See :any:`notmuch-hooks(5)`. @@ -88,7 +88,7 @@ paths are presumed relative to `$HOME` for items in section Default: See HOOKS, below. -**database.autocommit** +database.autocommit How often to commit transactions to disk. `0` means wait until command completes, otherwise an integer `n` specifies to commit to @@ -96,30 +96,30 @@ paths are presumed relative to `$HOME` for items in section History: this configuration value was introduced in notmuch 0.33. -**user.name** +user.name Your full name. Default: ``$NAME`` variable if set, otherwise read from ``/etc/passwd``. -**user.primary\_email** +user.primary\_email Your primary email address. Default: ``$EMAIL`` variable if set, otherwise constructed from the username and hostname of the current machine. -**user.other\_email** +user.other\_email A list of other email addresses at which you receive email. Default: not set. -**new.tags** +new.tags A list of tags that will be added to all messages incorporated by **notmuch new**. Default: ``unread;inbox``. -**new.ignore** +new.ignore A list to specify files and directories that will not be searched for messages by :any:`notmuch-new(1)`. Each entry in the list is either: @@ -137,7 +137,7 @@ paths are presumed relative to `$HOME` for items in section Default: empty list. -**search.exclude\_tags** +search.exclude\_tags A list of tags that will be excluded from search results by default. Using an excluded tag in a query will override that exclusion. @@ -145,7 +145,7 @@ paths are presumed relative to `$HOME` for items in section Default: empty list. Note that :any:`notmuch-setup(1)` puts ``deleted;spam`` here when creating new configuration file. -**maildir.synchronize\_flags** +maildir.synchronize\_flags If true, then the following maildir flags (in message filenames) will be synchronized with the corresponding notmuch tags: @@ -178,7 +178,7 @@ paths are presumed relative to `$HOME` for items in section Default: ``true``. -**index.decrypt** +index.decrypt Policy for decrypting encrypted messages during indexing. Must be one of: ``false``, ``auto``, ``nostash``, or ``true``. @@ -231,7 +231,7 @@ paths are presumed relative to `$HOME` for items in section Default: ``auto``. -**index.header.** +index.header. Define the query prefix , based on a mail header. For example ``index.header.List=List-Id`` will add a probabilistic prefix ``List:`` that searches the ``List-Id`` field. User @@ -240,18 +240,18 @@ paths are presumed relative to `$HOME` for items in section supported. See :any:`notmuch-search-terms(7)` for a list of existing prefixes, and an explanation of probabilistic prefixes. -**built_with.** +built_with. Compile time feature . Current possibilities include "retry_lock" (configure option, included by default). (since notmuch 0.30, "compact" and "field_processor" are always included.) -**query.** +query. Expansion for named query called . See :any:`notmuch-search-terms(7)` for more information about named queries. -**squery.** +squery. Expansion for named query called , using s-expression syntax. See :any:`notmuch-sexp-queries(7)` for more information about s-expression queries. diff --git a/doc/man1/notmuch-count.rst b/doc/man1/notmuch-count.rst index 9a7e4bac..4c9c9a1c 100644 --- a/doc/man1/notmuch-count.rst +++ b/doc/man1/notmuch-count.rst @@ -28,13 +28,13 @@ Supported options for **count** include .. option:: --output=(messages|threads|files) - **messages** + messages Output the number of matching messages. This is the default. - **threads** + threads Output the number of matching threads. - **files** + files Output the number of files associated with matching messages. This may be bigger than the number of matching messages due to duplicates (i.e. multiple files having the diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst index 4319c5c3..a7ca39d0 100644 --- a/doc/man1/notmuch-dump.rst +++ b/doc/man1/notmuch-dump.rst @@ -39,7 +39,7 @@ Supported options for **dump** include Notmuch restore supports two plain text dump formats, both with one message-id per line, followed by a list of tags. - **batch-tag** + batch-tag The default **batch-tag** dump format is intended to more robust against malformed message-ids and tags containing whitespace or non-\ :manpage:`ascii(7)` characters. Each line @@ -58,7 +58,7 @@ Supported options for **dump** include :any:`notmuch-tag(1)`; note that the single message-id query is mandatory for :any:`notmuch-restore(1)`. - **sup** + sup The **sup** dump file format is specifically chosen to be compatible with the format of files produced by :manpage:`sup-dump(1)`. So if you've previously been using sup @@ -77,18 +77,18 @@ Supported options for **dump** include Control what kind of metadata is included in the output. - **config** + config Output configuration data stored in the database. Each line starts with "#@ ", followed by a space separated key-value pair. Both key and value are hex encoded if needed. - **properties** + properties Output per-message (key,value) metadata. Each line starts with "#= ", followed by a message id, and a space separated list of key=value pairs. Ids, keys and values are hex encoded if needed. See :any:`notmuch-properties(7)` for more details. - **tags** + tags Output per-message boolean metadata, namely tags. See *format* above for description of the output. diff --git a/doc/man1/notmuch-reply.rst b/doc/man1/notmuch-reply.rst index 4a78a90b..fa0371f9 100644 --- a/doc/man1/notmuch-reply.rst +++ b/doc/man1/notmuch-reply.rst @@ -40,22 +40,22 @@ Supported options for **reply** include .. option:: --format=(default|json|sexp|headers-only) - **default** + default Includes subject and quoted message body as an RFC 2822 message. - **json** + json Produces JSON output containing headers for a reply message and the contents of the original message. This output can be used by a client to create a reply message intelligently. - **sexp** + sexp Produces S-Expression output containing headers for a reply message and the contents of the original message. This output can be used by a client to create a reply message intelligently. - **headers-only** + headers-only Only produces In-Reply-To, References, To, Cc, and Bcc headers. @@ -67,10 +67,10 @@ Supported options for **reply** include .. option:: --reply-to=(all|sender) - **all** (default) + all (default) Replies to all addresses. - **sender** + sender Replies only to the sender. If replying to user's own message (Reply-to: or From: header is one of the user's configured email addresses), try To:, Cc:, and Bcc: headers in this diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst index bd452475..ac6b4245 100644 --- a/doc/man1/notmuch-restore.rst +++ b/doc/man1/notmuch-restore.rst @@ -32,14 +32,14 @@ Supported options for **restore** include line specifying a message-id and a set of tags. For details of the actual formats, see :any:`notmuch-dump(1)`. - **sup** + sup The **sup** dump file format is specifically chosen to be compatible with the format of files produced by sup-dump. So if you've previously been using sup for mail, then the **notmuch restore** command provides you a way to import all of your tags (or labels as sup calls them). - **batch-tag** + batch-tag The **batch-tag** dump format is intended to more robust against malformed message-ids and tags containing whitespace or non-\ **ascii(7)** characters. See :any:`notmuch-dump(1)` for @@ -49,7 +49,7 @@ Supported options for **restore** include changes if the **maildir.synchronize\_flags** configuration option is enabled. See :any:`notmuch-config(1)` for details. - **auto** + auto This option (the default) tries to guess the format from the input. For correctly formed input in either supported format, this heuristic, based the fact that batch-tag format contains @@ -59,18 +59,18 @@ Supported options for **restore** include Control what kind of metadata is restored. - **config** + config Restore configuration data to the database. Each configuration line starts with "#@ ", followed by a space separated key-value pair. Both key and value are hex encoded if needed. - **properties** + properties Restore per-message (key,value) metadata. Each line starts with "#= ", followed by a message id, and a space separated list of key=value pairs. Ids, keys and values are hex encoded if needed. See :any:`notmuch-properties(7)` for more details. - **tags** + tags Restore per-message metadata, namely tags. See *format* above for more details. diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst index 2d9ca2d5..ad305efd 100644 --- a/doc/man1/notmuch-search.rst +++ b/doc/man1/notmuch-search.rst @@ -43,7 +43,7 @@ Supported options for **search** include .. option:: --output=(summary|threads|messages|files|tags) - **summary** + summary Output a summary of each thread with any message matching the search terms. The summary includes the thread ID, date, the number of messages in the thread (both the number matched and @@ -52,19 +52,19 @@ Supported options for **search** include for some messages, the total number of files is printed in parentheses (see below for an example). - **threads** + threads Output the thread IDs of all threads with any message matching the search terms, either one per line (``--format=text``), separated by null characters (``--format=text0``), as a JSON array (``--format=json``), or an S-Expression list (``--format=sexp``). - **messages** + messages Output the message IDs of all messages matching the search terms, either one per line (``--format=text``), separated by null characters (``--format=text0``), as a JSON array (``--format=json``), or as an S-Expression list (``--format=sexp``). - **files** + files Output the filenames of all messages matching the search terms, either one per line (``--format=text``), separated by null characters (``--format=text0``), as a JSON array (``--format=json``), @@ -78,7 +78,7 @@ Supported options for **search** include in other directories that are included in the output, although these files alone would not match the search. - **tags** + tags Output all tags that appear on any message matching the search terms, either one per line (``--format=text``), separated by null characters (``--format=text0``), as a JSON array (``--format=json``), @@ -115,20 +115,20 @@ Supported options for **search** include terms. This option specifies whether to omit excluded messages in the search process. - **true** (default) + true (default) Prevent excluded messages from matching the search terms. - **all** + all Additionally prevent excluded messages from appearing in displayed results, in effect behaving as though the excluded messages do not exist. - **false** + false Allow excluded messages to match search terms and appear in displayed results. Excluded messages are still marked in the relevant outputs. - **flag** + flag Only has an effect when ``--output=summary``. The output is almost identical to **false**, but the "match count" is the number of matching non-excluded messages in the thread, rather diff --git a/doc/man1/notmuch-show.rst b/doc/man1/notmuch-show.rst index 64639174..3d2a2c41 100644 --- a/doc/man1/notmuch-show.rst +++ b/doc/man1/notmuch-show.rst @@ -36,7 +36,7 @@ Supported options for **show** include .. option:: --format=(text|json|sexp|mbox|raw) - **text** (default for messages) + text (default for messages) The default plain-text format has all text-content MIME parts decoded. Various components in the output, (**message**, **header**, **body**, **attachment**, and MIME **part**), will @@ -46,7 +46,7 @@ Supported options for **show** include '}'), to either open or close the component. For a multipart MIME message, these parts will be nested. - **json** + json The output is formatted with Javascript Object Notation (JSON). This format is more robust than the text format for automated processing. The nested structure of multipart MIME @@ -58,7 +58,7 @@ Supported options for **show** include as UTF-8 and any message content included in the output will be charset-converted to UTF-8. - **sexp** + sexp The output is formatted as the Lisp s-expression (sexp) equivalent of the JSON format above. Objects are formatted as property lists whose keys are keywords (symbols preceded by a @@ -66,7 +66,7 @@ Supported options for **show** include formatted as ``nil``. As for JSON, the s-expression output is always encoded as UTF-8. - **mbox** + mbox All matching messages are output in the traditional, Unix mbox format with each message being prefixed by a line beginning with "From " and a blank line separating each message. Lines @@ -77,7 +77,7 @@ Supported options for **show** include http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/mail-mbox-formats.html - **raw** (default if ``--part`` is given) + raw (default if ``--part`` is given) Write the raw bytes of the given MIME part of a message to standard out. For this format, it is an error to specify a query that matches more than one message. diff --git a/doc/man5/notmuch-hooks.rst b/doc/man5/notmuch-hooks.rst index 268917cd..0ab5efbc 100644 --- a/doc/man5/notmuch-hooks.rst +++ b/doc/man5/notmuch-hooks.rst @@ -19,7 +19,7 @@ must have executable permissions. The currently available hooks are described below. -**pre-new** +pre-new This hook is invoked by the :any:`notmuch-new(1)` command before scanning or importing new messages into the database. If this hook exits with a non-zero status, notmuch will abort further @@ -28,7 +28,7 @@ The currently available hooks are described below. Typically this hook is used for fetching or delivering new mail to be imported into the database. -**post-new** +post-new This hook is invoked by the :any:`notmuch-new(1)` command after new messages have been imported into the database and initial tags have been applied. The hook will not be run if there have been any @@ -37,7 +37,7 @@ The currently available hooks are described below. Typically this hook is used to perform additional query-based tagging on the imported messages. -**post-insert** +post-insert This hook is invoked by the :any:`notmuch-insert(1)` command after the message has been delivered, added to the database, and initial tags have been applied. The hook will not be run if there have diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst index 7f128b5c..ff79f4c2 100644 --- a/doc/man7/notmuch-properties.rst +++ b/doc/man7/notmuch-properties.rst @@ -55,7 +55,7 @@ MESSAGE PROPERTIES The following properties are set by notmuch internally in the course of its normal activity. -**index.decryption** +index.decryption If a message contains encrypted content, and notmuch tries to decrypt that content during indexing, it will add the property ``index.decryption=success`` when the cleartext was successfully @@ -75,8 +75,7 @@ of its normal activity. :any:`notmuch-config(1)`), then this property will not be set on that message. -**session-key** - +session-key When :any:`notmuch-show(1)` or :any:`notmuch-reply(1)` encounters a message with an encrypted part, if notmuch finds a ``session-key`` property associated with the message, it will try @@ -111,8 +110,7 @@ of its normal activity. example, an AES-128 key might be stashed in a notmuch property as: ``session-key=7:14B16AF65536C28AF209828DFE34C9E0``. -**index.repaired** - +index.repaired Some messages arrive in forms that are confusing to view; they can be mangled by mail transport agents, or the sending mail user agent may structure them in a way that is confusing. If notmuch From 93104f0d9de4fa2919896a55dfdd207bbaf22589 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 11 Oct 2021 09:56:15 -0300 Subject: [PATCH 007/113] doc/sexp-queries: replace definition lists with block quotes. This document contains meaningful markup in the terms, which makeinfo complains about. Replace the use of definition lists with regular paragraphs containing quote blocks. This is accomplished by splitting the "term" from the definition with a blank line. --- doc/man7/notmuch-sexp-queries.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/man7/notmuch-sexp-queries.rst b/doc/man7/notmuch-sexp-queries.rst index 019d15f0..d3b18593 100644 --- a/doc/man7/notmuch-sexp-queries.rst +++ b/doc/man7/notmuch-sexp-queries.rst @@ -31,10 +31,12 @@ An *s-expression* is either an atom, or list of whitespace delimited s-expressions inside parentheses. Atoms are either *basic value* + A basic value is an unquoted string containing no whitespace, double quotes, or parentheses. *quoted string* + Double quotes (") delimit strings possibly containing whitespace or parentheses. These can contain double quote characters by escaping with backslash. E.g. ``"this is a quote \""``. @@ -48,9 +50,11 @@ a *field*, *logical operation*, or *modifier*, and 0 or more subqueries. ``*`` + "*" matches any non-empty string in the current field. ``()`` + The empty list matches all messages *term* @@ -62,19 +66,23 @@ subqueries. phrase splitting see :any:`fields`. ``(`` *field* |q1| |q2| ... |qn| ``)`` + Restrict the queries |q1| to |qn| to *field*, and combine with *and* (for most fields) or *or*. See :any:`fields` for more information. ``(`` *operator* |q1| |q2| ... |qn| ``)`` + Combine queries |q1| to |qn|. Currently supported operators are ``and``, ``or``, and ``not``. ``(not`` |q1| ... |qn| ``)`` is equivalent to ``(and (not`` |q1| ``) ... (not`` |qn| ``))``. ``(`` *modifier* |q1| |q2| ... |qn| ``)`` + Combine queries |q1| to |qn|, and reinterpret the result (e.g. as a regular expression). See :any:`modifiers` for more information. ``(macro (`` |p1| ... |pn| ``) body)`` + Define saved query with parameter substitution. The syntax is recognized only in saved s-expression queries (see ``squery.*`` in :any:`notmuch-config(1)`). Parameter names in ``body`` must be @@ -164,26 +172,31 @@ MODIFIERS that are neither operators nor fields. ``(infix`` *atom* ``)`` + Interpret *atom* as an infix notmuch query (see :any:`notmuch-search-terms(7)`). Not supported inside fields. ``(matching`` |q1| |q2| ... |qn| ``)`` ``(of`` |q1| |q2| ... |qn| ``)`` + Match all messages have the same values of the current field as those matching all of |q1| ... |qn|. Supported in most term [#not-path]_ or phrase fields. Most commonly used in the ``thread`` field. ``(query`` *atom* ``)`` + Expand to the saved query named by *atom*. See :any:`notmuch-config(1)` for more. Note that the saved query must be in infix syntax (:any:`notmuch-search-terms(7)`). Not supported inside fields. ``(regex`` *atom* ``)`` ``(rx`` *atom* ``)`` + Interpret *atom* as a POSIX.2 regular expression (see :manpage:`regex(7)`). This applies in term fields and a subset [#not-phrase]_ of phrase fields (see :any:`field-table`). ``(starts-with`` *subword* ``)`` + Matches any term starting with *subword*. This applies in either phrase or term :any:`fields `, or outside of fields [#not-body]_. Note that a ``starts-with`` query cannot be part of a phrase. The @@ -193,63 +206,80 @@ EXAMPLES ======== ``Wizard`` + Match all messages containing the word "wizard", ignoring case. ``added`` + Match all messages containing "added", but also those containing "add", "additional", "Additional", "adds", etc... via stemming. ``(and Bob Marley)`` + Match messages containing words "Bob" and "Marley", or their stems The words need not be adjacent. ``(not Bob Marley)`` + Match messages containing neither "Bob" nor "Marley", nor their stems, ``"quick fox"`` ``quick-fox`` ``quick@fox`` + Match the *phrase* "quick" followed by "fox" in phrase fields (or outside a field). Match the literal string in a term field. ``(folder (of (id 1234@invalid)))`` + Match any message in the same folder as the one with Message-Id "1234@invalid" ``(id 1234@invalid blah@test)`` + Matches Message-Id "1234@invalid" *or* Message-Id "blah@test" ``(and (infix "date:2009-11-18..2009-11-18") (tag unread))`` + Match messages in the given date range with tag unread. ``(starts-with prelim)`` + Match any words starting with "prelim". ``(subject quick "brown fox")`` + Match messages whose subject contains "quick" (anywhere, stemmed) and the phrase "brown fox". ``(subject (starts-with prelim))`` + Matches any word starting with "prelim", inside a message subject. ``(subject (starts-wih quick) "brown fox")`` + Match messages whose subject contains "quick brown fox", but also "brown fox quicksand". ``(thread (of (id 1234@invalid)))`` + Match any message in the same thread as the one with Message-Id "1234@invalid" ``(thread (matching (from bob@example.com) (to bob@example.com)))`` + Match any (messages in) a thread containing a message from "bob@example.com" and a (possibly distinct) message to "bob at example.com") ``(to (or bob@example.com mallory@example.org))`` ``(or (to bob@example.com) (to mallory@example.org))`` + Match in the "To" or "Cc" headers, "bob@example.com", "mallory@example.org", and also "bob@example.com.au" since it contains the adjacent triple "bob", "example", "com". ``(not (to *))`` + Match messages with an empty or invalid 'To' and 'Cc' field. ``(List *)`` + Match messages with a non-empty List-Id header, assuming configuration ``index.header.List=List-Id`` From c7705fb95e2f5d11fabb7b6a07f7b7558e0928e6 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 23 Oct 2021 09:27:01 -0300 Subject: [PATCH 008/113] lib/compact: replace deprecated notmuch_database_open_verbose It should not be necesary to have any config information here, hence passing "" to n_d_open_with_config. --- lib/database.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/database.cc b/lib/database.cc index 7eb0de79..6ef56d56 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -590,10 +590,12 @@ notmuch_database_compact (const char *path, notmuch_database_t *notmuch = NULL; char *message = NULL; - ret = notmuch_database_open_verbose (path, - NOTMUCH_DATABASE_MODE_READ_WRITE, - ¬much, - &message); + ret = notmuch_database_open_with_config (path, + NOTMUCH_DATABASE_MODE_READ_WRITE, + "", + NULL, + ¬much, + &message); if (ret) { if (status_cb) status_cb (message, closure); return ret; From efbf5bafaf15cc99e272bd75e71688c96130d9d5 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 23 Oct 2021 09:27:02 -0300 Subject: [PATCH 009/113] lib/open: replace call to deprecated notmuch_database_open_verbose Essentially inline the existing shim definition of notmuch_database_open_verbose. --- lib/open.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/open.cc b/lib/open.cc index ba32c2f1..46a01545 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -19,9 +19,8 @@ notmuch_database_open (const char *path, char *status_string = NULL; notmuch_status_t status; - status = notmuch_database_open_verbose (path, mode, database, - &status_string); - + status = notmuch_database_open_with_config (path, mode, "", NULL, + database, &status_string); if (status_string) { fputs (status_string, stderr); free (status_string); From 55c6570318a00f64a03d9aaff842c8551822c0a1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 11 Oct 2021 08:19:21 -0300 Subject: [PATCH 010/113] emacs: add minimal docstring for notmuch-unthreaded The missing docstring causes a blank in the notmuch-help display [1]. Since the function is a simple wrapper for notmuch-tree, it seems fair to forward the reader there for more detailed information. [1]: id:878sape5a9.fsf@disroot.org --- emacs/notmuch-tree.el | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el index 001a367d..7fa73d40 100644 --- a/emacs/notmuch-tree.el +++ b/emacs/notmuch-tree.el @@ -1206,6 +1206,9 @@ The arguments are: (defun notmuch-unthreaded (&optional query query-context target buffer-name open-target) + "Display threads matching QUERY in unthreaded view. + +See function NOTMUCH-TREE for documentation of the arguments" (interactive) (notmuch-tree query query-context target buffer-name open-target t)) From 78416a3e97fd19df5c89cdaf564c76be0edea740 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 23 Oct 2021 10:12:58 -0300 Subject: [PATCH 011/113] emacs: improve notmuch-*-from-current-query docstrings Err on the side of providing better user documentation, rather than documentation for developers. --- emacs/notmuch.el | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emacs/notmuch.el b/emacs/notmuch.el index 2ef67c0e..fa061693 100644 --- a/emacs/notmuch.el +++ b/emacs/notmuch.el @@ -535,12 +535,12 @@ thread." (message "End of search results.")))) (defun notmuch-tree-from-search-current-query () - "Call notmuch tree with the current query." + "Tree view of current query." (interactive) (notmuch-tree notmuch-search-query-string)) (defun notmuch-unthreaded-from-search-current-query () - "Call notmuch tree with the current query." + "Unthreaded view of current query." (interactive) (notmuch-unthreaded notmuch-search-query-string)) From b264a49be3dccb7d110eae6420019a052f1f665b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 30 Oct 2021 15:49:20 -0300 Subject: [PATCH 012/113] rename built_with.sexpr_query to built_with.sexp_queries It is confusing to use two different names (sexp vs sexpr) when compared with the command line option --query=sexp and (furthermore) singular vs plural when compared with the man page title. --- doc/man7/notmuch-sexp-queries.rst | 2 +- lib/built-with.c | 2 +- notmuch-config.c | 4 ++-- test/T030-config.sh | 2 +- test/T055-path-config.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/man7/notmuch-sexp-queries.rst b/doc/man7/notmuch-sexp-queries.rst index d3b18593..3c33232f 100644 --- a/doc/man7/notmuch-sexp-queries.rst +++ b/doc/man7/notmuch-sexp-queries.rst @@ -21,7 +21,7 @@ build of notmuch supports it with :: - $ notmuch config get built_with.sexpr_query + $ notmuch config get built_with.sexp_queries S-EXPRESSIONS diff --git a/lib/built-with.c b/lib/built-with.c index 89958e12..275e72b8 100644 --- a/lib/built-with.c +++ b/lib/built-with.c @@ -32,7 +32,7 @@ notmuch_built_with (const char *name) return HAVE_XAPIAN_DB_RETRY_LOCK; } else if (STRNCMP_LITERAL (name, "session_key") == 0) { return true; - } else if (STRNCMP_LITERAL (name, "sexpr_query") == 0) { + } else if (STRNCMP_LITERAL (name, "sexp_queries") == 0) { return HAVE_SFSEXP; } else { return false; diff --git a/notmuch-config.c b/notmuch-config.c index db00a26c..11d8d68b 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -680,9 +680,9 @@ _notmuch_config_list_built_with () printf ("%sretry_lock=%s\n", BUILT_WITH_PREFIX, notmuch_built_with ("retry_lock") ? "true" : "false"); - printf ("%ssexpr_query=%s\n", + printf ("%ssexp_queries=%s\n", BUILT_WITH_PREFIX, - notmuch_built_with ("sexpr_query") ? "true" : "false"); + notmuch_built_with ("sexp_queries") ? "true" : "false"); } static int diff --git a/test/T030-config.sh b/test/T030-config.sh index 3a585d1b..b99eb9e7 100755 --- a/test/T030-config.sh +++ b/test/T030-config.sh @@ -51,7 +51,7 @@ cat < EXPECTED built_with.compact=something built_with.field_processor=something built_with.retry_lock=something -built_with.sexpr_query=something +built_with.sexp_queries=something database.autocommit=8000 database.mail_root=MAIL_DIR database.path=MAIL_DIR diff --git a/test/T055-path-config.sh b/test/T055-path-config.sh index ef22e964..38f72e5e 100755 --- a/test/T055-path-config.sh +++ b/test/T055-path-config.sh @@ -277,7 +277,7 @@ EOF built_with.compact=something built_with.field_processor=something built_with.retry_lock=something -built_with.sexpr_query=something +built_with.sexp_queries=something database.autocommit=8000 database.backup_dir database.hook_dir From 20b2ae12183a5be79d2f3d8da7943bc358e8202c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 30 Oct 2021 15:58:16 -0300 Subject: [PATCH 013/113] emacs: drop C-tab binding in hello mode, document . The binding has always been there, but the docs were apparently mistakenly changed to say [1] Revert to in the documentation. The commit also drops the C- binding, since it seems redundant and it interferes with tab-bar-mode. [1]: 703dec7754da477b5683867c88cb940b8553be91. --- doc/notmuch-emacs.rst | 2 +- emacs/notmuch-hello.el | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index 12ee25e5..22aee340 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -56,7 +56,7 @@ notmuch-hello key bindings ```` Move to the next widget (button or text entry field) -```` +```` Move to the previous widget. ```` diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index 71487bd9..acd48c9b 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -702,7 +702,6 @@ with `notmuch-hello-query-counts'." ;; that when we modify map it does not modify widget-keymap). (let ((map (make-composed-keymap (list (make-sparse-keymap) widget-keymap)))) (set-keymap-parent map notmuch-common-keymap) - (define-key map (kbd "") 'widget-backward) map) "Keymap for \"notmuch hello\" buffers.") From eafb033d3220f1388f434dc29618924659fc2d33 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 12 Sep 2021 22:03:50 -0300 Subject: [PATCH 014/113] emacs: run notmuch-search-hook lazily In message id:YT3ueuZHKW931NW3@localhost, Fabio Natali isolated a visual glitch caused by running notmuch-search-hook too early. This change moves the running of that hook to notmuch-search-process-filter, which ensures there is some output in the buffer before running the hook. Since n-s-p-f can be called many times for a given buffer, add a buffer local flag to make sure it is only run once per buffer. --- emacs/notmuch.el | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/emacs/notmuch.el b/emacs/notmuch.el index fa061693..85a54706 100644 --- a/emacs/notmuch.el +++ b/emacs/notmuch.el @@ -880,6 +880,14 @@ sets the :orig-tag property." (setq notmuch-search-target-thread "found") (goto-char pos)))) +(defvar-local notmuch--search-hook-run nil + "Flag used to ensure the notmuch-search-hook is only run once per buffer") + +(defun notmuch--search-hook-wrapper () + (unless notmuch--search-hook-run + (setq notmuch--search-hook-run t) + (run-hooks 'notmuch-search-hook))) + (defun notmuch-search-process-filter (proc string) "Process and filter the output of \"notmuch search\"." (let ((results-buf (process-buffer proc)) @@ -892,7 +900,9 @@ sets the :orig-tag property." (goto-char (point-max)) (insert string)) (notmuch-sexp-parse-partial-list 'notmuch-search-append-result - results-buf))))) + results-buf)) + (with-current-buffer results-buf + (notmuch--search-hook-wrapper))))) ;;; Commands (and some helper functions used by them) @@ -1036,8 +1046,7 @@ the configured default sort order." (process-put proc 'parse-buf (generate-new-buffer " *notmuch search parse*")) (set-process-filter proc 'notmuch-search-process-filter) - (set-process-query-on-exit-flag proc nil)))) - (run-hooks 'notmuch-search-hook))) + (set-process-query-on-exit-flag proc nil)))))) (defun notmuch-search-refresh-view () "Refresh the current view. From e22bbb124e4f9191880e80d6517346f35bf7e2a9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 30 Sep 2021 14:17:47 -0300 Subject: [PATCH 015/113] test: known broken tests for leading/trailing ws in config These tests duplicate the bug/misfeature reported in id:CA+Tk8fzjPLaEd3vL1f9ebk_bF_RV8PDTLzDupraTkCLCpJAmCg@mail.gmail.com --- test/T050-new.sh | 13 +++++++++++++ test/T070-insert.sh | 13 +++++++++++++ test/T590-libconfig.sh | 24 ++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/test/T050-new.sh b/test/T050-new.sh index 1141c1e3..bc20440b 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -329,6 +329,19 @@ notmuch config set new.tags "foo;;bar" output=$(NOTMUCH_NEW --quiet 2>&1) test_expect_equal "$output" "" +test_begin_subtest "leading/trailing whitespace in new.tags is ignored" +test_subtest_known_broken +# avoid complications with leading spaces and "notmuch config" +sed -i 's/^tags=.*$/tags= fu bar ; ; bar /' notmuch-config +add_message +NOTMUCH_NEW --quiet +notmuch dump id:$gen_msg_id | sed 's/ --.*$//' > OUTPUT +cat <EXPECTED +#notmuch-dump batch-tag:3 config,properties,tags ++bar +fu%20bar +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "Tags starting with '-' in new.tags are forbidden" notmuch config set new.tags "-foo;bar" output=$(NOTMUCH_NEW --debug 2>&1) diff --git a/test/T070-insert.sh b/test/T070-insert.sh index 208deb1c..9d29c859 100755 --- a/test/T070-insert.sh +++ b/test/T070-insert.sh @@ -234,6 +234,19 @@ output=$(notmuch show --format=json id:$gen_msg_id) test_json_nodes <<<"$output" \ 'new_tags:[0][0][0]["tags"] = ["bar", "foo"]' +test_begin_subtest "leading/trailing whitespace in new.tags is ignored" +test_subtest_known_broken +# avoid complications with leading spaces and "notmuch config" +sed -i 's/^tags=.*$/tags= fu bar ; ; bar /' notmuch-config +gen_insert_msg +notmuch insert < $gen_msg_filename +notmuch dump id:$gen_msg_id | sed 's/ --.*$//' > OUTPUT +cat <EXPECTED +#notmuch-dump batch-tag:3 config,properties,tags ++bar +fu%20bar +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "Tags starting with '-' in new.tags are forbidden" notmuch config set new.tags "-foo;bar" gen_insert_msg diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 9fa51fc0..912aaa1f 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -272,6 +272,30 @@ EOF test_expect_equal_file EXPECTED OUTPUT restore_database +test_begin_subtest "notmuch_config_get_values (ignore leading/trailing whitespace)" +test_subtest_known_broken +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} %NULL% +{ + notmuch_config_values_t *values; + EXPECT0(notmuch_config_set (db, NOTMUCH_CONFIG_NEW_TAGS, " a ; b c ; d ")); + for (values = notmuch_config_get_values (db, NOTMUCH_CONFIG_NEW_TAGS); + notmuch_config_values_valid (values); + notmuch_config_values_move_to_next (values)) + { + puts (notmuch_config_values_get (values)); + } +} +EOF +cat <<'EOF' >EXPECTED +== stdout == +a +b c +d +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT +restore_database + test_begin_subtest "notmuch_config_get_values_string" cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} %NULL% { From bab633d3ac87167dc214094f9d340655885a01b1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 30 Sep 2021 14:17:48 -0300 Subject: [PATCH 016/113] config: ignore leading/trailing spaces in ';'-delimited lists In [1] Ciprian observed that it was easy for users to mistakenly introduce leading and trailing space to new.tags when editing a notmuch config file. This commit strips spaces on either side of the ';' delimiter when splitting. In principle it would be possible to support tags (or other config values) with leading or trailing spaces by processing '\s' escapes in the input string. Currently such processing is not done. [1]: id:CA+Tk8fzjPLaEd3vL1f9ebk_bF_RV8PDTLzDupraTkCLCpJAmCg@mail.gmail.com --- test/T050-new.sh | 1 - test/T070-insert.sh | 1 - test/T590-libconfig.sh | 1 - util/string-util.c | 10 ++++++---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/T050-new.sh b/test/T050-new.sh index bc20440b..69697c48 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -330,7 +330,6 @@ output=$(NOTMUCH_NEW --quiet 2>&1) test_expect_equal "$output" "" test_begin_subtest "leading/trailing whitespace in new.tags is ignored" -test_subtest_known_broken # avoid complications with leading spaces and "notmuch config" sed -i 's/^tags=.*$/tags= fu bar ; ; bar /' notmuch-config add_message diff --git a/test/T070-insert.sh b/test/T070-insert.sh index 9d29c859..ec170b30 100755 --- a/test/T070-insert.sh +++ b/test/T070-insert.sh @@ -235,7 +235,6 @@ test_json_nodes <<<"$output" \ 'new_tags:[0][0][0]["tags"] = ["bar", "foo"]' test_begin_subtest "leading/trailing whitespace in new.tags is ignored" -test_subtest_known_broken # avoid complications with leading spaces and "notmuch config" sed -i 's/^tags=.*$/tags= fu bar ; ; bar /' notmuch-config gen_insert_msg diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 912aaa1f..6c426ae8 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -273,7 +273,6 @@ test_expect_equal_file EXPECTED OUTPUT restore_database test_begin_subtest "notmuch_config_get_values (ignore leading/trailing whitespace)" -test_subtest_known_broken cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} %NULL% { notmuch_config_values_t *values; diff --git a/util/string-util.c b/util/string-util.c index 9c46a81a..03d7648d 100644 --- a/util/string-util.c +++ b/util/string-util.c @@ -42,13 +42,15 @@ const char * strsplit_len (const char *s, char delim, size_t *len) { bool escaping = false; - size_t count = 0; + size_t count = 0, last_nonspace = 0; - /* Skip initial unescaped delimiters */ - while (*s && *s == delim) + /* Skip initial unescaped delimiters and whitespace */ + while (*s && (*s == delim || isspace (*s))) s++; while (s[count] && (escaping || s[count] != delim)) { + if (! isspace (s[count])) + last_nonspace = count; escaping = (s[count] == '\\'); count++; } @@ -56,7 +58,7 @@ strsplit_len (const char *s, char delim, size_t *len) if (count == 0) return NULL; - *len = count; + *len = last_nonspace + 1; return s; } From 482bd3a46de34c5f6c5cae3696fd56ffdadc6299 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 30 Sep 2021 15:28:34 -0300 Subject: [PATCH 017/113] test: known broken tests for escape characters in config files. glib generates the following escape characters with their usual meanings: \n, \t, \r, and \\, along with \s for _leading_ spaces. Currently notmuch fails to unescape these on reading the config files. These tests demonstrate this bug; the one new test that passes is because apparently glib only escapes tabs at the beginning of a key. --- test/T030-config.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/T030-config.sh b/test/T030-config.sh index b99eb9e7..1fdde9fe 100755 --- a/test/T030-config.sh +++ b/test/T030-config.sh @@ -67,6 +67,37 @@ user.primary_email=test_suite@notmuchmail.org EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "Round trip config item with leading spaces" +test_subtest_known_broken +notmuch config set foo.bar " thing" +output=$(notmuch config get foo.bar) +test_expect_equal "${output}" " thing" + +test_begin_subtest "Round trip config item with leading tab" +test_subtest_known_broken +notmuch config set foo.bar " thing" +output=$(notmuch config get foo.bar) +test_expect_equal "${output}" " thing" + +test_begin_subtest "Round trip config item with embedded tab" +notmuch config set foo.bar "thing other" +output=$(notmuch config get foo.bar) +test_expect_equal "${output}" "thing other" + +test_begin_subtest "Round trip config item with embedded backslash" +test_subtest_known_broken +notmuch config set foo.bar 'thing\other' +output=$(notmuch config get foo.bar) +test_expect_equal "${output}" "thing\other" + +test_begin_subtest "Round trip config item with embedded NL/CR" +test_subtest_known_broken +notmuch config set foo.bar 'thing + other' +output=$(notmuch config get foo.bar) +test_expect_equal "${output}" "thing + other" + test_begin_subtest "Top level --config=FILE option" cp "${NOTMUCH_CONFIG}" alt-config notmuch --config=alt-config config set user.name "Another Name" From 18cdd21b8b2ef056062700607eade43909c32cd2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 30 Sep 2021 15:59:54 -0300 Subject: [PATCH 018/113] lib/config: use g_key_file_get_string to read config values Unlike the previous g_key_file_get_value, this version processes escape codes for whitespace and \. The remaining two broken tests from the last commit are because "notmuch config get" treats every value as a list, and thus the previously introduces stripping of leading whitespace applies. --- lib/config.cc | 2 +- lib/open.cc | 4 ++-- test/T030-config.sh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/config.cc b/lib/config.cc index e502858d..7a2882de 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -435,7 +435,7 @@ _notmuch_config_load_from_file (notmuch_database_t *notmuch, for (gchar **keys_p = keys; *keys_p; keys_p++) { char *absolute_key = talloc_asprintf (notmuch, "%s.%s", *grp, *keys_p); char *normalized_val; - val = g_key_file_get_value (file, *grp, *keys_p, NULL); + val = g_key_file_get_string (file, *grp, *keys_p, NULL); if (! val) { status = NOTMUCH_STATUS_FILE_ERROR; goto DONE; diff --git a/lib/open.cc b/lib/open.cc index c2cb2818..a91d22ef 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -198,7 +198,7 @@ _choose_database_path (void *ctx, } if (! *database_path && key_file) { - char *path = g_key_file_get_value (key_file, "database", "path", NULL); + char *path = g_key_file_get_string (key_file, "database", "path", NULL); if (path) { if (path[0] == '/') *database_path = talloc_strdup (ctx, path); @@ -642,7 +642,7 @@ notmuch_database_create_with_config (const char *database_path, if (key_file && ! split) { char *mail_root = notmuch_canonicalize_file_name ( - g_key_file_get_value (key_file, "database", "mail_root", NULL)); + g_key_file_get_string (key_file, "database", "mail_root", NULL)); char *db_path = notmuch_canonicalize_file_name (database_path); split = (mail_root && (0 != strcmp (mail_root, db_path))); diff --git a/test/T030-config.sh b/test/T030-config.sh index 1fdde9fe..43bbce31 100755 --- a/test/T030-config.sh +++ b/test/T030-config.sh @@ -85,13 +85,11 @@ output=$(notmuch config get foo.bar) test_expect_equal "${output}" "thing other" test_begin_subtest "Round trip config item with embedded backslash" -test_subtest_known_broken notmuch config set foo.bar 'thing\other' output=$(notmuch config get foo.bar) test_expect_equal "${output}" "thing\other" test_begin_subtest "Round trip config item with embedded NL/CR" -test_subtest_known_broken notmuch config set foo.bar 'thing other' output=$(notmuch config get foo.bar) From 1643c0459a496b45b601d91e0089fac507a2a6d1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 28 Oct 2021 09:32:34 -0300 Subject: [PATCH 019/113] test: move system includes to notmuch-test.h This removes some redudant includes, and will also make it easier to introduce "#define _GNU_SOURCE", which must come before all system includes. --- test/T562-lib-database.sh | 4 +--- test/T563-lib-directory.sh | 4 +--- test/T564-lib-query.sh | 4 +--- test/T566-lib-message.sh | 3 +-- test/T568-lib-thread.sh | 3 +-- test/T590-libconfig.sh | 4 ---- test/T595-reopen.sh | 2 -- test/T610-message-property.sh | 4 ---- test/T620-lock.sh | 3 --- test/T640-database-modified.sh | 5 +---- test/notmuch-test.h | 6 ++++++ 11 files changed, 12 insertions(+), 30 deletions(-) diff --git a/test/T562-lib-database.sh b/test/T562-lib-database.sh index 769fe86e..d9f5d18e 100755 --- a/test/T562-lib-database.sh +++ b/test/T562-lib-database.sh @@ -9,10 +9,8 @@ test_begin_subtest "building database" test_expect_success "NOTMUCH_NEW" cat < c_head -#include -#include #include -#include + int main (int argc, char** argv) { notmuch_database_t *db; diff --git a/test/T563-lib-directory.sh b/test/T563-lib-directory.sh index 28325ff2..ad390c1c 100755 --- a/test/T563-lib-directory.sh +++ b/test/T563-lib-directory.sh @@ -9,10 +9,8 @@ test_begin_subtest "building database" test_expect_success "NOTMUCH_NEW" cat < c_head -#include -#include #include -#include + int main (int argc, char** argv) { notmuch_database_t *db; diff --git a/test/T564-lib-query.sh b/test/T564-lib-query.sh index 50b0a88e..ff1d4984 100755 --- a/test/T564-lib-query.sh +++ b/test/T564-lib-query.sh @@ -9,10 +9,8 @@ test_begin_subtest "building database" test_expect_success "NOTMUCH_NEW" cat < c_head -#include -#include #include -#include + int main (int argc, char** argv) { notmuch_database_t *db; diff --git a/test/T566-lib-message.sh b/test/T566-lib-message.sh index ee55ef29..8b61d182 100755 --- a/test/T566-lib-message.sh +++ b/test/T566-lib-message.sh @@ -19,9 +19,8 @@ cat <<'EOF' > c_tail EOF cat < c_head0 -#include -#include #include + int main (int argc, char** argv) { notmuch_database_t *db; diff --git a/test/T568-lib-thread.sh b/test/T568-lib-thread.sh index 088e66dd..b45836cd 100755 --- a/test/T568-lib-thread.sh +++ b/test/T568-lib-thread.sh @@ -24,9 +24,8 @@ cat <<'EOF' > c_tail EOF cat < c_head -#include -#include #include + int main (int argc, char** argv) { notmuch_database_t *db; diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 6c426ae8..eb303444 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -23,8 +23,6 @@ EOF } cat < c_head -#include -#include #include int main (int argc, char** argv) @@ -639,8 +637,6 @@ cp notmuch-config.bak notmuch-config test_expect_equal_file EXPECTED OUTPUT cat < c_head2 -#include -#include #include int main (int argc, char** argv) diff --git a/test/T595-reopen.sh b/test/T595-reopen.sh index 7375e2ac..1a517423 100755 --- a/test/T595-reopen.sh +++ b/test/T595-reopen.sh @@ -6,8 +6,6 @@ test_description="library reopen API" add_email_corpus cat < c_head -#include -#include #include int main (int argc, char** argv) diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh index d0e52f4a..4ec85474 100755 --- a/test/T610-message-property.sh +++ b/test/T610-message-property.sh @@ -6,10 +6,6 @@ test_description="message property API" add_email_corpus cat < c_head -#include -#include -#include -#include #include void print_properties (notmuch_message_t *message, const char *prefix, notmuch_bool_t exact) { diff --git a/test/T620-lock.sh b/test/T620-lock.sh index 7aaaff2a..8f4c380f 100755 --- a/test/T620-lock.sh +++ b/test/T620-lock.sh @@ -9,9 +9,6 @@ if [ $NOTMUCH_HAVE_XAPIAN_DB_RETRY_LOCK -ne 1 ]; then test_subtest_known_broken fi test_C ${MAIL_DIR} <<'EOF' -#include -#include -#include #include void diff --git a/test/T640-database-modified.sh b/test/T640-database-modified.sh index 274105c7..636b20c7 100755 --- a/test/T640-database-modified.sh +++ b/test/T640-database-modified.sh @@ -10,11 +10,8 @@ test_begin_subtest "catching DatabaseModifiedError in _notmuch_message_ensure_me first_id=$(notmuch search --output=messages '*'| head -1 | sed s/^id://) test_C ${MAIL_DIR} < -#include #include -#include -#include + int main (int argc, char **argv) { diff --git a/test/notmuch-test.h b/test/notmuch-test.h index 34dbb8e0..3a0e90a3 100644 --- a/test/notmuch-test.h +++ b/test/notmuch-test.h @@ -1,7 +1,13 @@ #ifndef _NOTMUCH_TEST_H #define _NOTMUCH_TEST_H +#include #include #include +#include +#include +#include +#include + #include inline static void From 9397e7e8eb70f27b411f219449a201ac11e60b21 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 24 Oct 2021 22:15:14 -0300 Subject: [PATCH 020/113] test/count: replace use of gdb with a LD_PRELOAD shim There is a certain amount of boilerplate to pass the call on the original function, so abstract it out as a C preprocessor macro, plus some extra includes in notmuch-test.h --- test/T060-count.sh | 27 +++++++++++++++------------ test/notmuch-test.h | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/test/T060-count.sh b/test/T060-count.sh index 6ad80df9..48146706 100755 --- a/test/T060-count.sh +++ b/test/T060-count.sh @@ -102,22 +102,25 @@ output=$(sed 's/^\(A Xapian exception [^:]*\):.*$/\1/' OUTPUT) test_expect_equal "${output}" "A Xapian exception occurred opening database" restore_database -cat < count-files.gdb -set breakpoint pending on -set logging file count-files-gdb.log -set logging on -break count_files -commands -shell cp /dev/null ${MAIL_DIR}/.notmuch/xapian/postlist.* -continue -end -run +make_shim qsm-shim< + +WRAP_DLFUNC (notmuch_status_t, notmuch_query_search_messages, (notmuch_query_t *query, notmuch_messages_t **messages)) + + /* XXX WARNING THIS CORRUPTS THE DATABASE */ + int fd = open ("target_postlist", O_WRONLY|O_TRUNC); + if (fd < 0) + exit (8); + close (fd); + + return notmuch_query_search_messages_orig(query, messages); +} EOF backup_database test_begin_subtest "error message from query_search_messages" -${TEST_GDB} --batch-silent --return-child-result -x count-files.gdb \ - --args notmuch count --output=files '*' 2>OUTPUT 1>/dev/null +ln -s ${MAIL_DIR}/.notmuch/xapian/postlist.* target_postlist +notmuch_with_shim qsm-shim count --output=files '*' 2>OUTPUT 1>/dev/null cat < EXPECTED notmuch count: A Xapian exception occurred A Xapian exception occurred performing query diff --git a/test/notmuch-test.h b/test/notmuch-test.h index 3a0e90a3..ed713099 100644 --- a/test/notmuch-test.h +++ b/test/notmuch-test.h @@ -1,9 +1,17 @@ #ifndef _NOTMUCH_TEST_H #define _NOTMUCH_TEST_H +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include +#include +#include #include #include #include +#include +#include #include #include #include @@ -20,4 +28,23 @@ expect0 (int line, notmuch_status_t ret) } #define EXPECT0(v) expect0 (__LINE__, v); + +inline static void * +dlsym_next (const char *symbol) +{ + void *sym = dlsym (RTLD_NEXT, symbol); + char *str = dlerror (); + + if (str != NULL) { + fprintf (stderr, "finding symbol '%s' failed: %s", symbol, str); + exit (77); + } + return sym; +} + +#define WRAP_DLFUNC(_rtype, _func, _args) \ + _rtype _func _args; \ + _rtype _func _args { \ + static _rtype (*_func##_orig) _args = NULL; \ + if (! _func##_orig ) *(void **) (&_func##_orig) = dlsym_next (#_func); #endif From efa7f35d4a28bd8af20cc5dee3195dc8f2d142bc Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 24 Oct 2021 22:15:15 -0300 Subject: [PATCH 021/113] test/new: replace use of gdb in vanishing file test Unlike the similar change in T060-new, no symlink creation is needed here. --- test/T050-new.sh | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/test/T050-new.sh b/test/T050-new.sh index 69697c48..7ea127d3 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -380,31 +380,26 @@ chmod u+w ${MAIL_DIR}/.notmuch/xapian/*.* test_expect_equal "$output" "A Xapian exception occurred opening database" +make_shim dif-shim< + +WRAP_DLFUNC(notmuch_status_t, notmuch_database_index_file, \ + (notmuch_database_t *database, const char *filename, notmuch_indexopts_t *indexopts, notmuch_message_t **message)) + + if (unlink ("${MAIL_DIR}/vanish")) { + fprintf (stderr, "unlink failed\n"); + exit (42); + } + return notmuch_database_index_file_orig (database, filename, indexopts, message); +} +EOF + test_begin_subtest "Handle files vanishing between scandir and add_file" # A file for scandir to find. It won't get indexed, so can be empty. touch ${MAIL_DIR}/vanish - -# Breakpoint to remove the file before indexing -cat < notmuch-new-vanish.gdb -set breakpoint pending on -set logging file notmuch-new-vanish-gdb.log -set logging on -break notmuch_database_index_file -commands -shell rm -f ${MAIL_DIR}/vanish -continue -end -run -EOF - -${TEST_GDB} --batch-silent --return-child-result -x notmuch-new-vanish.gdb \ - --args notmuch new 2>OUTPUT 1>/dev/null +notmuch_with_shim dif-shim new 2>OUTPUT 1>/dev/null echo "exit status: $?" >> OUTPUT - -# Clean up the file in case gdb isn't available. -rm -f ${MAIL_DIR}/vanish - cat < EXPECTED Unexpected error with file ${MAIL_DIR}/vanish add_file: Something went wrong trying to read or write a file From 20b2150406e1f80af6417ac601ff366de7eb804b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 23 Oct 2021 14:05:24 -0300 Subject: [PATCH 022/113] emacs: don't add space to tag completion candidates. Apparently this messes up various third party completion frameworks. This change does mean that users will have to hit space after completing a tag change in order to enter another change. As a bonus, remove the call to #'delete, since completing-read-multiple already promises to remove empty strings. --- emacs/notmuch-tag.el | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el index 536315e9..145f309f 100644 --- a/emacs/notmuch-tag.el +++ b/emacs/notmuch-tag.el @@ -429,17 +429,9 @@ initial input in the minibuffer." (set-keymap-parent map crm-local-completion-map) (define-key map " " 'self-insert-command) map))) - (delete "" (completing-read-multiple - prompt - ;; Append the separator to each completion so when the - ;; user completes a tag they can immediately begin - ;; entering another. `completing-read-multiple' - ;; ultimately splits the input on crm-separator, so we - ;; don't need to strip this back off (we just need to - ;; delete "empty" entries caused by trailing spaces). - (mapcar (lambda (tag-op) (concat tag-op crm-separator)) tag-list) - nil nil initial-input - 'notmuch-read-tag-changes-history)))) + (completing-read-multiple prompt tag-list + nil nil initial-input + 'notmuch-read-tag-changes-history))) ;;; Tagging From 904c067f32ce29f582c711c0ce631df9b8d4edfc Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 30 Oct 2021 17:48:54 -0300 Subject: [PATCH 023/113] ruby: don't use a directory as a target. The directory is (neccesarily) not updated by the build, so it keeps trying to build. The proposed fix is to use the name of the dynamic library containing the extension. This is a partial fix for the rebuilding reported at [1]. [1]: id:87r29wwgq2.fsf@fifthhorseman.net --- bindings/Makefile.local | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bindings/Makefile.local b/bindings/Makefile.local index 3672e69f..1cdd28a0 100644 --- a/bindings/Makefile.local +++ b/bindings/Makefile.local @@ -3,21 +3,23 @@ dir := bindings # force the shared library to be built -ruby-bindings: lib/$(LINKER_NAME) +ruby-bindings: $(dir)/ruby.stamp + +$(dir)/ruby.stamp: lib/$(LINKER_NAME) ifeq ($(HAVE_RUBY_DEV),1) cd $(dir)/ruby && \ EXTRA_LDFLAGS="$(NO_UNDEFINED_LDFLAGS)" \ LIBNOTMUCH="../../lib/$(LINKER_NAME)" \ NOTMUCH_SRCDIR='$(NOTMUCH_SRCDIR)' \ $(RUBY) extconf.rb --vendor - $(MAKE) -C $(dir)/ruby CFLAGS="$(CFLAGS) -pipe -fno-plt -fPIC" + $(MAKE) -C $(dir)/ruby CFLAGS="$(CFLAGS) -pipe -fno-plt -fPIC" && touch $@ endif python-cffi-bindings: lib/$(LINKER_NAME) ifeq ($(HAVE_PYTHON3_CFFI),1) cd $(dir)/python-cffi && \ ${PYTHON} setup.py build --build-lib build/stage && \ - mkdir -p build/stage/tests && cp tests/*.py build/stage/tests + mkdir -p build/stage/tests && cp tests/*.py build/stage/tests && touch ../../$@ endif CLEAN += $(patsubst %,$(dir)/ruby/%, \ @@ -26,6 +28,6 @@ CLEAN += $(patsubst %,$(dir)/ruby/%, \ init.o message.o messages.o mkmf.log notmuch.so query.o \ status.o tags.o thread.o threads.o) -CLEAN += bindings/ruby/.vendorarchdir.time +CLEAN += bindings/ruby/.vendorarchdir.time $(dir)/ruby.stamp CLEAN += bindings/python-cffi/build From c01152885c565813aa9510481e425e7c61815b56 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 30 Oct 2021 17:48:55 -0300 Subject: [PATCH 024/113] python-cffi: introduce stamp file Although the rebuild does not take long, it is a bit noisy, so assume if it succeeds once, it doesn't need to re-invoke setup.py until the shared library is rebuilt. This is a partial fix for [1]. [1]: id:87r29wwgq2.fsf@fifthhorseman.net --- bindings/Makefile.local | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bindings/Makefile.local b/bindings/Makefile.local index 1cdd28a0..7b10af08 100644 --- a/bindings/Makefile.local +++ b/bindings/Makefile.local @@ -15,11 +15,14 @@ ifeq ($(HAVE_RUBY_DEV),1) $(MAKE) -C $(dir)/ruby CFLAGS="$(CFLAGS) -pipe -fno-plt -fPIC" && touch $@ endif -python-cffi-bindings: lib/$(LINKER_NAME) +python-cffi-bindings: $(dir)/python-cffi.stamp + +$(dir)/python-cffi.stamp: lib/$(LINKER_NAME) ifeq ($(HAVE_PYTHON3_CFFI),1) cd $(dir)/python-cffi && \ ${PYTHON} setup.py build --build-lib build/stage && \ - mkdir -p build/stage/tests && cp tests/*.py build/stage/tests && touch ../../$@ + mkdir -p build/stage/tests && cp tests/*.py build/stage/tests && \ + touch ../python-cffi.stamp endif CLEAN += $(patsubst %,$(dir)/ruby/%, \ @@ -30,4 +33,4 @@ CLEAN += $(patsubst %,$(dir)/ruby/%, \ CLEAN += bindings/ruby/.vendorarchdir.time $(dir)/ruby.stamp -CLEAN += bindings/python-cffi/build +CLEAN += bindings/python-cffi/build $(dir)/python-cffi.stamp From f17d75b83c90ae4ea75f79377f3acb873b9e564e Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 31 Oct 2021 07:03:54 -0300 Subject: [PATCH 025/113] python-cffi: fix out-of-tree build The main idea is to replace the hack of copying version.txt into the bindings source with a generated _notmuch_config.py file. This will mean that the bindings only build after configuring and building notmuch itself. Given those constraints, "pip install ." should work. --- Makefile.local | 1 - bindings/python-cffi/notmuch2/_build.py | 6 +++--- bindings/python-cffi/setup.py | 3 ++- bindings/python-cffi/version.txt | 1 - configure | 8 ++++++++ 5 files changed, 13 insertions(+), 6 deletions(-) delete mode 100644 bindings/python-cffi/version.txt diff --git a/Makefile.local b/Makefile.local index e12b94cd..10fb9908 100644 --- a/Makefile.local +++ b/Makefile.local @@ -54,7 +54,6 @@ update-versions: sed -i -e "s/^__VERSION__[[:blank:]]*=.*$$/__VERSION__ = \'${VERSION}\'/" \ -e "s/^SOVERSION[[:blank:]]*=.*$$/SOVERSION = \'${LIBNOTMUCH_VERSION_MAJOR}\'/" \ ${PV_FILE} - cp version.txt bindings/python-cffi # We invoke make recursively only to force ordering of our phony # targets in the case of parallel invocation of make (-j). diff --git a/bindings/python-cffi/notmuch2/_build.py b/bindings/python-cffi/notmuch2/_build.py index f6184b97..45eb20c0 100644 --- a/bindings/python-cffi/notmuch2/_build.py +++ b/bindings/python-cffi/notmuch2/_build.py @@ -1,5 +1,5 @@ import cffi - +from _notmuch_config import * ffibuilder = cffi.FFI() ffibuilder.set_source( @@ -16,8 +16,8 @@ ffibuilder.set_source( #ERROR libnotmuch version < 5.1 not supported #endif """, - include_dirs=['../../lib'], - library_dirs=['../../lib'], + include_dirs=[NOTMUCH_INCLUDE_DIR], + library_dirs=[NOTMUCH_LIB_DIR], libraries=['notmuch'], ) ffibuilder.cdef( diff --git a/bindings/python-cffi/setup.py b/bindings/python-cffi/setup.py index cda52338..55fb2d24 100644 --- a/bindings/python-cffi/setup.py +++ b/bindings/python-cffi/setup.py @@ -1,6 +1,7 @@ import setuptools +from _notmuch_config import * -with open('version.txt') as fp: +with open(NOTMUCH_VERSION_FILE) as fp: VERSION = fp.read().strip() setuptools.setup( diff --git a/bindings/python-cffi/version.txt b/bindings/python-cffi/version.txt deleted file mode 100644 index cd46610f..00000000 --- a/bindings/python-cffi/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.34.1 diff --git a/configure b/configure index 6c3a38f1..7d9df370 100755 --- a/configure +++ b/configure @@ -1579,6 +1579,14 @@ EOF printf "rsti_dir = '%s'\n" "$(cd emacs && pwd -P)" } > sphinx.config +cat > bindings/python-cffi/_notmuch_config.py < Date: Sun, 5 Dec 2021 10:40:07 -0400 Subject: [PATCH 026/113] test/emacs: mark one test as fixed in newer emacs. The remaining problem in this test is fixed upstream in Emacs 28. While most people are using earlier versions of emacs, the test still provides some documentation of a known bug. --- test/T450-emacs-show.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/T450-emacs-show.sh b/test/T450-emacs-show.sh index 4b5f5fde..057ad37e 100755 --- a/test/T450-emacs-show.sh +++ b/test/T450-emacs-show.sh @@ -220,7 +220,9 @@ test_emacs '(notmuch-show "id:basic-encrypted@crypto.notmuchmail.org") test_expect_equal_file $EXPECTED/notmuch-show-decrypted-message OUTPUT test_begin_subtest "show encrypted rfc822 message" -test_subtest_known_broken +if ${TEST_EMACS} --quick --batch --eval '(kill-emacs (if (version< emacs-version "28") 0 1))'; then + test_subtest_known_broken +fi test_emacs '(notmuch-show "id:encrypted-rfc822-attachment@crypto.notmuchmail.org") (test-visible-output)' test_expect_code 1 'fgrep "!!!" OUTPUT' From ab8d0e572537006a4e1dafa266075ed0d848c80f Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Tue, 7 Dec 2021 21:11:41 +0200 Subject: [PATCH 027/113] configure: have bash_absolute and perl_absolute always defined Since set -u is used, without bash or perl, configure would fail. This has gone unnoticed as (almost) everyone always had both bash and perl installed (and in $PATH). Thanks to FreeBSD ports this bug became visible; this change is verbatim copy of `patch-configure` in FreeBSD ports tree. --- configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure b/configure index 7d9df370..d399457a 100755 --- a/configure +++ b/configure @@ -734,6 +734,7 @@ if command -v ${BASHCMD} > /dev/null; then printf "Yes (%s).\n" "$bash_absolute" else have_bash=0 + bash_absolute= printf "No. (%s not found)\n" "${BASHCMD}" fi @@ -744,6 +745,7 @@ if command -v ${PERL} > /dev/null; then printf "Yes (%s).\n" "$perl_absolute" else have_perl=0 + perl_absolute= printf "No. (%s not found)\n" "${PERL}" fi From ed03babd053d679a85ea3baa1632d8ae1dff31b6 Mon Sep 17 00:00:00 2001 From: Kyle Meyer Date: Sun, 5 Dec 2021 15:48:36 -0500 Subject: [PATCH 028/113] emacs/tree: fix docstrings for author faces The docstrings for notmuch-tree-match-author-face and notmuch-tree-no-match-author-face incorrectly match the docstring of notmuch-tree-match-date-face. --- emacs/notmuch-tree.el | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el index 7fa73d40..d7486904 100644 --- a/emacs/notmuch-tree.el +++ b/emacs/notmuch-tree.el @@ -179,7 +179,7 @@ Note that the author string should not contain whitespace (:foreground "dark blue")) (t (:bold t))) - "Face used in tree mode for the date in messages matching the query." + "Face used in tree mode for the author in messages matching the query." :group 'notmuch-tree :group 'notmuch-faces) @@ -236,7 +236,7 @@ Note that the author string should not contain whitespace (defface notmuch-tree-no-match-author-face nil - "Face used in tree mode for the date in messages matching the query." + "Face used in tree mode for non-matching authors." :group 'notmuch-tree :group 'notmuch-faces) From d825847b522f2b84abf087d90b57275502f37163 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 4 Dec 2021 19:47:50 -0400 Subject: [PATCH 029/113] doc: replace phony target with variable Depending on a phony target seems like a good way to always trigger a recipe. --- doc/Makefile.local | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/Makefile.local b/doc/Makefile.local index 730ad4fb..1782c784 100644 --- a/doc/Makefile.local +++ b/doc/Makefile.local @@ -35,7 +35,7 @@ endif INFO_INFO_FILES := $(INFO_TEXI_FILES:.texi=.info) -.PHONY: sphinx-html sphinx-texinfo sphinx-info doc-prereqs +.PHONY: sphinx-html sphinx-texinfo sphinx-info .PHONY: install-man build-man apidocs install-apidocs @@ -47,18 +47,20 @@ $(DOCBUILDDIR)/.roff.stamp $(DOCBUILDDIR)/.html.stamp $(DOCBUILDDIR)/.texi.stamp endif ifeq ($(HAVE_PYTHON3_CFFI),1) -doc-prereqs: python-cffi-bindings +DOC_PREREQS=bindings/python-cffi.stamp +else +DOC_PREREQS= endif sphinx-html: $(DOCBUILDDIR)/.html.stamp -$(DOCBUILDDIR)/.html.stamp: $(ALL_RST_FILES) doc-prereqs +$(DOCBUILDDIR)/.html.stamp: $(ALL_RST_FILES) $(DOC_PREREQS) $(SPHINXBUILD) -b html -d $(DOCBUILDDIR)/html_doctrees $(ALLSPHINXOPTS) $(DOCBUILDDIR)/html touch $@ sphinx-texinfo: $(DOCBUILDDIR)/.texi.stamp -$(DOCBUILDDIR)/.texi.stamp: $(ALL_RST_FILES) doc-prereqs +$(DOCBUILDDIR)/.texi.stamp: $(ALL_RST_FILES) $(DOC_PREREQS) $(SPHINXBUILD) -b texinfo -d $(DOCBUILDDIR)/texinfo_doctrees $(ALLSPHINXOPTS) $(DOCBUILDDIR)/texinfo touch $@ From b7e08901e8ab695fb75de0528871771009f49aa8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 4 Dec 2021 19:47:51 -0400 Subject: [PATCH 030/113] doc: introduce stamp file for info build This partially fixes (i.e. just for sphinx) the problem reported by Daniel in id:87r29wwgq2.fsf@fifthhorseman.net. --- doc/Makefile.local | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/Makefile.local b/doc/Makefile.local index 1782c784..c2ae1743 100644 --- a/doc/Makefile.local +++ b/doc/Makefile.local @@ -64,8 +64,11 @@ $(DOCBUILDDIR)/.texi.stamp: $(ALL_RST_FILES) $(DOC_PREREQS) $(SPHINXBUILD) -b texinfo -d $(DOCBUILDDIR)/texinfo_doctrees $(ALLSPHINXOPTS) $(DOCBUILDDIR)/texinfo touch $@ -sphinx-info: sphinx-texinfo +sphinx-info: $(DOCBUILDDIR)/.info.stamp + +$(DOCBUILDDIR)/.info.stamp: $(DOCBUILDDIR)/.texi.stamp $(DOC_PREREQS) $(MAKE) -C $(DOCBUILDDIR)/texinfo info + touch $@ # Use the man page converter that is available. We should never depend # on MAN_ROFF_FILES if a converter is not available. @@ -129,7 +132,7 @@ ifneq ($(HAVE_SPHINX)$(HAVE_MAKEINFO),11) build-info: @echo "Missing sphinx or makeinfo, not building info pages" else -build-info: sphinx-info +build-info: $(DOCBUILDDIR)/.info.stamp endif ifneq ($(HAVE_SPHINX)$(HAVE_MAKEINFO)$(HAVE_INSTALL_INFO),111) @@ -147,5 +150,5 @@ $(dir)/config.dox: version.stamp echo "INPUT=${srcdir}/lib/notmuch.h" >> $@ CLEAN := $(CLEAN) $(DOCBUILDDIR) $(DOCBUILDDIR)/.roff.stamp $(DOCBUILDDIR)/.texi.stamp -CLEAN := $(CLEAN) $(DOCBUILDDIR)/.html.stamp +CLEAN := $(CLEAN) $(DOCBUILDDIR)/.html.stamp $(DOCBUILDDIR)/.info.stamp CLEAN := $(CLEAN) $(MAN_GZIP_FILES) $(MAN_ROFF_FILES) $(dir)/conf.pyc $(dir)/config.dox From 031f4b4da5b317c580df474d002a8300d35a818e Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Wed, 22 Dec 2021 01:10:39 +0200 Subject: [PATCH 031/113] emacs: add notmuch-logo.svg and use it in emacs mua on graphic displays emacs/notmuch-logo.svg is handcrafted scalable vector graphics version of the notmuch logo. Emacs on graphic displays render this image four times in size compared to the emacs/notmuch-logo.png, and the image is much sharper. The rendered image size, 100x100 pixels, fits reasonably well with text shown in various font sizes. Scaling the image -- if desired -- may come in the future. For now this is improvement on how the logo looked in notmuch-hello window. --- debian/elpa-notmuch.elpa | 2 +- emacs/Makefile.local | 2 +- emacs/notmuch-hello.el | 2 +- emacs/notmuch-logo.svg | 27 +++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 emacs/notmuch-logo.svg diff --git a/debian/elpa-notmuch.elpa b/debian/elpa-notmuch.elpa index 4712b73f..7b3ce0fa 100644 --- a/debian/elpa-notmuch.elpa +++ b/debian/elpa-notmuch.elpa @@ -1,3 +1,3 @@ debian/tmp/usr/share/emacs/site-lisp/*.el -debian/tmp/usr/share/emacs/site-lisp/notmuch-logo.png +debian/tmp/usr/share/emacs/site-lisp/notmuch-logo.svg emacs/notmuch-pkg.el diff --git a/emacs/Makefile.local b/emacs/Makefile.local index d1b320c3..0f1f0eb2 100644 --- a/emacs/Makefile.local +++ b/emacs/Makefile.local @@ -42,7 +42,7 @@ emacs_mua := $(dir)/notmuch-emacs-mua emacs_mua_desktop := $(dir)/notmuch-emacs-mua.desktop emacs_images := \ - $(srcdir)/$(dir)/notmuch-logo.png + $(srcdir)/$(dir)/notmuch-logo.svg emacs_bytecode = $(emacs_sources:.el=.elc) emacs_docstrings = $(emacs_sources:.el=.rsti) diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index acd48c9b..fc007c4c 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -198,7 +198,7 @@ fields of the search." (defvar notmuch-hello-indent 4 "How much to indent non-headers.") -(defimage notmuch-hello-logo ((:type png :file "notmuch-logo.png"))) +(defimage notmuch-hello-logo ((:type svg :file "notmuch-logo.svg"))) (defcustom notmuch-show-logo t "Should the notmuch logo be shown?" diff --git a/emacs/notmuch-logo.svg b/emacs/notmuch-logo.svg new file mode 100644 index 00000000..2c65a73b --- /dev/null +++ b/emacs/notmuch-logo.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + From cea1604a087645d07998c6986a8678b2af239322 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 5 Dec 2021 09:56:03 -0400 Subject: [PATCH 032/113] test: remove directory names from paths in exceptions These cause failures when building out of tree. --- test/T562-lib-database.sh | 4 ++-- test/T563-lib-directory.sh | 4 ++-- test/test-lib.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/T562-lib-database.sh b/test/T562-lib-database.sh index d9f5d18e..2314efd2 100755 --- a/test/T562-lib-database.sh +++ b/test/T562-lib-database.sh @@ -80,7 +80,7 @@ cat < EXPECTED == stdout == 0 == stderr == -A Xapian exception occurred at lib/database.cc:XXX: Database has been closed +A Xapian exception occurred at database.cc:XXX: Database has been closed EOF test_expect_equal_file EXPECTED OUTPUT @@ -145,7 +145,7 @@ cat < EXPECTED == stdout == 1 == stderr == -A Xapian exception occurred at lib/database.cc:XXX: Database has been closed +A Xapian exception occurred at database.cc:XXX: Database has been closed EOF test_expect_equal_file EXPECTED OUTPUT diff --git a/test/T563-lib-directory.sh b/test/T563-lib-directory.sh index ad390c1c..ebd7fcb2 100755 --- a/test/T563-lib-directory.sh +++ b/test/T563-lib-directory.sh @@ -51,7 +51,7 @@ cat < EXPECTED == stdout == 1 == stderr == -A Xapian exception occurred at lib/directory.cc:XXX: Database has been closed +A Xapian exception occurred at directory.cc:XXX: Database has been closed EOF test_expect_equal_file EXPECTED OUTPUT @@ -68,7 +68,7 @@ cat < EXPECTED == stdout == 1 == stderr == -A Xapian exception occurred at lib/directory.cc:XXX: Database has been closed +A Xapian exception occurred at directory.cc:XXX: Database has been closed EOF test_expect_equal_file EXPECTED OUTPUT diff --git a/test/test-lib.sh b/test/test-lib.sh index e476a69b..6bc0b723 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -529,7 +529,7 @@ notmuch_debug_sanitize () { } notmuch_exception_sanitize () { - perl -pe 's/(A Xapian exception occurred at .*[.]cc?):([0-9]*)/\1:XXX/' + perl -pe 's,(A Xapian exception occurred at) .*?([^/]*[.]cc?):([0-9]*),\1 \2:XXX,' } notmuch_search_sanitize () { From 02d8ff376d77e5d96389c30576221a7ac5b4bea1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 24 Dec 2021 12:20:31 -0400 Subject: [PATCH 033/113] doc: add dep. on stamp file for rebuilding gzipped man pages. In [1] Daniel observed that the gzipped man pages were only being rebuild every second time when building with `make -j4'. This may be caused by a race condition between sphinx-build rebuilding the roff files and the recipe to gzip them. This commit sequentializes these two steps by making the stamp file a prerequisite for (all of) the gzip files. [1]: id:87tveotn1g.fsf@fifthhorseman.net --- doc/Makefile.local | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/Makefile.local b/doc/Makefile.local index c2ae1743..d43ef269 100644 --- a/doc/Makefile.local +++ b/doc/Makefile.local @@ -117,6 +117,11 @@ build-man: install-man: @echo "No sphinx, will not install man pages." else + +# it should be safe to depend on the stamp file, because it is created +# after all roff files are moved into place. +${MAN_GZIP_FILES}: ${DOCBUILDDIR}/.roff.stamp + build-man: ${MAN_GZIP_FILES} install-man: ${MAN_GZIP_FILES} mkdir -m0755 -p "$(DESTDIR)$(mandir)/man1" From 063f5e98620e2f2adeaa8b3313cdac85eb4ef4db Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 26 Dec 2021 12:20:44 -0400 Subject: [PATCH 034/113] test: test folder renames In [1] Mark Walters reported a problem with messages being removed from the database when the parent directory was renamed. Jani Nikula proposed [2] these tests but observed This test is not suitable for merging since it's not deterministic. After applying Jani's patch [3], the tests now pass deterministically, and could usefully act as regression tests. [1]: id:87siray6th.fsf@qmul.ac.uk [2]: id:1393191650-28333-1-git-send-email-jani@nikula.org [3]: id:1441445731-4362-2-git-send-email-jani@nikula.org --- test/T051-new-renames.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 test/T051-new-renames.sh diff --git a/test/T051-new-renames.sh b/test/T051-new-renames.sh new file mode 100755 index 00000000..ebd06be1 --- /dev/null +++ b/test/T051-new-renames.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +test_description='"notmuch new" with directory renames' +. $(dirname "$0")/test-lib.sh || exit 1 + +for loop in {1..10}; do + +rm -rf ${MAIL_DIR} + +for i in {1..10}; do + generate_message '[dir]=foo' '[subject]="Message foo $i"' +done + +for i in {1..10}; do + generate_message '[dir]=bar' '[subject]="Message bar $i"' +done + +test_begin_subtest "Index the messages, round $loop" +output=$(NOTMUCH_NEW) +test_expect_equal "$output" "Added 20 new messages to the database." + +all_files=$(notmuch search --output=files \*) +count_foo=$(notmuch count folder:foo) + +test_begin_subtest "Rename folder" +mv ${MAIL_DIR}/foo ${MAIL_DIR}/baz +output=$(NOTMUCH_NEW) +test_expect_equal "$output" "No new mail. Detected $count_foo file renames." + +test_begin_subtest "Rename folder back" +mv ${MAIL_DIR}/baz ${MAIL_DIR}/foo +output=$(NOTMUCH_NEW) +test_expect_equal "$output" "No new mail. Detected $count_foo file renames." + +test_begin_subtest "Files remain the same" +output=$(notmuch search --output=files \*) +test_expect_equal "$output" "$all_files" + +done + +test_done From e43bad4883cf8a0c1d58d50238da599f9e170307 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 17:22:55 -0400 Subject: [PATCH 035/113] test/new: add known broken test for missing xapian directory. `notmuch new' should go ahead and create the xapian database if it is missing, even in the case where the parent .notmuch (or equivalent) directory exists. --- test/T055-path-config.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/T055-path-config.sh b/test/T055-path-config.sh index 6d9fb402..b6858d42 100755 --- a/test/T055-path-config.sh +++ b/test/T055-path-config.sh @@ -318,7 +318,15 @@ to=m.header('To') print(to) EOF test_expect_equal_file EXPECTED OUTPUT - ;& # fall through + ;; + *) + backup_database + test_begin_subtest ".notmuch without xapian/ handled gracefully ($config)" + test_subtest_known_broken + rm -r $XAPIAN_PATH + test_expect_success "notmuch new" + restore_database + ;; esac case $config in From 25e0f5e59293ce961549201fdc74f81a3cc1675c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 17:22:56 -0400 Subject: [PATCH 036/113] lib/open: do not consider .notmuch alone as an existing database. It makes perfect sense for users to want to pre-create .notmuch, e.g. to install hooks, so we should handle the case of a .notmuch directory without an actual xapian database more gracefully. --- lib/open.cc | 8 ++------ test/T055-path-config.sh | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/open.cc b/lib/open.cc index a942383b..4dd62a9f 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -662,16 +662,12 @@ notmuch_database_create_with_config (const char *database_path, err = mkdir (notmuch_path, 0755); if (err) { - if (errno == EEXIST) { - status = NOTMUCH_STATUS_DATABASE_EXISTS; - talloc_free (notmuch); - notmuch = NULL; - } else { + if (errno != EEXIST) { IGNORE_RESULT (asprintf (&message, "Error: Cannot create directory %s: %s.\n", notmuch_path, strerror (errno))); status = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; } - goto DONE; } } diff --git a/test/T055-path-config.sh b/test/T055-path-config.sh index b6858d42..b910e28e 100755 --- a/test/T055-path-config.sh +++ b/test/T055-path-config.sh @@ -322,7 +322,6 @@ EOF *) backup_database test_begin_subtest ".notmuch without xapian/ handled gracefully ($config)" - test_subtest_known_broken rm -r $XAPIAN_PATH test_expect_success "notmuch new" restore_database From 14c4f9441d7fe0893003a5e793e19f1e55c9f73f Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 22 Dec 2021 21:44:45 -0400 Subject: [PATCH 037/113] configure: calculate NOTMUCH_BUILDDIR, write to Makefile.config This will correct the current use of an undefined variable when setting LD_LIBRARY_PATH in doc/Makefile.local It is tempting to try to replace the use of test/export-dirs.sh, but this is not as simple as it looks, as NOTMUCH_BUILDDIR is used to locate sh.config, so probably cannot also sensibly be used to define it. --- configure | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure b/configure index 6c3a38f1..9b692b3e 100755 --- a/configure +++ b/configure @@ -55,6 +55,8 @@ subdirs="${subdirs} bindings" # the directory structure and copy Makefiles. if [ "$srcdir" != "." ]; then + NOTMUCH_BUILDDIR=$PWD + for dir in . ${subdirs}; do mkdir -p "$dir" cp "$srcdir"/"$dir"/Makefile.local "$dir" @@ -78,6 +80,8 @@ if [ "$srcdir" != "." ]; then "$srcdir"/bindings/python-cffi/notmuch2 \ "$srcdir"/bindings/python-cffi/setup.py \ bindings/python-cffi/ +else + NOTMUCH_BUILDDIR=$NOTMUCH_SRCDIR fi # Set several defaults (optionally specified by the user in @@ -1245,6 +1249,7 @@ cat > Makefile.config < Date: Sat, 25 Dec 2021 19:31:12 -0400 Subject: [PATCH 038/113] emacs: update coolj-line-prefix-regexp to make space optional Jani reported that lines prefixed with '>' (as opposed to '> ') are not highlighted properly [1]. David E responded with a updated regex [2]. This change implements David E's suggestion. [1]: id:87a8b5pcky.fsf@nikula.org [2]: id:m2pok1e3gv.fsf@dme.org --- emacs/coolj.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emacs/coolj.el b/emacs/coolj.el index d820525b..79d2a1b7 100644 --- a/emacs/coolj.el +++ b/emacs/coolj.el @@ -45,7 +45,7 @@ Otherwise respect `fill-column'." :group 'coolj :type 'boolean) -(defcustom coolj-line-prefix-regexp "^\\(>+ \\)*" +(defcustom coolj-line-prefix-regexp "^\\(>+ ?\\)*" "Regular expression that matches line prefixes." :group 'coolj :type 'regexp) From 2494e61b9ee6bbab4ac017136d7c6ee0de19b26d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 24 Dec 2021 14:29:03 -0400 Subject: [PATCH 039/113] build: move LDFLAGS after notmuch libraries. In [1] Ryan Schmidt reported a problem on macports [2] with notmuch finding an existing installed version of libnotmuch during the build when the user specified LDFLAGS including the libnotmuch install directory. This change should prevent that. LDFLAGS also occurs in FINAL_LIBNOTMUCH_LDFLAGS. The only built library linked to that is util/libtnotmuch_util.a, and that passed as explicit (relative) path, and is thus not affected by -L. [1]: id:7851CAB5-4556-4931-A0A2-37003E56C927@ryandesign.com [2]: The problem does not arise when libnotmuch is installed into a "system" library path that the compiler/linker searches by default. --- Makefile.global | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.global b/Makefile.global index fe79121d..e6b00815 100644 --- a/Makefile.global +++ b/Makefile.global @@ -52,7 +52,7 @@ PV_FILE=bindings/python/notmuch/version.py # Smash together user's values with our extra values FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS) FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS) -FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lnotmuch_util -Llib -lnotmuch +FINAL_NOTMUCH_LDFLAGS = -Lutil -lnotmuch_util -Llib -lnotmuch $(LDFLAGS) ifeq ($(LIBDIR_IN_LDCONFIG),0) FINAL_NOTMUCH_LDFLAGS += $(RPATH_LDFLAGS) endif From 911d9a916e4dc2de7348d138aabd0ebe51242efd Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 24 Dec 2021 14:29:04 -0400 Subject: [PATCH 040/113] build: move CPPFLAGS after source directory includes. In [1] Ryan Schmidt reported a problem on macports [2] with notmuch finding an existing installed version of notmuch.h during the build. This is a partial fix; the user might also specify -I in CFLAGS. [1]: id:7851CAB5-4556-4931-A0A2-37003E56C927@ryandesign.com [2]: The problem does not arise when notmuch.h is installed into a "system" include path that the compiler searches by default. --- Makefile.global | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.global b/Makefile.global index e6b00815..b712dd7b 100644 --- a/Makefile.global +++ b/Makefile.global @@ -50,8 +50,8 @@ DETACHED_SIG_FILE=$(TAR_FILE).asc PV_FILE=bindings/python/notmuch/version.py # Smash together user's values with our extra values -FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS) -FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS) +FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CPPFLAGS) $(CONFIGURE_CFLAGS) +FINAL_CXXFLAGS = $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CPPFLAGS) $(CONFIGURE_CXXFLAGS) FINAL_NOTMUCH_LDFLAGS = -Lutil -lnotmuch_util -Llib -lnotmuch $(LDFLAGS) ifeq ($(LIBDIR_IN_LDCONFIG),0) FINAL_NOTMUCH_LDFLAGS += $(RPATH_LDFLAGS) From e9c55864cde6e9b9653b9963a36f633f34a57779 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 24 Dec 2021 14:29:05 -0400 Subject: [PATCH 041/113] build: move {C,CXX}FLAGS to the end of FINAL_{C,CXX}FLAGS In addition to avoiding problems with user specified include paths picking up an installed version of notmuch.h, this should also enable users to override more options (in particular they could override warning options since the last one takes effect). [1]: id:7851CAB5-4556-4931-A0A2-37003E56C927@ryandesign.com --- Makefile.global | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.global b/Makefile.global index b712dd7b..7a7a3c6d 100644 --- a/Makefile.global +++ b/Makefile.global @@ -50,8 +50,8 @@ DETACHED_SIG_FILE=$(TAR_FILE).asc PV_FILE=bindings/python/notmuch/version.py # Smash together user's values with our extra values -FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CPPFLAGS) $(CONFIGURE_CFLAGS) -FINAL_CXXFLAGS = $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CPPFLAGS) $(CONFIGURE_CXXFLAGS) +FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(WARN_CFLAGS) $(extra_cflags) $(CPPFLAGS) $(CONFIGURE_CFLAGS) $(CFLAGS) +FINAL_CXXFLAGS = $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CPPFLAGS) $(CONFIGURE_CXXFLAGS) $(CXXFLAGS) FINAL_NOTMUCH_LDFLAGS = -Lutil -lnotmuch_util -Llib -lnotmuch $(LDFLAGS) ifeq ($(LIBDIR_IN_LDCONFIG),0) FINAL_NOTMUCH_LDFLAGS += $(RPATH_LDFLAGS) From 9cc026f3daaa7731527787f8c7e729c0a08c456c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 1 Jan 2022 10:36:43 -0400 Subject: [PATCH 042/113] bindings/python-cffi: add matched property to message objects Existing users of the legacy python bindings use message.get_flags(Message.FLAG.MATCH) to determine which messages in a thread matched. Since the bindings don't provide get_flags anymore, they should provide a property analogous to the existing "excluded" property. --- bindings/python-cffi/notmuch2/_message.py | 14 ++++++++++++++ bindings/python-cffi/tests/test_message.py | 3 +++ bindings/python-cffi/tests/test_thread.py | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/bindings/python-cffi/notmuch2/_message.py b/bindings/python-cffi/notmuch2/_message.py index 2f232076..a460d8c1 100644 --- a/bindings/python-cffi/notmuch2/_message.py +++ b/bindings/python-cffi/notmuch2/_message.py @@ -205,6 +205,20 @@ class Message(base.NotmuchObject): self._msg_p, capi.lib.NOTMUCH_MESSAGE_FLAG_EXCLUDED) return bool(ret) + @property + def matched(self): + """Indicates whether this message was matched by the query. + + When a thread is created from a search, some of the + messages may not match the original query. This property + is set to *True* for those that do match. + + :raises ObjectDestroyedError: if used after destroyed. + """ + ret = capi.lib.notmuch_message_get_flag( + self._msg_p, capi.lib.NOTMUCH_MESSAGE_FLAG_MATCH) + return bool(ret) + @property def date(self): """The message date as an integer. diff --git a/bindings/python-cffi/tests/test_message.py b/bindings/python-cffi/tests/test_message.py index 532bf921..56701d05 100644 --- a/bindings/python-cffi/tests/test_message.py +++ b/bindings/python-cffi/tests/test_message.py @@ -97,6 +97,9 @@ class TestMessage: def test_ghost_no(self, msg): assert not msg.ghost + def test_matched_no(self,msg): + assert not msg.matched + def test_date(self, msg): # XXX Someone seems to treat things as local time instead of # UTC or the other way around. diff --git a/bindings/python-cffi/tests/test_thread.py b/bindings/python-cffi/tests/test_thread.py index 1f44b35d..fbef73ac 100644 --- a/bindings/python-cffi/tests/test_thread.py +++ b/bindings/python-cffi/tests/test_thread.py @@ -57,6 +57,13 @@ def test_iter(thread): def test_matched(thread): assert thread.matched == 1 +def test_matched_iter(thread): + count = 0 + msgs = list(iter(thread)) + for msg in msgs: + if msg.matched: + count += 1 + assert count == thread.matched def test_authors_type(thread): assert isinstance(thread.authors, notmuch2.BinString) From f9ffc5f433ca9c1cb1e9b9929e4f39e549910328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Hannotier?= Date: Thu, 16 Dec 2021 18:40:52 +0100 Subject: [PATCH 043/113] doc: mail store is given by database.mail_root The mail store directory is given by database.mail_root, which can be different from database.path. However, notmuch-insert documentation was still referencing the latter as the provider of the maildir directory instead of the former. --- doc/man1/notmuch-insert.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst index 82c4a7a0..da9ca791 100644 --- a/doc/man1/notmuch-insert.rst +++ b/doc/man1/notmuch-insert.rst @@ -14,7 +14,7 @@ DESCRIPTION **notmuch insert** reads a message from standard input and delivers it into the maildir directory given by configuration option -**database.path**, then incorporates the message into the notmuch +**database.mail_root**, then incorporates the message into the notmuch database. It is an alternative to using a separate tool to deliver the message then running :any:`notmuch-new(1)` afterwards. @@ -38,7 +38,7 @@ Supported options for **insert** include .. option:: --folder= Deliver the message to the specified folder, relative to the - top-level directory given by the value of **database.path**. The + top-level directory given by the value of **database.mail_root**. The default is the empty string, which means delivering to the top-level directory. From d99b0d4dc8b9262373e2d0ae158dd8336fc28e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Hannotier?= Date: Thu, 16 Dec 2021 18:40:53 +0100 Subject: [PATCH 044/113] completion: use mail_root for path completion in bash/zsh Since mail store and database directory can be different, path and folder completions must use the directory given by database.mail_root, not by database.path. --- completion/notmuch-completion.bash | 6 +++--- completion/zsh/_notmuch | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 15425697..0022b54b 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -103,12 +103,12 @@ _notmuch_search_terms() COMPREPLY=( $(compgen -P "from:" -W "`_notmuch_email ${cur}`" -- ${cur##from:}) ) ;; path:*) - local path=`notmuch config get database.path` + local path=`notmuch config get database.mail_root` compopt -o nospace COMPREPLY=( $(compgen -d "$path/${cur##path:}" | sed "s|^$path/||" ) ) ;; folder:*) - local path=`notmuch config get database.path` + local path=`notmuch config get database.mail_root` compopt -o nospace COMPREPLY=( $(compgen -d "$path/${cur##folder:}" | \ sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) ) @@ -281,7 +281,7 @@ _notmuch_insert() $split && case "${prev}" in --folder) - local path=`notmuch config get database.path` + local path=`notmuch config get database.mail_root` compopt -o nospace COMPREPLY=( $(compgen -d "$path/${cur}" | \ sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) ) diff --git a/completion/zsh/_notmuch b/completion/zsh/_notmuch index e920f10b..e207d90b 100644 --- a/completion/zsh/_notmuch +++ b/completion/zsh/_notmuch @@ -69,8 +69,8 @@ _notmuch_term_mimetype() { _notmuch_term_path() { local ret=1 expl - local maildir="$(notmuch config get database.path)" - [[ -d $maildir ]] || { _message -e "database.path not found" ; return $ret } + local maildir="$(notmuch config get database.mail_root)" + [[ -d $maildir ]] || { _message -e "database.mail_root not found" ; return $ret } _description notmuch-folder expl 'maildir folder' _files "$expl[@]" -W $maildir -/ && ret=0 @@ -79,8 +79,8 @@ _notmuch_term_path() { _notmuch_term_folder() { local ret=1 expl - local maildir="$(notmuch config get database.path)" - [[ -d $maildir ]] || { _message -e "database.path not found" ; return $ret } + local maildir="$(notmuch config get database.mail_root)" + [[ -d $maildir ]] || { _message -e "database.mail_root not found" ; return $ret } _description notmuch-folder expl 'maildir folder' local ignoredfolders=( '*/(cur|new|tmp)' ) From d9a2b900b6525874b913276af91840983d81b3f1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 8 Jan 2022 10:03:15 -0400 Subject: [PATCH 045/113] test: add known broken tests for recursive traversal of replies. This reproduces the bug reported at [1]. The second test hints at the solution, making reply return OwnedMessage objects. [1]: id:87sfu6utxg.fsf@tethera.net --- test/T392-python-cffi-notmuch.sh | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 test/T392-python-cffi-notmuch.sh diff --git a/test/T392-python-cffi-notmuch.sh b/test/T392-python-cffi-notmuch.sh new file mode 100755 index 00000000..50012c55 --- /dev/null +++ b/test/T392-python-cffi-notmuch.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +test_description="python bindings (notmuch test suite)" +. $(dirname "$0")/test-lib.sh || exit 1 + +if [ $NOTMUCH_HAVE_PYTHON3_CFFI -eq 0 -o $NOTMUCH_HAVE_PYTHON3_PYTEST -eq 0 ]; then + test_done +fi + +add_email_corpus + +cat < recurse.py +from notmuch2 import Database +def show_msgs(msgs, level): + print('{:s} {:s}'.format(' ' * level*4, type(msgs).__name__)) + for msg in msgs: + print('{:s} {:s}'.format(' ' * (level*4+2), type(msg).__name__)) + replies=msg.replies() + show_msgs(replies, level+1) +db = Database(config=Database.CONFIG.SEARCH) +msg=db.find("87ocn0qh6d.fsf@yoom.home.cworth.org") +threads = db.threads(query="thread:"+msg.threadid) +thread = next (threads) +show_msgs(thread, 0) +EOF + +test_begin_subtest "recursive traversal of replies (no crash)" +test_subtest_known_broken +test_python < recurse.py +error=$? +test_expect_equal "${error}" 0 + +test_begin_subtest "recursive traversal of replies (output)" +test_subtest_known_broken +test_python < recurse.py +tail -n 10 < OUTPUT > OUTPUT.sample +cat < EXPECTED + OwnedMessage + MessageIter + OwnedMessage + MessageIter + OwnedMessage + MessageIter + OwnedMessage + MessageIter + OwnedMessage + MessageIter +EOF +test_expect_equal_file EXPECTED OUTPUT.sample + +test_done From 9e7ea628e6bddbd7345d053a3daf14af74896cc2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 8 Jan 2022 10:03:16 -0400 Subject: [PATCH 046/113] python-cffi: returned OwnedMessage objects from Message.replies If we return regular Message objects, python will try to destroy them, and the underlying notmuch object, causing e.g. the crash [1]. [1]: id:87sfu6utxg.fsf@tethera.net --- bindings/python-cffi/notmuch2/_message.py | 4 ++-- test/T392-python-cffi-notmuch.sh | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bindings/python-cffi/notmuch2/_message.py b/bindings/python-cffi/notmuch2/_message.py index 2f232076..b4f651fb 100644 --- a/bindings/python-cffi/notmuch2/_message.py +++ b/bindings/python-cffi/notmuch2/_message.py @@ -357,14 +357,14 @@ class Message(base.NotmuchObject): This method will only work if the message was created from a thread. Otherwise it will yield no results. - :returns: An iterator yielding :class:`Message` instances. + :returns: An iterator yielding :class:`OwnedMessage` instances. :rtype: MessageIter """ # The notmuch_messages_valid call accepts NULL and this will # become an empty iterator, raising StopIteration immediately. # Hence no return value checking here. msgs_p = capi.lib.notmuch_message_get_replies(self._msg_p) - return MessageIter(self, msgs_p, db=self._db) + return MessageIter(self, msgs_p, db=self._db, msg_cls=OwnedMessage) def __hash__(self): return hash(self.messageid) diff --git a/test/T392-python-cffi-notmuch.sh b/test/T392-python-cffi-notmuch.sh index 50012c55..15c8fc6b 100755 --- a/test/T392-python-cffi-notmuch.sh +++ b/test/T392-python-cffi-notmuch.sh @@ -24,13 +24,11 @@ show_msgs(thread, 0) EOF test_begin_subtest "recursive traversal of replies (no crash)" -test_subtest_known_broken test_python < recurse.py error=$? test_expect_equal "${error}" 0 test_begin_subtest "recursive traversal of replies (output)" -test_subtest_known_broken test_python < recurse.py tail -n 10 < OUTPUT > OUTPUT.sample cat < EXPECTED From 8b737af28bc377db3e661a5744f3b7479b7ce485 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 8 Jan 2022 17:21:43 -0400 Subject: [PATCH 047/113] bindings/python-cffi: search for config by default The previous (pre-0.34.2) constructor searched for a config file but only if the database path was not specified, and only to retrieve database.path. Neither of the available options (CONFIG.SEARCH or CONFIG.NONE) matches this semantics exactly, but CONFIG.SEARCH causes less breakage for people who relied on the old behaviour to set their database.path [1]. Since it also seems like the friendlier option in the long run, this commit switches to CONFIG.SEARCH as default. This requires a certain amount of updating the pytest tests, but most users will actually have a config file, unlike the test environment. [1]: id:87fsqijx7u.fsf@metapensiero.it --- bindings/python-cffi/notmuch2/_database.py | 2 +- bindings/python-cffi/tests/test_config.py | 4 ++-- bindings/python-cffi/tests/test_database.py | 4 ++-- bindings/python-cffi/tests/test_tags.py | 15 +++++++++------ bindings/python-cffi/tests/test_thread.py | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bindings/python-cffi/notmuch2/_database.py b/bindings/python-cffi/notmuch2/_database.py index 14a8f15c..d7485b4d 100644 --- a/bindings/python-cffi/notmuch2/_database.py +++ b/bindings/python-cffi/notmuch2/_database.py @@ -139,7 +139,7 @@ class Database(base.NotmuchObject): path = os.fsencode(path) return path - def __init__(self, path=None, mode=MODE.READ_ONLY, config=CONFIG.EMPTY): + def __init__(self, path=None, mode=MODE.READ_ONLY, config=CONFIG.SEARCH): if isinstance(mode, str): mode = self.STR_MODE_MAP[mode] self.mode = mode diff --git a/bindings/python-cffi/tests/test_config.py b/bindings/python-cffi/tests/test_config.py index 1b2695f5..67b0dea4 100644 --- a/bindings/python-cffi/tests/test_config.py +++ b/bindings/python-cffi/tests/test_config.py @@ -23,9 +23,9 @@ class TestIter: def test_set_get(self, maildir): # Ensure get-set works from different db objects - with dbmod.Database.create(maildir.path) as db0: + with dbmod.Database.create(maildir.path, config=dbmod.Database.CONFIG.EMPTY) as db0: db0.config['spam'] = 'ham' - with dbmod.Database(maildir.path) as db1: + with dbmod.Database(maildir.path, config=dbmod.Database.CONFIG.EMPTY) as db1: assert db1.config['spam'] == 'ham' def test_get_keyerror(self, db): diff --git a/bindings/python-cffi/tests/test_database.py b/bindings/python-cffi/tests/test_database.py index 9b3219c0..f1d12ea6 100644 --- a/bindings/python-cffi/tests/test_database.py +++ b/bindings/python-cffi/tests/test_database.py @@ -13,7 +13,7 @@ import notmuch2._message as message @pytest.fixture def db(maildir): - with dbmod.Database.create(maildir.path) as db: + with dbmod.Database.create(maildir.path, config=notmuch2.Database.CONFIG.EMPTY) as db: yield db @@ -293,7 +293,7 @@ class TestQuery: maildir.deliver(body='baz', headers=[('In-Reply-To', '<{}>'.format(msgid))]) notmuch('new') - with dbmod.Database(maildir.path, 'rw') as db: + with dbmod.Database(maildir.path, 'rw', config=notmuch2.Database.CONFIG.EMPTY) as db: yield db def test_count_messages(self, db): diff --git a/bindings/python-cffi/tests/test_tags.py b/bindings/python-cffi/tests/test_tags.py index faf3947b..f2c6209d 100644 --- a/bindings/python-cffi/tests/test_tags.py +++ b/bindings/python-cffi/tests/test_tags.py @@ -23,7 +23,7 @@ class TestImmutable: """ maildir.deliver() notmuch('new') - with database.Database(maildir.path) as db: + with database.Database(maildir.path, config=database.Database.CONFIG.EMPTY) as db: yield db.tags def test_type(self, tagset): @@ -33,7 +33,7 @@ class TestImmutable: def test_hash(self, tagset, maildir, notmuch): h0 = hash(tagset) notmuch('tag', '+foo', '*') - with database.Database(maildir.path) as db: + with database.Database(maildir.path, config=database.Database.CONFIG.EMPTY) as db: h1 = hash(db.tags) assert h0 != h1 @@ -42,7 +42,7 @@ class TestImmutable: def test_neq(self, tagset, maildir, notmuch): notmuch('tag', '+foo', '*') - with database.Database(maildir.path) as db: + with database.Database(maildir.path, config=database.Database.CONFIG.EMPTY) as db: assert tagset != db.tags def test_contains(self, tagset): @@ -159,7 +159,8 @@ class TestMutableTagset: _, pathname = maildir.deliver() notmuch('new') with database.Database(maildir.path, - mode=database.Mode.READ_WRITE) as db: + mode=database.Mode.READ_WRITE, + config=database.Database.CONFIG.EMPTY) as db: msg = db.get(pathname) yield msg.tags @@ -195,7 +196,8 @@ class TestMutableTagset: _, pathname = maildir.deliver(flagged=True) notmuch('new') with database.Database(maildir.path, - mode=database.Mode.READ_WRITE) as db: + mode=database.Mode.READ_WRITE, + config=database.Database.CONFIG.EMPTY) as db: msg = db.get(pathname) msg.tags.discard('flagged') msg.tags.from_maildir_flags() @@ -205,7 +207,8 @@ class TestMutableTagset: _, pathname = maildir.deliver(flagged=True) notmuch('new') with database.Database(maildir.path, - mode=database.Mode.READ_WRITE) as db: + mode=database.Mode.READ_WRITE, + config=database.Database.CONFIG.EMPTY) as db: msg = db.get(pathname) flags = msg.path.name.split(',')[-1] assert 'F' in flags diff --git a/bindings/python-cffi/tests/test_thread.py b/bindings/python-cffi/tests/test_thread.py index 1f44b35d..afdbcfe0 100644 --- a/bindings/python-cffi/tests/test_thread.py +++ b/bindings/python-cffi/tests/test_thread.py @@ -13,7 +13,7 @@ def thread(maildir, notmuch): maildir.deliver(body='bar', headers=[('In-Reply-To', '<{}>'.format(msgid))]) notmuch('new') - with notmuch2.Database(maildir.path) as db: + with notmuch2.Database(maildir.path, config=notmuch2.Database.CONFIG.EMPTY) as db: yield next(db.threads('foo')) From 9e62a0beaa3817b9534f871b896a8ec538a4f090 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 15:29:05 -0400 Subject: [PATCH 048/113] NEWS: add NEWS for 0.34.3 --- NEWS | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/NEWS b/NEWS index 27e43156..e39af344 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,24 @@ +Notmuch 0.34.3 (2022-01-09) +=========================== + +Library +------- + +Do not crash when presented with a .notmuch directory without a +xapian/ subdirectory. + +Python Bindings (notmuch2) +-------------------------- + +Database constructor now searches for configuration by default. Pass +`config=Database.CONFIG.EMPTY` to disable. + +The `Message.replies()` method now returns OwnedMessage objects, to +prevent certain memory de-allocation errors. + +Fix for importing `notmuch2` module when building bindings +documentation. + Notmuch 0.34.2 (2021-12-09) =========================== From ad147c02055982df25848735fc48be3fb378dd40 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 15:31:03 -0400 Subject: [PATCH 049/113] debian: changelog for 0.34.3-1 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index be7bc397..3880a004 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +notmuch (0.34.3-1) unstable; urgency=medium + + * New upstream bugfix release, with several fixes for the notmuch2 + python module. + + -- David Bremner Sun, 09 Jan 2022 15:30:38 -0400 + notmuch (0.34.2-1) unstable; urgency=medium * New upstream bugfix with release, with fixes database location in From a226b7a29b8977003081d96850a1f9e2b9a962b9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 15:35:12 -0400 Subject: [PATCH 050/113] version: bump to 0.34.3 --- bindings/python-cffi/version.txt | 2 +- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/python-cffi/version.txt b/bindings/python-cffi/version.txt index 3f8003cd..0aeaf413 100644 --- a/bindings/python-cffi/version.txt +++ b/bindings/python-cffi/version.txt @@ -1 +1 @@ -0.34.2 +0.34.3 diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 7a872f5f..d10f9654 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.34.2' +__VERSION__ = '0.34.3' SOVERSION = '5' diff --git a/version.txt b/version.txt index 3f8003cd..0aeaf413 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.34.2 +0.34.3 From 51c287ead807b6e3830bc5d393a7e9a89f36db86 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 15:35:52 -0400 Subject: [PATCH 051/113] doc: add 2022 to copyright years. --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index c7fd8f5a..e46e1d4e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -14,7 +14,7 @@ master_doc = 'index' # General information about the project. project = u'notmuch' -copyright = u'2009-2021, Carl Worth and many others' +copyright = u'2009-2022, Carl Worth and many others' location = os.path.dirname(__file__) From 332b3b639ebd797eac6980fc7bd547f6cd064e84 Mon Sep 17 00:00:00 2001 From: jao Date: Mon, 10 Jan 2022 03:05:23 +0000 Subject: [PATCH 052/113] emacs: make header line in notmuch-show buffers optional New notmuch-show-header-line customizable boolean to allow inhibiting a header line in notmuch-show-mode buffers (for instance, because one prefers to just include Subject in notmuch-message-headers). --- doc/notmuch-emacs.rst | 3 +++ emacs/notmuch-show.el | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index 22aee340..fed619b7 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -222,6 +222,9 @@ Display of messages can be controlled by the following variables :index:`notmuch-message-headers-visible` |docstring::notmuch-message-headers-visible| +:index:`notmuch-show-header-line` + |docstring::notmuch-show-header-line| + .. _show-copy: Copy to kill-ring diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el index 4de3e423..7c1f02c9 100644 --- a/emacs/notmuch-show.el +++ b/emacs/notmuch-show.el @@ -84,6 +84,11 @@ visible for any given message." :type 'boolean :group 'notmuch-show) +(defcustom notmuch-show-header-line t + "Show a header line with the current message's subject." + :type 'boolean + :group 'notmuch-show) + (defcustom notmuch-show-relative-dates t "Display relative dates in the message summary line." :type 'boolean @@ -1345,11 +1350,12 @@ If no messages match the query return NIL." (notmuch-show-mapc (lambda () (notmuch-show-set-prop :orig-tags (notmuch-show-get-tags)))) ;; Set the header line to the subject of the first message. - (setq header-line-format - (replace-regexp-in-string "%" "%%" - (notmuch-sanitize - (notmuch-show-strip-re - (notmuch-show-get-subject))))) + (when notmuch-show-header-line + (setq header-line-format + (replace-regexp-in-string "%" "%%" + (notmuch-sanitize + (notmuch-show-strip-re + (notmuch-show-get-subject)))))) (run-hooks 'notmuch-show-hook) (if state (notmuch-show-apply-state state) From 21e206e8b9bec15dbb4b1fa738a483eac4a135fa Mon Sep 17 00:00:00 2001 From: Michael J Gruber Date: Tue, 11 Jan 2022 11:13:34 +0100 Subject: [PATCH 053/113] configure: Ignore more options that Fedora spec macros expect Signed-off-by: Michael J Gruber --- configure | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/configure b/configure index e7bb7edc..9e45cbe6 100755 --- a/configure +++ b/configure @@ -312,12 +312,22 @@ for option; do true elif [ "${option%%=*}" = '--host' ] ; then true + elif [ "${option%%=*}" = '--bindir' ] ; then + true + elif [ "${option%%=*}" = '--sbindir' ] ; then + true elif [ "${option%%=*}" = '--datadir' ] ; then true elif [ "${option%%=*}" = '--localstatedir' ] ; then true + elif [ "${option%%=*}" = '--sharedstatedir' ] ; then + true elif [ "${option%%=*}" = '--libexecdir' ] ; then true + elif [ "${option%%=*}" = '--exec-prefix' ] ; then + true + elif [ "${option%%=*}" = '--program-prefix' ] ; then + true elif [ "${option}" = '--disable-maintainer-mode' ] ; then true elif [ "${option}" = '--disable-dependency-tracking' ] ; then From 114b985ba73da16d953f30dddf014be889db6e50 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 11 Jan 2022 19:52:30 -0400 Subject: [PATCH 054/113] test/gpgsm: use --with-colons when calculating fingerprint. As stressed by the gpg documentation, the non-'with-colons' output format is subject to change, and indeed it did in 2.3.x (x<=3). This should make the the test suite more robust against such changes. --- test/test-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-lib.sh b/test/test-lib.sh index e476a69b..2ba224c3 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -145,7 +145,7 @@ add_gpgsm_home () { mkdir -p -m 0700 "$GNUPGHOME" gpgsm --batch --no-tty --no-common-certs-import --pinentry-mode=loopback --passphrase-fd 3 \ --disable-dirmngr --import >"$GNUPGHOME"/import.log 2>&1 3<<<'' <$NOTMUCH_SRCDIR/test/smime/0xE0972A47.p12 - fpr=$(gpgsm --batch --list-key test_suite@notmuchmail.org | sed -n 's/.*fingerprint: //p') + fpr=$(gpgsm --batch --with-colons --list-key test_suite@notmuchmail.org | awk -F: '/^fpr/ {print $10}') echo "$fpr S relax" >> "$GNUPGHOME/trustlist.txt" gpgsm --quiet --batch --no-tty --no-common-certs-import --disable-dirmngr --import < $NOTMUCH_SRCDIR/test/smime/ca.crt echo "4D:E0:FF:63:C0:E9:EC:01:29:11:C8:7A:EE:DA:3A:9A:7F:6E:C1:0D S" >> "$GNUPGHOME/trustlist.txt" From c7c422ded29e73f7f6b53e82c40a36c93c041253 Mon Sep 17 00:00:00 2001 From: LdBeth Date: Fri, 14 Jan 2022 07:15:00 -0600 Subject: [PATCH 055/113] emacs/notmuch-tag.el: add xmlns attribute to svg icons emacs-mac that compiled with OS X system API for image display support cannot correctly render svg without xmlns parameter [1]. [1]: id:tencent_127AA231767438AC66FEE4DDB4BBF51DF909@qq.com --- emacs/notmuch-tag.el | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el index 145f309f..8af09e68 100644 --- a/emacs/notmuch-tag.el +++ b/emacs/notmuch-tag.el @@ -241,7 +241,7 @@ DATA is the content of an SVG picture (e.g., as returned by "Return SVG data representing a star icon. This can be used with `notmuch-tag-format-image-data'." " - + - + - + Date: Sat, 11 Dec 2021 08:49:11 -0400 Subject: [PATCH 056/113] test/libconfig: save and restore config file Currently the config file is unusable for further tests requiring a valid database path. --- test/T590-libconfig.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index eb303444..089b4e58 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -972,6 +972,7 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "open: database parameter overrides implicit config" +cp $NOTMUCH_CONFIG ${NOTMUCH_CONFIG}.bak notmuch config set database.path ${MAIL_DIR}/nonexistent cat c_head3 - c_tail3 <<'EOF' | test_C ${MAIL_DIR} const char *path = NULL; @@ -982,6 +983,7 @@ cat c_head3 - c_tail3 <<'EOF' | test_C ${MAIL_DIR} path = notmuch_database_get_path (db); printf ("path: %s\n", path ? path : "(null)"); EOF +cp ${NOTMUCH_CONFIG}.bak ${NOTMUCH_CONFIG} cat < EXPECTED == stdout == status: 0 From 237f803fdbf8978514d0f6b56e9d1aaf51cb3153 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 11 Dec 2021 08:49:12 -0400 Subject: [PATCH 057/113] test/libconfig: add two tests for the config = "" case If notmuch_database_open_with_config finds a database, but that database is not in a legacy, non-split configuration, then it currently incorrectly deduces the mail root and returns SUCCESS. Add to two tests to demonstrate this bug. --- test/T590-libconfig.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 089b4e58..9cc79e8c 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -994,4 +994,44 @@ EOF notmuch_dir_sanitize < OUTPUT > OUTPUT.clean test_expect_equal_file EXPECTED OUTPUT.clean +cat < c_body + notmuch_status_t st = notmuch_database_open_with_config(NULL, + NOTMUCH_DATABASE_MODE_READ_ONLY, + "", NULL, &db, NULL); + printf ("status == SUCCESS: %d\n", st == NOTMUCH_STATUS_SUCCESS); + if (db) { + const char *mail_root = NULL; + mail_root = notmuch_config_get (db, NOTMUCH_CONFIG_MAIL_ROOT); + printf ("mail_root: %s\n", mail_root ? mail_root : "(null)"); + } +EOF + +cat < EXPECTED.common +== stdout == +status == SUCCESS: 0 +db == NULL: 1 +== stderr == +EOF + +test_begin_subtest "open/error: config=empty with no mail root in db " +old_NOTMUCH_CONFIG=${NOTMUCH_CONFIG} +unset NOTMUCH_CONFIG +cat c_head3 c_body c_tail3 | test_C +export NOTMUCH_CONFIG=${old_NOTMUCH_CONFIG} +notmuch_dir_sanitize < OUTPUT > OUTPUT.clean +test_expect_equal_file EXPECTED.common OUTPUT.clean + +test_begin_subtest "open/error: config=empty with no mail root in db (xdg)" +test_subtest_known_broken +old_NOTMUCH_CONFIG=${NOTMUCH_CONFIG} +unset NOTMUCH_CONFIG +backup_database +mkdir -p home/.local/share/notmuch +mv mail/.notmuch home/.local/share/notmuch/default +cat c_head3 c_body c_tail3 | test_C +restore_database +export NOTMUCH_CONFIG=${old_NOTMUCH_CONFIG} +notmuch_dir_sanitize < OUTPUT > OUTPUT.clean +test_expect_equal_file EXPECTED.common OUTPUT.clean + test_done From fd0edeb561e5593d8a834fcc74602b2d8c637123 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 09:33:02 -0400 Subject: [PATCH 058/113] lib/open: use db struct as talloc ctx for choose_database_path The extra talloc struct "local" was left over from before the notmuch struct was allocated earlier. Having the notmuch struct available in this function will allow more flexibility to track the configuration variations (e.g. split vs. non-split). --- lib/open.cc | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/open.cc b/lib/open.cc index 54510eac..4e5d2545 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -186,7 +186,7 @@ _db_dir_exists (const char *database_path, char **message) } static notmuch_status_t -_choose_database_path (void *ctx, +_choose_database_path (notmuch_database_t *notmuch, const char *profile, GKeyFile *key_file, const char **database_path, @@ -201,16 +201,16 @@ _choose_database_path (void *ctx, char *path = g_key_file_get_string (key_file, "database", "path", NULL); if (path) { if (path[0] == '/') - *database_path = talloc_strdup (ctx, path); + *database_path = talloc_strdup (notmuch, path); else - *database_path = talloc_asprintf (ctx, "%s/%s", getenv ("HOME"), path); + *database_path = talloc_asprintf (notmuch, "%s/%s", getenv ("HOME"), path); g_free (path); } } if (! *database_path) { notmuch_status_t status; - *database_path = _xdg_dir (ctx, "XDG_DATA_HOME", ".local/share", profile); + *database_path = _xdg_dir (notmuch, "XDG_DATA_HOME", ".local/share", profile); status = _db_dir_exists (*database_path, message); if (status) { *database_path = NULL; @@ -226,7 +226,7 @@ _choose_database_path (void *ctx, if (! *database_path) { notmuch_status_t status; - *database_path = talloc_asprintf (ctx, "%s/mail", getenv ("HOME")); + *database_path = talloc_asprintf (notmuch, "%s/mail", getenv ("HOME")); status = _db_dir_exists (*database_path, message); if (status) { *database_path = NULL; @@ -510,7 +510,6 @@ notmuch_database_open_with_config (const char *database_path, char **status_string) { notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; - void *local = talloc_new (NULL); notmuch_database_t *notmuch = NULL; char *message = NULL; GKeyFile *key_file = NULL; @@ -530,7 +529,7 @@ notmuch_database_open_with_config (const char *database_path, goto DONE; } - if ((status = _choose_database_path (local, profile, key_file, + if ((status = _choose_database_path (notmuch, profile, key_file, &database_path, &split, &message))) goto DONE; @@ -549,8 +548,6 @@ notmuch_database_open_with_config (const char *database_path, status = _finish_open (notmuch, profile, mode, key_file, &message); DONE: - talloc_free (local); - if (key_file) g_key_file_free (key_file); @@ -612,7 +609,6 @@ notmuch_database_create_with_config (const char *database_path, const char *notmuch_path = NULL; char *message = NULL; GKeyFile *key_file = NULL; - void *local = talloc_new (NULL); int err; bool split = false; @@ -630,7 +626,7 @@ notmuch_database_create_with_config (const char *database_path, goto DONE; } - if ((status = _choose_database_path (local, profile, key_file, + if ((status = _choose_database_path (notmuch, profile, key_file, &database_path, &split, &message))) goto DONE; @@ -654,7 +650,7 @@ notmuch_database_create_with_config (const char *database_path, if (split) { notmuch_path = database_path; } else { - if (! (notmuch_path = talloc_asprintf (local, "%s/%s", database_path, ".notmuch"))) { + if (! (notmuch_path = talloc_asprintf (notmuch, "%s/%s", database_path, ".notmuch"))) { status = NOTMUCH_STATUS_OUT_OF_MEMORY; goto DONE; } @@ -707,8 +703,6 @@ notmuch_database_create_with_config (const char *database_path, } DONE: - talloc_free (local); - if (key_file) g_key_file_free (key_file); @@ -808,7 +802,6 @@ notmuch_database_load_config (const char *database_path, char **status_string) { notmuch_status_t status = NOTMUCH_STATUS_SUCCESS, warning = NOTMUCH_STATUS_SUCCESS; - void *local = talloc_new (NULL); notmuch_database_t *notmuch = NULL; char *message = NULL; GKeyFile *key_file = NULL; @@ -834,7 +827,7 @@ notmuch_database_load_config (const char *database_path, goto DONE; } - status = _choose_database_path (local, profile, key_file, + status = _choose_database_path (notmuch, profile, key_file, &database_path, &split, &message); switch (status) { case NOTMUCH_STATUS_NO_DATABASE: @@ -870,8 +863,6 @@ notmuch_database_load_config (const char *database_path, goto DONE; DONE: - talloc_free (local); - if (status_string) *status_string = message; From 63b4c469835133ac44b7db516fc54b8a4abb90a8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 09:33:03 -0400 Subject: [PATCH 059/113] lib/open: use notmuch->params to track split status Persisting this status will allow us to use the information in other compilation units, in particular when setting configuration defaults. --- lib/database-private.h | 7 ++++++- lib/open.cc | 19 ++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/database-private.h b/lib/database-private.h index 8133e364..5db8882f 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -191,12 +191,17 @@ operator& (notmuch_field_flag_t a, notmuch_field_flag_t b) Xapian::QueryParser::FLAG_PURE_NOT) /* - * Which parameters were explicit when the database was opened */ + * explicit and implied parameters to open */ typedef enum { NOTMUCH_PARAM_NONE = 0, + /* database passed explicitely */ NOTMUCH_PARAM_DATABASE = 1 << 0, + /* config file passed explicitely */ NOTMUCH_PARAM_CONFIG = 1 << 1, + /* profile name passed explicitely */ NOTMUCH_PARAM_PROFILE = 1 << 2, + /* split (e.g. XDG) configuration */ + NOTMUCH_PARAM_SPLIT = 1 << 3, } notmuch_open_param_t; /* diff --git a/lib/open.cc b/lib/open.cc index 4e5d2545..30cfcf9e 100644 --- a/lib/open.cc +++ b/lib/open.cc @@ -190,7 +190,6 @@ _choose_database_path (notmuch_database_t *notmuch, const char *profile, GKeyFile *key_file, const char **database_path, - bool *split, char **message) { if (! *database_path) { @@ -215,7 +214,7 @@ _choose_database_path (notmuch_database_t *notmuch, if (status) { *database_path = NULL; } else { - *split = true; + notmuch->params |= NOTMUCH_PARAM_SPLIT; } } @@ -513,7 +512,6 @@ notmuch_database_open_with_config (const char *database_path, notmuch_database_t *notmuch = NULL; char *message = NULL; GKeyFile *key_file = NULL; - bool split = false; _notmuch_init (); @@ -530,7 +528,7 @@ notmuch_database_open_with_config (const char *database_path, } if ((status = _choose_database_path (notmuch, profile, key_file, - &database_path, &split, + &database_path, &message))) goto DONE; @@ -610,7 +608,6 @@ notmuch_database_create_with_config (const char *database_path, char *message = NULL; GKeyFile *key_file = NULL; int err; - bool split = false; _notmuch_init (); @@ -627,7 +624,7 @@ notmuch_database_create_with_config (const char *database_path, } if ((status = _choose_database_path (notmuch, profile, key_file, - &database_path, &split, &message))) + &database_path, &message))) goto DONE; status = _db_dir_exists (database_path, &message); @@ -636,18 +633,19 @@ notmuch_database_create_with_config (const char *database_path, _set_database_path (notmuch, database_path); - if (key_file && ! split) { + if (key_file && ! (notmuch->params & NOTMUCH_PARAM_SPLIT)) { char *mail_root = notmuch_canonicalize_file_name ( g_key_file_get_string (key_file, "database", "mail_root", NULL)); char *db_path = notmuch_canonicalize_file_name (database_path); - split = (mail_root && (0 != strcmp (mail_root, db_path))); + if (mail_root && (0 != strcmp (mail_root, db_path))) + notmuch->params |= NOTMUCH_PARAM_SPLIT; free (mail_root); free (db_path); } - if (split) { + if (notmuch->params & NOTMUCH_PARAM_SPLIT) { notmuch_path = database_path; } else { if (! (notmuch_path = talloc_asprintf (notmuch, "%s/%s", database_path, ".notmuch"))) { @@ -805,7 +803,6 @@ notmuch_database_load_config (const char *database_path, notmuch_database_t *notmuch = NULL; char *message = NULL; GKeyFile *key_file = NULL; - bool split = false; _notmuch_init (); @@ -828,7 +825,7 @@ notmuch_database_load_config (const char *database_path, } status = _choose_database_path (notmuch, profile, key_file, - &database_path, &split, &message); + &database_path, &message); switch (status) { case NOTMUCH_STATUS_NO_DATABASE: case NOTMUCH_STATUS_SUCCESS: From 64212c7b91cdb7e65a2a28f994f8d060a50ae78c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 09:33:04 -0400 Subject: [PATCH 060/113] lib/config: make sure the config map exists when loading defaults We should not rely on one of the other "_notmuch_config_load_*" functions being called before this one. --- lib/config.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/config.cc b/lib/config.cc index 7a2882de..8f6ef110 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -658,6 +658,9 @@ _notmuch_config_load_defaults (notmuch_database_t *notmuch) { notmuch_config_key_t key; + if (notmuch->config == NULL) + notmuch->config = _notmuch_string_map_create (notmuch); + for (key = NOTMUCH_CONFIG_FIRST; key < NOTMUCH_CONFIG_LAST; key = notmuch_config_key_t (key + 1)) { From fad2e7540bf9309bfb335650ded753e9ed085eff Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 25 Dec 2021 09:33:05 -0400 Subject: [PATCH 061/113] lib/open: no default mail root in split configurations If we know the configuration is split, but there is no mail root defined, this indicates a (lack of) configuration error. Currently this can only arise in XDG configurations. --- bindings/python-cffi/notmuch2/_build.py | 1 + lib/config.cc | 6 +++++- lib/database.cc | 2 ++ lib/notmuch.h | 4 ++++ test/T590-libconfig.sh | 1 - 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bindings/python-cffi/notmuch2/_build.py b/bindings/python-cffi/notmuch2/_build.py index 45eb20c0..a55b484f 100644 --- a/bindings/python-cffi/notmuch2/_build.py +++ b/bindings/python-cffi/notmuch2/_build.py @@ -54,6 +54,7 @@ ffibuilder.cdef( NOTMUCH_STATUS_NO_DATABASE, NOTMUCH_STATUS_DATABASE_EXISTS, NOTMUCH_STATUS_BAD_QUERY_SYNTAX, + NOTMUCH_STATUS_NO_MAIL_ROOT, NOTMUCH_STATUS_LAST_STATUS } notmuch_status_t; typedef enum { diff --git a/lib/config.cc b/lib/config.cc index 8f6ef110..f61eb636 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -657,6 +657,7 @@ notmuch_status_t _notmuch_config_load_defaults (notmuch_database_t *notmuch) { notmuch_config_key_t key; + notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; if (notmuch->config == NULL) notmuch->config = _notmuch_string_map_create (notmuch); @@ -669,11 +670,14 @@ _notmuch_config_load_defaults (notmuch_database_t *notmuch) val = _notmuch_string_map_get (notmuch->config, key_string); if (! val) { + if (key == NOTMUCH_CONFIG_MAIL_ROOT && (notmuch->params & NOTMUCH_PARAM_SPLIT)) + status = NOTMUCH_STATUS_NO_MAIL_ROOT; + _notmuch_string_map_set (notmuch->config, key_string, _notmuch_config_default (notmuch, key)); } } - return NOTMUCH_STATUS_SUCCESS; + return status; } const char * diff --git a/lib/database.cc b/lib/database.cc index 6ef56d56..0effe978 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -311,6 +311,8 @@ notmuch_status_to_string (notmuch_status_t status) return "Database exists, not recreated"; case NOTMUCH_STATUS_BAD_QUERY_SYNTAX: return "Syntax error in query"; + case NOTMUCH_STATUS_NO_MAIL_ROOT: + return "No mail root found"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; diff --git a/lib/notmuch.h b/lib/notmuch.h index 1b2bdf3f..fef98b4b 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -224,6 +224,10 @@ typedef enum { * Syntax error in query */ NOTMUCH_STATUS_BAD_QUERY_SYNTAX, + /** + * No mail root could be deduced from parameters and environment + */ + NOTMUCH_STATUS_NO_MAIL_ROOT, /** * Not an actual status value. Just a way to find out how many * valid status values there are. diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 9cc79e8c..0b5291ab 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -1022,7 +1022,6 @@ notmuch_dir_sanitize < OUTPUT > OUTPUT.clean test_expect_equal_file EXPECTED.common OUTPUT.clean test_begin_subtest "open/error: config=empty with no mail root in db (xdg)" -test_subtest_known_broken old_NOTMUCH_CONFIG=${NOTMUCH_CONFIG} unset NOTMUCH_CONFIG backup_database From fd6d50b38f8d8529191f6c9b637ecb458018baa3 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 26 Dec 2021 08:24:09 -0400 Subject: [PATCH 062/113] test: add known broken test for ignoring non-toplevel .notmuch In [1] Rob observed that notmuch new ignored directories called .notmuch everywhere in the tree, where they should only (and now, with split configs, at most) be ignored at the top level. Add a test to demonstrate the problem. [1]: id:87mwhifu9a.fsf@trouble.defaultvalue.org --- test/T050-new.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/T050-new.sh b/test/T050-new.sh index 7ea127d3..db13090b 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -351,6 +351,17 @@ test_expect_code 1 "NOTMUCH_NEW --debug 2>&1" notmuch config set new.tags $OLDCONFIG +test_begin_subtest ".notmuch only ignored at top level" +test_subtest_known_broken +generate_message '[dir]=foo/bar/.notmuch/cur' '[subject]="Do not ignore, very important"' +NOTMUCH_NEW > OUTPUT +notmuch search subject:Do-not-ignore | notmuch_search_sanitize >> OUTPUT +cat < EXPECTED +Added 1 new message to the database. +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Do not ignore, very important (inbox unread) +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "RFC822 group names are indexed" test_subtest_known_broken generate_message [to]="undisclosed-recipients:" From 6472dbf4b7fdec3bd59d7622ef477a035e34c67a Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 26 Dec 2021 08:24:10 -0400 Subject: [PATCH 063/113] cli/new: only ignore .notmuch at top level Since the bug was first reported in [1], notmuch has gained the ability to have the database located outside the mail root, hence this this change differs slightly from Jani's proposed solution [2] in not using notmuch_database_get_path, but rather the already retrieved mail_root. [1]: id:87mwhifu9a.fsf@trouble.defaultvalue.org [2]: id:87ios5v59p.fsf@nikula.org --- notmuch-new.c | 5 +++-- test/T050-new.sh | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notmuch-new.c b/notmuch-new.c index 5b8fa340..346e6469 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -601,11 +601,12 @@ add_files (notmuch_database_t *notmuch, continue; } - /* Ignore the .notmuch directory and any "tmp" directory + /* Ignore any top level .notmuch directory and any "tmp" directory * that appears within a maildir. */ if ((is_maildir && strcmp (entry->d_name, "tmp") == 0) || - strcmp (entry->d_name, ".notmuch") == 0) + (strcmp (entry->d_name, ".notmuch") == 0 + && (strcmp (path, state->mail_root)) == 0)) continue; next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); diff --git a/test/T050-new.sh b/test/T050-new.sh index db13090b..6791f87c 100755 --- a/test/T050-new.sh +++ b/test/T050-new.sh @@ -352,7 +352,6 @@ test_expect_code 1 "NOTMUCH_NEW --debug 2>&1" notmuch config set new.tags $OLDCONFIG test_begin_subtest ".notmuch only ignored at top level" -test_subtest_known_broken generate_message '[dir]=foo/bar/.notmuch/cur' '[subject]="Do not ignore, very important"' NOTMUCH_NEW > OUTPUT notmuch search subject:Do-not-ignore | notmuch_search_sanitize >> OUTPUT From 78e6cf12c05222c324110aa1eb9df9d99baa91a1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 1 Jan 2022 08:47:16 -0400 Subject: [PATCH 064/113] test: fix deprecation warning in symbol-test Reduce the amount of noise in the build log. --- test/symbol-test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/symbol-test.cc b/test/symbol-test.cc index 9d73a571..9e956ddf 100644 --- a/test/symbol-test.cc +++ b/test/symbol-test.cc @@ -12,8 +12,10 @@ main (int argc, char **argv) if (argc != 3) return 1; - if (notmuch_database_open_verbose (argv[1], NOTMUCH_DATABASE_MODE_READ_ONLY, - ¬much, &message)) { + if (notmuch_database_open_with_config (argv[1], NOTMUCH_DATABASE_MODE_READ_ONLY, + "", + NULL, + ¬much, &message)) { if (message) { fputs (message, stderr); free (message); From 417d202e642e0af0ef692e3ce250a6af985c7442 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 1 Jan 2022 08:01:34 -0400 Subject: [PATCH 065/113] CLI: stash pointer to database in sprinter structs We already use an allocated (and presumably open) database as a talloc context. Keeping the pointer in the allocated struct will allow us to e.g. interrogate the configuration in a sprinter function without threading the database all the way through the various levels of function. --- notmuch-client.h | 2 +- sprinter-json.c | 5 +++-- sprinter-sexp.c | 5 +++-- sprinter-text.c | 9 +++++---- sprinter.h | 13 +++++++++---- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/notmuch-client.h b/notmuch-client.h index de318e1f..9f57ac5e 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -65,7 +65,7 @@ struct sprinter; struct notmuch_show_params; typedef struct notmuch_show_format { - struct sprinter *(*new_sprinter)(const void *ctx, FILE *stream); + struct sprinter *(*new_sprinter)(notmuch_database_t * db, FILE *stream); notmuch_status_t (*part)(const void *ctx, struct sprinter *sprinter, struct mime_node *node, int indent, const struct notmuch_show_params *params); diff --git a/sprinter-json.c b/sprinter-json.c index c7f4851c..502f89fb 100644 --- a/sprinter-json.c +++ b/sprinter-json.c @@ -172,7 +172,7 @@ json_separator (struct sprinter *sp) } struct sprinter * -sprinter_json_create (const void *ctx, FILE *stream) +sprinter_json_create (notmuch_database_t *db, FILE *stream) { static const struct sprinter_json template = { .vtable = { @@ -192,11 +192,12 @@ sprinter_json_create (const void *ctx, FILE *stream) }; struct sprinter_json *res; - res = talloc (ctx, struct sprinter_json); + res = talloc (db, struct sprinter_json); if (! res) return NULL; *res = template; + res->vtable.notmuch = db; res->stream = stream; return &res->vtable; } diff --git a/sprinter-sexp.c b/sprinter-sexp.c index 63b25428..e37cb1f9 100644 --- a/sprinter-sexp.c +++ b/sprinter-sexp.c @@ -207,7 +207,7 @@ sexp_separator (struct sprinter *sp) } struct sprinter * -sprinter_sexp_create (const void *ctx, FILE *stream) +sprinter_sexp_create (notmuch_database_t *db, FILE *stream) { static const struct sprinter_sexp template = { .vtable = { @@ -227,11 +227,12 @@ sprinter_sexp_create (const void *ctx, FILE *stream) }; struct sprinter_sexp *res; - res = talloc (ctx, struct sprinter_sexp); + res = talloc (db, struct sprinter_sexp); if (! res) return NULL; *res = template; + res->vtable.notmuch = db; res->stream = stream; return &res->vtable; } diff --git a/sprinter-text.c b/sprinter-text.c index c75ec5be..99330a94 100644 --- a/sprinter-text.c +++ b/sprinter-text.c @@ -114,7 +114,7 @@ text_map_key (unused (struct sprinter *sp), unused (const char *key)) } struct sprinter * -sprinter_text_create (const void *ctx, FILE *stream) +sprinter_text_create (notmuch_database_t *db, FILE *stream) { static const struct sprinter_text template = { .vtable = { @@ -134,21 +134,22 @@ sprinter_text_create (const void *ctx, FILE *stream) }; struct sprinter_text *res; - res = talloc (ctx, struct sprinter_text); + res = talloc (db, struct sprinter_text); if (! res) return NULL; *res = template; + res->vtable.notmuch = db; res->stream = stream; return &res->vtable; } struct sprinter * -sprinter_text0_create (const void *ctx, FILE *stream) +sprinter_text0_create (notmuch_database_t *db, FILE *stream) { struct sprinter *sp; - sp = sprinter_text_create (ctx, stream); + sp = sprinter_text_create (db, stream); if (! sp) return NULL; diff --git a/sprinter.h b/sprinter.h index 528d8a2d..fd08641c 100644 --- a/sprinter.h +++ b/sprinter.h @@ -9,6 +9,11 @@ * (strings, integers and booleans). */ typedef struct sprinter { + /* + * Open notmuch database + */ + notmuch_database_t *notmuch; + /* Start a new map/dictionary structure. This should be followed by * a sequence of alternating calls to map_key and one of the * value-printing functions until the map is ended by end. @@ -65,20 +70,20 @@ typedef struct sprinter { /* Create a new unstructured printer that emits the default text format * for "notmuch search". */ struct sprinter * -sprinter_text_create (const void *ctx, FILE *stream); +sprinter_text_create (notmuch_database_t *db, FILE *stream); /* Create a new unstructured printer that emits the text format for * "notmuch search", with each field separated by a null character * instead of the newline character. */ struct sprinter * -sprinter_text0_create (const void *ctx, FILE *stream); +sprinter_text0_create (notmuch_database_t *db, FILE *stream); /* Create a new structure printer that emits JSON. */ struct sprinter * -sprinter_json_create (const void *ctx, FILE *stream); +sprinter_json_create (notmuch_database_t *db, FILE *stream); /* Create a new structure printer that emits S-Expressions. */ struct sprinter * -sprinter_sexp_create (const void *ctx, FILE *stream); +sprinter_sexp_create (notmuch_database_t *db, FILE *stream); #endif // NOTMUCH_SPRINTER_H From 79936ac93e486d8ff82729840154bf8c2212ebb6 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 1 Jan 2022 08:01:35 -0400 Subject: [PATCH 066/113] lib/config: add known config key "show.extra_headers" Used in a following commit to enable including extra headers beyond the default in structured output. --- lib/config.cc | 3 +++ lib/notmuch.h | 1 + test/T590-libconfig.sh | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/lib/config.cc b/lib/config.cc index f61eb636..003ce679 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -596,6 +596,8 @@ _notmuch_config_key_to_string (notmuch_config_key_t key) return "user.name"; case NOTMUCH_CONFIG_AUTOCOMMIT: return "database.autocommit"; + case NOTMUCH_CONFIG_EXTRA_HEADERS: + return "show.extra_headers"; default: return NULL; } @@ -643,6 +645,7 @@ _notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key) return ""; case NOTMUCH_CONFIG_AUTOCOMMIT: return "8000"; + case NOTMUCH_CONFIG_EXTRA_HEADERS: case NOTMUCH_CONFIG_HOOK_DIR: case NOTMUCH_CONFIG_BACKUP_DIR: case NOTMUCH_CONFIG_OTHER_EMAIL: diff --git a/lib/notmuch.h b/lib/notmuch.h index fef98b4b..cd5c1246 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -2550,6 +2550,7 @@ typedef enum { NOTMUCH_CONFIG_OTHER_EMAIL, NOTMUCH_CONFIG_USER_NAME, NOTMUCH_CONFIG_AUTOCOMMIT, + NOTMUCH_CONFIG_EXTRA_HEADERS, NOTMUCH_CONFIG_LAST } notmuch_config_key_t; diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh index 0b5291ab..26a1f033 100755 --- a/test/T590-libconfig.sh +++ b/test/T590-libconfig.sh @@ -439,6 +439,7 @@ cat <<'EOF' >EXPECTED 09: 'NULL' 10: 'USER_FULL_NAME' 11: '8000' +12: 'NULL' == stderr == EOF unset MAILDIR @@ -749,6 +750,7 @@ cat <<'EOF' >EXPECTED 09: 'test_suite_other@notmuchmail.org;test_suite@otherdomain.org' 10: 'Notmuch Test Suite' 11: '8000' +12: 'NULL' == stderr == EOF test_expect_equal_file EXPECTED OUTPUT @@ -782,6 +784,7 @@ cat <<'EOF' >EXPECTED 09: 'NULL' 10: 'USER_FULL_NAME' 11: '8000' +12: 'NULL' == stderr == EOF test_expect_equal_file EXPECTED OUTPUT.clean @@ -858,6 +861,7 @@ maildir.synchronize_flags true new.ignore sekrit_junk new.tags unread;inbox search.exclude_tags foo;bar;fub +show.extra_headers (null) test.key1 testvalue1 test.key2 testvalue2 user.name Notmuch Test Suite From c5cf92aa3534b27c0dda4794d14571b1b5439da8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 1 Jan 2022 08:01:36 -0400 Subject: [PATCH 067/113] CLI: print extra headers in structured output This is based on a patch from Johan Parin [1], which is in turn responding to a bug report / feature requiest from Jan Malkhovski. The update to the structured output documented in schemata is intended to be upward compatible, so the format version stays the same [1]: id:20191116162723.18343-1-johan.parin@gmail.com [2]: id:87h8sdemnr.fsf@oxij.org --- devel/schemata | 4 +++- notmuch-show.c | 26 ++++++++++++++++++++++++++ test/T160-json.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ test/T170-sexp.sh | 14 ++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/devel/schemata b/devel/schemata index 01e3a3df..01810888 100644 --- a/devel/schemata +++ b/devel/schemata @@ -145,9 +145,11 @@ headers = { Cc?: string, Bcc?: string, Reply-To?: string, - Date: string + Date: string, + extra_header_pair* } +extra_header_pair= (header_name: string) # Encryption status (format_part_sprinter) encstatus = [{status: "good"|"bad"}] diff --git a/notmuch-show.c b/notmuch-show.c index 2848c9c3..136f4439 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -209,6 +209,30 @@ _is_from_line (const char *line) return 0; } +/* Output extra headers if configured with the `show.extra_headers' + * configuration option + */ +static void +format_extra_headers_sprinter (sprinter_t *sp, GMimeMessage *message) +{ + GMimeHeaderList *header_list = g_mime_object_get_header_list (GMIME_OBJECT (message)); + + for (notmuch_config_values_t *extra_headers = notmuch_config_get_values ( + sp->notmuch, NOTMUCH_CONFIG_EXTRA_HEADERS); + notmuch_config_values_valid (extra_headers); + notmuch_config_values_move_to_next (extra_headers)) { + GMimeHeader *header; + const char *header_name = notmuch_config_values_get (extra_headers); + + header = g_mime_header_list_get_header (header_list, header_name); + if (header == NULL) + continue; + + sp->map_key (sp, g_mime_header_get_name (header)); + sp->string (sp, g_mime_header_get_value (header)); + } +} + void format_headers_sprinter (sprinter_t *sp, GMimeMessage *message, bool reply, const _notmuch_message_crypto_t *msg_crypto) @@ -269,6 +293,8 @@ format_headers_sprinter (sprinter_t *sp, GMimeMessage *message, sp->string (sp, g_mime_message_get_date_string (sp, message)); } + /* Output extra headers the user has configured, if any */ + format_extra_headers_sprinter (sp, message); sp->end (sp); talloc_free (local); } diff --git a/test/T160-json.sh b/test/T160-json.sh index 6a3e5812..ec7b1461 100755 --- a/test/T160-json.sh +++ b/test/T160-json.sh @@ -156,4 +156,46 @@ EOF output=$(notmuch show --format=json --body=false --format-version=2 id:message-id@example.com) test_expect_equal_json "$output" "$(cat EXPECTED)" +test_begin_subtest "show extra headers" +add_message "[subject]=\"extra-headers\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[in-reply-to]=\"\"" "[body]=\"extra-headers test\""\ + "[header]=\"Received: from mail.example.com (mail.example.com [1.1.1.1]) + by mail.notmuchmail.org (some MTA) with ESMTP id 12345678 + for ; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)\"" \ + +notmuch config set show.extra_headers "in-reply-to;received" +output=$(notmuch show --format=json --body=false id:${gen_msg_id} | notmuch_json_show_sanitize) +cat < EXPECTED +[ + [ + [ + { + "crypto": {}, + "date_relative": "2000-01-01", + "excluded": false, + "filename": [ + "YYYYY" + ], + "headers": { + "Date": "Sat, 01 Jan 2000 12:00:00 +0000", + "From": "Notmuch Test Suite ", + "In-Reply-To": "", + "Received": "from mail.example.com (mail.example.com [1.1.1.1])\tby mail.notmuchmail.org (some MTA) with ESMTP id 12345678\tfor ; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)", + "Subject": "extra-headers", + "To": "Notmuch Test Suite " + }, + "id": "XXXXX", + "match": true, + "tags": [ + "inbox", + "unread" + ], + "timestamp": 946728000 + }, + [] + ] + ] +] +EOF +test_expect_equal_json "${output}" "$(cat EXPECTED)" + test_done diff --git a/test/T170-sexp.sh b/test/T170-sexp.sh index 18084273..0d32560c 100755 --- a/test/T170-sexp.sh +++ b/test/T170-sexp.sh @@ -47,4 +47,18 @@ filename=$(notmuch search --output=files "id:$id") attachment_length=$(( $(base64 $NOTMUCH_SRCDIR/test/README | wc -c) - 1 )) test_expect_equal "$output" "((((:id \"$id\" :match t :excluded nil :filename (\"$filename\") :timestamp 946728000 :date_relative \"2000-01-01\" :tags (\"inbox\") :body ((:id 1 :content-type \"multipart/mixed\" :content ((:id 2 :content-type \"text/plain\" :content \"This is a test message with inline attachment with a filename\") (:id 3 :content-type \"application/octet-stream\" :content-disposition \"inline\" :filename \"README\" :content-transfer-encoding \"base64\" :content-length $attachment_length)))) :crypto () :headers (:Subject \"sexp-show-inline-attachment-filename\" :From \"Notmuch Test Suite \" :To \"test_suite@notmuchmail.org\" :Date \"Sat, 01 Jan 2000 12:00:00 +0000\")) ())))" +test_begin_subtest "show extra headers" +add_message "[subject]=\"extra-headers\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[in-reply-to]=\"\"" "[body]=\"extra-headers test\""\ + "[header]=\"Received: from mail.example.com (mail.example.com [1.1.1.1]) + by mail.notmuchmail.org (some MTA) with ESMTP id 12345678 + for ; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)\"" \ + +notmuch config set show.extra_headers "in-reply-to;received" +notmuch show --format=sexp --body=false id:${gen_msg_id} | \ + notmuch_dir_sanitize | sed 's/msg-[0-9]*/MSG/g'> OUTPUT +cat < EXPECTED +((((:id "MSG@notmuch-test-suite" :match t :excluded nil :filename ("MAIL_DIR/MSG") :timestamp 946728000 :date_relative "2000-01-01" :tags ("inbox" "unread") :crypto () :headers (:Subject "extra-headers" :From "Notmuch Test Suite " :To "Notmuch Test Suite " :Date "Sat, 01 Jan 2000 12:00:00 +0000" :In-Reply-To "" :Received "from mail.example.com (mail.example.com [1.1.1.1])\011by mail.notmuchmail.org (some MTA) with ESMTP id 12345678\011for ; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)")) ()))) +EOF +test_expect_equal_file EXPECTED OUTPUT + test_done From 2d036dbc3b6e48b12ab3e8aa0cbe713d2ef96854 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 18 Jan 2022 09:39:05 -0400 Subject: [PATCH 068/113] test/emacs: known broken test for reply with extra headers set. Although it makes sense for the extra headers to be added to the copy of the message headers included in the sexp/json, it is a bit surprising for them to show in the new message constructed for the reply, especially when, as here, they are always missing/empty. --- test/T310-emacs.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/T310-emacs.sh b/test/T310-emacs.sh index 1b6660f0..831df1cc 100755 --- a/test/T310-emacs.sh +++ b/test/T310-emacs.sh @@ -485,6 +485,32 @@ Sender writes: EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "Reply with show.extra_headers set" +test_subtest_known_broken +notmuch config set show.extra_headers Received +add_message '[from]="Sender "' \ + [to]=test_suite_other@notmuchmail.org + +test_emacs "(let ((message-hidden-headers '())) + (notmuch-search \"id:\\\"${gen_msg_id}\\\"\") + (notmuch-test-wait) + (notmuch-search-reply-to-thread) + (test-output))" +cat <EXPECTED +From: Notmuch Test Suite +To: Sender +Subject: Re: ${test_subtest_name} +In-Reply-To: <${gen_msg_id}> +Fcc: ${MAIL_DIR}/sent +References: <${gen_msg_id}> +--text follows this line-- +Sender writes: + +> This is just a test message (#${gen_msg_cnt}) +EOF +notmuch config set show.extra_headers +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "Reply from address in named group list within emacs" add_message '[from]="Sender "' \ '[to]=group:test_suite@notmuchmail.org,someone@example.com\;' \ @@ -680,7 +706,7 @@ References: --text follows this line-- test_suite@notmuchmail.org writes: -> This is just a test message (#7) +> This is just a test message (#${gen_msg_cnt}) EOF test_expect_equal_file EXPECTED OUTPUT From 87d5a5a8aa323077c0b79fce42d062839eb2ff2d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 18 Jan 2022 21:26:45 -0400 Subject: [PATCH 069/113] CLI: print extra headers only for non-replies If in the future we decide to output extra headers for replies, this should be controlled by a separate configuration option. --- notmuch-show.c | 3 ++- test/T310-emacs.sh | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notmuch-show.c b/notmuch-show.c index 136f4439..6a54d9c1 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -294,7 +294,8 @@ format_headers_sprinter (sprinter_t *sp, GMimeMessage *message, } /* Output extra headers the user has configured, if any */ - format_extra_headers_sprinter (sp, message); + if (! reply) + format_extra_headers_sprinter (sp, message); sp->end (sp); talloc_free (local); } diff --git a/test/T310-emacs.sh b/test/T310-emacs.sh index 831df1cc..a05b828a 100755 --- a/test/T310-emacs.sh +++ b/test/T310-emacs.sh @@ -486,7 +486,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "Reply with show.extra_headers set" -test_subtest_known_broken notmuch config set show.extra_headers Received add_message '[from]="Sender "' \ [to]=test_suite_other@notmuchmail.org From 704aa76f5dbe64098963100ec0ed89c00340d89b Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 4 Jan 2022 21:06:04 -0400 Subject: [PATCH 070/113] test/emacs: add known broken test for hidden tags Gregor Zattler observed that tags could be unintentionally hidden in the "All tags" view, and Tomi Ollia worked out [2] that the issue was tags that only occured on excluded messages. This test reproduces that bug. [1]: id:87wox1vovj.fsf@len.workgroup [2]: id:m28t9faaim.fsf@guru.guru-group.fi --- test/T440-emacs-hello.sh | 11 +++++++++++ test/emacs.expected-output/notmuch-hello-all-tags | 15 +++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/emacs.expected-output/notmuch-hello-all-tags diff --git a/test/T440-emacs-hello.sh b/test/T440-emacs-hello.sh index a1ed1c2b..dc80cc75 100755 --- a/test/T440-emacs-hello.sh +++ b/test/T440-emacs-hello.sh @@ -68,6 +68,17 @@ test_emacs '(notmuch-hello) notmuch tag -$tag '*' test_expect_equal_file $EXPECTED/notmuch-hello-long-names OUTPUT +test_begin_subtest "All tags show up" +test_subtest_known_broken +tag=exclude_me +notmuch tag +$tag '*' +notmuch config set search.exclude_tags $tag +test_emacs '(notmuch-hello) + (test-output)' +notmuch tag -$tag '*' +test_expect_equal_file $EXPECTED/notmuch-hello-all-tags OUTPUT + +test_done test_begin_subtest "notmuch-hello with nonexistent CWD" test_emacs ' (notmuch-hello) diff --git a/test/emacs.expected-output/notmuch-hello-all-tags b/test/emacs.expected-output/notmuch-hello-all-tags new file mode 100644 index 00000000..2802a708 --- /dev/null +++ b/test/emacs.expected-output/notmuch-hello-all-tags @@ -0,0 +1,15 @@ + Welcome to notmuch. You have 52 messages. + +Saved searches: [edit] + + 52 inbox 52 unread 52 all mail + +Search: . + +All tags: [hide] + + 4 attachment 52 inbox 52 unread + 52 exclude_me 7 signed + + Hit `?' for context-sensitive help in any Notmuch screen. + Customize Notmuch or this page. From 8370e3cfe2dd8a79323613c2bbf2f11db6134dac Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 4 Jan 2022 21:06:05 -0400 Subject: [PATCH 071/113] emacs: use --exclude=false when checking for empty searches In particular tags that only occur on otherwise excluded messages do not show up in "All tags" without this change. The reported numbers _do_ take excludes into account, so it is possible that searches (e.g. the aformentioned tags) will show up with a count of zero. --- emacs/notmuch-hello.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index fc007c4c..50b6f303 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -570,7 +570,7 @@ options will be handled as specified for (plist-get options :filter)))) "\n"))) (unless (= (notmuch--call-process-region (point-min) (point-max) notmuch-command - t t nil "count" "--batch") 0) + t t nil "count" "--exclude=false" "--batch") 0) (notmuch-logged-error "notmuch count --batch failed" "Please check that the notmuch CLI is new enough to support `count From cc180507b03d9826c92d48ee91dbd9bb5f15cd56 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 4 Jan 2022 21:06:06 -0400 Subject: [PATCH 072/113] emacs: use --exclude=false when counting total messages Even if a user excludes a large fraction of their messages, they still nonetheless exist and are searchable. --- emacs/notmuch-hello.el | 2 +- test/T440-emacs-hello.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index 50b6f303..9ac52c11 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -785,7 +785,7 @@ Complete list of currently available key bindings: :help-echo "Refresh" (notmuch-hello-nice-number (string-to-number - (car (notmuch--process-lines notmuch-command "count"))))) + (car (notmuch--process-lines notmuch-command "count" "--exclude=false"))))) (widget-insert " messages.\n"))) (defun notmuch-hello-insert-saved-searches () diff --git a/test/T440-emacs-hello.sh b/test/T440-emacs-hello.sh index dc80cc75..842781a4 100755 --- a/test/T440-emacs-hello.sh +++ b/test/T440-emacs-hello.sh @@ -69,7 +69,6 @@ notmuch tag -$tag '*' test_expect_equal_file $EXPECTED/notmuch-hello-long-names OUTPUT test_begin_subtest "All tags show up" -test_subtest_known_broken tag=exclude_me notmuch tag +$tag '*' notmuch config set search.exclude_tags $tag From 5620dc142eded4dd210e8dc0d16dd0c257f88819 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 10:38:02 -0400 Subject: [PATCH 073/113] configure: check for ASAN support This will allow conditionally running tests that use the address sanitizer. --- configure | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configure b/configure index 9e45cbe6..36f3f606 100755 --- a/configure +++ b/configure @@ -410,6 +410,18 @@ EOF exit 1 fi +printf "C compiler supports address sanitizer... " +test_cmdline="${CC} ${CFLAGS} ${CPPFLAGS} -fsanitize=address minimal.c ${LDFLAGS} -o minimal" +if ${test_cmdline} >/dev/null 2>&1 && ./minimal +then + printf "Yes.\n" + have_asan=1 +else + printf "Nope, skipping those tests.\n" + have_asan=0 +fi +unset test_cmdline + printf "Reading libnotmuch version from source... " cat > _libversion.c < @@ -1548,6 +1560,9 @@ NOTMUCH_GMIME_X509_CERT_VALIDITY=${gmime_x509_cert_validity} # Whether GMime can verify signatures when decrypting with a session key: NOTMUCH_GMIME_VERIFY_WITH_SESSION_KEY=${gmime_verify_with_session_key} +# Does the C compiler support the address sanitizer +NOTMUCH_HAVE_ASAN=${have_asan} + # do we have man pages? NOTMUCH_HAVE_MAN=$((have_sphinx)) From 8dab460a082633903a0cf50938378ec12794819e Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 10:38:03 -0400 Subject: [PATCH 074/113] test: add known broken test for memory leaks in open This duplicates the memory leaks reported in [1] [1]: id:20220105224538.m36lnjn7rf3ieonc@athena --- test/T800-asan.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 test/T800-asan.sh diff --git a/test/T800-asan.sh b/test/T800-asan.sh new file mode 100755 index 00000000..8c294578 --- /dev/null +++ b/test/T800-asan.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +test_description='run code with ASAN enabled against the library' +. $(dirname "$0")/test-lib.sh || exit 1 + +if [ $NOTMUCH_HAVE_ASAN -ne 1 ]; then + printf "Skipping due to missing ASAN support\n" + test_done +fi + +add_email_corpus + +TEST_CFLAGS="-fsanitize=address" + +test_begin_subtest "open and destroy" +test_subtest_known_broken +test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} < +#include + +int main(int argc, char **argv) { + notmuch_database_t *db = NULL; + + notmuch_status_t st = notmuch_database_open_with_config(argv[1], + NOTMUCH_DATABASE_MODE_READ_ONLY, + argv[2], NULL, &db, NULL); + + printf("db != NULL: %d\n", db != NULL); + if (db != NULL) + notmuch_database_destroy(db); + return 0; +} +EOF +cat < EXPECTED +== stdout == +db != NULL: 1 +== stderr == +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_done From df7c5acd759f22fcb537490f62b85d39aa71d677 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 10:38:04 -0400 Subject: [PATCH 075/113] lib/config: move g_key_File_get_string before continue In [1] Austin Ray reported some memory leaks in notmuch_database_open. One of those leaks is caused by jumping to the next key without freeing val. This change avoids that leak. [1]: id:20220105224538.m36lnjn7rf3ieonc@athena --- lib/config.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/config.cc b/lib/config.cc index 003ce679..503a0c8b 100644 --- a/lib/config.cc +++ b/lib/config.cc @@ -435,11 +435,6 @@ _notmuch_config_load_from_file (notmuch_database_t *notmuch, for (gchar **keys_p = keys; *keys_p; keys_p++) { char *absolute_key = talloc_asprintf (notmuch, "%s.%s", *grp, *keys_p); char *normalized_val; - val = g_key_file_get_string (file, *grp, *keys_p, NULL); - if (! val) { - status = NOTMUCH_STATUS_FILE_ERROR; - goto DONE; - } /* If we opened from a given path, do not overwrite it */ if (strcmp (absolute_key, "database.path") == 0 && @@ -447,6 +442,12 @@ _notmuch_config_load_from_file (notmuch_database_t *notmuch, notmuch->xapian_db) continue; + val = g_key_file_get_string (file, *grp, *keys_p, NULL); + if (! val) { + status = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + normalized_val = _expand_path (notmuch, absolute_key, val); _notmuch_string_map_set (notmuch->config, absolute_key, normalized_val); g_free (val); From 2786aa4d548d28579c761e9358d44c84dfb29068 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 9 Jan 2022 10:38:05 -0400 Subject: [PATCH 076/113] lib/database: delete stemmer on destroy Commit [0] left the stemmer object accessible, but did not add de-allocation code to notmuch_database_destroy. This commit corrects that oversight. Leak originally reported by Austin Ray [1]. [0]: 3202e0d1feba1ab955ba1c07098c00208f8f0ada [1]: id:20220105224538.m36lnjn7rf3ieonc@athena --- lib/database.cc | 2 ++ test/T800-asan.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/database.cc b/lib/database.cc index 0effe978..df83e204 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -755,6 +755,8 @@ notmuch_database_destroy (notmuch_database_t *notmuch) notmuch->date_range_processor = NULL; delete notmuch->last_mod_range_processor; notmuch->last_mod_range_processor = NULL; + delete notmuch->stemmer; + notmuch->stemmer = NULL; talloc_free (notmuch); diff --git a/test/T800-asan.sh b/test/T800-asan.sh index 8c294578..8607732e 100755 --- a/test/T800-asan.sh +++ b/test/T800-asan.sh @@ -12,7 +12,6 @@ add_email_corpus TEST_CFLAGS="-fsanitize=address" test_begin_subtest "open and destroy" -test_subtest_known_broken test_C ${MAIL_DIR} ${NOTMUCH_CONFIG} < #include From 68640fd9670d12ea80d313fe30bfe42619e73f47 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 22 Jan 2022 13:44:39 -0400 Subject: [PATCH 077/113] Revert "emacs: use --exclude=false when checking for empty searches" Revert commit 8370e3cfe2dd8a79323613c2bbf2f11db6134dac, and remark the corresponding test as broken. Also update the expected output of the broken test to show excludes active in the user defined saved searches. --- emacs/notmuch-hello.el | 2 +- test/T440-emacs-hello.sh | 1 + test/emacs.expected-output/notmuch-hello-all-tags | 4 ---- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index 9ac52c11..71e91093 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -570,7 +570,7 @@ options will be handled as specified for (plist-get options :filter)))) "\n"))) (unless (= (notmuch--call-process-region (point-min) (point-max) notmuch-command - t t nil "count" "--exclude=false" "--batch") 0) + t t nil "count" "--batch") 0) (notmuch-logged-error "notmuch count --batch failed" "Please check that the notmuch CLI is new enough to support `count diff --git a/test/T440-emacs-hello.sh b/test/T440-emacs-hello.sh index 842781a4..dc80cc75 100755 --- a/test/T440-emacs-hello.sh +++ b/test/T440-emacs-hello.sh @@ -69,6 +69,7 @@ notmuch tag -$tag '*' test_expect_equal_file $EXPECTED/notmuch-hello-long-names OUTPUT test_begin_subtest "All tags show up" +test_subtest_known_broken tag=exclude_me notmuch tag +$tag '*' notmuch config set search.exclude_tags $tag diff --git a/test/emacs.expected-output/notmuch-hello-all-tags b/test/emacs.expected-output/notmuch-hello-all-tags index 2802a708..65e479fa 100644 --- a/test/emacs.expected-output/notmuch-hello-all-tags +++ b/test/emacs.expected-output/notmuch-hello-all-tags @@ -1,9 +1,5 @@ Welcome to notmuch. You have 52 messages. -Saved searches: [edit] - - 52 inbox 52 unread 52 all mail - Search: . All tags: [hide] From 21cce961b99fb32a3d7056fce7e3e890c618333c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 22 Jan 2022 13:44:40 -0400 Subject: [PATCH 078/113] emacs: define, use option :disable-excludes for n-h-query-counts Initially only use in notmuch-hello-insert-alltags. This is a more narrow resolution of [1], which (unlike [2]) does not disable exclude processing for regular saved searches. [1]: id:87wox1vovj.fsf@len.workgroup [2]: id:20220105010606.2034601-2-david@tethera.net --- emacs/notmuch-hello.el | 12 +++++++++--- test/T440-emacs-hello.sh | 1 - 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index 71e91093..581e7f3a 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -557,7 +557,8 @@ with any properties in the original saved-search. The values :show-empty-searches, :filter and :filter-count from options will be handled as specified for -`notmuch-hello-insert-searches'." +`notmuch-hello-insert-searches'. :disable-includes can be used to +turn off the default exclude processing in `notmuch-count(1)'" (with-temp-buffer (dolist (elem query-list nil) (let ((count-query (or (notmuch-saved-search-get elem :count-query) @@ -570,7 +571,11 @@ options will be handled as specified for (plist-get options :filter)))) "\n"))) (unless (= (notmuch--call-process-region (point-min) (point-max) notmuch-command - t t nil "count" "--batch") 0) + t t nil "count" + (if (plist-get options :disable-excludes) + "--exclude=false" + "--exclude=true") + "--batch") 0) (notmuch-logged-error "notmuch count --batch failed" "Please check that the notmuch CLI is new enough to support `count @@ -917,7 +922,8 @@ following: nil :initially-hidden (not notmuch-show-all-tags-list) :hide-tags notmuch-hello-hide-tags - :filter notmuch-hello-tag-list-make-query)) + :filter notmuch-hello-tag-list-make-query + :disable-excludes t)) (defun notmuch-hello-insert-footer () "Insert the notmuch-hello footer." diff --git a/test/T440-emacs-hello.sh b/test/T440-emacs-hello.sh index dc80cc75..842781a4 100755 --- a/test/T440-emacs-hello.sh +++ b/test/T440-emacs-hello.sh @@ -69,7 +69,6 @@ notmuch tag -$tag '*' test_expect_equal_file $EXPECTED/notmuch-hello-long-names OUTPUT test_begin_subtest "All tags show up" -test_subtest_known_broken tag=exclude_me notmuch tag +$tag '*' notmuch config set search.exclude_tags $tag From 15207652a1e52f995d08eb5645f28531b5e19d46 Mon Sep 17 00:00:00 2001 From: jao Date: Sun, 23 Jan 2022 19:54:17 +0000 Subject: [PATCH 079/113] emacs: customizable names for search buffers Customizable names for buffers presenting search results, via two custom variables (notmuch-search-buffer-name-format and notmuch-saved-search-buffer-name-format), defaulting to values currently used for plain searches and including too tree and unthreaded search buffers. Amended by db: spelling fix. --- doc/notmuch-emacs.rst | 10 ++++++++ emacs/notmuch-hello.el | 11 +++++---- emacs/notmuch-tree.el | 10 ++++---- emacs/notmuch.el | 53 ++++++++++++++++++++++++++++++++++-------- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst index fed619b7..85b2c0ea 100644 --- a/doc/notmuch-emacs.rst +++ b/doc/notmuch-emacs.rst @@ -175,6 +175,16 @@ variables. :index:`notmuch-search-oldest-first` Display the oldest threads at the top of the buffer +It is also possible to customize how the name of buffers containing +search results is formatted using the following variables: + +:index:`notmuch-search-buffer-name-format` + |docstring::notmuch-search-buffer-name-format| + +:index:`notmuch-saved-search-buffer-name-format` + |docstring::notmuch-saved-search-buffer-name-format| + + .. _notmuch-show: notmuch-show diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el index 581e7f3a..beb25382 100644 --- a/emacs/notmuch-hello.el +++ b/emacs/notmuch-hello.el @@ -486,11 +486,14 @@ diagonal." (defun notmuch-hello-widget-search (widget &rest _ignore) (cl-case (widget-get widget :notmuch-search-type) (tree - (notmuch-tree (widget-get widget :notmuch-search-terms) - nil nil nil nil nil nil - (widget-get widget :notmuch-search-oldest-first))) + (let ((n (notmuch-search-format-buffer-name (widget-value widget) "tree" t))) + (notmuch-tree (widget-get widget :notmuch-search-terms) + nil nil n nil nil nil + (widget-get widget :notmuch-search-oldest-first)))) (unthreaded - (notmuch-unthreaded (widget-get widget :notmuch-search-terms))) + (let ((n (notmuch-search-format-buffer-name (widget-value widget) + "unthreaded" t))) + (notmuch-unthreaded (widget-get widget :notmuch-search-terms) nil nil n))) (t (notmuch-search (widget-get widget :notmuch-search-terms) (widget-get widget :notmuch-search-oldest-first))))) diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el index d7486904..303c6fad 100644 --- a/emacs/notmuch-tree.el +++ b/emacs/notmuch-tree.el @@ -1191,11 +1191,11 @@ The arguments are: (setq query (notmuch-read-query (concat "Notmuch " (if unthreaded "unthreaded " "tree ") "view search: ")))) - (let ((buffer (get-buffer-create (generate-new-buffer-name - (or buffer-name - (concat "*notmuch-" - (if unthreaded "unthreaded-" "tree-") - query "*"))))) + (let* ((name + (or buffer-name + (notmuch-search-buffer-title query + (if unthreaded "unthreaded" "tree")))) + (buffer (get-buffer-create (generate-new-buffer-name name))) (inhibit-read-only t)) (pop-to-buffer-same-window buffer)) ;; Don't track undo information for this buffer diff --git a/emacs/notmuch.el b/emacs/notmuch.el index 85a54706..6abb17ff 100644 --- a/emacs/notmuch.el +++ b/emacs/notmuch.el @@ -915,7 +915,39 @@ See `notmuch-tag' for information on the format of TAG-CHANGES." (notmuch-search-get-tags-region (point-min) (point-max)) "Tag all"))) (notmuch-search-tag tag-changes (point-min) (point-max) t)) -(defun notmuch-search-buffer-title (query) +(defcustom notmuch-search-buffer-name-format "*notmuch-%t-%s*" + "Format for the name of search results buffers. + +In this spec, %s will be replaced by a description of the search +query and %t by its type (search, tree or unthreaded). The +buffer name is formatted using `format-spec': see its docstring +for additional parameters for the s and t format specifiers. + +See also `notmuch-saved-search-buffer-name-format'" + :type 'string + :group 'notmuch-search) + +(defcustom notmuch-saved-search-buffer-name-format "*notmuch-saved-%t-%s*" + "Format for the name of search results buffers for saved searches. + +In this spec, %s will be replaced by the saved search name and %t +by its type (search, tree or unthreaded). The buffer name is +formatted using `format-spec': see its docstring for additional +parameters for the s and t format specifiers. + +See also `notmuch-search-buffer-name-format'" + :type 'string + :group 'notmuch-search) + +(defun notmuch-search-format-buffer-name (query type saved) + "Compose a buffer name for the given QUERY, TYPE (search, tree, +unthreaded) and whether it's SAVED (t or nil)." + (let ((fmt (if saved + notmuch-saved-search-buffer-name-format + notmuch-search-buffer-name-format))) + (format-spec fmt `((?t . ,(or type "search")) (?s . ,query))))) + +(defun notmuch-search-buffer-title (query &optional type) "Returns the title for a buffer with notmuch search results." (let* ((saved-search (let (longest @@ -930,19 +962,20 @@ See `notmuch-tag' for information on the format of TAG-CHANGES." do (setq longest tuple)) longest)) (saved-search-name (notmuch-saved-search-get saved-search :name)) + (saved-search-type (notmuch-saved-search-get saved-search :search-type)) (saved-search-query (notmuch-saved-search-get saved-search :query))) (cond ((and saved-search (equal saved-search-query query)) ;; Query is the same as saved search (ignoring case) - (concat "*notmuch-saved-search-" saved-search-name "*")) + (notmuch-search-format-buffer-name saved-search-name + saved-search-type + t)) (saved-search - (concat "*notmuch-search-" - (replace-regexp-in-string - (concat "^" (regexp-quote saved-search-query)) - (concat "[ " saved-search-name " ]") - query) - "*")) - (t - (concat "*notmuch-search-" query "*"))))) + (let ((query (replace-regexp-in-string + (concat "^" (regexp-quote saved-search-query)) + (concat "[ " saved-search-name " ]") + query))) + (notmuch-search-format-buffer-name query saved-search-type t))) + (t (notmuch-search-format-buffer-name query type nil))))) (defun notmuch-read-query (prompt) "Read a notmuch-query from the minibuffer with completion. From 82e1279790bb352c5716ad25703283b09f42730f Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 17 Jan 2022 19:54:52 -0400 Subject: [PATCH 080/113] test/setup: add known broken test for single items In [1] Ian observed that notmuch setup was inconsistent with notmuch config set when adding single items, namely adding an unneeded semi-colon at the end. This test replicates that bug. [1]: id:6O3LTUhoXlrnkPWCtPJCP4cagU7mFVEdyTpcC_37BoSzStlARXDBa7oczy6hB0jyjGjBQvgj_jFV58cw0aNx-jUg1h1O-FQ7820k68C0X4M=@protonmail.com --- test/T040-setup.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/T040-setup.sh b/test/T040-setup.sh index 42c621c8..36a42562 100755 --- a/test/T040-setup.sh +++ b/test/T040-setup.sh @@ -23,6 +23,14 @@ EOF expected_dir=$NOTMUCH_SRCDIR/test/setup.expected-output test_expect_equal_file ${expected_dir}/config-with-comments new-notmuch-config +test_begin_subtest "setup consistent with config-set for single items" +test_subtest_known_broken +# note this relies on the config state from the previous test. +notmuch --config=new-notmuch-config config list > list.setup +notmuch --config=new-notmuch-config config set search.exclude_tags baz +notmuch --config=new-notmuch-config config list > list.config +test_expect_equal_file list.setup list.config + test_begin_subtest "notmuch with a config but without a database suggests notmuch new" notmuch 2>&1 | notmuch_dir_sanitize > OUTPUT cat < EXPECTED From 88633bc7a79d47ba96879f698ec2267f2d3f6766 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Mon, 17 Jan 2022 19:54:53 -0400 Subject: [PATCH 081/113] CLI/setup: special case single item lists This matches the heuristic used by "notmuch config set" to decide if something is a list. This change fixes the bug reported at [1]. [1]: id:6O3LTUhoXlrnkPWCtPJCP4cagU7mFVEdyTpcC_37BoSzStlARXDBa7oczy6hB0jyjGjBQvgj_jFV58cw0aNx-jUg1h1O-FQ7820k68C0X4M=@protonmail.com --- notmuch-config.c | 5 ++++- test/T040-setup.sh | 1 - test/setup.expected-output/config-with-comments | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/notmuch-config.c b/notmuch-config.c index 11d8d68b..e9456d79 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -383,7 +383,10 @@ _config_set_list (notmuch_conffile_t *config, const char *list[], size_t length) { - g_key_file_set_string_list (config->key_file, group, key, list, length); + if (length > 1) + g_key_file_set_string_list (config->key_file, group, key, list, length); + else + g_key_file_set_string (config->key_file, group, key, list[0]); } void diff --git a/test/T040-setup.sh b/test/T040-setup.sh index 36a42562..10b29ec3 100755 --- a/test/T040-setup.sh +++ b/test/T040-setup.sh @@ -24,7 +24,6 @@ expected_dir=$NOTMUCH_SRCDIR/test/setup.expected-output test_expect_equal_file ${expected_dir}/config-with-comments new-notmuch-config test_begin_subtest "setup consistent with config-set for single items" -test_subtest_known_broken # note this relies on the config state from the previous test. notmuch --config=new-notmuch-config config list > list.setup notmuch --config=new-notmuch-config config set search.exclude_tags baz diff --git a/test/setup.expected-output/config-with-comments b/test/setup.expected-output/config-with-comments index 56c628e5..d8397714 100644 --- a/test/setup.expected-output/config-with-comments +++ b/test/setup.expected-output/config-with-comments @@ -31,7 +31,7 @@ path=/path/to/maildir [user] name=Test Suite primary_email=test.suite@example.com -other_email=another.suite@example.com; +other_email=another.suite@example.com # Configuration for "notmuch new" # @@ -60,7 +60,7 @@ tags=foo;bar; # query will override that exclusion. # [search] -exclude_tags=baz; +exclude_tags=baz # Maildir compatibility configuration # From 3bf6487359b693af572867083da06de2663fb1f8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Tue, 18 Jan 2022 09:26:59 -0400 Subject: [PATCH 082/113] doc: document new option `show.extra_headers` Increase discoverability by cross referencing from the notmuch-show manual entry to the notmuch-config manual entry. --- doc/man1/notmuch-config.rst | 14 ++++++++++++++ doc/man1/notmuch-show.rst | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index ed188a25..41e1338b 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -145,6 +145,20 @@ search.exclude\_tags Default: empty list. Note that :any:`notmuch-setup(1)` puts ``deleted;spam`` here when creating new configuration file. +.. _show.extra_headers: + +show.extra\_headers + + By default :any:`notmuch-show(1)` includes the following headers + in structured output if they are present in the message: + `Subject`, `From`, `To`, `Cc`, `Bcc`, `Reply-To`, `Date`. This + option allows the specification of a list of further + headers to output. + + History: This configuration value was introduced in notmuch 0.35. + + Default: empty list. + maildir.synchronize\_flags If true, then the following maildir flags (in message filenames) will be synchronized with the corresponding notmuch tags: diff --git a/doc/man1/notmuch-show.rst b/doc/man1/notmuch-show.rst index 3d2a2c41..1b02d407 100644 --- a/doc/man1/notmuch-show.rst +++ b/doc/man1/notmuch-show.rst @@ -221,6 +221,13 @@ email messages. For this, use a search term of "thread:" as can be seen in the first column of output from the :any:`notmuch-search(1)` command. +CONFIGURATION +============= + +Structured output (json / sexp) is influenced by the configuration +option :ref:`show.extra_headers `. See +:any:`notmuch-config(1)` for details. + EXIT STATUS =========== From c8d292e25b9e571c0ab5274cc20e23b4f0e979d1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 19 Jan 2022 20:50:49 -0400 Subject: [PATCH 083/113] test/emacs: match mml settings in emacs_{fcc,deliver}_message This will allow the sending of signed messages via smtp in the test suite. --- test/test-lib-emacs.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test-lib-emacs.sh b/test/test-lib-emacs.sh index dde32177..a298526d 100644 --- a/test/test-lib-emacs.sh +++ b/test/test-lib-emacs.sh @@ -54,8 +54,9 @@ emacs_deliver_message () { (message-goto-body) (insert \"${body}\") $* - (notmuch-mua-send-and-exit))" - + (let ((mml-secure-smime-sign-with-sender t) + (mml-secure-openpgp-sign-with-sender t)) + (notmuch-mua-send-and-exit)))" # In case message was sent properly, client waits for confirmation # before exiting and resuming control here; therefore making sure # that server exits by sending (KILL) signal to it is safe. From b559240bb1cbc3d0b18589bd3e5aa996ac84382d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 07:48:35 -0400 Subject: [PATCH 084/113] test: define test_expect_equal_message_body This is a relatively simple sed invocation, but rather than write a comment everywhere, give it a descriptive name. --- test/test-lib.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/test-lib.sh b/test/test-lib.sh index f1275b85..af017ec2 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -432,6 +432,20 @@ test_expect_equal_file () { test_diff_file_ "$1" "$2" } +# Like test_expect_equal_file, but compare the part of the two files after the first blank line +test_expect_equal_message_body () { + exec 1>&6 2>&7 # Restore stdout and stderr + if [ -z "$inside_subtest" ]; then + error "bug in the test script: test_expect_equal_file without test_begin_subtest" + fi + test "$#" = 2 || + error "bug in the test script: not 2 parameters to test_expect_equal_file" + + expected=$(sed '1,/^$/d' "$1") + output=$(sed '1,/^$/d' "$2") + test_expect_equal "$expected" "$output" +} + # Like test_expect_equal, but takes two filenames. Fails if either is empty test_expect_equal_file_nonempty () { exec 1>&6 2>&7 # Restore stdout and stderr From 8179c3d1146b6b19c64200f0617c4a1ba7588569 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 19 Jan 2022 21:23:30 -0400 Subject: [PATCH 085/113] test/emacs: known broken test for matching fcc and sent message Based on the method outlined by Daniel Kahn Gillmor in id:87k1zm225v.fsf@fifthhorseman.net. With a delay of 0.2 seconds the test becomes flaky on my machine. With a 1 second delay it fails consistently for more than 1600 iterations. --- test/T350-crypto.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 8dbf8935..209b8a74 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -13,16 +13,30 @@ test_description='PGP/MIME signature verification and decryption' test_require_emacs add_gnupg_home -test_begin_subtest "emacs delivery of signed message" +test_begin_subtest "emacs delivery of signed message via fcc" test_expect_success \ 'emacs_fcc_message \ "test signed message 001" \ "This is a test signed message." \ "(mml-secure-message-sign)"' +test_begin_subtest "emacs delivery of signed message via fcc and smtp" +test_subtest_known_broken +emacs_deliver_message \ + 'signed message sent via SMTP' \ + 'This is a test that messages are sent via SMTP' \ + "(add-hook 'message-send-mail-hook (lambda () (sleep-for 1))) + (mml-secure-message-sign)" +msg_file=$(notmuch search --output=files subject:signed-message-sent-via-SMTP) +test_expect_equal_message_body sent_message "$msg_file" + test_begin_subtest "signed part content-type indexing" -output=$(notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize) -test_expect_equal "$output" "thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed)" +notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize > OUTPUT +cat <EXPECTED +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed) +thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; signed message sent via SMTP (inbox signed) +EOF +test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "signature verification" output=$(notmuch show --format=json --verify subject:"test signed message 001" \ From da302e1cbaaab89b2bbb32c0f59e1aa6ee708455 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 19 Jan 2022 23:22:07 -0400 Subject: [PATCH 086/113] emacs: use cached encoded copy for fcc This fixes the bug reported by dkg in [1]. The movement of the call to n-m-setup-message-for-saving is so the cleanup of Fcc headers happens in the encoded version (otherwise Fcc headers may be saved to disk). [1]: id:87k1zm225v.fsf@fifthhorseman.net --- emacs/notmuch-maildir-fcc.el | 8 ++++++-- test/T350-crypto.sh | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/emacs/notmuch-maildir-fcc.el b/emacs/notmuch-maildir-fcc.el index 7e177bf7..51020788 100644 --- a/emacs/notmuch-maildir-fcc.el +++ b/emacs/notmuch-maildir-fcc.el @@ -149,6 +149,7 @@ Otherwise set it according to `notmuch-fcc-dirs'." (buf (current-buffer)) (mml-externalize-attachments message-fcc-externalize-attachments)) (with-current-buffer (get-buffer-create " *message temp*") + (message-clone-locals buf) ;; for message-encoded-mail-cache (erase-buffer) (insert-buffer-substring buf) ,@body))) @@ -158,7 +159,10 @@ Otherwise set it according to `notmuch-fcc-dirs'." This should be called on a temporary copy. This is taken from the function message-do-fcc." - (message-encode-message-body) + (if (not message-encoded-mail-cache) + (message-encode-message-body) + (erase-buffer) + (insert message-encoded-mail-cache)) (save-restriction (message-narrow-to-headers) (mail-encode-encoded-word-buffer)) @@ -179,12 +183,12 @@ This is a rearranged version of message mode's message-do-fcc." (setq file (message-fetch-field "fcc" t))) (when file (with-temporary-notmuch-message-buffer + (notmuch-maildir-setup-message-for-saving) (save-restriction (message-narrow-to-headers) (while (setq file (message-fetch-field "fcc" t)) (push file files) (message-remove-header "fcc" nil t))) - (notmuch-maildir-setup-message-for-saving) ;; Process FCC operations. (mapc #'notmuch-fcc-handler files) (kill-buffer (current-buffer))))))) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 209b8a74..3c6626b4 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -21,7 +21,6 @@ test_expect_success \ "(mml-secure-message-sign)"' test_begin_subtest "emacs delivery of signed message via fcc and smtp" -test_subtest_known_broken emacs_deliver_message \ 'signed message sent via SMTP' \ 'This is a test that messages are sent via SMTP' \ From e02bf15a4f22b0c2eace7f604a1c84d54eaa7f4d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:00 -0400 Subject: [PATCH 087/113] test/search: add known broken tests for 'date' prefix This is documented, but apparently not implemented. Add tests for planned supported syntax and error messages. --- test/T081-sexpr-search.sh | 72 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 2a8ad5f1..d8f8872d 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -768,6 +768,78 @@ notmuch search date:2009-11-18..2009-11-18 and tag:unread > EXPECTED notmuch search --query=sexp '(and (infix "date:2009-11-18..2009-11-18") (infix "tag:unread"))' > OUTPUT test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "date query, empty" +test_subtest_known_broken +notmuch search from:keithp | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp '(and (date) (from keithp))'| notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, one argument" +test_subtest_known_broken +notmuch search date:2009-11-18 and from:keithp | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp '(and (date 2009-11-18) (from keithp))' | notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, two arguments" +test_subtest_known_broken +notmuch search date:2009-11-17..2009-11-18 and from:keithp | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp '(and (date 2009-11-17 2009-11-18) (from keithp))' | notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, illegal nesting 1" +test_subtest_known_broken +notmuch search --query=sexp '(to (date))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +nested field: 'date' inside 'to' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, illegal nesting 2" +test_subtest_known_broken +notmuch search --query=sexp '(to (date 2021-11-18))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +nested field: 'date' inside 'to' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, illegal nesting 3" +test_subtest_known_broken +notmuch search --query=sexp '(date (to))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +expected atom as first argument of 'date' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, illegal nesting 4" +test_subtest_known_broken +notmuch search --query=sexp '(date today (to))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +expected atom as second argument of 'date' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, too many arguments" +test_subtest_known_broken +notmuch search --query=sexp '(date yesterday and tommorow)' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +'date' expects maximum of two arguments +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "date query, bad date" +test_subtest_known_broken +notmuch search --query=sexp '(date "hawaiian-pizza-day")' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +Didn't understand date specification 'hawaiian-pizza-day' +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "user header (unknown header)" notmuch search --query=sexp '(FooBar)' >& OUTPUT cat < EXPECTED From 303f207a54325158105aba400702637e670decda Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:01 -0400 Subject: [PATCH 088/113] lib/parse-sexp: support zero argument date queries These are not too practical, although they may simplify some user query generation code. Mainly this adds a new prefix keyword to the parser. --- lib/parse-sexp.cc | 22 +++++++++++++++++++++- test/T081-sexpr-search.sh | 3 --- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc index 356c32ea..f36d18a6 100644 --- a/lib/parse-sexp.cc +++ b/lib/parse-sexp.cc @@ -32,6 +32,7 @@ typedef enum { SEXP_FLAG_EXPAND = 1 << 6, SEXP_FLAG_DO_EXPAND = 1 << 7, SEXP_FLAG_ORPHAN = 1 << 8, + SEXP_FLAG_RANGE = 1 << 9, } _sexp_flag_t; /* @@ -66,6 +67,8 @@ static _sexp_prefix_t prefixes[] = SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND }, { "body", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD }, + { "date", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, + SEXP_FLAG_RANGE }, { "from", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "folder", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, @@ -446,6 +449,19 @@ _sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } +static notmuch_status_t +_sexp_parse_date (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output) +{ + /* empty date matches everything */ + if (! sx) { + output = Xapian::Query::MatchAll; + return NOTMUCH_STATUS_SUCCESS; + } + + _notmuch_database_log (notmuch, "unimplemented date query\n"); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; +} + /* Here we expect the s-expression to be a proper list, with first * element defining and operation, or as a special case the empty * list */ @@ -519,7 +535,7 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) { if (strcmp (prefix->name, sx->list->val) == 0) { - if (prefix->flags & SEXP_FLAG_FIELD) { + if (prefix->flags & (SEXP_FLAG_FIELD | SEXP_FLAG_RANGE)) { if (parent) { _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n", prefix->name, parent->name); @@ -541,6 +557,10 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } + if (strcmp (prefix->name, "date") == 0) { + return _sexp_parse_date (notmuch, sx->list->next, output); + } + if (strcmp (prefix->name, "infix") == 0) { return _sexp_parse_infix (notmuch, sx->list->next, output); } diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index d8f8872d..a6e2ee82 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -769,7 +769,6 @@ notmuch search --query=sexp '(and (infix "date:2009-11-18..2009-11-18") (infix test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, empty" -test_subtest_known_broken notmuch search from:keithp | notmuch_search_sanitize > EXPECTED notmuch search --query=sexp '(and (date) (from keithp))'| notmuch_search_sanitize > OUTPUT test_expect_equal_file EXPECTED OUTPUT @@ -787,7 +786,6 @@ notmuch search --query=sexp '(and (date 2009-11-17 2009-11-18) (from keithp))' test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, illegal nesting 1" -test_subtest_known_broken notmuch search --query=sexp '(to (date))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -796,7 +794,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, illegal nesting 2" -test_subtest_known_broken notmuch search --query=sexp '(to (date 2021-11-18))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query From bf3cc5eed2d7f3386946b0c51e45c057705d24b8 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:02 -0400 Subject: [PATCH 089/113] lib/date: factor out date range parsing. This will allow re-using the same logic in the s-expression parser. --- lib/database-private.h | 5 +++++ lib/parse-time-vrp.cc | 51 +++++++++++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/lib/database-private.h b/lib/database-private.h index 5db8882f..0c08fa15 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -379,5 +379,10 @@ notmuch_status_t _notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr, Xapian::Query &output); #endif + +/* parse-time-vrp.h */ +notmuch_status_t +_notmuch_date_strings_to_query (Xapian::valueno slot, const std::string &from, const std::string &to, + Xapian::Query &output, std::string &msg); #endif #endif diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc index 22bf2ab5..6b07970b 100644 --- a/lib/parse-time-vrp.cc +++ b/lib/parse-time-vrp.cc @@ -24,22 +24,28 @@ #include "parse-time-vrp.h" #include "parse-time-string.h" -Xapian::Query -ParseTimeRangeProcessor::operator() (const std::string &begin, const std::string &end) +notmuch_status_t +_notmuch_date_strings_to_query (Xapian::valueno slot, + const std::string &begin, const std::string &end, + Xapian::Query &output, std::string &msg) { double from = DBL_MIN, to = DBL_MAX; time_t parsed_time, now; std::string str; /* Use the same 'now' for begin and end. */ - if (time (&now) == (time_t) -1) - throw Xapian::QueryParserError ("unable to get current time"); + if (time (&now) == (time_t) -1) { + msg = "unable to get current time"; + return NOTMUCH_STATUS_ILLEGAL_ARGUMENT; + } if (! begin.empty ()) { - if (parse_time_string (begin.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_DOWN)) - throw Xapian::QueryParserError ("Didn't understand date specification '" + begin + "'"); - else - from = (double) parsed_time; + if (parse_time_string (begin.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_DOWN)) { + msg = "Didn't understand date specification '" + begin + "'"; + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + from = (double) parsed_time; } if (! end.empty ()) { @@ -48,15 +54,30 @@ ParseTimeRangeProcessor::operator() (const std::string &begin, const std::string else str = end; - if (parse_time_string (str.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_UP_INCLUSIVE)) - throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'"); - else - to = (double) parsed_time; + if (parse_time_string (str.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_UP_INCLUSIVE)) { + msg = "Didn't understand date specification '" + str + "'"; + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + to = (double) parsed_time; } - return Xapian::Query (Xapian::Query::OP_VALUE_RANGE, slot, - Xapian::sortable_serialise (from), - Xapian::sortable_serialise (to)); + output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, slot, + Xapian::sortable_serialise (from), + Xapian::sortable_serialise (to)); + return NOTMUCH_STATUS_SUCCESS; +} + +Xapian::Query +ParseTimeRangeProcessor::operator() (const std::string &begin, const std::string &end) +{ + + Xapian::Query output; + std::string msg; + + if (_notmuch_date_strings_to_query (slot, begin, end, output, msg)) + throw Xapian::QueryParserError (msg); + + return output; } /* XXX TODO: is throwing an exception the right thing to do here? */ From 77ab961a1dce3a31d1f0edf5c9bd83c295575835 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:03 -0400 Subject: [PATCH 090/113] lib/parse-sexp: support actual date queries. The default argument processing overlaps somewhat with what is already done in _notmuch_date_strings_to_query, but we can give more specific error messages for the s-expression context. The extra generality of _sexp_parse_range will be useful when we implement additional range prefixes (at least 'lastmod' is needed). --- lib/parse-sexp.cc | 48 ++++++++++++++++++++++++++++++++++----- test/T081-sexpr-search.sh | 6 ----- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc index f36d18a6..dbc3f89d 100644 --- a/lib/parse-sexp.cc +++ b/lib/parse-sexp.cc @@ -450,15 +450,52 @@ _sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, } static notmuch_status_t -_sexp_parse_date (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output) +_sexp_parse_range (notmuch_database_t *notmuch, const _sexp_prefix_t *prefix, + const sexp_t *sx, Xapian::Query &output) { - /* empty date matches everything */ + const char *from, *to; + std::string msg; + + /* empty range matches everything */ if (! sx) { output = Xapian::Query::MatchAll; return NOTMUCH_STATUS_SUCCESS; } - _notmuch_database_log (notmuch, "unimplemented date query\n"); + if (sx->ty == SEXP_LIST) { + _notmuch_database_log (notmuch, "expected atom as first argument of '%s'\n", prefix->name); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + from = sx->val; + to = from; + + if (sx->next) { + if (sx->next->ty == SEXP_LIST) { + _notmuch_database_log (notmuch, "expected atom as second argument of '%s'\n", + prefix->name); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + if (sx->next->next) { + _notmuch_database_log (notmuch, "'%s' expects maximum of two arguments\n", prefix->name); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + to = sx->next->val; + } + + if (strcmp (prefix->name, "date") == 0) { + notmuch_status_t status; + status = _notmuch_date_strings_to_query (NOTMUCH_VALUE_TIMESTAMP, from, to, output, msg); + if (status) { + if (! msg.empty ()) + _notmuch_database_log (notmuch, "%s\n", msg.c_str ()); + } + return status; + } + + _notmuch_database_log (notmuch, "unimplimented range prefix: '%s'\n", prefix->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } @@ -557,9 +594,8 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } - if (strcmp (prefix->name, "date") == 0) { - return _sexp_parse_date (notmuch, sx->list->next, output); - } + if (prefix->flags & SEXP_FLAG_RANGE) + return _sexp_parse_range (notmuch, prefix, sx->list->next, output); if (strcmp (prefix->name, "infix") == 0) { return _sexp_parse_infix (notmuch, sx->list->next, output); diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index a6e2ee82..32b30493 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -774,13 +774,11 @@ notmuch search --query=sexp '(and (date) (from keithp))'| notmuch_search_saniti test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, one argument" -test_subtest_known_broken notmuch search date:2009-11-18 and from:keithp | notmuch_search_sanitize > EXPECTED notmuch search --query=sexp '(and (date 2009-11-18) (from keithp))' | notmuch_search_sanitize > OUTPUT test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, two arguments" -test_subtest_known_broken notmuch search date:2009-11-17..2009-11-18 and from:keithp | notmuch_search_sanitize > EXPECTED notmuch search --query=sexp '(and (date 2009-11-17 2009-11-18) (from keithp))' | notmuch_search_sanitize > OUTPUT test_expect_equal_file EXPECTED OUTPUT @@ -802,7 +800,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, illegal nesting 3" -test_subtest_known_broken notmuch search --query=sexp '(date (to))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -811,7 +808,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, illegal nesting 4" -test_subtest_known_broken notmuch search --query=sexp '(date today (to))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -820,7 +816,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, too many arguments" -test_subtest_known_broken notmuch search --query=sexp '(date yesterday and tommorow)' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -829,7 +824,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "date query, bad date" -test_subtest_known_broken notmuch search --query=sexp '(date "hawaiian-pizza-day")' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query From 341016c8ec34f838c2d72dc1bbfac567df054cd9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:04 -0400 Subject: [PATCH 091/113] test/sexpr-search: add known broken tests for lastmod queries These are loosely modelled on the tests just above for date ranges, since the error conditions are similar. Some ideas also borrowed from T570-revision-tracking. --- test/T081-sexpr-search.sh | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 32b30493..4231dd38 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -831,6 +831,91 @@ Didn't understand date specification 'hawaiian-pizza-day' EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "lastmod query, empty" +test_subtest_known_broken +notmuch search from:keithp | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp '(and (lastmod) (from keithp))'| notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, one argument" +test_subtest_known_broken +notmuch tag +4EFC743A.3060609@april.org id:4EFC743A.3060609@april.org +revision=$(notmuch count --lastmod '*' | cut -f3) +notmuch search lastmod:$revision..$revision | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp "(and (lastmod $revision))" | notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, two arguments" +test_subtest_known_broken +notmuch tag +keithp from:keithp +revision2=$(notmuch count --lastmod '*' | cut -f3) +notmuch search lastmod:$revision..$revision2 | notmuch_search_sanitize > EXPECTED +notmuch search --query=sexp "(and (lastmod $revision $revision2))" | notmuch_search_sanitize > OUTPUT +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, illegal nesting 1" +test_subtest_known_broken +notmuch search --query=sexp '(to (lastmod))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +nested field: 'lastmod' inside 'to' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, bad from revision" +test_subtest_known_broken +notmuch search --query=sexp '(lastmod apples)' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +bad 'from' revision: 'apples' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, bad to revision" +test_subtest_known_broken +notmuch search --query=sexp '(lastmod 0 apples)' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +bad 'to' revision: 'apples' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, illegal nesting 2" +test_subtest_known_broken +notmuch search --query=sexp '(to (lastmod 2021-11-18))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +nested field: 'lastmod' inside 'to' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, illegal nesting 3" +test_subtest_known_broken +notmuch search --query=sexp '(lastmod (to))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +expected atom as first argument of 'lastmod' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, illegal nesting 4" +test_subtest_known_broken +notmuch search --query=sexp '(lastmod today (to))' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +expected atom as second argument of 'lastmod' +EOF +test_expect_equal_file EXPECTED OUTPUT + +test_begin_subtest "lastmod query, too many arguments" +test_subtest_known_broken +notmuch search --query=sexp '(lastmod yesterday and tommorow)' > OUTPUT 2>&1 +cat < EXPECTED +notmuch search: Syntax error in query +'lastmod' expects maximum of two arguments +EOF +test_expect_equal_file EXPECTED OUTPUT + test_begin_subtest "user header (unknown header)" notmuch search --query=sexp '(FooBar)' >& OUTPUT cat < EXPECTED From 0a32741fceb7778ced34064eacb7b5aac2c71638 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 20 Jan 2022 09:36:05 -0400 Subject: [PATCH 092/113] lib/parse-sexp: handle lastmod queries. This particular choice of converting strings to integers requires C++11. --- lib/parse-sexp.cc | 25 +++++++++++++++++++++++++ test/T081-sexpr-search.sh | 10 ---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc index dbc3f89d..06825dc4 100644 --- a/lib/parse-sexp.cc +++ b/lib/parse-sexp.cc @@ -79,6 +79,8 @@ static _sexp_prefix_t prefixes[] = SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN }, { "is", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, + { "lastmod", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, + SEXP_FLAG_RANGE }, { "matching", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_DO_EXPAND }, { "mid", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, @@ -495,6 +497,29 @@ _sexp_parse_range (notmuch_database_t *notmuch, const _sexp_prefix_t *prefix, return status; } + if (strcmp (prefix->name, "lastmod") == 0) { + long from_idx, to_idx; + + try { + from_idx = std::stol (from); + } catch (std::logic_error &e) { + _notmuch_database_log (notmuch, "bad 'from' revision: '%s'\n", from); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + try { + to_idx = std::stol (to); + } catch (std::logic_error &e) { + _notmuch_database_log (notmuch, "bad 'to' revision: '%s'\n", to); + return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; + } + + output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, NOTMUCH_VALUE_LAST_MOD, + Xapian::sortable_serialise (from_idx), + Xapian::sortable_serialise (to_idx)); + return NOTMUCH_STATUS_SUCCESS; + } + _notmuch_database_log (notmuch, "unimplimented range prefix: '%s'\n", prefix->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 4231dd38..73f45041 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -832,13 +832,11 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, empty" -test_subtest_known_broken notmuch search from:keithp | notmuch_search_sanitize > EXPECTED notmuch search --query=sexp '(and (lastmod) (from keithp))'| notmuch_search_sanitize > OUTPUT test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, one argument" -test_subtest_known_broken notmuch tag +4EFC743A.3060609@april.org id:4EFC743A.3060609@april.org revision=$(notmuch count --lastmod '*' | cut -f3) notmuch search lastmod:$revision..$revision | notmuch_search_sanitize > EXPECTED @@ -846,7 +844,6 @@ notmuch search --query=sexp "(and (lastmod $revision))" | notmuch_search_saniti test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, two arguments" -test_subtest_known_broken notmuch tag +keithp from:keithp revision2=$(notmuch count --lastmod '*' | cut -f3) notmuch search lastmod:$revision..$revision2 | notmuch_search_sanitize > EXPECTED @@ -854,7 +851,6 @@ notmuch search --query=sexp "(and (lastmod $revision $revision2))" | notmuch_se test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, illegal nesting 1" -test_subtest_known_broken notmuch search --query=sexp '(to (lastmod))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -863,7 +859,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, bad from revision" -test_subtest_known_broken notmuch search --query=sexp '(lastmod apples)' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -872,7 +867,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, bad to revision" -test_subtest_known_broken notmuch search --query=sexp '(lastmod 0 apples)' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -881,7 +875,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, illegal nesting 2" -test_subtest_known_broken notmuch search --query=sexp '(to (lastmod 2021-11-18))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -890,7 +883,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, illegal nesting 3" -test_subtest_known_broken notmuch search --query=sexp '(lastmod (to))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -899,7 +891,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, illegal nesting 4" -test_subtest_known_broken notmuch search --query=sexp '(lastmod today (to))' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query @@ -908,7 +899,6 @@ EOF test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "lastmod query, too many arguments" -test_subtest_known_broken notmuch search --query=sexp '(lastmod yesterday and tommorow)' > OUTPUT 2>&1 cat < EXPECTED notmuch search: Syntax error in query From 0756105832612ad12661d1dd61b7e7b5bfe99c6c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:48 -0400 Subject: [PATCH 093/113] test: sanitize generated message files names It is fragile to encode the generated names into tests, as it makes tests break when e.g. new tests are added. There is a possibility that this will hide certain failures; in that case meaningful filenames should be chosen for the generated messages. --- test/T081-sexpr-search.sh | 2 +- test/T100-search-by-folder.sh | 6 +++--- test/T370-search-folder-coherence.sh | 4 ++-- test/T750-gzip.sh | 14 +++++++------- test/test-lib.sh | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 73f45041..0e5af635 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -346,7 +346,7 @@ output=$(notmuch search --query=sexp '(attachment (starts-with not))' | notmuch_ test_expect_equal "$output" 'thread:XXX 2009-11-18 [2/2] Lars Kellogg-Stedman; [notmuch] "notmuch help" outputs to stderr? (attachment inbox signed unread)' test_begin_subtest "starts-with, folder" -notmuch search --output=files --query=sexp '(folder (starts-with bad))' | notmuch_dir_sanitize | sed 's/[0-9]*$/XXX/' > OUTPUT +notmuch search --output=files --query=sexp '(folder (starts-with bad))' | notmuch_search_files_sanitize > OUTPUT cat < EXPECTED MAIL_DIR/bad/msg-XXX MAIL_DIR/bad/news/msg-XXX diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh index 409cfdcc..fc9ad5f8 100755 --- a/test/T100-search-by-folder.sh +++ b/test/T100-search-by-folder.sh @@ -28,8 +28,8 @@ test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite test_begin_subtest "Folder search with --output=files" output=$(notmuch search --output=files folder:bad/news | notmuch_search_files_sanitize) -test_expect_equal "$output" "MAIL_DIR/bad/news/msg-003 -MAIL_DIR/duplicate/bad/news/msg-003" +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" test_begin_subtest "After removing duplicate instance of matching path" rm -r "${MAIL_DIR}/bad/news" @@ -39,7 +39,7 @@ test_expect_equal "$output" "" test_begin_subtest "Folder search with --output=files part #2" output=$(notmuch search --output=files folder:duplicate/bad/news | notmuch_search_files_sanitize) -test_expect_equal "$output" "MAIL_DIR/duplicate/bad/news/msg-003" +test_expect_equal "$output" "MAIL_DIR/duplicate/bad/news/msg-XXX" test_begin_subtest "After removing duplicate instance of matching path part #2" output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize) diff --git a/test/T370-search-folder-coherence.sh b/test/T370-search-folder-coherence.sh index 0a2727e7..cf202bb3 100755 --- a/test/T370-search-folder-coherence.sh +++ b/test/T370-search-folder-coherence.sh @@ -24,8 +24,8 @@ test_expect_equal "$output" "No new mail." test_begin_subtest "Multiple files for same message" cat <EXPECTED -MAIL_DIR/msg-001 -MAIL_DIR/spam/msg-001 +MAIL_DIR/msg-XXX +MAIL_DIR/spam/msg-XXX EOF notmuch search --output=files id:$id_x | notmuch_search_files_sanitize >OUTPUT test_expect_equal_file EXPECTED OUTPUT diff --git a/test/T750-gzip.sh b/test/T750-gzip.sh index 4408d085..5648896f 100755 --- a/test/T750-gzip.sh +++ b/test/T750-gzip.sh @@ -58,13 +58,13 @@ test_expect_equal_file EXPECTED OUTPUT test_begin_subtest "notmuch search --output=files with partially gzipped mail store" notmuch search --output=files '*' | notmuch_search_files_sanitize > OUTPUT cat < EXPECTED -MAIL_DIR/msg-001.gz -MAIL_DIR/msg-002.gz -MAIL_DIR/msg-003.gz -MAIL_DIR/msg-004 -MAIL_DIR/msg-005.gz -MAIL_DIR/msg-006 -MAIL_DIR/msg-007.gz +MAIL_DIR/msg-XXX.gz +MAIL_DIR/msg-XXX.gz +MAIL_DIR/msg-XXX.gz +MAIL_DIR/msg-XXX +MAIL_DIR/msg-XXX.gz +MAIL_DIR/msg-XXX +MAIL_DIR/msg-XXX.gz EOF test_expect_equal_file EXPECTED OUTPUT diff --git a/test/test-lib.sh b/test/test-lib.sh index af017ec2..833bf5fe 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -551,7 +551,7 @@ notmuch_search_sanitize () { } notmuch_search_files_sanitize () { - notmuch_dir_sanitize + notmuch_dir_sanitize | sed 's/msg-[0-9][0-9][0-9]/msg-XXX/' } notmuch_dir_sanitize () { From 053a390370922a213b57f4f2e4bc2c5c8fb88c74 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:49 -0400 Subject: [PATCH 094/113] test: known broken tests for trailing / in path search (infix) In [1], David Edmondson observed that the trailing / added by many completion mechanisms causes path searches to silently fail. This test reproduces that bug for both `path:' and `folder:' searches. [1]: id:cunoasuolcv.fsf@gargravarr.hh.sledj.net --- test/T100-search-by-folder.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh index fc9ad5f8..fbc68266 100755 --- a/test/T100-search-by-folder.sh +++ b/test/T100-search-by-folder.sh @@ -31,6 +31,12 @@ output=$(notmuch search --output=files folder:bad/news | notmuch_search_files_sa test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" +test_begin_subtest "Folder search with --output=files (trailing /)" +test_subtest_known_broken +output=$(notmuch search --output=files folder:bad/news/ | notmuch_search_files_sanitize) +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" + test_begin_subtest "After removing duplicate instance of matching path" rm -r "${MAIL_DIR}/bad/news" notmuch new @@ -120,6 +126,14 @@ test_expect_equal "$output" "MAIL_DIR/bar/17:2, MAIL_DIR/bar/18:2, MAIL_DIR/cur/51:2," +test_begin_subtest "path: search (trailing /)" +test_subtest_known_broken +output=$(notmuch search --output=files path:"bar/" | notmuch_search_files_sanitize | sort) +# cur/51:2, is a duplicate of bar/18:2, +test_expect_equal "$output" "MAIL_DIR/bar/17:2, +MAIL_DIR/bar/18:2, +MAIL_DIR/cur/51:2," + test_begin_subtest "top level path: search" output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort) test_expect_equal "$output" "MAIL_DIR/01:2, From c73e273aaf7410c16a9597a041bd09f49352ced3 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:50 -0400 Subject: [PATCH 095/113] test/sexp: tests for path, folder, including trailing '/' (sexp) This duplicates the bug reported in [1], as well as adding some simple regression tests for 'path' and 'folder' searches which were previously missing for sexp syntax. [1]: id:cunoasuolcv.fsf@gargravarr.hh.sledj.net --- test/T081-sexpr-search.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index 0e5af635..a355f8b6 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -185,6 +185,28 @@ notmuch search folder:'""' > EXPECTED notmuch search --query=sexp '(folder "")' > OUTPUT test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest "Search by 'folder' with --output=files" +output=$(notmuch search --output=files --query=sexp '(folder bad/news)' | notmuch_search_files_sanitize) +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" + +test_begin_subtest "Search by 'folder' with --output=files (trailing /)" +test_subtest_known_broken +output=$(notmuch search --output=files --query=sexp '(folder bad/news/)' | notmuch_search_files_sanitize) +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" + +test_begin_subtest "Search by 'path' with --output=files" +output=$(notmuch search --output=files --query=sexp '(path bad/news)' | notmuch_search_files_sanitize) +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" + +test_begin_subtest "Search by 'path' with --output=files (trailing /)" +test_subtest_known_broken +output=$(notmuch search --output=files --query=sexp '(path bad/news/)' | notmuch_search_files_sanitize) +test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX +MAIL_DIR/duplicate/bad/news/msg-XXX" + test_begin_subtest "Search by 'id'" add_message '[subject]="search by id"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' output=$(notmuch search --query=sexp "(id ${gen_msg_id})" | notmuch_search_sanitize) From c62c22c9fb222d43d9b9956ce6b6e9985019ea2d Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:51 -0400 Subject: [PATCH 096/113] lib: drop trailing slash for path and folder searches (infix) This resolves an old bug reported by David Edmondson in 2014. The fix is only needed for the "boolean" case, as probabilistic / phrase searching already ignores punctuation. This fix is only for the infix (xapian provided) query parser. [1]: id:cunoasuolcv.fsf@gargravarr.hh.sledj.net --- lib/database-private.h | 1 + lib/prefix.cc | 4 ++-- lib/regexp-fields.cc | 10 +++++++++- test/T100-search-by-folder.sh | 2 -- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/database-private.h b/lib/database-private.h index 0c08fa15..657b1aa1 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -165,6 +165,7 @@ typedef enum { NOTMUCH_FIELD_EXTERNAL = 1 << 0, NOTMUCH_FIELD_PROBABILISTIC = 1 << 1, NOTMUCH_FIELD_PROCESSOR = 1 << 2, + NOTMUCH_FIELD_STRIP_TRAILING_SLASH = 1 << 3, } notmuch_field_flag_t; /* diff --git a/lib/prefix.cc b/lib/prefix.cc index 0d92bdd7..857c05b9 100644 --- a/lib/prefix.cc +++ b/lib/prefix.cc @@ -46,7 +46,7 @@ prefix_t prefix_table[] = { { "mid", "Q", NOTMUCH_FIELD_EXTERNAL | NOTMUCH_FIELD_PROCESSOR }, { "path", "P", NOTMUCH_FIELD_EXTERNAL | - NOTMUCH_FIELD_PROCESSOR }, + NOTMUCH_FIELD_PROCESSOR | NOTMUCH_FIELD_STRIP_TRAILING_SLASH }, { "property", "XPROPERTY", NOTMUCH_FIELD_EXTERNAL }, /* * Unconditionally add ':' to reduce potential ambiguity with @@ -55,7 +55,7 @@ prefix_t prefix_table[] = { * discussion. */ { "folder", "XFOLDER:", NOTMUCH_FIELD_EXTERNAL | - NOTMUCH_FIELD_PROCESSOR }, + NOTMUCH_FIELD_PROCESSOR | NOTMUCH_FIELD_STRIP_TRAILING_SLASH }, { "date", NULL, NOTMUCH_FIELD_EXTERNAL | NOTMUCH_FIELD_PROCESSOR }, { "query", NULL, NOTMUCH_FIELD_EXTERNAL | diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc index c6d9d94f..7e9d959c 100644 --- a/lib/regexp-fields.cc +++ b/lib/regexp-fields.cc @@ -235,7 +235,15 @@ RegexpFieldProcessor::operator() (const std::string & str) return parser.parse_query (query_str, NOTMUCH_QUERY_PARSER_FLAGS, term_prefix); } else { /* Boolean prefix */ - std::string term = term_prefix + str; + std::string query_str; + std::string term; + + if (str.length () > 1 && str.at (str.size () - 1) == '/') + query_str = str.substr (0, str.size () - 1); + else + query_str = str; + + term = term_prefix + query_str; return Xapian::Query (term); } } diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh index fbc68266..0123d0b2 100755 --- a/test/T100-search-by-folder.sh +++ b/test/T100-search-by-folder.sh @@ -32,7 +32,6 @@ test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" test_begin_subtest "Folder search with --output=files (trailing /)" -test_subtest_known_broken output=$(notmuch search --output=files folder:bad/news/ | notmuch_search_files_sanitize) test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" @@ -127,7 +126,6 @@ MAIL_DIR/bar/18:2, MAIL_DIR/cur/51:2," test_begin_subtest "path: search (trailing /)" -test_subtest_known_broken output=$(notmuch search --output=files path:"bar/" | notmuch_search_files_sanitize | sort) # cur/51:2, is a duplicate of bar/18:2, test_expect_equal "$output" "MAIL_DIR/bar/17:2, From 8358841c66ee11110a784f2559e07477c067ba10 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:52 -0400 Subject: [PATCH 097/113] test: add multiple path, folder sexp query tests This is mainly to make sure we get trailing / removal correct. Also add regression test for path: in the infix parser matching the existing one for folder:. --- test/T081-sexpr-search.sh | 26 ++++++++++++++++++++++++++ test/T100-search-by-folder.sh | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index a355f8b6..ebf2c59f 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -196,6 +196,19 @@ output=$(notmuch search --output=files --query=sexp '(folder bad/news/)' | notmu test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" +test_begin_subtest "Search by 'folder' (multiple)" +output=$(notmuch search --query=sexp '(folder bad bad/news things/bad)' | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) +thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + +test_begin_subtest "Search by 'folder' (multiple, trailing /)" +test_subtest_known_broken +output=$(notmuch search --query=sexp '(folder bad bad/news/ things/bad)' | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) +thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + test_begin_subtest "Search by 'path' with --output=files" output=$(notmuch search --output=files --query=sexp '(path bad/news)' | notmuch_search_files_sanitize) test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX @@ -207,6 +220,19 @@ output=$(notmuch search --output=files --query=sexp '(path bad/news/)' | notmuch test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" +test_begin_subtest "Search by 'path' specification (multiple)" +output=$(notmuch search --query=sexp '(path bad bad/news things/bad)' | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) +thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + +test_begin_subtest "Search by 'path' specification (multiple, trailing /)" +test_subtest_known_broken +output=$(notmuch search --query=sexp '(path bad bad/news/ things/bad)' | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) +thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + test_begin_subtest "Search by 'id'" add_message '[subject]="search by id"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' output=$(notmuch search --query=sexp "(id ${gen_msg_id})" | notmuch_search_sanitize) diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh index 0123d0b2..b4f6294e 100755 --- a/test/T100-search-by-folder.sh +++ b/test/T100-search-by-folder.sh @@ -18,6 +18,12 @@ test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; T thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" +test_begin_subtest "search by path: specification (multiple)" +output=$(notmuch search path:bad path:bad/news path:things/bad | notmuch_search_sanitize) +test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) +thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) +thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + test_begin_subtest "Top level folder" output=$(notmuch search folder:'""' | notmuch_search_sanitize) test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Top level (inbox unread)" From 2c1d1107f5dacdb4a2c514909fd96f45f83e2f3c Mon Sep 17 00:00:00 2001 From: David Bremner Date: Fri, 21 Jan 2022 19:38:53 -0400 Subject: [PATCH 098/113] lib: strip trailing '/' from pathnames (sexp queries). This changes makes the sexp query parser consistent with the infix one in ignoring trailing '/'. Here we do a bit better and ignore any number of trailing '/'. --- lib/parse-sexp.cc | 14 +++++++++++--- test/T081-sexpr-search.sh | 4 ---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc index 06825dc4..08fd7037 100644 --- a/lib/parse-sexp.cc +++ b/lib/parse-sexp.cc @@ -33,6 +33,7 @@ typedef enum { SEXP_FLAG_DO_EXPAND = 1 << 7, SEXP_FLAG_ORPHAN = 1 << 8, SEXP_FLAG_RANGE = 1 << 9, + SEXP_FLAG_PATHNAME = 1 << 10, } _sexp_flag_t; /* @@ -72,7 +73,8 @@ static _sexp_prefix_t prefixes[] = { "from", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "folder", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, - SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, + SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND | + SEXP_FLAG_PATHNAME }, { "id", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX }, { "infix", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, @@ -94,7 +96,8 @@ static _sexp_prefix_t prefixes[] = { "or", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_NONE }, { "path", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, - SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX }, + SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | + SEXP_FLAG_PATHNAME }, { "property", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "query", Xapian::Query::OP_INVALID, Xapian::Query::MatchNothing, @@ -545,8 +548,13 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent return _sexp_parse_wildcard (notmuch, parent, env, "", output); } + char *atom = sx->val; + + if (parent && parent->flags & SEXP_FLAG_PATHNAME) + strip_trailing (atom, '/'); + if (parent && (parent->flags & SEXP_FLAG_BOOLEAN)) { - output = Xapian::Query (term_prefix + sx->val); + output = Xapian::Query (term_prefix + atom); return NOTMUCH_STATUS_SUCCESS; } diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh index ebf2c59f..e2936cd7 100755 --- a/test/T081-sexpr-search.sh +++ b/test/T081-sexpr-search.sh @@ -191,7 +191,6 @@ test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" test_begin_subtest "Search by 'folder' with --output=files (trailing /)" -test_subtest_known_broken output=$(notmuch search --output=files --query=sexp '(folder bad/news/)' | notmuch_search_files_sanitize) test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" @@ -203,7 +202,6 @@ thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" test_begin_subtest "Search by 'folder' (multiple, trailing /)" -test_subtest_known_broken output=$(notmuch search --query=sexp '(folder bad bad/news/ things/bad)' | notmuch_search_sanitize) test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) @@ -215,7 +213,6 @@ test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" test_begin_subtest "Search by 'path' with --output=files (trailing /)" -test_subtest_known_broken output=$(notmuch search --output=files --query=sexp '(path bad/news/)' | notmuch_search_files_sanitize) test_expect_equal "$output" "MAIL_DIR/bad/news/msg-XXX MAIL_DIR/duplicate/bad/news/msg-XXX" @@ -227,7 +224,6 @@ thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" test_begin_subtest "Search by 'path' specification (multiple, trailing /)" -test_subtest_known_broken output=$(notmuch search --query=sexp '(path bad bad/news/ things/bad)' | notmuch_search_sanitize) test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) thread:XXX 2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread) From 5c183d814b2cb3155cb8eec1c8ab44868ff75d30 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 26 Jan 2022 09:30:45 -0400 Subject: [PATCH 099/113] doc: replace :math: for subscripts in notmuch-sexp-queries(7) Sphinx pulls in mathjax to do :math:, which complicates viewing the html version offline. The direct :sub: is supported directly in html. --- doc/man7/notmuch-sexp-queries.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/man7/notmuch-sexp-queries.rst b/doc/man7/notmuch-sexp-queries.rst index 3c33232f..bc8e5086 100644 --- a/doc/man7/notmuch-sexp-queries.rst +++ b/doc/man7/notmuch-sexp-queries.rst @@ -336,10 +336,10 @@ NOTES in notmuch, this modifier is not supported in the ``path`` field. -.. |q1| replace:: :math:`q_1` -.. |q2| replace:: :math:`q_2` -.. |qn| replace:: :math:`q_n` +.. |q1| replace:: `q`\ :sub:`1` +.. |q2| replace:: `q`\ :sub:`2` +.. |qn| replace:: `q`\ :sub:`n` -.. |p1| replace:: :math:`p_1` -.. |p2| replace:: :math:`p_2` -.. |pn| replace:: :math:`p_n` +.. |p1| replace:: `p`\ :sub:`1` +.. |p2| replace:: `p`\ :sub:`2` +.. |pn| replace:: `p`\ :sub:`n` From 11f03c87a205ffa99ff5e7b301b03bf996a8d562 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Wed, 26 Jan 2022 09:30:46 -0400 Subject: [PATCH 100/113] debian: package HTML docs This is particularly useful for the docs for the notmuch2 python bindings, which are otherwise not packaged. --- debian/control | 18 +++++++++++++++++- debian/rules | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index 9872d602..a11d4130 100644 --- a/debian/control +++ b/debian/control @@ -56,7 +56,8 @@ Recommends: gnupg-agent, gpgsm, Suggests: - mailscripts + mailscripts, + notmuch-doc, Description: thread-based email index, search and tagging Notmuch is a system for indexing, searching, reading, and tagging large collections of email messages in maildir or mh format. It uses @@ -65,6 +66,21 @@ Description: thread-based email index, search and tagging . This package contains the notmuch command-line interface +Package: notmuch-doc +Architecture: all +Depends: + ${misc:Depends}, + ${sphinxdoc:Depends}, +Suggests: + notmuch +Description: thread-based email index, search and tagging + Notmuch is a system for indexing, searching, reading, and tagging + large collections of email messages in maildir or mh format. It uses + the Xapian library to provide fast, full-text search with a very + convenient search syntax. + . + This package contains the HTML documentation + Package: libnotmuch5 Section: libs Architecture: any diff --git a/debian/rules b/debian/rules index fa0551a9..55867126 100755 --- a/debian/rules +++ b/debian/rules @@ -3,7 +3,7 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: - dh $@ --with python3,elpa + dh $@ --with python3,elpa,sphinxdoc override_dh_auto_configure: BASHCMD=/bin/bash ./configure --prefix=/usr \ @@ -19,7 +19,7 @@ override_dh_auto_test: dh_auto_test -- V=1 override_dh_auto_build: - dh_auto_build -- V=1 + dh_auto_build -- V=1 all sphinx-html PYBUILD_NAME=notmuch dh_auto_build --buildsystem=pybuild --sourcedirectory bindings/python PYBUILD_NAME=notmuch2 dh_auto_build --buildsystem=pybuild --sourcedirectory bindings/python-cffi $(MAKE) -C contrib/notmuch-mutt From addeb11e37b0fff06211b1756d021732730ce0ee Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 18:12:58 -0400 Subject: [PATCH 101/113] version: bump to 0.35~rc0 Start release process for 0.35 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index d10f9654..9e974b10 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.34.3' +__VERSION__ = '0.35~rc0' SOVERSION = '5' diff --git a/version.txt b/version.txt index 0aeaf413..9881671b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.34.3 +0.35~rc0 From 7f8af14bdc53a4a793143651e708a44e29bb9522 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 18:13:26 -0400 Subject: [PATCH 102/113] lib: bump minor version to 6. One new status value and one configuration value added. --- lib/notmuch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/notmuch.h b/lib/notmuch.h index cd5c1246..2e6ec2af 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -58,7 +58,7 @@ NOTMUCH_BEGIN_DECLS * version in Makefile.local. */ #define LIBNOTMUCH_MAJOR_VERSION 5 -#define LIBNOTMUCH_MINOR_VERSION 5 +#define LIBNOTMUCH_MINOR_VERSION 6 #define LIBNOTMUCH_MICRO_VERSION 0 From 87000367362971130a09174e8ec1cc112356b564 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 18:15:05 -0400 Subject: [PATCH 103/113] debian: add changelog for 0.34~rc0-1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3880a004..95dda536 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +notmuch (0.35~rc0-1) experimental; urgency=medium + + * New upstream release candidate + + -- David Bremner Sat, 29 Jan 2022 18:14:57 -0400 + notmuch (0.34.3-1) unstable; urgency=medium * New upstream bugfix release, with several fixes for the notmuch2 From 6a9f66ef324aa286d4fb8721a10837784e87ee98 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 18:17:23 -0400 Subject: [PATCH 104/113] NEWS: add stub for 0.35 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index e39af344..84567592 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +Notmuch 0.35 (UNRELEASED) +========================= + Notmuch 0.34.3 (2022-01-09) =========================== From ade8202c1860d1b62bacedd2c54033800d0697b1 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 18:50:48 -0400 Subject: [PATCH 105/113] debian: add missing install file for notmuch-doc This was missed from commit 11f03c87a205ffa99ff5e7b301b03bf996a8d562 --- debian/notmuch-doc.install | 1 + 1 file changed, 1 insertion(+) create mode 100644 debian/notmuch-doc.install diff --git a/debian/notmuch-doc.install b/debian/notmuch-doc.install new file mode 100644 index 00000000..fa902fe1 --- /dev/null +++ b/debian/notmuch-doc.install @@ -0,0 +1 @@ +doc/_build/html usr/share/doc/notmuch From 28600c6ea1ed79f1b372c06fcdc839e7a6787dfc Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Jan 2022 21:54:00 -0400 Subject: [PATCH 106/113] debian: changelog for 0.35~rc0-2 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 95dda536..b0ce7baf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +notmuch (0.35~rc0-2) experimental; urgency=medium + + * Reupload with binaries + + -- David Bremner Sat, 29 Jan 2022 21:53:29 -0400 + notmuch (0.35~rc0-1) experimental; urgency=medium * New upstream release candidate From 437d15044ccbadf54f4e6f8183e6232ac00ead93 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 30 Jan 2022 09:23:26 -0400 Subject: [PATCH 107/113] NEWS: my changes for 0.35 --- NEWS | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/NEWS b/NEWS index 84567592..bc2a80de 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,74 @@ Notmuch 0.35 (UNRELEASED) ========================= +Library +------- + +Implement the `date` and `lastmod` fields in the S-expression parser. + +Ignore trailing `/` for pathnames in both query parsers. + +Rename configuration option `built_with.sexpr_query` to +`built_with.sexp_queries`. + +Do not assume a default mail root in split (e.g. XDG) configurations. + +Fix some small memory leaks in `notmuch_database_open_with_config`. + +CLI +--- + +Improve handling of leading/trailing punctation and space for +configuration lists. + +Only ignore `.notmuch` at the top level in `notmuch new`. + +Optionally show extra headers in `notmuch show`. See +`show.extra_headers` in notmuch-config(1). + +Emacs +----- + +Drop `C-TAB` binding in hello mode, document `backtab`. + +Fix visual glitch in search mode by running `notmuch-search-hook` +lazily. + +Don't add space to completion candidates, improves compatibility with +third party completion frameworks. + +Make citation formating more robust against whitespace. + +Use `--excludes=false` when generating the 'All tags' section. + +Use cached copy of message body for `Fcc`, avoiding variant bodies for +signed and/or encrypted messages. + +Build +----- + +Fix out-of-tree build for `python-cffi` bindings. + +Rearrange position of {C,CXX,CPP,LD}FLAGS, prevent some clashes with +installed version of notmuch. + +Test Suite +---------- + +Replace some uses of `gdb` in the test suite with `LD_PRELOAD` based +shims. + +Use `--with-colons` for gpgsm, fix compatibility with newer gnupg. + +Python bindings +--------------- + +Add `matched` property to message objects. + +Users are reminded that the old python bindings in bindings/python are +deprecated; this will probably be the last major release that ships +them. + Notmuch 0.34.3 (2022-01-09) =========================== From 0c357ce78516ca176156a12eb362fb1d854074b6 Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Mon, 31 Jan 2022 22:47:43 +0200 Subject: [PATCH 108/113] NEWS: emacs/notmuch-logo.svg is there in 0.35 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index bc2a80de..d03d0a33 100644 --- a/NEWS +++ b/NEWS @@ -44,6 +44,9 @@ Use `--excludes=false` when generating the 'All tags' section. Use cached copy of message body for `Fcc`, avoiding variant bodies for signed and/or encrypted messages. +Add notmuch-logo.svg and use it in notmuch-hello view, replacing +the .png version. + Build ----- From a4c58cbc1d9b0580c904a4b533f4147fa20fc7d9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 5 Feb 2022 08:43:28 -0400 Subject: [PATCH 109/113] NEWS: remaining items for 0.35 release These are my best guesses based on git commit messages. --- NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS b/NEWS index d03d0a33..9545a1bb 100644 --- a/NEWS +++ b/NEWS @@ -47,6 +47,10 @@ signed and/or encrypted messages. Add notmuch-logo.svg and use it in notmuch-hello view, replacing the .png version. +Make header line in show buffers optional. + +Add customizable names for search buffers. + Build ----- @@ -55,6 +59,8 @@ Fix out-of-tree build for `python-cffi` bindings. Rearrange position of {C,CXX,CPP,LD}FLAGS, prevent some clashes with installed version of notmuch. +Ignore more configure options. + Test Suite ---------- @@ -72,6 +78,11 @@ Users are reminded that the old python bindings in bindings/python are deprecated; this will probably be the last major release that ships them. +Completion +---------- + +Use `database.mail_root` for path completion in bash/zsh. + Notmuch 0.34.3 (2022-01-09) =========================== From 0bdd9ffd844bb6abef19d204fcfc72bb48939e30 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 6 Feb 2022 12:15:44 -0400 Subject: [PATCH 110/113] debian: start changelog for 0.35 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index b0ce7baf..35bb98cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +notmuch (0.35-1) unstable; urgency=medium + + * New upstream release + + -- David Bremner Sun, 06 Feb 2022 12:15:19 -0400 + notmuch (0.35~rc0-2) experimental; urgency=medium * Reupload with binaries From 748154646eb44c8db3a7887e0ae8e4701e552bc9 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 6 Feb 2022 13:10:43 -0400 Subject: [PATCH 111/113] NEWS: set release date for 0.35 --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9545a1bb..c6ce2eea 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -Notmuch 0.35 (UNRELEASED) +Notmuch 0.35 (2022-02-06) ========================= Library From b87373f8c41fb8014b7e756fba1e942410209bfb Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 6 Feb 2022 13:11:32 -0400 Subject: [PATCH 112/113] version: bump to 0.35 --- bindings/python/notmuch/version.py | 2 +- version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py index 9e974b10..f7392174 100644 --- a/bindings/python/notmuch/version.py +++ b/bindings/python/notmuch/version.py @@ -1,3 +1,3 @@ # this file should be kept in sync with ../../../version -__VERSION__ = '0.35~rc0' +__VERSION__ = '0.35' SOVERSION = '5' diff --git a/version.txt b/version.txt index 9881671b..c74e8a04 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.35~rc0 +0.35 From 7b5921877e748338359a25dae578771f768183af Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sun, 6 Feb 2022 13:14:54 -0400 Subject: [PATCH 113/113] gitignore: add bindings/python-cffi/_notmuch_config.py This is generated by configure and should not be committed. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3edd1768..f846ebec 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,6 @@ /sh.config /sphinx.config /version.stamp +/bindings/python-cffi/_notmuch_config.py TAGS tags