CLI/show: support --duplicate for structured output

This introduces a new mandatory key for message structures, namely
"duplicate". Per convention in devel/schemata this does _not_ increase
the format version. This means that clients are responsible for
checking that it exists, and not crashing if it does not.

The main functional change is teaching mime_node_open to understand a
'duplicate' argument.

Support for --duplicate in notmuch-reply would make sense, but we
defer it to a later commit.

devel/schemata

@@ -83,6 +83,7 @@ message = {
 
     headers:        headers,
     crypto:         crypto,
+    duplicate:      integer,
     body?:          [part]    # omitted if --body=false
 }
 

mime-node.c

@@ -78,13 +78,14 @@ mime_node_get_message_crypto_status (mime_node_t *node)
 
 notmuch_status_t
 mime_node_open (const void *ctx, notmuch_message_t *message,
+		int duplicate,
 		_notmuch_crypto_t *crypto, mime_node_t **root_out)
 {
     const char *filename = notmuch_message_get_filename (message);
     mime_node_context_t *mctx;
     mime_node_t *root;
     notmuch_status_t status;
-    int fd;
+    int fd = -1;
 
     root = talloc_zero (ctx, mime_node_t);
     if (root == NULL) {
@@ -103,20 +104,33 @@ mime_node_open (const void *ctx, notmuch_message_t *message,
     talloc_set_destructor (mctx, _mime_node_context_free);
 
     /* Fast path */
-    fd = open (filename, O_RDONLY);
+    if (duplicate <= 0)
+	fd = open (filename, O_RDONLY);
     if (fd == -1) {
-	/* Slow path - for some reason the first file in the list is
+	/* Slow path - Either we are trying to open a specific file, or
-	 * not available anymore. This is clearly a problem in the
+	 * for some reason the first file in the list is
+	 * not available anymore. The latter is clearly a problem in the
 	 * database, but we are not going to let this problem be a
 	 * show stopper */
 	notmuch_filenames_t *filenames;
+	int i = 1;
+
 	for (filenames = notmuch_message_get_filenames (message);
 	     notmuch_filenames_valid (filenames);
-	     notmuch_filenames_move_to_next (filenames)) {
+	     notmuch_filenames_move_to_next (filenames), i++) {
-	    filename = notmuch_filenames_get (filenames);
+	    if (i >= duplicate) {
-	    fd = open (filename, O_RDONLY);
+		filename = notmuch_filenames_get (filenames);
-	    if (fd != -1)
+		fd = open (filename, O_RDONLY);
-		break;
+		if (fd != -1) {
+		    break;
+		} else {
+		    if (duplicate > 0) {
+			fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno));
+			status = NOTMUCH_STATUS_FILE_ERROR;
+			goto DONE;
+		    }
+		}
+	    }
 	}
 
 	talloc_free (filenames);

notmuch-client.h

@@ -230,6 +230,7 @@ show_one_part (const char *filename, int part);
 
 void
 format_part_sprinter (const void *ctx, struct sprinter *sp, mime_node_t *node,
+		      int duplicate,
 		      bool output_body,
 		      bool include_html);
 
@@ -389,7 +390,8 @@ struct mime_node {
 };
 
 /* Construct a new MIME node pointing to the root message part of
- * message. If crypto->verify is true, signed child parts will be
+ * message. Use the duplicate-th filename if that parameter is
+ * positive. If crypto->verify is true, signed child parts will be
  * verified. If crypto->decrypt is NOTMUCH_DECRYPT_TRUE, encrypted
  * child parts will be decrypted using either stored session keys or
  * asymmetric crypto.  If crypto->decrypt is NOTMUCH_DECRYPT_AUTO,
@@ -407,6 +409,7 @@ struct mime_node {
  */
 notmuch_status_t
 mime_node_open (const void *ctx, notmuch_message_t *message,
+		int duplicate,
 		_notmuch_crypto_t *crypto, mime_node_t **node_out);
 
 /* Return a new MIME node for the requested child part of parent.

notmuch-reply.c

@@ -663,7 +663,7 @@ do_reply (notmuch_database_t *notmuch,
 	 notmuch_messages_move_to_next (messages)) {
 	message = notmuch_messages_get (messages);
 
-	if (mime_node_open (notmuch, message, &params->crypto, &node))
+	if (mime_node_open (notmuch, message, -1, &params->crypto, &node))
 	    return 1;
 
 	reply = create_reply_message (notmuch, message,
@@ -683,7 +683,7 @@ do_reply (notmuch_database_t *notmuch,
 
 	    /* Start the original */
 	    sp->map_key (sp, "original");
-	    format_part_sprinter (notmuch, sp, node, true, false);
+	    format_part_sprinter (notmuch, sp, node, -1, true, false);
 
 	    /* End */
 	    sp->end (sp);

notmuch-show.c

@@ -673,6 +673,7 @@ format_omitted_part_meta_sprinter (sprinter_t *sp, GMimeObject *meta, GMimePart
 
 void
 format_part_sprinter (const void *ctx, sprinter_t *sp, mime_node_t *node,
+		      int duplicate,
 		      bool output_body,
 		      bool include_html)
 {
@@ -684,10 +685,13 @@ format_part_sprinter (const void *ctx, sprinter_t *sp, mime_node_t *node,
 	sp->begin_map (sp);
 	format_message_sprinter (sp, node->envelope_file);
 
+	sp->map_key (sp, "duplicate");
+	sp->integer (sp, duplicate > 0 ? duplicate : 1);
+
 	if (output_body) {
 	    sp->map_key (sp, "body");
 	    sp->begin_list (sp);
-	    format_part_sprinter (ctx, sp, mime_node_child (node, 0), true, include_html);
+	    format_part_sprinter (ctx, sp, mime_node_child (node, 0), -1, true, include_html);
 	    sp->end (sp);
 	}
 
@@ -851,7 +855,7 @@ format_part_sprinter (const void *ctx, sprinter_t *sp, mime_node_t *node,
     }
 
     for (i = 0; i < node->nchildren; i++)
-	format_part_sprinter (ctx, sp, mime_node_child (node, i), true, include_html);
+	format_part_sprinter (ctx, sp, mime_node_child (node, i), -1, true, include_html);
 
     /* Close content structures */
     for (i = 0; i < nclose; i++)
@@ -865,7 +869,8 @@ format_part_sprinter_entry (const void *ctx, sprinter_t *sp,
 			    mime_node_t *node, unused (int indent),
 			    const notmuch_show_params_t *params)
 {
-    format_part_sprinter (ctx, sp, node, params->output_body, params->include_html);
+    format_part_sprinter (ctx, sp, node, params->duplicate, params->output_body,
+			  params->include_html);
 
     return NOTMUCH_STATUS_SUCCESS;
 }
@@ -1019,7 +1024,7 @@ show_message (void *ctx,
 	session_key_count_error = notmuch_message_count_properties (message, "session-key",
 								    &session_keys);
 
-    status = mime_node_open (local, message, &(params->crypto), &root);
+    status = mime_node_open (local, message, params->duplicate, &(params->crypto), &root);
     if (status)
 	goto DONE;
     part = mime_node_seek_dfs (root, (params->part < 0 ? 0 : params->part));

test/T160-json.sh

@@ -49,7 +49,7 @@ output=$(notmuch show --format=json "id:$id")
 filename=$(notmuch search --output=files "id:$id")
 # Get length of README after base64-encoding, minus additional newline.
 attachment_length=$(( $(base64 $NOTMUCH_SRCDIR/test/README | wc -c) - 1 ))
-test_expect_equal_json "$output" "[[[{\"id\": \"$id\",  \"crypto\": {}, \"match\": true, \"excluded\": false, \"filename\": [\"$filename\"], \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"content-length\": $attachment_length, \"content-transfer-encoding\": \"base64\", \"content-disposition\": \"inline\", \"filename\": \"README\"}]}]}, []]]]"
+test_expect_equal_json "$output" "[[[{\"id\": \"$id\", \"duplicate\": 1, \"crypto\": {}, \"match\": true, \"excluded\": false, \"filename\": [\"$filename\"], \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"content-length\": $attachment_length, \"content-transfer-encoding\": \"base64\", \"content-disposition\": \"inline\", \"filename\": \"README\"}]}]}, []]]]"
 
 test_begin_subtest "Search message: json, utf-8"
 add_message "[subject]=\"json-search-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-search-méssage\""
@@ -97,6 +97,7 @@ cat <<EOF > EXPECTED
         [
             {
                 "date_relative": "2001-01-05",
+		"duplicate": 1,
                 "excluded": false,
                 "filename": [
                     "${MAIL_DIR}/copy1",
@@ -132,6 +133,7 @@ cat <<EOF > EXPECTED
         [
             {
                 "date_relative": "2001-01-05",
+		"duplicate": 1,
                 "excluded": false,
                 "filename": "${MAIL_DIR}/copy1",
                 "headers": {

test/T170-sexp.sh

@@ -45,7 +45,7 @@ output=$(notmuch show --format=sexp "id:$id")
 filename=$(notmuch search --output=files "id:$id")
 # Get length of README after base64-encoding, minus additional newline.
 attachment_length=$(( $(base64 $NOTMUCH_SRCDIR/test/README | wc -c) - 1 ))
-test_expect_equal "$output" "((((:id \"$id\" :match t :excluded nil :filename (\"$filename\") :timestamp 946728000 :date_relative \"2000-01-01\" :tags (\"inbox\") :body ((:id 1 :content-type \"multipart/mixed\" :content ((:id 2 :content-type \"text/plain\" :content \"This is a test message with inline attachment with a filename\") (:id 3 :content-type \"application/octet-stream\" :content-disposition \"inline\" :filename \"README\" :content-transfer-encoding \"base64\" :content-length $attachment_length)))) :crypto () :headers (:Subject \"sexp-show-inline-attachment-filename\" :From \"Notmuch Test Suite <test_suite@notmuchmail.org>\" :To \"test_suite@notmuchmail.org\" :Date \"Sat, 01 Jan 2000 12:00:00 +0000\")) ())))"
+test_expect_equal "$output" "((((:id \"$id\" :match t :excluded nil :filename (\"$filename\") :timestamp 946728000 :date_relative \"2000-01-01\" :tags (\"inbox\") :duplicate 1 :body ((:id 1 :content-type \"multipart/mixed\" :content ((:id 2 :content-type \"text/plain\" :content \"This is a test message with inline attachment with a filename\") (:id 3 :content-type \"application/octet-stream\" :content-disposition \"inline\" :filename \"README\" :content-transfer-encoding \"base64\" :content-length $attachment_length)))) :crypto () :headers (:Subject \"sexp-show-inline-attachment-filename\" :From \"Notmuch Test Suite <test_suite@notmuchmail.org>\" :To \"test_suite@notmuchmail.org\" :Date \"Sat, 01 Jan 2000 12:00:00 +0000\")) ())))"
 
 test_begin_subtest "show extra headers"
 add_message "[subject]=\"extra-headers\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[in-reply-to]=\"<parent@notmuch-test-suite>\"" "[body]=\"extra-headers test\""\

test/T520-show.sh

@@ -45,4 +45,40 @@ if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then
 
 fi
 
+add_email_corpus duplicate
+
+ID1=debian/2.6.1.dfsg-4-1-g87ea161@87ea161e851dfb1ea324af00e4ecfccc18875e15
+
+test_begin_subtest "format json, --duplicate=2, duplicate key"
+output=$(notmuch show --format=json --duplicate=2 id:${ID1})
+test_json_nodes <<<"$output" "dup:['duplicate']=2"
+
+test_begin_subtest "format json, subject, --duplicate=1"
+output=$(notmuch show --format=json --duplicate=1 id:${ID1})
+file=$(notmuch search --output=files id:${ID1} | head -n 1)
+subject=$(sed -n 's/^Subject: \(.*\)$/\1/p' < $file)
+test_json_nodes <<<"$output" "subject:['headers']['Subject']=\"$subject\""
+
+test_begin_subtest "format json, subject, --duplicate=2"
+output=$(notmuch show --format=json --duplicate=2 id:${ID1})
+file=$(notmuch search --output=files id:${ID1} | tail -n 1)
+subject=$(sed -n 's/^Subject: \(.*\)$/\1/p' < $file)
+test_json_nodes <<<"$output" "subject:['headers']['Subject']=\"$subject\""
+
+ID2=87r2geywh9.fsf@tethera.net
+for dup in {1..2}; do
+    test_begin_subtest "format json, body, --duplicate=${dup}"
+    output=$(notmuch show --format=json --duplicate=${dup} id:${ID2} | \
+	     $NOTMUCH_PYTHON -B "$NOTMUCH_SRCDIR"/test/json_check_nodes.py "body:['body'][0]['content']" | \
+	     grep '^# body')
+    test_expect_equal "$output" "# body ${dup}"
+done
+
+ID3=87r2ecrr6x.fsf@zephyr.silentflame.com
+for dup in {1..5}; do
+    test_begin_subtest "format json, --duplicate=${dup}, 'duplicate' key"
+    output=$(notmuch show --format=json --duplicate=${dup} id:${ID3})
+    test_json_nodes <<<"$output" "dup:['duplicate']=${dup}"
+done
+
 test_done

test/test-lib.sh

@@ -522,6 +522,7 @@ notmuch_json_show_sanitize () {
 	-e 's|"id": "[^"]*",|"id": "XXXXX",|g' \
 	-e 's|"Date": "Fri, 05 Jan 2001 [^"]*0000"|"Date": "GENERATED_DATE"|g' \
 	-e 's|"filename": "signature.asc",||g' \
+	-e 's|"duplicate": 1,||g' \
 	-e 's|"filename": \["/[^"]*"\],|"filename": \["YYYYY"\],|g' \
 	-e 's|"timestamp": 97.......|"timestamp": 42|g' \
 	-e 's|"content-length": [1-9][0-9]*|"content-length": "NONZERO"|g'
@@ -532,6 +533,7 @@ notmuch_sexp_show_sanitize () {
 	-e 's|:id "[^"]*"|:id "XXXXX"|g' \
 	-e 's|:Date "Sat, 01 Jan 2000 [^"]*0000"|:Date "GENERATED_DATE"|g' \
 	-e 's|:filename "signature.asc"||g' \
+	-e 's|:duplicate 1 ||g' \
 	-e 's|:filename ("/[^"]*")|:filename ("YYYYY")|g' \
 	-e 's|:timestamp 9........|:timestamp 42|g' \
 	-e 's|:content-length [1-9][0-9]*|:content-length "NONZERO"|g'