notmuch show: Properly nest MIME parts within mulipart parts

Previously, notmuch show flattened all output, losing information
about the nesting of the MIME hierarchy. Now, the output is properly
nested, (both in the --format=text and --format=json output), so that
clients can analyze the original MIME structure.

Internally, this required splitting the final closing delimiter out of
the various show_part functions and putting it into a new
show_part_end function instead. Also, the show_part function now
accepts a new "first" argument that is set not only for the first MIME
part of a message, but also for each first MIME part within a series
of multipart parts. This "first" argument controls the omission of a
preceding comma when printing a part (for json).

Many thanks to David Edmondson <dme@dme.org> for originally
identifying the lack of nesting in the json output and submitting an
early implementation of this feature. Thanks as well to Jameson Graef
Rollins <jrollins@finestructure.net> for carefully shepherding David's
patches through a remarkably long review process, patiently explaining
them, and providing a cleaned up series that led to this final
implementation. Jameson also provided the new emacs code here.

emacs/notmuch-show.el

@@ -280,6 +280,15 @@ current buffer, if possible."
 	      t)
 	  nil)))))
 
+(defun notmuch-show-insert-part-multipart/* (msg part content-type nth depth declared-type)
+  (let ((inner-parts (plist-get part :content)))
+    (notmuch-show-insert-part-header nth declared-type content-type nil)
+    ;; Show all of the parts.
+    (mapc (lambda (inner-part)
+	    (notmuch-show-insert-bodypart msg inner-part depth))
+	  inner-parts))
+  t)
+
 (defun notmuch-show-insert-part-text/plain (msg part content-type nth depth declared-type)
   (let ((start (point)))
     ;; If this text/plain part is not the first part in the message,

notmuch-client.h

@@ -133,7 +133,8 @@ query_string_from_args (void *ctx, int argc, char *argv[]);
 
 notmuch_status_t
 show_message_body (const char *filename,
-		   void (*show_part) (GMimeObject *part, int *part_count));
+		   void (*show_part) (GMimeObject *part, int *part_count, int first),
+		   void (*show_part_end) (GMimeObject *part));
 
 notmuch_status_t
 show_one_part (const char *filename, int part);

notmuch-reply.c

@@ -72,7 +72,7 @@ show_reply_headers (GMimeMessage *message)
 }
 
 static void
-reply_part (GMimeObject *part, int *part_count)
+reply_part (GMimeObject *part, int *part_count, unused (int first))
 {
     GMimeContentDisposition *disposition;
     GMimeContentType *content_type;
@@ -505,7 +505,8 @@ notmuch_reply_format_default(void *ctx, notmuch_config_t *config, notmuch_query_
 		notmuch_message_get_header (message, "date"),
 		notmuch_message_get_header (message, "from"));
 
-	show_message_body (notmuch_message_get_filename (message), reply_part);
+	show_message_body (notmuch_message_get_filename (message),
+			   reply_part, NULL);
 
 	notmuch_message_destroy (message);
     }

notmuch-show.c

@@ -32,7 +32,8 @@ typedef struct show_format {
     const char *header_end;
     const char *body_start;
     void (*part) (GMimeObject *part,
-		  int *part_count);
+		  int *part_count, int first);
+    void (*part_end) (GMimeObject *part);
     const char *body_end;
     const char *message_end;
     const char *message_set_sep;
@@ -46,14 +47,20 @@ format_message_text (unused (const void *ctx),
 static void
 format_headers_text (const void *ctx,
 		     notmuch_message_t *message);
+
 static void
 format_part_text (GMimeObject *part,
-		  int *part_count);
+		  int *part_count,
+		  int first);
+
+static void
+format_part_end_text (GMimeObject *part);
+
 static const show_format_t format_text = {
     "",
 	"\fmessage{ ", format_message_text,
 	    "\fheader{\n", format_headers_text, "\fheader}\n",
-	    "\fbody{\n", format_part_text, "\fbody}\n",
+	    "\fbody{\n", format_part_text, format_part_end_text, "\fbody}\n",
 	"\fmessage}\n", "",
     ""
 };
@@ -65,14 +72,20 @@ format_message_json (const void *ctx,
 static void
 format_headers_json (const void *ctx,
 		     notmuch_message_t *message);
+
 static void
 format_part_json (GMimeObject *part,
-		  int *part_count);
+		  int *part_count,
+		  int first);
+
+static void
+format_part_end_json (GMimeObject *part);
+
 static const show_format_t format_json = {
     "[",
 	"{", format_message_json,
 	    ", \"headers\": {", format_headers_json, "}",
-	    ", \"body\": [", format_part_json, "]",
+	    ", \"body\": [", format_part_json, format_part_end_json, "]",
 	"}", ", ",
     "]"
 };
@@ -86,7 +99,7 @@ static const show_format_t format_mbox = {
     "",
         "", format_message_mbox,
             "", NULL, "",
-            "", NULL, "",
+	    "", NULL, NULL, "",
         "", "",
     ""
 };
@@ -364,7 +377,7 @@ show_part_content (GMimeObject *part, GMimeStream *stream_out)
 }
 
 static void
-format_part_text (GMimeObject *part, int *part_count)
+format_part_text (GMimeObject *part, int *part_count, unused (int first))
 {
     GMimeContentDisposition *disposition;
     GMimeContentType *content_type;
@@ -391,8 +404,6 @@ format_part_text (GMimeObject *part, int *part_count)
 	    g_object_unref(stream_stdout);
 	}
 
-	printf ("\fattachment}\n");
-
 	return;
     }
 
@@ -420,12 +431,27 @@ format_part_text (GMimeObject *part, int *part_count)
 	printf ("Non-text part: %s\n",
 		g_mime_content_type_to_string (content_type));
     }
-
-    printf ("\fpart}\n");
 }
 
 static void
-format_part_json (GMimeObject *part, int *part_count)
+format_part_end_text (GMimeObject *part)
+{
+    GMimeContentDisposition *disposition;
+
+    disposition = g_mime_object_get_content_disposition (part);
+    if (disposition &&
+	strcmp (disposition->disposition, GMIME_DISPOSITION_ATTACHMENT) == 0)
+    {
+	printf ("\fattachment}\n");
+    }
+    else
+    {
+	printf ("\fpart}\n");
+    }
+}
+
+static void
+format_part_json (GMimeObject *part, int *part_count, int first)
 {
     GMimeContentType *content_type;
     GMimeContentDisposition *disposition;
@@ -435,7 +461,7 @@ format_part_json (GMimeObject *part, int *part_count)
 
     content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
 
-    if (*part_count > 1)
+    if (! first)
 	fputs (", ", stdout);
 
     printf ("{\"id\": %d, \"content-type\": %s",
@@ -459,14 +485,29 @@ format_part_json (GMimeObject *part, int *part_count)
 
 	printf (", \"content\": %s", json_quote_chararray (ctx, (char *) part_content->data, part_content->len));
     }
-
+    else if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-    fputs ("}", stdout);
+    {
+	printf (", \"content\": [");
+    }
 
     talloc_free (ctx);
     if (stream_memory)
 	g_object_unref (stream_memory);
 }
 
+static void
+format_part_end_json (GMimeObject *part)
+{
+    GMimeContentType *content_type;
+
+    content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
+
+    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
+	printf ("]");
+
+    printf ("}");
+}
+
 static void
 show_message (void *ctx, const show_format_t *format, notmuch_message_t *message, int indent)
 {
@@ -481,7 +522,8 @@ show_message (void *ctx, const show_format_t *format, notmuch_message_t *message
 
     fputs (format->body_start, stdout);
     if (format->part)
-	show_message_body (notmuch_message_get_filename (message), format->part);
+	show_message_body (notmuch_message_get_filename (message),
+			   format->part, format->part_end);
     fputs (format->body_end, stdout);
 
     fputs (format->message_end, stdout);

notmuch.1

@@ -260,7 +260,8 @@ decoded. Various components in the output,
 will be delimited by easily-parsed markers. Each marker consists of a
 Control-L character (ASCII decimal 12), the name of the marker, and
 then either an opening or closing brace, ('{' or '}'), to either open
-or close the component.
+or close the component. For a multipart MIME message, these parts will
+be nested.
 .RE
 .RS 4
 .TP 4
@@ -268,8 +269,9 @@ or close the component.
 
 The output is formatted with Javascript Object Notation (JSON). This
 format is more robust than the text format for automated
-processing. JSON output always includes all messages in a matching
+processing. The nested structure of multipart MIME messages is
-thread; in effect
+reflected in nested JSON output. JSON output always includes all
+messages in a matching thread; in effect
 .B \-\-format=json
 implies
 .B \-\-entire\-thread

notmuch.c

@@ -238,15 +238,17 @@ command_t commands[] = {
       "\t\teasily-parsed markers. Each marker consists of a Control-L\n"
       "\t\tcharacter (ASCII decimal 12), the name of the marker, and\n"
       "\t\tthen either an opening or closing brace, '{' or '}' to\n"
-      "\t\teither open or close the component.\n"
+      "\t\teither open or close the component. For a multipart MIME\n"
+      "\t\tmessage, these parts will be nested.\n"
       "\n"
       "\t\tjson\n"
       "\n"
       "\t\tThe output is formatted with Javascript Object Notation\n"
       "\t\t(JSON). This format is more robust than the text format\n"
-      "\t\tfor automated processing. JSON output always includes all\n"
+      "\t\tfor automated processing. The nested structure of multipart\n"
-      "\t\tmessages in a matching thread; in effect '--format=json'\n"
+      "\t\tMIME messages is reflected in nested JSON output. JSON\n"
-      "\t\timplies '--entire-thread'\n"
+      "\t\toutput always includes all messages in a matching thread;\n"
+      "\t\tin effect '--format=json' implies '--entire-thread'\n"
       "\n"
       "\t\tmbox\n"
       "\n"

show-message.c

@@ -23,20 +23,27 @@
 #include "notmuch-client.h"
 
 static void
-show_message_part (GMimeObject *part, int *part_count,
+show_message_part (GMimeObject *part,
-		   void (*show_part) (GMimeObject *part, int *part_count))
+		   int *part_count,
+		   void (*show_part) (GMimeObject *part, int *part_count, int first),
+		   void (*show_part_end) (GMimeObject *part),
+		   int first)
 {
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
 	int i;
 
 	*part_count = *part_count + 1;
-	(*show_part) (part, part_count);
+	(*show_part) (part, part_count, first);
 
 	for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
 	    show_message_part (g_mime_multipart_get_part (multipart, i),
-			       part_count, show_part);
+			       part_count, show_part, show_part_end, i == 0);
 	}
+
+	if (show_part_end)
+	    (*show_part_end) (part);
+
 	return;
     }
 
@@ -46,7 +53,7 @@ show_message_part (GMimeObject *part, int *part_count,
 	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
 
 	show_message_part (g_mime_message_get_mime_part (mime_message),
-			   part_count, show_part);
+			   part_count, show_part, show_part_end, first);
 
 	return;
     }
@@ -59,12 +66,15 @@ show_message_part (GMimeObject *part, int *part_count,
 
     *part_count = *part_count + 1;
 
-    (*show_part) (part, part_count);
+    (*show_part) (part, part_count, first);
+    if (show_part_end)
+	(*show_part_end) (part);
 }
 
 notmuch_status_t
 show_message_body (const char *filename,
-		   void (*show_part) (GMimeObject *part, int *part_count))
+		   void (*show_part) (GMimeObject *part, int *part_count, int first),
+		   void (*show_part_end) (GMimeObject *part))
 {
     GMimeStream *stream = NULL;
     GMimeParser *parser = NULL;
@@ -88,7 +98,7 @@ show_message_body (const char *filename,
     mime_message = g_mime_parser_construct_message (parser);
 
     show_message_part (g_mime_message_get_mime_part (mime_message),
-		       &part_count, show_part);
+		       &part_count, show_part, show_part_end, TRUE);
 
   DONE:
     if (mime_message)

test/multipart

@@ -59,9 +59,7 @@ Date: Tue, 05 Jan 2001 15:43:57 -0000
 header}
 body{
 part{ ID: 1, Content-type: multipart/signed
-part}
 part{ ID: 2, Content-type: multipart/mixed
-part}
 part{ ID: 3, Content-type: text/plain
 This is an inline text part.
 part}
@@ -74,15 +72,17 @@ And this message is signed.
 
 -Carl
 part}
+part}
 part{ ID: 6, Content-type: application/pgp-signature
 Non-text part: application/pgp-signature
 part}
+part}
 body}
 message}"
 
 test_begin_subtest "Show multipart MIME message (--format=json)"
 output=$(notmuch show --format=json 'id:87liy5ap00.fsf@yoom.home.cworth.org')
-test_expect_equal "$output" '[[[{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "/home/cworth/src/notmuch/test/tmp.multipart/mail/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Tue, 05 Jan 2001 15:43:57 -0000"}, "body": [{"id": 1, "content-type": "multipart/signed"}, {"id": 2, "content-type": "multipart/mixed"}, {"id": 3, "content-type": "text/plain", "content": "This is an inline text part.\n"}, {"id": 4, "content-type": "text/plain", "filename": "attachment", "content": "This is a text attachment.\n"}, {"id": 5, "content-type": "text/plain", "content": "And this message is signed.\n\n-Carl\n"}, {"id": 6, "content-type": "application/pgp-signature"}]}, []]]]'
+test_expect_equal "$output" '[[[{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "/home/cworth/src/notmuch/test/tmp.multipart/mail/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Tue, 05 Jan 2001 15:43:57 -0000"}, "body": [{"id": 1, "content-type": "multipart/signed", "content": [{"id": 2, "content-type": "multipart/mixed", "content": [{"id": 3, "content-type": "text/plain", "content": "This is an inline text part.\n"}, {"id": 4, "content-type": "text/plain", "filename": "attachment", "content": "This is a text attachment.\n"}, {"id": 5, "content-type": "text/plain", "content": "And this message is signed.\n\n-Carl\n"}]}, {"id": 6, "content-type": "application/pgp-signature"}]}]}, []]]]'
 
 test_done