Implement 'notmuch dump'.

This is a fairly big milestone for notmuch. It's our first command
to do anything besides building the index, so it proves we can
actually read valid results out from the index.

It also puts in place almost all of the API and infrastructure we
will need to allow searching of the database.

Finally, with this change we are now using talloc inside of notmuch
which is truly a delight to use. And now that I figured out how
to use C++ objects with talloc allocation, (it requires grotty
parts of C++ such as "placement new" and "explicit destructors"),
we are valgrind-clean for "notmuch dump", (as in "no leaks are
possible").

Makefile

@@ -1,9 +1,9 @@
 PROGS=notmuch
 
-MYCFLAGS=-Wall -O0 -g `pkg-config --cflags glib-2.0`
+MYCFLAGS=-Wall -O0 -g `pkg-config --cflags glib-2.0 talloc`
 MYCXXFLAGS=$(MYCFLAGS) `xapian-config --cxxflags`
 
-MYLDFLAGS=`pkg-config --libs glib-2.0` `xapian-config --libs`
+MYLDFLAGS=`pkg-config --libs glib-2.0 talloc` `xapian-config --libs`
 
 all: $(PROGS)
 
@@ -13,7 +13,7 @@ all: $(PROGS)
 %.o: %.c
 	$(CC) -c $(CFLAGS) $(MYCFLAGS) $< -o $@
 
-notmuch: notmuch.o database.o date.o message-file.o xutil.o
+notmuch: notmuch.o database.o date.o message.o message-file.o query.o xutil.o
 	$(CC) $(MYLDFLAGS) $^ -o $@
 
 Makefile.dep: *.c *.cc

database-private.h

@@ -0,0 +1,34 @@
+/* database-private.h - For peeking into the internals of notmuch_database_t
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_DATABASE_PRIVATE_H
+#define NOTMUCH_DATABASE_PRIVATE_H
+
+#include "notmuch-private.h"
+
+#include <xapian.h>
+
+struct _notmuch_database {
+    char *path;
+    Xapian::WritableDatabase *xapian_db;
+    Xapian::TermGenerator *term_gen;
+};
+
+#endif

database.cc

@@ -18,7 +18,7 @@
  * Author: Carl Worth <cworth@cworth.org>
  */
 
-#include "notmuch-private.h"
+#include "database-private.h"
 
 #include <iostream>
 
@@ -28,12 +28,6 @@
 
 using namespace std;
 
-struct _notmuch_database {
-    char *path;
-    Xapian::WritableDatabase *xapian_db;
-    Xapian::TermGenerator *term_gen;
-};
-
 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
 
 /* Xapian complains if we provide a term longer than this. */
@@ -463,6 +457,7 @@ notmuch_database_open (const char *path)
     notmuch->path = xstrdup (path);
 
     try {
+	Xapian::PostingIterator i;
 	notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
 							   Xapian::DB_CREATE_OR_OPEN);
     } catch (const Xapian::Error &error) {

message.cc

@@ -0,0 +1,131 @@
+/* message.cc - Results of message-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <xapian.h>
+
+struct _notmuch_message {
+    Xapian::Document doc;
+};
+
+struct _notmuch_tags {
+    Xapian::TermIterator iterator;
+    Xapian::TermIterator iterator_end;
+};
+
+static int
+_notmuch_message_destroy (notmuch_message_t *message)
+{
+    message->doc.~Document ();
+
+    return 0;
+}
+
+notmuch_message_t *
+_notmuch_message_create (notmuch_results_t *owner,
+			 notmuch_database_t *notmuch,
+			 Xapian::docid doc_id)
+{
+    notmuch_message_t *message;
+
+    message = talloc (owner, notmuch_message_t);
+    if (unlikely (message == NULL))
+	return NULL;
+
+    new (&message->doc) Xapian::Document;
+
+    talloc_set_destructor (message, _notmuch_message_destroy);
+
+    message->doc = notmuch->xapian_db->get_document (doc_id);
+
+    return message;
+}
+
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message)
+{
+    Xapian::TermIterator i;
+
+    i = message->doc.termlist_begin ();
+    i.skip_to ("Q");
+    if (i != message->doc.termlist_end ())
+	return talloc_strdup (message, (*i).c_str () + 1);
+    else
+	return NULL;
+}
+
+static int
+_notmuch_tags_destroy (notmuch_tags_t *tags)
+{
+    tags->iterator.~TermIterator ();
+    tags->iterator_end.~TermIterator ();
+
+    return 0;
+}
+
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message)
+{
+    notmuch_tags_t *tags;
+
+    tags = talloc (message, notmuch_tags_t);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    new (&tags->iterator) Xapian::TermIterator;
+    new (&tags->iterator_end) Xapian::TermIterator;
+
+    talloc_set_destructor (tags, _notmuch_tags_destroy);
+
+    tags->iterator = message->doc.termlist_begin ();
+    tags->iterator.skip_to ("L");
+    tags->iterator_end = message->doc.termlist_end ();
+
+    return tags;
+}
+
+notmuch_bool_t
+notmuch_tags_has_more (notmuch_tags_t *tags)
+{
+    std::string s;
+
+    if (tags->iterator == tags->iterator_end)
+	return FALSE;
+
+    s = *tags->iterator;
+    if (s.size () && s[0] == 'L')
+	return TRUE;
+    else
+	return FALSE;
+}
+
+const char *
+notmuch_tags_get (notmuch_tags_t *tags)
+{
+    return talloc_strdup (tags, (*tags->iterator).c_str () + 1);
+}
+
+void
+notmuch_tags_advance (notmuch_tags_t *tags)
+{
+    tags->iterator++;
+}

notmuch-private.h

@@ -41,6 +41,31 @@
 
 NOTMUCH_BEGIN_DECLS
 
+#include <talloc.h>
+
+
+/* Thanks to Andrew Tridgell's (SAMBA's) talloc for this definition of
+ * unlikely. The talloc source code comes to us via the GNU LGPL v. 3.
+ */
+/* these macros gain us a few percent of speed on gcc */
+#if (__GNUC__ >= 3)
+/* the strange !! is to ensure that __builtin_expect() takes either 0 or 1
+   as its first argument */
+#ifndef likely
+#define likely(x)   __builtin_expect(!!(x), 1)
+#endif
+#ifndef unlikely
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+#else
+#ifndef likely
+#define likely(x) (x)
+#endif
+#ifndef unlikely
+#define unlikely(x) (x)
+#endif
+#endif
+
 /* xutil.c */
 void *
 xcalloc (size_t nmemb, size_t size);
@@ -57,6 +82,13 @@ xstrdup (const char *s);
 char *
 xstrndup (const char *s, size_t n);
 
+/* message.cc */
+
+notmuch_message_t *
+_notmuch_message_create (notmuch_results_t *owner,
+			 notmuch_database_t *notmuch,
+			 unsigned int doc_id);
+
 /* message-file.c */
 
 /* XXX: I haven't decided yet whether these will actually get exported

notmuch.c

@@ -35,6 +35,8 @@
 #include <dirent.h>
 #include <errno.h>
 
+#include <talloc.h>
+
 #include <glib.h> /* g_strdup_printf */
 
 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
@@ -370,8 +372,76 @@ show_command (int argc, char *argv[])
 int
 dump_command (int argc, char *argv[])
 {
-    fprintf (stderr, "Error: dump is not implemented yet.\n");
+    FILE *output;
-    return 1;
+    notmuch_database_t *notmuch = NULL;
+    notmuch_query_t *query;
+    notmuch_results_t *results;
+    notmuch_message_t *message;
+    notmuch_tags_t *tags;
+    int ret = 0;
+
+    if (argc) {
+	output = fopen (argv[0], "w");
+	if (output == NULL) {
+	    fprintf (stderr, "Error opening %s for writing: %s\n",
+		     argv[1], strerror (errno));
+	    ret = 1;
+	    goto DONE;
+	}
+    } else {
+	output = stdout;
+    }
+
+    notmuch = notmuch_database_open (NULL);
+    if (notmuch == NULL) {
+	ret = 1;
+	goto DONE;
+    }
+
+    query = notmuch_query_create (notmuch, NOTMUCH_QUERY_ALL);
+    if (query == NULL) {
+	fprintf (stderr, "Out of memory\n");
+	ret = 1;
+	goto DONE;
+    }
+
+    notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
+
+    for (results = notmuch_query_search (query);
+	 notmuch_results_has_more (results);
+	 notmuch_results_advance (results))
+    {
+	message = notmuch_results_get (results);
+
+	fprintf (output,
+		 "%s (", notmuch_message_get_message_id (message));
+
+	for (tags = notmuch_message_get_tags (message);
+	     notmuch_tags_has_more (tags);
+	     notmuch_tags_advance (tags))
+	{
+	    int first = 1;
+
+	    if (! first)
+		fprintf (output, " ");
+
+	    fprintf (output, "%s", notmuch_tags_get (tags));
+
+	    first = 0;
+	}
+
+	fprintf (output, ")\n");
+    }
+
+    notmuch_query_destroy (query);
+
+  DONE:
+    if (notmuch)
+	notmuch_database_close (notmuch);
+    if (output != stdout)
+	fclose (output);
+
+    return ret;
 }
 
 int

notmuch.h

@@ -31,6 +31,16 @@
 
 NOTMUCH_BEGIN_DECLS
 
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+
+typedef int notmuch_bool_t;
+
 /* Status codes used for the return values of most functions.
  *
  * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function
@@ -50,10 +60,13 @@ typedef enum _notmuch_status {
     NOTMUCH_STATUS_FILE_NOT_EMAIL
 } notmuch_status_t;
 
-/* An opaque data structure representing a notmuch database. See
+/* Various opaque data types. For each notmuch_<foo>_t see the various
- * notmuch_database_open and other notmuch_database functions
+ * notmuch_<foo> functions below. */
- * below. */
 typedef struct _notmuch_database notmuch_database_t;
+typedef struct _notmuch_query notmuch_query_t;
+typedef struct _notmuch_results notmuch_results_t;
+typedef struct _notmuch_message notmuch_message_t;
+typedef struct _notmuch_tags notmuch_tags_t;
 
 /* Create a new, empty notmuch database located at 'path'.
  *
@@ -150,6 +163,185 @@ notmuch_status_t
 notmuch_database_add_message (notmuch_database_t *database,
 			      const char *filename);
 
+/* Create a new query for 'database'.
+ *
+ * Here, 'database' should be an open database, (see
+ * notmuch_database_open and notmuch_database_create).
+ *
+ * For the query string, we'll document the syntax here more
+ * completely in the future, but it's likely to be a specialized
+ * version of the general Xapian query syntax:
+ *
+ * http://xapian.org/docs/queryparser.html
+ *
+ * As a special case, passing a value of NOTMUCH_QUERY_ALL for the
+ * query string will result in a query that returns all messages in
+ * the database.
+ *
+ * See notmuch_query_set_sort for controlling the order of results and
+ * notmuch_query_search to actually execute the query.
+ *
+ * User should call notmuch_query_destroy when finished with this
+ * query.
+ *
+ * Will return NULL if insufficient memory is available.
+ */
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *database,
+		      const char *query_string);
+
+/* Special value to cause notmuch_query_create to return all
+ * messages. */
+extern const char *NOTMUCH_QUERY_ALL;
+
+/* Sort values for notmuch_query_set_sort */
+typedef enum {
+    NOTMUCH_SORT_DATE_OLDEST_FIRST,
+    NOTMUCH_SORT_DATE_NEWEST_FIRST,
+    NOTMUCH_SORT_MESSAGE_ID
+} notmuch_sort_t;
+
+/* Specify the sorting desired for this query. */
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort);
+
+/* Execute a query, returning a notmuch_results_t object which can be
+ * used to iterate over the results. The results object is owned by
+ * the query and as such, will only be valid until notmuch_query_destroy.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_query_t *query;
+ *     notmuch_results_t *results;
+ *
+ *     query = notmuch_query_create (database, query_string);
+ *
+ *     for (results = notmuch_query_search (query);
+ *          notmuch_results_has_more (results);
+ *          notmuch_result_advance (results))
+ *     {
+ *         message = notmuch_results_get (results);
+ *         ....
+ *     }
+ *
+ *     notmuch_query_destroy (query);
+ *
+ * Note that there's no explicit destructor for the notmuch_results_t
+ * object.
+ */
+notmuch_results_t *
+notmuch_query_search (notmuch_query_t *query);
+
+/* Destroy a notmuch_query_t along with any associated resources.
+ *
+ * This will in turn destroy any notmuch_results_t objects generated
+ * by this query, (and in turn any notmuch_message_t objects generated
+ * from those results, etc.).
+ */
+void
+notmuch_query_destroy (notmuch_query_t *query);
+
+/* Does the given notmuch_results_t object contain any more results.
+ *
+ * When this function returns TRUE, notmuch_results_get will return a
+ * valid object. Whereas when this function returns FALSE,
+ * notmuch_results_get will return NULL.
+ *
+ * See the documentation of notmuch_query_search for example code
+ * showing how to iterate over a notmuch_results_t object.
+ */
+notmuch_bool_t
+notmuch_results_has_more (notmuch_results_t *results);
+
+/* Get the current result from 'results' as a notmuch_message_t.
+ *
+ * Note: The returned message belongs to 'results' and has a lifetime
+ * identical to it (and the query to which it belongs).
+ *
+ * See the documentation of notmuch_query_search for example code
+ * showing how to iterate over a notmuch_results_t object.
+ */
+notmuch_message_t *
+notmuch_results_get (notmuch_results_t *results);
+
+/* Advance the 'results' iterator to the next result.
+ *
+ * See the documentation of notmuch_query_search for example code
+ * showing how to iterate over a notmuch_results_t object.
+ */
+void
+notmuch_results_advance (notmuch_results_t *results);
+
+/* Get the message ID of 'message'.
+ *
+ * The returned string belongs to 'message' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * message is valid, (which is until the query from which it derived
+ * is destroyed).
+ */
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message);
+
+/* Get the tags for 'message', returning a notmuch_tags_t object which
+ * can be used to iterate over all tags.
+ *
+ * The tags object is owned by the message and as such, will only be
+ * valid for as long as the message is valid, (which is until the
+ * query from which it derived is destroyed).
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_t *message;
+ *     notmuch_tags_t *tags;
+ *     const char *tag;
+ *
+ *     message = notmuch_results_get (results);
+ *
+ *     for (tags = notmuch_message_get_tags (message);
+ *          notmuch_tags_has_more (tags);
+ *          notmuch_result_advance (tags))
+ *     {
+ *         tag = notmuch_tags_get_string (tags);
+ *         ....
+ *     }
+ *
+ * Note that there's no explicit destructor for the notmuch_tags_t
+ * object.
+ */
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message);
+
+/* Does the given notmuch_tags_t object contain any more results.
+ *
+ * When this function returns TRUE, notmuch_tags_get will return a
+ * valid string. Whereas when this function returns FALSE,
+ * notmuch_tags_get will return NULL.
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+notmuch_bool_t
+notmuch_tags_has_more (notmuch_tags_t *tags);
+
+/* Get the current result from 'tags' as a string.
+ *
+ * Note: The returned string belongs to 'tags' and has a lifetime
+ * identical to it (and the query to which it utlimately belongs).
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+const char *
+notmuch_tags_get (notmuch_tags_t *tags);
+
+/* Advance the 'tags' iterator to the next tag.
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+void
+notmuch_tags_advance (notmuch_tags_t *results);
+
 NOTMUCH_END_DECLS
 
 #endif

query.cc

@@ -0,0 +1,137 @@
+/* query.cc - Support for searching a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <xapian.h>
+
+const char *NOTMUCH_QUERY_ALL = "";
+
+struct _notmuch_query {
+    notmuch_database_t *notmuch;
+    const char *query_string;
+    notmuch_sort_t sort;
+};
+
+struct _notmuch_results {
+    notmuch_database_t *notmuch;
+    Xapian::PostingIterator iterator;
+    Xapian::PostingIterator iterator_end;
+};
+
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *notmuch,
+		      const char *query_string)
+{
+    notmuch_query_t *query;
+
+    query = talloc (NULL, notmuch_query_t);
+    if (unlikely (query == NULL))
+	return NULL;
+
+    query->notmuch = notmuch;
+
+    /* Special-case NOTMUCH_QUERY_ALL so we see it and not a copy. */
+    if (query_string == NOTMUCH_QUERY_ALL)
+	query->query_string = query_string;
+    else
+	query->query_string = talloc_strdup (query, query_string);
+
+    query->sort = NOTMUCH_SORT_DATE_OLDEST_FIRST;
+
+    return query;
+}
+
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort)
+{
+    query->sort = sort;
+}
+
+static int
+_notmuch_results_destroy (notmuch_results_t *results)
+{
+    results->iterator.~PostingIterator ();
+    results->iterator_end.~PostingIterator ();
+
+    return 0;
+}
+
+notmuch_results_t *
+notmuch_query_search (notmuch_query_t *query)
+{
+    notmuch_results_t *results;
+
+    results = talloc (query, notmuch_results_t);
+    if (unlikely (results == NULL))
+	return NULL;
+
+    try {
+	if (query->query_string != NOTMUCH_QUERY_ALL) {
+	    fprintf (stderr, "Error: Arbitrary search strings are not supported yet. Come back soon!\n");
+	    exit (1);
+	}
+
+	results->notmuch = query->notmuch;
+	new (&results->iterator) Xapian::PostingIterator ();
+	new (&results->iterator_end) Xapian::PostingIterator ();
+
+	talloc_set_destructor (results, _notmuch_results_destroy);
+
+	results->iterator = query->notmuch->xapian_db->postlist_begin ("");
+	results->iterator_end = query->notmuch->xapian_db->postlist_end ("");
+
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred: %s\n",
+		 error.get_msg().c_str());
+    }
+
+    return results;
+}
+
+void
+notmuch_query_destroy (notmuch_query_t *query)
+{
+    talloc_free (query);
+}
+
+notmuch_bool_t
+notmuch_results_has_more (notmuch_results_t *results)
+{
+    return (results->iterator != results->iterator_end);
+}
+
+notmuch_message_t *
+notmuch_results_get (notmuch_results_t *results)
+{
+    Xapian::docid doc_id;
+
+    doc_id = *results->iterator;
+
+    return _notmuch_message_create (results,
+				    results->notmuch, doc_id);
+}
+
+void
+notmuch_results_advance (notmuch_results_t *results)
+{
+    results->iterator++;
+}