notmuch: Start actually adding messages to the index.

This is the beginning of the notmuch library as well, with its
interface in notmuch.h. So far we've got create, open, close, and
add_message (all with a notmuch_database prefix).

The current add_message function has already been whittled down from
what we have in notmuch-index-message to add only references,
message-id, and thread-id to the index, (that is---just enough to do
thread-linkage but nothing for full-text searching).

The concept here is to do something quickly so that the user can get
some data into notmuch and start using it. (The most interesting stuff
is then thread-linkage and labels like inbox and unread.)  We can
defer the full-text indexing of the body of the messages for later,
(such as in the background while the user is reading mail).

The initial thread-stitching step is still slower than I would like.
We may have to stop using libgmime for this step as its overhead is
not worth it for the simple case of just parsing the message-id,
references, and in-reply-to headers.

.gitignore

@@ -1,3 +1,4 @@
 xapian-dump
 notmuch-index-message
+notmuch
 

Makefile

@@ -1,12 +1,20 @@
 PROGS=notmuch notmuch-index-message xapian-dump
 
-MYCFLAGS=-Wall -O0 -g
-MYCXXFLAGS=-Wall -O0 -g
+MYCFLAGS=-Wall -O0 -g `pkg-config --cflags gmime-2.4`
+MYCXXFLAGS=$(MYCFLAGS) `xapian-config --cxxflags`
+
+MYLDFLAGS=`pkg-config --libs gmime-2.4` `xapian-config --libs`
 
 all: $(PROGS)
 
-notmuch: notmuch.c
-	$(CC) $(CFLAGS) $(MYCFLAGS) notmuch.c `pkg-config --cflags --libs glib-2.0` -o notmuch
+%.o: %.cc
+	$(CXX) -c $(CXXFLAGS) $(MYCXXFLAGS) $^ -o $@
+
+%.o: %.c
+	$(CC) -c $(CFLAGS) $(MYCFLAGS) $^ -o $@
+
+notmuch: notmuch.o database.o xutil.o
+	$(CC) $(MYLDFLAGS) $^ -o $@
 
 notmuch-index-message: notmuch-index-message.cc
 	$(CXX) $(CXXFLAGS) $(MYCXXFLAGS) notmuch-index-message.cc `pkg-config --cflags --libs gmime-2.4` `xapian-config --cxxflags --libs` -o notmuch-index-message
@@ -15,4 +23,4 @@ xapian-dump: xapian-dump.cc
 	$(CXX) $(CXXFLAGS) $(MYCXXFLAGS) xapian-dump.cc `xapian-config --libs --cxxflags` -o xapian-dump
 
 clean:
-	rm -f $(PROGS)
+	rm -f $(PROGS) *.o

database.cc

@@ -0,0 +1,496 @@
+/* database.cc - The database interfaces of the notmuch mail library
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <stdio.h>
+#include <errno.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include <iostream>
+
+#include <gmime/gmime.h>
+
+#include <xapian.h>
+
+using namespace std;
+
+struct _notmuch_database {
+    char *path;
+    Xapian::WritableDatabase *xapian_db;
+    Xapian::TermGenerator *term_gen;
+};
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
+/* Xapian complains if we provide a term longer than this. */
+#define NOTMUCH_MAX_TERM 245
+
+/* These prefix values are specifically chosen to be compatible
+ * with sup, (http://sup.rubyforge.org), written by
+ * William Morgan <wmorgan-sup@masanjin.net>, and released
+ * under the GNU GPL v2.
+ */
+
+typedef struct {
+    const char *name;
+    const char *prefix;
+} prefix_t;
+
+prefix_t NORMAL_PREFIX[] = {
+    { "subject", "S" },
+    { "body", "B" },
+    { "from_name", "FN" },
+    { "to_name", "TN" },
+    { "name", "N" },
+    { "attachment", "A" }
+};
+
+prefix_t BOOLEAN_PREFIX[] = {
+    { "type", "K" },
+    { "from_email", "FE" },
+    { "to_email", "TE" },
+    { "email", "E" },
+    { "date", "D" },
+    { "label", "L" },
+    { "source_id", "I" },
+    { "attachment_extension", "O" },
+    { "msgid", "Q" },
+    { "thread", "H" },
+    { "ref", "R" }
+};
+
+/* Similarly, these value numbers are also chosen to be sup
+ * compatible. */
+
+typedef enum {
+    NOTMUCH_VALUE_MESSAGE_ID = 0,
+    NOTMUCH_VALUE_THREAD = 1,
+    NOTMUCH_VALUE_DATE = 2
+} notmuch_value_t;
+
+static const char *
+find_prefix (const char *name)
+{
+    unsigned int i;
+
+    for (i = 0; i < ARRAY_SIZE (NORMAL_PREFIX); i++)
+	if (strcmp (name, NORMAL_PREFIX[i].name) == 0)
+	    return NORMAL_PREFIX[i].prefix;
+
+    for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
+	if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
+	    return BOOLEAN_PREFIX[i].prefix;
+
+    return "";
+}
+
+/* "128 bits of thread-id ought to be enough for anybody" */
+#define NOTMUCH_THREAD_ID_BITS	 128
+#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4)
+typedef struct _thread_id {
+    char str[NOTMUCH_THREAD_ID_DIGITS + 1];
+} thread_id_t;
+
+static void
+thread_id_generate (thread_id_t *thread_id)
+{
+    static int seeded = 0;
+    FILE *dev_random;
+    uint32_t value;
+    char *s;
+    int i;
+
+    if (! seeded) {
+	dev_random = fopen ("/dev/random", "r");
+	if (dev_random == NULL) {
+	    srand (time (NULL));
+	} else {
+	    fread ((void *) &value, sizeof (value), 1, dev_random);
+	    srand (value);
+	    fclose (dev_random);
+	}
+	seeded = 1;
+    }
+
+    s = thread_id->str;
+    for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) {
+	value = rand ();
+	sprintf (s, "%08x", value);
+	s += 8;
+    }
+}
+
+static void
+add_term (Xapian::Document doc,
+	  const char *prefix_name,
+	  const char *value)
+{
+    const char *prefix;
+    char *term;
+
+    if (value == NULL)
+	return;
+
+    prefix = find_prefix (prefix_name);
+
+    term = g_strdup_printf ("%s%s", prefix, value);
+
+    if (strlen (term) <= NOTMUCH_MAX_TERM)
+	doc.add_term (term);
+
+    g_free (term);
+}
+
+static void
+find_messages_by_term (Xapian::Database *db,
+		       const char *prefix_name,
+		       const char *value,
+		       Xapian::PostingIterator *begin,
+		       Xapian::PostingIterator *end)
+{
+    Xapian::PostingIterator i;
+    char *term;
+
+    term = g_strdup_printf ("%s%s", find_prefix (prefix_name), value);
+
+    *begin = db->postlist_begin (term);
+
+    if (end)
+	*end = db->postlist_end (term);
+
+    free (term);
+}
+
+Xapian::Document
+find_message_by_docid (Xapian::Database *db, Xapian::docid docid)
+{
+    return db->get_document (docid);
+}
+
+Xapian::Document
+find_message_by_message_id (Xapian::Database *db, const char *message_id)
+{
+    Xapian::PostingIterator i, end;
+
+    find_messages_by_term (db, "msgid", message_id, &i, &end);
+
+    if (i != end)
+	return find_message_by_docid (db, *i);
+    else
+	return Xapian::Document ();
+}
+
+static void
+insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
+{
+    string value_string;
+    const char *value, *id, *comma;
+
+    value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
+    value = value_string.c_str();
+    if (strlen (value)) {
+	id = value;
+	while (*id) {
+	    comma = strchr (id, ',');
+	    if (comma == NULL)
+		comma = id + strlen (id);
+	    g_hash_table_insert (thread_ids,
+				 strndup (id, comma - id), NULL);
+	    id = comma;
+	    if (*id)
+		id++;
+	}
+    }
+}
+
+/* Return one or more thread_ids, (as a GPtrArray of strings), for the
+ * given message based on looking into the database for any messages
+ * referenced in parents, and also for any messages in the database
+ * referencing message_id.
+ *
+ * Caller should free all strings in the array and the array itself,
+ * (g_ptr_array_free) when done. */
+static GPtrArray *
+find_thread_ids (Xapian::Database *db,
+		 GPtrArray *parents,
+		 const char *message_id)
+{
+    Xapian::PostingIterator child, children_end;
+    Xapian::Document doc;
+    GHashTable *thread_ids;
+    GList *keys, *l;
+    unsigned int i;
+    const char *parent_message_id;
+    GPtrArray *result;
+
+    thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
+					free, NULL);
+
+    find_messages_by_term (db, "ref", message_id, &child, &children_end);
+    for ( ; child != children_end; child++) {
+	doc = find_message_by_docid (db, *child);
+	insert_thread_id (thread_ids, doc);
+    }
+
+    for (i = 0; i < parents->len; i++) {
+	parent_message_id = (char *) g_ptr_array_index (parents, i);
+	doc = find_message_by_message_id (db, parent_message_id);
+	insert_thread_id (thread_ids, doc);
+    }
+
+    result = g_ptr_array_new ();
+
+    keys = g_hash_table_get_keys (thread_ids);
+    for (l = keys; l; l = l->next) {
+	char *id = (char *) l->data;
+	g_ptr_array_add (result, id);
+    }
+    g_list_free (keys);
+
+    /* We're done with the hash table, but we've taken the pointers to
+     * the allocated strings and put them into our result array, so
+     * tell the hash not to free them on its way out. */
+    g_hash_table_steal_all (thread_ids);
+    g_hash_table_unref (thread_ids);
+
+    return result;
+}
+
+/* Add a term for each message-id in the References header of the
+ * message. */
+static void
+parse_references (GPtrArray *array,
+		  const char *refs_str)
+{
+    GMimeReferences *refs, *r;
+    const char *message_id;
+
+    if (refs_str == NULL)
+	return;
+
+    refs = g_mime_references_decode (refs_str);
+
+    for (r = refs; r; r = r->next) {
+	message_id = g_mime_references_get_message_id (r);
+	g_ptr_array_add (array, g_strdup (message_id));
+    }
+
+    g_mime_references_free (refs);
+}
+
+notmuch_database_t *
+notmuch_database_create (const char *path)
+{
+    char *notmuch_path;
+    struct stat st;
+    int err;
+
+    err = stat (path, &st);
+    if (err) {
+	fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
+		 path, strerror (errno));
+	return NULL;
+    }
+
+    if (! S_ISDIR (st.st_mode)) {
+	fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
+		 path);
+	return NULL;
+    }
+
+    notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
+
+    err = mkdir (notmuch_path, 0755);
+
+    if (err) {
+	fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
+		 notmuch_path, strerror (errno));
+	free (notmuch_path);
+	return NULL;
+    }
+
+    free (notmuch_path);
+
+    return notmuch_database_open (path);
+}
+
+notmuch_database_t *
+notmuch_database_open (const char *path)
+{
+    notmuch_database_t *notmuch;
+    char *notmuch_path, *xapian_path;
+    struct stat st;
+    int err;
+
+    g_mime_init (0);
+
+    notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
+
+    err = stat (notmuch_path, &st);
+    if (err) {
+	fprintf (stderr, "Error: Cannot stat %s: %s\n",
+		 notmuch_path, strerror (err));
+	free (notmuch_path);
+	return NULL;
+    }
+
+    xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
+    free (notmuch_path);
+
+    /* C++ is so nasty in requiring these casts. I'm almost tempted to
+     * write a C wrapper for Xapian... */
+    notmuch = (notmuch_database_t *) xmalloc (sizeof (notmuch_database_t));
+    notmuch->path = xstrdup (path);
+
+    try {
+	notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
+							   Xapian::DB_CREATE_OR_OPEN);
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred: %s\n",
+		 error.get_msg().c_str());
+    }
+    
+    free (xapian_path);
+
+    return notmuch;
+}
+
+void
+notmuch_database_close (notmuch_database_t *notmuch)
+{
+    delete notmuch->xapian_db;
+    free (notmuch->path);
+    free (notmuch);
+}
+
+const char *
+notmuch_database_get_path (notmuch_database_t *notmuch)
+{
+    return notmuch->path;
+}
+
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *notmuch,
+			      const char *filename)
+{
+    Xapian::WritableDatabase *db = notmuch->xapian_db;
+    Xapian::Document doc;
+
+    GMimeStream *stream;
+    GMimeParser *parser;
+    GMimeMessage *message;
+    GPtrArray *parents, *thread_ids;
+
+    FILE *file;
+
+    const char *refs, *in_reply_to;
+    const char *message_id;
+
+    time_t time;
+    unsigned int i;
+
+    file = fopen (filename, "r");
+    if (! file) {
+	fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno));
+	exit (1);
+    }
+
+    stream = g_mime_stream_file_new (file);
+
+    parser = g_mime_parser_new_with_stream (stream);
+
+    message = g_mime_parser_construct_message (parser);
+
+    try {
+	doc = Xapian::Document ();
+
+	doc.set_data (filename);
+
+	parents = g_ptr_array_new ();
+
+	refs = g_mime_object_get_header (GMIME_OBJECT (message), "references");
+	parse_references (parents, refs);
+
+	in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message),
+						"in-reply-to");
+	parse_references (parents, in_reply_to);
+	for (i = 0; i < parents->len; i++)
+	    add_term (doc, "ref", (char *) g_ptr_array_index (parents, i));
+
+	message_id = g_mime_message_get_message_id (message);
+
+	thread_ids = find_thread_ids (db, parents, message_id);
+
+	for (i = 0; i < parents->len; i++)
+	    g_free (g_ptr_array_index (parents, i));
+	g_ptr_array_free (parents, TRUE);
+	if (message_id) {
+	    add_term (doc, "msgid", message_id);
+	    doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
+	}
+
+	if (thread_ids->len) {
+	    unsigned int i;
+	    GString *thread_id;
+	    char *id;
+
+	    for (i = 0; i < thread_ids->len; i++) {
+		id = (char *) thread_ids->pdata[i];
+		add_term (doc, "thread", id);
+		if (i == 0)
+		    thread_id = g_string_new (id);
+		else
+		    g_string_append_printf (thread_id, ",%s", id);
+
+		free (id);
+	    }
+	    g_ptr_array_free (thread_ids, TRUE);
+	    doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str);
+	    g_string_free (thread_id, TRUE);
+	} else if (message_id) {
+	    /* If not part of any existing thread, generate a new thread_id. */
+	    thread_id_t thread_id;
+
+	    thread_id_generate (&thread_id);
+	    add_term (doc, "thread", thread_id.str);
+	    doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str);
+	}
+
+	g_mime_message_get_date (message, &time, NULL);
+	doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time));
+
+	db->add_document (doc);
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred: %s.\n",
+		 error.get_msg().c_str());
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    g_object_unref (message);
+    g_object_unref (parser);
+    g_object_unref (stream);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}

notmuch-private.h

@@ -0,0 +1,49 @@
+/* notmuch-private.h - Internal interfaces for notmuch.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_PRIVATE_H
+#define NOTMUCH_PRIVATE_H
+
+#include "notmuch.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+NOTMUCH_BEGIN_DECLS
+
+/* xutil.c */
+void *
+xcalloc (size_t nmemb, size_t size);
+
+void *
+xmalloc (size_t size);
+
+void *
+xrealloc (void *ptrr, size_t size);
+
+char *
+xstrdup (const char *s);
+
+char *
+xstrndup (const char *s, size_t n);
+
+NOTMUCH_END_DECLS
+
+#endif

notmuch.c

@@ -18,11 +18,14 @@
  * Author: Carl Worth <cworth@cworth.org>
  */
 
+#include "notmuch.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/time.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <errno.h>
@@ -75,6 +78,143 @@ read_line (void)
     return result;
 }
 
+typedef struct {
+    int messages_total;
+    int count;
+    int count_last;
+    struct timeval tv_start;
+    struct timeval tv_last;
+} add_files_state_t;
+
+/* Compute the number of seconds elapsed from start to end. */
+double
+tv_elapsed (struct timeval start, struct timeval end)
+{
+    return ((end.tv_sec - start.tv_sec) +
+	    (end.tv_usec - start.tv_usec) / 1e6);
+}
+
+void
+print_formatted_seconds (double seconds)
+{
+    int hours;
+    int minutes;
+
+    if (seconds > 3600) {
+	hours = (int) seconds / 3600;
+	printf ("%d:", hours);
+	seconds -= hours * 3600;
+    }
+
+    if (seconds > 60)
+	minutes = (int) seconds / 60;
+    else
+	minutes = 0;
+
+    printf ("%02d:", minutes);
+    seconds -= minutes * 60;
+
+    printf ("%02d", (int) seconds);
+}
+
+void
+add_files_print_progress (add_files_state_t *state)
+{
+    struct timeval tv_now;
+    double ratio_complete;
+    double elapsed_current, rate_current;
+    double elapsed_overall;
+
+    gettimeofday (&tv_now, NULL);
+
+    ratio_complete = (double) state->count / state->messages_total;
+    elapsed_current = tv_elapsed (state->tv_last, tv_now);
+    rate_current = (state->count - state->count_last) / elapsed_current;
+    elapsed_overall = tv_elapsed (state->tv_start, tv_now);
+
+    printf ("Added %d messages at %d messages/sec. ",
+	    state->count, (int) rate_current);
+    print_formatted_seconds (elapsed_overall);
+    printf ("/");
+    print_formatted_seconds (elapsed_overall / ratio_complete);
+    printf (" elapsed (%.2f%%).     \r", 100 * ratio_complete);
+
+    fflush (stdout);
+
+    state->tv_last = tv_now;
+    state->count_last = state->count;
+}
+
+/* Recursively find all regular files in 'path' and add them to the
+ * database. */
+void
+add_files (notmuch_database_t *notmuch, const char *path,
+	   add_files_state_t *state)
+{
+    DIR *dir;
+    struct dirent *entry, *e;
+    int entry_length;
+    int err;
+    char *next;
+    struct stat st;
+
+    dir = opendir (path);
+
+    if (dir == NULL) {
+	fprintf (stderr, "Warning: failed to open directory %s: %s\n",
+		 path, strerror (errno));
+	return;
+    }
+
+    entry_length = offsetof (struct dirent, d_name) +
+	pathconf (path, _PC_NAME_MAX) + 1;
+    entry = malloc (entry_length);
+
+    while (1) {
+	err = readdir_r (dir, entry, &e);
+	if (err) {
+	    fprintf (stderr, "Error reading directory: %s\n",
+		     strerror (errno));
+	    free (entry);
+	    return;
+	}
+
+	if (e == NULL)
+	    break;
+
+	/* Ignore special directories to avoid infinite recursion.
+	 * Also ignore the .notmuch directory.
+	 */
+	/* XXX: Eventually we'll want more sophistication to let the
+	 * user specify files to be ignored. */
+	if (strcmp (entry->d_name, ".") == 0 ||
+	    strcmp (entry->d_name, "..") == 0 ||
+	    strcmp (entry->d_name, ".notmuch") ==0)
+	{
+	    continue;
+	}
+
+	next = g_strdup_printf ("%s/%s", path, entry->d_name);
+
+	stat (next, &st);
+
+	if (S_ISREG (st.st_mode)) {
+	    notmuch_database_add_message (notmuch, next);
+	    state->count++;
+	    if (state->count % 1000 == 0)
+		add_files_print_progress (state);
+	} else if (S_ISDIR (st.st_mode)) {
+	    add_files (notmuch, next, state);
+	}
+
+	free (next);
+    }
+
+    free (entry);
+
+    closedir (dir);
+}
+
 /* Recursively count all regular files in path and all sub-direcotries
  * of path.  The result is added to *count (which should be
  * initialized to zero by the top-level caller before calling
@@ -124,14 +264,14 @@ count_files (const char *path, int *count)
 
 	stat (next, &st);
 
-	if (S_ISREG (st.st_mode))
+	if (S_ISREG (st.st_mode)) {
 	    *count = *count + 1;
-	else if (S_ISDIR (st.st_mode))
+	    if (*count % 1000 == 0) {
+		printf ("Found %d files so far.\r", *count);
+		fflush (stdout);
+	    }
+	} else if (S_ISDIR (st.st_mode)) {
 	    count_files (next, count);
-
-	if (*count % 1000 == 0) {
-	    printf ("Found %d files so far.\r", *count);
-	    fflush (stdout);
 	}
 
 	free (next);
@@ -145,8 +285,11 @@ count_files (const char *path, int *count)
 int
 setup_command (int argc, char *argv[])
 {
+    notmuch_database_t *notmuch;
     char *mail_directory;
     int count;
+    add_files_state_t add_files_state;
+    double elapsed;
 
     printf ("Welcome to notmuch!\n\n");
 
@@ -187,13 +330,40 @@ setup_command (int argc, char *argv[])
 	mail_directory = g_strdup_printf ("%s/mail", home);
     }
 
+    notmuch = notmuch_database_create (mail_directory);
+    if (notmuch == NULL) {
+	fprintf (stderr, "Failed to create new notmuch database at %s\n",
+		 mail_directory);
+	free (mail_directory);
+	return 1;
+    }
+
     printf ("OK. Let's take a look at the mail we can find in the directory\n");
     printf ("%s ...\n", mail_directory);
 
     count = 0;
     count_files (mail_directory, &count);
 
-    printf ("Found %d total files. That's not much mail.\n", count);
+    printf ("Found %d total files. That's not much mail.\n\n", count);
+
+    printf ("Next, we'll inspect the messages and create a database of threads:\n");
+
+    add_files_state.messages_total = count;
+    add_files_state.count = 0;
+    add_files_state.count_last = 0;
+    gettimeofday (&add_files_state.tv_start, NULL);
+    add_files_state.tv_last = add_files_state.tv_start;
+
+    add_files (notmuch, mail_directory, &add_files_state);
+
+    gettimeofday (&add_files_state.tv_last, NULL);
+    elapsed = tv_elapsed (add_files_state.tv_start,
+			  add_files_state.tv_last);
+    printf ("Added %d total messages in ", add_files_state.count);
+    print_formatted_seconds (elapsed);
+    printf (" (%d messages/sec.).                 \n", (int) (add_files_state.count / elapsed));
+
+    notmuch_database_close (notmuch);
 
     free (mail_directory);
     

notmuch.h

@@ -0,0 +1,126 @@
+/* notmuch - Not much of an email library, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_H
+#define NOTMUCH_H
+
+#ifdef  __cplusplus
+# define NOTMUCH_BEGIN_DECLS  extern "C" {
+# define NOTMUCH_END_DECLS    }
+#else
+# define NOTMUCH_BEGIN_DECLS
+# define NOTMUCH_END_DECLS
+#endif
+
+NOTMUCH_BEGIN_DECLS
+
+/* Status codes used for the return values of most functions.
+ *
+ * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function
+ * completed without error. Any other value indicates an error as
+ * follows:
+ *
+ * NOTMUCH_STATUS_SUCCESS: No error occurred.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ */
+typedef enum _notmuch_status {
+    NOTMUCH_STATUS_SUCCESS = 0,
+    NOTMUCH_STATUS_XAPIAN_EXCEPTION
+} notmuch_status_t;
+
+/* An opaque data structure representing a notmuch database. See
+ * notmuch_database_open and other notmuch_database functions
+ * below. */
+typedef struct _notmuch_database notmuch_database_t;
+
+/* Create a new, empty notmuch database located at 'path'.
+ *
+ * The path should be a top-level directory to a collection of
+ * plain-text email messages (one message per file). This call will
+ * create a new ".notmuch" directory within 'path' where notmuch will
+ * store its data.
+ *
+ * Passing a value of NULL for 'path' will cause notmuch to open the
+ * default database. The default database path can be specified by the
+ * NOTMUCH_BASE environment variable, and is equivalent to
+ * ${HOME}/mail if NOTMUCH_BASE is not set.
+ *
+ * After a successful call to notmuch_database_create, the returned
+ * database will be open so the caller should call
+ * notmuch_database_close when finished with it.
+ *
+ * The database will not yet have any data in it
+ * (notmuch_database_create itself is a very cheap function). Messages
+ * contained within 'path' can be added to the database by calling
+ * notmuch_database_add_message.
+ *
+ * In case of any failure, this function returns NULL, (after printing
+ * an error message on stderr).
+ */
+notmuch_database_t *
+notmuch_database_create (const char *path);
+
+/* Open a an existing notmuch database located at 'path'.
+ *
+ * The database should have been created at some time in the past,
+ * (not necessarily by this process), by calling
+ * notmuch_database_create with 'path'.
+ *
+ * An existing notmuch database can be identified by the presence of a
+ * directory named ".notmuch" below 'path'.
+ *
+ * Passing a value of NULL for 'path' will cause notmuch to open the
+ * default database. The default database path can be specified by the
+ * NOTMUCH_BASE environment variable, and is equivalent to
+ * ${HOME}/mail if NOTMUCH_BASE is not set.
+ *
+ * The caller should call notmuch_database_close when finished with
+ * this database.
+ *
+ * In case of any failure, this function returns NULL, (after printing
+ * an error message on stderr).
+ */
+notmuch_database_t *
+notmuch_database_open (const char *path);
+
+/* Close the given notmuch database, freeing all associated
+ * resources. See notmuch_database_open. */
+void
+notmuch_database_close (notmuch_database_t *database);
+
+const char *
+notmuch_database_get_path (notmuch_database_t *database);
+
+/* Add a new message to the given notmuch database.
+ *
+ * Here,'filename' should be a path relative to the the path of
+ * 'database' (see notmuch_database_get_path). The file should be a
+ * single mail message (not a multi-message mbox) that is expected to
+ * remain at its current location, (since the notmuch database will
+ * reference the filename, and will not copy the entire contents of
+ * the file. */
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *database,
+			      const char *filename);
+
+NOTMUCH_END_DECLS
+
+#endif

xutil.c

@@ -0,0 +1,94 @@
+/* xutil.c - Various wrapper functions to abort on error.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#define _GNU_SOURCE /* For strndup */
+#include "notmuch-private.h"
+
+#include <stdio.h>
+
+void *
+xcalloc (size_t nmemb, size_t size)
+{
+    void *ret;
+
+    ret = calloc (nmemb, size);
+    if (ret == NULL) {
+	fprintf (stderr, "Out of memory.\n");
+	exit (1);
+    }
+
+    return ret;
+}
+
+void *
+xmalloc (size_t size)
+{
+    void *ret;
+
+    ret = malloc (size);
+    if (ret == NULL) {
+	fprintf (stderr, "Out of memory.\n");
+	exit (1);
+    }
+
+    return ret;
+}
+
+void *
+xrealloc (void *ptr, size_t size)
+{
+    void *ret;
+
+    ret = realloc (ptr, size);
+    if (ret == NULL) {
+	fprintf (stderr, "Out of memory.\n");
+	exit (1);
+    }
+
+    return ret;
+}
+
+char *
+xstrdup (const char *s)
+{
+    char *ret;
+
+    ret = strdup (s);
+    if (ret == NULL) {
+	fprintf (stderr, "Out of memory.\n");
+	exit (1);
+    }
+
+    return ret;
+}
+
+char *
+xstrndup (const char *s, size_t n)
+{
+    char *ret;
+
+    ret = strndup (s, n);
+    if (ret == NULL) {
+	fprintf (stderr, "Out of memory.\n");
+	exit (1);
+    }
+
+    return ret;
+}