notmuch/lib/add-message.cc

#include "database-private.h"

/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
 * a (potentially nested) parenthesized sequence with '\' used to
 * escape any character (including parentheses).
 *
 * If the sequence to be skipped continues to the end of the string,
 * then 'str' will be left pointing at the final terminating '\0'
 * character.
 */
static void
skip_space_and_comments (const char **str)
{
    const char *s;

    s = *str;
    while (*s && (isspace (*s) || *s == '(')) {
	while (*s && isspace (*s))
	    s++;
	if (*s == '(') {
	    int nesting = 1;
	    s++;
	    while (*s && nesting) {
		if (*s == '(') {
		    nesting++;
		} else if (*s == ')') {
		    nesting--;
		} else if (*s == '\\') {
		    if (*(s+1))
			s++;
		}
		s++;
	    }
	}
    }

    *str = s;
}

/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
 * comments, and the '<' and '>' delimiters.
 *
 * If not NULL, then *next will be made to point to the first character
 * not parsed, (possibly pointing to the final '\0' terminator.
 *
 * Returns a newly talloc'ed string belonging to 'ctx'.
 *
 * Returns NULL if there is any error parsing the message-id. */
static char *
_parse_message_id (void *ctx, const char *message_id, const char **next)
{
    const char *s, *end;
    char *result;

    if (message_id == NULL || *message_id == '\0')
	return NULL;

    s = message_id;

    skip_space_and_comments (&s);

    /* Skip any unstructured text as well. */
    while (*s && *s != '<')
	s++;

    if (*s == '<') {
	s++;
    } else {
	if (next)
	    *next = s;
	return NULL;
    }

    skip_space_and_comments (&s);

    end = s;
    while (*end && *end != '>')
	end++;
    if (next) {
	if (*end)
	    *next = end + 1;
	else
	    *next = end;
    }

    if (end > s && *end == '>')
	end--;
    if (end <= s)
	return NULL;

    result = talloc_strndup (ctx, s, end - s + 1);

    /* Finally, collapse any whitespace that is within the message-id
     * itself. */
    {
	char *r;
	int len;

	for (r = result, len = strlen (r); *r; r++, len--)
	    if (*r == ' ' || *r == '\t')
		memmove (r, r+1, len);
    }

    return result;
}

/* Parse a References header value, putting a (talloc'ed under 'ctx')
 * copy of each referenced message-id into 'hash'.
 *
 * We explicitly avoid including any reference identical to
 * 'message_id' in the result (to avoid mass confusion when a single
 * message references itself cyclically---and yes, mail messages are
 * not infrequent in the wild that do this---don't ask me why).
 *
 * Return the last reference parsed, if it is not equal to message_id.
 */
static char *
parse_references (void *ctx,
		  const char *message_id,
		  GHashTable *hash,
		  const char *refs)
{
    char *ref, *last_ref = NULL;

    if (refs == NULL || *refs == '\0')
	return NULL;

    while (*refs) {
	ref = _parse_message_id (ctx, refs, &refs);

	if (ref && strcmp (ref, message_id)) {
	    g_hash_table_add (hash, ref);
	    last_ref = ref;
	}
    }

    /* The return value of this function is used to add a parent
     * reference to the database.  We should avoid making a message
     * its own parent, thus the above check.
     */
    return talloc_strdup(ctx, last_ref);
}

static const char *
_notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
{
    /* 16 bytes (+ terminator) for hexadecimal representation of
     * a 64-bit integer. */
    static char thread_id[17];
    Xapian::WritableDatabase *db;

    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);

    notmuch->last_thread_id++;

    sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);

    db->set_metadata ("last_thread_id", thread_id);

    return thread_id;
}

static char *
_get_metadata_thread_id_key (void *ctx, const char *message_id)
{
    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
	message_id = _notmuch_message_id_compressed (ctx, message_id);

    return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
			    message_id);
}


static notmuch_status_t
_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
				      void *ctx,
				      const char *message_id,
				      const char **thread_id_ret);


/* Find the thread ID to which the message with 'message_id' belongs.
 *
 * Note: 'thread_id_ret' must not be NULL!
 * On success '*thread_id_ret' is set to a newly talloced string belonging to
 * 'ctx'.
 *
 * Note: If there is no message in the database with the given
 * 'message_id' then a new thread_id will be allocated for this
 * message ID and stored in the database metadata so that the
 * thread ID can be looked up if the message is added to the database
 * later.
 */
static notmuch_status_t
_resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
				  void *ctx,
				  const char *message_id,
				  const char **thread_id_ret)
{
    notmuch_private_status_t status;
    notmuch_message_t *message;

    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
	return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
						     thread_id_ret);

    /* Look for this message (regular or ghost) */
    message = _notmuch_message_create_for_message_id (
	notmuch, message_id, &status);
    if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
	/* Message exists */
	*thread_id_ret = talloc_steal (
	    ctx, notmuch_message_get_thread_id (message));
    } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
	/* Message did not exist.  Give it a fresh thread ID and
	 * populate this message as a ghost message. */
	*thread_id_ret = talloc_strdup (
	    ctx, _notmuch_database_generate_thread_id (notmuch));
	if (! *thread_id_ret) {
	    status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
	} else {
	    status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
	    if (status == 0)
		/* Commit the new ghost message */
		_notmuch_message_sync (message);
	}
    } else {
	/* Create failed. Fall through. */
    }

    notmuch_message_destroy (message);

    return COERCE_STATUS (status, "Error creating ghost message");
}

/* Pre-ghost messages _resolve_message_id_to_thread_id */
static notmuch_status_t
_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
				      void *ctx,
				      const char *message_id,
				      const char **thread_id_ret)
{
    notmuch_status_t status;
    notmuch_message_t *message;
    std::string thread_id_string;
    char *metadata_key;
    Xapian::WritableDatabase *db;

    status = notmuch_database_find_message (notmuch, message_id, &message);

    if (status)
	return status;

    if (message) {
	*thread_id_ret = talloc_steal (ctx,
				       notmuch_message_get_thread_id (message));

	notmuch_message_destroy (message);

	return NOTMUCH_STATUS_SUCCESS;
    }

    /* Message has not been seen yet.
     *
     * We may have seen a reference to it already, in which case, we
     * can return the thread ID stored in the metadata. Otherwise, we
     * generate a new thread ID and store it there.
     */
    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
    thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);

    if (thread_id_string.empty()) {
	*thread_id_ret = talloc_strdup (ctx,
					_notmuch_database_generate_thread_id (notmuch));
	db->set_metadata (metadata_key, *thread_id_ret);
    } else {
	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
    }

    talloc_free (metadata_key);

    return NOTMUCH_STATUS_SUCCESS;
}

static notmuch_status_t
_merge_threads (notmuch_database_t *notmuch,
		const char *winner_thread_id,
		const char *loser_thread_id)
{
    Xapian::PostingIterator loser, loser_end;
    notmuch_message_t *message = NULL;
    notmuch_private_status_t private_status;
    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;

    _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);

    for ( ; loser != loser_end; loser++) {
	message = _notmuch_message_create (notmuch, notmuch,
					   *loser, &private_status);
	if (message == NULL) {
	    ret = COERCE_STATUS (private_status,
				 "Cannot find document for doc_id from query");
	    goto DONE;
	}

	_notmuch_message_remove_term (message, "thread", loser_thread_id);
	_notmuch_message_add_term (message, "thread", winner_thread_id);
	_notmuch_message_sync (message);

	notmuch_message_destroy (message);
	message = NULL;
    }

  DONE:
    if (message)
	notmuch_message_destroy (message);

    return ret;
}

static void
_my_talloc_free_for_g_hash (void *ptr)
{
    talloc_free (ptr);
}

static notmuch_status_t
_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
					   notmuch_message_t *message,
					   notmuch_message_file_t *message_file,
					   const char **thread_id)
{
    GHashTable *parents = NULL;
    const char *refs, *in_reply_to, *in_reply_to_message_id;
    const char *last_ref_message_id, *this_message_id;
    GList *l, *keys = NULL;
    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;

    parents = g_hash_table_new_full (g_str_hash, g_str_equal,
				     _my_talloc_free_for_g_hash, NULL);
    this_message_id = notmuch_message_get_message_id (message);

    refs = _notmuch_message_file_get_header (message_file, "references");
    last_ref_message_id = parse_references (message,
					    this_message_id,
					    parents, refs);

    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
    in_reply_to_message_id = parse_references (message,
					       this_message_id,
					       parents, in_reply_to);

    /* For the parent of this message, use the last message ID of the
     * References header, if available.  If not, fall back to the
     * first message ID in the In-Reply-To header. */
    if (last_ref_message_id) {
	_notmuch_message_add_term (message, "replyto",
				   last_ref_message_id);
    } else if (in_reply_to_message_id) {
	_notmuch_message_add_term (message, "replyto",
			     in_reply_to_message_id);
    }

    keys = g_hash_table_get_keys (parents);
    for (l = keys; l; l = l->next) {
	char *parent_message_id;
	const char *parent_thread_id = NULL;

	parent_message_id = (char *) l->data;

	_notmuch_message_add_term (message, "reference",
				   parent_message_id);

	ret = _resolve_message_id_to_thread_id (notmuch,
						message,
						parent_message_id,
						&parent_thread_id);
	if (ret)
	    goto DONE;

	if (*thread_id == NULL) {
	    *thread_id = talloc_strdup (message, parent_thread_id);
	    _notmuch_message_add_term (message, "thread", *thread_id);
	} else if (strcmp (*thread_id, parent_thread_id)) {
	    ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
	    if (ret)
		goto DONE;
	}
    }

  DONE:
    if (keys)
	g_list_free (keys);
    if (parents)
	g_hash_table_unref (parents);

    return ret;
}

static notmuch_status_t
_notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
					    notmuch_message_t *message,
					    const char **thread_id)
{
    const char *message_id = notmuch_message_get_message_id (message);
    Xapian::PostingIterator child, children_end;
    notmuch_message_t *child_message = NULL;
    const char *child_thread_id;
    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
    notmuch_private_status_t private_status;

    _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);

    for ( ; child != children_end; child++) {

	child_message = _notmuch_message_create (message, notmuch,
						 *child, &private_status);
	if (child_message == NULL) {
	    ret = COERCE_STATUS (private_status,
				 "Cannot find document for doc_id from query");
	    goto DONE;
	}

	child_thread_id = notmuch_message_get_thread_id (child_message);
	if (*thread_id == NULL) {
	    *thread_id = talloc_strdup (message, child_thread_id);
	    _notmuch_message_add_term (message, "thread", *thread_id);
	} else if (strcmp (*thread_id, child_thread_id)) {
	    _notmuch_message_remove_term (child_message, "reference",
					  message_id);
	    _notmuch_message_sync (child_message);
	    ret = _merge_threads (notmuch, *thread_id, child_thread_id);
	    if (ret)
		goto DONE;
	}

	notmuch_message_destroy (child_message);
	child_message = NULL;
    }

  DONE:
    if (child_message)
	notmuch_message_destroy (child_message);

    return ret;
}

/* Fetch and clear the stored thread_id for message, or NULL if none. */
static char *
_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
			     notmuch_message_t *message)
{
    const char *message_id;
    std::string stored_id;
    char *metadata_key;

    message_id = notmuch_message_get_message_id (message);
    metadata_key = _get_metadata_thread_id_key (ctx, message_id);

    /* Check if we have already seen related messages to this one.
     * If we have then use the thread_id that we stored at that time.
     */
    stored_id = notmuch->xapian_db->get_metadata (metadata_key);
    if (stored_id.empty ()) {
	return NULL;
    } else {
	Xapian::WritableDatabase *db;

	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);

	/* Clear the metadata for this message ID. We don't need it
	 * anymore. */
	db->set_metadata (metadata_key, "");

	return talloc_strdup (ctx, stored_id.c_str ());
    }
}

/* Given a blank or ghost 'message' and its corresponding
 * 'message_file' link it to existing threads in the database.
 *
 * First, if is_ghost, this retrieves the thread ID already stored in
 * the message (which will be the case if a message was previously
 * added that referenced this one).  If the message is blank
 * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
 * later in this function).  If the database does not support ghost
 * messages, this checks for a thread ID stored in database metadata
 * for this message ID.
 *
 * Second, we look at 'message_file' and its link-relevant headers
 * (References and In-Reply-To) for message IDs.
 *
 * Finally, we look in the database for existing message that
 * reference 'message'.
 *
 * In all cases, we assign to the current message the first thread ID
 * found. We will also merge any existing, distinct threads where this
 * message belongs to both, (which is not uncommon when messages are
 * processed out of order).
 *
 * Finally, if no thread ID has been found through referenced messages, we
 * call _notmuch_message_generate_thread_id to generate a new thread
 * ID. This should only happen for new, top-level messages, (no
 * References or In-Reply-To header in this message, and no previously
 * added message refers to this message).
 */
static notmuch_status_t
_notmuch_database_link_message (notmuch_database_t *notmuch,
				notmuch_message_t *message,
				notmuch_message_file_t *message_file,
				notmuch_bool_t is_ghost)
{
    void *local = talloc_new (NULL);
    notmuch_status_t status;
    const char *thread_id = NULL;

    /* Check if the message already had a thread ID */
    if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
	if (is_ghost)
	    thread_id = notmuch_message_get_thread_id (message);
    } else {
	thread_id = _consume_metadata_thread_id (local, notmuch, message);
	if (thread_id)
	    _notmuch_message_add_term (message, "thread", thread_id);
    }

    status = _notmuch_database_link_message_to_parents (notmuch, message,
							message_file,
							&thread_id);
    if (status)
	goto DONE;

    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
	/* In general, it shouldn't be necessary to link children,
	 * since the earlier indexing of those children will have
	 * stored a thread ID for the missing parent.  However, prior
	 * to ghost messages, these stored thread IDs were NOT
	 * rewritten during thread merging (and there was no
	 * performant way to do so), so if indexed children were
	 * pulled into a different thread ID by a merge, it was
	 * necessary to pull them *back* into the stored thread ID of
	 * the parent.  With ghost messages, we just rewrite the
	 * stored thread IDs during merging, so this workaround isn't
	 * necessary. */
	status = _notmuch_database_link_message_to_children (notmuch, message,
							     &thread_id);
	if (status)
	    goto DONE;
    }

    /* If not part of any existing thread, generate a new thread ID. */
    if (thread_id == NULL) {
	thread_id = _notmuch_database_generate_thread_id (notmuch);

	_notmuch_message_add_term (message, "thread", thread_id);
    }

 DONE:
    talloc_free (local);

    return status;
}

notmuch_status_t
notmuch_database_add_message (notmuch_database_t *notmuch,
			      const char *filename,
			      notmuch_message_t **message_ret)
{
    notmuch_message_file_t *message_file;
    notmuch_message_t *message = NULL;
    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
    notmuch_private_status_t private_status;
    notmuch_bool_t is_ghost = FALSE, is_new = FALSE;

    const char *date, *header;
    const char *from, *to, *subject;
    char *message_id = NULL;

    if (message_ret)
	*message_ret = NULL;

    ret = _notmuch_database_ensure_writable (notmuch);
    if (ret)
	return ret;

    message_file = _notmuch_message_file_open (notmuch, filename);
    if (message_file == NULL)
	return NOTMUCH_STATUS_FILE_ERROR;

    /* Adding a message may change many documents.  Do this all
     * atomically. */
    ret = notmuch_database_begin_atomic (notmuch);
    if (ret)
	goto DONE;

    /* Parse message up front to get better error status. */
    ret = _notmuch_message_file_parse (message_file);
    if (ret)
	goto DONE;

    /* Before we do any real work, (especially before doing a
     * potential SHA-1 computation on the entire file's contents),
     * let's make sure that what we're looking at looks like an
     * actual email message.
     */
    from = _notmuch_message_file_get_header (message_file, "from");
    subject = _notmuch_message_file_get_header (message_file, "subject");
    to = _notmuch_message_file_get_header (message_file, "to");

    if ((from == NULL || *from == '\0') &&
	(subject == NULL || *subject == '\0') &&
	(to == NULL || *to == '\0')) {
	ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
	goto DONE;
    }

    /* Now that we're sure it's mail, the first order of business
     * is to find a message ID (or else create one ourselves).
     */
    header = _notmuch_message_file_get_header (message_file, "message-id");
    if (header && *header != '\0') {
	message_id = _parse_message_id (message_file, header, NULL);

	/* So the header value isn't RFC-compliant, but it's
	 * better than no message-id at all.
	 */
	if (message_id == NULL)
	    message_id = talloc_strdup (message_file, header);
    }

    if (message_id == NULL ) {
	/* No message-id at all, let's generate one by taking a
	 * hash over the file's contents.
	 */
	char *sha1 = _notmuch_sha1_of_file (filename);

	/* If that failed too, something is really wrong. Give up. */
	if (sha1 == NULL) {
	    ret = NOTMUCH_STATUS_FILE_ERROR;
	    goto DONE;
	}

	message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
	free (sha1);
    }

    try {
	/* Now that we have a message ID, we get a message object,
	 * (which may or may not reference an existing document in the
	 * database). */

	message = _notmuch_message_create_for_message_id (notmuch,
							  message_id,
							  &private_status);

	talloc_free (message_id);

	/* We cannot call notmuch_message_get_flag for a new message */
	switch (private_status) {
	case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
	    is_ghost = FALSE;
	    is_new = TRUE;
	    break;
	case NOTMUCH_PRIVATE_STATUS_SUCCESS:
	    is_ghost = notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_GHOST);
	    is_new = FALSE;
	    break;
	default:
	    ret = COERCE_STATUS (private_status,
				 "Unexpected status value from _notmuch_message_create_for_message_id");
	    goto DONE;
	}

	_notmuch_message_add_filename (message, filename);

	if (is_new || is_ghost) {
	    _notmuch_message_add_term (message, "type", "mail");
	    if (is_ghost)
		/* Convert ghost message to a regular message */
		_notmuch_message_remove_term (message, "type", "ghost");
	    ret = _notmuch_database_link_message (notmuch, message,
						  message_file, is_ghost);
	    if (ret)
		goto DONE;

	    date = _notmuch_message_file_get_header (message_file, "date");
	    _notmuch_message_set_header_values (message, date, from, subject);

	    ret = _notmuch_message_index_file (message, message_file);
	    if (ret)
		goto DONE;
	} else {
	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
	}

	_notmuch_message_sync (message);
    } catch (const Xapian::Error &error) {
	_notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
		 error.get_msg().c_str());
	notmuch->exception_reported = TRUE;
	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
	goto DONE;
    }

  DONE:
    if (message) {
	if ((ret == NOTMUCH_STATUS_SUCCESS ||
	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
	    *message_ret = message;
	else
	    notmuch_message_destroy (message);
    }

    if (message_file)
	_notmuch_message_file_close (message_file);

    ret2 = notmuch_database_end_atomic (notmuch);
    if ((ret == NOTMUCH_STATUS_SUCCESS ||
	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
	ret2 != NOTMUCH_STATUS_SUCCESS)
	ret = ret2;

    return ret;
}