From d807e28f43579ecc91aa40ae3e42760991c2f810 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 5 Jan 2010 13:29:23 -0800 Subject: [PATCH] lib: Implement new notmuch_directory_t API. This new directory ojbect provides all the infrastructure needed to detect when files or directories are deleted or renamed. There's still code needed on top of this (within "notmuch new") to actually do that detection. --- lib/Makefile.local | 1 + lib/database.cc | 234 ++++++++---------------------- lib/directory.cc | 329 ++++++++++++++++++++++++++++++++++++++++++ lib/message.cc | 4 +- lib/notmuch-private.h | 19 +++ notmuch-new.c | 7 +- 6 files changed, 413 insertions(+), 181 deletions(-) create mode 100644 lib/directory.cc diff --git a/lib/Makefile.local b/lib/Makefile.local index a7562c9b..70489e17 100644 --- a/lib/Makefile.local +++ b/lib/Makefile.local @@ -11,6 +11,7 @@ libnotmuch_c_srcs = \ libnotmuch_cxx_srcs = \ $(dir)/database.cc \ + $(dir)/directory.cc \ $(dir)/index.cc \ $(dir)/message.cc \ $(dir)/query.cc \ diff --git a/lib/database.cc b/lib/database.cc index a3824956..510d13cb 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -63,10 +63,11 @@ typedef struct { * * tag: Any tags associated with this message by the user. * - * direntry: A colon-separated pair of values (INTEGER:STRING), - * where INTEGER is the document ID of a directory - * document, and STRING is the name of a file within - * that directory for this mail message. + * file-direntry: A colon-separated pair of values + * (INTEGER:STRING), where INTEGER is the + * document ID of a directory document, and + * STRING is the name of a file within that + * directory for this mail message. * * A mail document also has two values: * @@ -88,22 +89,28 @@ typedef struct { * maintain data necessary to allow for efficient polling of mail * directories. * - * The directory document contains the following terms: + * All directory documents contain one term: * * directory: The directory path (relative to the database path) * Or the SHA1 sum of the directory path (if the * path itself is too long to fit in a Xapian * term). * - * parent: The document ID of the parent directory document. - * Top-level directories will have a parent value of 0. + * And all directory documents for directories other than top-level + * directories also contain the following term: * - * and has a single value: + * directory-direntry: A colon-separated pair of values + * (INTEGER:STRING), where INTEGER is the + * document ID of the parent directory + * document, and STRING is the name of this + * directory within that parent. + * + * All directory documents have a single value: * * TIMESTAMP: The mtime of the directory (at last scan) * * The data portion of a directory document contains the path of the - * directory (relative to the datbase path). + * directory (relative to the database path). */ /* With these prefix values we follow the conventions published here: @@ -122,25 +129,25 @@ typedef struct { */ prefix_t BOOLEAN_PREFIX_INTERNAL[] = { - { "type", "T" }, - { "reference", "XREFERENCE" }, - { "replyto", "XREPLYTO" }, - { "directory", "XDIRECTORY" }, - { "direntry", "XDIRENTRY" }, - { "parent", "XPARENT" }, + { "type", "T" }, + { "reference", "XREFERENCE" }, + { "replyto", "XREPLYTO" }, + { "directory", "XDIRECTORY" }, + { "file-direntry", "XFDIRENTRY" }, + { "directory-direntry", "XDDIRENTRY" }, }; prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { - { "thread", "G" }, - { "tag", "K" }, - { "id", "Q" } + { "thread", "G" }, + { "tag", "K" }, + { "id", "Q" } }; prefix_t PROBABILISTIC_PREFIX[]= { - { "from", "XFROM" }, - { "to", "XTO" }, - { "attachment", "XATTACHMENT" }, - { "subject", "XSUBJECT"} + { "from", "XFROM" }, + { "to", "XTO" }, + { "attachment", "XATTACHMENT" }, + { "subject", "XSUBJECT"} }; int @@ -241,11 +248,11 @@ find_doc_ids (notmuch_database_t *notmuch, talloc_free (term); } -static notmuch_private_status_t -find_unique_doc_id (notmuch_database_t *notmuch, - const char *prefix_name, - const char *value, - unsigned int *doc_id) +notmuch_private_status_t +_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch, + const char *prefix_name, + const char *value, + unsigned int *doc_id) { Xapian::PostingIterator i, end; @@ -275,26 +282,6 @@ find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id) return notmuch->xapian_db->get_document (doc_id); } -static notmuch_private_status_t -find_unique_document (notmuch_database_t *notmuch, - const char *prefix_name, - const char *value, - Xapian::Document *document, - unsigned int *doc_id) -{ - notmuch_private_status_t status; - - status = find_unique_doc_id (notmuch, prefix_name, value, doc_id); - - if (status) { - *document = Xapian::Document (); - return status; - } - - *document = find_document_for_doc_id (notmuch, *doc_id); - return NOTMUCH_PRIVATE_STATUS_SUCCESS; -} - notmuch_message_t * notmuch_database_find_message (notmuch_database_t *notmuch, const char *message_id) @@ -302,7 +289,8 @@ notmuch_database_find_message (notmuch_database_t *notmuch, notmuch_private_status_t status; unsigned int doc_id; - status = find_unique_doc_id (notmuch, "id", message_id, &doc_id); + status = _notmuch_database_find_unique_doc_id (notmuch, "id", + message_id, &doc_id); if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) return NULL; @@ -593,13 +581,6 @@ notmuch_database_get_path (notmuch_database_t *notmuch) return notmuch->path; } -static notmuch_private_status_t -find_directory_document (notmuch_database_t *notmuch, const char *db_path, - Xapian::Document *doc, unsigned int *doc_id) -{ - return find_unique_document (notmuch, "directory", db_path, doc, doc_id); -} - /* We allow the user to use arbitrarily long paths for directories. But * we have a term-length limit. So if we exceed that, we'll use the * SHA-1 of the path for the database term. @@ -608,8 +589,8 @@ find_directory_document (notmuch_database_t *notmuch, const char *db_path, * does not, then the caller is responsible to free() the returned * value. */ -static const char * -directory_db_path (const char *path) +const char * +_notmuch_database_get_directory_db_path (const char *path) { int term_len = strlen (_find_prefix ("directory")) + strlen (path); @@ -705,38 +686,25 @@ _notmuch_database_find_directory_id (notmuch_database_t *notmuch, const char *path, unsigned int *directory_id) { - notmuch_private_status_t private_status; - notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; - const char *db_path; + notmuch_directory_t *directory; + notmuch_status_t status; if (path == NULL) { *directory_id = 0; return NOTMUCH_STATUS_SUCCESS; } - db_path = directory_db_path (path); - - private_status = find_unique_doc_id (notmuch, "directory", - db_path, directory_id); - if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { - status = notmuch_database_set_directory_mtime (notmuch, - path, 0); - if (status) - goto DONE; - - private_status = find_unique_doc_id (notmuch, "directory", - db_path, directory_id); - status = COERCE_STATUS (private_status, "_find_directory_id"); + directory = _notmuch_directory_create (notmuch, path, &status); + if (status) { + *directory_id = -1; + return status; } - DONE: - if (db_path != path) - free ((char *) db_path); + *directory_id = _notmuch_directory_get_document_id (directory); - if (status) - *directory_id = -1; + notmuch_directory_destroy (directory); - return status; + return NOTMUCH_STATUS_SUCCESS; } const char * @@ -817,101 +785,13 @@ _notmuch_database_relative_path (notmuch_database_t *notmuch, return relative; } -notmuch_status_t -notmuch_database_set_directory_mtime (notmuch_database_t *notmuch, - const char *path, - time_t mtime) +notmuch_directory_t * +notmuch_database_get_directory (notmuch_database_t *notmuch, + const char *path) { - Xapian::Document doc; - Xapian::WritableDatabase *db; - unsigned int doc_id; - notmuch_private_status_t status; - notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; - const char *parent, *db_path = NULL; - unsigned int parent_id; - void *local = talloc_new (notmuch); + notmuch_status_t status; - if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) { - fprintf (stderr, "Attempted to update a read-only database.\n"); - return NOTMUCH_STATUS_READONLY_DATABASE; - } - - path = _notmuch_database_relative_path (notmuch, path); - - db = static_cast (notmuch->xapian_db); - db_path = directory_db_path (path); - - try { - status = find_directory_document (notmuch, db_path, &doc, &doc_id); - - doc.add_value (NOTMUCH_VALUE_TIMESTAMP, - Xapian::sortable_serialise (mtime)); - - if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { - char *term = talloc_asprintf (local, "%s%s", - _find_prefix ("directory"), db_path); - doc.add_term (term); - - doc.set_data (path); - - _notmuch_database_split_path (local, path, &parent, NULL); - - _notmuch_database_find_directory_id (notmuch, parent, &parent_id); - - term = talloc_asprintf (local, "%s%u", - _find_prefix ("parent"), - parent_id); - doc.add_term (term); - - db->add_document (doc); - } else { - db->replace_document (doc_id, doc); - } - - } catch (const Xapian::Error &error) { - fprintf (stderr, "A Xapian exception occurred setting directory mtime: %s.\n", - error.get_msg().c_str()); - notmuch->exception_reported = TRUE; - ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION; - } - - if (db_path != path) - free ((char *) db_path); - - talloc_free (local); - - return ret; -} - -time_t -notmuch_database_get_directory_mtime (notmuch_database_t *notmuch, - const char *path) -{ - Xapian::Document doc; - unsigned int doc_id; - notmuch_private_status_t status; - const char *db_path = NULL; - time_t ret = 0; - - db_path = directory_db_path (path); - - try { - status = find_directory_document (notmuch, db_path, &doc, &doc_id); - - if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) - goto DONE; - - ret = Xapian::sortable_unserialise (doc.get_value (NOTMUCH_VALUE_TIMESTAMP)); - } catch (Xapian::Error &error) { - ret = 0; - goto DONE; - } - - DONE: - if (db_path != path) - free ((char *) db_path); - - return ret; + return _notmuch_directory_create (notmuch, path, &status); } /* Find the thread ID to which the message with 'message_id' belongs. @@ -1284,7 +1164,7 @@ notmuch_database_remove_message (notmuch_database_t *notmuch, { Xapian::WritableDatabase *db; void *local = talloc_new (notmuch); - const char *direntry_prefix = _find_prefix ("direntry"); + const char *prefix = _find_prefix ("file-direntry"); char *direntry, *term; Xapian::PostingIterator i, end; Xapian::Document document; @@ -1302,7 +1182,7 @@ notmuch_database_remove_message (notmuch_database_t *notmuch, if (status) return status; - term = talloc_asprintf (notmuch, "%s%s", direntry_prefix, direntry); + term = talloc_asprintf (notmuch, "%s%s", prefix, direntry); find_doc_ids_for_term (notmuch, term, &i, &end); @@ -1314,11 +1194,11 @@ notmuch_database_remove_message (notmuch_database_t *notmuch, document.remove_term (term); j = document.termlist_begin (); - j.skip_to (direntry_prefix); + j.skip_to (prefix); - /* Was this the last direntry in the message? */ + /* Was this the last file-direntry in the message? */ if (j == document.termlist_end () || - strncmp ((*j).c_str (), direntry_prefix, strlen (direntry_prefix))) + strncmp ((*j).c_str (), prefix, strlen (prefix))) { db->delete_document (document.get_docid ()); } else { diff --git a/lib/directory.cc b/lib/directory.cc new file mode 100644 index 00000000..196f7805 --- /dev/null +++ b/lib/directory.cc @@ -0,0 +1,329 @@ +/* directory.cc - Results of directory-based searches from a notmuch database + * + * Copyright © 2009 Carl Worth + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/ . + * + * Author: Carl Worth + */ + +#include "notmuch-private.h" +#include "database-private.h" + +#include + +struct _notmuch_filenames { + Xapian::TermIterator iterator; + Xapian::TermIterator end; + int prefix_len; + char *filename; +}; + +/* We end up having to call the destructors explicitly because we had + * to use "placement new" in order to initialize C++ objects within a + * block that we allocated with talloc. So C++ is making talloc + * slightly less simple to use, (we wouldn't need + * talloc_set_destructor at all otherwise). + */ +static int +_notmuch_filenames_destructor (notmuch_filenames_t *filenames) +{ + filenames->iterator.~TermIterator (); + filenames->end.~TermIterator (); + + return 0; +} + +/* Create an iterator to iterate over the basenames of files (or + * directories) that all share a common parent directory. + * + * The code here is general enough to be reused for any case of + * iterating over the non-prefixed portion of terms sharing a common + * prefix. + */ +notmuch_filenames_t * +_notmuch_filenames_create (void *ctx, + notmuch_database_t *notmuch, + const char *prefix) +{ + notmuch_filenames_t *filenames; + + filenames = talloc (ctx, notmuch_filenames_t); + if (unlikely (filenames == NULL)) + return NULL; + + new (&filenames->iterator) Xapian::TermIterator (); + new (&filenames->end) Xapian::TermIterator (); + + talloc_set_destructor (filenames, _notmuch_filenames_destructor); + + filenames->iterator = notmuch->xapian_db->allterms_begin (prefix); + filenames->end = notmuch->xapian_db->allterms_end (prefix); + + filenames->prefix_len = strlen (prefix); + + filenames->filename = NULL; + + return filenames; +} + +notmuch_bool_t +notmuch_filenames_has_more (notmuch_filenames_t *filenames) +{ + return (filenames->iterator != filenames->end); +} + +const char * +notmuch_filenames_get (notmuch_filenames_t *filenames) +{ + if (filenames->filename == NULL) { + std::string term = *filenames->iterator; + + filenames->filename = talloc_strdup (filenames, + term.c_str () + + filenames->prefix_len); + } + + return filenames->filename; +} + +void +notmuch_filenames_advance (notmuch_filenames_t *filenames) +{ + if (filenames->filename) { + talloc_free (filenames->filename); + filenames->filename = NULL; + } + + if (filenames->iterator != filenames->end) + filenames->iterator++; +} + +void +notmuch_filenames_destroy (notmuch_filenames_t *filenames) +{ + talloc_free (filenames); +} + +struct _notmuch_directory { + notmuch_database_t *notmuch; + Xapian::docid document_id; + Xapian::Document doc; + time_t mtime; +}; + +/* We end up having to call the destructor explicitly because we had + * to use "placement new" in order to initialize C++ objects within a + * block that we allocated with talloc. So C++ is making talloc + * slightly less simple to use, (we wouldn't need + * talloc_set_destructor at all otherwise). + */ +static int +_notmuch_directory_destructor (notmuch_directory_t *directory) +{ + directory->doc.~Document (); + + return 0; +} + +static notmuch_private_status_t +find_directory_document (notmuch_database_t *notmuch, + const char *db_path, + Xapian::Document *document) +{ + notmuch_private_status_t status; + Xapian::docid doc_id; + + status = _notmuch_database_find_unique_doc_id (notmuch, "directory", + db_path, &doc_id); + if (status) { + *document = Xapian::Document (); + return status; + } + + *document = notmuch->xapian_db->get_document (doc_id); + return NOTMUCH_PRIVATE_STATUS_SUCCESS; +} + +notmuch_directory_t * +_notmuch_directory_create (notmuch_database_t *notmuch, + const char *path, + notmuch_status_t *status_ret) +{ + Xapian::WritableDatabase *db; + notmuch_directory_t *directory; + notmuch_private_status_t private_status; + const char *db_path; + + *status_ret = NOTMUCH_STATUS_SUCCESS; + + path = _notmuch_database_relative_path (notmuch, path); + + if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) { + fprintf (stderr, "Attempted to update a read-only database.\n"); + *status_ret = NOTMUCH_STATUS_READONLY_DATABASE; + return NULL; + } + + db = static_cast (notmuch->xapian_db); + + directory = talloc (notmuch, notmuch_directory_t); + if (unlikely (directory == NULL)) + return NULL; + + directory->notmuch = notmuch; + + /* "placement new"---not actually allocating memory */ + new (&directory->doc) Xapian::Document; + + talloc_set_destructor (directory, _notmuch_directory_destructor); + + db_path = _notmuch_database_get_directory_db_path (path); + + try { + Xapian::TermIterator i, end; + + private_status = find_directory_document (notmuch, db_path, + &directory->doc); + directory->document_id = directory->doc.get_docid (); + + if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + void *local = talloc_new (directory); + const char *parent, *basename; + Xapian::docid parent_id; + char *term = talloc_asprintf (local, "%s%s", + _find_prefix ("directory"), db_path); + directory->doc.add_term (term); + + directory->doc.set_data (path); + + _notmuch_database_split_path (local, path, &parent, &basename); + + _notmuch_database_find_directory_id (notmuch, parent, &parent_id); + + if (basename) { + term = talloc_asprintf (local, "%s%u:%s", + _find_prefix ("directory-direntry"), + parent_id, basename); + directory->doc.add_term (term); + } + + directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP, + Xapian::sortable_serialise (0)); + + directory->document_id = db->add_document (directory->doc); + talloc_free (local); + } + + directory->mtime = Xapian::sortable_unserialise ( + directory->doc.get_value (NOTMUCH_VALUE_TIMESTAMP)); + } catch (const Xapian::Error &error) { + fprintf (stderr, + "A Xapian exception occurred creating a directory: %s.\n", + error.get_msg().c_str()); + notmuch->exception_reported = TRUE; + notmuch_directory_destroy (directory); + *status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION; + return NULL; + } + + if (db_path != path) + free ((char *) db_path); + + return directory; +} + +unsigned int +_notmuch_directory_get_document_id (notmuch_directory_t *directory) +{ + return directory->document_id; +} + +notmuch_status_t +notmuch_directory_set_mtime (notmuch_directory_t *directory, + time_t mtime) +{ + notmuch_database_t *notmuch = directory->notmuch; + Xapian::WritableDatabase *db; + + if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) { + fprintf (stderr, "Attempted to update a read-only database.\n"); + return NOTMUCH_STATUS_READONLY_DATABASE; + } + + db = static_cast (notmuch->xapian_db); + + try { + directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP, + Xapian::sortable_serialise (mtime)); + + db->replace_document (directory->document_id, directory->doc); + } catch (const Xapian::Error &error) { + fprintf (stderr, + "A Xapian exception occurred setting directory mtime: %s.\n", + error.get_msg().c_str()); + notmuch->exception_reported = TRUE; + return NOTMUCH_STATUS_XAPIAN_EXCEPTION; + } + + return NOTMUCH_STATUS_SUCCESS; +} + +time_t +notmuch_directory_get_mtime (notmuch_directory_t *directory) +{ + return directory->mtime; +} + +notmuch_filenames_t * +notmuch_directory_get_child_files (notmuch_directory_t *directory) +{ + char *term; + notmuch_filenames_t *child_files; + + term = talloc_asprintf (directory, "%s%u:", + _find_prefix ("file-direntry"), + directory->document_id); + + child_files = _notmuch_filenames_create (directory, + directory->notmuch, term); + + talloc_free (term); + + return child_files; +} + +notmuch_filenames_t * +notmuch_directory_get_child_directories (notmuch_directory_t *directory) +{ + char *term; + notmuch_filenames_t *child_directories; + + term = talloc_asprintf (directory, "%s%u:", + _find_prefix ("directory-direntry"), + directory->document_id); + + child_directories = _notmuch_filenames_create (directory, + directory->notmuch, term); + + talloc_free (term); + + return child_directories; +} + +void +notmuch_directory_destroy (notmuch_directory_t *directory) +{ + talloc_free (directory); +} diff --git a/lib/message.cc b/lib/message.cc index bd179519..82e8fce7 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -411,7 +411,7 @@ _notmuch_message_add_filename (notmuch_message_t *message, if (status) return status; - _notmuch_message_add_term (message, "direntry", direntry); + _notmuch_message_add_term (message, "file-direntry", direntry); talloc_free (local); @@ -421,7 +421,7 @@ _notmuch_message_add_filename (notmuch_message_t *message, const char * notmuch_message_get_filename (notmuch_message_t *message) { - const char *prefix = _find_prefix ("direntry"); + const char *prefix = _find_prefix ("file-direntry"); int prefix_len = strlen (prefix); Xapian::TermIterator i; char *direntry, *colon; diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index cb93c397..8b582640 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -161,6 +161,15 @@ _notmuch_database_split_path (void *ctx, const char **directory, const char **basename); +const char * +_notmuch_database_get_directory_db_path (const char *path); + +notmuch_private_status_t +_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch, + const char *prefix_name, + const char *value, + unsigned int *doc_id); + notmuch_status_t _notmuch_database_find_directory_id (notmuch_database_t *database, const char *path, @@ -177,6 +186,16 @@ _notmuch_database_filename_to_direntry (void *ctx, const char *filename, char **direntry); +/* directory.cc */ + +notmuch_directory_t * +_notmuch_directory_create (notmuch_database_t *notmuch, + const char *path, + notmuch_status_t *status_ret); + +unsigned int +_notmuch_directory_get_document_id (notmuch_directory_t *directory); + /* thread.cc */ notmuch_thread_t * diff --git a/notmuch-new.c b/notmuch-new.c index 2a929c56..ee6f196e 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -143,10 +143,13 @@ add_files_recursive (notmuch_database_t *notmuch, notmuch_message_t *message = NULL; struct dirent **namelist = NULL; int num_entries; + notmuch_directory_t *directory; path_mtime = st->st_mtime; - path_dbtime = notmuch_database_get_directory_mtime (notmuch, path); + directory = notmuch_database_get_directory (notmuch, path); + path_dbtime = notmuch_directory_get_mtime (directory); + num_entries = scandir (path, &namelist, 0, ino_cmp); if (num_entries == -1) { @@ -277,7 +280,7 @@ add_files_recursive (notmuch_database_t *notmuch, next = NULL; } - status = notmuch_database_set_directory_mtime (notmuch, path, path_mtime); + status = notmuch_directory_set_mtime (directory, path_mtime); if (status && ret == NOTMUCH_STATUS_SUCCESS) ret = status;