debian/notmuch
1
0
Fork 0
mirror of https://git.notmuchmail.org/git/notmuch synced 2024-11-25 12:28:09 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'upstream'

Browse source
This commit is contained in:
martin f. krafft 2010-01-21 13:58:55 +13:00
commit 0ea5f3fc0e
17 changed files with 1881 additions and 516 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
TODO
View file

@ -74,6 +74,8 @@ for selecting what gets printed).
Add a "--count-only" (or so?) option to "notmuch search" for returning
the count of search results.
Add documented syntax for searching all threads/messages.
Give "notmuch restore" some progress indicator. Until we get the
Xapian bugs fixed that are making this operation slow, we really need
to let the user know that things are still moving.
@ -146,6 +148,10 @@ notmuch initially sees all changes as interesting, and quickly learns
from the user which changes are not interesting (such as the very
common mailing-list footer).
Fix notmuch_query_count_messages to share code with
notmuch_query_search_messages rather than duplicating code. (And
consider renaming it as well.)
General
-------
Audit everything for dealing with out-of-memory (and drop xutil.c).

4
compat/Makefile
View file

@ -1,5 +1,5 @@
all:
$(MAKE) -C .. all
clean:
$(MAKE) -C .. clean
.DEFAULT:
$(MAKE) -C .. $@

26
configure vendored
View file

@ -6,6 +6,7 @@ CC=${CC:-gcc}
CXX=${CXX:-g++}
CFLAGS=${CFLAGS:--O2}
CXXFLAGS=${CXXFLAGS:-\$(CFLAGS)}
XAPIAN_CONFIG=${XAPIAN_CONFIG:-xapian-config-1.1 xapian-config}
# Set the defaults for values the user can specify with command-line
# options.
@ -37,6 +38,13 @@ First, some common variables can specified via environment variables:
Each of these values can further be controlled by specifying them
later on the "make" command line.
Other environment variables can be used to control configure itself,
(and for which there is no equivalent build-time control):
XAPIAN_CONFIG The program to use to determine flags for
compiling and linking against the Xapian
library. [$XAPIAN_CONFIG]
Additionally, various options can be specified on the configure
command line.
@ -91,14 +99,18 @@ else
fi
printf "Checking for Xapian development files... "
if xapian-config --version > /dev/null 2>&1; then
printf "Yes.\n"
have_xapian=1
xapian_cxxflags=$(xapian-config --cxxflags)
xapian_ldflags=$(xapian-config --libs)
else
have_xapian=0
for xapian_config in ${XAPIAN_CONFIG}; do
if ${xapian_config} --version > /dev/null 2>&1; then
printf "Yes.\n"
have_xapian=1
xapian_cxxflags=$(${xapian_config} --cxxflags)
xapian_ldflags=$(${xapian_config} --libs)
break
fi
done
if [ ${have_xapian} = "0" ]; then
printf "No.\n"
have_xapian=0
errors=$((errors + 1))
fi

6
lib/Makefile
View file

@ -1,5 +1,7 @@
# See Makfefile.local for the list of files to be compiled in this
# directory.
all:
$(MAKE) -C .. all
clean:
$(MAKE) -C .. clean
.DEFAULT:
$(MAKE) -C .. $@

1
lib/Makefile.local
View file

@ -11,6 +11,7 @@ libnotmuch_c_srcs = \
libnotmuch_cxx_srcs = \
$(dir)/database.cc \
$(dir)/directory.cc \
$(dir)/index.cc \
$(dir)/message.cc \
$(dir)/query.cc \

2
lib/database-private.h
View file

@ -33,6 +33,8 @@ struct _notmuch_database {
Xapian::QueryParser *query_parser;
Xapian::TermGenerator *term_gen;
Xapian::ValueRangeProcessor *value_range_processor;
notmuch_bool_t needs_upgrade;
};
/* Convert tags from Xapian internal format to notmuch format.

785
lib/database.cc
View file

@ -22,6 +22,8 @@
#include <iostream>
#include <sys/time.h>
#include <signal.h>
#include <xapian.h>
#include <glib.h> /* g_free, GPtrArray, GHashTable */
@ -35,9 +37,14 @@ typedef struct {
const char *prefix;
} prefix_t;
/* Here's the current schema for our database:
#define NOTMUCH_DATABASE_VERSION 1
#define STRINGIFY(s) _SUB_STRINGIFY(s)
#define _SUB_STRINGIFY(s) #s
/* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
*
* We currently have two different types of documents: mail and timestamps.
* We currently have two different types of documents: mail and directory.
*
* Mail document
* -------------
@ -63,6 +70,12 @@ typedef struct {
*
* tag: Any tags associated with this message by the user.
*
* file-direntry: A colon-separated pair of values
* (INTEGER:STRING), where INTEGER is the
* document ID of a directory document, and
* STRING is the name of a file within that
* directory for this mail message.
*
* A mail document also has two values:
*
* TIMESTAMP: The time_t value corresponding to the message's
@ -75,21 +88,36 @@ typedef struct {
* user in searching. But the database doesn't really care itself
* about any of these.
*
* Timestamp document
* The data portion of a mail document is empty.
*
* Directory document
* ------------------
* A timestamp document is used by a client of the notmuch library to
* A directory document is used by a client of the notmuch library to
* maintain data necessary to allow for efficient polling of mail
* directories. The notmuch library does no interpretation of
* timestamps, but merely allows the user to store and retrieve
* timestamps as name/value pairs.
* directories.
*
* The timestamp document is indexed with a single prefixed term:
* All directory documents contain one term:
*
* timestamp: The user's key value (likely a directory name)
* directory: The directory path (relative to the database path)
* Or the SHA1 sum of the directory path (if the
* path itself is too long to fit in a Xapian
* term).
*
* and has a single value:
* And all directory documents for directories other than top-level
* directories also contain the following term:
*
* TIMESTAMP: The time_t value from the user.
* directory-direntry: A colon-separated pair of values
* (INTEGER:STRING), where INTEGER is the
* document ID of the parent directory
* document, and STRING is the name of this
* directory within that parent.
*
* All directory documents have a single value:
*
* TIMESTAMP: The mtime of the directory (at last scan)
*
* The data portion of a directory document contains the path of the
* directory (relative to the database path).
*/
/* With these prefix values we follow the conventions published here:
@ -108,23 +136,25 @@ typedef struct {
*/
prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
{ "type", "T" },
{ "reference", "XREFERENCE" },
{ "replyto", "XREPLYTO" },
{ "timestamp", "XTIMESTAMP" },
{ "type", "T" },
{ "reference", "XREFERENCE" },
{ "replyto", "XREPLYTO" },
{ "directory", "XDIRECTORY" },
{ "file-direntry", "XFDIRENTRY" },
{ "directory-direntry", "XDDIRENTRY" },
};
prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
{ "thread", "G" },
{ "tag", "K" },
{ "id", "Q" }
{ "thread", "G" },
{ "tag", "K" },
{ "id", "Q" }
};
prefix_t PROBABILISTIC_PREFIX[]= {
{ "from", "XFROM" },
{ "to", "XTO" },
{ "attachment", "XATTACHMENT" },
{ "subject", "XSUBJECT"}
{ "from", "XFROM" },
{ "to", "XTO" },
{ "attachment", "XATTACHMENT" },
{ "subject", "XSUBJECT"}
};
int
@ -175,8 +205,8 @@ notmuch_status_to_string (notmuch_status_t status)
return "No error occurred";
case NOTMUCH_STATUS_OUT_OF_MEMORY:
return "Out of memory";
case NOTMUCH_STATUS_READONLY_DATABASE:
return "The database is read-only";
case NOTMUCH_STATUS_READ_ONLY_DATABASE:
return "Attempt to write to a read-only database";
case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
return "A Xapian exception occurred";
case NOTMUCH_STATUS_FILE_ERROR:
@ -197,6 +227,17 @@ notmuch_status_to_string (notmuch_status_t status)
}
}
static void
find_doc_ids_for_term (notmuch_database_t *notmuch,
const char *term,
Xapian::PostingIterator *begin,
Xapian::PostingIterator *end)
{
*begin = notmuch->xapian_db->postlist_begin (term);
*end = notmuch->xapian_db->postlist_end (term);
}
static void
find_doc_ids (notmuch_database_t *notmuch,
const char *prefix_name,
@ -204,24 +245,21 @@ find_doc_ids (notmuch_database_t *notmuch,
Xapian::PostingIterator *begin,
Xapian::PostingIterator *end)
{
Xapian::PostingIterator i;
char *term;
term = talloc_asprintf (notmuch, "%s%s",
_find_prefix (prefix_name), value);
*begin = notmuch->xapian_db->postlist_begin (term);
*end = notmuch->xapian_db->postlist_end (term);
find_doc_ids_for_term (notmuch, term, begin, end);
talloc_free (term);
}
static notmuch_private_status_t
find_unique_doc_id (notmuch_database_t *notmuch,
const char *prefix_name,
const char *value,
unsigned int *doc_id)
notmuch_private_status_t
_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
const char *prefix_name,
const char *value,
unsigned int *doc_id)
{
Xapian::PostingIterator i, end;
@ -230,10 +268,19 @@ find_unique_doc_id (notmuch_database_t *notmuch,
if (i == end) {
*doc_id = 0;
return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
} else {
*doc_id = *i;
return NOTMUCH_PRIVATE_STATUS_SUCCESS;
}
*doc_id = *i;
#if DEBUG_DATABASE_SANITY
i++;
if (i != end)
INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
prefix_name, value);
#endif
return NOTMUCH_PRIVATE_STATUS_SUCCESS;
}
static Xapian::Document
@ -242,26 +289,6 @@ find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
return notmuch->xapian_db->get_document (doc_id);
}
static notmuch_private_status_t
find_unique_document (notmuch_database_t *notmuch,
const char *prefix_name,
const char *value,
Xapian::Document *document,
unsigned int *doc_id)
{
notmuch_private_status_t status;
status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
if (status) {
*document = Xapian::Document ();
return status;
}
*document = find_document_for_doc_id (notmuch, *doc_id);
return NOTMUCH_PRIVATE_STATUS_SUCCESS;
}
notmuch_message_t *
notmuch_database_find_message (notmuch_database_t *notmuch,
const char *message_id)
@ -269,7 +296,8 @@ notmuch_database_find_message (notmuch_database_t *notmuch,
notmuch_private_status_t status;
unsigned int doc_id;
status = find_unique_doc_id (notmuch, "id", message_id, &doc_id);
status = _notmuch_database_find_unique_doc_id (notmuch, "id",
message_id, &doc_id);
if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
return NULL;
@ -446,6 +474,7 @@ notmuch_database_create (const char *path)
notmuch = notmuch_database_open (path,
NOTMUCH_DATABASE_MODE_READ_WRITE);
notmuch_database_upgrade (notmuch, NULL, NULL);
DONE:
if (notmuch_path)
@ -454,6 +483,17 @@ notmuch_database_create (const char *path)
return notmuch;
}
notmuch_status_t
_notmuch_database_ensure_writable (notmuch_database_t *notmuch)
{
if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
fprintf (stderr, "Cannot write to a read-only database.\n");
return NOTMUCH_STATUS_READ_ONLY_DATABASE;
}
return NOTMUCH_STATUS_SUCCESS;
}
notmuch_database_t *
notmuch_database_open (const char *path,
notmuch_database_mode_t mode)
@ -462,7 +502,7 @@ notmuch_database_open (const char *path,
char *notmuch_path = NULL, *xapian_path = NULL;
struct stat st;
int err;
unsigned int i;
unsigned int i, version;
if (asprintf (&notmuch_path, "%s/%s", path, ".notmuch") == -1) {
notmuch_path = NULL;
@ -490,13 +530,42 @@ notmuch_database_open (const char *path,
if (notmuch->path[strlen (notmuch->path) - 1] == '/')
notmuch->path[strlen (notmuch->path) - 1] = '\0';
notmuch->needs_upgrade = FALSE;
notmuch->mode = mode;
try {
if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
Xapian::DB_CREATE_OR_OPEN);
version = notmuch_database_get_version (notmuch);
if (version > NOTMUCH_DATABASE_VERSION) {
fprintf (stderr,
"Error: Notmuch database at %s\n"
" has a newer database format version (%u) than supported by this\n"
" version of notmuch (%u). Refusing to open this database in\n"
" read-write mode.\n",
notmuch_path, version, NOTMUCH_DATABASE_VERSION);
notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
notmuch_database_close (notmuch);
notmuch = NULL;
goto DONE;
}
if (version < NOTMUCH_DATABASE_VERSION)
notmuch->needs_upgrade = TRUE;
} else {
notmuch->xapian_db = new Xapian::Database (xapian_path);
version = notmuch_database_get_version (notmuch);
if (version > NOTMUCH_DATABASE_VERSION)
{
fprintf (stderr,
"Warning: Notmuch database at %s\n"
" has a newer database format version (%u) than supported by this\n"
" version of notmuch (%u). Some operations may behave incorrectly,\n"
" (but the database will not be harmed since it is being opened\n"
" in read-only mode).\n",
notmuch_path, version, NOTMUCH_DATABASE_VERSION);
}
}
notmuch->query_parser = new Xapian::QueryParser;
notmuch->term_gen = new Xapian::TermGenerator;
@ -560,110 +629,469 @@ notmuch_database_get_path (notmuch_database_t *notmuch)
return notmuch->path;
}
static notmuch_private_status_t
find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
Xapian::Document *doc, unsigned int *doc_id)
unsigned int
notmuch_database_get_version (notmuch_database_t *notmuch)
{
return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
unsigned int version;
string version_string;
const char *str;
char *end;
version_string = notmuch->xapian_db->get_metadata ("version");
if (version_string.empty ())
return 0;
str = version_string.c_str ();
if (str == NULL || *str == '\0')
return 0;
version = strtoul (str, &end, 10);
if (*end != '\0')
INTERNAL_ERROR ("Malformed database version: %s", str);
return version;
}
/* We allow the user to use arbitrarily long keys for timestamps,
* (they're for filesystem paths after all, which have no limit we
* know about). But we have a term-length limit. So if we exceed that,
* we'll use the SHA-1 of the user's key as the actual key for
* constructing a database term.
*
* Caution: This function returns a newly allocated string which the
* caller should free() when finished.
*/
static char *
timestamp_db_key (const char *key)
notmuch_bool_t
notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
{
int term_len = strlen (_find_prefix ("timestamp")) + strlen (key);
return notmuch->needs_upgrade;
}
static volatile sig_atomic_t do_progress_notify = 0;
static void
handle_sigalrm (unused (int signal))
{
do_progress_notify = 1;
}
/* Upgrade the current database.
*
* After opening a database in read-write mode, the client should
* check if an upgrade is needed (notmuch_database_needs_upgrade) and
* if so, upgrade with this function before making any modifications.
*
* The optional progress_notify callback can be used by the caller to
* provide progress indication to the user. If non-NULL it will be
* called periodically with 'count' as the number of messages upgraded
* so far and 'total' the overall number of messages that will be
* converted.
*/
notmuch_status_t
notmuch_database_upgrade (notmuch_database_t *notmuch,
void (*progress_notify) (void *closure,
double progress),
void *closure)
{
Xapian::WritableDatabase *db;
struct sigaction action;
struct itimerval timerval;
notmuch_bool_t timer_is_active = FALSE;
unsigned int version;
notmuch_status_t status;
unsigned int count = 0, total = 0;
status = _notmuch_database_ensure_writable (notmuch);
if (status)
return status;
db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
version = notmuch_database_get_version (notmuch);
if (version >= NOTMUCH_DATABASE_VERSION)
return NOTMUCH_STATUS_SUCCESS;
if (progress_notify) {
/* Setup our handler for SIGALRM */
memset (&action, 0, sizeof (struct sigaction));
action.sa_handler = handle_sigalrm;
sigemptyset (&action.sa_mask);
action.sa_flags = SA_RESTART;
sigaction (SIGALRM, &action, NULL);
/* Then start a timer to send SIGALRM once per second. */
timerval.it_interval.tv_sec = 1;
timerval.it_interval.tv_usec = 0;
timerval.it_value.tv_sec = 1;
timerval.it_value.tv_usec = 0;
setitimer (ITIMER_REAL, &timerval, NULL);
timer_is_active = TRUE;
}
/* Before version 1, each message document had its filename in the
* data field. Copy that into the new format by calling
* notmuch_message_add_filename.
*/
if (version < 1) {
notmuch_query_t *query = notmuch_query_create (notmuch, "");
notmuch_messages_t *messages;
notmuch_message_t *message;
char *filename;
Xapian::TermIterator t, t_end;
total = notmuch_query_count_messages (query);
for (messages = notmuch_query_search_messages (query);
notmuch_messages_has_more (messages);
notmuch_messages_advance (messages))
{
if (do_progress_notify) {
progress_notify (closure, (double) count / total);
do_progress_notify = 0;
}
message = notmuch_messages_get (messages);
filename = _notmuch_message_talloc_copy_data (message);
if (filename && *filename != '\0') {
_notmuch_message_add_filename (message, filename);
_notmuch_message_sync (message);
}
talloc_free (filename);
notmuch_message_destroy (message);
count++;
}
notmuch_query_destroy (query);
/* Also, before version 1 we stored directory timestamps in
* XTIMESTAMP documents instead of the current XDIRECTORY
* documents. So copy those as well. */
t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
t != t_end;
t++)
{
Xapian::PostingIterator p, p_end;
std::string term = *t;
p_end = notmuch->xapian_db->postlist_end (term);
for (p = notmuch->xapian_db->postlist_begin (term);
p != p_end;
p++)
{
Xapian::Document document;
time_t mtime;
notmuch_directory_t *directory;
if (do_progress_notify) {
progress_notify (closure, (double) count / total);
do_progress_notify = 0;
}
document = find_document_for_doc_id (notmuch, *p);
mtime = Xapian::sortable_unserialise (
document.get_value (NOTMUCH_VALUE_TIMESTAMP));
directory = notmuch_database_get_directory (notmuch,
term.c_str() + 10);
notmuch_directory_set_mtime (directory, mtime);
notmuch_directory_destroy (directory);
}
}
}
db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
db->flush ();
/* Now that the upgrade is complete we can remove the old data
* and documents that are no longer needed. */
if (version < 1) {
notmuch_query_t *query = notmuch_query_create (notmuch, "");
notmuch_messages_t *messages;
notmuch_message_t *message;
char *filename;
for (messages = notmuch_query_search_messages (query);
notmuch_messages_has_more (messages);
notmuch_messages_advance (messages))
{
if (do_progress_notify) {
progress_notify (closure, (double) count / total);
do_progress_notify = 0;
}
message = notmuch_messages_get (messages);
filename = _notmuch_message_talloc_copy_data (message);
if (filename && *filename != '\0') {
_notmuch_message_clear_data (message);
_notmuch_message_sync (message);
}
talloc_free (filename);
notmuch_message_destroy (message);
}
notmuch_query_destroy (query);
}
if (version < 1) {
Xapian::TermIterator t, t_end;
t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
t != t_end;
t++)
{
Xapian::PostingIterator p, p_end;
std::string term = *t;
p_end = notmuch->xapian_db->postlist_end (term);
for (p = notmuch->xapian_db->postlist_begin (term);
p != p_end;
p++)
{
if (do_progress_notify) {
progress_notify (closure, (double) count / total);
do_progress_notify = 0;
}
db->delete_document (*p);
}
}
}
if (timer_is_active) {
/* Now stop the timer. */
timerval.it_interval.tv_sec = 0;
timerval.it_interval.tv_usec = 0;
timerval.it_value.tv_sec = 0;
timerval.it_value.tv_usec = 0;
setitimer (ITIMER_REAL, &timerval, NULL);
/* And disable the signal handler. */
action.sa_handler = SIG_IGN;
sigaction (SIGALRM, &action, NULL);
}
return NOTMUCH_STATUS_SUCCESS;
}
/* We allow the user to use arbitrarily long paths for directories. But
* we have a term-length limit. So if we exceed that, we'll use the
* SHA-1 of the path for the database term.
*
* Note: This function may return the original value of 'path'. If it
* does not, then the caller is responsible to free() the returned
* value.
*/
const char *
_notmuch_database_get_directory_db_path (const char *path)
{
int term_len = strlen (_find_prefix ("directory")) + strlen (path);
if (term_len > NOTMUCH_TERM_MAX)
return notmuch_sha1_of_string (key);
return notmuch_sha1_of_string (path);
else
return strdup (key);
return path;
}
/* Given a path, split it into two parts: the directory part is all
* components except for the last, and the basename is that last
* component. Getting the return-value for either part is optional
* (the caller can pass NULL).
*
* The original 'path' can represent either a regular file or a
* directory---the splitting will be carried out in the same way in
* either case. Trailing slashes on 'path' will be ignored, and any
* cases of multiple '/' characters appearing in series will be
* treated as a single '/'.
*
* Allocation (if any) will have 'ctx' as the talloc owner. But
* pointers will be returned within the original path string whenever
* possible.
*
* Note: If 'path' is non-empty and contains no non-trailing slash,
* (that is, consists of a filename with no parent directory), then
* the directory returned will be an empty string. However, if 'path'
* is an empty string, then both directory and basename will be
* returned as NULL.
*/
notmuch_status_t
_notmuch_database_split_path (void *ctx,
const char *path,
const char **directory,
const char **basename)
{
const char *slash;
if (path == NULL || *path == '\0') {
if (directory)
*directory = NULL;
if (basename)
*basename = NULL;
return NOTMUCH_STATUS_SUCCESS;
}
/* Find the last slash (not counting a trailing slash), if any. */
slash = path + strlen (path) - 1;
/* First, skip trailing slashes. */
while (slash != path) {
if (*slash != '/')
break;
--slash;
}
/* Then, find a slash. */
while (slash != path) {
if (*slash == '/')
break;
if (basename)
*basename = slash;
--slash;
}
/* Finally, skip multiple slashes. */
while (slash != path) {
if (*slash != '/')
break;
--slash;
}
if (slash == path) {
if (directory)
*directory = talloc_strdup (ctx, "");
if (basename)
*basename = path;
} else {
if (directory)
*directory = talloc_strndup (ctx, path, slash - path + 1);
}
return NOTMUCH_STATUS_SUCCESS;
}
notmuch_status_t
notmuch_database_set_timestamp (notmuch_database_t *notmuch,
const char *key, time_t timestamp)
_notmuch_database_find_directory_id (notmuch_database_t *notmuch,
const char *path,
unsigned int *directory_id)
{
Xapian::Document doc;
Xapian::WritableDatabase *db;
unsigned int doc_id;
notmuch_private_status_t status;
notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
char *db_key = NULL;
notmuch_directory_t *directory;
notmuch_status_t status;
if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
fprintf (stderr, "Attempted to update a read-only database.\n");
return NOTMUCH_STATUS_READONLY_DATABASE;
if (path == NULL) {
*directory_id = 0;
return NOTMUCH_STATUS_SUCCESS;
}
db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
db_key = timestamp_db_key (key);
try {
status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
Xapian::sortable_serialise (timestamp));
if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
char *term = talloc_asprintf (NULL, "%s%s",
_find_prefix ("timestamp"), db_key);
doc.add_term (term);
talloc_free (term);
db->add_document (doc);
} else {
db->replace_document (doc_id, doc);
}
} catch (const Xapian::Error &error) {
fprintf (stderr, "A Xapian exception occurred setting timestamp: %s.\n",
error.get_msg().c_str());
notmuch->exception_reported = TRUE;
ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
directory = _notmuch_directory_create (notmuch, path, &status);
if (status) {
*directory_id = -1;
return status;
}
if (db_key)
free (db_key);
*directory_id = _notmuch_directory_get_document_id (directory);
return ret;
notmuch_directory_destroy (directory);
return NOTMUCH_STATUS_SUCCESS;
}
time_t
notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
const char *
_notmuch_database_get_directory_path (void *ctx,
notmuch_database_t *notmuch,
unsigned int doc_id)
{
Xapian::Document doc;
unsigned int doc_id;
notmuch_private_status_t status;
char *db_key = NULL;
time_t ret = 0;
Xapian::Document document;
db_key = timestamp_db_key (key);
document = find_document_for_doc_id (notmuch, doc_id);
try {
status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
return talloc_strdup (ctx, document.get_data ().c_str ());
}
if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
goto DONE;
/* Given a legal 'filename' for the database, (either relative to
* database path or absolute with initial components identical to
* database path), return a new string (with 'ctx' as the talloc
* owner) suitable for use as a direntry term value.
*
* The necessary directory documents will be created in the database
* as needed.
*/
notmuch_status_t
_notmuch_database_filename_to_direntry (void *ctx,
notmuch_database_t *notmuch,
const char *filename,
char **direntry)
{
const char *relative, *directory, *basename;
Xapian::docid directory_id;
notmuch_status_t status;
ret = Xapian::sortable_unserialise (doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
} catch (Xapian::Error &error) {
ret = 0;
goto DONE;
relative = _notmuch_database_relative_path (notmuch, filename);
status = _notmuch_database_split_path (ctx, relative,
&directory, &basename);
if (status)
return status;
status = _notmuch_database_find_directory_id (notmuch, directory,
&directory_id);
if (status)
return status;
*direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
return NOTMUCH_STATUS_SUCCESS;
}
/* Given a legal 'path' for the database, return the relative path.
*
* The return value will be a pointer to the originl path contents,
* and will be either the original string (if 'path' was relative) or
* a portion of the string (if path was absolute and begins with the
* database path).
*/
const char *
_notmuch_database_relative_path (notmuch_database_t *notmuch,
const char *path)
{
const char *db_path, *relative;
unsigned int db_path_len;
db_path = notmuch_database_get_path (notmuch);
db_path_len = strlen (db_path);
relative = path;
if (*relative == '/') {
while (*relative == '/' && *(relative+1) == '/')
relative++;
if (strncmp (relative, db_path, db_path_len) == 0)
{
relative += db_path_len;
while (*relative == '/')
relative++;
}
}
DONE:
if (db_key)
free (db_key);
return relative;
}
return ret;
notmuch_directory_t *
notmuch_database_get_directory (notmuch_database_t *notmuch,
const char *path)
{
notmuch_status_t status;
return _notmuch_directory_create (notmuch, path, &status);
}
/* Find the thread ID to which the message with 'message_id' belongs.
@ -903,11 +1331,13 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (message_ret)
*message_ret = NULL;
ret = _notmuch_database_ensure_writable (notmuch);
if (ret)
return ret;
message_file = notmuch_message_file_open (filename);
if (message_file == NULL) {
ret = NOTMUCH_STATUS_FILE_ERROR;
goto DONE;
}
if (message_file == NULL)
return NOTMUCH_STATUS_FILE_ERROR;
notmuch_message_file_restrict_headers (message_file,
"date",
@ -988,24 +1418,25 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
goto DONE;
}
_notmuch_message_add_filename (message, filename);
/* Is this a newly created message object? */
if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
_notmuch_message_set_filename (message, filename);
_notmuch_message_add_term (message, "type", "mail");
ret = _notmuch_database_link_message (notmuch, message,
message_file);
if (ret)
goto DONE;
date = notmuch_message_file_get_header (message_file, "date");
_notmuch_message_set_date (message, date);
_notmuch_message_index_file (message, filename);
} else {
ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
goto DONE;
}
ret = _notmuch_database_link_message (notmuch, message, message_file);
if (ret)
goto DONE;
date = notmuch_message_file_get_header (message_file, "date");
_notmuch_message_set_date (message, date);
_notmuch_message_index_file (message, filename);
_notmuch_message_sync (message);
} catch (const Xapian::Error &error) {
fprintf (stderr, "A Xapian exception occurred adding message: %s.\n",
@ -1029,6 +1460,60 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
return ret;
}
notmuch_status_t
notmuch_database_remove_message (notmuch_database_t *notmuch,
const char *filename)
{
Xapian::WritableDatabase *db;
void *local = talloc_new (notmuch);
const char *prefix = _find_prefix ("file-direntry");
char *direntry, *term;
Xapian::PostingIterator i, end;
Xapian::Document document;
notmuch_status_t status;
status = _notmuch_database_ensure_writable (notmuch);
if (status)
return status;
db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
status = _notmuch_database_filename_to_direntry (local, notmuch,
filename, &direntry);
if (status)
return status;
term = talloc_asprintf (notmuch, "%s%s", prefix, direntry);
find_doc_ids_for_term (notmuch, term, &i, &end);
for ( ; i != end; i++) {
Xapian::TermIterator j;
document = find_document_for_doc_id (notmuch, *i);
document.remove_term (term);
j = document.termlist_begin ();
j.skip_to (prefix);
/* Was this the last file-direntry in the message? */
if (j == document.termlist_end () ||
strncmp ((*j).c_str (), prefix, strlen (prefix)))
{
db->delete_document (document.get_docid ());
status = NOTMUCH_STATUS_SUCCESS;
} else {
db->replace_document (document.get_docid (), document);
status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
}
}
talloc_free (local);
return status;
}
notmuch_tags_t *
_notmuch_convert_tags (void *ctx, Xapian::TermIterator &i,
Xapian::TermIterator &end)

338
lib/directory.cc Normal file
View file

@ -0,0 +1,338 @@
/* directory.cc - Results of directory-based searches from a notmuch database
*
* Copyright © 2009 Carl Worth
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/ .
*
* Author: Carl Worth <cworth@cworth.org>
*/
#include "notmuch-private.h"
#include "database-private.h"
#include <xapian.h>
struct _notmuch_filenames {
Xapian::TermIterator iterator;
Xapian::TermIterator end;
int prefix_len;
char *filename;
};
/* We end up having to call the destructors explicitly because we had
* to use "placement new" in order to initialize C++ objects within a
* block that we allocated with talloc. So C++ is making talloc
* slightly less simple to use, (we wouldn't need
* talloc_set_destructor at all otherwise).
*/
static int
_notmuch_filenames_destructor (notmuch_filenames_t *filenames)
{
filenames->iterator.~TermIterator ();
filenames->end.~TermIterator ();
return 0;
}
/* Create an iterator to iterate over the basenames of files (or
* directories) that all share a common parent directory.
*
* The code here is general enough to be reused for any case of
* iterating over the non-prefixed portion of terms sharing a common
* prefix.
*/
notmuch_filenames_t *
_notmuch_filenames_create (void *ctx,
notmuch_database_t *notmuch,
const char *prefix)
{
notmuch_filenames_t *filenames;
filenames = talloc (ctx, notmuch_filenames_t);
if (unlikely (filenames == NULL))
return NULL;
new (&filenames->iterator) Xapian::TermIterator ();
new (&filenames->end) Xapian::TermIterator ();
talloc_set_destructor (filenames, _notmuch_filenames_destructor);
filenames->iterator = notmuch->xapian_db->allterms_begin (prefix);
filenames->end = notmuch->xapian_db->allterms_end (prefix);
filenames->prefix_len = strlen (prefix);
filenames->filename = NULL;
return filenames;
}
notmuch_bool_t
notmuch_filenames_has_more (notmuch_filenames_t *filenames)
{
if (filenames == NULL)
return NULL;
return (filenames->iterator != filenames->end);
}
const char *
notmuch_filenames_get (notmuch_filenames_t *filenames)
{
if (filenames == NULL || filenames->iterator == filenames->end)
return NULL;
if (filenames->filename == NULL) {
std::string term = *filenames->iterator;
filenames->filename = talloc_strdup (filenames,
term.c_str () +
filenames->prefix_len);
}
return filenames->filename;
}
void
notmuch_filenames_advance (notmuch_filenames_t *filenames)
{
if (filenames == NULL)
return;
if (filenames->filename) {
talloc_free (filenames->filename);
filenames->filename = NULL;
}
if (filenames->iterator != filenames->end)
filenames->iterator++;
}
void
notmuch_filenames_destroy (notmuch_filenames_t *filenames)
{
if (filenames == NULL)
return;
talloc_free (filenames);
}
struct _notmuch_directory {
notmuch_database_t *notmuch;
Xapian::docid document_id;
Xapian::Document doc;
time_t mtime;
};
/* We end up having to call the destructor explicitly because we had
* to use "placement new" in order to initialize C++ objects within a
* block that we allocated with talloc. So C++ is making talloc
* slightly less simple to use, (we wouldn't need
* talloc_set_destructor at all otherwise).
*/
static int
_notmuch_directory_destructor (notmuch_directory_t *directory)
{
directory->doc.~Document ();
return 0;
}
static notmuch_private_status_t
find_directory_document (notmuch_database_t *notmuch,
const char *db_path,
Xapian::Document *document)
{
notmuch_private_status_t status;
Xapian::docid doc_id;
status = _notmuch_database_find_unique_doc_id (notmuch, "directory",
db_path, &doc_id);
if (status) {
*document = Xapian::Document ();
return status;
}
*document = notmuch->xapian_db->get_document (doc_id);
return NOTMUCH_PRIVATE_STATUS_SUCCESS;
}
notmuch_directory_t *
_notmuch_directory_create (notmuch_database_t *notmuch,
const char *path,
notmuch_status_t *status_ret)
{
Xapian::WritableDatabase *db;
notmuch_directory_t *directory;
notmuch_private_status_t private_status;
const char *db_path;
*status_ret = NOTMUCH_STATUS_SUCCESS;
path = _notmuch_database_relative_path (notmuch, path);
if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
INTERNAL_ERROR ("Failure to ensure database is writable");
db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
directory = talloc (notmuch, notmuch_directory_t);
if (unlikely (directory == NULL))
return NULL;
directory->notmuch = notmuch;
/* "placement new"---not actually allocating memory */
new (&directory->doc) Xapian::Document;
talloc_set_destructor (directory, _notmuch_directory_destructor);
db_path = _notmuch_database_get_directory_db_path (path);
try {
Xapian::TermIterator i, end;
private_status = find_directory_document (notmuch, db_path,
&directory->doc);
directory->document_id = directory->doc.get_docid ();
if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
void *local = talloc_new (directory);
const char *parent, *basename;
Xapian::docid parent_id;
char *term = talloc_asprintf (local, "%s%s",
_find_prefix ("directory"), db_path);
directory->doc.add_term (term, 0);
directory->doc.set_data (path);
_notmuch_database_split_path (local, path, &parent, &basename);
_notmuch_database_find_directory_id (notmuch, parent, &parent_id);
if (basename) {
term = talloc_asprintf (local, "%s%u:%s",
_find_prefix ("directory-direntry"),
parent_id, basename);
directory->doc.add_term (term, 0);
}
directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
Xapian::sortable_serialise (0));
directory->document_id = db->add_document (directory->doc);
talloc_free (local);
}
directory->mtime = Xapian::sortable_unserialise (
directory->doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
} catch (const Xapian::Error &error) {
fprintf (stderr,
"A Xapian exception occurred creating a directory: %s.\n",
error.get_msg().c_str());
notmuch->exception_reported = TRUE;
notmuch_directory_destroy (directory);
*status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
return NULL;
}
if (db_path != path)
free ((char *) db_path);
return directory;
}
unsigned int
_notmuch_directory_get_document_id (notmuch_directory_t *directory)
{
return directory->document_id;
}
notmuch_status_t
notmuch_directory_set_mtime (notmuch_directory_t *directory,
time_t mtime)
{
notmuch_database_t *notmuch = directory->notmuch;
Xapian::WritableDatabase *db;
notmuch_status_t status;
status = _notmuch_database_ensure_writable (notmuch);
if (status)
return status;
db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
try {
directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
Xapian::sortable_serialise (mtime));
db->replace_document (directory->document_id, directory->doc);
} catch (const Xapian::Error &error) {
fprintf (stderr,
"A Xapian exception occurred setting directory mtime: %s.\n",
error.get_msg().c_str());
notmuch->exception_reported = TRUE;
return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
}
return NOTMUCH_STATUS_SUCCESS;
}
time_t
notmuch_directory_get_mtime (notmuch_directory_t *directory)
{
return directory->mtime;
}
notmuch_filenames_t *
notmuch_directory_get_child_files (notmuch_directory_t *directory)
{
char *term;
notmuch_filenames_t *child_files;
term = talloc_asprintf (directory, "%s%u:",
_find_prefix ("file-direntry"),
directory->document_id);
child_files = _notmuch_filenames_create (directory,
directory->notmuch, term);
talloc_free (term);
return child_files;
}
notmuch_filenames_t *
notmuch_directory_get_child_directories (notmuch_directory_t *directory)
{
char *term;
notmuch_filenames_t *child_directories;
term = talloc_asprintf (directory, "%s%u:",
_find_prefix ("directory-direntry"),
directory->document_id);
child_directories = _notmuch_filenames_create (directory,
directory->notmuch, term);
talloc_free (term);
return child_directories;
}
void
notmuch_directory_destroy (notmuch_directory_t *directory)
{
talloc_free (directory);
}

62
lib/index.cc
View file

@ -31,7 +31,7 @@ _index_address_mailbox (notmuch_message_t *message,
{
InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
const char *name, *addr;
void *local = talloc_new (NULL);
void *local = talloc_new (message);
name = internet_address_get_name (address);
addr = internet_address_mailbox_get_addr (mailbox);
@ -123,60 +123,6 @@ skip_re_in_subject (const char *subject)
return s;
}
/* Given a string representing the body of a message, generate terms
* for it, (skipping quoted portions and signatures).
*
* This function is evil in that it modifies the string passed to it,
* (changing some newlines into '\0').
*/
static void
_index_body_text (notmuch_message_t *message, char *body)
{
char *line, *line_end, *next_line;
if (body == NULL)
return;
next_line = body;
while (1) {
line = next_line;
if (*line == '\0')
break;
next_line = strchr (line, '\n');
if (next_line == NULL) {
next_line = line + strlen (line);
}
line_end = next_line - 1;
/* Get to the next non-blank line. */
while (*next_line == '\n')
next_line++;
/* Skip blank lines. */
if (line_end < line)
continue;
/* Skip lines that are quotes. */
if (*line == '>')
continue;
/* Also skip lines introducing a quote on the next line. */
if (*line_end == ':' && *next_line == '>')
continue;
/* Finally, bail as soon as we see a signature. */
/* XXX: Should only do this if "near" the end of the message. */
if (strncmp (line, "-- ", 3) == 0)
break;
*(line_end + 1) = '\0';
_notmuch_message_gen_terms (message, NULL, line);
}
}
/* Callback to generate terms for each mime part of a message. */
static void
_index_mime_part (notmuch_message_t *message,
@ -249,9 +195,11 @@ _index_mime_part (notmuch_message_t *message,
g_byte_array_append (byte_array, (guint8 *) "\0", 1);
body = (char *) g_byte_array_free (byte_array, FALSE);
_index_body_text (message, body);
if (body) {
_notmuch_message_gen_terms (message, NULL, body);
free (body);
free (body);
}
}
notmuch_status_t

181
lib/message.cc
View file

@ -174,11 +174,6 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
unsigned int doc_id;
char *term;
if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
*status_ret = NOTMUCH_PRIVATE_STATUS_READONLY_DATABASE;
return NULL;
}
*status_ret = NOTMUCH_PRIVATE_STATUS_SUCCESS;
message = notmuch_database_find_message (notmuch, message_id);
@ -192,9 +187,12 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
return NULL;
}
if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
INTERNAL_ERROR ("Failure to ensure database is writable.");
db = static_cast<Xapian::WritableDatabase *> (notmuch->xapian_db);
try {
doc.add_term (term);
doc.add_term (term, 0);
talloc_free (term);
doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
@ -385,20 +383,17 @@ notmuch_message_get_replies (notmuch_message_t *message)
return _notmuch_messages_create (message->replies);
}
/* Set the filename for 'message' to 'filename'.
*
* XXX: We should still figure out if we think it's important to store
* multiple filenames for email messages with identical message IDs.
/* Add an additional 'filename' for 'message'.
*
* This change will not be reflected in the database until the next
* call to _notmuch_message_set_sync. */
void
_notmuch_message_set_filename (notmuch_message_t *message,
notmuch_status_t
_notmuch_message_add_filename (notmuch_message_t *message,
const char *filename)
{
const char *s;
const char *db_path;
unsigned int db_path_len;
notmuch_status_t status;
void *local = talloc_new (message);
char *direntry;
if (message->filename) {
talloc_free (message->filename);
@ -408,41 +403,98 @@ _notmuch_message_set_filename (notmuch_message_t *message,
if (filename == NULL)
INTERNAL_ERROR ("Message filename cannot be NULL.");
s = filename;
status = _notmuch_database_filename_to_direntry (local,
message->notmuch,
filename, &direntry);
if (status)
return status;
db_path = notmuch_database_get_path (message->notmuch);
db_path_len = strlen (db_path);
_notmuch_message_add_term (message, "file-direntry", direntry);
if (*s == '/' && strlen (s) > db_path_len
&& strncmp (s, db_path, db_path_len) == 0)
{
s += db_path_len;
while (*s == '/') s++;
talloc_free (local);
if (!*s)
INTERNAL_ERROR ("Message filename was same as db prefix.");
}
return NOTMUCH_STATUS_SUCCESS;
}
message->doc.set_data (s);
char *
_notmuch_message_talloc_copy_data (notmuch_message_t *message)
{
return talloc_strdup (message, message->doc.get_data ().c_str ());
}
void
_notmuch_message_clear_data (notmuch_message_t *message)
{
message->doc.set_data ("");
}
const char *
notmuch_message_get_filename (notmuch_message_t *message)
{
std::string filename_str;
const char *db_path;
const char *prefix = _find_prefix ("file-direntry");
int prefix_len = strlen (prefix);
Xapian::TermIterator i;
char *direntry, *colon;
const char *db_path, *directory, *basename;
unsigned int directory_id;
void *local = talloc_new (message);
if (message->filename)
return message->filename;
filename_str = message->doc.get_data ();
i = message->doc.termlist_begin ();
i.skip_to (prefix);
if (i != message->doc.termlist_end ())
direntry = talloc_strdup (local, (*i).c_str ());
if (i == message->doc.termlist_end () ||
strncmp (direntry, prefix, prefix_len))
{
/* A message document created by an old version of notmuch
* (prior to rename support) will have the filename in the
* data of the document rather than as a file-direntry term.
*
* It would be nice to do the upgrade of the document directly
* here, but the database is likely open in read-only mode. */
const char *data;
data = message->doc.get_data ().c_str ();
if (data == NULL)
INTERNAL_ERROR ("message with no filename");
message->filename = talloc_strdup (message, data);
return message->filename;
}
direntry += prefix_len;
directory_id = strtol (direntry, &colon, 10);
if (colon == NULL || *colon != ':')
INTERNAL_ERROR ("malformed direntry");
basename = colon + 1;
*colon = '\0';
db_path = notmuch_database_get_path (message->notmuch);
if (filename_str[0] != '/')
message->filename = talloc_asprintf (message, "%s/%s", db_path,
filename_str.c_str ());
directory = _notmuch_database_get_directory_path (local,
message->notmuch,
directory_id);
if (strlen (directory))
message->filename = talloc_asprintf (message, "%s/%s/%s",
db_path, directory, basename);
else
message->filename = talloc_strdup (message, filename_str.c_str ());
message->filename = talloc_asprintf (message, "%s/%s",
db_path, basename);
talloc_free ((void *) directory);
talloc_free (local);
return message->filename;
}
@ -594,7 +646,7 @@ _notmuch_message_add_term (notmuch_message_t *message,
if (strlen (term) > NOTMUCH_TERM_MAX)
return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
message->doc.add_term (term);
message->doc.add_term (term, 0);
talloc_free (term);
@ -667,7 +719,12 @@ _notmuch_message_remove_term (notmuch_message_t *message,
notmuch_status_t
notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
{
notmuch_private_status_t status;
notmuch_private_status_t private_status;
notmuch_status_t status;
status = _notmuch_database_ensure_writable (message->notmuch);
if (status)
return status;
if (tag == NULL)
return NOTMUCH_STATUS_NULL_POINTER;
@ -675,10 +732,10 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
if (strlen (tag) > NOTMUCH_TAG_MAX)
return NOTMUCH_STATUS_TAG_TOO_LONG;
status = _notmuch_message_add_term (message, "tag", tag);
if (status) {
private_status = _notmuch_message_add_term (message, "tag", tag);
if (private_status) {
INTERNAL_ERROR ("_notmuch_message_add_term return unexpected value: %d\n",
status);
private_status);
}
if (! message->frozen)
@ -690,7 +747,12 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
notmuch_status_t
notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
{
notmuch_private_status_t status;
notmuch_private_status_t private_status;
notmuch_status_t status;
status = _notmuch_database_ensure_writable (message->notmuch);
if (status)
return status;
if (tag == NULL)
return NOTMUCH_STATUS_NULL_POINTER;
@ -698,10 +760,10 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
if (strlen (tag) > NOTMUCH_TAG_MAX)
return NOTMUCH_STATUS_TAG_TOO_LONG;
status = _notmuch_message_remove_term (message, "tag", tag);
if (status) {
private_status = _notmuch_message_remove_term (message, "tag", tag);
if (private_status) {
INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
status);
private_status);
}
if (! message->frozen)
@ -710,39 +772,60 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
return NOTMUCH_STATUS_SUCCESS;
}
void
notmuch_status_t
notmuch_message_remove_all_tags (notmuch_message_t *message)
{
notmuch_private_status_t status;
notmuch_private_status_t private_status;
notmuch_status_t status;
notmuch_tags_t *tags;
const char *tag;
status = _notmuch_database_ensure_writable (message->notmuch);
if (status)
return status;
for (tags = notmuch_message_get_tags (message);
notmuch_tags_has_more (tags);
notmuch_tags_advance (tags))
{
tag = notmuch_tags_get (tags);
status = _notmuch_message_remove_term (message, "tag", tag);
if (status) {
private_status = _notmuch_message_remove_term (message, "tag", tag);
if (private_status) {
INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
status);
private_status);
}
}
if (! message->frozen)
_notmuch_message_sync (message);
return NOTMUCH_STATUS_SUCCESS;
}
void
notmuch_status_t
notmuch_message_freeze (notmuch_message_t *message)
{
notmuch_status_t status;
status = _notmuch_database_ensure_writable (message->notmuch);
if (status)
return status;
message->frozen++;
return NOTMUCH_STATUS_SUCCESS;
}
notmuch_status_t
notmuch_message_thaw (notmuch_message_t *message)
{
notmuch_status_t status;
status = _notmuch_database_ensure_writable (message->notmuch);
if (status)
return status;
if (message->frozen > 0) {
message->frozen--;
if (message->frozen == 0)

74
lib/notmuch-private.h
View file

@ -112,14 +112,15 @@ typedef enum _notmuch_private_status {
/* First, copy all the public status values. */
NOTMUCH_PRIVATE_STATUS_SUCCESS = NOTMUCH_STATUS_SUCCESS,
NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY = NOTMUCH_STATUS_OUT_OF_MEMORY,
NOTMUCH_PRIVATE_STATUS_READONLY_DATABASE = NOTMUCH_STATUS_READONLY_DATABASE,
NOTMUCH_PRIVATE_STATUS_READ_ONLY_DATABASE = NOTMUCH_STATUS_READ_ONLY_DATABASE,
NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION = NOTMUCH_STATUS_XAPIAN_EXCEPTION,
NOTMUCH_PRIVATE_STATUS_FILE_NOT_EMAIL = NOTMUCH_STATUS_FILE_NOT_EMAIL,
NOTMUCH_PRIVATE_STATUS_NULL_POINTER = NOTMUCH_STATUS_NULL_POINTER,
NOTMUCH_PRIVATE_STATUS_TAG_TOO_LONG = NOTMUCH_STATUS_TAG_TOO_LONG,
NOTMUCH_PRIVATE_STATUS_UNBALANCED_FREEZE_THAW = NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
/* Then add our own private values. */
NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG,
NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG = NOTMUCH_STATUS_LAST_STATUS,
NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND,
NOTMUCH_PRIVATE_STATUS_LAST_STATUS
@ -150,6 +151,54 @@ typedef enum _notmuch_private_status {
const char *
_find_prefix (const char *name);
notmuch_status_t
_notmuch_database_ensure_writable (notmuch_database_t *notmuch);
const char *
_notmuch_database_relative_path (notmuch_database_t *notmuch,
const char *path);
notmuch_status_t
_notmuch_database_split_path (void *ctx,
const char *path,
const char **directory,
const char **basename);
const char *
_notmuch_database_get_directory_db_path (const char *path);
notmuch_private_status_t
_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
const char *prefix_name,
const char *value,
unsigned int *doc_id);
notmuch_status_t
_notmuch_database_find_directory_id (notmuch_database_t *database,
const char *path,
unsigned int *directory_id);
const char *
_notmuch_database_get_directory_path (void *ctx,
notmuch_database_t *notmuch,
unsigned int doc_id);
notmuch_status_t
_notmuch_database_filename_to_direntry (void *ctx,
notmuch_database_t *notmuch,
const char *filename,
char **direntry);
/* directory.cc */
notmuch_directory_t *
_notmuch_directory_create (notmuch_database_t *notmuch,
const char *path,
notmuch_status_t *status_ret);
unsigned int
_notmuch_directory_get_document_id (notmuch_directory_t *directory);
/* thread.cc */
notmuch_thread_t *
@ -190,7 +239,10 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
const char *text);
void
_notmuch_message_set_filename (notmuch_message_t *message,
_notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
notmuch_status_t
_notmuch_message_add_filename (notmuch_message_t *message,
const char *filename);
void
@ -206,6 +258,22 @@ _notmuch_message_sync (notmuch_message_t *message);
void
_notmuch_message_close (notmuch_message_t *message);
/* Get a copy of the data in this message document.
*
* Caller should talloc_free the result when done.
*
* This function is intended to support database upgrade and really
* shouldn't be used otherwise. */
char *
_notmuch_message_talloc_copy_data (notmuch_message_t *message);
/* Clear the data in this message document.
*
* This function is intended to support database upgrade and really
* shouldn't be used otherwise. */
void
_notmuch_message_clear_data (notmuch_message_t *message);
/* index.cc */
notmuch_status_t

255
lib/notmuch.h
View file

@ -57,6 +57,9 @@ typedef int notmuch_bool_t;
* value. Instead we should map to things like DATABASE_LOCKED or
* whatever.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: An attempt was made to write to
* a database opened in read-only mode.
*
* NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
*
* NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to read or
@ -86,7 +89,7 @@ typedef int notmuch_bool_t;
typedef enum _notmuch_status {
NOTMUCH_STATUS_SUCCESS = 0,
NOTMUCH_STATUS_OUT_OF_MEMORY,
NOTMUCH_STATUS_READONLY_DATABASE,
NOTMUCH_STATUS_READ_ONLY_DATABASE,
NOTMUCH_STATUS_XAPIAN_EXCEPTION,
NOTMUCH_STATUS_FILE_ERROR,
NOTMUCH_STATUS_FILE_NOT_EMAIL,
@ -114,6 +117,8 @@ typedef struct _notmuch_thread notmuch_thread_t;
typedef struct _notmuch_messages notmuch_messages_t;
typedef struct _notmuch_message notmuch_message_t;
typedef struct _notmuch_tags notmuch_tags_t;
typedef struct _notmuch_directory notmuch_directory_t;
typedef struct _notmuch_filenames notmuch_filenames_t;
/* Create a new, empty notmuch database located at 'path'.
*
@ -178,56 +183,46 @@ notmuch_database_close (notmuch_database_t *database);
const char *
notmuch_database_get_path (notmuch_database_t *database);
/* Store a timestamp within the database.
/* Return the database format version of the given database. */
unsigned int
notmuch_database_get_version (notmuch_database_t *database);
/* Does this database need to be upgraded before writing to it?
*
* The Notmuch database will not interpret this key nor the timestamp
* values at all. It will merely store them together and return the
* timestamp when notmuch_database_get_timestamp is called with the
* same value for 'key'.
* If this function returns TRUE then no functions that modify the
* database (notmuch_database_add_message, notmuch_message_add_tag,
* notmuch_directory_set_mtime, etc.) will work unless the function
* notmuch_database_upgrade is called successfully first. */
notmuch_bool_t
notmuch_database_needs_upgrade (notmuch_database_t *database);
/* Upgrade the current database.
*
* The intention is for the caller to use the timestamp to allow
* efficient identification of new messages to be added to the
* database. The recommended usage is as follows:
* After opening a database in read-write mode, the client should
* check if an upgrade is needed (notmuch_database_needs_upgrade) and
* if so, upgrade with this function before making any modifications.
*
* o Read the mtime of a directory from the filesystem
*
* o Call add_message for all mail files in the directory
*
* o Call notmuch_database_set_timestamp with the path of the
* directory as 'key' and the originally read mtime as 'value'.
*
* Then, when wanting to check for updates to the directory in the
* future, the client can call notmuch_database_get_timestamp and know
* that it only needs to add files if the mtime of the directory and
* files are newer than the stored timestamp.
*
* Note: The notmuch_database_get_timestamp function does not allow
* the caller to distinguish a timestamp of 0 from a non-existent
* timestamp. So don't store a timestamp of 0 unless you are
* comfortable with that.
*
* Return value:
*
* NOTMUCH_STATUS_SUCCESS: Timestamp successfully stored in database.
*
* NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
* occurred. Timestamp not stored.
* The optional progress_notify callback can be used by the caller to
* provide progress indication to the user. If non-NULL it will be
* called periodically with 'progress' as a floating-point value in
* the range of [0.0 .. 1.0] indicating the progress made so far in
* the upgrade process.
*/
notmuch_status_t
notmuch_database_set_timestamp (notmuch_database_t *database,
const char *key, time_t timestamp);
notmuch_database_upgrade (notmuch_database_t *database,
void (*progress_notify) (void *closure,
double progress),
void *closure);
/* Retrieve a timestamp from the database.
/* Retrieve a directory object from the database for 'path'.
*
* Returns the timestamp value previously stored by calling
* notmuch_database_set_timestamp with the same value for 'key'.
*
* Returns 0 if no timestamp is stored for 'key' or if any error
* occurred querying the database.
* Here, 'path' should be a path relative to the path of 'database'
* (see notmuch_database_get_path), or else should be an absolute path
* with initial components that match the path of 'database'.
*/
time_t
notmuch_database_get_timestamp (notmuch_database_t *database,
const char *key);
notmuch_directory_t *
notmuch_database_get_directory (notmuch_database_t *database,
const char *path);
/* Add a new message to the given notmuch database.
*
@ -252,8 +247,8 @@ notmuch_database_get_timestamp (notmuch_database_t *database,
* NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
*
* NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
* ID as another message already in the database. Nothing added
* to the database.
* ID as another message already in the database. The new filename
* was successfully added to the message in the database.
*
* NOTMUCH_STATUS_FILE_ERROR: an error occurred trying to open the
* file, (such as permission denied, or file not found,
@ -261,12 +256,40 @@ notmuch_database_get_timestamp (notmuch_database_t *database,
*
* NOTMUCH_STATUS_FILE_NOT_EMAIL: the contents of filename don't look
* like an email message. Nothing added to the database.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so no message can be added.
*/
notmuch_status_t
notmuch_database_add_message (notmuch_database_t *database,
const char *filename,
notmuch_message_t **message);
/* Remove a message from the given notmuch database.
*
* Note that only this particular filename association is removed from
* the database. If the same message (as determined by the message ID)
* is still available via other filenames, then the message will
* persist in the database for those filenames. When the last filename
* is removed for a particular message, the database content for that
* message will be entirely removed.
*
* Return value:
*
* NOTMUCH_STATUS_SUCCESS: The last filename was removed and the
* message was removed from the database.
*
* NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: This filename was removed but
* the message persists in the database with at least one other
* filename.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so no message can be removed.
*/
notmuch_status_t
notmuch_database_remove_message (notmuch_database_t *database,
const char *filename);
/* Find a message with the given message_id.
*
* If the database contains a message with the given message_id, then
@ -698,14 +721,20 @@ notmuch_message_get_thread_id (notmuch_message_t *message);
notmuch_messages_t *
notmuch_message_get_replies (notmuch_message_t *message);
/* Get the filename for the email corresponding to 'message'.
/* Get a filename for the email corresponding to 'message'.
*
* The returned filename is an absolute filename, (the initial
* component will match notmuch_database_get_path() ).
*
* The returned string belongs to the message so should not be
* modified or freed by the caller (nor should it be referenced after
* the message is destroyed). */
* the message is destroyed).
*
* Note: If this message corresponds to multiple files in the mail
* store, (that is, multiple files contain identical message IDs),
* this function will arbitrarily return a single one of those
* filenames.
*/
const char *
notmuch_message_get_filename (notmuch_message_t *message);
@ -793,6 +822,9 @@ notmuch_message_get_tags (notmuch_message_t *message);
*
* NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
* (exceeds NOTMUCH_TAG_MAX)
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so message cannot be modified.
*/
notmuch_status_t
notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
@ -807,6 +839,9 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
*
* NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
* (exceeds NOTMUCH_TAG_MAX)
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so message cannot be modified.
*/
notmuch_status_t
notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
@ -815,8 +850,11 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
*
* See notmuch_message_freeze for an example showing how to safely
* replace tag values.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so message cannot be modified.
*/
void
notmuch_status_t
notmuch_message_remove_all_tags (notmuch_message_t *message);
/* Freeze the current state of 'message' within the database.
@ -851,8 +889,15 @@ notmuch_message_remove_all_tags (notmuch_message_t *message);
* somehow getting interrupted. This could result in the message being
* left with no tags if the interruption happened after
* notmuch_message_remove_all_tags but before notmuch_message_add_tag.
*
* Return value:
*
* NOTMUCH_STATUS_SUCCESS: Message successfully frozen.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so message cannot be modified.
*/
void
notmuch_status_t
notmuch_message_freeze (notmuch_message_t *message);
/* Thaw the current 'message', synchronizing any changes that may have
@ -929,6 +974,118 @@ notmuch_tags_advance (notmuch_tags_t *tags);
void
notmuch_tags_destroy (notmuch_tags_t *tags);
/* Store an mtime within the database for 'directory'.
*
* The 'directory' should be an object retrieved from the database
* with notmuch_database_get_directory for a particular path.
*
* The intention is for the caller to use the mtime to allow efficient
* identification of new messages to be added to the database. The
* recommended usage is as follows:
*
* o Read the mtime of a directory from the filesystem
*
* o Call add_message for all mail files in the directory
*
* o Call notmuch_directory_set_mtime with the mtime read from the
* filesystem.
*
* Then, when wanting to check for updates to the directory in the
* future, the client can call notmuch_directory_get_mtime and know
* that it only needs to add files if the mtime of the directory and
* files are newer than the stored timestamp.
*
* Note: The notmuch_directory_get_mtime function does not allow the
* caller to distinguish a timestamp of 0 from a non-existent
* timestamp. So don't store a timestamp of 0 unless you are
* comfortable with that.
*
* Return value:
*
* NOTMUCH_STATUS_SUCCESS: mtime successfully stored in database.
*
* NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
* occurred, mtime not stored.
*
* NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
* mode so directory mtime cannot be modified.
*/
notmuch_status_t
notmuch_directory_set_mtime (notmuch_directory_t *directory,
time_t mtime);
/* Get the mtime of a directory, (as previously stored with
* notmuch_directory_set_mtime).
*
* Returns 0 if no mtime has previously been stored for this
* directory.*/
time_t
notmuch_directory_get_mtime (notmuch_directory_t *directory);
/* Get a notmuch_filenames_t iterator listing all the filenames of
* messages in the database within the given directory.
*
* The returned filenames will be the basename-entries only (not
* complete paths). */
notmuch_filenames_t *
notmuch_directory_get_child_files (notmuch_directory_t *directory);
/* Get a notmuch_filenams_t iterator listing all the filenames of
* sub-directories in the database within the given directory.
*
* The returned filenames will be the basename-entries only (not
* complete paths). */
notmuch_filenames_t *
notmuch_directory_get_child_directories (notmuch_directory_t *directory);
/* Destroy a notmuch_directory_t object. */
void
notmuch_directory_destroy (notmuch_directory_t *directory);
/* Does the given notmuch_filenames_t object contain any more
* filenames.
*
* When this function returns TRUE, notmuch_filenames_get will return
* a valid string. Whereas when this function returns FALSE,
* notmuch_filenames_get will return NULL.
*
* It is acceptable to pass NULL for 'filenames', in which case this
* function will always return FALSE.
*/
notmuch_bool_t
notmuch_filenames_has_more (notmuch_filenames_t *filenames);
/* Get the current filename from 'filenames' as a string.
*
* Note: The returned string belongs to 'filenames' and has a lifetime
* identical to it (and the directory to which it ultimately belongs).
*
* It is acceptable to pass NULL for 'filenames', in which case this
* function will always return NULL.
*/
const char *
notmuch_filenames_get (notmuch_filenames_t *filenames);
/* Advance the 'filenames' iterator to the next filename.
*
* It is acceptable to pass NULL for 'filenames', in which case this
* function will do nothing.
*/
void
notmuch_filenames_advance (notmuch_filenames_t *filenames);
/* Destroy a notmuch_filenames_t object.
*
* It's not strictly necessary to call this function. All memory from
* the notmuch_filenames_t object will be reclaimed when the
* containing directory object is destroyed.
*
* It is acceptable to pass NULL for 'filenames', in which case this
* function will do nothing.
*/
void
notmuch_filenames_destroy (notmuch_filenames_t *filenames);
NOTMUCH_END_DECLS
#endif

2
lib/query.cc
View file

@ -134,6 +134,8 @@ notmuch_query_search_messages (notmuch_query_t *query)
mail_query, string_query);
}
enquire.set_weighting_scheme (Xapian::BoolWeight());
switch (query->sort) {
case NOTMUCH_SORT_OLDEST_FIRST:
enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE);

20
notmuch-client.h
View file

@ -70,22 +70,6 @@
#define STRNCMP_LITERAL(var, literal) \
strncmp ((var), (literal), sizeof (literal) - 1)
typedef void (*add_files_callback_t) (notmuch_message_t *message);
typedef struct {
int ignore_read_only_directories;
int saw_read_only_directory;
int output_is_a_tty;
int verbose;
int total_files;
int processed_files;
int added_messages;
struct timeval tv_start;
add_files_callback_t callback;
} add_files_state_t;
static inline void
chomp_newline (char *str)
{
@ -132,10 +116,6 @@ notmuch_time_print_formatted_seconds (double seconds);
double
notmuch_time_elapsed (struct timeval start, struct timeval end);
notmuch_status_t
add_files (notmuch_database_t *notmuch, const char *path,
add_files_state_t *state);
char *
query_string_from_args (void *ctx, int argc, char *argv[]);

622
notmuch-new.c
View file

@ -22,6 +22,29 @@
#include <unistd.h>
typedef struct _filename_node {
char *filename;
struct _filename_node *next;
} _filename_node_t;
typedef struct _filename_list {
_filename_node_t *head;
_filename_node_t **tail;
} _filename_list_t;
typedef struct {
int output_is_a_tty;
int verbose;
int total_files;
int processed_files;
int added_messages;
struct timeval tv_start;
_filename_list_t *removed_files;
_filename_list_t *removed_directories;
} add_files_state_t;
static volatile sig_atomic_t do_add_files_print_progress = 0;
static void
@ -42,6 +65,34 @@ handle_sigint (unused (int sig))
interrupted = 1;
}
static _filename_list_t *
_filename_list_create (const void *ctx)
{
_filename_list_t *list;
list = talloc (ctx, _filename_list_t);
if (list == NULL)
return NULL;
list->head = NULL;
list->tail = &list->head;
return list;
}
static void
_filename_list_add (_filename_list_t *list,
const char *filename)
{
_filename_node_t *node = talloc (list, _filename_node_t);
node->filename = talloc_strdup (list, filename);
node->next = NULL;
*(list->tail) = node;
list->tail = &node->next;
}
static void
tag_inbox_and_unread (notmuch_message_t *message)
{
@ -77,11 +128,18 @@ add_files_print_progress (add_files_state_t *state)
fflush (stdout);
}
static int ino_cmp(const struct dirent **a, const struct dirent **b)
static int
dirent_sort_inode (const struct dirent **a, const struct dirent **b)
{
return ((*a)->d_ino < (*b)->d_ino) ? -1 : 1;
}
static int
dirent_sort_strcmp_name (const struct dirent **a, const struct dirent **b)
{
return strcmp ((*a)->d_name, (*b)->d_name);
}
/* Test if the directory looks like a Maildir directory.
*
* Search through the array of directory entries to see if we can find all
@ -90,12 +148,14 @@ static int ino_cmp(const struct dirent **a, const struct dirent **b)
* Return 1 if the directory looks like a Maildir and 0 otherwise.
*/
static int
is_maildir (struct dirent **entries, int count)
_entries_resemble_maildir (struct dirent **entries, int count)
{
int i, found = 0;
for (i = 0; i < count; i++) {
if (entries[i]->d_type != DT_DIR) continue;
if (entries[i]->d_type != DT_DIR)
continue;
if (strcmp(entries[i]->d_name, "new") == 0 ||
strcmp(entries[i]->d_name, "cur") == 0 ||
strcmp(entries[i]->d_name, "tmp") == 0)
@ -111,186 +171,301 @@ is_maildir (struct dirent **entries, int count)
/* Examine 'path' recursively as follows:
*
* o Ask the filesystem for the mtime of 'path' (path_mtime)
* o Ask the filesystem for the mtime of 'path' (fs_mtime)
* o Ask the database for its timestamp of 'path' (db_mtime)
*
* o Ask the database for its timestamp of 'path' (path_dbtime)
* o Ask the filesystem for files and directories within 'path'
* (via scandir and stored in fs_entries)
* o Ask the database for files and directories within 'path'
* (db_files and db_subdirs)
*
* o If 'path_mtime' > 'path_dbtime'
* o Pass 1: For each directory in fs_entries, recursively call into
* this same function.
*
* o For each regular file in 'path' with mtime newer than the
* 'path_dbtime' call add_message to add the file to the
* database.
* o Pass 2: If 'fs_mtime' > 'db_mtime', then walk fs_entries
* simultaneously with db_files and db_subdirs. Look for one of
* three interesting cases:
*
* o For each sub-directory of path, recursively call into this
* same function.
* 1. Regular file in fs_entries and not in db_files
* This is a new file to add_message into the database.
*
* o Tell the database to update its time of 'path' to 'path_mtime'
* 2. Filename in db_files not in fs_entries.
* This is a file that has been removed from the mail store.
*
* The 'struct stat *st' must point to a structure that has already
* been initialized for 'path' by calling stat().
* 3. Directory in db_subdirs not in fs_entries
* This is a directory that has been removed from the mail store.
*
* Note that the addition of a directory is not interesting here,
* since that will have been taken care of in pass 1. Also, we
* don't immediately act on file/directory removal since we must
* ensure that in the case of a rename that the new filename is
* added before the old filename is removed, (so that no
* information is lost from the database).
*
* o Tell the database to update its time of 'path' to 'fs_mtime'
*/
static notmuch_status_t
add_files_recursive (notmuch_database_t *notmuch,
const char *path,
struct stat *st,
add_files_state_t *state)
{
DIR *dir = NULL;
struct dirent *entry = NULL;
char *next = NULL;
time_t path_mtime, path_dbtime;
time_t fs_mtime, db_mtime;
notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
notmuch_message_t *message = NULL;
struct dirent **namelist = NULL;
int num_entries;
struct dirent **fs_entries = NULL;
int i, num_fs_entries;
notmuch_directory_t *directory;
notmuch_filenames_t *db_files = NULL;
notmuch_filenames_t *db_subdirs = NULL;
struct stat st;
notmuch_bool_t is_maildir, new_directory;
/* If we're told to, we bail out on encountering a read-only
* directory, (with this being a clear clue from the user to
* Notmuch that new mail won't be arriving there and we need not
* look. */
if (state->ignore_read_only_directories &&
(st->st_mode & S_IWUSR) == 0)
{
state->saw_read_only_directory = TRUE;
goto DONE;
if (stat (path, &st)) {
fprintf (stderr, "Error reading directory %s: %s\n",
path, strerror (errno));
return NOTMUCH_STATUS_FILE_ERROR;
}
path_mtime = st->st_mtime;
/* This is not an error since we may have recursed based on a
* symlink to a regular file, not a directory, and we don't know
* that until this stat. */
if (! S_ISDIR (st.st_mode))
return NOTMUCH_STATUS_SUCCESS;
path_dbtime = notmuch_database_get_timestamp (notmuch, path);
num_entries = scandir (path, &namelist, 0, ino_cmp);
fs_mtime = st.st_mtime;
if (num_entries == -1) {
directory = notmuch_database_get_directory (notmuch, path);
db_mtime = notmuch_directory_get_mtime (directory);
if (db_mtime == 0) {
new_directory = TRUE;
db_files = NULL;
db_subdirs = NULL;
} else {
new_directory = FALSE;
db_files = notmuch_directory_get_child_files (directory);
db_subdirs = notmuch_directory_get_child_directories (directory);
}
/* If the database knows about this directory, then we sort based
* on strcmp to match the database sorting. Otherwise, we can do
* inode-based sorting for faster filesystem operation. */
num_fs_entries = scandir (path, &fs_entries, 0,
new_directory ?
dirent_sort_inode : dirent_sort_strcmp_name);
if (num_fs_entries == -1) {
fprintf (stderr, "Error opening directory %s: %s\n",
path, strerror (errno));
ret = NOTMUCH_STATUS_FILE_ERROR;
goto DONE;
}
int i=0;
/* Pass 1: Recurse into all sub-directories. */
is_maildir = _entries_resemble_maildir (fs_entries, num_fs_entries);
while (!interrupted) {
if (i == num_entries)
for (i = 0; i < num_fs_entries; i++) {
if (interrupted)
break;
entry= namelist[i++];
entry = fs_entries[i];
/* If this directory hasn't been modified since the last
* add_files, then we only need to look further for
* sub-directories. */
if (path_mtime <= path_dbtime && entry->d_type == DT_REG)
if (entry->d_type != DT_DIR && entry->d_type != DT_LNK)
continue;
/* Ignore special directories to avoid infinite recursion.
* Also ignore the .notmuch directory.
* Also ignore the .notmuch directory and any "tmp" directory
* that appears within a maildir.
*/
/* XXX: Eventually we'll want more sophistication to let the
* user specify files to be ignored. */
if (strcmp (entry->d_name, ".") == 0 ||
strcmp (entry->d_name, "..") == 0 ||
(entry->d_type == DT_DIR &&
(strcmp (entry->d_name, "tmp") == 0) &&
is_maildir (namelist, num_entries)) ||
(is_maildir && strcmp (entry->d_name, "tmp") == 0) ||
strcmp (entry->d_name, ".notmuch") ==0)
{
continue;
}
next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
status = add_files_recursive (notmuch, next, state);
if (status && ret == NOTMUCH_STATUS_SUCCESS)
ret = status;
talloc_free (next);
next = NULL;
}
if (stat (next, st)) {
int err = errno;
/* If this directory hasn't been modified since the last
* "notmuch new", then we can skip the second pass entirely. */
if (fs_mtime <= db_mtime)
goto DONE;
switch (err) {
case ENOENT:
/* The file was removed between scandir and now... */
case EPERM:
case EACCES:
/* We can't read this file so don't add it to the cache. */
continue;
/* Pass 2: Scan for new files, removed files, and removed directories. */
for (i = 0; i < num_fs_entries; i++)
{
if (interrupted)
break;
entry = fs_entries[i];
/* Check if we've walked past any names in db_files or
* db_subdirs. If so, these have been deleted. */
while (notmuch_filenames_has_more (db_files) &&
strcmp (notmuch_filenames_get (db_files), entry->d_name) < 0)
{
char *absolute = talloc_asprintf (state->removed_files,
"%s/%s", path,
notmuch_filenames_get (db_files));
_filename_list_add (state->removed_files, absolute);
notmuch_filenames_advance (db_files);
}
while (notmuch_filenames_has_more (db_subdirs) &&
strcmp (notmuch_filenames_get (db_subdirs), entry->d_name) <= 0)
{
const char *filename = notmuch_filenames_get (db_subdirs);
if (strcmp (filename, entry->d_name) < 0)
{
char *absolute = talloc_asprintf (state->removed_directories,
"%s/%s", path, filename);
_filename_list_add (state->removed_directories, absolute);
}
fprintf (stderr, "Error reading %s: %s\n",
next, strerror (errno));
ret = NOTMUCH_STATUS_FILE_ERROR;
notmuch_filenames_advance (db_subdirs);
}
/* If we're looking at a symlink, we only want to add it if it
* links to a regular file, (and not to a directory, say). */
if (entry->d_type == DT_LNK) {
int err;
next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
err = stat (next, &st);
talloc_free (next);
next = NULL;
/* Don't emit an error for a link pointing nowhere, since
* the directory-traversal pass will have already done
* that. */
if (err)
continue;
if (! S_ISREG (st.st_mode))
continue;
} else if (entry->d_type != DT_REG) {
continue;
}
/* Don't add a file that we've added before. */
if (notmuch_filenames_has_more (db_files) &&
strcmp (notmuch_filenames_get (db_files), entry->d_name) == 0)
{
notmuch_filenames_advance (db_files);
continue;
}
/* We're now looking at a regular file that doesn't yet exist
* in the database, so add it. */
next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
state->processed_files++;
if (state->verbose) {
if (state->output_is_a_tty)
printf("\r\033[K");
printf ("%i/%i: %s",
state->processed_files,
state->total_files,
next);
putchar((state->output_is_a_tty) ? '\r' : '\n');
fflush (stdout);
}
status = notmuch_database_add_message (notmuch, next, &message);
switch (status) {
/* success */
case NOTMUCH_STATUS_SUCCESS:
state->added_messages++;
tag_inbox_and_unread (message);
break;
/* Non-fatal issues (go on to next file) */
case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
/* Stay silent on this one. */
break;
case NOTMUCH_STATUS_FILE_NOT_EMAIL:
fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
next);
break;
/* Fatal issues. Don't process anymore. */
case NOTMUCH_STATUS_READ_ONLY_DATABASE:
case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
case NOTMUCH_STATUS_OUT_OF_MEMORY:
fprintf (stderr, "Error: %s. Halting processing.\n",
notmuch_status_to_string (status));
ret = status;
goto DONE;
default:
case NOTMUCH_STATUS_FILE_ERROR:
case NOTMUCH_STATUS_NULL_POINTER:
case NOTMUCH_STATUS_TAG_TOO_LONG:
case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
case NOTMUCH_STATUS_LAST_STATUS:
INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
goto DONE;
}
if (S_ISREG (st->st_mode)) {
/* If the file hasn't been modified since the last
* add_files, then we need not look at it. */
if (path_dbtime == 0 || st->st_mtime > path_dbtime) {
state->processed_files++;
if (message) {
notmuch_message_destroy (message);
message = NULL;
}
if (state->verbose) {
if (state->output_is_a_tty)
printf("\r\033[K");
printf ("%i/%i: %s",
state->processed_files,
state->total_files,
next);
putchar((state->output_is_a_tty) ? '\r' : '\n');
fflush (stdout);
}
status = notmuch_database_add_message (notmuch, next, &message);
switch (status) {
/* success */
case NOTMUCH_STATUS_SUCCESS:
state->added_messages++;
tag_inbox_and_unread (message);
break;
/* Non-fatal issues (go on to next file) */
case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
/* Stay silent on this one. */
break;
case NOTMUCH_STATUS_FILE_NOT_EMAIL:
fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
next);
break;
/* Fatal issues. Don't process anymore. */
case NOTMUCH_STATUS_READONLY_DATABASE:
case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
case NOTMUCH_STATUS_OUT_OF_MEMORY:
fprintf (stderr, "Error: %s. Halting processing.\n",
notmuch_status_to_string (status));
ret = status;
goto DONE;
default:
case NOTMUCH_STATUS_FILE_ERROR:
case NOTMUCH_STATUS_NULL_POINTER:
case NOTMUCH_STATUS_TAG_TOO_LONG:
case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
case NOTMUCH_STATUS_LAST_STATUS:
INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
goto DONE;
}
if (message) {
notmuch_message_destroy (message);
message = NULL;
}
if (do_add_files_print_progress) {
do_add_files_print_progress = 0;
add_files_print_progress (state);
}
}
} else if (S_ISDIR (st->st_mode)) {
status = add_files_recursive (notmuch, next, st, state);
if (status && ret == NOTMUCH_STATUS_SUCCESS)
ret = status;
if (do_add_files_print_progress) {
do_add_files_print_progress = 0;
add_files_print_progress (state);
}
talloc_free (next);
next = NULL;
}
status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
if (status && ret == NOTMUCH_STATUS_SUCCESS)
ret = status;
/* Now that we've walked the whole filesystem list, anything left
* over in the database lists has been deleted. */
while (notmuch_filenames_has_more (db_files))
{
char *absolute = talloc_asprintf (state->removed_files,
"%s/%s", path,
notmuch_filenames_get (db_files));
_filename_list_add (state->removed_files, absolute);
notmuch_filenames_advance (db_files);
}
while (notmuch_filenames_has_more (db_subdirs))
{
char *absolute = talloc_asprintf (state->removed_directories,
"%s/%s", path,
notmuch_filenames_get (db_subdirs));
_filename_list_add (state->removed_directories, absolute);
notmuch_filenames_advance (db_subdirs);
}
if (! interrupted) {
status = notmuch_directory_set_mtime (directory, fs_mtime);
if (status && ret == NOTMUCH_STATUS_SUCCESS)
ret = status;
}
DONE:
if (next)
@ -299,8 +474,14 @@ add_files_recursive (notmuch_database_t *notmuch,
free (entry);
if (dir)
closedir (dir);
if (namelist)
free (namelist);
if (fs_entries)
free (fs_entries);
if (db_subdirs)
notmuch_filenames_destroy (db_subdirs);
if (db_files)
notmuch_filenames_destroy (db_files);
if (directory)
notmuch_directory_destroy (directory);
return ret;
}
@ -308,27 +489,16 @@ add_files_recursive (notmuch_database_t *notmuch,
/* This is the top-level entry point for add_files. It does a couple
* of error checks, sets up the progress-printing timer and then calls
* into the recursive function. */
notmuch_status_t
static notmuch_status_t
add_files (notmuch_database_t *notmuch,
const char *path,
add_files_state_t *state)
{
struct stat st;
notmuch_status_t status;
struct sigaction action;
struct itimerval timerval;
notmuch_bool_t timer_is_active = FALSE;
if (stat (path, &st)) {
fprintf (stderr, "Error reading directory %s: %s\n",
path, strerror (errno));
return NOTMUCH_STATUS_FILE_ERROR;
}
if (! S_ISDIR (st.st_mode)) {
fprintf (stderr, "Error: %s is not a directory.\n", path);
return NOTMUCH_STATUS_FILE_ERROR;
}
struct stat st;
if (state->output_is_a_tty && ! debugger_is_active () && ! state->verbose) {
/* Setup our handler for SIGALRM */
@ -348,7 +518,18 @@ add_files (notmuch_database_t *notmuch,
timer_is_active = TRUE;
}
status = add_files_recursive (notmuch, path, &st, state);
if (stat (path, &st)) {
fprintf (stderr, "Error reading directory %s: %s\n",
path, strerror (errno));
return NOTMUCH_STATUS_FILE_ERROR;
}
if (! S_ISDIR (st.st_mode)) {
fprintf (stderr, "Error: %s is not a directory.\n", path);
return NOTMUCH_STATUS_FILE_ERROR;
}
status = add_files_recursive (notmuch, path, state);
if (timer_is_active) {
/* Now stop the timer. */
@ -378,21 +559,21 @@ count_files (const char *path, int *count)
struct dirent *entry = NULL;
char *next;
struct stat st;
struct dirent **namelist = NULL;
int n_entries = scandir (path, &namelist, 0, ino_cmp);
struct dirent **fs_entries = NULL;
int num_fs_entries = scandir (path, &fs_entries, 0, dirent_sort_inode);
int i = 0;
if (n_entries == -1) {
if (num_fs_entries == -1) {
fprintf (stderr, "Warning: failed to open directory %s: %s\n",
path, strerror (errno));
goto DONE;
}
while (!interrupted) {
if (i == n_entries)
if (i == num_fs_entries)
break;
entry= namelist[i++];
entry = fs_entries[i++];
/* Ignore special directories to avoid infinite recursion.
* Also ignore the .notmuch directory.
@ -431,8 +612,77 @@ count_files (const char *path, int *count)
DONE:
if (entry)
free (entry);
if (namelist)
free (namelist);
if (fs_entries)
free (fs_entries);
}
static void
upgrade_print_progress (void *closure,
double progress)
{
add_files_state_t *state = closure;
printf ("Upgrading database: %.2f%% complete", progress * 100.0);
if (progress > 0) {
struct timeval tv_now;
double elapsed, time_remaining;
gettimeofday (&tv_now, NULL);
elapsed = notmuch_time_elapsed (state->tv_start, tv_now);
time_remaining = (elapsed / progress) * (1.0 - progress);
printf (" (");
notmuch_time_print_formatted_seconds (time_remaining);
printf (" remaining)");
}
printf (". \r");
fflush (stdout);
}
/* Recursively remove all filenames from the database referring to
* 'path' (or to any of its children). */
static void
_remove_directory (void *ctx,
notmuch_database_t *notmuch,
const char *path,
int *renamed_files,
int *removed_files)
{
notmuch_directory_t *directory;
notmuch_filenames_t *files, *subdirs;
notmuch_status_t status;
char *absolute;
directory = notmuch_database_get_directory (notmuch, path);
for (files = notmuch_directory_get_child_files (directory);
notmuch_filenames_has_more (files);
notmuch_filenames_advance (files))
{
absolute = talloc_asprintf (ctx, "%s/%s", path,
notmuch_filenames_get (files));
status = notmuch_database_remove_message (notmuch, absolute);
if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
*renamed_files = *renamed_files + 1;
else
*removed_files = *removed_files + 1;
talloc_free (absolute);
}
for (subdirs = notmuch_directory_get_child_directories (directory);
notmuch_filenames_has_more (subdirs);
notmuch_filenames_advance (subdirs))
{
absolute = talloc_asprintf (ctx, "%s/%s", path,
notmuch_filenames_get (subdirs));
_remove_directory (ctx, notmuch, absolute, renamed_files, removed_files);
talloc_free (absolute);
}
notmuch_directory_destroy (directory);
}
int
@ -448,6 +698,9 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
const char *db_path;
char *dot_notmuch_path;
struct sigaction action;
_filename_node_t *f;
int renamed_files, removed_files;
notmuch_status_t status;
int i;
add_files_state.verbose = 0;
@ -462,13 +715,6 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
}
}
/* Setup our handler for SIGINT */
memset (&action, 0, sizeof (struct sigaction));
action.sa_handler = handle_sigint;
sigemptyset (&action.sa_mask);
action.sa_flags = SA_RESTART;
sigaction (SIGINT, &action, NULL);
config = notmuch_config_open (ctx, NULL, NULL);
if (config == NULL)
return 1;
@ -485,33 +731,73 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
if (interrupted)
return 1;
printf ("Found %d total files. \n", count);
printf ("Found %d total files (that's not much mail).\n", count);
notmuch = notmuch_database_create (db_path);
add_files_state.ignore_read_only_directories = FALSE;
add_files_state.total_files = count;
} else {
notmuch = notmuch_database_open (db_path,
NOTMUCH_DATABASE_MODE_READ_WRITE);
add_files_state.ignore_read_only_directories = TRUE;
if (notmuch == NULL)
return 1;
if (notmuch_database_needs_upgrade (notmuch)) {
printf ("Welcome to a new version of notmuch! Your database will now be upgraded.\n");
gettimeofday (&add_files_state.tv_start, NULL);
notmuch_database_upgrade (notmuch, upgrade_print_progress,
&add_files_state);
printf ("Your notmuch database has now been upgraded to database format version %u.\n",
notmuch_database_get_version (notmuch));
}
add_files_state.total_files = 0;
}
if (notmuch == NULL)
return 1;
/* Setup our handler for SIGINT. We do this after having
* potentially done a database upgrade we this interrupt handler
* won't support. */
memset (&action, 0, sizeof (struct sigaction));
action.sa_handler = handle_sigint;
sigemptyset (&action.sa_mask);
action.sa_flags = SA_RESTART;
sigaction (SIGINT, &action, NULL);
talloc_free (dot_notmuch_path);
dot_notmuch_path = NULL;
add_files_state.saw_read_only_directory = FALSE;
add_files_state.processed_files = 0;
add_files_state.added_messages = 0;
gettimeofday (&add_files_state.tv_start, NULL);
add_files_state.removed_files = _filename_list_create (ctx);
add_files_state.removed_directories = _filename_list_create (ctx);
ret = add_files (notmuch, db_path, &add_files_state);
removed_files = 0;
renamed_files = 0;
for (f = add_files_state.removed_files->head; f; f = f->next) {
status = notmuch_database_remove_message (notmuch, f->filename);
if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
renamed_files++;
else
removed_files++;
}
for (f = add_files_state.removed_directories->head; f; f = f->next) {
_remove_directory (ctx, notmuch, f->filename,
&renamed_files, &removed_files);
}
talloc_free (add_files_state.removed_files);
talloc_free (add_files_state.removed_directories);
gettimeofday (&tv_now, NULL);
elapsed = notmuch_time_elapsed (add_files_state.tv_start,
tv_now);
if (add_files_state.processed_files) {
printf ("Processed %d %s in ", add_files_state.processed_files,
add_files_state.processed_files == 1 ?
@ -524,22 +810,30 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
printf (". \n");
}
}
if (add_files_state.added_messages) {
printf ("Added %d new %s to the database (not much, really).\n",
printf ("Added %d new %s to the database.",
add_files_state.added_messages,
add_files_state.added_messages == 1 ?
"message" : "messages");
} else {
printf ("No new mail---and that's not much.\n");
printf ("No new mail.");
}
if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
"they will never receive new mail), marking these directories as\n"
"read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
"much more efficient (it won't even look in those directories).\n");
if (removed_files) {
printf (" Removed %d %s.",
removed_files,
removed_files == 1 ? "message" : "messages");
}
if (renamed_files) {
printf (" Detected %d file %s.",
renamed_files,
renamed_files == 1 ? "rename" : "renames");
}
printf ("\n");
if (ret) {
printf ("\nNote: At least one error was encountered: %s\n",
notmuch_status_to_string (ret));

8
notmuch.1
View file

@ -109,14 +109,6 @@ whenever new mail is delivered and you wish to incorporate it into the
database. These subsequent runs will be much quicker than the initial
run.
Note:
.B notmuch new
runs (other than the first run) will skip any read-only directories,
so you can use that to mark directories that will not receive any new
mail (and make
.B notmuch new
even faster).
Invoking
.B notmuch
with no command argument will run

5
notmuch.c
View file

@ -145,11 +145,6 @@ command_t commands[] = {
"\t\t\tVerbose operation. Shows paths of message files as\n"
"\t\t\tthey are being indexed.\n"
"\n"
"\t\tNote: \"notmuch new\" runs (other than the first run) will\n"
"\t\tskip any read-only directories, so you can use that to mark\n"
"\t\tdirectories that will not receive any new mail (and make\n"
"\t\t\"notmuch new\" even faster).\n"
"\n"
"\t\tInvoking notmuch with no command argument will run new if\n"
"\t\tthe setup command has previously been completed, but new has\n"
"\t\tnot previously been run." },