From d12801c8b4d04a50fcb912b75017244fb09658e8 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Sat, 9 Jan 2010 11:13:12 -0800 Subject: [PATCH] lib: Split the database upgrade into two phases for safer operation. The first phase copies data from the old format to the new format without deleting anything. This allows an old notmuch to still use the database if the upgrade process gets interrupted. The second phase performs the deletion (after updating the database version number). If the second phase is interrupted, there will be some unused data in the database, but it shouldn't cause any actual harm. --- lib/database.cc | 78 ++++++++++++++++++++++++++++++++++++++++--- lib/message.cc | 22 +++++------- lib/notmuch-private.h | 16 +++++++++ 3 files changed, 97 insertions(+), 19 deletions(-) diff --git a/lib/database.cc b/lib/database.cc index d0262722..19f960e2 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -722,7 +722,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } /* Before version 1, each message document had its filename in the - * data field. Move that into the new format by calling + * data field. Copy that into the new format by calling * notmuch_message_add_filename. */ if (version < 1) { @@ -730,6 +730,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, notmuch_query_t *query = notmuch_query_create (notmuch, ""); notmuch_messages_t *messages; notmuch_message_t *message; + char *filename; total = notmuch_query_count_messages (query); @@ -744,15 +745,24 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, message = notmuch_messages_get (messages); - _notmuch_message_upgrade_filename_storage (message); + filename = _notmuch_message_talloc_copy_data (message); + if (filename && *filename != '\0') { + _notmuch_message_add_filename (message, filename); + _notmuch_message_sync (message); + } + talloc_free (filename); + + notmuch_message_destroy (message); count++; } + + notmuch_query_destroy (query); } /* Also, before version 1 we stored directory timestamps in * XTIMESTAMP documents instead of the current XDIRECTORY - * documents. So convert those as well. */ + * documents. So copy those as well. */ if (version < 1) { Xapian::TermIterator t, t_end; @@ -783,8 +793,6 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, term.c_str() + 10); notmuch_directory_set_mtime (directory, mtime); notmuch_directory_destroy (directory); - - db->delete_document (*p); } } } @@ -792,6 +800,66 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); db->flush (); + /* Now that the upgrade is complete we can remove the old data + * and documents that are no longer needed. */ + if (version < 1) { + unsigned int count = 0, total; + notmuch_query_t *query = notmuch_query_create (notmuch, ""); + notmuch_messages_t *messages; + notmuch_message_t *message; + char *filename; + + total = notmuch_query_count_messages (query); + + for (messages = notmuch_query_search_messages (query); + notmuch_messages_has_more (messages); + notmuch_messages_advance (messages)) + { + if (do_progress_notify) { + progress_notify (closure, count, total); + do_progress_notify = 0; + } + + message = notmuch_messages_get (messages); + + filename = _notmuch_message_talloc_copy_data (message); + if (filename && *filename != '\0') { + _notmuch_message_clear_data (message); + _notmuch_message_sync (message); + } + talloc_free (filename); + + notmuch_message_destroy (message); + + count++; + } + + notmuch_query_destroy (query); + } + + if (version < 1) { + Xapian::TermIterator t, t_end; + + t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP"); + + for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP"); + t != t_end; + t++) + { + Xapian::PostingIterator p, p_end; + std::string term = *t; + + p_end = notmuch->xapian_db->postlist_end (term); + + for (p = notmuch->xapian_db->postlist_begin (term); + p != p_end; + p++) + { + db->delete_document (*p); + } + } + } + if (timer_is_active) { /* Now stop the timer. */ timerval.it_interval.tv_sec = 0; diff --git a/lib/message.cc b/lib/message.cc index baeaa469..1cda55a1 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -416,22 +416,16 @@ _notmuch_message_add_filename (notmuch_message_t *message, return NOTMUCH_STATUS_SUCCESS; } -/* Move the filename from the data field (as it was in database format - * version 0) to a file-direntry term instead (as in database format - * version 1). - */ -void -_notmuch_message_upgrade_filename_storage (notmuch_message_t *message) +char * +_notmuch_message_talloc_copy_data (notmuch_message_t *message) { - char *filename; + return talloc_strdup (message, message->doc.get_data ().c_str ()); +} - filename = talloc_strdup (message, message->doc.get_data ().c_str ()); - if (filename && *filename != '\0') { - _notmuch_message_add_filename (message, filename); - message->doc.set_data (""); - _notmuch_message_sync (message); - } - talloc_free (filename); +void +_notmuch_message_clear_data (notmuch_message_t *message) +{ + message->doc.set_data (""); } const char * diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index f9adea72..c7fb0ef8 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -258,6 +258,22 @@ _notmuch_message_sync (notmuch_message_t *message); void _notmuch_message_close (notmuch_message_t *message); +/* Get a copy of the data in this message document. + * + * Caller should talloc_free the result when done. + * + * This function is intended to support database upgrade and really + * shouldn't be used otherwise. */ +char * +_notmuch_message_talloc_copy_data (notmuch_message_t *message); + +/* Clear the data in this message document. + * + * This function is intended to support database upgrade and really + * shouldn't be used otherwise. */ +void +_notmuch_message_clear_data (notmuch_message_t *message); + /* index.cc */ notmuch_status_t