Merge remote branch 'origin/master' into vim

Makefile.local

@@ -13,6 +13,7 @@ notmuch_client_srcs =		\
 	notmuch-reply.c		\
 	notmuch-restore.c	\
 	notmuch-search.c	\
+	notmuch-search-tags.c   \
 	notmuch-setup.c		\
 	notmuch-show.c		\
 	notmuch-tag.c		\

TODO

@@ -4,9 +4,7 @@ Fix the things that are causing the most pain to new users
 
 2. Allow an easy way to get tags from directory names (if the user has them)
 
-3. Make emacs fast for big search results (see "lazy searching" below)
+3. Fix Xapian defect #250 so tagging is fast.
-
-4. Fix Xapian defect #250 so tagging is fast.
 
 Emacs interface (notmuch.el)
 ----------------------------
@@ -18,10 +16,6 @@ tables that add to it.
 
 Add a command to archive all threads in a search view.
 	
-Lazy searching: call "notmuch search" with --first and --max to fill
-just a screenful of results, and then fill in more as ther user pages
-through the buffer.
-
 Add a '|' binding from the search view.
 
 Add a binding to run a search from notmuch-show-mode.
@@ -119,6 +113,10 @@ any of this we're probably going to need to break down an write our
 own parser for the query string rather than using Xapian's QueryParser
 class.
 
+Make failure to read a file (such as a permissions problem) a warning
+rather than an error (should be similar to the existing warning for a
+non-mail file).
+
 Add support for files that are moved or deleted (which obviously need
 to be handled differently).
 

lib/database-private.h

@@ -35,4 +35,17 @@ struct _notmuch_database {
     Xapian::ValueRangeProcessor *value_range_processor;
 };
 
+/* Convert tags from Xapian internal format to notmuch format.
+ *
+ * The function gets a TermIterator as argument and uses that iterator to find
+ * all tag terms in the object. The tags are then converted to a
+ * notmuch_tags_t list and returned. The function needs to allocate memory for
+ * the resulting list and it uses the argument ctx as talloc context.
+ *
+ * The function returns NULL on failure.
+ */
+notmuch_tags_t *
+_notmuch_convert_tags (void *ctx, Xapian::TermIterator &i,
+		       Xapian::TermIterator &end);
+
 #endif

lib/database.cc

@@ -1029,3 +1029,46 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
     return ret;
 }
+
+notmuch_tags_t *
+_notmuch_convert_tags (void *ctx, Xapian::TermIterator &i,
+		       Xapian::TermIterator &end)
+{
+    const char *prefix = _find_prefix ("tag");
+    notmuch_tags_t *tags;
+    std::string tag;
+
+    /* Currently this iteration is written with the assumption that
+     * "tag" has a single-character prefix. */
+    assert (strlen (prefix) == 1);
+
+    tags = _notmuch_tags_create (ctx);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    i.skip_to (prefix);
+
+    while (i != end) {
+	tag = *i;
+
+	if (tag.empty () || tag[0] != *prefix)
+	    break;
+
+	_notmuch_tags_add_tag (tags, tag.c_str () + 1);
+
+	i++;
+    }
+
+    _notmuch_tags_prepare_iterator (tags);
+
+    return tags;
+}
+
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db)
+{
+    Xapian::TermIterator i, end;
+    i = db->xapian_db->allterms_begin();
+    end = db->xapian_db->allterms_end();
+    return _notmuch_convert_tags(db, i, end);
+}

lib/message.cc

@@ -482,38 +482,10 @@ notmuch_message_get_date (notmuch_message_t *message)
 notmuch_tags_t *
 notmuch_message_get_tags (notmuch_message_t *message)
 {
-    const char *prefix = _find_prefix ("tag");
     Xapian::TermIterator i, end;
-    notmuch_tags_t *tags;
+    i = message->doc.termlist_begin();
-    std::string tag;
+    end = message->doc.termlist_end();
-
+    return _notmuch_convert_tags(message, i, end);
-    /* Currently this iteration is written with the assumption that
-     * "tag" has a single-character prefix. */
-    assert (strlen (prefix) == 1);
-
-    tags = _notmuch_tags_create (message);
-    if (unlikely (tags == NULL))
-	return NULL;
-
-    i = message->doc.termlist_begin ();
-    end = message->doc.termlist_end ();
-
-    i.skip_to (prefix);
-
-    while (i != end) {
-	tag = *i;
-
-	if (tag.empty () || tag[0] != *prefix)
-	    break;
-
-	_notmuch_tags_add_tag (tags, tag.c_str () + 1);
-
-	i++;
-    }
-
-    _notmuch_tags_prepare_iterator (tags);
-
-    return tags;
 }
 
 void

lib/messages.c

@@ -20,6 +20,8 @@
 
 #include "notmuch-private.h"
 
+#include <glib.h>
+
 /* Create a new notmuch_message_list_t object, with 'ctx' as its
  * talloc owner.
  *
@@ -140,3 +142,41 @@ notmuch_messages_destroy (notmuch_messages_t *messages)
 {
     talloc_free (messages);
 }
+
+
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages)
+{
+    notmuch_tags_t *tags, *msg_tags;
+    notmuch_message_t *msg;
+    GHashTable *htable;
+    GList *keys, *l;
+    const char *tag;
+
+    tags = _notmuch_tags_create (messages);
+    if (tags == NULL) return NULL;
+
+    htable = g_hash_table_new_full (g_str_hash, g_str_equal, free, NULL);
+
+    while ((msg = notmuch_messages_get (messages))) {
+	msg_tags = notmuch_message_get_tags (msg);
+	while ((tag = notmuch_tags_get (msg_tags))) {
+	    g_hash_table_insert (htable, xstrdup (tag), NULL);
+	    notmuch_tags_advance (msg_tags);
+	}
+	notmuch_tags_destroy (msg_tags);
+	notmuch_message_destroy (msg);
+	notmuch_messages_advance (messages);
+    }
+
+    keys = g_hash_table_get_keys (htable);
+    for (l = keys; l; l = l->next) {
+	_notmuch_tags_add_tag (tags, (char *)l->data);
+    }
+
+    g_list_free (keys);
+    g_hash_table_destroy (htable);
+
+    _notmuch_tags_prepare_iterator (tags);
+    return tags;
+}

lib/notmuch.h

@@ -280,6 +280,16 @@ notmuch_message_t *
 notmuch_database_find_message (notmuch_database_t *database,
 			       const char *message_id);
 
+/* Return a list of all tags found in the database.
+ *
+ * This function creates a list of all tags found in the database. The
+ * resulting list contains all tags from all messages found in the database.
+ *
+ * On error this function returns NULL.
+ */
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db);
+
 /* Create a new query for 'database'.
  *
  * Here, 'database' should be an open database, (see
@@ -625,6 +635,21 @@ notmuch_messages_advance (notmuch_messages_t *messages);
 void
 notmuch_messages_destroy (notmuch_messages_t *messages);
 
+/* Return a list of tags from all messages.
+ *
+ * The resulting list is guaranteed not to contain duplicated tags.
+ *
+ * WARNING: You can no longer iterate over messages after calling this
+ * function, because the iterator will point at the end of the list.
+ * We do not have a function to reset the iterator yet and the only
+ * way how you can iterate over the list again is to recreate the
+ * message list.
+ *
+ * The function returns NULL on error.
+ */
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages);
+
 /* Get the message ID of 'message'.
  *
  * The returned string belongs to 'message' and as such, should not be

notmuch-client.h

@@ -119,6 +119,9 @@ notmuch_show_command (void *ctx, int argc, char *argv[]);
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[]);
 
+int
+notmuch_search_tags_command (void *ctx, int argc, char *argv[]);
+
 const char *
 notmuch_time_relative_date (const void *ctx, time_t then);
 

notmuch-search-tags.c

@@ -0,0 +1,98 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ * Copyright © 2009 Jan Janak
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Jan Janak <jan@ryngle.com>
+ */
+
+#include "notmuch-client.h"
+
+static void
+print_tags (notmuch_tags_t *tags)
+{
+    const char *t;
+
+    while ((t = notmuch_tags_get (tags))) {
+	printf ("%s\n", t);
+	notmuch_tags_advance (tags);
+    }
+}
+
+int
+notmuch_search_tags_command (void *ctx, int argc, char *argv[])
+{
+    notmuch_messages_t *msgs;
+    notmuch_tags_t *tags;
+    notmuch_config_t *config;
+    notmuch_database_t *db;
+    notmuch_query_t *query;
+    char *query_str;
+
+    tags = NULL;
+    config = NULL;
+    db = NULL;
+    query = NULL;
+
+    if ((config = notmuch_config_open (ctx, NULL, NULL)) == NULL) {
+	goto error;
+    }
+
+    db = notmuch_database_open (notmuch_config_get_database_path (config),
+				NOTMUCH_DATABASE_MODE_READ_ONLY);
+    if (db == NULL) {
+	goto error;
+    }
+
+    if (argc > 0) {
+	if ((query_str = query_string_from_args (ctx, argc, argv)) == NULL) {
+	    fprintf (stderr, "Out of memory.\n");
+	    goto error;
+	}
+
+	if (*query_str == '\0') {
+	    fprintf (stderr, "Error: Invalid search string.\n");
+	    goto error;
+	}
+
+	if ((query = notmuch_query_create (db, query_str)) == NULL) {
+	    fprintf (stderr, "Out of memory\n");
+	    goto error;
+	}
+
+
+	msgs = notmuch_query_search_messages (query);
+	if ((tags = notmuch_messages_collect_tags (msgs)) == NULL) goto error;
+    } else {
+	if ((tags = notmuch_database_get_all_tags (db)) == NULL) {
+	    fprintf (stderr, "Error while getting tags from the database.\n");
+	    goto error;
+	}
+    }
+
+    print_tags (tags);
+
+    notmuch_tags_destroy (tags);
+    if (query) notmuch_query_destroy (query);
+    notmuch_database_close (db);
+    return 0;
+
+error:
+    if (tags) notmuch_tags_destroy (tags);
+    if (query) notmuch_query_destroy (query);
+    if (db) notmuch_database_close (db);
+    return 1;
+}

notmuch-setup.c

@@ -156,10 +156,11 @@ notmuch_setup_command (unused (void *ctx),
 	notmuch_config_set_database_path (config, absolute_path);
     }
 
-    notmuch_config_save (config);
+    if (! notmuch_config_save (config)) {
-
 	if (is_new)
 	  welcome_message_post_setup ();
-
 	return 0;
+    } else {
+	return 1;
+    }
 }

notmuch.c

@@ -254,6 +254,15 @@ command_t commands[] = {
       "\t\tSo if you've previously been using sup for mail, then the\n"
       "\t\t\"notmuch restore\" command provides you a way to import\n"
       "\t\tall of your tags (or labels as sup calls them)." },
+    { "search-tags", notmuch_search_tags_command,
+      "[<search-terms> [...] ]",
+      "\t\tList all tags found in the database or matching messages.",
+      "\t\tRun this command without any search-term(s) to obtain a list\n"
+      "\t\tof all tags found in the database. If you provide one or more\n"
+      "\t\tsearch-terms as argument(s) then the resulting list will\n"
+      "\t\tcontain tags only from messages that match the search-term(s).\n"
+      "\n"
+      "\t\tIn both cases the list will be alphabetically sorted." },
     { "help", notmuch_help_command,
       "[<command>]",
       "\t\tThis message, or more detailed help for the named command.",

notmuch.el

@@ -137,6 +137,13 @@ within the current window."
       (or (memq prop buffer-invisibility-spec)
 	  (assq prop buffer-invisibility-spec)))))
 
+(defun notmuch-select-tag-with-completion (prompt &rest search-terms)
+  (let ((tag-list
+	 (with-output-to-string
+	   (with-current-buffer standard-output
+	     (apply 'call-process notmuch-command nil t nil "search-tags" search-terms)))))
+    (completing-read prompt (split-string tag-list "\n+" t) nil nil nil)))
+
 (defun notmuch-show-next-line ()
   "Like builtin `next-line' but ensuring we end on a visible character.
 
@@ -167,7 +174,7 @@ Unlike builtin `next-line' this version accepts no arguments."
     (if (not (looking-at notmuch-show-message-begin-regexp))
 	(re-search-backward notmuch-show-message-begin-regexp))
     (re-search-forward notmuch-show-id-regexp)
-    (buffer-substring (match-beginning 1) (match-end 1))))
+    (buffer-substring-no-properties (match-beginning 1) (match-end 1))))
 
 (defun notmuch-show-get-filename ()
   (save-excursion
@@ -175,7 +182,7 @@ Unlike builtin `next-line' this version accepts no arguments."
     (if (not (looking-at notmuch-show-message-begin-regexp))
 	(re-search-backward notmuch-show-message-begin-regexp))
     (re-search-forward notmuch-show-filename-regexp)
-    (buffer-substring (match-beginning 1) (match-end 1))))
+    (buffer-substring-no-properties (match-beginning 1) (match-end 1))))
 
 (defun notmuch-show-set-tags (tags)
   (save-excursion
@@ -200,7 +207,8 @@ Unlike builtin `next-line' this version accepts no arguments."
 
 (defun notmuch-show-add-tag (&rest toadd)
   "Add a tag to the current message."
-  (interactive "sTag to add: ")
+  (interactive
+   (list (notmuch-select-tag-with-completion "Tag to add: ")))
   (apply 'notmuch-call-notmuch-process
 	 (append (cons "tag"
 		       (mapcar (lambda (s) (concat "+" s)) toadd))
@@ -209,7 +217,8 @@ Unlike builtin `next-line' this version accepts no arguments."
 
 (defun notmuch-show-remove-tag (&rest toremove)
   "Remove a tag from the current message."
-  (interactive "sTag to remove: ")
+  (interactive
+   (list (notmuch-select-tag-with-completion "Tag to remove: " (notmuch-show-get-message-id))))
   (let ((tags (notmuch-show-get-tags)))
     (if (intersection tags toremove :test 'string=)
 	(progn
@@ -804,7 +813,8 @@ thread from that buffer can be show when done with this one)."
 (fset 'notmuch-search-mode-map notmuch-search-mode-map)
 
 (defvar notmuch-search-query-string)
-(defvar notmuch-search-oldest-first)
+(defvar notmuch-search-oldest-first t
+  "Show the oldest mail first in the search-mode")
 
 
 (defun notmuch-search-scroll-up ()
@@ -871,31 +881,6 @@ global search.
   "Return the thread for the current thread"
   (get-text-property (point) 'notmuch-search-thread-id))
 
-(defun notmuch-search-markup-this-thread-id ()
-  (beginning-of-line)
-  (let ((beg (point)))
-    (if (re-search-forward "thread:[a-fA-F0-9]*" nil t)
-	(progn
-	  (forward-char)
-	  (overlay-put (make-overlay beg (point)) 'invisible 'notmuch-search)
-	  (re-search-forward ".*\\[[0-9]*/[0-9]*\\] \\([^;]*\\)\\(;\\)")
-	  (let* ((authors (buffer-substring (match-beginning 1) (match-end 1)))
-		 (authors-length (length authors)))
-	    ;; Drop the semi-colon
-	    (replace-match "" t nil nil 2)
-	    (if (<= authors-length notmuch-search-authors-width)
-		(replace-match (concat authors (make-string
-						(- notmuch-search-authors-width
-						   authors-length) ? )) t t nil 1)
-	      (replace-match (concat (substring authors 0 (- notmuch-search-authors-width 3)) "...") t t nil 1)))))))
-
-(defun notmuch-search-markup-thread-ids ()
-  (save-excursion
-    (goto-char (point-min))
-    (while (not (eobp))
-      (notmuch-search-markup-this-thread-id)
-      (forward-line))))
-
 (defun notmuch-search-show-thread ()
   (interactive)
   (let ((thread-id (notmuch-search-find-thread-id)))
@@ -949,12 +934,14 @@ and will also appear in a buffer named \"*Notmuch errors*\"."
 	(split-string (buffer-substring beg end))))))
 
 (defun notmuch-search-add-tag (tag)
-  (interactive "sTag to add: ")
+  (interactive
+   (list (notmuch-select-tag-with-completion "Tag to add: ")))
   (notmuch-call-notmuch-process "tag" (concat "+" tag) (notmuch-search-find-thread-id))
   (notmuch-search-set-tags (delete-dups (sort (cons tag (notmuch-search-get-tags)) 'string<))))
 
 (defun notmuch-search-remove-tag (tag)
-  (interactive "sTag to remove: ")
+  (interactive
+   (list (notmuch-select-tag-with-completion "Tag to remove: " (notmuch-search-find-thread-id))))
   (notmuch-call-notmuch-process "tag" (concat "-" tag) (notmuch-search-find-thread-id))
   (notmuch-search-set-tags (delete tag (notmuch-search-get-tags))))
 
@@ -1089,13 +1076,14 @@ current search results AND the additional query string provided."
 
 Runs a new search matching only messages that match both the
 current search results AND that are tagged with the given tag."
-  (interactive "sFilter by tag: ")
+  (interactive
+   (list (notmuch-select-tag-with-completion "Filter by tag: ")))
   (notmuch-search (concat notmuch-search-query-string " and tag:" tag) notmuch-search-oldest-first))
 
 (defun notmuch ()
   "Run notmuch to display all mail with tag of 'inbox'"
   (interactive)
-  (notmuch-search "tag:inbox" t))
+  (notmuch-search "tag:inbox" notmuch-search-oldest-first))
 
 (setq mail-user-agent 'message-user-agent)
 
@@ -1165,7 +1153,7 @@ results for the search terms in that line.
       (setq folder (notmuch-folder-find-name)))
   (let ((search (assoc folder notmuch-folders)))
     (if search
-	(notmuch-search (cdr search) t))))
+	(notmuch-search (cdr search) notmuch-search-oldest-first))))
 
 (defun notmuch-folder ()
   "Show the notmuch folder view and update the displayed counts."