debian/notmuch
1
0
Fork 0
mirror of https://git.notmuchmail.org/git/notmuch synced 2024-11-24 20:08:10 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

dump: support gzipped and atomic output

Browse source 
The main goal is to support gzipped output for future internal
calls (e.g. from notmuch-new) to notmuch_database_dump.

The additional dependency is not very heavy since xapian already pulls
in zlib.

We want the dump to be "atomic", in the sense that after running the
dump file is either present and complete, or not present.  This avoids
certain classes of mishaps involving overwriting a good backup with a
bad or partial one.
This commit is contained in:
David Bremner 2014-03-28 22:14:51 -03:00
parent 69867c33fa
commit 3c13bc0321
7 changed files with 142 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
INSTALL
View file

@ -20,8 +20,8 @@ configure stage.
Dependencies
------------
Notmuch depends on three libraries: Xapian, GMime 2.4 or 2.6, and
Talloc which are each described below:
Notmuch depends on four libraries: Xapian, GMime 2.4 or 2.6,
Talloc, and zlib which are each described below:
Xapian
------
@ -60,6 +60,18 @@ Talloc which are each described below:
Talloc is available from http://talloc.samba.org/
zlib
----
zlib is an extremely popular compression library. It is used
by Xapian, so if you installed that you will already have
zlib. You may need to install the zlib headers separately.
Notmuch needs the transparent write feature of zlib introduced
in version 1.2.5.2 (Dec. 2011).
zlib is available from http://zlib.net
Building Documentation
----------------------
@ -79,11 +91,11 @@ dependencies with a simple simple command line. For example:
For Debian and similar:
sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev python-sphinx
sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev python-sphinx
For Fedora and similar:
sudo yum install xapian-core-devel gmime-devel libtalloc-devel python-sphinx
sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel python-sphinx
On other systems, a similar command can be used, but the details of
the package names may be different.

2
Makefile.local
View file

@ -46,7 +46,7 @@ PV_FILE=bindings/python/notmuch/version.py
# Smash together user's values with our extra values
FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)
FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)
FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)
FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS)
FINAL_NOTMUCH_LINKER = CC
ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)

28
configure vendored
View file

@ -340,6 +340,18 @@ else
errors=$((errors + 1))
fi
printf "Checking for zlib (>= 1.2.5.2)... "
have_zlib=0
if pkg-config --atleast-version=1.2.5.2 zlib; then
printf "Yes.\n"
have_zlib=1
zlib_cflags=$(pkg-config --cflags zlib)
zlib_ldflags=$(pkg-config --libs zlib)
else
printf "No.\n"
errors=$((errors + 1))
fi
printf "Checking for talloc development files... "
if pkg-config --exists talloc; then
printf "Yes.\n"
@ -496,6 +508,11 @@ EOF
echo " Xapian library (including development files such as headers)"
echo " http://xapian.org/"
fi
if [ $have_zlib -eq 0 ]; then
echo " zlib library (>= version 1.2.5.2, including development files such as headers)"
echo " http://zlib.net/"
echo
fi
if [ $have_gmime -eq 0 ]; then
echo " Either GMime 2.4 library" $GMIME_24_VERSION_CTR "or GMime 2.6 library" $GMIME_26_VERSION_CTR
echo " (including development files such as headers)"
@ -519,11 +536,11 @@ case a simple command will install everything you need. For example:
On Debian and similar systems:
sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev
sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev
Or on Fedora and similar systems:
sudo yum install xapian-core-devel gmime-devel libtalloc-devel
sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel
On other systems, similar commands can be used, but the details of the
package names may be different.
@ -860,6 +877,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags}
GMIME_CFLAGS = ${gmime_cflags}
GMIME_LDFLAGS = ${gmime_ldflags}
# Flags needed to compile and link against zlib
ZLIB_CFLAGS = ${zlib_cflags}
ZLIB_LDFLAGS = ${zlib_ldflags}
# Flags needed to compile and link against talloc
TALLOC_CFLAGS = ${talloc_cflags}
TALLOC_LDFLAGS = ${talloc_ldflags}
@ -898,6 +919,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
-DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
\$(ZLIB_CFLAGS) \\
\$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\
\$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\
-DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\
@ -908,5 +930,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
-DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\
-DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS)
CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)
EOF

3
doc/man1/notmuch-dump.rst
View file

@ -19,6 +19,9 @@ recreated from the messages themselves. The output of notmuch dump is
therefore the only critical thing to backup (and much more friendly to
incremental backup than the native database files.)
``--gzip``
Compress the output in a format compatible with **gzip(1)**.
``--format=(sup|batch-tag)``
Notmuch restore supports two plain text dump formats, both with one
message-id per line, followed by a list of tags.

4
notmuch-client.h
View file

@ -450,7 +450,9 @@ typedef enum dump_formats {
int
notmuch_database_dump (notmuch_database_t *notmuch,
const char *output_file_name,
const char *query_str, dump_format_t output_format);
const char *query_str,
dump_format_t output_format,
notmuch_bool_t gzip_output);
#include "command-line-arguments.h"
#endif

99
notmuch-dump.c
View file

@ -21,9 +21,11 @@
#include "notmuch-client.h"
#include "hex-escape.h"
#include "string-util.h"
#include <zlib.h>
static int
database_dump_file (notmuch_database_t *notmuch, FILE *output,
database_dump_file (notmuch_database_t *notmuch, gzFile output,
const char *query_str, int output_format)
{
notmuch_query_t *query;
@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
}
if (output_format == DUMP_FORMAT_SUP) {
fprintf (output, "%s (", message_id);
gzprintf (output, "%s (", message_id);
}
for (tags = notmuch_message_get_tags (message);
@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
const char *tag_str = notmuch_tags_get (tags);
if (! first)
fputs (" ", output);
gzputs (output, " ");
first = 0;
if (output_format == DUMP_FORMAT_SUP) {
fputs (tag_str, output);
gzputs (output, tag_str);
} else {
if (hex_encode (notmuch, tag_str,
&buffer, &buffer_size) != HEX_SUCCESS) {
@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
tag_str);
return EXIT_FAILURE;
}
fprintf (output, "+%s", buffer);
gzprintf (output, "+%s", buffer);
}
}
if (output_format == DUMP_FORMAT_SUP) {
fputs (")\n", output);
gzputs (output, ")\n");
} else {
if (make_boolean_term (notmuch, "id", message_id,
&buffer, &buffer_size)) {
@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
message_id, strerror (errno));
return EXIT_FAILURE;
}
fprintf (output, " -- %s\n", buffer);
gzprintf (output, " -- %s\n", buffer);
}
notmuch_message_destroy (message);
@ -121,24 +123,83 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
int
notmuch_database_dump (notmuch_database_t *notmuch,
const char *output_file_name,
const char *query_str, dump_format_t output_format)
const char *query_str,
dump_format_t output_format,
notmuch_bool_t gzip_output)
{
FILE *output = stdout;
int ret;
gzFile output = NULL;
const char *mode = gzip_output ? "w9" : "wT";
const char *name_for_error = output_file_name ? output_file_name : "stdout";
char *tempname = NULL;
int outfd = -1;
int ret = -1;
if (output_file_name) {
output = fopen (output_file_name, "w");
if (output == NULL) {
fprintf (stderr, "Error opening %s for writing: %s\n",
output_file_name, strerror (errno));
return EXIT_FAILURE;
tempname = talloc_asprintf (notmuch, "%s.XXXXXX", output_file_name);
outfd = mkstemp (tempname);
} else {
outfd = dup (STDOUT_FILENO);
}
if (outfd < 0) {
fprintf (stderr, "Bad output file %s\n", name_for_error);
goto DONE;
}
output = gzdopen (outfd, mode);
if (output == NULL) {
fprintf (stderr, "Error opening %s for (gzip) writing: %s\n",
name_for_error, strerror (errno));
if (close (outfd))
fprintf (stderr, "Error closing %s during shutdown: %s\n",
name_for_error, strerror (errno));
goto DONE;
}
ret = database_dump_file (notmuch, output, query_str, output_format);
if (ret) goto DONE;
if (output != stdout)
fclose (output);
ret = gzflush (output, Z_FINISH);
if (ret) {
fprintf (stderr, "Error flushing output: %s\n", gzerror (output, NULL));
goto DONE;
}
if (output_file_name) {
ret = fdatasync (outfd);
if (ret) {
fprintf (stderr, "Error syncing %s to disk: %s\n",
name_for_error, strerror (errno));
goto DONE;
}
}
if (gzclose_w (output) != Z_OK) {
fprintf (stderr, "Error closing %s: %s\n", name_for_error,
gzerror (output, NULL));
ret = EXIT_FAILURE;
output = NULL;
goto DONE;
}
if (output_file_name) {
ret = rename (tempname, output_file_name);
if (ret) {
fprintf (stderr, "Error renaming %s to %s: %s\n",
tempname, output_file_name, strerror (errno));
goto DONE;
}
}
DONE:
if (ret != EXIT_SUCCESS && output)
(void) gzclose_w (output);
if (ret != EXIT_SUCCESS && output_file_name)
(void) unlink (tempname);
return ret;
}
@ -158,6 +219,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
int opt_index;
int output_format = DUMP_FORMAT_BATCH_TAG;
notmuch_bool_t gzip_output = 0;
notmuch_opt_desc_t options[] = {
{ NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
@ -165,6 +227,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
{ "batch-tag", DUMP_FORMAT_BATCH_TAG },
{ 0, 0 } } },
{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 },
{ NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
{ 0, 0, 0, 0, 0 }
};
@ -181,7 +244,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
}
ret = notmuch_database_dump (notmuch, output_file_name, query_str,
output_format);
output_format, gzip_output);
notmuch_database_destroy (notmuch);

12
test/T240-dump-restore.sh
View file

@ -68,6 +68,18 @@ test_begin_subtest "dump --output=outfile --"
notmuch dump --output=dump-1-arg-dash.actual --
test_expect_equal_file dump.expected dump-1-arg-dash.actual
# gzipped output
test_begin_subtest "dump --gzip"
notmuch dump --gzip > dump-gzip.gz
gunzip dump-gzip.gz
test_expect_equal_file dump.expected dump-gzip
test_begin_subtest "dump --gzip --output=outfile"
notmuch dump --gzip --output=dump-gzip-outfile.gz
gunzip dump-gzip-outfile.gz
test_expect_equal_file dump.expected dump-gzip-outfile
# Note, we assume all messages from cworth have a message-id
# containing cworth.org