notmuch/bindings/python-cffi/notdb/_database.py

import collections
import configparser
import enum
import functools
import os
import pathlib
import weakref

import notdb._base as base
import notdb._capi as capi
import notdb._errors as errors
import notdb._message as message
import notdb._query as querymod
import notdb._tags as tags


__all__ = ['Database', 'AtomicContext', 'DbRevision']


def _config_pathname():
    """Return the path of the configuration file.

    :rtype: pathlib.Path
    """
    cfgfname = os.getenv('NOTMUCH_CONFIG', '~/.notmuch-config')
    return pathlib.Path(os.path.expanduser(cfgfname))


class Mode(enum.Enum):
    READ_ONLY = capi.lib.NOTMUCH_DATABASE_MODE_READ_ONLY
    READ_WRITE = capi.lib.NOTMUCH_DATABASE_MODE_READ_WRITE


class QuerySortOrder(enum.Enum):
    OLDEST_FIRST = capi.lib.NOTMUCH_SORT_OLDEST_FIRST
    NEWEST_FIRST = capi.lib.NOTMUCH_SORT_NEWEST_FIRST
    MESSAGE_ID = capi.lib.NOTMUCH_SORT_MESSAGE_ID
    UNSORTED = capi.lib.NOTMUCH_SORT_UNSORTED


class QueryExclude(enum.Enum):
    TRUE = capi.lib.NOTMUCH_EXCLUDE_TRUE
    FLAG = capi.lib.NOTMUCH_EXCLUDE_FLAG
    FALSE = capi.lib.NOTMUCH_EXCLUDE_FALSE
    ALL = capi.lib.NOTMUCH_EXCLUDE_ALL


class Database(base.NotmuchObject):
    """Toplevel access to notmuch.

    A :class:`Database` can be opened read-only or read-write.
    Modifications are not atomic by default, use :meth:`begin_atomic`
    for atomic updates.  If the underlying database has been modified
    outside of this class a :exc:`XapianError` will be raised and the
    instance must be closed and a new one created.

    You can use an instance of this class as a context-manager.

    :cvar MODE: The mode a database can be opened with, an enumeration
       of ``READ_ONLY`` and ``READ_WRITE``
    :cvar SORT: The sort order for search results, ``OLDEST_FIRST``,
       ``NEWEST_FIRST``, ``MESSAGE_ID`` or ``UNSORTED``.
    :cvar EXCLUDE: Which messages to exclude from queries, ``TRUE``,
       ``FLAG``, ``FALSE`` or ``ALL``.  See the query documentation
       for details.
    :cvar AddedMessage: A namedtuple ``(msg, dup)`` used by
       :meth:`add` as return value.
    :cvar STR_MODE_MAP: A map mapping strings to :attr:`MODE` items.
       This is used to implement the ``ro`` and ``rw`` string
       variants.

    :ivar closed: Boolean indicating if the database is closed or
       still open.

    :param path: The directory of where the database is stored.  If
       ``None`` the location will be read from the user's
       configuration file, respecting the ``NOTMUCH_CONFIG``
       environment variable if set.
    :type path: str, bytes, os.PathLike or pathlib.Path
    :param mode: The mode to open the database in.  One of
       :attr:`MODE.READ_ONLY` OR :attr:`MODE.READ_WRITE`.  For
       convenience you can also use the strings ``ro`` for
       :attr:`MODE.READ_ONLY` and ``rw`` for :attr:`MODE.READ_WRITE`.
    :type mode: :attr:`MODE` or str.

    :raises KeyError: if an unknown mode string is used.
    :raises OSError: or subclasses if the configuration file can not
       be opened.
    :raises configparser.Error: or subclasses if the configuration
       file can not be parsed.
    :raises NotmuchError: or subclasses for other failures.
    """

    MODE = Mode
    SORT = QuerySortOrder
    EXCLUDE = QueryExclude
    AddedMessage = collections.namedtuple('AddedMessage', ['msg', 'dup'])
    _db_p = base.MemoryPointer()
    STR_MODE_MAP = {
        'ro': MODE.READ_ONLY,
        'rw': MODE.READ_WRITE,
    }

    def __init__(self, path=None, mode=MODE.READ_ONLY):
        if isinstance(mode, str):
            mode = self.STR_MODE_MAP[mode]
        self.mode = mode
        if path is None:
            path = self.default_path()
        if not hasattr(os, 'PathLike') and isinstance(path, pathlib.Path):
            path = bytes(path)
        db_pp = capi.ffi.new('notmuch_database_t **')
        cmsg = capi.ffi.new('char**')
        ret = capi.lib.notmuch_database_open_verbose(os.fsencode(path),
                                                     mode.value, db_pp, cmsg)
        if cmsg[0]:
            msg = capi.ffi.string(cmsg[0]).decode(errors='replace')
            capi.lib.free(cmsg[0])
        else:
            msg = None
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret, msg)
        self._db_p = db_pp[0]
        self.closed = False

    @classmethod
    def create(cls, path=None):
        """Create and open database in READ_WRITE mode.

        This is creates a new notmuch database and returns an opened
        instance in :attr:`MODE.READ_WRITE` mode.

        :param path: The directory of where the database is stored.  If
           ``None`` the location will be read from the user's
           configuration file, respecting the ``NOTMUCH_CONFIG``
           environment variable if set.
        :type path: str, bytes or os.PathLike

        :raises OSError: or subclasses if the configuration file can not
           be opened.
        :raises configparser.Error: or subclasses if the configuration
           file can not be parsed.
        :raises NotmuchError: if the config file does not have the
           database.path setting.
        :raises FileError: if the database already exists.

        :returns: The newly created instance.
        """
        if path is None:
            path = cls.default_path()
        if not hasattr(os, 'PathLike') and isinstance(path, pathlib.Path):
            path = bytes(path)
        db_pp = capi.ffi.new('notmuch_database_t **')
        cmsg = capi.ffi.new('char**')
        ret = capi.lib.notmuch_database_create_verbose(os.fsencode(path),
                                                       db_pp, cmsg)
        if cmsg[0]:
            msg = capi.ffi.string(cmsg[0]).decode(errors='replace')
            capi.lib.free(cmsg[0])
        else:
            msg = None
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret, msg)

        # Now close the db and let __init__ open it.  Inefficient but
        # creating is not a hot loop while this allows us to have a
        # clean API.
        ret = capi.lib.notmuch_database_destroy(db_pp[0])
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)
        return cls(path, cls.MODE.READ_WRITE)

    @staticmethod
    def default_path(cfg_path=None):
        """Return the path of the user's default database.

        This reads the user's configuration file and returns the
        default path of the database.

        :param cfg_path: The pathname of the notmuch configuration file.
           If not specified tries to use the pathname provided in the
           :env:`NOTMUCH_CONFIG` environment variable and falls back
           to :file:`~/.notmuch-config.
        :type cfg_path: str, bytes, os.PathLike or pathlib.Path.

        :returns: The path of the database, which does not necessarily
           exists.
        :rtype: pathlib.Path
        :raises OSError: or subclasses if the configuration file can not
           be opened.
        :raises configparser.Error: or subclasses if the configuration
           file can not be parsed.
        :raises NotmuchError if the config file does not have the
           database.path setting.
        """
        if not cfg_path:
            cfg_path = _config_pathname()
        if not hasattr(os, 'PathLike') and isinstance(cfg_path, pathlib.Path):
            cfg_path = bytes(cfg_path)
        parser = configparser.ConfigParser()
        with open(cfg_path) as fp:
            parser.read_file(fp)
        try:
            return pathlib.Path(parser.get('database', 'path'))
        except configparser.Error:
            raise errors.NotmuchError(
                'No database.path setting in {}'.format(cfg_path))

    def __del__(self):
        self._destroy()

    @property
    def alive(self):
        try:
            self._db_p
        except errors.ObjectDestroyedError:
            return False
        else:
            return True

    def _destroy(self):
        try:
            ret = capi.lib.notmuch_database_destroy(self._db_p)
        except errors.ObjectDestroyedError:
            ret = capi.lib.NOTMUCH_STATUS_SUCCESS
        else:
            self._db_p = None
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)

    def close(self):
        """Close the notmuch database.

        Once closed most operations will fail.  This can still be
        useful however to explicitly close a database which is opened
        read-write as this would otherwise stop other processes from
        reading the database while it is open.

        :raises ObjectDestroyedError: if used after destroyed.
        """
        ret = capi.lib.notmuch_database_close(self._db_p)
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)
        self.closed = True

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    @property
    def path(self):
        """The pathname of the notmuch database.

        This is returned as a :class:`pathlib.Path` instance.

        :raises ObjectDestroyedError: if used after destoryed.
        """
        try:
            return self._cache_path
        except AttributeError:
            ret = capi.lib.notmuch_database_get_path(self._db_p)
            self._cache_path = pathlib.Path(os.fsdecode(capi.ffi.string(ret)))
            return self._cache_path

    @property
    def version(self):
        """The database format version.

        This is a positive integer.

        :raises ObjectDestroyedError: if used after destoryed.
        """
        try:
            return self._cache_version
        except AttributeError:
            ret = capi.lib.notmuch_database_get_version(self._db_p)
            self._cache_version = ret
            return ret

    @property
    def needs_upgrade(self):
        """Whether the database should be upgraded.

        If *True* the database can be upgraded using :meth:`upgrade`.
        Not doing so may result in some operations raising
        :exc:`UpgradeRequiredError`.

        A read-only database will never be upgradable.

        :raises ObjectDestroyedError: if used after destoryed.
        """
        ret = capi.lib.notmuch_database_needs_upgrade(self._db_p)
        return bool(ret)

    def upgrade(self, progress_cb=None):
        """Upgrade the database to the latest version.

        Upgrade the database, optionally with a progress callback
        which should be a callable which will be called with a
        floating point number in the range of [0.0 .. 1.0].
        """
        raise NotImplementedError

    def atomic(self):
        """Return a context manager to perform atomic operations.

        The returned context manager can be used to perform atomic
        operations on the database.

        .. note:: Unlinke a traditional RDBMS transaction this does
           not imply durability, it only ensures the changes are
           performed atomically.

        :raises ObjectDestroyedError: if used after destoryed.
        """
        ctx = AtomicContext(self, '_db_p')
        return ctx

    def revision(self):
        """The currently committed revision in the database.

        Returned as a ``(revision, uuid)`` namedtuple.

        :raises ObjectDestroyedError: if used after destoryed.
        """
        raw_uuid = capi.ffi.new('char**')
        rev = capi.lib.notmuch_database_get_revision(self._db_p, raw_uuid)
        return DbRevision(rev, capi.ffi.string(raw_uuid[0]))

    def get_directory(self, path):
        raise NotImplementedError

    def add(self, filename, *, sync_flags=False):
        """Add a message to the database.

        Add a new message to the notmuch database.  The message is
        referred to by the pathname of the maildir file.  If the
        message ID of the new message already exists in the database,
        this adds ``pathname`` to the list of list of files for the
        existing message.

        :param filename: The path of the file containing the message.
        :type filename: str, bytes, os.PathLike or pathlib.Path.
        :param sync_flags: Whether to sync the known maildir flags to
           notmuch tags.  See :meth:`Message.flags_to_tags` for
           details.

        :returns: A tuple where the first item is the newly inserted
           messages as a :class:`Message` instance, and the second
           item is a boolean indicating if the message inserted was a
           duplicate.  This is the namedtuple ``AddedMessage(msg,
           dup)``.
        :rtype: Database.AddedMessage

        If an exception is raised, no message was added.

        :raises XapianError: A Xapian exception occurred.
        :raises FileError: The file referred to by ``pathname`` could
           not be opened.
        :raises FileNotEmailError: The file referreed to by
           ``pathname`` is not recognised as an email message.
        :raises ReadOnlyDatabaseError: The database is opened in
           READ_ONLY mode.
        :raises UpgradeRequiredError: The database must be upgraded
           first.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        if not hasattr(os, 'PathLike') and isinstance(filename, pathlib.Path):
            filename = bytes(filename)
        msg_pp = capi.ffi.new('notmuch_message_t **')
        ret = capi.lib.notmuch_database_add_message(self._db_p,
                                                    os.fsencode(filename),
                                                    msg_pp)
        ok = [capi.lib.NOTMUCH_STATUS_SUCCESS,
              capi.lib.NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID]
        if ret not in ok:
            raise errors.NotmuchError(ret)
        msg = message.Message(self, msg_pp[0], db=self)
        if sync_flags:
            msg.tags.from_maildir_flags()
        return self.AddedMessage(
            msg, ret == capi.lib.NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)

    def remove(self, filename):
        """Remove a message from the notmuch database.

        Removing a message which is not in the database is just a
        silent nop-operation.

        :param filename: The pathname of the file containing the
           message to be removed.
        :type filename: str, bytes, os.PathLike or pathlib.Path.

        :returns: True if the message is still in the database.  This
           can happen when multiple files contain the same message ID.
           The true/false distinction is fairly arbitrary, but think
           of it as ``dup = db.remove_message(name); if dup: ...``.
        :rtype: bool

        :raises XapianError: A Xapian exception ocurred.
        :raises ReadOnlyDatabaseError: The database is opened in
           READ_ONLY mode.
        :raises UpgradeRequiredError: The database must be upgraded
           first.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        if not hasattr(os, 'PathLike') and isinstance(filename, pathlib.Path):
            filename = bytes(filename)
        ret = capi.lib.notmuch_database_remove_message(self._db_p,
                                                       os.fsencode(filename))
        ok = [capi.lib.NOTMUCH_STATUS_SUCCESS,
              capi.lib.NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID]
        if ret not in ok:
            raise errors.NotmuchError(ret)
        if ret == capi.lib.NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
            return True
        else:
            return False

    def find(self, msgid):
        """Return the message matching the given message ID.

        If a message with the given message ID is found a
        :class:`Message` instance is returned.  Otherwise a
        :exc:`LookupError` is raised.

        :param msgid: The message ID to look for.
        :type msgid: str

        :returns: The message instance.
        :rtype: Message

        :raises LookupError: If no message was found.
        :raises OutOfMemoryError: When there is no memory to allocate
           the message instance.
        :raises XapianError: A Xapian exception ocurred.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        msg_pp = capi.ffi.new('notmuch_message_t **')
        ret = capi.lib.notmuch_database_find_message(self._db_p,
                                                     msgid.encode(), msg_pp)
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)
        msg_p = msg_pp[0]
        if msg_p == capi.ffi.NULL:
            raise LookupError
        msg = message.Message(self, msg_p, db=self)
        return msg

    def get(self, filename):
        """Return the :class:`Message` given a pathname.

        If a message with the given pathname exists in the database
        return the :class:`Message` instance for the message.
        Otherwise raise a :exc:`LookupError` exception.

        :param filename: The pathname of the message.
        :type filename: str, bytes, os.PathLike or pathlib.Path

        :returns: The message instance.
        :rtype: Message

        :raises LookupError: If no message was found.  This is also
           a subclass of :exc:`KeyError`.
        :raises OutOfMemoryError: When there is no memory to allocate
           the message instance.
        :raises XapianError: A Xapian exception ocurred.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        if not hasattr(os, 'PathLike') and isinstance(filename, pathlib.Path):
            filename = bytes(filename)
        msg_pp = capi.ffi.new('notmuch_message_t **')
        ret = capi.lib.notmuch_database_find_message_by_filename(
            self._db_p, os.fsencode(filename), msg_pp)
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)
        msg_p = msg_pp[0]
        if msg_p == capi.ffi.NULL:
            raise LookupError
        msg = message.Message(self, msg_p, db=self)
        return msg

    @property
    def tags(self):
        """Return an immutable set with all tags used in this database.

        This returns an immutable set-like object implementing the
        collections.abc.Set Abstract Base Class.  Due to the
        underlying libnotmuch implementation some operations have
        different performance characteristics then plain set objects.
        Mainly any lookup operation is O(n) rather then O(1).

        Normal usage treats tags as UTF-8 encoded unicode strings so
        they are exposed to Python as normal unicode string objects.
        If you need to handle tags stored in libnotmuch which are not
        valid unicode do check the :class:`ImmutableTagSet` docs for
        how to handle this.

        :rtype: ImmutableTagSet

        :raises ObjectDestroyedError: if used after destoryed.
        """
        try:
            ref = self._cached_tagset
        except AttributeError:
            tagset = None
        else:
            tagset = ref()
        if tagset is None:
            tagset = tags.ImmutableTagSet(
                self, '_db_p', capi.lib.notmuch_database_get_all_tags)
            self._cached_tagset = weakref.ref(tagset)
        return tagset

    def _create_query(self, query, *,
                      omit_excluded=EXCLUDE.TRUE,
                      sort=SORT.UNSORTED,  # Check this default
                      exclude_tags=None):
        """Create an internal query object.

        :raises OutOfMemoryError: if no memory is available to
           allocate the query.
        """
        if isinstance(query, str):
            query = query.encode('utf-8')
        query_p = capi.lib.notmuch_query_create(self._db_p, query)
        if query_p == capi.ffi.NULL:
            raise errors.OutOfMemoryError()
        capi.lib.notmuch_query_set_omit_excluded(query_p, omit_excluded.value)
        capi.lib.notmuch_query_set_sort(query_p, sort.value)
        if exclude_tags is not None:
            for tag in exclude_tags:
                if isinstance(tag, str):
                    tag = str.encode('utf-8')
                capi.lib.notmuch_query_add_tag_exclude(query_p, tag)
        return querymod.Query(self, query_p)

    def messages(self, query, *,
                 omit_excluded=EXCLUDE.TRUE,
                 sort=SORT.UNSORTED,  # Check this default
                 exclude_tags=None):
        """Search the database for messages.

        :returns: An iterator over the messages found.
        :rtype: MessageIter

        :raises OutOfMemoryError: if no memory is available to
           allocate the query.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        query = self._create_query(query,
                                   omit_excluded=omit_excluded,
                                   sort=sort,
                                   exclude_tags=exclude_tags)
        return query.messages()

    def count_messages(self, query, *,
                       omit_excluded=EXCLUDE.TRUE,
                       sort=SORT.UNSORTED,  # Check this default
                       exclude_tags=None):
        """Search the database for messages.

        :returns: An iterator over the messages found.
        :rtype: MessageIter

        :raises ObjectDestroyedError: if used after destoryed.
        """
        query = self._create_query(query,
                                   omit_excluded=omit_excluded,
                                   sort=sort,
                                   exclude_tags=exclude_tags)
        return query.count_messages()

    def threads(self,  query, *,
                omit_excluded=EXCLUDE.TRUE,
                sort=SORT.UNSORTED,  # Check this default
                exclude_tags=None):
        query = self._create_query(query,
                                   omit_excluded=omit_excluded,
                                   sort=sort,
                                   exclude_tags=exclude_tags)
        return query.threads()

    def count_threads(self, query, *,
                      omit_excluded=EXCLUDE.TRUE,
                      sort=SORT.UNSORTED,  # Check this default
                      exclude_tags=None):
        query = self._create_query(query,
                                   omit_excluded=omit_excluded,
                                   sort=sort,
                                   exclude_tags=exclude_tags)
        return query.count_threads()

    def status_string(self):
        raise NotImplementedError

    def __repr__(self):
        return 'Database(path={self.path}, mode={self.mode})'.format(self=self)


class AtomicContext:
    """Context manager for atomic support.

    This supports the notmuch_database_begin_atomic and
    notmuch_database_end_atomic API calls.  The object can not be
    directly instantiated by the user, only via ``Database.atomic``.
    It does keep a reference to the :class:`Database` instance to keep
    the C memory alive.

    :raises XapianError: When this is raised at enter time the atomic
       section is not active.  When it is raised at exit time the
       atomic section is still active and you may need to try using
       :meth:`force_end`.
    :raises ObjectDestroyedError: if used after destoryed.
    """

    def __init__(self, db, ptr_name):
        self._db = db
        self._ptr = lambda: getattr(db, ptr_name)

    def __del__(self):
        self._destroy()

    @property
    def alive(self):
        return self.parent.alive

    def _destroy(self):
        pass

    def __enter__(self):
        ret = capi.lib.notmuch_database_begin_atomic(self._ptr())
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        ret = capi.lib.notmuch_database_end_atomic(self._ptr())
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)

    def force_end(self):
        """Force ending the atomic section.

        This can only be called once __exit__ has been called.  It
        will attept to close the atomic section (again).  This is
        useful if the original exit raised an exception and the atomic
        section is still open.  But things are pretty ugly by now.

        :raises XapianError: If exiting fails, the atomic section is
           not ended.
        :raises UnbalancedAtomicError: If the database was currently
           not in an atomic section.
        :raises ObjectDestroyedError: if used after destoryed.
        """
        ret = capi.lib.notmuch_database_end_atomic(self._ptr())
        if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
            raise errors.NotmuchError(ret)


@functools.total_ordering
class DbRevision:
    """A database revision.

    The database revision number increases monotonically with each
    commit to the database.  Which means user-visible changes can be
    ordered.  This object is sortable with other revisions.  It
    carries the UUID of the database to ensure it is only ever
    compared with revisions from the same database.
    """

    def __init__(self, rev, uuid):
        self._rev = rev
        self._uuid = uuid

    @property
    def rev(self):
        """The revision number, a positive integer."""
        return self._rev

    @property
    def uuid(self):
        """The UUID of the database, consider this opaque."""
        return self._uuid

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            if self.uuid != other.uuid:
                return False
            return self.rev == other.rev
        else:
            return NotImplemented

    def __lt__(self, other):
        if self.__class__ is other.__class__:
            if self.uuid != other.uuid:
                return False
            return self.rev < other.rev
        else:
            return NotImplemented

    def __repr__(self):
        return 'DbRevision(rev={self.rev}, uuid={self.uuid})'.format(self=self)