notmuch/devel/nmbug/notmuch-report

#!/usr/bin/env python3
#
# Copyright (c) 2011-2012 David Bremner <david@tethera.net>
#
# dependencies
#       - python3 or python2.7
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see https://www.gnu.org/licenses/ .

"""Generate text and/or HTML for one or more notmuch searches.

Messages matching each search are grouped by thread.  Each message
that contains both a subject and message-id will have the displayed
subject link to an archive view of the message (defaulting to Gmane).
"""

from __future__ import print_function
from __future__ import unicode_literals

import codecs
import collections
import datetime
import email.utils
try:  # Python 3
    from urllib.parse import quote
except ImportError:  # Python 2
    from urllib import quote
import json
import argparse
import os
import re
import sys
import subprocess
import xml.sax.saxutils


_ENCODING = 'UTF-8'
_PAGES = {}


if not hasattr(collections, 'OrderedDict'):  # Python 2.6 or earlier
    class _OrderedDict (dict):
        "Just enough of a stub to get through Page._get_threads"
        def __init__(self, *args, **kwargs):
            super(_OrderedDict, self).__init__(*args, **kwargs)
            self._keys = []  # record key order

        def __setitem__(self, key, value):
            super(_OrderedDict, self).__setitem__(key, value)
            self._keys.append(key)

        def values(self):
            for key in self._keys:
                yield self[key]


    collections.OrderedDict = _OrderedDict


class ConfigError (Exception):
    """Errors with config file usage
    """
    pass


def read_config(path=None, encoding=None):
    "Read config from json file"
    if not encoding:
        encoding = _ENCODING
    if path:
        try:
            with open(path, 'rb') as f:
                config_bytes = f.read()
        except IOError as e:
            raise ConfigError('Could not read config from {}'.format(path))
    else:
        nmbhome = os.getenv('NMBGIT', os.path.expanduser('~/.nmbug'))
        branch = 'config'
        filename = 'notmuch-report.json'

        # read only the first line from the pipe
        sha1_bytes = subprocess.Popen(
            ['git', '--git-dir', nmbhome, 'show-ref', '-s', '--heads', branch],
            stdout=subprocess.PIPE).stdout.readline()
        sha1 = sha1_bytes.decode(encoding).rstrip()
        if not sha1:
            raise ConfigError(
                ("No local branch '{branch}' in {nmbgit}.  "
                 'Checkout a local {branch} branch or explicitly set --config.'
                ).format(branch=branch, nmbgit=nmbhome))

        p = subprocess.Popen(
            ['git', '--git-dir', nmbhome, 'cat-file', 'blob',
             '{}:{}'.format(sha1, filename)],
            stdout=subprocess.PIPE)
        config_bytes, err = p.communicate()
        status = p.wait()
        if status != 0:
            raise ConfigError(
                ("Missing {filename} in branch '{branch}' of {nmbgit}.  "
                 'Add the file or explicitly set --config.'
                ).format(filename=filename, branch=branch, nmbgit=nmbhome))

    config_json = config_bytes.decode(encoding)
    try:
        return json.loads(config_json)
    except ValueError as e:
        if not path:
            path = "{} in branch '{}' of {}".format(
                filename, branch, nmbhome)
        raise ConfigError(
            'Could not parse JSON from the config file {}:\n{}'.format(
                path, e))


class Thread (list):
    def __init__(self):
        self.running_data = {}


class Page (object):
    def __init__(self, header=None, footer=None):
        self.header = header
        self.footer = footer

    def write(self, database, views, stream=None):
        if not stream:
            try:  # Python 3
                byte_stream = sys.stdout.buffer
            except AttributeError:  # Python 2
                byte_stream = sys.stdout
            stream = codecs.getwriter(encoding=_ENCODING)(stream=byte_stream)
        self._write_header(views=views, stream=stream)
        for view in views:
            self._write_view(database=database, view=view, stream=stream)
        self._write_footer(views=views, stream=stream)

    def _write_header(self, views, stream):
        if self.header:
            stream.write(self.header)

    def _write_footer(self, views, stream):
        if self.footer:
            stream.write(self.footer)

    def _write_view(self, database, view, stream):
        # sort order, default to oldest-first
        sort_key = view.get('sort', 'oldest-first')
        # dynamically accept all values in Query.SORT
        sort_attribute = sort_key.upper().replace('-', '_')
        try:
            sort = getattr(notmuch.Query.SORT, sort_attribute)
        except AttributeError:
            raise ConfigError('Invalid sort setting for {}: {!r}'.format(
                view['title'], sort_key))
        if 'query-string' not in view:
            query = view['query']
            view['query-string'] = ' and '.join(
                '( {} )'.format(q) for q in query)
        q = notmuch.Query(database, view['query-string'])
        q.set_sort(sort)
        threads = self._get_threads(messages=q.search_messages())
        self._write_view_header(view=view, stream=stream)
        self._write_threads(threads=threads, stream=stream)

    def _get_threads(self, messages):
        threads = collections.OrderedDict()
        for message in messages:
            thread_id = message.get_thread_id()
            if thread_id in threads:
                thread = threads[thread_id]
            else:
                thread = Thread()
                threads[thread_id] = thread
            thread.running_data, display_data = self._message_display_data(
                running_data=thread.running_data, message=message)
            thread.append(display_data)
        return list(threads.values())

    def _write_view_header(self, view, stream):
        pass

    def _write_threads(self, threads, stream):
        for thread in threads:
            for message_display_data in thread:
                stream.write(
                    ('{date:10.10s} {from:20.20s} {subject:40.40s}\n'
                     '{message-id-term:>72}\n'
                     ).format(**message_display_data))
            if thread != threads[-1]:
                stream.write('\n')

    def _message_display_data(self, running_data, message):
        headers = ('thread-id', 'message-id', 'date', 'from', 'subject')
        data = {}
        for header in headers:
            if header == 'thread-id':
                value = message.get_thread_id()
            elif header == 'message-id':
                value = message.get_message_id()
                data['message-id-term'] = 'id:"{0}"'.format(value)
            elif header == 'date':
                value = str(datetime.datetime.utcfromtimestamp(
                    message.get_date()).date())
            else:
                value = message.get_header(header)
            if header == 'from':
                (value, addr) = email.utils.parseaddr(value)
                if not value:
                    value = addr.split('@')[0]
            data[header] = value
        next_running_data = data.copy()
        for header, value in data.items():
            if header in ['message-id', 'subject']:
                continue
            if value == running_data.get(header, None):
                data[header] = ''
        return (next_running_data, data)


class HtmlPage (Page):
    _slug_regexp = re.compile('\W+')

    def __init__(self, message_url_template, **kwargs):
        self.message_url_template = message_url_template
        super(HtmlPage, self).__init__(**kwargs)

    def _write_header(self, views, stream):
        super(HtmlPage, self)._write_header(views=views, stream=stream)
        stream.write('<ul>\n')
        for view in views:
            if 'id' not in view:
                view['id'] = self._slug(view['title'])
            stream.write(
                '<li><a href="#{id}">{title}</a></li>\n'.format(**view))
        stream.write('</ul>\n')

    def _write_view_header(self, view, stream):
        stream.write('<h3 id="{id}">{title}</h3>\n'.format(**view))
        stream.write('<p>\n')
        if 'comment' in view:
            stream.write(view['comment'])
            stream.write('\n')
        for line in [
                'The view is generated from the following query:',
                '</p>',
                '<p>',
                '  <code>',
                view['query-string'],
                '  </code>',
                '</p>',
                ]:
            stream.write(line)
            stream.write('\n')

    def _write_threads(self, threads, stream):
        if not threads:
            return
        stream.write('<table>\n')
        for thread in threads:
            stream.write('  <tbody>\n')
            for message_display_data in thread:
                stream.write((
                    '    <tr class="message-first">\n'
                    '      <td>{date}</td>\n'
                    '      <td><code>{message-id-term}</code></td>\n'
                    '    </tr>\n'
                    '    <tr class="message-last">\n'
                    '      <td>{from}</td>\n'
                    '      <td>{subject}</td>\n'
                    '    </tr>\n'
                    ).format(**message_display_data))
            stream.write('  </tbody>\n')
            if thread != threads[-1]:
                stream.write(
                    '  <tbody><tr><td colspan="2"><br /></td></tr></tbody>\n')
        stream.write('</table>\n')

    def _message_display_data(self, *args, **kwargs):
        running_data, display_data = super(
            HtmlPage, self)._message_display_data(
                *args, **kwargs)
        if 'subject' in display_data and 'message-id' in display_data:
            d = {
                'message-id': quote(display_data['message-id']),
                'subject': xml.sax.saxutils.escape(display_data['subject']),
                }
            d['url'] = self.message_url_template.format(**d)
            display_data['subject'] = (
                '<a href="{url}">{subject}</a>'
                ).format(**d)
        for key in ['message-id', 'from']:
            if key in display_data:
                display_data[key] = xml.sax.saxutils.escape(display_data[key])
        return (running_data, display_data)

    def _slug(self, string):
        return self._slug_regexp.sub('-', string)

parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
    '--text', action='store_true', help='output plain text format')
parser.add_argument(
    '--config', metavar='PATH',
    help='load config from given file.  '
        'The format is described in notmuch-report.json(5).')
parser.add_argument(
    '--list-views', action='store_true', help='list views')
parser.add_argument(
    '--get-query', metavar='VIEW', help='get query for view')


args = parser.parse_args()

try:
    config = read_config(path=args.config)
except ConfigError as e:
    print(e, file=sys.stderr)
    sys.exit(1)

header_template = config['meta'].get('header', '''<!DOCTYPE html>
<html lang="en">
<head>
  <meta http-equiv="Content-Type" content="text/html; charset={encoding}" />
  <title>{title}</title>
  <style media="screen" type="text/css">
    h1 {{
      font-size: 1.5em;
    }}
    h2 {{
      font-size: 1.17em;
    }}
    h3 {{
      font-size: 100%;
    }}
    table {{
      border-spacing: 0;
    }}
    tr.message-first td {{
      padding-top: {inter_message_padding};
    }}
    tr.message-last td {{
      padding-bottom: {inter_message_padding};
    }}
    td {{
      padding-left: {border_radius};
      padding-right: {border_radius};
    }}
    tr:first-child td:first-child {{
      border-top-left-radius: {border_radius};
    }}
    tr:first-child td:last-child {{
      border-top-right-radius: {border_radius};
    }}
    tr:last-child td:first-child {{
      border-bottom-left-radius: {border_radius};
    }}
    tr:last-child td:last-child {{
      border-bottom-right-radius: {border_radius};
    }}
    tbody:nth-child(4n+1) tr td {{
      background-color: #ffd96e;
    }}
    tbody:nth-child(4n+3) tr td {{
      background-color: #bce;
    }}
    hr {{
      border: 0;
      height: 1px;
      color: #ccc;
      background-color: #ccc;
    }}
  </style>
</head>
<body>
<h1>{title}</h1>
<p>
{blurb}
</p>
<h2>Views</h2>
''')

footer_template = config['meta'].get('footer', '''
<hr>
<p>Generated: {datetime}</p>
</body>
</html>
''')

now = datetime.datetime.utcnow()
context = {
    'date': now,
    'datetime': now.strftime('%Y-%m-%d %H:%M:%SZ'),
    'title': config['meta']['title'],
    'blurb': config['meta']['blurb'],
    'encoding': _ENCODING,
    'inter_message_padding': '0.25em',
    'border_radius': '0.5em',
    }

_PAGES['text'] = Page()
_PAGES['html'] = HtmlPage(
    header=header_template.format(**context),
    footer=footer_template.format(**context),
    message_url_template=config['meta'].get(
        'message-url', 'https://mid.gmane.org/{message-id}'),
    )

if args.list_views:
    for view in config['views']:
        print(view['title'])
    sys.exit(0)
elif args.get_query != None:
    for view in config['views']:
        if args.get_query == view['title']:
            print(' and '.join('( {} )'.format(q) for q in view['query']))
    sys.exit(0)
else:
    # only import notmuch if needed
    import notmuch

if args.text:
    page = _PAGES['text']
else:
    page = _PAGES['html']

db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY)
page.write(database=db, views=config['views'])