devel/notmuch-web: single user web front end using python-cffi

Originally contributed by Brian Sniffen [1]. Quite a few unpublished
fixes from Daniel Kahn Gilmour. We've been running it a few years now
as the "official" archive of the notmuch mailing list. There also a
few fixes from myself and Austin Ray. Finally I converted it to the
new python bindings.

This commit squashes the several years of development history and
moves it under devel, in recognition of the now established role it
plays in the project infrastructure.

[1]: id:87tvyvp4f2.fsf@istari.evenmere.org

NEWS

@@ -1,3 +1,15 @@
+New add-on tool: notmuch-web
+-----------------------------
+
+The new contrib/ tool `notmuch-web` is a very thin web client.  It
+supports a full search interface for one user: there is no facility
+for multiple users provided today.  See the notmuch-web README file
+for more information.
+
+Be careful about running it on a network-connected system: it will
+expose a web interface that requires no authentication but exposes
+your mail store.
+
 Notmuch 0.35 (2022-02-06)
 =========================
 

devel/notmuch-web/nmgunicorn.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+# to launch nmweb from gunicorn.
+
+from nmweb import urls, index, search, show
+import web
+
+app = web.application(urls, globals())
+
+# get the wsgi app from web.py application object
+wsgiapp = app.wsgifunc()

devel/notmuch-web/nmweb.py

@@ -0,0 +1,366 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+
+try:
+  from urllib.parse import quote_plus
+  from urllib.parse import unquote_plus
+except ImportError:
+  from urllib import quote_plus
+  from urllib import unquote_plus
+
+from datetime import datetime
+from mailbox import MaildirMessage
+import mimetypes
+import email
+import re
+import html
+import os
+
+import bleach
+import web
+from notmuch2 import Database
+from jinja2 import Environment, FileSystemLoader # FIXME to PackageLoader
+from jinja2 import Markup
+try:
+  import bjoern # from https://github.com/jonashaag/bjoern/
+  use_bjoern = True
+except:
+  use_bjoern = False
+
+# Configuration options
+safe_tags = bleach.sanitizer.ALLOWED_TAGS + \
+            [u'div', u'span', u'p', u'br', u'table', u'tr', u'td', u'th']
+linkify_plaintext = True # delays page load by about 0.02s of 0.20s budget
+show_thread_nav = True   # delays page load by about 0.04s of 0.20s budget
+
+prefix = os.environ.get('NMWEB_PREFIX', "http://localhost:8080")
+webprefix = os.environ.get('NMWEB_STATIC', prefix + "/static")
+cachedir = os.environ.get('NMWEB_CACHE', "static/cache") # special for webpy server; changeable if using your own
+cachepath = os.environ.get('NMWEB_CACHE_PATH', cachedir) # location of static cache in the local filesystem
+
+if 'NMWEB_DEBUG' in os.environ:
+  web.config.debug = True
+else:
+  web.config.debug = False
+
+# End of config options
+
+env = Environment(autoescape=True,
+                  loader=FileSystemLoader('templates'))
+
+urls = (
+  '/', 'index',
+  '/search/(.*)', 'search',
+  '/show/(.*)', 'show',
+)
+
+def urlencode_filter(s):
+  if type(s) == 'Markup':
+    s = s.unescape()
+  s = s.encode('utf8')
+  s = quote_plus(s)
+  return Markup(s)
+env.filters['url'] = urlencode_filter
+
+class index:
+  def GET(self):
+    web.header('Content-type', 'text/html')
+    base = env.get_template('base.html')
+    template = env.get_template('index.html')
+    db = Database()
+    tags = db.tags
+    return template.render(tags=tags,
+                           title="Notmuch webmail",
+                           prefix=prefix,
+                           sprefix=webprefix)
+
+class search:
+  def GET(self, terms):
+    redir = False
+    if web.input(terms=None).terms:
+      redir = True
+      terms = web.input().terms
+    terms = unquote_plus (terms)
+    if web.input(afters=None).afters:
+      afters = web.input(afters=None).afters[:-3]
+    else:
+      afters = '0'
+    if web.input(befores=None).befores:
+      befores = web.input(befores=None).befores
+    else:
+      befores = '4294967296' # 2^32
+    try:
+      if int(afters) > 0 or int(befores) < 4294967296:
+        redir = True
+        terms += ' date:@%s..@%s' % (int(afters), int(befores))
+    except ValueError:
+      pass
+    if redir:
+      raise web.seeother('/search/%s' % quote_plus(terms.encode('utf8')))
+    web.header('Content-type', 'text/html')
+    db = Database()
+    ts = db.threads(query=terms, sort=Database.SORT.NEWEST_FIRST)
+    template = env.get_template('search.html')
+    return template.generate(terms=terms,
+                             ts=ts,
+                             title=terms,
+                             prefix=prefix,
+                             sprefix=webprefix)
+
+def format_time_range(start, end):
+  if end-start < (60*60*24):
+    time = datetime.fromtimestamp(start).strftime('%Y %b %d %H:%M')
+  else:
+    start = datetime.fromtimestamp(start).strftime("%Y %b %d")
+    end = datetime.fromtimestamp(end).strftime("%Y %b %d")
+    time = "%s through %s" % (start, end)
+  return time
+env.globals['format_time_range'] = format_time_range
+
+def mailto_addrs(msg,header_name):
+  try:
+    hdr = msg.header(header_name)
+  except LookupError:
+    return ''
+
+  frm = email.utils.getaddresses([hdr])
+  return ','.join(['<a href="mailto:%s">%s</a> ' % ((l, p) if p else (l, l)) for (p, l) in frm])
+env.globals['mailto_addrs'] = mailto_addrs
+
+def link_msg(msg):
+  lnk = quote_plus(msg.messageid.encode('utf8'))
+  try:
+    subj = msg.header('Subject')
+  except LookupError:
+    subj = ""
+  out = '<a href="%s/show/%s">%s</a>' % (prefix, lnk, subj)
+  return out
+env.globals['link_msg'] = link_msg
+
+def show_msgs(msgs):
+  r = '<ul>'
+  for msg in msgs:
+    red = 'color:black; font-style:normal'
+    if msg.matched:
+      red = 'color:red; font-style:italic'
+    frm = mailto_addrs(msg,'From')
+    lnk = link_msg(msg)
+    tags = ", ".join(msg.tags)
+    rs = show_msgs(msg.replies())
+    r += '<li><span style="%s">%s&mdash;%s</span> [%s] %s</li>' % (red, frm, lnk, tags, rs)
+  r += '</ul>'
+  return r
+env.globals['show_msgs'] = show_msgs
+
+# As email.message.walk, but showing close tags as well
+def mywalk(self):
+  yield self
+  if self.is_multipart():
+    for subpart in self.get_payload():
+      for subsubpart in mywalk(subpart):
+        yield subsubpart
+    yield 'close-div'
+
+class show:
+  def GET(self, mid):
+    web.header('Content-type', 'text/html')
+    db = Database()
+    try:
+      m = db.find(mid)
+    except:
+      raise web.notfound("No such message id.")
+    template = env.get_template('show.html')
+    # FIXME add reply-all link with email.urils.getaddresses
+    # FIXME add forward link using mailto with body parameter?
+    return template.render(m=m,
+                           mid=mid,
+                           title=m.header('Subject'),
+                           prefix=prefix,
+                           sprefix=webprefix)
+
+def thread_nav(m):
+  if not show_thread_nav: return
+  db = Database()
+  thread = next(db.threads('thread:'+m.threadid))
+  prv = None
+  found = False
+  nxt = None
+  for msg in thread:
+    if m == msg:
+      found = True
+    elif not found:
+      prv = msg
+    else: # found message, but not on this loop
+      nxt = msg
+      break
+  yield "<hr><ul>"
+  if prv: yield "<li>Previous message (by thread): %s</li>" % link_msg(prv)
+  if nxt: yield "<li>Next message (by thread): %s</li>" % link_msg(nxt)
+  yield "</ul><h3>Thread:</h3>"
+  # FIXME show now takes three queries instead of 1;
+  # can we yield the message body while computing the thread shape?
+  thread = next(db.threads('thread:'+m.threadid))
+  yield show_msgs(thread.toplevel())
+  return
+env.globals['thread_nav'] = thread_nav
+
+def format_message(nm_msg, mid):
+  fn = list(nm_msg.filenames())[0]
+  msg = MaildirMessage(open(fn))
+  return format_message_walk(msg, mid)
+
+def decodeAnyway(txt, charset='ascii'):
+  try:
+    out = txt.decode(charset)
+  except:
+    try:
+      out = txt.decode('utf-8')
+    except UnicodeDecodeError:
+      out = txt.decode('latin1')
+  return out
+
+def require_protocol_prefix(attrs, new=False):
+  if not new:
+    return attrs
+  link_text = attrs[u'_text']
+  if link_text.startswith(('http:', 'https:', 'mailto:', 'git:', 'id:')):
+    return attrs
+  return None
+
+# Bleach doesn't even try to linkify id:... text, so no point invoking this yet
+def modify_id_links(attrs, new=False):
+  if attrs[(None, u'href')].startswith(u'id:'):
+    attrs[(None, u'href')] = prefix + "/show/" + attrs[(None, u'href')][3:]
+  return attrs
+
+def css_part_id(content_type, parts=[]):
+  c = content_type.replace('/', '-')
+  out = "-".join(parts + [c])
+  return out
+
+def format_message_walk(msg, mid):
+  counter = 0
+  cid_refd = []
+  parts = ['main']
+  for part in mywalk(msg):
+    if part == 'close-div':
+      parts.pop()
+      yield '</div>'
+    elif part.get_content_maintype() == 'multipart':
+      yield '<div class="multipart-%s" id="%s">' % \
+          (part.get_content_subtype(), css_part_id(part.get_content_type(), parts))
+      parts.append(part.get_content_subtype())
+      if part.get_content_subtype() == 'alternative':
+        yield '<ul>'
+        for subpart in part.get_payload():
+          yield ('<li><a href="#%s">%s</a></li>' %
+                 (css_part_id(subpart.get_content_type(), parts),
+                  subpart.get_content_type()))
+        yield '</ul>'
+    elif part.get_content_type() == 'message/rfc822':
+      # FIXME extract subject, date, to/cc/from into a separate template and use it here
+      yield '<div class="message-rfc822">'
+    elif part.get_content_maintype() == 'text':
+      if part.get_content_subtype() == 'plain':
+        yield '<div id="%s">' % css_part_id(part.get_content_type(), parts)
+        yield '<pre>'
+        out = part.get_payload(decode=True)
+        out = decodeAnyway(out, part.get_content_charset('ascii'))
+        out = html.escape(out)
+        out = out.encode('ascii', 'xmlcharrefreplace').decode('ascii')
+        if linkify_plaintext: out = bleach.linkify(out, callbacks=[require_protocol_prefix])
+        yield out
+        yield '</pre></div>'
+      elif part.get_content_subtype() == 'html':
+        yield '<div id="%s">' % css_part_id(part.get_content_type(), parts)
+        unb64 = part.get_payload(decode=True)
+        decoded = decodeAnyway(unb64, part.get_content_charset('ascii'))
+        cid_refd += find_cids(decoded)
+        part.set_payload(bleach.clean(replace_cids(decoded, mid), tags=safe_tags).
+                         encode(part.get_content_charset('ascii'), 'xmlcharrefreplace'))
+        (filename, cid) = link_to_cached_file(part, mid, counter)
+        counter += 1
+        yield '<iframe class="embedded-html" src="%s"></iframe>' % \
+            os.path.join(prefix, cachedir, mid, filename)
+        yield '</div>'
+      else:
+        yield '<div id="%s">' % css_part_id(part.get_content_type(), parts)
+        (filename, cid) = link_to_cached_file(part, mid, counter)
+        counter += 1
+        yield '<a href="%s">%s (%s)</a>' % (os.path.join(prefix,
+                                                         cachedir,
+                                                         mid,
+                                                         filename),
+                                            filename,
+                                            part.get_content_type())
+        yield '</div>'
+    elif part.get_content_maintype() == 'image':
+      (filename, cid) = link_to_cached_file(part, mid, counter)
+      if cid not in cid_refd:
+        counter += 1
+        yield '<img src="%s" alt="%s">' % (os.path.join(prefix,
+                                                        cachedir,
+                                                        mid,
+                                                        filename),
+                                           filename)
+    else:
+      (filename, cid) = link_to_cached_file(part, mid, counter)
+      counter += 1
+      yield '<a href="%s">%s (%s)</a>' % (os.path.join(prefix,
+                                                       cachedir,
+                                                       mid,
+                                                       filename),
+                                          filename,
+                                          part.get_content_type())
+env.globals['format_message'] = format_message
+
+def replace_cids(body, mid):
+  return body.replace('cid:', os.path.join(prefix, cachedir, mid)+'/')
+
+def find_cids(body):
+  return re.findall(r'cid:([^ "\'>]*)', body)
+
+def link_to_cached_file(part, mid, counter):
+  filename = part.get_filename()
+  if not filename:
+    ext = mimetypes.guess_extension(part.get_content_type())
+    if not ext:
+      ext = '.bin'
+    filename = 'part-%03d%s' % (counter, ext)
+  try:
+    os.makedirs(os.path.join(cachepath, mid))
+  except OSError:
+    pass
+  fn = os.path.join(cachepath, mid, filename) # FIXME escape mid, filename
+  fp = open(fn, 'wb')
+  if part.get_content_maintype() == 'text':
+    data = part.get_payload(decode=True)
+    data = decodeAnyway(data, part.get_content_charset('ascii')).encode('utf-8')
+  else:
+    try:
+      data = part.get_payload(decode=True)
+    except:
+      data = part.get_payload(decode=False)
+  if data:
+    fp.write(data)
+  fp.close()
+  if 'Content-ID' in part:
+    cid = part['Content-ID']
+    if cid[0] == '<' and cid[-1] == '>': cid = cid[1:-1]
+    cid_fn = os.path.join(cachepath, mid, cid) # FIXME escape mid, cid
+    try:
+      os.unlink(cid_fn)
+    except OSError:
+      pass
+    os.link(fn, cid_fn)
+    return (filename, cid)
+  else:
+    return (filename, None)
+
+if __name__ == '__main__':
+  app = web.application(urls, globals())
+  if use_bjoern:
+    bjoern.run(app.wsgifunc(), "127.0.0.1", 8080)
+  else:
+    app.run()

devel/notmuch-web/static/css/jquery-ui.css

@@ -0,0 +1 @@
+/usr/share/javascript/jquery-ui/themes/base/jquery-ui.min.css

devel/notmuch-web/static/css/notmuch-0.1.css

@@ -0,0 +1,15 @@
+pre {
+  white-space: pre-wrap;
+}
+
+.message-rfc822 {
+  border: 1px solid;
+  border-radius: 25px;
+}
+
+.embedded-html {
+  frameborder: 0;
+  border: 0;
+  scrolling: no;
+  width: 100%;
+}

devel/notmuch-web/static/js/jquery-ui.js

@@ -0,0 +1 @@
+/usr/share/javascript/jquery-ui/jquery-ui.min.js

devel/notmuch-web/static/js/jquery.js

@@ -0,0 +1 @@
+/usr/share/javascript/jquery/jquery.min.js

devel/notmuch-web/static/js/notmuch-0.1.js

@@ -0,0 +1,35 @@
+$(function(){
+  $("#after").datepicker({
+    altField: "#afters",
+    altFormat: "@",
+    changeMonth: true,
+    changeYear: true,
+    defaultDate: "-7d",
+    minDate: "01/01/1970",
+    yearRange: "2000:+0",
+    onSelect: function(selectedDate) {
+      $("#before").datepicker("option","minDate",selectedDate);
+    }
+  });
+  $("#before").datepicker({
+    altField: "#befores",
+    altFormat: "@",
+    changeMonth: true,
+    changeYear: true,
+    defaultDate: "+1d",
+    maxDate: "+1d",
+    yearRange: "2000:+0",
+    onSelect: function(selectedDate) {
+      $("#after").datepicker("option","maxDate",selectedDate);
+    }
+  });
+  $(function(){
+  $('.multipart-alternative').tabs()
+  });
+  $(function(){
+      $('.embedded-html').on('load',function(){
+      this.style.height = this.contentWindow.document.body.offsetHeight + 'px';
+    });
+  });
+});
+

devel/notmuch-web/templates/base.html

@@ -0,0 +1,39 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html lang="en">
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"
+  />
+	<meta name="viewport" content="width=device-width, initial-scale=1"> 
+<link type="text/css" href="{{sprefix}}/css/jquery-ui.css" rel="stylesheet" />
+<link type="text/css" href="{{sprefix}}/css/notmuch-0.1.css" rel="stylesheet" />
+<script type="text/javascript" src="{{sprefix}}/js/jquery.js"></script>
+<script type="text/javascript" src="{{sprefix}}/js/jquery-ui.js"></script>
+<script type="text/javascript" src="{{sprefix}}/js/notmuch-0.1.js"></script>
+<title>{{title}}</title>
+</head><body>
+<div data-role="page">
+<div data-role="header">
+{% block searchform %}
+<form action="{{prefix}}/search/" method="GET" data-ajax="false">
+<label for="terms">Terms</label><input id="terms" name="terms">
+<label for="after">After</label><input id="after"
+name="after"><input type="hidden" id="afters" name="afters">
+<label for="before">Before</label><input id="before"
+name="before"><input id="befores" type="hidden" name="befores">
+<input type="submit" name="submit" id="submit" value="Search">
+</form>
+{% endblock searchform %}
+<h2>{{title}}</h2>
+</div>
+<div data-role="content">
+{% block content %}
+<h2>Common tags</h2>
+<ul>
+{% for tag in tags %}
+  <li><a href="search/tag:{{ tag|url }}">{{ tag|e }}</a></li>
+{% endfor %}
+</ul>
+</div>
+{% endblock content %}
+</div>
+</body></html>

devel/notmuch-web/templates/index.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block content %}
+<h2>Common tags</h2>
+<ul>
+{% for tag in tags %}
+  <li><a href="search/tag:{{ tag|url }}">{{ tag|e }}</a></li>
+{% endfor %}
+</ul>
+{% endblock content %}

devel/notmuch-web/templates/search.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+<h1>{{ terms|e }}</h1>
+{% block content %}
+{% for t in ts %}
+  <h2>{{ t.subject|e }}</h2>
+  <p><i>{{ t.authors|e }}</i></p>
+  <p><b>{{ format_time_range(t.first,t.last)|e }}</b></p>
+  {{ show_msgs(t.toplevel())|safe }}
+{% endfor %}
+{% endblock content %}

devel/notmuch-web/templates/show.html

@@ -0,0 +1,15 @@
+{% extends "base.html" %}
+{% block content %}
+{% set headers = ['Subject', 'Date'] %}
+{% set addr_headers = ['To', 'Cc', 'From'] %}
+{% for header in headers: %}
+<p><b>{{header}}:</b>{{m.header(header)|e}}</p>
+{% endfor %}
+{% for header in addr_headers: %}
+<p><b>{{header}}:</b>{{mailto_addrs(m,header)|safe}}</p>
+{% endfor %}
+<hr>
+{% for part in format_message(m,mid): %}{{ part|safe }}{% endfor %}
+{% for b in thread_nav(m): %}{{b|safe}}{% endfor %}
+<hr>
+{% endblock content %}

devel/notmuch-web/todo

@@ -0,0 +1,14 @@
+review escaping and safety handling mail from Bad People
+
+revise template loader---can we make this faster?
+
+add reply-all link with email.urils.getaddresses
+
+change current reply links to quote body
+
+add forward link using mailto with body parameter?
+
+unescape the current search term, including translating back dates
+
+
+later: json support, iOS app?