lib/parse-sexp: apply macros

Macros implement lazy evaluation and lexical scope.  The former is
needed to make certain natural constructs work sensibly (e.g. (tag
,param)) but the latter is mainly future-proofing in case the DSL is
is extended to allow local bindings.

For technical background, see chapters 6 and 17 of [1] (or some other
intermediate programming languages textbook).

[1] http://cs.brown.edu/courses/cs173/2012/book/

doc/man7/notmuch-sexp-queries.rst

@@ -63,6 +63,14 @@ subqueries.
     Combine queries |q1| to |qn|, and reinterpret the result (e.g. as a regular expression).
     See :any:`modifiers` for more information.
 
+``(macro (`` |p1| ... |pn| ``) body)``
+    Define saved query with parameter substitution. The syntax is
+    recognized only in saved s-expression queries (see ``squery.*`` in
+    :any:`notmuch-config(1)`). Parameter names in ``body`` must be
+    prefixed with ``,`` to be expanded (see :any:`macro_examples`).
+    Macros may refer to other macros, but only to their own
+    parameters [#macro-details]_.
+
 .. _fields:
 
 FIELDS
@@ -234,9 +242,43 @@ EXAMPLES
     Match messages with a non-empty List-Id header, assuming
     configuration ``index.header.List=List-Id``
 
+.. _macro_examples:
+
+MACRO EXAMPLES
+--------------
+
+A macro that takes two parameters and applies different fields to them.
+
+::
+
+   $ notmuch config set squery.TagSubject '(macro (tagname subj) (and (tag ,tagname) (subject ,subj)))'
+   $ notmuch search --query=sexp '(TagSubject inbox maildir)'
+
+Nested macros are allowed.
+
+::
+
+    $ notmuch config set squery.Inner '(macro (x) (subject ,x))'
+    $ notmuch config set squery.Outer  '(macro (x y) (and (tag ,x) (Inner ,y)))'
+    $ notmuch search --query=sexp '(Outer inbox maildir)'
+
+Parameters can be re-used to reduce boilerplate. Any field, including
+user defined fields is permitted within a macro.
+
+::
+
+    $ notmuch config set squery.About '(macro (name) (or (subject ,name) (List ,name)))'
+    $ notmuch search --query=sexp '(About notmuch)'
+
+
 NOTES
 =====
 
+.. [#macro-details] Technically macros impliment lazy evaluation and
+                    lexical scope. There is one top level scope
+                    containing all macro definitions, but all
+                    parameter definitions are local to a given macro.
+
 .. [#aka-pref] a.k.a. prefixes
 
 .. [#aka-prob] a.k.a. probabilistic prefixes
@@ -256,3 +298,7 @@ NOTES
 .. |q1| replace:: :math:`q_1`
 .. |q2| replace:: :math:`q_2`
 .. |qn| replace:: :math:`q_n`
+
+.. |p1| replace:: :math:`p_1`
+.. |p2| replace:: :math:`p_2`
+.. |pn| replace:: :math:`p_n`

lib/parse-sexp.cc

@@ -7,9 +7,18 @@
 /* _sexp is used for file scope symbols to avoid clashing with
  * definitions from sexp.h */
 
-typedef struct {
+/* sexp_binding structs attach name to a sexp and a defining
+ * context. The latter allows lazy evaluation of parameters whose
+ * definition contains other parameters.  Lazy evaluation is needed
+ * because a primary goal of macros is to change the parent field for
+ * a sexp.
+ */
+
+typedef struct sexp_binding {
     const char *name;
     const sexp_t *sx;
+    const struct sexp_binding *context;
+    const struct sexp_binding *next;
 } _sexp_binding_t;
 
 typedef enum {
@@ -302,6 +311,81 @@ _sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 				sx->list->next, output);
 }
 
+static _sexp_binding_t *
+_sexp_bind (void *ctx, const _sexp_binding_t *env, const char *name, const sexp_t *sx, const
+	    _sexp_binding_t *context)
+{
+    _sexp_binding_t *binding = talloc (ctx, _sexp_binding_t);
+
+    binding->name = talloc_strdup (ctx, name);
+    binding->sx = sx;
+    binding->context = context;
+    binding->next = env;
+    return binding;
+}
+
+static notmuch_status_t
+maybe_apply_macro (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		   const _sexp_binding_t *env, const sexp_t *sx, const sexp_t *args,
+		   Xapian::Query &output)
+{
+    const sexp_t *params, *param, *arg, *body;
+    void *local = talloc_new (notmuch);
+    _sexp_binding_t *new_env = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    if (sx->list->ty != SEXP_VALUE || strcmp (sx->list->val, "macro") != 0) {
+	status = NOTMUCH_STATUS_IGNORED;
+	goto DONE;
+    }
+
+    params = sx->list->next;
+
+    if (! params || (params->ty != SEXP_LIST)) {
+	_notmuch_database_log (notmuch, "missing (possibly empty) list of arguments to macro\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    body = params->next;
+
+    if (! body) {
+	_notmuch_database_log (notmuch, "missing body of macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    for (param = params->list, arg = args;
+	 param && arg;
+	 param = param->next, arg = arg->next) {
+	if (param->ty != SEXP_VALUE || param->aty != SEXP_BASIC) {
+	    _notmuch_database_log (notmuch, "macro parameters must be unquoted atoms\n");
+	    status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	    goto DONE;
+	}
+	new_env = _sexp_bind (local, new_env, param->val, arg, env);
+    }
+
+    if (param && ! arg) {
+	_notmuch_database_log (notmuch, "too few arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    if (! param && arg) {
+	_notmuch_database_log (notmuch, "too many arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    status = _sexp_to_xapian_query (notmuch, parent, new_env, body, output);
+
+  DONE:
+    if (local)
+	talloc_free (local);
+
+    return status;
+}
+
 static notmuch_status_t
 maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 		    const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
@@ -336,7 +420,9 @@ maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 	goto DONE;
     }
 
-    status =  _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output);
+    status = maybe_apply_macro (notmuch, parent, env, saved_sexp, sx->list->next, output);
+    if (status == NOTMUCH_STATUS_IGNORED)
+	status =  _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output);
 
   DONE:
     if (local)
@@ -345,6 +431,21 @@ maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
     return status;
 }
 
+static notmuch_status_t
+_sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const char *name,
+		    Xapian::Query &output)
+{
+    for (; env; env = env->next) {
+	if (strcmp (name, env->name) == 0) {
+	    return _sexp_to_xapian_query (notmuch, parent, env->context, env->sx,
+					  output);
+	}
+    }
+    _notmuch_database_log (notmuch, "undefined parameter %s\n", name);
+    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+}
+
 /* Here we expect the s-expression to be a proper list, with first
  * element defining and operation, or as a special case the empty
  * list */
@@ -355,6 +456,10 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent
 {
     notmuch_status_t status;
 
+    if (sx->ty == SEXP_VALUE && sx->aty == SEXP_BASIC && sx->val[0] == ',') {
+	return _sexp_expand_param (notmuch, parent, env, sx->val + 1, output);
+    }
+
     if (sx->ty == SEXP_VALUE) {
 	std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
 
@@ -407,6 +512,11 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent
 	return _sexp_parse_header (notmuch, parent, env, sx, output);
     }
 
+    if (strcmp (sx->list->val, "macro") == 0) {
+	_notmuch_database_log (notmuch, "macro definition not permitted here\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
     for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) {
 	if (strcmp (prefix->name, sx->list->val) == 0) {
 	    if (prefix->flags & SEXP_FLAG_FIELD) {

test/T081-sexpr-search.sh

@@ -857,4 +857,130 @@ nested field: 'tag' inside 'subject'
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "Saved Search: list as prefix"
+notmuch config set squery.Bad2  '((and) (tag inbox) (subject maildir))'
+notmuch search --query=sexp '(Bad2)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+unexpected list in field/operation position
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Saved Search: bad parameter syntax"
+notmuch config set squery.Bad3  '(macro a b)'
+notmuch search --query=sexp '(Bad3)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+missing (possibly empty) list of arguments to macro
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Saved Search: bad parameter syntax 2"
+notmuch config set squery.Bad4  '(macro ((a b)) a)'
+notmuch search --query=sexp '(Bad4 1)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+macro parameters must be unquoted atoms
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Saved Search: bad parameter syntax 3"
+notmuch config set squery.Bad5  '(macro (a b) a)'
+notmuch search --query=sexp '(Bad5 1)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+too few arguments to macro
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Saved Search: bad parameter syntax 4"
+notmuch search --query=sexp '(Bad5 1 2 3)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+too many arguments to macro
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Saved Search: macro without body"
+notmuch config set squery.Bad3  '(macro (a b))'
+notmuch search --query=sexp '(Bad3)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+missing body of macro
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "macro in query"
+notmuch search --query=sexp '(macro (a) (and ,b (subject maildir)))' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+macro definition not permitted here
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "zero argument macro"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.TagSubject0  '(macro () (and (tag inbox) (subject maildir)))'
+notmuch search --query=sexp '(TagSubject0)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "undefined argument"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.Bad6  '(macro (a) (and ,b (subject maildir)))'
+notmuch search --query=sexp '(Bad6 foo)' >OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+undefined parameter b
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Single argument macro"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.TagSubject1  '(macro (tagname) (and (tag ,tagname) (subject maildir)))'
+notmuch search --query=sexp '(TagSubject1 inbox)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Single argument macro, list argument"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.ThingSubject  '(macro (thing) (and ,thing (subject maildir)))'
+notmuch search --query=sexp '(ThingSubject (tag inbox))' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "two argument macro"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.TagSubject2  '(macro (tagname subj) (and (tag ,tagname) (subject ,subj)))'
+notmuch search --query=sexp '(TagSubject2 inbox maildir)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "nested macros (shadowing)"
+notmuch search tag:inbox and subject:maildir | notmuch_search_sanitize > EXPECTED
+notmuch config set squery.Inner '(macro (x) (subject ,x))'
+notmuch config set squery.Outer  '(macro (x y) (and (tag ,x) (Inner ,y)))'
+notmuch search --query=sexp '(Outer inbox maildir)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "nested macros (no dynamic scope)"
+notmuch config set squery.Inner2 '(macro (x) (subject ,y))'
+notmuch config set squery.Outer2  '(macro (x y) (and (tag ,x) (Inner2 ,y)))'
+notmuch search --query=sexp '(Outer2 inbox maildir)' > OUTPUT 2>&1
+cat <<EOF > EXPECTED
+notmuch search: Syntax error in query
+undefined parameter y
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "combine macro and user defined header"
+notmuch config set squery.About '(macro (name) (or (subject ,name) (List ,name)))'
+notmuch search subject:notmuch or List:notmuch | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp '(About notmuch)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+
+test_begin_subtest "combine macro and user defined header"
+notmuch config set squery.About '(macro (name) (or (subject ,name) (List ,name)))'
+notmuch search subject:notmuch or List:notmuch | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp '(About notmuch)' | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+
 test_done