Merge pull request #53 from jonathanhogg/enh_pure_expression_tree

Convert language into a pure expression tree

Author: jonathanhogg

docs/language.md

@@ -692,11 +692,11 @@ an insufficient number of matching arguments, otherwise any parameters lacking
 matching arguments will be bound to `null`. The result of evaluating all
 body expressions will be returned as a single vector to the caller.
 
-Functions may only be declared at the top-level of a program and so cannot be
-nested within another function definition. Functions may refer to names defined
-above the function definition at the top level. These will be captured at
-definition time. Thus rebinding a name later in the same scope will be ignored.
-For example:
+Functions may be declared anywhere in a program including within another
+function definition. Functions may refer to names defined outside of the
+function definition at the top level. The values of these names will be
+captured at definition time and so rebinding a name later in the same scope
+will be ignored. For example:
 
 ```flitter
 let x=10
@@ -732,13 +732,41 @@ the value `1` and `z` taking the value `3`. Named arguments should be given
 *after* any positional arguments and should not repeat positional arguments.
 
 Functions that have all literal (or unspecified) default parameter values, and
-that do not reference any non-local names within the body, will be in-lined by
+that do not reference any non-local names within the body, will be inlined by
 the simplifier at each call site. The simplifier is then able to bind the
 parameters to the argument expressions and continue simplifying the body on
 that basis. Therefore, it is often more efficient to pass dynamic values (such
 as `beat`) into the function as parameters than allow them to be captured from
 the environment.
 
+## Anonymous functions
+
+**Flitter** also allows anonymous functions to be defined and used as values
+with the syntax:
+
+```flitter
+func (parameter《=default》《, parameter…》) body
+```
+
+In the grammar rules, anonymous functions sit above inline `if`/`else`
+expressions, but below inline `for` and `where` expressions. Therefore, the
+following creates a vector of 10 anonymous functions, each of which multiplies
+by a different value, not a single function that does 10 multiplications in
+a loop:
+
+```flitter
+let f = func(x) x*y for y in ..10
+```
+
+Note that as function calls are allowed to accept a vector of functions, it is
+valid to call `f`. So `f(3)` will still evaluate to `0;3;6;9;12;15;18;21;24;27`.
+
+It is wise to use parentheses to make the precedence explicit.
+
+As with regular functions, any captured names are bound at the point of
+definition. An anonymous function cannot be recursive as it has no function
+name to use within the body.
+
 ## Template function calls
 
 The special `@` operator provides syntactic sugar for calling a function using
@@ -813,8 +841,11 @@ between these different uses.
 
 ## Pragmas
 
-There are three supported pragmas that may be placed at the top-level of a
-source file:
+One or more pragmas may be placed at the top of a source file before any other
+expressions. Pragmas take a name and a value, which must be a literal number
+or string.
+
+There are three currently supported pragmas:
 
 ```flitter
 %pragma tempo 110
@@ -830,8 +861,8 @@ option).
 ## Imports
 
 Top-level definitions (`let`s and `func`s) may be imported from one **Flitter**
-program file into the top level of another. This allows common definitions to
-be collected into *modules* that can be used elsewhere. For example:
+program file into another. This allows common definitions to be collected into
+*modules* that can be used elsewhere. For example:
 
 ```flitter
 import SIZE;thing from 'common.fl'

src/flitter/engine/control.py

@@ -186,6 +186,7 @@ async def run(self):
                     level = 'SUCCESS' if current_program is None else 'INFO'
                     logger.log(level, "Loaded page {}: {}", self.current_page, self.current_path)
                     run_program = current_program = program
+                    self.handle_pragmas(program.pragmas, frame_time)
                     errors = set()
                     logs = set()
                     self.state_generation0 ^= self.state_generation1
@@ -243,7 +244,6 @@ async def run(self):
                     run_program.run(context, record_stats=self.vm_stats)
                 else:
                     context = Context()
-                self.handle_pragmas(context.pragmas, frame_time)
 
                 new_errors = context.errors.difference(errors) if errors is not None else context.errors
                 errors = context.errors

src/flitter/language/grammar.lark

@@ -2,7 +2,7 @@
 %import common.WS_INLINE
 %ignore WS_INLINE
 %ignore /\\$\n/m
-%declare _INDENT _DEDENT
+%declare _INDENT _DEDENT _EOF
 
 _NL: /((--[^\r\n]*)?\r?\n[\t ]*)+/
 
@@ -16,35 +16,36 @@ _HASHBANG : /#!.*\r?\n/
 _LPAREN : "("
 _RPAREN : ")"
 
-top : _HASHBANG? _NL? top_expressions
+top : _HASHBANG? _NL? pragmas _NL? sequence
 
-top_expressions : top_expression* -> tuple
+pragmas : pragma* -> tuple
 
-?top_expression : expression
-                | "let" multiline_bindings -> let
-                | "%pragma" NAME node _NL -> pragma
-                | "import" name_list "from" composition _NL ->  file_import
-                | "func" NAME _LPAREN parameters _RPAREN _NL _INDENT sequence _DEDENT -> function
+pragma : "%pragma" NAME literal _NL -> binding
 
-sequence : expressions [let_sequence]
+sequence : expressions
 
-let_sequence : "let" multiline_bindings sequence
+expressions : expression* let_expression? -> tuple
 
-expressions : expression* -> tuple
+let_expression : "let" multiline_bindings sequence -> let
+               | function sequence -> let_function
+               | "import" name_list "from" composition _NL sequence ->  let_import
 
 ?expression : node _NL
             | node _NL _INDENT sequence _DEDENT -> append
             | "@" atom [attribute_bindings] _NL [_INDENT sequence _DEDENT] -> template_call
             | "for" name_list "in" range _NL _INDENT sequence _DEDENT -> loop
-            | "if" tests ["else" _NL _INDENT sequence _DEDENT] -> if_else
+            | "if" conditions ["else" _NL _INDENT sequence _DEDENT] -> if_else
+            | _EOF -> export
+
+function : "func" NAME _LPAREN parameters _RPAREN _NL _INDENT sequence _DEDENT
 
 parameters : (parameter ("," parameter)*)? -> tuple
 
 parameter : NAME ["=" node] -> binding
 
-tests : test ("elif" test)* -> tuple
+conditions : condition ("elif" condition)* -> tuple
 
-test : composition _NL _INDENT sequence _DEDENT
+condition : composition _NL _INDENT sequence _DEDENT
 
 ?node : composition
       | node TAG -> tag
@@ -68,6 +69,7 @@ compositions : comprehension (";" comprehension)+ -> tuple
 ?comprehension : conditional
                | comprehension "for" name_list "in" conditional -> inline_loop
                | comprehension "where" inline_bindings -> inline_let
+               | "func" _LPAREN parameters _RPAREN conditional -> anonymous_function
 
 inline_bindings: binding+ -> tuple
 

src/flitter/language/parser.py

@@ -9,6 +9,7 @@
 from lark import Lark, Transformer
 from lark.exceptions import UnexpectedInput
 from lark.indenter import Indenter
+from lark.lexer import Token
 from lark.visitors import v_args
 
 from .. import model
@@ -26,6 +27,10 @@ class FlitterIndenter(Indenter):
     DEDENT_type = '_DEDENT'
     tab_len = 8
 
+    def _process(self, stream):
+        yield from super()._process(stream)
+        yield Token('_EOF', '')
+
 
 class ParseError(Exception):
     def __init__(self, msg, line, column, context):
@@ -68,14 +73,6 @@ def inline_if_else(self, then, condition, else_):
     def inline_loop(self, body, names, source):
         return tree.For(names, source, body)
 
-    def sequence(self, expressions, let_sequence=None):
-        if let_sequence is not None:
-            expressions = expressions + (let_sequence,)
-        return tree.Sequence(expressions)
-
-    def let_sequence(self, bindings, expressions):
-        return tree.InlineLet(expressions, bindings)
-
     def call(self, function, args):
         args = list(args)
         bindings = []
@@ -91,24 +88,31 @@ def template_call(self, function, bindings, sequence):
             return tree.Call(function, (sequence,), bindings)
         return tree.Call(function, (tree.Literal(model.null),), bindings or None)
 
+    def let_function(self, function, sequence):
+        return tree.Let((tree.PolyBinding((function.name,), function),), sequence)
+
+    def anonymous_function(self, parameters, body):
+        return tree.Function('<anon>', parameters, body)
+
     tuple = v_args(inline=False)(tuple)
 
     add = tree.Add
     append = tree.Append
     attributes = tree.Attributes
     binding = tree.Binding
     bool = tree.Literal
+    condition = tree.IfCondition
     divide = tree.Divide
+    export = tree.Export
     eq = tree.EqualTo
-    file_import = tree.Import
-    floor_divide = tree.FloorDivide
     function = tree.Function
+    floor_divide = tree.FloorDivide
     ge = tree.GreaterThanOrEqualTo
     gt = tree.GreaterThan
     if_else = tree.IfElse
-    inline_let = tree.InlineLet
     le = tree.LessThanOrEqualTo
     let = tree.Let
+    let_import = tree.Import
     literal = tree.Literal
     logical_and = tree.And
     logical_not = tree.Not
@@ -125,11 +129,10 @@ def template_call(self, function, bindings, sequence):
     poly_binding = tree.PolyBinding
     pos = tree.Positive
     power = tree.Power
-    pragma = tree.Pragma
+    sequence = tree.Sequence
     slice = tree.Slice
     subtract = tree.Subtract
     tag = tree.Tag
-    test = tree.IfCondition
     top = tree.Top