Disallow trailing junk in generators (#497)

Disallow generator syntax with trailing non-delimited expressions

    (y for x in xs a)
    (y for x in xs a, b)

Allow parameter syntax such as

    f(y for x in xs; a)

because the flisp parser allowed this it's used in Base in at least one
location.

Disallow the almost-equivalent block syntax for now:

    (y for x in xs; a)

because the flisp parser disallowed it and users can always get the same
thing with some extra parentheses.

src/parser.jl

@@ -2706,7 +2706,7 @@ end
 function parse_call_arglist(ps::ParseState, closer)
     ps = ParseState(ps, for_generator=true)
 
-    parse_brackets(ps, closer) do _, _, _, _
+    parse_brackets(ps, closer, false) do _, _, _, _
         return (needs_parameters=true,)
     end
 end
@@ -3123,7 +3123,7 @@ end
 #
 # flisp: parts of parse-paren- and parse-arglist
 function parse_brackets(after_parse::Function,
-                        ps::ParseState, closing_kind)
+                        ps::ParseState, closing_kind, generator_is_last=true)
     ps = ParseState(ps, range_colon_enabled=true,
                     space_sensitive=false,
                     where_enabled=true,
@@ -3159,18 +3159,22 @@ function parse_brackets(after_parse::Function,
             if num_subexprs == 1
                 had_splat = peek_behind(ps).kind == K"..."
             end
-            t = peek_token(ps, skip_newlines=true)
-            k = kind(t)
+            k = peek(ps, skip_newlines=true)
+            if k == K"for"
+                # Generator syntax
+                # (x for a in as)  ==>  (parens (generator x (iteration (in a as))))
+                parse_generator(ps, mark)
+                if generator_is_last
+                    break
+                end
+                k = peek(ps, skip_newlines=true)
+            end
             if k == K","
                 had_commas = true
                 bump(ps, TRIVIA_FLAG)
             elseif k == K";" || k == closing_kind
                 # Handled above
                 continue
-            elseif k == K"for"
-                # Generator syntax
-                # (x for a in as)  ==>  (parens (generator x (iteration (in a as))))
-                parse_generator(ps, mark)
             else
                 # Error - recovery done when consuming closing_kind
                 break

test/parser.jl

@@ -371,6 +371,7 @@ tests = [
         "@x(a, b)"   =>  "(macrocall-p @x a b)"
         "A.@x(y)"    =>  "(macrocall-p (. A @x) y)"
         "A.@x(y).z"  =>  "(. (macrocall-p (. A @x) y) z)"
+        "f(y for x = xs; a)" => "(call f (generator y (iteration (in x xs))) (parameters a))"
         # do
         "f() do\nend"         =>  "(call f (do (tuple) (block)))"
         "f() do ; body end"   =>  "(call f (do (tuple) (block body)))"
@@ -435,6 +436,7 @@ tests = [
         "A.@S{a}" => "(macrocall (. A @S) (braces a))"
         "@S{a}.b" => "(. (macrocall @S (braces a)) b)"
         "S{a,b}"  => "(curly S a b)"
+        "T{y for x = xs; a}" => "(curly T (generator y (iteration (in x xs))) (parameters a))"
         # String macros
         "x\"str\""   => """(macrocall @x_str (string-r "str"))"""
         "x`str`"     => """(macrocall @x_cmd (cmdstring-r "str"))"""
@@ -729,6 +731,9 @@ tests = [
         "(a=1;)"      =>  "(block-p (= a 1))"
         "(a;b;;c)"    =>  "(block-p a b c)"
         "(a=1; b=2)"  =>  "(block-p (= a 1) (= b 2))"
+        # Following is an error for flisp compatibility. But it could be
+        # allowed as valid block syntax in the future?
+        "(y for x = xs; a)" => "(parens (generator y (iteration (in x xs))) (error-t ✘ a))"
         # Parentheses used for grouping
         "(a * b)"     =>  "(parens (call-i a * b))"
         "(a=1)"       =>  "(parens (= a 1))"
@@ -1075,6 +1080,11 @@ parsestmt_test_specs = [
     "x in'``\$" => "(call-i x in (call-i (juxtapose (char '`' (error-t)) (cmdstring-r (error-t))) \$ (error)))"
     "var\"#\"`str`" => "(juxtapose (var # (error-t)) (cmdstring-r \"str\"))"
     "var\"#\"\"str\"" => "(juxtapose (var # (error-t)) (error-t) (string \"str\"))"
+
+    # trailing junk in generators (issue #407)
+    "(x for x = xs a)"      =>  "(parens (generator x (iteration (in x xs))) (error-t a))"
+    "(x for x = xs a, b)"   =>  "(parens (generator x (iteration (in x xs))) (error-t a ✘ b))"
+    "f(x for x = xs a)"     =>  "(call f (generator x (iteration (in x xs))) (error-t a))"
 ]
 
 @testset "Parser does not crash on broken code" begin