Support left recursion syntax

generator/src/generator.rs

@@ -240,6 +240,16 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream {
         generate_expr(rule.expr)
     };
 
+    let expr = if rule.rec {
+        quote! {
+            state.recursive(Rule::#name, |state| {
+                #expr
+            })
+        }
+    } else {
+        expr
+    };
+
     match rule.ty {
         RuleType::Normal => quote! {
             #[inline]
@@ -636,6 +646,7 @@ mod tests {
         let rules = vec![OptimizedRule {
             name: "f".to_owned(),
             ty: RuleType::Normal,
+            rec: false,
             expr: OptimizedExpr::Ident("g".to_owned()),
         }];
 
@@ -926,13 +937,45 @@ mod tests {
         );
     }
 
+    #[test]
+    fn generate_from_recursive_rule() {
+        let rule = OptimizedRule {
+            name: "f".to_owned(),
+            ty: RuleType::Normal,
+            rec: true,
+            expr: OptimizedExpr::Choice(
+                Box::new(OptimizedExpr::Ident("f".to_owned())),
+                Box::new(OptimizedExpr::Ident("g".to_owned())),
+            ),
+        };
+
+        assert_eq!(
+            generate_rule(rule).to_string(),
+            quote! {
+                #[inline]
+                #[allow (non_snake_case , unused_variables)]
+                pub fn f (state: Box<::pest::ParserState<Rule>>) -> ::pest::ParseResult<Box<::pest::ParserState<Rule>>> {
+                    state.rule(Rule::f, |state| {
+                        state.recursive(Rule::f, |state| {
+                            self::f(state).or_else(|state| {
+                                self::g(state)
+                            })
+                        })
+                    })
+                }
+            }
+            .to_string()
+        );
+    }
+
     #[test]
     fn generate_complete() {
         let name = Ident::new("MyParser", Span::call_site());
         let generics = Generics::default();
         let rules = vec![OptimizedRule {
             name: "a".to_owned(),
             ty: RuleType::Silent,
+            rec: false,
             expr: OptimizedExpr::Str("b".to_owned()),
         }];
         let defaults = vec!["ANY"];

generator/src/lib.rs

@@ -64,6 +64,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream {
                 Rule::grammar_rule => "rule".to_owned(),
                 Rule::_push => "PUSH".to_owned(),
                 Rule::assignment_operator => "`=`".to_owned(),
+                Rule::recursive_modifier => "`*`".to_owned(),
                 Rule::silent_modifier => "`_`".to_owned(),
                 Rule::atomic_modifier => "`@`".to_owned(),
                 Rule::compound_atomic_modifier => "`$`".to_owned(),

meta/src/ast.rs

@@ -11,6 +11,7 @@
 pub struct Rule {
     pub name: String,
     pub ty: RuleType,
+    pub rec: bool,
     pub expr: Expr,
 }
 

meta/src/grammar.pest

@@ -10,7 +10,7 @@
 grammar_rules = _{ SOI ~ grammar_rule+ ~ EOI }
 
 grammar_rule = {
-    identifier ~ assignment_operator ~ modifier? ~
+    identifier ~ assignment_operator ~ recursive_modifier? ~ modifier? ~
     opening_brace ~ expression ~ closing_brace
 }
 
@@ -29,6 +29,7 @@ modifier = _{
     non_atomic_modifier
 }
 
+recursive_modifier       = { "*" }
 silent_modifier          = { "_" }
 atomic_modifier          = { "@" }
 compound_atomic_modifier = { "$" }

meta/src/optimizer/concatenator.rs

@@ -10,10 +10,11 @@
 use ast::*;
 
 pub fn concatenate(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule { name, ty, rec, expr } = rule;
     Rule {
         name,
         ty,
+        rec,
         expr: expr.map_bottom_up(|expr| {
             if ty == RuleType::Atomic {
                 // TODO: Use box syntax when it gets stabilized.

meta/src/optimizer/factorizer.rs

@@ -10,10 +10,11 @@
 use ast::*;
 
 pub fn factor(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule { name, ty, rec, expr } = rule;
     Rule {
         name,
         ty,
+        rec,
         expr: expr.map_top_down(|expr| {
             // TODO: Use box syntax when it gets stabilized.
             match expr {

meta/src/optimizer/lister.rs

@@ -10,10 +10,11 @@
 use ast::*;
 
 pub fn list(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule { name, ty, rec, expr } = rule;
     Rule {
         name,
         ty,
+        rec,
         expr: expr.map_bottom_up(|expr| {
             // TODO: Use box syntax when it gets stabilized.
             match expr {

meta/src/optimizer/mod.rs

@@ -76,6 +76,7 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
     OptimizedRule {
         name: rule.name,
         ty: rule.ty,
+        rec: rule.rec,
         expr: to_optimized(rule.expr),
     }
 }
@@ -91,6 +92,7 @@ fn to_hash_map(rules: &[OptimizedRule]) -> HashMap<String, OptimizedExpr> {
 pub struct OptimizedRule {
     pub name: String,
     pub ty: RuleType,
+    pub rec: bool,
     pub expr: OptimizedExpr,
 }
 
@@ -285,6 +287,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Normal,
+                rec: false,
                 expr: box_tree!(Choice(
                     Choice(
                         Choice(Str(String::from("a")), Str(String::from("b"))),
@@ -299,6 +302,7 @@ mod tests {
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Normal,
+                rec: false,
                 expr: box_tree!(Choice(
                     Str(String::from("a")),
                     Choice(
@@ -319,6 +323,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Rep(Seq(
                     NegPred(Choice(Str(String::from("a")), Str(String::from("b")))),
                     Ident(String::from("ANY"))
@@ -328,6 +333,7 @@ mod tests {
         let skipped = vec![OptimizedRule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: OptimizedExpr::Skip(vec![String::from("a"), String::from("b")]),
         }];
 
@@ -341,6 +347,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Seq(
                     Seq(Str(String::from("a")), Str(String::from("b"))),
                     Seq(Str(String::from("c")), Str(String::from("d")))
@@ -350,6 +357,7 @@ mod tests {
         let concatenated = vec![OptimizedRule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: OptimizedExpr::Str(String::from("abcd")),
         }];
 
@@ -361,13 +369,15 @@ mod tests {
         let rules = vec![Rule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: Expr::RepExact(Box::new(Expr::Ident(String::from("a"))), 3),
         }];
         let unrolled = {
             use optimizer::OptimizedExpr::*;
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Seq(
                     Ident(String::from("a")),
                     Seq(Ident(String::from("a")), Ident(String::from("a")))
@@ -383,13 +393,15 @@ mod tests {
         let rules = vec![Rule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: Expr::RepMax(Box::new(Expr::Str("a".to_owned())), 3),
         }];
         let unrolled = {
             use optimizer::OptimizedExpr::*;
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Seq(
                     Opt(Str(String::from("a"))),
                     Seq(Opt(Str(String::from("a"))), Opt(Str(String::from("a"))))
@@ -405,13 +417,15 @@ mod tests {
         let rules = vec![Rule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: Expr::RepMin(Box::new(Expr::Str("a".to_owned())), 2),
         }];
         let unrolled = {
             use optimizer::OptimizedExpr::*;
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Seq(
                     Str(String::from("a")),
                     Seq(Str(String::from("a")), Rep(Str(String::from("a"))))
@@ -427,13 +441,15 @@ mod tests {
         let rules = vec![Rule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: Expr::RepMinMax(Box::new(Expr::Str("a".to_owned())), 2, 3),
         }];
         let unrolled = {
             use optimizer::OptimizedExpr::*;
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 /* TODO possible room for improvement here:
                  * if the sequences were rolled out in the opposite
                  * order, we could further optimize the strings
@@ -458,6 +474,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Atomic,
+                rec: false,
                 expr: box_tree!(Seq(
                     Seq(Insens(String::from("a")), Insens(String::from("b"))),
                     Seq(Insens(String::from("c")), Insens(String::from("d")))
@@ -467,6 +484,7 @@ mod tests {
         let concatenated = vec![OptimizedRule {
             name: "rule".to_owned(),
             ty: RuleType::Atomic,
+            rec: false,
             expr: OptimizedExpr::Insens(String::from("abcd")),
         }];
 
@@ -480,6 +498,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Choice(
                     Seq(
                         Ident(String::from("a")),
@@ -497,6 +516,7 @@ mod tests {
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Seq(
                     Ident(String::from("a")),
                     Seq(
@@ -517,6 +537,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Choice(
                     Seq(Ident(String::from("a")), Ident(String::from("b"))),
                     Ident(String::from("a"))
@@ -528,6 +549,7 @@ mod tests {
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))),
             }]
         };
@@ -542,6 +564,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Choice(
                     Ident(String::from("a")),
                     Seq(Ident(String::from("a")), Ident(String::from("b")))
@@ -553,6 +576,7 @@ mod tests {
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Ident(String::from("a"))),
             }]
         };
@@ -567,6 +591,7 @@ mod tests {
             vec![Rule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Seq(
                     Rep(Seq(Ident(String::from("a")), Ident(String::from("b")))),
                     Ident(String::from("a"))
@@ -578,6 +603,7 @@ mod tests {
             vec![OptimizedRule {
                 name: "rule".to_owned(),
                 ty: RuleType::Silent,
+                rec: false,
                 expr: box_tree!(Seq(
                     Ident(String::from("a")),
                     Rep(Seq(Ident(String::from("b")), Ident(String::from("a"))))