diff --git a/generator/src/generator.rs b/generator/src/generator.rs index ccbaa34f..7a693477 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -240,6 +240,16 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { generate_expr(rule.expr) }; + let expr = if rule.rec { + quote! { + state.recursive(Rule::#name, |state| { + #expr + }) + } + } else { + expr + }; + match rule.ty { RuleType::Normal => quote! { #[inline] @@ -636,6 +646,7 @@ mod tests { let rules = vec![OptimizedRule { name: "f".to_owned(), ty: RuleType::Normal, + rec: false, expr: OptimizedExpr::Ident("g".to_owned()), }]; @@ -926,6 +937,37 @@ mod tests { ); } + #[test] + fn generate_from_recursive_rule() { + let rule = OptimizedRule { + name: "f".to_owned(), + ty: RuleType::Normal, + rec: true, + expr: OptimizedExpr::Choice( + Box::new(OptimizedExpr::Ident("f".to_owned())), + Box::new(OptimizedExpr::Ident("g".to_owned())), + ), + }; + + assert_eq!( + generate_rule(rule).to_string(), + quote! { + #[inline] + #[allow (non_snake_case , unused_variables)] + pub fn f (state: Box<::pest::ParserState>) -> ::pest::ParseResult>> { + state.rule(Rule::f, |state| { + state.recursive(Rule::f, |state| { + self::f(state).or_else(|state| { + self::g(state) + }) + }) + }) + } + } + .to_string() + ); + } + #[test] fn generate_complete() { let name = Ident::new("MyParser", Span::call_site()); @@ -933,6 +975,7 @@ mod tests { let rules = vec![OptimizedRule { name: "a".to_owned(), ty: RuleType::Silent, + rec: false, expr: OptimizedExpr::Str("b".to_owned()), }]; let defaults = vec!["ANY"]; diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 27b4d816..f7a19185 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -64,6 +64,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { Rule::grammar_rule => "rule".to_owned(), Rule::_push => "PUSH".to_owned(), Rule::assignment_operator => "`=`".to_owned(), + Rule::recursive_modifier => "`*`".to_owned(), Rule::silent_modifier => "`_`".to_owned(), Rule::atomic_modifier => "`@`".to_owned(), Rule::compound_atomic_modifier => "`$`".to_owned(), diff --git a/meta/src/ast.rs b/meta/src/ast.rs index b80c5963..aded324c 100644 --- a/meta/src/ast.rs +++ b/meta/src/ast.rs @@ -11,6 +11,7 @@ pub struct Rule { pub name: String, pub ty: RuleType, + pub rec: bool, pub expr: Expr, } diff --git a/meta/src/grammar.pest b/meta/src/grammar.pest index 0d03ba89..3bb952ec 100644 --- a/meta/src/grammar.pest +++ b/meta/src/grammar.pest @@ -10,7 +10,7 @@ grammar_rules = _{ SOI ~ grammar_rule+ ~ EOI } grammar_rule = { - identifier ~ assignment_operator ~ modifier? ~ + identifier ~ assignment_operator ~ recursive_modifier? ~ modifier? ~ opening_brace ~ expression ~ closing_brace } @@ -29,6 +29,7 @@ modifier = _{ non_atomic_modifier } +recursive_modifier = { "*" } silent_modifier = { "_" } atomic_modifier = { "@" } compound_atomic_modifier = { "$" } diff --git a/meta/src/optimizer/concatenator.rs b/meta/src/optimizer/concatenator.rs index 8ff7590c..ac0c16a7 100644 --- a/meta/src/optimizer/concatenator.rs +++ b/meta/src/optimizer/concatenator.rs @@ -10,10 +10,11 @@ use ast::*; pub fn concatenate(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: expr.map_bottom_up(|expr| { if ty == RuleType::Atomic { // TODO: Use box syntax when it gets stabilized. diff --git a/meta/src/optimizer/factorizer.rs b/meta/src/optimizer/factorizer.rs index 573f566f..3a40b73f 100644 --- a/meta/src/optimizer/factorizer.rs +++ b/meta/src/optimizer/factorizer.rs @@ -10,10 +10,11 @@ use ast::*; pub fn factor(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: expr.map_top_down(|expr| { // TODO: Use box syntax when it gets stabilized. match expr { diff --git a/meta/src/optimizer/lister.rs b/meta/src/optimizer/lister.rs index 4ca2830a..49cae564 100644 --- a/meta/src/optimizer/lister.rs +++ b/meta/src/optimizer/lister.rs @@ -10,10 +10,11 @@ use ast::*; pub fn list(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: expr.map_bottom_up(|expr| { // TODO: Use box syntax when it gets stabilized. match expr { diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs index 0712fbec..eab8cdd0 100644 --- a/meta/src/optimizer/mod.rs +++ b/meta/src/optimizer/mod.rs @@ -76,6 +76,7 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule { OptimizedRule { name: rule.name, ty: rule.ty, + rec: rule.rec, expr: to_optimized(rule.expr), } } @@ -91,6 +92,7 @@ fn to_hash_map(rules: &[OptimizedRule]) -> HashMap { pub struct OptimizedRule { pub name: String, pub ty: RuleType, + pub rec: bool, pub expr: OptimizedExpr, } @@ -285,6 +287,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Choice( Choice( Choice(Str(String::from("a")), Str(String::from("b"))), @@ -299,6 +302,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Choice( Str(String::from("a")), Choice( @@ -319,6 +323,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Rep(Seq( NegPred(Choice(Str(String::from("a")), Str(String::from("b")))), Ident(String::from("ANY")) @@ -328,6 +333,7 @@ mod tests { let skipped = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: OptimizedExpr::Skip(vec![String::from("a"), String::from("b")]), }]; @@ -341,6 +347,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Seq( Seq(Str(String::from("a")), Str(String::from("b"))), Seq(Str(String::from("c")), Str(String::from("d"))) @@ -350,6 +357,7 @@ mod tests { let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: OptimizedExpr::Str(String::from("abcd")), }]; @@ -361,6 +369,7 @@ mod tests { let rules = vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: Expr::RepExact(Box::new(Expr::Ident(String::from("a"))), 3), }]; let unrolled = { @@ -368,6 +377,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Seq( Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("a"))) @@ -383,6 +393,7 @@ mod tests { let rules = vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: Expr::RepMax(Box::new(Expr::Str("a".to_owned())), 3), }]; let unrolled = { @@ -390,6 +401,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Seq( Opt(Str(String::from("a"))), Seq(Opt(Str(String::from("a"))), Opt(Str(String::from("a")))) @@ -405,6 +417,7 @@ mod tests { let rules = vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: Expr::RepMin(Box::new(Expr::Str("a".to_owned())), 2), }]; let unrolled = { @@ -412,6 +425,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Seq( Str(String::from("a")), Seq(Str(String::from("a")), Rep(Str(String::from("a")))) @@ -427,6 +441,7 @@ mod tests { let rules = vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: Expr::RepMinMax(Box::new(Expr::Str("a".to_owned())), 2, 3), }]; let unrolled = { @@ -434,6 +449,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, /* TODO possible room for improvement here: * if the sequences were rolled out in the opposite * order, we could further optimize the strings @@ -458,6 +474,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: box_tree!(Seq( Seq(Insens(String::from("a")), Insens(String::from("b"))), Seq(Insens(String::from("c")), Insens(String::from("d"))) @@ -467,6 +484,7 @@ mod tests { let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, + rec: false, expr: OptimizedExpr::Insens(String::from("abcd")), }]; @@ -480,6 +498,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Choice( Seq( Ident(String::from("a")), @@ -497,6 +516,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Seq( Ident(String::from("a")), Seq( @@ -517,6 +537,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Choice( Seq(Ident(String::from("a")), Ident(String::from("b"))), Ident(String::from("a")) @@ -528,6 +549,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))), }] }; @@ -542,6 +564,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Choice( Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("b"))) @@ -553,6 +576,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Ident(String::from("a"))), }] }; @@ -567,6 +591,7 @@ mod tests { vec![Rule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Seq( Rep(Seq(Ident(String::from("a")), Ident(String::from("b")))), Ident(String::from("a")) @@ -578,6 +603,7 @@ mod tests { vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Silent, + rec: false, expr: box_tree!(Seq( Ident(String::from("a")), Rep(Seq(Ident(String::from("b")), Ident(String::from("a")))) diff --git a/meta/src/optimizer/restorer.rs b/meta/src/optimizer/restorer.rs index fd67790e..3b5434f4 100644 --- a/meta/src/optimizer/restorer.rs +++ b/meta/src/optimizer/restorer.rs @@ -14,9 +14,9 @@ pub fn restore_on_err( rule: OptimizedRule, rules: &HashMap, ) -> OptimizedRule { - let OptimizedRule { name, ty, expr } = rule; + let OptimizedRule { name, ty, rec, expr } = rule; let expr = expr.map_bottom_up(|expr| wrap_branching_exprs(expr, rules)); - OptimizedRule { name, ty, expr } + OptimizedRule { name, ty, rec, expr } } fn wrap_branching_exprs( @@ -99,6 +99,7 @@ mod tests { let rules = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Opt(Str("a".to_string()))), }]; @@ -113,12 +114,14 @@ mod tests { let rules = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Rep(Push(Str("a".to_string())))), }]; let restored = OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Rep(RestoreOnErr(Push(Str("a".to_string()))))), }; @@ -133,12 +136,14 @@ mod tests { let rules = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Choice(Push(Str("a".to_string())), Str("a".to_string()))), }]; let restored = OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, + rec: false, expr: box_tree!(Choice( RestoreOnErr(Push(Str("a".to_string()))), Str("a".to_string()) diff --git a/meta/src/optimizer/rotater.rs b/meta/src/optimizer/rotater.rs index f313ae06..5ab6206a 100644 --- a/meta/src/optimizer/rotater.rs +++ b/meta/src/optimizer/rotater.rs @@ -35,10 +35,11 @@ pub fn rotate(rule: Rule) -> Rule { } } - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: expr.map_top_down(rotate_internal), } } diff --git a/meta/src/optimizer/skipper.rs b/meta/src/optimizer/skipper.rs index cad31246..34ecd991 100644 --- a/meta/src/optimizer/skipper.rs +++ b/meta/src/optimizer/skipper.rs @@ -28,10 +28,11 @@ pub fn skip(rule: Rule) -> Rule { } } - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: if ty == RuleType::Atomic { expr.map_top_down(|expr| { // TODO: Use box syntax when it gets stabilized. diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs index b1e84e40..17a80b6c 100644 --- a/meta/src/optimizer/unroller.rs +++ b/meta/src/optimizer/unroller.rs @@ -10,10 +10,11 @@ use ast::*; pub fn unroll(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { name, ty, rec, expr } = rule; Rule { name, ty, + rec, expr: expr.map_bottom_up(|expr| match expr { Expr::RepOnce(expr) => Expr::Seq(expr.clone(), Box::new(Expr::Rep(expr))), Expr::RepExact(expr, num) => (1..num + 1) diff --git a/meta/src/parser.rs b/meta/src/parser.rs index 8d8abbf0..c58a23f1 100644 --- a/meta/src/parser.rs +++ b/meta/src/parser.rs @@ -34,6 +34,7 @@ pub fn parse(rule: Rule, data: &str) -> Result, Error> { pub struct ParserRule<'i> { pub name: String, pub span: Span<'i>, + pub rec: bool, pub ty: RuleType, pub node: ParserNode<'i>, } @@ -131,9 +132,9 @@ pub enum ParserExpr<'i> { } fn convert_rule(rule: ParserRule) -> AstRule { - let ParserRule { name, ty, node, .. } = rule; + let ParserRule { name, ty, rec, node, .. } = rule; let expr = convert_node(node); - AstRule { name, ty, expr } + AstRule { name, ty, rec, expr } } fn convert_node(node: ParserNode) -> Expr { @@ -192,6 +193,11 @@ fn consume_rules_with_spans(pairs: Pairs) -> Result, Vec RuleType::Silent, @@ -211,6 +217,7 @@ fn consume_rules_with_spans(pairs: Pairs) -> Result, Vec(rules: &'a Vec>) -> Vec( expr: &ParserExpr<'i>, - rules: &HashMap>, + rules: &HashMap>, trace: &mut Vec, ) -> bool { match *expr { @@ -266,7 +266,7 @@ fn is_non_progressing<'i>( } if !trace.contains(ident) { - if let Some(node) = rules.get(ident) { + if let Some(ParserRule { node, .. }) = rules.get(ident) { trace.push(ident.clone()); let result = is_non_progressing(&node.expr, rules, trace); trace.pop().unwrap(); @@ -293,14 +293,14 @@ fn is_non_progressing<'i>( fn is_non_failing<'i>( expr: &ParserExpr<'i>, - rules: &HashMap>, + rules: &HashMap>, trace: &mut Vec, ) -> bool { match *expr { ParserExpr::Str(ref string) => string.is_empty(), ParserExpr::Ident(ref ident) => { if !trace.contains(ident) { - if let Some(node) = rules.get(ident) { + if let Some(ParserRule { node, .. }) = rules.get(ident) { trace.push(ident.clone()); let result = is_non_failing(&node.expr, rules, trace); trace.pop().unwrap(); @@ -445,20 +445,21 @@ fn validate_left_recursion<'a, 'i: 'a>(rules: &'a [ParserRule<'i>]) -> Vec(rules: &'a [ParserRule<'i>]) -> HashMap> { - rules.iter().map(|r| (r.name.clone(), &r.node)).collect() +fn to_hash_map<'a, 'i: 'a>(rules: &'a [ParserRule<'i>]) -> HashMap> { + rules.iter().map(|r| (r.name.clone(), r)).collect() } #[allow(clippy::needless_pass_by_value)] -fn left_recursion<'a, 'i: 'a>(rules: HashMap>) -> Vec> { +fn left_recursion<'a, 'i: 'a>(rules: HashMap>) -> Vec> { fn check_expr<'a, 'i: 'a>( node: &'a ParserNode<'i>, - rules: &'a HashMap>, + rules: &'a HashMap>, trace: &mut Vec, ) -> Option> { match node.expr.clone() { ParserExpr::Ident(other) => { - if trace[0] == other { + let is_marked_recursive = rules.get(&other).as_ref().map_or(false, |r| r.rec); + if trace[0] == other && !is_marked_recursive { trace.push(other); let chain = trace .iter() @@ -469,18 +470,21 @@ fn left_recursion<'a, 'i: 'a>(rules: HashMap>) -> Vec return Some(Error::new_from_span( ErrorVariant::CustomError { message: format!( - "rule {} is left-recursive ({}); pest::prec_climber might be useful \ - in this case", + "rule {} is left-recursive ({}). Consider using pest::prec_climber for operators \ + or recursive marker `*`, i.e. `{} = *{{ ... }}`.", + node.span.as_str(), + chain, node.span.as_str(), - chain ) }, node.span.clone() )); + } else if trace[0] == other { + trace.push(other.clone()); } if !trace.contains(&other) { - if let Some(node) = rules.get(&other) { + if let Some(ParserRule { node, .. } ) = rules.get(&other) { trace.push(other); let result = check_expr(node, rules, trace); trace.pop().unwrap(); @@ -513,7 +517,7 @@ fn left_recursion<'a, 'i: 'a>(rules: HashMap>) -> Vec let mut errors = vec![]; - for (ref name, ref node) in &rules { + for (ref name, ParserRule { ref node, .. }) in &rules { let name = (*name).clone(); if let Some(error) = check_expr(node, &rules, &mut vec![name]) { @@ -651,6 +655,23 @@ mod tests { )); } + #[test] + #[should_panic(expected = "grammar error + + --> 1:7 + | +1 | a = { (\"hello\"?)* } + | ^---------^ + | + = expression inside repetition cannot fail and will repeat infinitely")] + fn non_failing_optional_repetition() { + let input = "a = { (\"hello\"?)* }"; + unwrap_or_report(consume_rules( + PestParser::parse(Rule::grammar_rules, input).unwrap(), + )); + } + + #[test] #[should_panic(expected = "grammar error @@ -723,7 +744,7 @@ mod tests { 1 | a = { a } | ^ | - = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")] + = rule a is left-recursive (a -> a). Consider using pest::prec_climber for operators or recursive marker `*`, i.e. `a = *{ ... }`.")] fn simple_left_recursion() { let input = "a = { a }"; unwrap_or_report(consume_rules( @@ -739,14 +760,14 @@ mod tests { 1 | a = { b } b = { a } | ^ | - = rule b is left-recursive (b -> a -> b); pest::prec_climber might be useful in this case + = rule b is left-recursive (b -> a -> b). Consider using pest::prec_climber for operators or recursive marker `*`, i.e. `b = *{ ... }`. --> 1:17 | 1 | a = { b } b = { a } | ^ | - = rule a is left-recursive (a -> b -> a); pest::prec_climber might be useful in this case")] + = rule a is left-recursive (a -> b -> a). Consider using pest::prec_climber for operators or recursive marker `*`, i.e. `a = *{ ... }`.")] fn indirect_left_recursion() { let input = "a = { b } b = { a }"; unwrap_or_report(consume_rules( @@ -762,7 +783,7 @@ mod tests { 1 | a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a } | ^ | - = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")] + = rule a is left-recursive (a -> a). Consider using pest::prec_climber for operators or recursive marker `*`, i.e. `a = *{ ... }`.")] fn non_failing_left_recursion() { let input = "a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a }"; unwrap_or_report(consume_rules( @@ -778,7 +799,7 @@ mod tests { 1 | a = { \"a\" | a } | ^ | - = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")] + = rule a is left-recursive (a -> a). Consider using pest::prec_climber for operators or recursive marker `*`, i.e. `a = *{ ... }`.")] fn non_primary_choice_left_recursion() { let input = "a = { \"a\" | a }"; unwrap_or_report(consume_rules( diff --git a/pest/src/iterators/queueable_token.rs b/pest/src/iterators/queueable_token.rs index 7d56749b..3fbaa41c 100644 --- a/pest/src/iterators/queueable_token.rs +++ b/pest/src/iterators/queueable_token.rs @@ -13,7 +13,7 @@ // increased speed when pushing to the queue // * it finds its pair in O(1) time instead of O(N), since pair positions are known at parse time // and can easily be stored instead of recomputed -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum QueueableToken { Start { end_token_index: usize, diff --git a/pest/src/parser_state.rs b/pest/src/parser_state.rs index ead92cac..b0fe875c 100644 --- a/pest/src/parser_state.rs +++ b/pest/src/parser_state.rs @@ -12,6 +12,7 @@ use alloc::rc::Rc; use alloc::vec; use alloc::vec::Vec; use core::ops::Range; +use std::collections::HashMap; use error::{Error, ErrorVariant}; use iterators::{pairs, QueueableToken}; @@ -53,7 +54,7 @@ pub enum MatchDir { /// The complete state of a [`Parser`]. /// /// [`Parser`]: trait.Parser.html -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ParserState<'i, R: RuleType> { position: Position<'i>, queue: Vec>, @@ -63,6 +64,9 @@ pub struct ParserState<'i, R: RuleType> { attempt_pos: usize, atomicity: Atomicity, stack: Stack>, + + // (count, max) + recursive_bounds: HashMap, } /// Creates a `ParserState` from a `&str`, supplying it to a closure `f`. @@ -125,6 +129,8 @@ impl<'i, R: RuleType> ParserState<'i, R> { attempt_pos: 0, atomicity: Atomicity::NonAtomic, stack: Stack::new(), + + recursive_bounds: HashMap::new(), }) } @@ -459,6 +465,58 @@ impl<'i, R: RuleType> ParserState<'i, R> { } } + #[inline] + /// Process a left recursive rule `f` by simulating the input against a finite bound recursion. + /// Then increase the bound on successful trial (and continue the match). + /// Returns state when error occurred after recursing on deepest depth possible. + /// + /// It's called simulation because this function run the parent rule where it is contained. + /// + pub fn recursive(mut self: Box, rule: R, mut f: F) -> ParseResult> + where + F: FnMut(Box) -> ParseResult>, + { + let bound = self.recursive_bounds.get(&rule); + if let Some((count, max)) = bound { + if count >= max { + return Err(self); + } + } + + if bound.is_none() { + self.recursive_bounds.insert(rule, (0, 0)); + let init_state = self.clone(); + let mut lastiter_state = self.clone(); + loop { + let r = f(self); + if let Err(_) = r { + return Ok(lastiter_state); + } + + lastiter_state = r.as_ref().unwrap().clone(); + let b = lastiter_state.recursive_bounds.get(&rule).unwrap(); + if b.0 != b.1 { + // It doesn't hit any deeper + break Ok(lastiter_state); + } + + let rb = lastiter_state.recursive_bounds.clone(); + self = init_state.clone(); + self.recursive_bounds = rb; + + let b = self.recursive_bounds.get_mut(&rule).unwrap(); + b.0 = 0; + b.1 += 1; + } + } else { + let b = self.recursive_bounds.get_mut(&rule).unwrap(); + b.0 += 1; + + return f(self); + } + } + + /// Attempts to match a single character based on a filter function. Returns `Ok` with the /// updated `Box` if successful, or `Err` with the updated `Box` /// otherwise. diff --git a/pest/src/stack.rs b/pest/src/stack.rs index 05ac11f6..5f337365 100644 --- a/pest/src/stack.rs +++ b/pest/src/stack.rs @@ -13,7 +13,7 @@ use core::ops::{Index, Range}; /// Implementation of a `Stack` which maintains an log of `StackOp`s in order to rewind the stack /// to a previous state. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Stack { ops: Vec>, cache: Vec, @@ -110,7 +110,7 @@ impl Index> for Stack { } } -#[derive(Debug)] +#[derive(Debug, Clone)] enum StackOp { Push(T), Pop(T), diff --git a/pest/tests/leftrecursive.rs b/pest/tests/leftrecursive.rs new file mode 100644 index 00000000..908408a3 --- /dev/null +++ b/pest/tests/leftrecursive.rs @@ -0,0 +1,220 @@ +// pest. The Elegant Parser +// Copyright (c) 2018 DragoČ™ Tiselice +// +// Licensed under the Apache License, Version 2.0 +// or the MIT +// license , at your +// option. All files in the project carrying such notice may not be copied, +// modified, or distributed except according to those terms. + +#[macro_use] +extern crate pest; + +use pest::error::Error; +use pest::iterators::Pairs; +use pest::{state, ParseResult, Parser, ParserState}; + +#[allow(dead_code, non_camel_case_types)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +enum Rule { + expression, + argument_list, + function_call, + callable_var, + variable, + symbol, +} + +struct LeftRecursiveParser; + +/// Represent these indirect recursive syntax. +/// +/// function_call = *{ +/// callable_variable ~ argument_list +/// } +/// +/// callable_variable = { +/// variable +/// | symbol+ +/// | function_call +/// } +/// +impl Parser for LeftRecursiveParser { + fn parse(rule: Rule, input: &str) -> Result, Error> { + fn expression(state: Box>) -> ParseResult>> { + state.rule(Rule::expression, |s| { + s.sequence(|s| { + variable(s) + .or_else(|s| function_call(s)) + }) + }) + } + + fn argument_list(state: Box>) -> ParseResult>> { + state.rule(Rule::argument_list, |s| { + s.match_string("(") + .and_then(|s| { + s.optional(|s| { + expression(s) + .and_then(|s| { + s.optional(|s| s.repeat(|s| { + s.match_string(",") + .and_then(|s| expression(s)) + })) + }) + }) + }) + .and_then(|s| s.match_string(")")) + }) + } + + fn callable_var(state: Box>) -> ParseResult>> { + state.rule(Rule::callable_var, |s| { + function_call(s) + .or_else(|s| variable(s)) + .or_else(|s| symbol(s)) + }) + } + + fn function_call(state: Box>) -> ParseResult>> { + state.rule(Rule::function_call, |s| { + s.recursive(Rule::callable_var, |s| { + callable_var(s) + .and_then(|s| argument_list(s)) + }) + }) + } + + fn variable(state: Box>) -> ParseResult>> { + state.rule(Rule::variable, |s| { + s.sequence(|s| { + s.match_string("$") + .and_then(|s| symbol(s)) + }) + }) + } + + fn symbol(state: Box>) -> ParseResult>> { + state.rule(Rule::symbol, |s| { + s.match_range('A'..'z') + .and_then(|s| s.repeat(|s| s.match_range('A'..'z'))) + }) + } + + state(input, |state| match rule { + Rule::expression => expression(state), + // For troubleshoot at first, but then a test + Rule::argument_list => argument_list(state), + _ => unreachable!(), + }) + } +} + +#[test] +fn variable() { + parses_to! { + parser: LeftRecursiveParser, + input: "$variable", + rule: Rule::expression, + tokens: [ + expression(0, 9, [ + variable(0, 9, [ + symbol(1, 9) + ]) + ]) + ] + }; +} + +#[test] +fn argument_list() { + parses_to! { + parser: LeftRecursiveParser, + input: "($abc)", + rule: Rule::argument_list, + tokens: [ + argument_list(0, 6, [ + expression(1, 5, [ + variable(1, 5, [ + symbol(2, 5) + ]) + ]) + ]) + ] + }; + + parses_to! { + parser: LeftRecursiveParser, + input: "($abc,$cde)", + rule: Rule::argument_list, + tokens: [ + argument_list(0, 11, [ + expression(1, 5, [ + variable(1, 5, [ + symbol(2, 5) + ]), + ]), + expression(6, 10, [ + variable(6, 10, [ + symbol(7, 10) + ]) + ]) + ]) + ] + }; +} + +#[test] +fn function_call() { + parses_to! { + parser: LeftRecursiveParser, + input: "func($arga)", + rule: Rule::expression, + tokens: [ + expression(0, 11, [ + function_call(0, 11, [ + callable_var(0, 4, [ + symbol(0, 4), + ]), + argument_list(4, 11, [ + expression(5, 10, [ + variable(5, 10, [ + symbol(6, 10) + ]) + ]) + ]) + ]) + ]) + ] + }; +} + +#[test] +fn recursive_function_call() { + parses_to! { + parser: LeftRecursiveParser, + input: "func()($arga)", + rule: Rule::expression, + tokens: [ + expression(0, 13, [ + function_call(0, 13, [ + callable_var(0, 6, [ + function_call(0, 6, [ + callable_var(0, 4, [ + symbol(0, 4), + ]), + argument_list(4, 6) + ]), + ]), + argument_list(6, 13, [ + expression(7, 12, [ + variable(7, 12, [ + symbol(8, 12) + ]) + ]) + ]) + ]) + ]) + ] + }; +} diff --git a/pest/tests/leftrecursive_simple.rs b/pest/tests/leftrecursive_simple.rs new file mode 100644 index 00000000..f11514e6 --- /dev/null +++ b/pest/tests/leftrecursive_simple.rs @@ -0,0 +1,157 @@ +// pest. The Elegant Parser +// Copyright (c) 2018 DragoČ™ Tiselice +// +// Licensed under the Apache License, Version 2.0 +// or the MIT +// license , at your +// option. All files in the project carrying such notice may not be copied, +// modified, or distributed except according to those terms. + +#[macro_use] +extern crate pest; + +use pest::error::Error; +use pest::iterators::Pairs; +use pest::{state, ParseResult, Parser, ParserState}; + +#[allow(dead_code, non_camel_case_types)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +enum Rule { + expression, + argument_list, + function_call, + callable_var, + variable, + symbol, +} + +struct LeftRecursiveParser; + + +/// Represent this simple recursive syntax. +/// +/// function_call = *{ +/// ( function_call +/// | symbol+ +/// | variable +/// ) ~ argument_list +/// } +/// +impl Parser for LeftRecursiveParser { + fn parse(rule: Rule, input: &str) -> Result, Error> { + fn expression(state: Box>) -> ParseResult>> { + state.rule(Rule::expression, |s| { + s.sequence(|s| { + variable(s) + .or_else(|s| function_call(s)) + }) + }) + } + + fn argument_list(state: Box>) -> ParseResult>> { + state.rule(Rule::argument_list, |s| { + s.match_string("(") + .and_then(|s| s.match_string(")")) + }) + } + + fn function_call(state: Box>) -> ParseResult>> { + state.rule(Rule::function_call, |s| { + s.recursive(Rule::function_call, |s| { + function_call(s) + .or_else(|s| variable(s)) + .or_else(|s| symbol(s)) + .and_then(|s| argument_list(s)) + }) + }) + } + + fn variable(state: Box>) -> ParseResult>> { + state.rule(Rule::variable, |s| { + s.sequence(|s| { + s.match_string("$") + .and_then(|s| symbol(s)) + }) + }) + } + + fn symbol(state: Box>) -> ParseResult>> { + state.rule(Rule::symbol, |s| { + s.match_range('A'..'z') + .and_then(|s| s.repeat(|s| s.match_range('A'..'z'))) + }) + } + + state(input, |state| match rule { + Rule::expression => expression(state), + // For troubleshoot at first, but then a test + Rule::argument_list => argument_list(state), + _ => unreachable!(), + }) + } +} + +#[test] +fn variable() { + parses_to! { + parser: LeftRecursiveParser, + input: "$variable", + rule: Rule::expression, + tokens: [ + expression(0, 9, [ + variable(0, 9, [ + symbol(1, 9) + ]) + ]) + ] + }; +} + +#[test] +fn argument_list() { + parses_to! { + parser: LeftRecursiveParser, + input: "()", + rule: Rule::argument_list, + tokens: [ + argument_list(0, 2) + ] + }; +} + +#[test] +fn function_call() { + parses_to! { + parser: LeftRecursiveParser, + input: "func()", + rule: Rule::expression, + tokens: [ + expression(0, 6, [ + function_call(0, 6, [ + symbol(0, 4), + argument_list(4, 6) + ]) + ]) + ] + }; +} + +#[test] +fn recursive_function_call() { + parses_to! { + parser: LeftRecursiveParser, + input: "func()()", + rule: Rule::expression, + tokens: [ + expression(0, 8, [ + function_call(0, 8, [ + function_call(0, 6, [ + symbol(0, 4), + argument_list(4, 6) + ]), + argument_list(6, 8) + ]) + ]) + ] + }; +}