diff --git a/Cargo.lock b/Cargo.lock index 9e75da7c..d10c7e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,7 +178,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chumsky" -version = "0.11.2" +version = "0.12.0" dependencies = [ "ariadne", "bytes", diff --git a/examples/mini_ml.rs b/examples/mini_ml.rs index 3fa7280e..e318e46d 100644 --- a/examples/mini_ml.rs +++ b/examples/mini_ml.rs @@ -4,7 +4,11 @@ //! cargo run --features=pratt,label --example mini_ml -- examples/sample.mini_ml use ariadne::{sources, Color, Label, Report, ReportKind}; -use chumsky::{input::BorrowInput, pratt::*, prelude::*}; +use chumsky::{ + input::{Input as _, MappedInput}, + pratt::*, + prelude::*, +}; use std::{env, fmt, fs}; // Tokens and lexer @@ -109,15 +113,12 @@ pub enum Expr<'src> { }, } -fn parser<'tokens, 'src: 'tokens, I, M>( - make_input: M, -) -> impl Parser<'tokens, I, Spanned>, extra::Err>>> -where - I: BorrowInput<'tokens, Token = Token<'src>, Span = SimpleSpan>, - // Because this function is generic over the input type, we need the caller to tell us how to create a new input, - // `I`, from a nested token tree. This function serves that purpose. - M: Fn(SimpleSpan, &'tokens [Spanned>]) -> I + Clone + 'src, -{ +fn parser<'tokens, 'src: 'tokens>() -> impl Parser< + 'tokens, + MappedInput<'tokens, Token<'src>, SimpleSpan, &'tokens [Spanned>]>, + Spanned>, + extra::Err>>, +> { recursive(|expr| { let ident = select_ref! { Token::Ident(x) => *x }; let atom = choice(( @@ -157,7 +158,7 @@ where ), ), // ( x ) - expr.nested_in(select_ref! { Token::Parens(ts) = e => make_input(e.span(), ts) }), + expr.nested_in(select_ref! { Token::Parens(ts) = e => ts.split_token_span(e.span()) }), )) .pratt(vec![ // Multiply @@ -445,13 +446,6 @@ fn parse_failure(err: &Rich, src: &str) -> ! { ) } -fn make_input<'src>( - eoi: SimpleSpan, - toks: &'src [Spanned>], -) -> impl BorrowInput<'src, Token = Token<'src>, Span = SimpleSpan> { - toks.map(eoi, |(t, s)| (t, s)) -} - fn main() { let filename = env::args().nth(1).expect("Expected file argument"); let src = &fs::read_to_string(&filename).expect("Failed to read file"); @@ -461,8 +455,8 @@ fn main() { .into_result() .unwrap_or_else(|errs| parse_failure(&errs[0], src)); - let expr = parser(make_input) - .parse(make_input((0..src.len()).into(), &tokens)) + let expr = parser() + .parse(tokens[..].split_token_span((0..src.len()).into())) .into_result() .unwrap_or_else(|errs| parse_failure(&errs[0], src)); diff --git a/examples/nested_spans.rs b/examples/nested_spans.rs index 1cff24fc..6a487b59 100644 --- a/examples/nested_spans.rs +++ b/examples/nested_spans.rs @@ -1,4 +1,4 @@ -use chumsky::{input::BorrowInput, prelude::*}; +use chumsky::{input::MappedInput, prelude::*}; // This token is a tree: it contains within it a sub-tree of tokens #[derive(PartialEq, Debug)] @@ -10,16 +10,13 @@ enum Token { } #[allow(clippy::let_and_return)] -fn parser<'src, I, M>(make_input: M) -> impl Parser<'src, I, i64> -where - I: BorrowInput<'src, Token = Token, Span = SimpleSpan>, - M: Fn(SimpleSpan, &'src [(Token, SimpleSpan)]) -> I + Clone + 'src, -{ +fn parser<'src>( +) -> impl Parser<'src, MappedInput<'src, Token, SimpleSpan, &'src [(Token, SimpleSpan)]>, i64> { recursive(|expr| { let num = select_ref! { Token::Num(x) => *x }; let parens = expr // Here we specify that `expr` should appear *inside* the parenthesised token tree - .nested_in(select_ref! { Token::Parens(xs) = e => make_input(e.span(), xs) }); + .nested_in(select_ref! { Token::Parens(xs) = e => xs.split_token_span(e.span()) }); let atom = num.or(parens); @@ -37,13 +34,6 @@ where }) } -fn make_input( - eoi: SimpleSpan, - toks: &[(Token, SimpleSpan)], -) -> impl BorrowInput<'_, Token = Token, Span = SimpleSpan> { - toks.map(eoi, |(t, s)| (t, s)) -} - fn main() { // This token tree represents the expression `(2 + 3) * 4` let tokens = [ @@ -62,8 +52,8 @@ fn main() { let eoi = SimpleSpan::new((), 11..11); // Example EoI assert_eq!( - parser(make_input) - .parse(make_input(eoi, &tokens)) + parser() + .parse(tokens[..].split_token_span(eoi)) .into_result(), Ok(20) ); diff --git a/src/combinator.rs b/src/combinator.rs index 7018d205..104342ea 100644 --- a/src/combinator.rs +++ b/src/combinator.rs @@ -2513,6 +2513,61 @@ where go_extra!(OA); } +/// See [`IterParser::fold`]. +pub struct Fold { + pub(crate) parser: A, + pub(crate) init: B, + pub(crate) folder: F, + #[allow(dead_code)] + pub(crate) phantom: EmptyPhantom<(O, E)>, +} + +impl Copy for Fold {} +impl Clone for Fold { + fn clone(&self) -> Self { + Self { + parser: self.parser.clone(), + init: self.init.clone(), + folder: self.folder.clone(), + phantom: EmptyPhantom::new(), + } + } +} + +impl<'src, I, F, A, B, O, E> Parser<'src, I, B, E> for Fold +where + I: Input<'src>, + A: IterParser<'src, I, O, E>, + E: ParserExtra<'src, I>, + B: Clone, + F: Fn(B, O) -> B, +{ + #[inline(always)] + fn go(&self, inp: &mut InputRef<'src, '_, I, E>) -> PResult + where + Self: Sized, + { + let mut acc = M::bind(|| self.init.clone()); + let mut iter_state = self.parser.make_iter::(inp)?; + loop { + match self + .parser + .next::(inp, &mut iter_state, IterParserDebug::new(false)) + { + Ok(Some(out)) => { + acc = M::combine(acc, out, |acc, item| (self.folder)(acc, item)); + } + Ok(None) => break, + Err(()) => return Err(()), + } + } + + Ok(acc) + } + + go_extra!(B); +} + /// See [`IterParser::foldr`]. pub struct Foldr { pub(crate) parser_a: A, diff --git a/src/input.rs b/src/input.rs index 0449098f..80a69636 100644 --- a/src/input.rs +++ b/src/input.rs @@ -192,7 +192,7 @@ pub trait Input<'src>: 'src { /// /// Although `MappedInput` does implement [`SliceInput`], please be aware that, as you might anticipate, the slices /// will be those of the original input and not `&[T]`, to avoid the need to copy around sections of the input. - fn map(self, eoi: S, f: F) -> MappedInput + fn map(self, eoi: S, f: F) -> MappedInput<'src, T, S, Self, F> where Self: Sized, F: Fn( @@ -212,6 +212,51 @@ pub trait Input<'src>: 'src { } } + /// Take an input (such as a slice) with token type `(T, S)` and turns it into one that produces tokens of type `T` and spans of type `S`. + /// + /// This is useful when you can't make your parser generic over an input type and need to name the input type, such as with [`Parser::nested_in`]. + /// + /// This is largely an ergonomic convenience over calling [`input.map(eoi, |(t, s)| (t, s))`](`Input::map`). + /// + /// # Examples + /// ``` + /// use chumsky::{input::{Input as _, MappedInput}, prelude::*}; + /// + /// enum TokenTree<'src> { Num(i64), Sum(&'src [(Self, SimpleSpan)]) } + /// + /// type Input<'src> = MappedInput<'src, TokenTree<'src>, SimpleSpan, &'src [(TokenTree<'src>, SimpleSpan)]>; + /// + /// // This parser will parse a recursive input composed of `TokenTree`s. + /// fn eval<'src>() -> impl Parser<'src, Input<'src>, i64> { + /// recursive(|tree| { + /// let sum = tree.repeated().fold(0, |sum, x| sum + x) + /// .nested_in(select_ref! { TokenTree::Sum(xs) = e => xs.split_token_span(e.span()) }); + /// + /// select_ref! { TokenTree::Num(x) => *x }.or(sum) + /// }) + /// } + /// + /// let example = [ + /// (TokenTree::Sum(&[ + /// (TokenTree::Num(1), SimpleSpan::from(0..1)), // 1 + /// (TokenTree::Sum(&[ + /// (TokenTree::Num(2), SimpleSpan::from(2..3)), // 2 + /// (TokenTree::Num(3), SimpleSpan::from(4..5)), // 3 + /// ]), SimpleSpan::from(2..5)) + /// ]), SimpleSpan::from(0..5)), + /// ]; + /// + /// assert_eq!(eval().parse(example[..].split_token_span((0..4).into())).into_result(), Ok(6)); + /// ``` + fn split_token_span(self, eoi: S) -> MappedInput<'src, T, S, Self> + where + Self: Input<'src, Token = (T, S), MaybeToken = &'src (T, S)> + Sized, + T: 'src, + S: Span + 'src, + { + self.map(eoi, |(t, s)| (t, s)) + } + /// Map the spans output for this input to a different output span. /// /// This is useful if you wish to include extra context that applies to all spans emitted during a parse, such as @@ -582,15 +627,26 @@ impl<'src, T: 'src, const N: usize> BorrowInput<'src> for &'src [T; N] { /// See [`Input::map`]. #[derive(Copy, Clone)] -pub struct MappedInput { +pub struct MappedInput< + 'src, + T, + S, + I, + F = fn( + >::MaybeToken, + ) -> ( + <>::MaybeToken as IntoMaybe<'src, >::Token>>::Proj, + <>::MaybeToken as IntoMaybe<'src, >::Token>>::Proj, + ), +> { input: I, eoi: S, mapper: F, #[allow(dead_code)] - phantom: EmptyPhantom, + phantom: EmptyPhantom<&'src T>, } -impl<'src, T, S, I, F> Input<'src> for MappedInput +impl<'src, T, S, I, F> Input<'src> for MappedInput<'src, T, S, I, F> where I: Input<'src>, T: 'src, @@ -649,7 +705,7 @@ where } } -impl<'src, T, S, I, F> ExactSizeInput<'src> for MappedInput +impl<'src, T, S, I, F> ExactSizeInput<'src> for MappedInput<'src, T, S, I, F> where I: ExactSizeInput<'src>, T: 'src, @@ -673,7 +729,7 @@ where } } -impl<'src, T, S, I, F> ValueInput<'src> for MappedInput +impl<'src, T, S, I, F> ValueInput<'src> for MappedInput<'src, T, S, I, F> where I: ValueInput<'src>, T: Clone + 'src, @@ -698,7 +754,7 @@ where } } -impl<'src, T, S, I, F> BorrowInput<'src> for MappedInput +impl<'src, T, S, I, F> BorrowInput<'src> for MappedInput<'src, T, S, I, F> where I: Input<'src> + BorrowInput<'src>, I::MaybeToken: From<&'src I::Token>, @@ -725,7 +781,7 @@ where } } -impl<'src, T, S, I, F> SliceInput<'src> for MappedInput +impl<'src, T, S, I, F> SliceInput<'src> for MappedInput<'src, T, S, I, F> where I: Input<'src> + SliceInput<'src, Token = (T, S)>, T: 'src, diff --git a/src/lib.rs b/src/lib.rs index 2eb6f77d..d4ae72ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2601,10 +2601,43 @@ where } } - /// Right-fold the output of the parser into a single value. + /// Fold the output of the parser into the given accumulator. + /// + /// The output type of this iterable parser is `B`, the accumulator type. + /// + /// # Examples + /// + /// ``` + /// # use chumsky::{prelude::*, error::Simple}; + /// let int = text::int::<_, extra::Err>>(10) + /// .from_str::() + /// .unwrapped(); + /// + /// let sum = int + /// .padded() + /// .repeated() + /// .fold(0, |sum, x| sum + x); /// - /// The output of the original parser must be of type `(impl IntoIterator, B)`. Because right-folds work - /// backwards, the iterator must implement [`DoubleEndedIterator`] so that it can be reversed. + /// assert_eq!(sum.parse("3 7 2").into_result(), Ok(12)); + /// assert_eq!(sum.parse("").into_result(), Ok(0)); + /// assert_eq!(sum.parse("42 1").into_result(), Ok(43)); + /// ``` + #[cfg_attr(debug_assertions, track_caller)] + fn fold(self, init: B, f: F) -> Fold + where + B: Clone, + F: Fn(B, O) -> B, + Self: Sized, + { + Fold { + parser: self, + init, + folder: f, + phantom: EmptyPhantom::new(), + } + } + + /// Right-fold the output of the parser into a single value. /// /// The output type of this iterable parser is `B`, the right-hand component of the original parser's output. /// @@ -2642,9 +2675,6 @@ where /// Right-fold the output of the parser into a single value, making use of the parser's state when doing so. /// - /// The output of the original parser must be of type `(impl IntoIterator, B)`. Because right-folds work - /// backwards, the iterator must implement [`DoubleEndedIterator`] so that it can be reversed. - /// /// The output type of this parser is `B`, the right-hand component of the original parser's output. /// /// # Examples