Logos 11 rc2

Cargo.toml

@@ -9,8 +9,9 @@ edition = "2018"
 [dependencies]
 lalrpop = "0.18"
 lalrpop-util = "0.18"
-logos = "0.10.0"
+logos = "0.11.0-rc2"
 regex = "1"
+logos-derive = "0.11.0-rc3"
 codespan-reporting = "0.9" 
 structopt = "0.3.12"
 

src/codespan.rs

@@ -51,7 +51,7 @@ pub fn codespan<'a>(
             .with_message("Extra token"),
         User { error } => Diagnostic::error()
             .with_message("Invalid token")
-            .with_labels(vec![Label::primary(file_id, error.0.clone())])
+            .with_labels(vec![Label::primary(file_id, *error..*error)])
             .with_message("Invalid token"),
     };
 

src/error.rs

@@ -1,6 +1,6 @@
-use crate::token_wrap::*;
+use crate::lex::Token;
 
-pub type Error<'a> = lalrpop_util::ParseError<usize, Token<'a>, LexicalError>;
+pub type Error<'a> = lalrpop_util::ParseError<usize, Token<'a>, usize>;
 
 #[derive(Debug)]
 pub enum MainError {

src/lex.rs

@@ -1,12 +1,9 @@
+pub use logos::Lexer;
 use logos::Logos;
 
-// Notably absent from the above, present in the below are
-// Whitespace, EOF, LexError
-#[derive(Logos, Debug)]
-pub enum Token {
-    #[end]
-    EOF,
-
+#[derive(Logos, Debug, Clone, PartialEq)]
+#[logos(trivia = r"(\p{Whitespace}+|#.*\n)")]
+pub enum Token<'a> {
     #[token = "."]
     Dot,
 
@@ -87,23 +84,17 @@ pub enum Token {
     // \x{1d62}-\x{1d6a}
     //
     // FancyNameAscii ↔ FancyNameUnicode
-    #[regex = r"[\\][a-zA-Z][_a-zA-Z0-9]*"]
-    FancyNameAscii,
-    #[regex = r"[a-zA-Z\p{Greek}\x{1d49c}-\x{1d59f}\x{2100}-\x{214f}][_a-zA-Z0-9\x{207f}-\x{2089}\x{2090}-\x{209c}\x{1d62}-\x{1d6a}]*"]
-    Name,
+    #[regex(r"[\\][a-zA-Z][_a-zA-Z0-9]*", |lex| lex.slice())]
+    FancyNameAscii(&'a str),
+    #[regex(r"[a-zA-Z\p{Greek}\x{1d49c}-\x{1d59f}\x{2100}-\x{214f}][_a-zA-Z0-9\x{207f}-\x{2089}\x{2090}-\x{209c}\x{1d62}-\x{1d6a}]*", |lex| lex.slice())]
+    Name(&'a str),
 
     #[token = ":"]
     Colon,
 
     #[token = ";"]
     Semi,
 
-    #[regex = r"#.*\n"]
-    Comment,
-
-    #[regex = r"\p{Whitespace}+"]
-    Whitespace,
-
     #[error]
     LexError,
 }

src/main.rs

@@ -2,7 +2,6 @@ mod ast;
 mod codespan;
 mod error;
 mod lex;
-mod token_wrap;
 
 #[cfg(test)]
 mod test;
@@ -12,9 +11,10 @@ mod test_util;
 use codespan_reporting::term::termcolor::StandardStream;
 use codespan_reporting::term::{self, ColorArg};
 use error::*;
+use logos::Logos;
 use std::io::Read;
 use structopt::StructOpt;
-use token_wrap::*;
+
 #[derive(Debug, StructOpt)]
 #[structopt(name = "prop")]
 pub struct Opts {
@@ -59,8 +59,15 @@ fn main() -> Result<(), MainError> {
 
     // Not really how i'd like this to be.
     buf.read_to_string(&mut s)?;
-    let lexer = Tokens::from_string(&s);
-    let parse_result = parser::propParser::new().parse(lexer);
+
+    let lex = lex::Token::lexer(&s).spanned();
+    let parse_result = parser::propParser::new().parse(lex.map(|(t, r)| {
+        if t == lex::Token::LexError {
+            Err(r.start)
+        } else {
+            Ok((r.start, t, r.end))
+        }
+    }));
 
     match parse_result {
         Err(e) => {

src/prop.lalrpop

@@ -1,12 +1,11 @@
-use crate::token_wrap;
+use crate::lex::Token;
 use crate::ast::{Prop, Expr, Binding, Typ};
 use std::rc::Rc;
-use token_wrap::*;
 grammar<'a>;
 
 extern {
   type Location = usize;
-  type Error = LexicalError;
+  type Error = usize;
 
   enum Token<'a> {
 	"⊥" => Token::Bot,
@@ -23,10 +22,16 @@ extern {
 	")" => Token::RParen,
 	":" => Token::Colon,
 	";" => Token::Semi,
-	name => Token::Name(<&'a str>),
+	fancy_name_unicode => Token::Name(<&'a str>),
+	fancy_name_ascii => Token::FancyNameAscii(<&'a str>),
   }
 }
 
+name: &'a str = {
+  fancy_name_unicode,
+  fancy_name_ascii,
+}
+
 pub prop = Semi<Binding>;
 
 Binding: Rc<Binding> = {

src/test.rs

@@ -1,6 +1,7 @@
 use crate::error::*;
-use crate::token_wrap::*;
+use crate::lex;
 use crate::{parser, test_util};
+use logos::Logos;
 
 use unindent::unindent;
 
@@ -108,16 +109,24 @@ fn bad_ascii() -> Result<(), &'static str> {
 
     let mut num_fail = 0;
     for s in invalid_source.iter() {
-        let lexer = Tokens::from_string(&s);
-        match parser::propParser::new().parse(lexer) {
+        let lex = lex::Token::lexer(&s).spanned();
+        let parse_result = parser::propParser::new().parse(lex.map(|(t, r)| {
+            if t == lex::Token::LexError {
+                Err(r.start)
+            } else {
+                Ok((r.start, t, r.end))
+            }
+        }));
+
+        match parse_result {
             Ok(_) => {
                 // bad
                 println!("parsed but shouldn't: {}", s);
                 num_fail += 1;
             }
             Err(e) => {
                 // good
-                println!("expected error: {}", e);
+                println!("expected error: {:?}", e);
                 ()
             }
         }

src/test_util.rs

@@ -1,19 +1,24 @@
 use crate::codespan;
 use crate::error::*;
+use crate::lex;
 use crate::parser;
-use crate::token_wrap::*;
 use codespan_reporting::term;
-use codespan_reporting::term::termcolor::{ColorChoice, StandardStream};
+use logos::Logos;
 
 pub fn do_test<'a>(sources: &[&'a str]) -> Result<(), Vec<(&'a str, Error<'a>)>> {
     let (_pass, fail): (Vec<_>, Vec<_>) = sources
         .iter()
         .enumerate()
         .map(|(index, s)| {
-            (
-                index,
-                parser::propParser::new().parse(Tokens::from_string(s)),
-            )
+            (index, {
+                parser::propParser::new().parse(lex::Token::lexer(&s).spanned().map(|(t, r)| {
+                    if t == lex::Token::LexError {
+                        Err(r.start)
+                    } else {
+                        Ok((r.start, t, r.end))
+                    }
+                }))
+            })
         })
         .partition(|(_, r)| r.is_ok());
     if fail.is_empty() {
@@ -27,21 +32,17 @@ pub fn do_test<'a>(sources: &[&'a str]) -> Result<(), Vec<(&'a str, Error<'a>)>>
     }
 }
 
-// FIXME these 2 and print_errors can involve less duplication of slightly different code
-// The difference: stdout vs stderr and ColorChoice::Never vs structopt which
-// causes problems with cargo test implicit arguments.
 pub fn expect_success<'a>(result: Result<(), Vec<(&'a str, Error<'a>)>>) -> Result<(), MainError> {
     match result {
         Ok(()) => Ok(()),
         Err(e) => {
             for (source, error) in e.iter() {
-                let writer = StandardStream::stdout(ColorChoice::Never);
+                let mut writer = codespan_reporting::term::termcolor::Buffer::no_color();
                 let config = codespan_reporting::term::Config::default();
                 let (files, diagnostic) = codespan::codespan("foo", source, error);
 
-                eprintln!("capture stderr?");
-                println!("capture stdout?");
-                term::emit(&mut writer.lock(), &config, &files, &diagnostic)?;
+                term::emit(&mut writer, &config, &files, &diagnostic)?;
+                eprintln!("{}", std::str::from_utf8(writer.as_slice()).unwrap())
             }
             Err(MainError::SomethingWentAwryAndStuffWasPrinted)
         }
@@ -57,11 +58,12 @@ pub fn expect_fail<'a>(result: Result<(), Vec<(&'a str, Error<'a>)>>) -> Result<
 
         Err(e) => {
             for (source, error) in e.iter() {
-                let writer = StandardStream::stdout(ColorChoice::Never);
+                let mut writer = codespan_reporting::term::termcolor::Buffer::no_color();
                 let config = codespan_reporting::term::Config::default();
                 let (files, diagnostic) = codespan::codespan("foo", source, error);
 
-                term::emit(&mut writer.lock(), &config, &files, &diagnostic)?;
+                term::emit(&mut writer, &config, &files, &diagnostic)?;
+                eprintln!("{}", std::str::from_utf8(writer.as_slice()).unwrap())
             }
             Ok(())
         }