Fixed lexing escaped regions (#72)

Author: KrosFire

src/compiling/lexing/lexer.rs

@@ -104,16 +104,6 @@ impl Lexer {
         lex_state.word = String::new()
     }
 
-    /// Checks whether this is a nontokenizable region
-    #[inline]
-    fn is_tokenized_region(&self, reaction: &RegionReaction, lex_state: &mut LexState) -> bool {
-        if let Some(region) = lex_state.region_handler.get_region() {
-            region.tokenize && *reaction == RegionReaction::Pass
-        } else {
-            false
-        }
-    }
-
     /// Pattern code for adding a symbol
     /// **[*]**
     #[inline]
@@ -191,9 +181,14 @@ impl Lexer {
 
             // Reaction stores the reaction of the region handler
             // Have we just opened or closed some region?
-            let reaction = lex_state
-                .region_handler
-                .handle_region(&lex_state.reader, lex_state.is_escaped);
+            let reaction = if lex_state.is_escaped {
+                RegionReaction::Pass
+            } else {
+                lex_state.region_handler.handle_region(&lex_state.reader)
+            };
+
+            lex_state.is_escaped = !lex_state.is_escaped && letter == self.escape_symbol;
+
             match reaction {
                 // If the region has been opened
                 // Finish the part that we have been parsing
@@ -236,7 +231,9 @@ impl Lexer {
                     }
                 }
                 RegionReaction::Pass => {
-                    let is_tokenized_region = self.is_tokenized_region(&reaction, &mut lex_state);
+                    let region = lex_state.region_handler.get_region().unwrap();
+                    let is_tokenized_region = region.tokenize;
+
                     match lex_state.compound_handler.handle_compound(
                         letter,
                         &lex_state.reader,
@@ -247,15 +244,9 @@ impl Lexer {
                         CompoundReaction::End => self.pattern_end(&mut lex_state, letter),
                         CompoundReaction::Pass => {
                             // Handle region scope
-                            if !self.is_tokenized_region(&reaction, &mut lex_state) {
-                                let region = lex_state.region_handler.get_region().unwrap();
-                                // Flip escaped key
-                                lex_state.is_escaped = (!lex_state.is_escaped
-                                    && letter == self.escape_symbol)
-                                    .then(|| !lex_state.is_escaped)
-                                    .unwrap_or(false);
+                            if !is_tokenized_region {
                                 // Handle singleline attribute
-                                if letter == '\n' && region.singleline {
+                                if region.singleline && letter == '\n' {
                                     let pos = lex_state.reader.get_position();
                                     return Err((
                                         LexerErrorType::Singleline,
@@ -509,4 +500,61 @@ mod test {
         }
         assert_eq!(expected, result);
     }
+
+    // Test if comments are tokenized in the string (it should not be)
+    #[test]
+    fn test_lexer_tokenized_regions() {
+        let symbols = vec!['/', '"', '{', '}'];
+        let regions = reg![
+            reg!(string as "String" => {
+                begin: "\"",
+                end: "\"",
+                tokenize: true
+            } => [
+                reg!(str_interp as "string interpolation" => {
+                    begin: "{",
+                    end: "}",
+                    tokenize: true
+                } ref global)
+            ]),
+            reg!(comment as "Comment" => {
+                begin: "//",
+                end: "\n",
+                allow_unclosed_region: true
+            })
+        ];
+        let compounds = vec![('/', '/')];
+        let rules = Rules::new(symbols, compounds, regions);
+        let lexer = super::Lexer::new(rules);
+
+        let text = r#""\{should not be interpolated // should not be a comment}""#;
+
+        let res = lexer.tokenize(&vec![text].join("\n"));
+        assert!(res.is_ok());
+
+        let mut result = vec![];
+        for lex in res.unwrap() {
+            result.push((lex.word, lex.pos.0, lex.pos.1));
+        }
+
+        let expected = vec![
+            ("\"", 1, 1),
+            ("\\", 1, 2),
+            ("{", 1, 3),
+            ("should", 1, 4),
+            ("not", 1, 11),
+            ("be", 1, 15),
+            ("interpolated", 1, 18),
+            ("//", 1, 31),
+            ("should", 1, 34),
+            ("not", 1, 41),
+            ("be", 1, 45),
+            ("a", 1, 48),
+            ("comment", 1, 50),
+            ("}", 1, 57),
+            ("\"", 1, 58),
+        ].iter().map(|(word, row, col)| (word.to_string(), *row, *col)).collect::<Vec<_>>();
+
+        assert_eq!(result, expected);
+    }
 }

src/compiling/lexing/region_handler.rs

@@ -1,6 +1,6 @@
-use crate::compiling_rules::{Region, Rules, RegionMap};
-use super::reader::Reader;
 use super::reader::ReadMode;
+use super::reader::Reader;
+use crate::compiling_rules::{Region, RegionMap, Rules};
 
 #[cfg(feature = "serde")]
 use serde::{Serialize, Deserialize};
@@ -10,20 +10,20 @@ use serde::{Serialize, Deserialize};
 pub enum RegionReaction {
     Begin(bool),
     End(bool),
-    Pass
+    Pass,
 }
 
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct RegionHandler {
     region_stack: Vec<Region>,
-    region_map: RegionMap
+    region_map: RegionMap,
 }
 
 impl RegionHandler {
     pub fn new(rules: &Rules) -> Self {
         RegionHandler {
             region_stack: vec![rules.region_tree.clone()],
-            region_map: rules.region_tree.clone().generate_region_map()
+            region_map: rules.region_tree.clone().generate_region_map(),
         }
     }
 
@@ -35,7 +35,7 @@ impl RegionHandler {
     // Error if after code lexing
     // some region was left unclosed
     #[inline]
-    pub fn is_region_closed(&self, reader: &Reader) -> Result<(),((usize, usize), Region)> {
+    pub fn is_region_closed(&self, reader: &Reader) -> Result<(), ((usize, usize), Region)> {
         if let Some(region) = self.region_stack.last() {
             if !region.allow_unclosed_region {
                 let pos = reader.get_position();
@@ -46,12 +46,12 @@ impl RegionHandler {
     }
 
     // Check where we are in code and open / close some region if matched
-    pub fn handle_region(&mut self, reader: &Reader, is_escaped: bool) -> RegionReaction {
+    pub fn handle_region(&mut self, reader: &Reader) -> RegionReaction {
         // If we are not in the global scope
         if let Some(region) = self.get_region() {
             for interp_region in region.interp.iter() {
                 // The region that got matched based on current code lexing state
-                if let Some(mut begin_region) = self.match_region_by_begin(reader, is_escaped) {
+                if let Some(mut begin_region) = self.match_region_by_begin(reader) {
                     if begin_region.name == *interp_region.name {
                         // Save the tokenize state here to preserve borrow rules
                         let tokenize = begin_region.tokenize;
@@ -64,7 +64,7 @@ impl RegionHandler {
                                 // If success, then we want to do the replace
                                 Some(target_region) => {
                                     begin_region.interp = target_region.interp.clone();
-                                },
+                                }
                                 // If fail then it means that we have invalid reference name
                                 None => {
                                     panic!("Could not find region with id '{}'", reference_name);
@@ -77,12 +77,12 @@ impl RegionHandler {
                 }
             }
             // Let's check if we can close current region
-            if let Some(end_region) = self.match_region_by_end(reader, is_escaped) {
+            if let Some(end_region) = self.match_region_by_end(reader) {
                 if end_region.name == region.name {
                     // Save the tokenize state here to preserve borrow rules
                     let tokenize = end_region.tokenize;
                     self.region_stack.pop();
-                    return RegionReaction::End(tokenize)
+                    return RegionReaction::End(tokenize);
                 }
             }
         }
@@ -97,48 +97,49 @@ impl RegionHandler {
         cb: impl Fn(&Region) -> &String,
         candidates: &[Region],
         read_mode: ReadMode,
-        is_escaped: bool
     ) -> Option<Region> {
         // Closure that checks if for each given Region is there any that matches current history state
-        let predicate = |candidate: &Region| match reader.get_history_or_future(cb(candidate).len(), &read_mode) {
-            Some(code_chunk) => !is_escaped && &code_chunk == cb(candidate),
-            None => false
+        let predicate = |candidate: &Region| match reader
+            .get_history_or_future(cb(candidate).len(), &read_mode)
+        {
+            Some(code_chunk) => &code_chunk == cb(candidate),
+            None => false,
         };
         self.get_region_by(predicate, candidates)
     }
 
     #[inline]
-    fn match_region_by_begin(&self, reader: &Reader, is_escaped: bool) -> Option<Region> {
+    fn match_region_by_begin(&self, reader: &Reader) -> Option<Region> {
         let region = self.get_region().unwrap();
         self.match_region_by(
             reader,
             |candidate: &Region| &candidate.begin,
             &region.interp,
             ReadMode::Future,
-            is_escaped
         )
     }
 
     #[inline]
-    fn match_region_by_end(&self, reader: &Reader, is_escaped: bool) -> Option<Region> {
+    fn match_region_by_end(&self, reader: &Reader) -> Option<Region> {
         let region = self.get_region().unwrap();
         if !region.global {
             self.match_region_by(
                 reader,
                 |candidate: &Region| &candidate.end,
                 &[region.clone()],
                 ReadMode::History,
-                is_escaped
             )
-        } else { None }
+        } else {
+            None
+        }
     }
 
     // Target region is a region on which we want to search the interpolations
     #[inline]
     fn get_region_by(&self, cb: impl Fn(&Region) -> bool, candidates: &[Region]) -> Option<Region> {
         for region in candidates.iter() {
             if cb(region) {
-                return Some(region.clone())
+                return Some(region.clone());
             }
         }
         None
@@ -147,30 +148,21 @@ impl RegionHandler {
 
 #[cfg(test)]
 mod test {
-    use crate::reg;
-    use crate::compiling_rules::Region;
-    use super::{ RegionHandler, RegionReaction };
     use super::Reader;
+    use super::{RegionHandler, RegionReaction};
+    use crate::compiling_rules::Region;
+    use crate::reg;
 
     #[test]
     fn match_region() {
-        let lines = vec![
-            "begin",
-            "\\begin",
-            "end"
-        ];
-        let expected = vec![
-            (0, String::from("begin")),
-            (15, String::from("end"))
-        ];
+        let lines = vec!["begin", "\\begin", "end"];
+        let expected = vec![(0, String::from("begin")), (15, String::from("end"))];
         let code = lines.join(" ");
         let mut reader = Reader::new(&code);
-        let region = reg![
-            reg!(module as "Module literal" => {
-                begin: "begin",
-                end: "end"
-            })
-        ];
+        let region = reg![reg!(module as "Module literal" => {
+            begin: "begin",
+            end: "end"
+        })];
         let mut rh = RegionHandler {
             region_stack: vec![region.clone()],
             region_map: region.generate_region_map(),
@@ -180,58 +172,58 @@ mod test {
         let mut is_escaped = false;
         // Simulate matching regions
         while let Some(letter) = reader.next() {
-            if let Some(begin) = rh.match_region_by_begin(&reader, is_escaped) {
+            if is_escaped {
+                is_escaped = !is_escaped && letter == '\\';
+                continue;
+            }
+
+            if let Some(begin) = rh.match_region_by_begin(&reader) {
                 rh.region_stack.push(begin.clone());
                 result.push((reader.get_index(), begin.begin));
             }
-            if let Some(end) = rh.match_region_by_end(&reader, is_escaped) {
+            if let Some(end) = rh.match_region_by_end(&reader) {
                 result.push((reader.get_index(), end.end));
             }
             // Handle the escape key
-            is_escaped = (!is_escaped && letter == '\\')
-                .then(|| !is_escaped)
-                .unwrap_or(false);
+            is_escaped = !is_escaped && letter == '\\';
         }
         assert_eq!(expected, result);
     }
 
     #[test]
     fn handle_region() {
-        let lines = vec![
-            "'My name is \\\\\\'{name}.\\\\'"
-        ];
-        let expected = vec![
-            0, 16, 21, 25
-        ];
+        let lines = vec!["'My name is \\\\\\'{name}.\\\\'"];
+        let expected = vec![0, 16, 21, 25];
         let code = lines.join("\n");
-        let region = reg![
-            reg!(string as "String literal" => {
-                begin: "'",
-                end: "'"
-            } => [
-                reg!(interp as "Interpolation" => {
-                    begin: "{",
-                    end: "}"
-                })
-            ])
-        ];
+        let region = reg![reg!(string as "String literal" => {
+            begin: "'",
+            end: "'"
+        } => [
+            reg!(interp as "Interpolation" => {
+                begin: "{",
+                end: "}"
+            })
+        ])];
         let mut reader = Reader::new(&code);
         let mut rh = RegionHandler {
             region_stack: vec![region.clone()],
-            region_map: region.generate_region_map()
+            region_map: region.generate_region_map(),
         };
         let mut result = vec![];
         let mut is_escaped = false;
         // Simulate matching regions
         while let Some(letter) = reader.next() {
-            let region_mutated = rh.handle_region(&reader, is_escaped);
+            if is_escaped {
+                is_escaped = !is_escaped && letter == '\\';
+                continue;
+            }
+
+            let region_mutated = rh.handle_region(&reader);
             if let RegionReaction::Begin(_) | RegionReaction::End(_) = region_mutated {
                 result.push(reader.get_index());
             }
             // Handle the escape key
-            is_escaped = (!is_escaped && letter == '\\')
-                .then(|| !is_escaped)
-                .unwrap_or(false);
+            is_escaped = !is_escaped && letter == '\\';
         }
         assert_eq!(expected, result);
     }