Implement branch_point/branch/fail backtracking

.github/workflows/CI.yml

@@ -67,9 +67,13 @@ jobs:
         cd bat
         sed -i 's%\[dependencies.syntect\]%[dependencies.syntect]\npath = "../syntect"%' Cargo.toml
         cargo build --release # Build bat so we can update the assets
-        PATH=target/release:$PATH ./assets/create.sh # Update assets with newly built bat
+        PATH=$PWD/target/release:$PATH ./assets/create.sh # Update assets with newly built bat
         cargo build --release # Build bat using the newly updated assets
-        PATH=./target/release:$PATH tests/syntax-tests/regression_test.sh
+        # Regenerate the expected highlighted output so the regression test
+        # compares two runs of the *same* syntect, not our build against an
+        # old baseline committed in the bat repo.
+        PATH=$PWD/target/release:$PATH tests/syntax-tests/update.sh
+        PATH=$PWD/target/release:$PATH tests/syntax-tests/regression_test.sh
 
   build-and-test:
     name: Build and test

benches/loading.rs

@@ -3,11 +3,11 @@ use syntect::highlighting::ThemeSet;
 use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
 
 fn bench_load_internal_dump(b: &mut Bencher) {
-    b.iter(|| SyntaxSet::load_defaults_newlines());
+    b.iter(SyntaxSet::load_defaults_newlines);
 }
 
 fn bench_load_internal_themes(b: &mut Bencher) {
-    b.iter(|| ThemeSet::load_defaults());
+    b.iter(ThemeSet::load_defaults);
 }
 
 fn bench_load_theme(b: &mut Bencher) {

benches/parsing.rs

@@ -9,7 +9,7 @@ fn do_parse(s: &str, ss: &SyntaxSet, syntax: &SyntaxReference) -> usize {
     let mut count = 0;
     for line in s.lines() {
         let ops = state.parse_line(line, ss).unwrap();
-        count += ops.len();
+        count += ops.ops.len();
     }
     count
 }

examples/synstats.rs

@@ -191,9 +191,9 @@ fn count(ss: &SyntaxSet, path: &Path, stats: &mut Stats) {
     let mut stack = ScopeStack::new();
     while reader.read_line(&mut line).unwrap() > 0 {
         {
-            let ops = state.parse_line(&line, ss).unwrap();
+            let output = state.parse_line(&line, ss).unwrap();
             stats.chars += line.len();
-            count_line(&ops, &line, &mut stack, stats);
+            count_line(&output.ops, &line, &mut stack, stats);
         }
         line.clear();
     }

examples/syntest.rs

@@ -274,7 +274,8 @@ fn test_file(
                     current_line_number, stack, state
                 );
             }
-            let ops = state.parse_line(&line, ss).unwrap();
+            let output = state.parse_line(&line, ss).unwrap();
+            let ops = output.ops;
             if out_opts.debug && !line_only_has_assertion {
                 if ops.is_empty() && !line.is_empty() {
                     println!("no operations for this line...");

src/easy.rs

@@ -77,7 +77,7 @@ impl<'a> HighlightLines<'a> {
         syntax_set: &SyntaxSet,
     ) -> Result<Vec<(Style, &'b str)>, Error> {
         // println!("{}", self.highlight_state.path);
-        let ops = self.parse_state.parse_line(line, syntax_set)?;
+        let ops = self.parse_state.parse_line(line, syntax_set)?.ops;
         // use util::debug_print_ops;
         // debug_print_ops(line, &ops);
         let iter =
@@ -330,7 +330,7 @@ mod tests {
         let ss = SyntaxSet::load_defaults_nonewlines();
         let mut state = ParseState::new(ss.find_syntax_by_extension("rb").unwrap());
         let line = "lol =5+2";
-        let ops = state.parse_line(line, &ss).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, &ss).expect("#[cfg(test)]").ops;
 
         let mut stack = ScopeStack::new();
         let mut token_count = 0;
@@ -362,7 +362,7 @@ mod tests {
         let mut stack = ScopeStack::new();
 
         for line in lines.iter() {
-            let ops = state.parse_line(line, &ss).expect("#[cfg(test)]");
+            let ops = state.parse_line(line, &ss).expect("#[cfg(test)]").ops;
             println!("{:?}", ops);
 
             let mut iterated_ops: Vec<&ScopeStackOp> = Vec::new();

src/highlighting/highlighter.rs

@@ -405,7 +405,7 @@ mod tests {
 
         let mut highlight_state = HighlightState::new(&highlighter, ScopeStack::new());
         let line = "module Bob::Wow::Troll::Five; 5; end";
-        let ops = state.parse_line(line, &ps).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, ps).expect("#[cfg(test)]").ops;
         let iter = HighlightIterator::new(&mut highlight_state, &ops[..], line, &highlighter);
         let regions: Vec<(Style, &str)> = iter.collect();
         // println!("{:#?}", regions);
@@ -447,15 +447,15 @@ mod tests {
         // We start by parsing a python multiline-comment: """
         let mut highlight_state = HighlightState::new(&highlighter, ScopeStack::new());
         let line = r#"""""#;
-        let ops = state.parse_line(line, &ps).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, ps).expect("#[cfg(test)]").ops;
         let iter = HighlightIterator::new(&mut highlight_state, &ops[..], line, &highlighter);
         assert_eq!(1, iter.count());
         let path = highlight_state.path;
 
         // We then parse the next line with a highlight state built from the previous state
         let mut highlight_state = HighlightState::new(&highlighter, path);
         let line = "multiline comment";
-        let ops = state.parse_line(line, &ps).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, ps).expect("#[cfg(test)]").ops;
         let iter = HighlightIterator::new(&mut highlight_state, &ops[..], line, &highlighter);
         let regions: Vec<(Style, &str)> = iter.collect();
 
@@ -617,7 +617,7 @@ mod tests {
 
         let mut highlight_state = HighlightState::new(&highlighter, ScopeStack::new());
         let line = "module Bob::Wow::Troll::Five; 5; end";
-        let ops = state.parse_line(line, &ps).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, ps).expect("#[cfg(test)]").ops;
         let iter = RangedHighlightIterator::new(&mut highlight_state, &ops[..], line, &highlighter);
         let regions: Vec<(Style, &str, Range<usize>)> = iter.collect();
         // println!("{:#?}", regions);

src/html.rs

@@ -91,7 +91,7 @@ impl<'a> ClassedHTMLGenerator<'a> {
     /// *Note:* This function requires `line` to include a newline at the end and
     /// also use of the `load_defaults_newlines` version of the syntaxes.
     pub fn parse_html_for_line_which_includes_newline(&mut self, line: &str) -> Result<(), Error> {
-        let parsed_line = self.parse_state.parse_line(line, self.syntax_set)?;
+        let parsed_line = self.parse_state.parse_line(line, self.syntax_set)?.ops;
         let (formatted_line, delta) = line_tokens_to_classed_spans(
             line,
             parsed_line.as_slice(),
@@ -584,7 +584,7 @@ mod tests {
         let syntax = ss.find_syntax_by_name("Markdown").unwrap();
         let mut state = ParseState::new(syntax);
         let line = "[w](t.co) *hi* **five**";
-        let ops = state.parse_line(line, &ss).expect("#[cfg(test)]");
+        let ops = state.parse_line(line, &ss).expect("#[cfg(test)]").ops;
         let mut stack = ScopeStack::new();
 
         // use util::debug_print_ops;

src/parsing/metadata.rs

@@ -547,33 +547,27 @@ mod tests {
         assert!(metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_START")
-            .is_some());
-        assert!(metadata
+            .contains_key("TM_COMMENT_START"));
+        assert!(!metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_END")
-            .is_none());
+            .contains_key("TM_COMMENT_END"));
         assert!(metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_START_2")
-            .is_some());
+            .contains_key("TM_COMMENT_START_2"));
         assert!(metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_START_3")
-            .is_some());
+            .contains_key("TM_COMMENT_START_3"));
         assert!(metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_END_3")
-            .is_some());
+            .contains_key("TM_COMMENT_END_3"));
         assert!(metadata
             .items
             .shell_variables
-            .get("TM_COMMENT_DISABLE_INDENT_3")
-            .is_some());
+            .contains_key("TM_COMMENT_DISABLE_INDENT_3"));
         assert!(metadata.items.line_comment.is_some());
         assert!(metadata.items.block_comment.is_some());
         assert!(metadata.items.increase_indent_pattern.is_none());