perf: find tab groups in doc comments without allocating (#17410)

`get_chunks_of_tabs` collected the char indices of every doc comment
into a vector and inspected them pairwise, allocating for every doc
comment in the crate. A tab is a single byte in UTF-8 that cannot appear
inside the encoding of any other character, so the groups can be found
on the raw bytes with a fast single-byte search instead.

| crate | change |
|---|---|
| syn 2.0.71 | -0.20% |
| serde 1.0.204 | -0.08% |
| wasmi 0.35.0 | -0.11% |
| regex 1.10.5 | -1.61% |
| ryu 1.0.18 | -0.12% |

changelog: none

Author: llogiq

clippy_lints/Cargo.toml

@@ -15,6 +15,7 @@ clippy_config = { path = "../clippy_config" }
 clippy_utils = { path = "../clippy_utils" }
 declare_clippy_lint = { path = "../declare_clippy_lint" }
 itertools = "0.15"
+memchr = "2"
 quine-mc_cluskey = "0.2"
 regex-syntax = "0.8"
 serde = { version = "1.0", features = ["derive"] }

clippy_lints/src/tabs_in_doc_comments.rs

@@ -1,3 +1,5 @@
+use std::iter;
+
 use clippy_utils::diagnostics::span_lint_and_sugg;
 use rustc_ast::ast;
 use rustc_errors::Applicability;
@@ -94,57 +96,30 @@ impl EarlyLintPass for TabsInDocComments {
 ///
 /// scans the string for groups of tabs and returns the start(inclusive) and end positions
 /// (exclusive) of all groups
-/// e.g. "sd\tasd\t\taa" will be converted to [(2, 3), (6, 8)] as
+/// e.g. "sd\tasd\t\taa" will yield [(2, 3), (6, 8)] as
 ///       012 3456 7 89
 ///         ^-^  ^---^
-fn get_chunks_of_tabs(the_str: &str) -> Vec<(u32, u32)> {
+fn get_chunks_of_tabs(the_str: &str) -> impl Iterator<Item = (u32, u32)> {
     let line_length_way_to_long = "doc comment longer than 2^32 chars";
-    let mut spans: Vec<(u32, u32)> = vec![];
-    let mut current_start: u32 = 0;
-
-    // tracker to decide if the last group of tabs is not closed by a non-tab character
-    let mut is_active = false;
-
-    // Note that we specifically need the char _byte_ indices here, not the positional indexes
-    // within the char array to deal with multi-byte characters properly. `char_indices` does
-    // exactly that. It provides an iterator over tuples of the form `(byte position, char)`.
-    let char_indices: Vec<_> = the_str.char_indices().collect();
-
-    if let [(_, '\t')] = char_indices.as_slice() {
-        return vec![(0, 1)];
-    }
-
-    for entry in char_indices.windows(2) {
-        match entry {
-            [(_, '\t'), (_, '\t')] => {
-                // either string starts with double tab, then we have to set it active,
-                // otherwise is_active is true anyway
-                is_active = true;
-            },
-            [(_, _), (index_b, '\t')] => {
-                // as ['\t', '\t'] is excluded, this has to be a start of a tab group,
-                // set indices accordingly
-                is_active = true;
-                current_start = u32::try_from(*index_b).unwrap();
-            },
-            [(_, '\t'), (index_b, _)] => {
-                // this now has to be an end of the group, hence we have to push a new tuple
-                is_active = false;
-                spans.push((current_start, u32::try_from(*index_b).unwrap()));
-            },
-            _ => {},
+    let mut haystack = the_str.as_bytes();
+    let mut offset = 0;
+
+    iter::from_fn(move || {
+        if let Some(i) = memchr::memchr(b'\t', haystack) {
+            let len = 1 + haystack[i + 1..].iter().take_while(|&&x| x == b'\t').count();
+            let start = offset + i;
+            let end = start + len;
+            haystack = &haystack[i + len..];
+            offset = end;
+            Some((
+                u32::try_from(start).expect(line_length_way_to_long),
+                u32::try_from(end).expect(line_length_way_to_long),
+            ))
+        } else {
+            haystack = &[];
+            None
         }
-    }
-
-    // only possible when tabs are at the end, insert last group
-    if is_active {
-        spans.push((
-            current_start,
-            u32::try_from(char_indices.last().unwrap().0 + 1).expect(line_length_way_to_long),
-        ));
-    }
-
-    spans
+    })
 }
 
 #[cfg(test)]
@@ -153,77 +128,77 @@ mod tests_for_get_chunks_of_tabs {
 
     #[test]
     fn test_unicode_han_string() {
-        let res = get_chunks_of_tabs(" \u{4f4d}\t");
+        let res: Vec<_> = get_chunks_of_tabs(" \u{4f4d}\t").collect();
 
         assert_eq!(res, vec![(4, 5)]);
     }
 
     #[test]
     fn test_empty_string() {
-        let res = get_chunks_of_tabs("");
+        let res: Vec<_> = get_chunks_of_tabs("").collect();
 
         assert_eq!(res, vec![]);
     }
 
     #[test]
     fn test_simple() {
-        let res = get_chunks_of_tabs("sd\t\t\taa");
+        let res: Vec<_> = get_chunks_of_tabs("sd\t\t\taa").collect();
 
         assert_eq!(res, vec![(2, 5)]);
     }
 
     #[test]
     fn test_only_t() {
-        let res = get_chunks_of_tabs("\t\t");
+        let res: Vec<_> = get_chunks_of_tabs("\t\t").collect();
 
         assert_eq!(res, vec![(0, 2)]);
     }
 
     #[test]
     fn test_only_one_t() {
-        let res = get_chunks_of_tabs("\t");
+        let res: Vec<_> = get_chunks_of_tabs("\t").collect();
 
         assert_eq!(res, vec![(0, 1)]);
     }
 
     #[test]
     fn test_double() {
-        let res = get_chunks_of_tabs("sd\tasd\t\taa");
+        let res: Vec<_> = get_chunks_of_tabs("sd\tasd\t\taa").collect();
 
         assert_eq!(res, vec![(2, 3), (6, 8)]);
     }
 
     #[test]
     fn test_start() {
-        let res = get_chunks_of_tabs("\t\taa");
+        let res: Vec<_> = get_chunks_of_tabs("\t\taa").collect();
 
         assert_eq!(res, vec![(0, 2)]);
     }
 
     #[test]
     fn test_end() {
-        let res = get_chunks_of_tabs("aa\t\t");
+        let res: Vec<_> = get_chunks_of_tabs("aa\t\t").collect();
 
         assert_eq!(res, vec![(2, 4)]);
     }
 
     #[test]
     fn test_start_single() {
-        let res = get_chunks_of_tabs("\taa");
+        let res: Vec<_> = get_chunks_of_tabs("\taa").collect();
 
         assert_eq!(res, vec![(0, 1)]);
     }
 
     #[test]
     fn test_end_single() {
-        let res = get_chunks_of_tabs("aa\t");
+        let res: Vec<_> = get_chunks_of_tabs("aa\t").collect();
 
         assert_eq!(res, vec![(2, 3)]);
     }
 
     #[test]
     fn test_no_tabs() {
-        let res = get_chunks_of_tabs("dsfs");
+        let res: Vec<_> = get_chunks_of_tabs("dsfs").collect();
 
         assert_eq!(res, vec![]);
     }