rewrite impl after blog post

Author: friendlymatthew

README.md

@@ -3,67 +3,6 @@
 simdcsv is a CSV parser that evaluates 64 bytes at a time. There are many kinds of CSV files; this project adheres to the format described
 in [RFC 4180](https://www.rfc-editor.org/rfc/rfc4180.html).
 
-**Introduction**
-
-We can classify every character in CSV into the following: a COMMA, QUOTATION, NEW_LINE, OTHER. We can build a perfect lookup table and use `vqtbl1q_u8` to classify 16 characters at once. Daniel Lemire calls this "vectorized classification" in the simdjson paper. [[code pointer]](https://github.com/friendlymatthew/simdcsv/blob/main/src/classifier.rs)
-
-Once we classify every character, we can build a bitset for each class. We chunk through 64 characters at a time, building a `u64` for every chunk. Here is a naive case:
-
-```
-[//]: # COMMA = 0, QUOTATION = 1, NEW_LINE = 2, OTHER = 3
-
-aaa,bbb,ccc
-33303330333
-```
-
-Then the bitsets look like:
-
-```rs
-comma_bitset = 0b00010001000
-other_bitset = 0b11101110111
-```
-
-Now, we can just [count the number of leading zeros](https://doc.rust-lang.org/std/primitive.u64.html#method.leading_zeros) in the comma bitset to pull the csv entries.
-
-Using a bitset is pretty powerful in cases where one wants to check if there exists a symbol, count the # of symbols, or remove escaped symbols.
-
-**Detecting Escaped Quotations and Commas**
-
-Consider the csv row: `"aaa,norm","b""bb","ccc"`
-
-In CSV, quotes are escaped by doubling them (`""`). The `""` in `b""bb` is field content, not a structural delimiter. We detect escaped pairs by finding adjacent quotes:
-
-```rs
-let escaped = q & (q << 1);        // Find adjacent quote pairs
-let escaped = escaped | (escaped >> 1);  // Mark both quotes in each pair
-let valid_quotes = q & !escaped;   // Remove escaped quotes
-```
-
-```rs
-quote_bitset   = 0b100010010011010000000001
-q << 1         = 0b000100100110100000000010
-escaped        = 0b000000010000000000000000  // Found the "" pair
-escaped | >> 1 = 0b000000011000000000000000  // Both quotes marked
-valid_quotes   = 0b100010000011010000000001  // Only structural quotes remain
-```
-
-**Marking Inside Quotations**
-
-With only structural quotes, we use parallel prefix XOR to mark all bits between quote pairs:
-
-```rs
-valid_quotes  = 0b100010000011010000000001  // Structural quotes only
-inside_quotes = 0b011100011111000011111110  // All bits between quote pairs marked as 1
-```
-
-Masking out commas inside quotes:
-
-```rs
-comma_bitset  = 0b000001000000100000010000  // Commas at positions 4, 10, 18
-valid_commas  = comma_bitset & !inside_quotes
-              = 0b000001000000100000000000  // Comma at 4 masked out
-```
-
 ## Reading
 
 https://www.rfc-editor.org/rfc/rfc4180.html<br>

src/classify.rs

@@ -1,110 +1,45 @@
-pub const COMMA_CLASS: u8 = 1;
-pub const NEW_LINE_CLASS: u8 = 2;
-pub const QUOTATION_CLASS: u8 = 4;
-
-pub const LO_LOOKUP: [u8; 16] = [
-    0,
-    0,
-    QUOTATION_CLASS,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    NEW_LINE_CLASS,
-    0,
-    COMMA_CLASS,
-    NEW_LINE_CLASS,
-    0,
-    0,
-];
-
-pub const HI_LOOKUP: [u8; 16] = [
-    NEW_LINE_CLASS,
-    0,
-    COMMA_CLASS | QUOTATION_CLASS,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-];
-
-use crate::u8x16::u8x16;
-
-#[derive(Debug)]
-pub struct CsvClassifier<'a> {
-    cursor: usize,
-    data: &'a [u8],
-}
+use crate::u8x16;
 
-impl<'a> CsvClassifier<'a> {
-    pub const fn new(data: &'a [u8]) -> Self {
-        Self { cursor: 0, data }
-    }
+pub const COMMA: u8 = 1;
+pub const NEWLINE: u8 = 2;
+pub const QUOTES: u8 = 4;
 
-    pub fn classify(&mut self) -> Vec<u8x16> {
-        let mut bitsets = Vec::new();
+pub const LOW_NIBBLES: [u8; 16] = {
+    let mut out = [0u8; 16];
+    out[0x2] = QUOTES;
+    out[0xA] = NEWLINE;
+    out[0xC] = COMMA;
+    out[0xD] = NEWLINE;
 
-        let high_nibble_lookup = u8x16::from_slice_unchecked(&HI_LOOKUP);
-        let low_nibble_lookup = u8x16::from_slice_unchecked(&LO_LOOKUP);
+    out
+};
 
-        while self.cursor < self.data.len() {
-            let (lanes, _aligned) = self.load_u8x16();
+pub const HIGH_NIBBLES: [u8; 16] = {
+    let mut out = [0u8; 16];
+    out[0x0] = NEWLINE;
+    out[0x2] = COMMA | QUOTES;
 
-            let (hi_lanes, lo_lanes) = lanes.nibbles();
-            let res = high_nibble_lookup.classify(hi_lanes) & low_nibble_lookup.classify(lo_lanes);
+    out
+};
 
-            bitsets.push(res);
-        }
+// note: data must be a multiple of 16
+pub fn classify(data: &[u8]) -> Vec<u8x16> {
+    let low_nibbles = u8x16::from_slice_unchecked(&LOW_NIBBLES);
+    let high_nibbles = u8x16::from_slice_unchecked(&HIGH_NIBBLES);
 
-        // if data length is exactly aligned to 16 bytes and doesn't end with a \n
-        // the last load_u8x16 took the fast path and no new line was appended
-        let last = self.data[self.data.len() - 1];
-        if self.data.len().is_multiple_of(16) && last != 0x0A && last != 0x0D {
-            let mut temp = [0u8; 16];
-            temp[0] = 0x0A;
+    let chunks = data.chunks_exact(u8x16::LANE_COUNT);
 
-            let (hi, lo) = u8x16::from_slice_unchecked(&temp).nibbles();
+    let mut out = Vec::with_capacity(data.len() / 16);
 
-            bitsets.push(high_nibble_lookup.classify(hi) & low_nibble_lookup.classify(lo));
-        }
+    for chunk in chunks {
+        let v = u8x16::from_slice_unchecked(chunk);
+        let (high, low) = v.nibbles();
 
-        bitsets
+        let res = high_nibbles.classify(high) & low_nibbles.classify(low);
+        out.push(res);
     }
 
-    fn load_u8x16(&mut self) -> (u8x16, bool) {
-        if self.cursor + u8x16::LANE_COUNT < self.data.len() {
-            let slice = &self.data[self.cursor..self.cursor + u8x16::LANE_COUNT];
-            self.cursor += u8x16::LANE_COUNT;
-
-            return (u8x16::from_slice_unchecked(slice), true);
-        }
-
-        let slice = &self.data[self.cursor..];
-        self.cursor = self.data.len();
-
-        let mut temp = [0u8; 16];
-        let copy_len = slice.len().min(16);
-        temp[..copy_len].copy_from_slice(&slice[..copy_len]);
-
-        let last = self.data[self.data.len() - 1];
-        if last != 0x0A && last != 0x0D && copy_len < 16 {
-            temp[copy_len] = 0x0A;
-        }
-
-        (u8x16::from_slice_unchecked(&temp), false)
-    }
+    out
 }
 
 #[cfg(test)]
@@ -113,31 +48,18 @@ mod tests {
 
     #[test]
     fn test_basic_classify() {
-        let mut classifier = CsvClassifier::new(b"a,b,c\nf,\"g\"");
-        let bitsets = classifier.classify();
+        let mut data = b"a,b,c\nf,\"g\"".to_vec();
+        data.resize(data.len().next_multiple_of(16), 0);
+
+        let bitsets = classify(&data);
 
         assert_eq!(bitsets.len(), 1);
         let res: [u8; 16] = bitsets[0].into();
 
         assert_eq!(
             res,
             [
-                0,
-                COMMA_CLASS,
-                0,
-                COMMA_CLASS,
-                0,
-                NEW_LINE_CLASS,
-                0,
-                COMMA_CLASS,
-                QUOTATION_CLASS,
-                0,
-                QUOTATION_CLASS,
-                NEW_LINE_CLASS, // we always add a \n at the end
-                0,
-                0,
-                0,
-                0,
+                0, COMMA, 0, COMMA, 0, NEWLINE, 0, COMMA, QUOTES, 0, QUOTES, 0, 0, 0, 0, 0,
             ]
         );
     }

src/lib.rs

@@ -2,6 +2,7 @@
 
 mod classify;
 mod read;
-mod u8x16;
+mod simd;
 
 pub use read::*;
+pub(crate) use simd::*;

src/main.rs

@@ -2,8 +2,8 @@ use simdcsv::read;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let path = std::env::args().nth(1).expect("expect .csv file path");
-    let data = std::fs::read(path)?;
-    let rows = read(&data);
+    let mut data = std::fs::read(path)?;
+    let rows = read(&mut data);
 
     println!("{:?}", rows);