add benchmarks, rename project

Author: friendlymatthew

.gitignore

@@ -1,2 +1,3 @@
 /target
 hits.csv
+hits_100mb.csv

Cargo.lock

@@ -1,5 +1,5 @@
 [package]
-name = "simdcsv"
+name = "arrow-csv2"
 authors = ["Matthew Kim"]
 version = "0.1.0"
 edition = "2024"
@@ -14,6 +14,11 @@ arrow-cast = "55"
 [dev-dependencies]
 insta = "1.46.3"
 criterion = "0.8.2"
+arrow-csv = "55"
+
+[[bin]]
+name = "slice_clickbench"
+path = "bin/slice_clickbench.rs"
 
 [[bench]]
 name = "parse"

Cargo.toml

@@ -1,16 +1,78 @@
+use std::sync::Arc;
+
+use arrow_csv2::{ReaderBuilder, read};
+use arrow_schema::{DataType, Field, Schema};
 use criterion::{Criterion, criterion_group, criterion_main};
-use simdcsv::read;
 
-fn parse_majestic_million(c: &mut Criterion) {
-    let raw = std::fs::read("majestic_million.csv").expect("csv not found");
+const NUM_COLUMNS: usize = 105;
+fn clickbench_schema() -> Arc<Schema> {
+    Arc::new(Schema::new(
+        (0..NUM_COLUMNS)
+            .map(|i| Field::new(format!("c{i}"), DataType::Utf8, true))
+            .collect::<Vec<_>>(),
+    ))
+}
+
+fn bench_clickbench(c: &mut Criterion) {
+    let raw = std::fs::read("hits_100mb.csv")
+        .expect("hits_100mb.csv not found — run: cargo run --release --bin slice_clickbench");
+    let schema = clickbench_schema();
 
-    c.bench_function("parse majestic_million.csv", |b| {
+    c.bench_function("arrow-csv2::read (clickbench 100MB)", |b| {
         b.iter(|| {
             let mut data = raw.clone();
             read(&mut data)
         });
     });
+
+    c.bench_function("arrow-csv2::Decoder (clickbench 100MB)", |b| {
+        b.iter(|| {
+            let mut decoder = ReaderBuilder::new(schema.clone())
+                .with_batch_size(8192)
+                .build_decoder();
+
+            let mut offset = 0;
+            let mut batches = Vec::new();
+            loop {
+                let consumed = decoder.decode(&raw[offset..]).unwrap();
+                offset += consumed;
+                if consumed == 0 || decoder.capacity() == 0 {
+                    if let Some(batch) = decoder.flush().unwrap() {
+                        batches.push(batch);
+                    }
+                    if consumed == 0 && decoder.capacity() > 0 {
+                        break;
+                    }
+                }
+            }
+            batches
+        });
+    });
+
+    c.bench_function("arrow-csv::Decoder (clickbench 100MB)", |b| {
+        b.iter(|| {
+            let mut decoder = arrow_csv::ReaderBuilder::new(schema.clone())
+                .with_batch_size(8192)
+                .build_decoder();
+
+            let mut offset = 0;
+            let mut batches = Vec::new();
+            loop {
+                let consumed = decoder.decode(&raw[offset..]).unwrap();
+                offset += consumed;
+                if consumed == 0 || decoder.capacity() == 0 {
+                    if let Some(batch) = decoder.flush().unwrap() {
+                        batches.push(batch);
+                    }
+                    if consumed == 0 {
+                        break;
+                    }
+                }
+            }
+            batches
+        });
+    });
 }
 
-criterion_group!(benches, parse_majestic_million);
+criterion_group!(benches, bench_clickbench);
 criterion_main!(benches);

benches/parse.rs

@@ -0,0 +1,16 @@
+use std::io::Read;
+
+const BENCH_BYTES: usize = 100 * 1024 * 1024;
+
+fn main() {
+    let mut f =
+        std::fs::File::open("hits.csv").expect("hits.csv not found. run ./download_clickbench.sh");
+
+    let mut buf = vec![0u8; BENCH_BYTES];
+    f.read_exact(&mut buf).expect("hits.csv too small");
+
+    let end = buf.iter().rposition(|&b| b == b'\n').unwrap_or(buf.len());
+    buf.truncate(end + 1);
+
+    std::fs::write("hits_100mb.csv", &buf).expect("failed to write");
+}