set up profile

Author: friendlymatthew

.gitignore

@@ -1,3 +1,4 @@
 /target
 hits.csv
 hits_100mb.csv
+profile.json.gz

Cargo.toml

@@ -4,7 +4,7 @@ authors = ["Matthew Kim"]
 version = "0.1.0"
 edition = "2024"
 description = "A CSV parser"
-license = "MIT"
+license = "Apache 2.0"
 
 [dependencies]
 arrow-schema = "55"
@@ -25,6 +25,7 @@ name = "parse"
 harness = false
 
 [profile.release]
-lto = "fat"
+debug = 2
+# lto = "fat"
 panic = "abort"
 codegen-units = 1

profile.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <file>"
+    exit 1
+fi
+
+FILE=$1
+
+if [ ! -f "$FILE" ]; then
+    echo "Error: File '$FILE' not found!"
+    exit 1
+fi
+
+cargo b --release && samply record ./target/release/arrow-csv2 "$FILE"

src/main.rs

@@ -1,11 +1,41 @@
-use arrow_csv2::read;
+use std::sync::Arc;
+
+use arrow_csv2::ReaderBuilder;
+use arrow_schema::{DataType, Field, Schema};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let path = std::env::args().nth(1).expect("expect .csv file path");
-    let mut data = std::fs::read(path)?;
-    let rows = read(&mut data);
+    let raw = std::fs::read(path)?;
+
+    let num_columns = raw
+        .iter()
+        .position(|&b| b == b'\n')
+        .map(|nl| raw[..nl].iter().filter(|&&b| b == b',').count() + 1)
+        .unwrap_or(1);
+
+    let schema = Arc::new(Schema::new(
+        (0..num_columns)
+            .map(|i| Field::new(format!("c{i}"), DataType::Utf8, true))
+            .collect::<Vec<_>>(),
+    ));
+
+    let mut decoder = ReaderBuilder::new(schema)
+        .with_batch_size(8192)
+        .build_decoder();
 
-    println!("{:?}", rows);
+    let mut offset = 0;
+    loop {
+        let consumed = decoder.decode(&raw[offset..])?;
+        offset += consumed;
+        if consumed == 0 || decoder.capacity() == 0 {
+            if let Some(batch) = decoder.flush()? {
+                std::hint::black_box(batch.num_rows());
+            }
+            if consumed == 0 && decoder.capacity() > 0 {
+                break;
+            }
+        }
+    }
 
     Ok(())
 }