test-code-tools.jsonl

{"type":"user","message":{"content":"Please create a new Rust file for the voyage API integration","role":"user"},"timestamp":"2025-01-10T10:00:00Z","sessionId":"code-session","gitBranch":"main","cwd":"/project"}
{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","id":"write-001","input":{"file_path":"/Users/linda/Documents/DEV/MaxMin-Contextual-Chunking/src/main.rs","content":"mod maxmin_chunker;\\nmod voyage_api;\\n\\nuse anyhow::{Result, Context};\\nuse std::env;\\nuse std::path::Path;\\nuse maxmin_chunker::{MaxMinChunker, ChunkerConfig, visualize_chunks};\\n\\n#[tokio::main]\\nasync fn main() -> Result<()> {\\n    // Load .env file if it exists\\n    dotenv::dotenv().ok();\\n    \\n    let args: Vec<String> = env::args().collect();\\n    \\n    // Parse command line arguments\\n    if args.len() < 2 {\\n        eprintln!(\\\"Usage: {} <text_file> [options]\\\", args[0]);\\n        eprintln!(\\\"\\\\nOptions:\\\");\\n        eprintln!(\\\"  --api-key <key>            Voyage API key (or set VOYAGE_API_KEY env var)\\\");\\n        eprintln!(\\\"  --fixed-threshold <float>  Fixed similarity threshold (default: 0.6)\\\");\\n        eprintln!(\\\"  --c <float>                Coefficient for threshold adjustment (default: 0.9)\\\");\\n        eprintln!(\\\"  --init-constant <float>    Initial constant for single-sentence clusters (default: 1.5)\\\");\\n        eprintln!(\\\"\\\\nExamples:\\\");\\n        eprintln!(\\\"  {} sample.txt\\\", args[0]);\\n        eprintln!(\\\"  {} sample.txt --api-key YOUR_KEY\\\", args[0]);\\n        eprintln!(\\\"  {} sample.txt --fixed-threshold 0.5 --c 0.8\\\", args[0]);\\n        std::process::exit(1);\\n    }\\n    \\n    let file_path = &args[1];\\n    \\n    // Get API key from environment or command line\\n    let mut api_key = env::var(\\\"VOYAGE_API_KEY\\\").ok();\\n    let mut fixed_threshold = 0.6;\\n    let mut c = 0.9;\\n    let mut init_constant = 1.5;\\n    \\n    // Parse optional arguments\\n    let mut i = 2;\\n    while i < args.len() {\\n        match args[i].as_str() {\\n            \\\"--api-key\\\" => {\\n                if i + 1 < args.len() {\\n                    api_key = Some(args[i + 1].clone());\\n                    i += 2;\\n                } else {\\n                    eprintln!(\\\"Error: --api-key requires a value\\\");\\n                    std::process::exit(1);\\n                }\\n            }\\n            \\\"--fixed-threshold\\\" => {\\n                if i + 1 < args.len() {\\n                    fixed_threshold = args[i + 1].parse()\\n                        .context(\\\"Invalid fixed_threshold value\\\")?;\\n                    i += 2;\\n                } else {\\n                    eprintln!(\\\"Error: --fixed-threshold requires a value\\\");\\n                    std::process::exit(1);\\n                }\\n            }\\n            \\\"--c\\\" => {\\n                if i + 1 < args.len() {\\n                    c = args[i + 1].parse()\\n                        .context(\\\"Invalid c coefficient value\\\")?;\\n                    i += 2;\\n                } else {\\n                    eprintln!(\\\"Error: --c requires a value\\\");\\n                    std::process::exit(1);\\n                }\\n            }\\n            \\\"--init-constant\\\" => {\\n                if i + 1 < args.len() {\\n                    init_constant = args[i + 1].parse()\\n                        .context(\\\"Invalid init_constant value\\\")?;\\n                    i += 2;\\n                } else {\\n                    eprintln!(\\\"Error: --init-constant requires a value\\\");\\n                    std::process::exit(1);\\n                }\\n            }\\n            _ => {\\n                eprintln!(\\\"Unknown argument: {}\\\", args[i]);\\n                std::process::exit(1);\\n            }\\n        }\\n    }\\n    \\n    // Ensure API key is available\\n    let api_key = api_key.context(\\\"VOYAGE_API_KEY not found. Set it via environment variable or --api-key argument\\\")?;\\n    \\n    // Read the input file\\n    let text = std::fs::read_to_string(file_path)\\n        .with_context(|| format!(\\\"Failed to read file: {}\\\", file_path))?;\\n    \\n    // Create chunker configuration\\n    let config = ChunkerConfig {\\n        api_key,\\n        fixed_threshold,\\n        c,\\n        init_constant,\\n    };\\n    \\n    println!(\\\"Configuration:\\\");\\n    println!(\\\"  Fixed threshold: {}\\\", config.fixed_threshold);\\n    println!(\\\"  Coefficient (c): {}\\\", config.c);\\n    println!(\\\"  Initial constant: {}\\\", config.init_constant);\\n    println!(\\\"  Using Voyage AI API (voyage-context-3)\\\\n\\\");\\n    \\n    // Create the chunker\\n    let chunker = MaxMinChunker::new(config);\\n    \\n    // Process the text\\n    println!(\\\"Processing text...\\\");\\n    let result = chunker.chunk_text(&text).await?;\\n    \\n    // Display results\\n    println!(\\\"\\\\n=== Chunking Results ===\\\\n\\\");\\n    println!(\\\"Total chunks: {}\\\", result.chunks.len());\\n    println!(\\\"Total tokens used: {} (estimated)\\\", result.total_tokens);\\n    println!(\\\"\\\\n=== Chunks ===\\\\n\\\");\\n    \\n    for (i, chunk) in result.chunks.iter().enumerate() {\\n        println!(\\\"Chunk {} (sentences {}-{}):\\\", \\n                 i + 1, \\n                 chunk.start_idx + 1, \\n                 chunk.end_idx + 1);\\n        println!(\\\"Adaptive threshold: {:.3}\\\", chunk.adaptive_threshold);\\n        println!(\\\"Content:\\\");\\n        println!(\\\"{}\\\", chunk.text);\\n        println!(\\\"\\\\n{}\\\", \\\"-\\\".repeat(80));\\n        println!();\\n    }\\n    \\n    // Visualize the chunks\\n    println!(\\\"\\\\n=== Chunk Visualization ===\\\\n\\\");\\n    visualize_chunks(&result.chunks);\\n    \\n    Ok(())\\n}"}}],"role":"assistant"},"timestamp":"2025-01-10T10:01:00Z"}
{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Edit","id":"edit-002","input":{"file_path":"/Users/linda/Documents/DEV/MaxMin-Contextual-Chunking/Cargo.toml","old_string":"[dependencies]\\nmodel2vec = \\\"0.2.4\\\"\\nserde = { version = \\\"1.0\\\", features = [\\\"derive\\\"] }\\nserde_json = \\\"1.0\\\"","new_string":"[dependencies]\\nreqwest = { version = \\\"0.11\\\", features = [\\\"json\\\"] }\\ntokio = { version = \\\"1.35\\\", features = [\\\"full\\\"] }\\nserde = { version = \\\"1.0\\\", features = [\\\"derive\\\"] }\\nserde_json = \\\"1.0\\\"\\nanyhow = \\\"1.0\\\"\\ndotenv = \\\"0.15\\\""}}],"role":"assistant"},"timestamp":"2025-01-10T10:02:00Z"}
{"type":"assistant","message":{"content":[{"type":"tool_use","name":"MultiEdit","id":"multi-003","input":{"file_path":"/Users/linda/Documents/DEV/MaxMin-Contextual-Chunking/src/voyage_api.rs","edits":[{"old_string":"","new_string":"use anyhow::{Result, Context};\\nuse serde::{Deserialize, Serialize};\\nuse std::time::Duration;\\n\\npub const VOYAGE_API_URL: &str = \\\"https://api.voyageai.com/v1/embeddings\\\";\\npub const MODEL_NAME: &str = \\\"voyage-context-3\\\";\\npub const MAX_BATCH_SIZE: usize = 128;\\npub const EMBEDDING_DIM: usize = 1024;"},{"old_string":"pub const EMBEDDING_DIM: usize = 1024;","new_string":"pub const EMBEDDING_DIM: usize = 1024;\\n\\n#[derive(Debug, Serialize)]\\npub struct EmbeddingRequest {\\n    pub input: Vec<String>,\\n    pub model: String,\\n}\\n\\n#[derive(Debug, Deserialize)]\\npub struct EmbeddingResponse {\\n    pub data: Vec<EmbeddingData>,\\n    pub model: String,\\n    pub usage: Usage,\\n}"},{"old_string":"pub usage: Usage,\\n}","new_string":"pub usage: Usage,\\n}\\n\\n#[derive(Debug, Deserialize)]\\npub struct EmbeddingData {\\n    pub embedding: Vec<f32>,\\n    pub index: usize,\\n}\\n\\n#[derive(Debug, Deserialize)]\\npub struct Usage {\\n    pub total_tokens: usize,\\n}"}]}}],"role":"assistant"},"timestamp":"2025-01-10T10:03:00Z"}
{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Grep","id":"grep-004","input":{"pattern":"TODO|FIXME|XXX","path":"/Users/linda/Documents/DEV/MaxMin-Contextual-Chunking","glob":"*.rs","output_mode":"files_with_matches"}}],"role":"assistant"},"timestamp":"2025-01-10T10:04:00Z"}
{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Read","id":"read-005","input":{"file_path":"/Users/linda/Documents/DEV/MaxMin-Contextual-Chunking/README.md","offset":0,"limit":50}}],"role":"assistant"},"timestamp":"2025-01-10T10:05:00Z"}