run_benchmarks.sh

#!/bin/bash

set -e

# TODO: delete the bucket from s3 after benchmarks are done

echo "Building VecPuff server..."
cargo build --release --bin server

# Also kill any other server processes on port 3000
pkill -f "target/release/server" 2>/dev/null || true

echo "Starting VecPuff server (runtime logs captured to server.log)..."

RUST_LOG=info ./target/release/server > server.log 2>&1 &
SERVER_PID=$!
sleep 3

echo "Running benchmarks..."
cd vecpuff-bench

# Create results directory
mkdir -p benchmark_results

# Latency benchmark with SIFT1M dataset - realistic scale testing
echo "Running latency benchmark with SIFT1M dataset..."
uv run vecpuff-bench latency --dataset-size 1000000 --dataset-type sift --query-count 100 --warmup-queries 10 --output benchmark_results/latency_results.json

# Throughput benchmark - measures concurrent query performance
echo "Running throughput benchmark..."
uv run vecpuff-bench throughput --dataset-size 1000000 --concurrency 10 --duration 30 --output benchmark_results/throughput_results.json

# Accuracy benchmark with ground truth validation
echo "Running accuracy benchmark..."
uv run vecpuff-bench accuracy --dataset-size 500000 --dataset-type sift --query-count 50 --output benchmark_results/accuracy_results.json

# Generate comprehensive report
echo "Generating comprehensive benchmark report..."
uv run vecpuff-bench report benchmark_results/latency_results.json benchmark_results/throughput_results.json benchmark_results/accuracy_results.json --output-dir benchmark_results

cd ..

echo "Done!"
echo ""
echo "BENCHMARK RESULTS SAVED:"
echo "   Individual results: vecpuff-bench/benchmark_results/"
echo "     - latency_results.json"
echo "     - throughput_results.json"
echo "     - accuracy_results.json"
echo "   Comprehensive report: vecpuff-bench/benchmark_results/"
echo "     - results.json (all data)"
echo "     - results.csv (spreadsheet)"
echo "     - summary.txt (text summary)"
echo "   Server logs: server.log"
echo ""
echo "View summary: cat vecpuff-bench/benchmark_results/summary.txt"

# Cleanup - more robust process killing
echo "Cleaning up server process..."
if [ ! -z "$SERVER_PID" ]; then
    kill $SERVER_PID 2>/dev/null || true
    # Wait a bit and force kill if still running
    sleep 2
    kill -9 $SERVER_PID 2>/dev/null || true
fi

# Also kill any other server processes on port 3000
pkill -f "target/release/server" 2>/dev/null || true