test_whisper_perf.sh

#!/usr/bin/env bash
# =============================================================================
#  VOXD Whisper Performance Test
#
#  Records audio for 5 seconds, transcribes it with whisper-cli,
#  and reports transcription time and results.
# =============================================================================
set -euo pipefail

# Colors
YEL=$'\033[1;33m'
GRN=$'\033[1;32m'
RED=$'\033[0;31m'
NC=$'\033[0m'

# Configuration
RECORD_DURATION=5
TEMP_DIR=$(mktemp -d)
AUDIO_FILE="$TEMP_DIR/test_audio.wav"
OUTPUT_PREFIX="$TEMP_DIR/transcript"
OUTPUT_TXT="${OUTPUT_PREFIX}.txt"

# Cleanup on exit
cleanup() {
    rm -rf "$TEMP_DIR"
}
trap cleanup EXIT

# Find whisper-cli
find_whisper_cli() {
    # Check environment variable
    if [[ -n "${VOXD_WC_BIN:-}" ]] && [[ -f "$VOXD_WC_BIN" ]]; then
        echo "$VOXD_WC_BIN"
        return 0
    fi
    # Check repo-local build
    if [[ -f "whisper.cpp/build/bin/whisper-cli" ]]; then
        echo "$(pwd)/whisper.cpp/build/bin/whisper-cli"
        return 0
    fi
    # Check PATH
    if command -v whisper-cli >/dev/null 2>&1; then
        command -v whisper-cli
        return 0
    fi
    return 1
}

# Find whisper model
find_whisper_model() {
    # First: Check config file (~/.config/voxd/config.yaml)
    local config_dir="${XDG_CONFIG_HOME:-$HOME/.config}"
    local config_file="$config_dir/voxd/config.yaml"
    if [[ -f "$config_file" ]]; then
        # Extract whisper_model_path value (handle quoted/unquoted, strip comments)
        local model_path
        model_path=$(grep -E "^[[:space:]]*whisper_model_path[[:space:]]*:" "$config_file" 2>/dev/null | \
            head -n1 | \
            sed -E 's/^[[:space:]]*whisper_model_path[[:space:]]*:[[:space:]]*//' | \
            sed -E 's/[[:space:]]*#.*$//' | \
            sed -E 's/^["'\''](.*)["'\'']$/\1/' | \
            sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
        if [[ -n "$model_path" ]]; then
            echo "$model_path"
            return 0
        fi
    fi
    # Fallback: Check repo-local
    if [[ -f "whisper.cpp/models/ggml-base.en.bin" ]]; then
        echo "$(pwd)/whisper.cpp/models/ggml-base.en.bin"
        return 0
    fi
    return 1
}

# Record audio
record_audio() {
    local duration="$1"
    local output="$2"
    
    # Try ffmpeg first (most reliable)
    if command -v ffmpeg >/dev/null 2>&1; then
        echo "Recording ${duration}s with ffmpeg..."
        ffmpeg -f pulse -i default -t "$duration" -ar 16000 -ac 1 -sample_fmt s16 "$output" -y >/dev/null 2>&1
        return $?
    fi
    # Try arecord (ALSA)
    if command -v arecord >/dev/null 2>&1; then
        echo "Recording ${duration}s with arecord..."
        arecord -f cd -t wav -d "$duration" -r 16000 -c 1 "$output" >/dev/null 2>&1
        return $?
    fi
    # Try sox
    if command -v sox >/dev/null 2>&1; then
        echo "Recording ${duration}s with sox..."
        sox -d -r 16000 -c 1 -b 16 "$output" trim 0 "$duration" >/dev/null 2>&1
        return $?
    fi
    echo "${RED}Error:${NC} No audio recording tool found (ffmpeg, arecord, or sox)" >&2
    return 1
}

# Main execution
main() {
    echo "${YEL}VOXD Whisper Performance Test${NC}"

    # Find whisper-cli
    WHISPER_CLI=$(find_whisper_cli) || {
        echo "${RED}Error:${NC} whisper-cli not found" >&2
        echo "  Checked: \$VOXD_WC_BIN, repo-local, and \$PATH" >&2
        exit 1
    }
    echo "whisper-cli: $WHISPER_CLI"

    # Find model
    MODEL_PATH=$(find_whisper_model) || {
        echo "${RED}Error:${NC} Whisper model not found" >&2
        echo "  Checked: config.yaml and repo-local" >&2
        exit 1
    }
    echo " used model: $MODEL_PATH"
    echo ""

    # Record audio
    echo "${YEL}Recording audio for ${RECORD_DURATION} seconds...${NC}"
    echo "  (Please speak now)"
    RECORD_START=$(date +%s.%N)
    if ! record_audio "$RECORD_DURATION" "$AUDIO_FILE"; then
        echo "${RED}Error:${NC} Failed to record audio" >&2
        exit 1
    fi
    RECORD_END=$(date +%s.%N)
    RECORD_DURATION_ACTUAL=$(echo "$RECORD_END - $RECORD_START" | bc)
    echo "  ${GRN}✓${NC} Recording complete (${RECORD_DURATION_ACTUAL}s)"
    echo ""

    # Transcribe
    echo "${YEL}Transcribing audio...${NC}"
    TRANS_START=$(date +%s.%N)

    "$WHISPER_CLI" \
        -m "$MODEL_PATH" \
        -f "$AUDIO_FILE" \
        -l en \
        -of "$OUTPUT_PREFIX" \
        -otxt >/dev/null 2>&1
    
    TRANS_END=$(date +%s.%N)
    TRANS_DURATION=$(echo "$TRANS_END - $TRANS_START" | bc)

    # Check if transcription succeeded
    if [[ ! -f "$OUTPUT_TXT" ]]; then
        echo "${RED}Error:${NC} Transcription failed - output file not found" >&2
        exit 1
    fi

    # Read transcription
    TRANSCRIPT=$(cat "$OUTPUT_TXT" 2>/dev/null || echo "")

    # Calculate speed ratio
    SPEED_RATIO=$(echo "scale=2; $RECORD_DURATION_ACTUAL / $TRANS_DURATION" | bc)

    # Display results
    echo "  ${GRN}✓${NC} Transcription complete"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "  Transcription:"
    if [[ -n "$TRANSCRIPT" ]]; then
        echo "  $TRANSCRIPT"
    else
        echo "  ${YEL}(empty transcription)${NC}"
    fi
    echo "  Speech duration:    ${RECORD_DURATION_ACTUAL} seconds"
    echo "  Transcription time:  ${TRANS_DURATION} seconds"
    echo "  Speed ratio:        ${SPEED_RATIO}x (${SPEED_RATIO} seconds of speech per second of transcription)"
    echo "  ──────────────────────────────────────────────────────────────────────────────"

    # Performance assessment
    if (( $(echo "$SPEED_RATIO >= 1.0" | bc -l) )); then
        echo "${GRN}✓ Real-time or faster transcription${NC}"
    else
        echo "${YEL}⚠ Slower than real-time (${SPEED_RATIO}x)${NC}"
    fi
}

# Check for bc (required for calculations)
if ! command -v bc >/dev/null 2>&1; then
    echo "${RED}Error:${NC} 'bc' command not found. Please install it:" >&2
    echo "  apt: sudo apt install bc" >&2
    echo "  dnf: sudo dnf install bc" >&2
    echo "  pacman: sudo pacman -S bc" >&2
    exit 1
fi

# Run main function
main