samexporter/test_comprehensive.sh at main · vietanhdev/samexporter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/bin/bash
# test_comprehensive.sh

set -euo pipefail

OUT_DIR="${SAMEXPORTER_RESULTS_DIR:-visual_results/runs/comprehensive}"
mkdir -p "$OUT_DIR"

# Parameters: variant, encoder, decoder, image, prompt, suffix, extra_args
run_single_test() {
    local variant=$1
    local encoder=$2
    local decoder=$3
    local image=$4
    local prompt=$5
    local suffix=$6
    local extra_args=$7

    local img_base=$(basename "$image" | cut -d. -f1)
    local enc_base=$(basename "$encoder" | cut -d. -f1)
    local output="${OUT_DIR}/${variant}_${enc_base}_${img_base}_${suffix}.png"
    local log="${output%.png}.log"

    echo "Testing: $variant | Model: $enc_base | Image: $img_base | Mode: $suffix"

    python -m samexporter.inference \
        --sam_variant "$variant" \
        --encoder_model "$encoder" \
        --decoder_model "$decoder" \
        --image "$image" \
        --prompt "$prompt" \
        --output "$output" \
        $extra_args 2>&1 | tee "$log"

    if [ -f "$output" ]; then
        echo "  [OK] -> $output"
    else
        echo "  [FAIL] -> $output"
        exit 1
    fi
}

MODELS_SAM1=("sam_vit_h_4b8939")
MODELS_SAM2=("sam2_hiera_tiny" "sam2_hiera_large")
MODELS_SAM21=("sam2.1_hiera_tiny" "sam2.1_hiera_large")

IMAGES=("images/truck.jpg" "images/plants.png")

echo "=== Starting Comprehensive Tests ==="

# 1. Test SAM 1 & Mobile SAM
for img in "${IMAGES[@]}"; do
    img_name=$(basename "$img" | cut -d. -f1)
    # Point
    run_single_test "sam" "output_models/sam_vit_h_4b8939.encoder.onnx" "output_models/sam_vit_h_4b8939.decoder.onnx" "$img" "images/${img_name}_point.json" "point"
    # Box
    run_single_test "sam" "output_models/sam_vit_h_4b8939.encoder.onnx" "output_models/sam_vit_h_4b8939.decoder.onnx" "$img" "images/${img_name}_box.json" "box"
    if [ "$img_name" = "plants" ]; then
        run_single_test "sam" "output_models/sam_vit_h_4b8939.encoder.onnx" "output_models/sam_vit_h_4b8939.decoder.onnx" "$img" "images/plants_box_refined.json" "refined"
    fi
    # Mobile SAM
    run_single_test "sam" "output_models/mobile_sam/mobile_sam.encoder.onnx" "output_models/mobile_sam/mobile_sam.decoder.onnx" "$img" "images/${img_name}_box.json" "mobile_box"
done

# 2. Test EfficientSAM-Ti
for img in "${IMAGES[@]}"; do
    img_name=$(basename "$img" | cut -d. -f1)
    text_prompt="$img_name"
    if [ "$img_name" = "plants" ]; then
        text_prompt="plant"
    fi
    run_single_test "efficient_sam" "output_models/efficient_sam/efficientsam_ti_encoder.onnx" "output_models/efficient_sam/efficientsam_ti_decoder.onnx" "$img" "images/${img_name}_point.json" "point"
    run_single_test "efficient_sam" "output_models/efficient_sam/efficientsam_ti_encoder.onnx" "output_models/efficient_sam/efficientsam_ti_decoder.onnx" "$img" "images/${img_name}_box.json" "box"
    if [ "$img_name" = "plants" ]; then
        run_single_test "efficient_sam" "output_models/efficient_sam/efficientsam_ti_encoder.onnx" "output_models/efficient_sam/efficientsam_ti_decoder.onnx" "$img" "images/plants_box_refined.json" "refined"
    fi
done

# 3. Test SAM 2 & 2.1
for model in "${MODELS_SAM2[@]}" "${MODELS_SAM21[@]}"; do
    for img in "${IMAGES[@]}"; do
        img_name=$(basename "$img" | cut -d. -f1)
        run_single_test "sam2" "output_models/${model}.encoder.onnx" "output_models/${model}.decoder.onnx" "$img" "images/${img_name}_point.json" "point"
        run_single_test "sam2" "output_models/${model}.encoder.onnx" "output_models/${model}.decoder.onnx" "$img" "images/${img_name}_box.json" "box"
        if [ "$img_name" = "plants" ]; then
            run_single_test "sam2" "output_models/${model}.encoder.onnx" "output_models/${model}.decoder.onnx" "$img" "images/plants_box_refined.json" "refined"
        fi
    done
done

# 4. Test SAM 3 (Point, Box, Text)
for img in "${IMAGES[@]}"; do
    img_name=$(basename "$img" | cut -d. -f1)
    # Point
    run_single_test "sam3" "output_models/sam3/sam3_image_encoder.onnx" "output_models/sam3/sam3_decoder.onnx" "$img" "images/${img_name}_point.json" "point" "--language_encoder_model output_models/sam3/sam3_language_encoder.onnx --text_prompt $text_prompt"
    # Box
    run_single_test "sam3" "output_models/sam3/sam3_image_encoder.onnx" "output_models/sam3/sam3_decoder.onnx" "$img" "images/${img_name}_box.json" "box" "--language_encoder_model output_models/sam3/sam3_language_encoder.onnx --text_prompt $text_prompt"
    if [ "$img_name" = "plants" ]; then
        run_single_test "sam3" "output_models/sam3/sam3_image_encoder.onnx" "output_models/sam3/sam3_decoder.onnx" "$img" "images/plants_box_refined.json" "refined" "--language_encoder_model output_models/sam3/sam3_language_encoder.onnx --text_prompt plant"
    fi
done

# SAM 3 specific Text tests
run_single_test "sam3" "output_models/sam3/sam3_image_encoder.onnx" "output_models/sam3/sam3_decoder.onnx" "images/truck.jpg" "images/truck_sam3.json" "text_truck" "--language_encoder_model output_models/sam3/sam3_language_encoder.onnx"
run_single_test "sam3" "output_models/sam3/sam3_image_encoder.onnx" "output_models/sam3/sam3_decoder.onnx" "images/plants.png" "images/plants_text.json" "text_plant" "--language_encoder_model output_models/sam3/sam3_language_encoder.onnx"

echo -e "
All comprehensive tests passed!"