run_text_only_eval.py

#!/usr/bin/env python3
"""Text-only evaluation: no image is passed to the model.
Establishes a text-baseline (prior-knowledge) performance figure.
Uses a text-only VLM judge (no image) when exact match fails.
"""
from __future__ import annotations

import argparse
import functools
import logging
from pathlib import Path
from typing import Optional

from visualneedle_eval.dataset import DEFAULT_COMPLEX_IMG_250_JSONL, parse_index_list
from visualneedle_eval.runner import (
    MAX_NULL_RETRIES,
    _generate_default_log_dir,
    _suppress_noisy_logs,
    evaluate_split,
)
from visualneedle_eval._retry import retry_existing_run as _retry_existing_run
from visualneedle_eval.cli import configure_models
from visualneedle_eval.visualneedle_agent import DEFAULT_MODEL_CONFIG_FILE

print = functools.partial(print, flush=True)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
            "Evaluate text-only VQA accuracy on a prepared JSONL dataset "
            "(no images, text-only judge)."
        )
    )
    parser.add_argument(
        "--dataset-file",
        type=str,
        default=str(DEFAULT_COMPLEX_IMG_250_JSONL),
        help=f"Path to prepared JSONL dataset (default: {DEFAULT_COMPLEX_IMG_250_JSONL}).",
    )
    parser.add_argument("--limit", type=int, default=None,
                        help="Evaluate at most this many samples (ignored when --indices is set).")
    parser.add_argument("--start", type=int, default=0,
                        help="Start index within the dataset (ignored when --indices is set).")
    parser.add_argument("--indices", type=str, default=None,
                        help="Comma-separated dataset indices (e.g. '0,12,99'). Overrides --start/--limit.")
    parser.add_argument("--verbose", action="store_true",
                        help="Print detailed per-sample logs.")
    parser.add_argument("--log-dir", type=str, default=None,
                        help="Directory to write per-sample reasoning logs.")
    parser.add_argument(
        "--retry-dir",
        type=str,
        default=None,
        help="Retry an existing experiment directory in-place: rerun missing/pred=None samples and merge results back.",
    )
    parser.add_argument(
        "--auto-log-dir",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Auto-generate a log directory name (default: enabled). Disable with --no-auto-log-dir.",
    )
    parser.add_argument("--workers", type=int, default=1,
                        help="Number of parallel workers (default: 1).")
    parser.add_argument(
        "--concurrency", type=int, default=0,
        help="Max concurrent async requests (0=disabled). Recommended: 20-50.",
    )
    parser.add_argument(
        "--ordered-output", action=argparse.BooleanOptionalAction, default=None,
        help="Print results in index order (default: auto-detected from TTY).",
    )
    parser.add_argument(
        "--no-tools", action="store_true",
        help="Accepted for CLI compatibility; text-only evaluation always disables tools.",
    )
    parser.add_argument(
        "--judge", action=argparse.BooleanOptionalAction, default=True,
        help="Use the text-only judge when exact match fails (default: enabled).",
    )
    parser.add_argument(
        "--config", type=str, default=None,
        help=f"YAML model config file. Defaults to $VISUALNEEDLE_MODEL_CONFIG or {DEFAULT_MODEL_CONFIG_FILE}.",
    )
    parser.add_argument("--multiple-choice", action="store_true",
                        help="Accept answer letter (A/B/C/D) as correct match.")
    parser.add_argument(
        "--data-format", type=str, choices=["local", "url", "file"], default="local",
        help="Accepted for CLI compatibility; text-only evaluation ignores images.",
    )
    parser.add_argument("--model", type=str, default=None,
                        help="Override the evaluation model via get_model_config(...).")
    parser.add_argument("--judge-model", type=str, default=None,
                        help="Judge model config name. Defaults to defaults.judge_model from --config.")
    parser.add_argument(
        "--max-null-retries",
        type=int,
        default=MAX_NULL_RETRIES,
        help=f"Maximum prompt retries when prediction is None (default: {MAX_NULL_RETRIES}).",
    )
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    _suppress_noisy_logs(logging.WARNING)

    if args.workers < 1:
        print(f"Error: --workers must be >= 1, got {args.workers}")
        return
    if args.concurrency < 0:
        print(f"Error: --concurrency must be >= 0, got {args.concurrency}")
        return
    if args.max_null_retries < 1:
        print(f"Error: --max-null-retries must be >= 1, got {args.max_null_retries}")
        return
    if args.workers > 1 and args.concurrency > 0:
        print("Warning: both --workers and --concurrency are set. Using async mode (--concurrency).")

    if not configure_models(args):
        return

    if args.retry_dir:
        args.mode = "text_only"
        args.no_tools = True
        args.data_format = "local"
        _retry_existing_run(args)
        return

    indices = parse_index_list(args.indices)
    if args.log_dir:
        log_dir: Optional[Path] = Path(args.log_dir)
    elif args.auto_log_dir:
        log_dir = Path(_generate_default_log_dir(mode="text_only"))
        print(f"Auto-generated log directory: {log_dir} (disable with --no-auto-log-dir)")
    else:
        log_dir = None

    correct, match_correct, total, judge_used_count, judge_stats, tool_stats = evaluate_split(
        args.dataset_file,
        args.limit,
        args.start,
        indices,
        args.verbose,
        args.workers,
        args.concurrency,
        args.ordered_output,
        log_dir=log_dir,
        no_tools=True,   # text_only always disables tools
        judge=args.judge,
        model_name=args.model,
        max_null_retries=args.max_null_retries,
        multiple_choice=args.multiple_choice,
        data_format="local",  # irrelevant; no image is loaded
        mode="text_only",
    )
    match_acc = (match_correct / total) * 100 if total else 0.0
    final_acc = (correct / total) * 100 if total else 0.0
    print(f"Evaluated {total} samples.")
    print(f"Exact Match: {match_acc:.2f}% ({match_correct}/{total})")
    print(f"Final Accuracy: {final_acc:.2f}% ({correct}/{total}) | judge_used={judge_used_count}")
    judge_stats.print_stats()
    tool_stats.print_stats()


if __name__ == "__main__":
    main()