evaluate_example.py

"""Minimal template evaluator for EgoDyn-Bench.

A copy-paste starting point for adding a new model. Replace `call_my_model`
with your own inference code, point `--output` at a new path, and you have
a working evaluator that produces the JSONL contract the leaderboard expects.

How the evaluator pipeline works
--------------------------------
1. `build_common_parser` gives you all shared CLI flags (selected_clips,
   QA paths, video dirs, trajectory mode, output paths, resume, etc.).
2. You implement a single closure:

       call_api(prompt: str, image_data: list[tuple[str, str]]) -> str

   where `image_data` is a list of (base64-encoded-bytes, mime-type) pairs.
   Returning an empty string is allowed — the harness records it as an
   unparseable answer.
3. `run_evaluation` does all the rest: data loading, prompt building, frame
   extraction, resume logic, per-clip progress, optional metrics computation.

Smoke test (no model required)
------------------------------
Run as-is with the dummy model and `--max_samples 5` to verify your
environment can produce JSONL rows in the right format. You need either
`--selected_clips` (and the matching `output/{nuscenes,carla}_clips/qa.jsonl`
generated by `dataset/scripts/generate_qa.py`) or `--qa_jsonl` pointing at
an existing QA file:

    # With --selected_clips (preferred; loads the curated 1000 benchmark clips)
    python evaluation/evaluate_example.py \\
        --selected_clips selected_clips.json \\
        --output generated/example_answers.jsonl \\
        --max_samples 5 --no_trajectory --no_images

    # With an existing QA JSONL (e.g. a training split)
    python evaluation/evaluate_example.py \\
        --qa_jsonl output/.../val_qa.jsonl \\
        --clips_index output/.../clips_index.jsonl \\
        --output generated/example_answers.jsonl \\
        --max_samples 5 --no_trajectory --no_images

The dummy `call_my_model` returns ``"yes"`` for every question. The
output JSONL will obviously score badly — the goal is just to verify the
schema is correct and the harness can find your prompts.

Real run
--------
Once you've swapped in your real inference call:

    python evaluation/evaluate_example.py \\
        --selected_clips selected_clips.json \\
        --output generated/<your_model>_answers.jsonl \\
        --resume --run_eval \\
        --metrics_output results/<your_model>.json
"""
from __future__ import annotations

import logging
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(PROJECT_ROOT))

from evaluation.evaluator_common import build_common_parser, run_evaluation

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# The only function you need to replace.
# ---------------------------------------------------------------------------
def call_my_model(prompt: str, image_data: list[tuple[str, str]]) -> str:
    """Run inference with your model.

    Parameters
    ----------
    prompt:
        Full text prompt assembled by the harness. Already contains the
        question, the answer-choice list, and (if `--trajectory_mode` is set)
        the trajectory-text block. You don't normally need to modify it.
    image_data:
        List of `(base64_string, mime_type)` pairs — one per video frame
        sampled from the clip. `mime_type` is typically ``"image/jpeg"``.
        Length depends on `--num_frames` (default 10).

    Returns
    -------
    The model's free-text answer. The harness parses it later with
    `evaluation.parsers.parse_answer`, which uses last-line matching and
    a word-boundary regex against the question's canonical choices, so
    you usually don't need to post-process here.

    Tip
    ---
    To smoke-test the pipeline without a real model, returning a constant
    string like "yes" gets you a JSONL with valid schema (you just won't
    get good metrics). That's exactly what the default below does.
    """
    # === REPLACE EVERYTHING IN THIS FUNCTION ===========================
    #
    # Typical implementations:
    #
    #   # Local HuggingFace model:
    #   inputs = self.processor(text=prompt, images=image_data, ...)
    #   out = self.model.generate(**inputs, max_new_tokens=64)
    #   return self.processor.decode(out[0], skip_special_tokens=True)
    #
    #   # API-based model:
    #   resp = client.chat.completions.create(
    #       model="gpt-4o", messages=[...], max_tokens=64
    #   )
    #   return resp.choices[0].message.content.strip()
    #
    # For now, return a placeholder so you can verify the harness runs
    # end-to-end before plugging in your model.
    return "yes"
    # ====================================================================


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main() -> int:
    parser = build_common_parser(
        description="Example EgoDyn-Bench evaluator template",
        default_model="my-model-v1",
        api_key_env_var="MY_MODEL_API_KEY",  # ignored if your model is local
    )
    # Add evaluator-specific arguments here, e.g.:
    #   parser.add_argument("--my_checkpoint", type=str, default=None)
    args = parser.parse_args()

    # Optional one-time setup (load model, open API client, etc.):
    #
    #   model = load_my_model(args.model)
    #
    # Then capture it in a closure so `call_api` has access:
    #
    #   def call_api(prompt, image_data):
    #       return call_my_model(prompt, image_data, model=model)
    #
    # For the dummy version we just pass the bare function:
    call_api = call_my_model

    return run_evaluation(args, call_api)


if __name__ == "__main__":
    sys.exit(main())