feat: add ICE-Bench OpenRouter smoke pipeline

examples/models/openrouter_ice_smoke.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:?Error: OPENROUTER_API_KEY not set}"
+
+MODEL_VERSION="${MODEL_VERSION:-google/gemini-2.5-flash-image}"
+TASKS="${TASKS:-ice_bench}"
+LIMIT="${LIMIT:-1}"
+OUTPUT_PATH="${OUTPUT_PATH:-./logs/openrouter_ice_smoke}"
+IMAGE_OUTPUT_DIR="${IMAGE_OUTPUT_DIR:-./logs/openrouter_ice_images}"
+USE_OFFICIAL_ICE_SAMPLE="${USE_OFFICIAL_ICE_SAMPLE:-1}"
+
+mkdir -p "${OUTPUT_PATH}" "${IMAGE_OUTPUT_DIR}"
+
+if [[ "${USE_OFFICIAL_ICE_SAMPLE}" == "1" ]]; then
+uv run python - <<'PY'
+import json
+import zipfile
+from pathlib import Path
+
+from huggingface_hub import hf_hub_download
+
+zip_path = hf_hub_download(
+    repo_id="ali-vilab/ICE-Bench",
+    repo_type="dataset",
+    filename="dataset.zip",
+    token=False,
+)
+
+target_jsonl = Path("/tmp/ice_bench_smoke.jsonl")
+target_dir = Path("/tmp/ice_bench_smoke_data")
+target_dir.mkdir(parents=True, exist_ok=True)
+
+with zipfile.ZipFile(zip_path) as zf:
+    with zf.open("data/data.jsonl") as fh:
+        first = json.loads(next(fh))
+
+    src_rel = first["SourceImage"]
+    instruction = first["Instruction"]
+    item_id = first["ItemID"]
+
+    src_out = target_dir / f"{item_id}_src.png"
+    with zf.open(src_rel) as src_in:
+        src_out.write_bytes(src_in.read())
+
+record = {
+    "item_id": item_id,
+    "instruction": instruction,
+    "source_image": str(src_out),
+}
+target_jsonl.write_text(json.dumps(record, ensure_ascii=False) + "\n", encoding="utf-8")
+print(f"Prepared smoke data at {target_jsonl}")
+print(f"Source image at {src_out}")
+PY
+fi
+
+echo "[INFO] Running ICE smoke with model=${MODEL_VERSION} tasks=${TASKS}"
+
+uv run python -m lmms_eval \
+  --model openrouter_image_gen \
+  --model_args "model_version=${MODEL_VERSION},output_dir=${IMAGE_OUTPUT_DIR},max_new_tokens=4096,image_size=1024x1024" \
+  --tasks "${TASKS}" \
+  --batch_size 1 \
+  --limit "${LIMIT}" \
+  --output_path "${OUTPUT_PATH}" \
+  --log_samples \
+  --verbosity INFO
+
+echo "[INFO] Done. Generated images in ${IMAGE_OUTPUT_DIR}/ice_bench"

examples/models/openrouter_image_smoke.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export OPENAI_API_KEY="${OPENAI_API_KEY:-${OPENROUTER_API_KEY:?Error: OPENROUTER_API_KEY not set}}"
+export OPENAI_API_BASE="${OPENAI_API_BASE:-https://openrouter.ai/api/v1}"
+
+MODEL_VERSION="${MODEL_VERSION:-google/gemini-2.5-flash-image}"
+TASKS="${TASKS:-ice_bench}"
+LIMIT="${LIMIT:-1}"
+OUTPUT_PATH="${OUTPUT_PATH:-./logs/openrouter_image_smoke}"
+IMAGE_OUTPUT_DIR="${IMAGE_OUTPUT_DIR:-./logs/openrouter_image_outputs}"
+
+echo "[INFO] OpenRouter image smoke"
+echo "[INFO] model=${MODEL_VERSION} tasks=${TASKS} limit=${LIMIT}"
+echo "[INFO] output_path=${OUTPUT_PATH} image_output_dir=${IMAGE_OUTPUT_DIR}"
+
+uv run python -m lmms_eval \
+  --model openrouter_image_gen \
+  --model_args "model_version=${MODEL_VERSION},output_dir=${IMAGE_OUTPUT_DIR},max_new_tokens=900,image_size=1024x1024" \
+  --tasks "${TASKS}" \
+  --batch_size 1 \
+  --limit "${LIMIT}" \
+  --output_path "${OUTPUT_PATH}" \
+  --log_samples \
+  --process_with_media \
+  --verbosity INFO
+
+echo "[INFO] Done. Generated images under: ${IMAGE_OUTPUT_DIR}"

lmms_eval/models/__init__.py

@@ -66,6 +66,7 @@
     "ola": "Ola",
     "omnivinci": "OmniVinci",
     "openai": "OpenAICompatible",
+    "openrouter_image_gen": "OpenRouterImageGen",
     "oryx": "Oryx",
     "phi3v": "Phi3v",
     "phi4_multimodal": "Phi4",

lmms_eval/models/simple/audio_flamingo_3.py

@@ -5,10 +5,10 @@
 import numpy as np
 import soundfile as sf
 import torch
+import transformers
 from accelerate import Accelerator, DistributedType
 from loguru import logger as eval_logger
 from tqdm import tqdm
-import transformers
 from transformers import AutoProcessor
 
 try:
@@ -53,11 +53,7 @@ def __init__(
             self.device_map = f"cuda:{accelerator.local_process_index}"
 
         if AudioFlamingo3ForConditionalGeneration is None:
-            raise ImportError(
-                "AudioFlamingo3ForConditionalGeneration is not available in transformers "
-                f"{transformers.__version__}. Please upgrade transformers/accelerate in this env, e.g. "
-                "`pip install -U transformers accelerate`."
-            )
+            raise ImportError("AudioFlamingo3ForConditionalGeneration is not available in transformers " f"{transformers.__version__}. Please upgrade transformers/accelerate in this env, e.g. " "`pip install -U transformers accelerate`.")
 
         self._model = AudioFlamingo3ForConditionalGeneration.from_pretrained(
             pretrained,

lmms_eval/models/simple/openrouter_image_gen.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+import requests as http_requests
+from PIL import Image
+
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+
+
+@register_model("openrouter_image_gen")
+class OpenRouterImageGen(lmms):
+    is_simple = True
+
+    def __init__(
+        self,
+        model_version: str = "openai/gpt-5-image-mini",
+        output_dir: str = "./logs/openrouter_image_gen",
+        max_new_tokens: int = 1024,
+        temperature: Optional[float] = None,
+        image_size: str = "1024x1024",
+        max_retries: int = 3,
+        timeout: int = 180,
+        **_: Any,
+    ) -> None:
+        super().__init__()
+        self.model_version = model_version
+        self.output_dir = output_dir
+        self.max_new_tokens = max_new_tokens
+        self.temperature = None if temperature is None else float(temperature)
+        self.image_size = image_size
+        self.max_retries = max_retries
+        self.timeout = timeout
+
+        self.api_key = os.getenv("OPENROUTER_API_KEY")
+        if not self.api_key:
+            raise EnvironmentError("OPENROUTER_API_KEY is required for openrouter_image_gen")
+
+        self.base_url = "https://openrouter.ai/api/v1/chat/completions"
+        self.session = http_requests.Session()
+        self.session.headers.update(
+            {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+        )
+
+        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
+
+    def _encode_image(self, image: Image.Image) -> str:
+        from io import BytesIO
+
+        buf = BytesIO()
+        image.convert("RGB").save(buf, format="PNG")
+        return base64.b64encode(buf.getvalue()).decode("utf-8")
+
+    def _decode_data_url(self, data_url: str) -> bytes:
+        marker = "base64,"
+        idx = data_url.find(marker)
+        if idx == -1:
+            raise ValueError("Image data URL missing base64 payload")
+        payload = data_url[idx + len(marker) :]
+        return base64.b64decode(payload)
+
+    def _extract_images(self, payload: dict[str, Any]) -> list[str]:
+        out: list[str] = []
+        try:
+            images = payload["choices"][0]["message"].get("images", [])
+        except (KeyError, IndexError, TypeError):
+            return out
+
+        for item in images:
+            if not isinstance(item, dict):
+                continue
+            image_url = item.get("image_url", {})
+            if not isinstance(image_url, dict):
+                continue
+            url = image_url.get("url")
+            if isinstance(url, str) and url.startswith("data:image"):
+                out.append(url)
+        return out
+
+    def _request_generation(self, prompt: str, visuals: list[Image.Image]) -> dict[str, Any]:
+        content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
+        for img in visuals:
+            b64 = self._encode_image(img)
+            content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}})
+
+        payload: dict[str, Any] = {
+            "model": self.model_version,
+            "messages": [{"role": "user", "content": content}],
+            "modalities": ["text", "image"],
+            "image": {"size": self.image_size},
+            "max_tokens": self.max_new_tokens,
+        }
+        if self.temperature is not None:
+            payload["temperature"] = self.temperature
+
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                resp = self.session.post(self.base_url, json=payload, timeout=self.timeout)
+                resp.raise_for_status()
+                return resp.json()
+            except http_requests.HTTPError as exc:
+                detail = ""
+                if exc.response is not None:
+                    detail = exc.response.text
+                if attempt == self.max_retries:
+                    raise RuntimeError(f"OpenRouter HTTPError: {detail}") from exc
+                time.sleep(min(2 * attempt, 8))
+            except Exception:
+                if attempt == self.max_retries:
+                    raise
+                time.sleep(min(2 * attempt, 8))
+        raise RuntimeError("Unreachable retry loop")
+
+    def _save_images(self, image_data_urls: list[str], task: str, doc_id: int) -> list[str]:
+        task_dir = Path(self.output_dir) / str(task).replace("/", "_")
+        task_dir.mkdir(parents=True, exist_ok=True)
+
+        saved_paths: list[str] = []
+        for idx, data_url in enumerate(image_data_urls):
+            raw = self._decode_data_url(data_url)
+            path = task_dir / f"{doc_id}_{idx}.png"
+            path.write_bytes(raw)
+            saved_paths.append(str(path))
+        return saved_paths
+
+    def generate_until(self, requests: list[Instance]) -> list[str]:
+        outputs: list[str] = []
+        for req in requests:
+            args = req.args
+            if len(args) < 6:
+                outputs.append(json.dumps({"text": "", "images": []}, ensure_ascii=False))
+                continue
+            ctx, gen_kwargs, doc_to_visual, doc_id, task, split = args[:6]
+            prompt = str(ctx)
+            local_gen_kwargs = dict(gen_kwargs or {})
+
+            visuals_raw = doc_to_visual(self.task_dict[task][split][doc_id])
+            visuals: list[Image.Image] = []
+            for item in visuals_raw:
+                if isinstance(item, Image.Image):
+                    visuals.append(item)
+
+            if "max_new_tokens" in local_gen_kwargs:
+                self.max_new_tokens = int(local_gen_kwargs["max_new_tokens"])
+            if "temperature" in local_gen_kwargs:
+                value = local_gen_kwargs["temperature"]
+                self.temperature = None if value is None else float(value)
+
+            try:
+                data = self._request_generation(prompt=prompt, visuals=visuals)
+            except Exception:
+                data = self._request_generation(prompt=prompt, visuals=[])
+            image_urls = self._extract_images(data)
+            saved_images = self._save_images(image_urls, task=str(task), doc_id=int(doc_id))
+
+            text = ""
+            try:
+                text = data["choices"][0]["message"].get("content", "")
+            except (KeyError, IndexError, TypeError):
+                text = ""
+
+            result = {"text": text, "images": saved_images}
+            outputs.append(json.dumps(result, ensure_ascii=False))
+            self.cache_hook.add_partial("generate_until", (ctx, local_gen_kwargs), outputs[-1])
+
+        return outputs
+
+    def loglikelihood(self, requests: list[Instance]) -> list[tuple[float, bool]]:
+        raise NotImplementedError("openrouter_image_gen does not support loglikelihood")
+
+    def generate_until_multi_round(self, requests: list[Instance]) -> list[str]:
+        raise NotImplementedError("openrouter_image_gen does not support multi-round generation")