From 4cdef353b3d0ccbfa040bf3085e5763dd2e6e77e Mon Sep 17 00:00:00 2001 From: Tai An Date: Thu, 3 Sep 2026 00:17:19 -0700 Subject: [PATCH] fix(inference): unpack structured records per sample, not all-or-nothing infer_data_job() only splits the prediction / extra_records columns when EVERY sample is a structured record: if all(_is_structured_record(data_all[x]) for x in data['index']): But a sample that exhausts its retries comes back from BaseAPI.generate as a plain string (`return self.fail_msg if answer in ['', None] else answer`), while a successful one returns {"prediction": ..., "extra_records": ...}. A single failed sample therefore makes all(...) False, and every successful record in the run is stringified into its own dict repr - the extra_records column is never created, and the prediction column holds "{'prediction': ..., 'extra_records': {...}}" for downstream scoring. Unpack each sample on its own via a shared helper, and gate on any(...) so that a run with no structured records at all still takes the old path. Fully homogeneous runs (all structured, none structured) are unchanged. The SPLIT_THINK branch had the same all-or-nothing gate and is fixed too; there the corruption is worse, because split_thinking() then cuts the dict repr at '' and leaves the tail glued to the answer. Fixes #1665 --- vlmeval/inference.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/vlmeval/inference.py b/vlmeval/inference.py index c29c002c8..22211b572 100644 --- a/vlmeval/inference.py +++ b/vlmeval/inference.py @@ -206,6 +206,26 @@ def _is_structured_record(v): return isinstance(v, dict) and 'prediction' in v and 'extra_records' in v +def _unpack_structured_records(data_all, indices): + """Split per-sample results into prediction / extra_records columns. + + A sample whose retries are exhausted comes back from `BaseAPI.generate` as + a plain string rather than a structured record, so a single run can mix + both shapes. Unpack each sample on its own instead of letting one string + discard the structured output of every other sample. + """ + predictions, extra_records = [], [] + for x in indices: + value = data_all[x] + if _is_structured_record(value): + predictions.append(value['prediction']) + extra_records.append(value['extra_records']) + else: + predictions.append(str(value)) + extra_records.append({}) + return predictions, extra_records + + # A wrapper for infer_data, do the pre & post processing def infer_data_job( model, work_dir, model_name, dataset, verbose=False, api_nproc=4, retry_failed=True, use_vllm=False @@ -245,9 +265,8 @@ def infer_data_job( for x in data['index']: assert x in data_all if os.getenv('SPLIT_THINK', False): - if all(_is_structured_record(data_all[x]) for x in data['index']): - prediction = [data_all[x]['prediction'] for x in data['index']] - extra_records = [data_all[x]['extra_records'] for x in data['index']] + if any(_is_structured_record(data_all[x]) for x in data['index']): + prediction, extra_records = _unpack_structured_records(data_all, data['index']) data['extra_records'] = extra_records else: prediction = [str(data_all[x]) for x in data['index']] @@ -274,9 +293,10 @@ def split_thinking(s): else: # data['prediction'] = [str(data_all[x]) for x in data['index']] # Add for agent evaluation - if all(_is_structured_record(data_all[x]) for x in data['index']): - data['prediction'] = [data_all[x]['prediction'] for x in data['index']] - data['extra_records'] = [data_all[x]['extra_records'] for x in data['index']] + if any(_is_structured_record(data_all[x]) for x in data['index']): + predictions, extra_records = _unpack_structured_records(data_all, data['index']) + data['prediction'] = predictions + data['extra_records'] = extra_records else: data['prediction'] = [str(data_all[x]) for x in data['index']] if 'image' in data: