Skip to content

fix(inference): unpack structured records per sample, not all-or-nothing #1669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Anai-Guo wants to merge 1 commit into
base: main
Choose a base branch
from
+26 −6
Open

fix(inference): unpack structured records per sample, not all-or-nothing #1669

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 26 additions & 6 deletions vlmeval/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,26 @@ def _is_structured_record(v):
return isinstance(v, dict) and 'prediction' in v and 'extra_records' in v


def _unpack_structured_records(data_all, indices):
"""Split per-sample results into prediction / extra_records columns.

A sample whose retries are exhausted comes back from `BaseAPI.generate` as
a plain string rather than a structured record, so a single run can mix
both shapes. Unpack each sample on its own instead of letting one string
discard the structured output of every other sample.
"""
predictions, extra_records = [], []
for x in indices:
value = data_all[x]
if _is_structured_record(value):
predictions.append(value['prediction'])
extra_records.append(value['extra_records'])
else:
predictions.append(str(value))
extra_records.append({})
return predictions, extra_records


# A wrapper for infer_data, do the pre & post processing
def infer_data_job(
model, work_dir, model_name, dataset, verbose=False, api_nproc=4, retry_failed=True, use_vllm=False
Expand Down Expand Up @@ -245,9 +265,8 @@ def infer_data_job(
for x in data['index']:
assert x in data_all
if os.getenv('SPLIT_THINK', False):
if all(_is_structured_record(data_all[x]) for x in data['index']):
prediction = [data_all[x]['prediction'] for x in data['index']]
extra_records = [data_all[x]['extra_records'] for x in data['index']]
if any(_is_structured_record(data_all[x]) for x in data['index']):
prediction, extra_records = _unpack_structured_records(data_all, data['index'])
data['extra_records'] = extra_records
else:
prediction = [str(data_all[x]) for x in data['index']]
Expand All @@ -274,9 +293,10 @@ def split_thinking(s):
else:
# data['prediction'] = [str(data_all[x]) for x in data['index']]
# Add for agent evaluation
if all(_is_structured_record(data_all[x]) for x in data['index']):
data['prediction'] = [data_all[x]['prediction'] for x in data['index']]
data['extra_records'] = [data_all[x]['extra_records'] for x in data['index']]
if any(_is_structured_record(data_all[x]) for x in data['index']):
predictions, extra_records = _unpack_structured_records(data_all, data['index'])
data['prediction'] = predictions
data['extra_records'] = extra_records
else:
data['prediction'] = [str(data_all[x]) for x in data['index']]
if 'image' in data:
Expand Down
Loading