diff --git a/benchmarks/job_bench/README.md b/benchmarks/job_bench/README.md new file mode 100644 index 0000000000..6fc9b63b72 --- /dev/null +++ b/benchmarks/job_bench/README.md @@ -0,0 +1,24 @@ +# Job-Bench + +Runs the official Job-Bench `main` split with OpenCode 1.14.18 and its weighted-rubric evaluator. Grok 4.3 is the +default judge. + +```bash +uv run gym eval prepare --benchmark job_bench + +export NVIDIA_API_KEY=... +export XAI_API_KEY=... +export RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 +uv run gym eval run \ + --benchmark job_bench \ + --model-type inference_provider \ + --model-url https://inference-api.nvidia.com/v1 \ + --model nvidia/moonshotai/kimi-k3 \ + --model-api-key "$NVIDIA_API_KEY" \ + --split benchmark \ + --output results/job_bench_kimi_k3.jsonl \ + +default_host="$ROUTABLE_HOST_IP" +``` + +Set `OPENSANDBOX_DOMAIN`, `OPENSANDBOX_API_KEY`, and a routable `ROUTABLE_HOST_IP`. Use `JOB_BENCH_JUDGE_*` to +override the judge. Set `JOB_BENCH_SPLIT=easy` for the smaller non-leaderboard split. diff --git a/benchmarks/job_bench/__init__.py b/benchmarks/job_bench/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/benchmarks/job_bench/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmarks/job_bench/config.yaml b/benchmarks/job_bench/config.yaml new file mode 100644 index 0000000000..09217dae71 --- /dev/null +++ b/benchmarks/job_bench/config.yaml @@ -0,0 +1,36 @@ +config_paths: + - responses_api_agents/opencode_sandboxed_agent/configs/opencode_sandboxed_agent.yaml + - resources_servers/job_bench/configs/job_bench.yaml + - nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml + +policy_model: + responses_api_models: + inference_provider: + uses_reasoning_parser: true + +job_bench_opencode_agent: + _inherit_from: opencode_sandboxed_agent + responses_api_agents: + opencode_sandboxed_agent: + opencode_version: 1.14.18 + opencode_max_output_tokens: 131072 + resources_server: + type: resources_servers + name: job_bench_resources_server + opencode_max_context_window: 1000000 + sandbox_timeout: 7200 + opencode_config: + permission: + external_directory: + "*": deny + /workspace: allow + /workspace/**: allow + tools: + webfetch: true + websearch: true + datasets: + - name: job_bench + type: benchmark + jsonl_fpath: benchmarks/job_bench/data/job_bench.jsonl + prepare_script: benchmarks/job_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/job_bench/data/.gitignore b/benchmarks/job_bench/data/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/benchmarks/job_bench/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/benchmarks/job_bench/prepare.py b/benchmarks/job_bench/prepare.py new file mode 100644 index 0000000000..3e4309bc98 --- /dev/null +++ b/benchmarks/job_bench/prepare.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +from pathlib import Path + + +DATA_DIR = Path(__file__).parent / "data" +OUTPUT_FPATH = DATA_DIR / "job_bench.jsonl" + + +def prepare() -> Path: + from huggingface_hub import snapshot_download + + split = os.environ.get("JOB_BENCH_SPLIT", "main") + source_dir = "dataset" if split == "main" else "dataset_easy" + root = ( + Path( + snapshot_download( + "JobBench/job-bench", + repo_type="dataset", + allow_patterns=f"{source_dir}/**", + ) + ) + / source_dir + ) + tasks = sorted(root.glob("*/task[0-9]*")) + DATA_DIR.mkdir(parents=True, exist_ok=True) + with OUTPUT_FPATH.open("w", encoding="utf-8") as output: + for task in tasks: + task_id = f"{task.parent.name}/{task.name}" + prompt = """=== TASK FOLDER === +/workspace/task + +=== INSTRUCTIONS === +1. Read TASK_INSTRUCTIONS.txt in the task folder +2. Read the files named in its Reference Files section +3. Complete the task as specified +4. Save only final deliverables in the output directory + +=== OUTPUT DIRECTORY === +/workspace/output + +All reference files are in /workspace/task. Only access /workspace or search online for needed references. +If information conflicts, explain and justify the chosen approach. Use appropriate tools to read office files.""" + output.write( + json.dumps( + { + "responses_create_params": {"input": [{"role": "user", "content": prompt}]}, + "task_id": task_id, + "task_dir": str(task), + "rubrics_file": str(task / "RUBRICS.json"), + } + ) + + "\n" + ) + print(f"Wrote {len(tasks)} {split} tasks to {OUTPUT_FPATH}") + return OUTPUT_FPATH + + +if __name__ == "__main__": + prepare() diff --git a/resources_servers/job_bench/__init__.py b/resources_servers/job_bench/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/resources_servers/job_bench/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/resources_servers/job_bench/app.py b/resources_servers/job_bench/app.py new file mode 100644 index 0000000000..ba4a374737 --- /dev/null +++ b/resources_servers/job_bench/app.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +import tarfile +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +from fastapi import Request +from pydantic import ConfigDict + +from nemo_gym.base_resources_server import ( + BaseResourcesServerConfig, + BaseSeedSessionRequest, + BaseSeedSessionResponse, + BaseVerifyRequest, + BaseVerifyResponse, + SimpleResourcesServer, +) +from nemo_gym.global_config import get_global_config_dict +from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec, create_provider +from nemo_gym.sandbox.config import resolve_provider_config, resolve_provider_metadata +from nemo_gym.server_utils import SESSION_ID_KEY, is_nemo_gym_fastapi_entrypoint +from resources_servers.job_bench.vendor import judge + + +class JobBenchConfig(BaseResourcesServerConfig): + judge_base_url: str + judge_api_key: str + judge_model: str + max_judge_workers: int = 10 + sandbox_provider: str + sandbox_config: dict[str, Any] + + +class JobBenchRequest(BaseSeedSessionRequest): + model_config = ConfigDict(extra="allow") + task_id: str + task_dir: str + rubrics_file: str + + +class JobBenchSeedResponse(BaseSeedSessionResponse): + sandbox_handle: str + + +class JobBenchVerifyRequest(BaseVerifyRequest): + model_config = ConfigDict(extra="allow") + task_id: str + task_dir: str + rubrics_file: str + + +class JobBenchVerifyResponse(BaseVerifyResponse): + model_config = ConfigDict(extra="allow") + task_id: str + score: float + max_score: float + passed_count: int + total_count: int + judge_model: str + rubrics: list[dict[str, Any]] + + +class JobBenchResourcesServer(SimpleResourcesServer): + config: JobBenchConfig + + def model_post_init(self, context: Any, /) -> None: + self._sandboxes: dict[str, AsyncSandbox] = {} + + async def seed_session(self, request: Request, body: JobBenchRequest) -> JobBenchSeedResponse: + task_dir = Path(body.task_dir) + if not (task_dir / "task_folder" / "TASK_INSTRUCTIONS.txt").is_file(): + raise ValueError(f"Invalid Job-Bench task directory: {task_dir}") + + global_config = get_global_config_dict() + provider = create_provider(resolve_provider_config(self.config.sandbox_provider, global_config)) + sandbox = AsyncSandbox(provider) + resources = SandboxResources.from_mapping(self.config.sandbox_config.get("resources", {})) + spec = SandboxSpec( + image=self.config.sandbox_config["image"], + ttl_s=self.config.sandbox_config.get("ttl_s"), + ready_timeout_s=self.config.sandbox_config.get("ready_timeout_s"), + workdir="/workspace", + env={}, + files={}, + metadata={ + **resolve_provider_metadata(self.config.sandbox_provider, global_config), + **self.config.sandbox_config.get("metadata", {}), + "task_id": body.task_id[:63], + }, + resources=resources, + entrypoint=None, + provider_options=self.config.sandbox_config.get("provider_options", {}), + ) + await sandbox.start(spec) + + with tempfile.TemporaryDirectory() as temporary_dir: + archive = Path(temporary_dir) / "task.tar.gz" + with tarfile.open(archive, "w:gz", dereference=True) as tar: + tar.add(task_dir / "task_folder", arcname="task") + await sandbox.upload(archive, "/tmp/task.tar.gz") + result = await sandbox.exec( + "mkdir -p /workspace/output && tar -xzf /tmp/task.tar.gz -C /workspace", + cwd="/", + ) + if result.return_code != 0: + await sandbox.stop() + raise RuntimeError(f"Failed to seed Job-Bench task: {result.stderr}") + + session_id = request.session[SESSION_ID_KEY] + self._sandboxes[session_id] = sandbox + return JobBenchSeedResponse(sandbox_handle=sandbox._handle.sandbox_id) + + def _judge(self, output_dir: Path, rubrics_file: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + rubrics_data = json.loads(rubrics_file.read_text(encoding="utf-8")) + rubrics = rubrics_data.get("rubrics") or rubrics_data.get("evaluation_rubrics") or [] + if not any(path.is_file() for path in output_dir.rglob("*")): + results = [ + judge.build_failed_rubric_result(index, rubric, "No output files found in the model output directory.") + for index, rubric in enumerate(rubrics) + ] + return judge.build_scorecard(results), results + file_contents = judge.extract_all_file_contents(output_dir) + if not file_contents.strip(): + results = [ + judge.build_failed_rubric_result( + index, rubric, "Output files were unreadable or empty after conversion." + ) + for index, rubric in enumerate(rubrics) + ] + return judge.build_scorecard(results), results + images = judge.collect_image_attachments(output_dir) + + with ThreadPoolExecutor(max_workers=self.config.max_judge_workers) as executor: + futures = [ + executor.submit( + judge.judge_rubric, + index, + rubric, + file_contents, + self.config.judge_model, + self.config.judge_base_url, + self.config.judge_api_key, + 300, + 3, + images, + ) + for index, rubric in enumerate(rubrics) + ] + results = [future.result()[0] for future in futures] + return judge.build_scorecard(results), results + + async def verify(self, request: Request, body: JobBenchVerifyRequest) -> JobBenchVerifyResponse: + sandbox = self._sandboxes.pop(request.session[SESSION_ID_KEY]) + try: + with tempfile.TemporaryDirectory() as temporary_dir: + local_dir = Path(temporary_dir) + archive = local_dir / "output.tar.gz" + result = await sandbox.exec("tar -czf /tmp/output.tar.gz -C /workspace/output .") + if result.return_code != 0: + raise RuntimeError(f"Failed to collect Job-Bench output: {result.stderr}") + await sandbox.download("/tmp/output.tar.gz", archive) + output_dir = local_dir / "output" + output_dir.mkdir() + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(output_dir, filter="data") + scorecard, rubrics = await asyncio.to_thread(self._judge, output_dir, Path(body.rubrics_file)) + finally: + await sandbox.stop() + + return JobBenchVerifyResponse( + **body.model_dump(), + reward=float(scorecard["normalized_score"]), + score=float(scorecard["total_score"]), + max_score=float(scorecard["max_score"]), + passed_count=int(scorecard["passed_count"]), + total_count=int(scorecard["total_count"]), + judge_model=self.config.judge_model, + rubrics=rubrics, + ) + + +if __name__ == "__main__": + JobBenchResourcesServer.run_webserver() +elif is_nemo_gym_fastapi_entrypoint(__file__): + app = JobBenchResourcesServer.run_webserver() # noqa: F401 diff --git a/resources_servers/job_bench/configs/job_bench.yaml b/resources_servers/job_bench/configs/job_bench.yaml new file mode 100644 index 0000000000..66337b8a73 --- /dev/null +++ b/resources_servers/job_bench/configs/job_bench.yaml @@ -0,0 +1,24 @@ +job_bench_resources_server: + resources_servers: + job_bench: + entrypoint: app.py + domain: other + verified: false + allowed_agents: [opencode_sandboxed_agent] + judge_base_url: ${oc.env:JOB_BENCH_JUDGE_BASE_URL,https://api.x.ai/v1} + judge_api_key: ${oc.env:JOB_BENCH_JUDGE_API_KEY,${oc.env:XAI_API_KEY,''}} + judge_model: ${oc.env:JOB_BENCH_JUDGE_MODEL,grok-4.3} + max_judge_workers: 10 + sandbox_provider: sandbox + sandbox_config: + image: ${oc.env:JOB_BENCH_SANDBOX_IMAGE,python:3.13-bookworm} + ttl_s: 10800 + ready_timeout_s: 1200 + resources: + cpu: 1 + memory_mib: 2048 + disk_gib: 30 + provider_options: {} + metadata: + benchmark: job-bench + harness: opencode diff --git a/resources_servers/job_bench/requirements.txt b/resources_servers/job_bench/requirements.txt new file mode 100644 index 0000000000..7607cc550e --- /dev/null +++ b/resources_servers/job_bench/requirements.txt @@ -0,0 +1,9 @@ +-e nemo-gym[dev,sandbox] @ ../.. +huggingface-hub>=0.24.0 +mammoth>=1.8.0 +openai>=1.0.0 +openpyxl>=3.1.0 +pandas>=2.0.0 +pdfplumber>=0.11.0 +pyarrow>=15.0.0 +python-pptx>=0.6.0 diff --git a/resources_servers/job_bench/task_data.py b/resources_servers/job_bench/task_data.py new file mode 100644 index 0000000000..13cb7c3abf --- /dev/null +++ b/resources_servers/job_bench/task_data.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pydantic import BaseModel, ConfigDict + + +class TaskData(BaseModel): + model_config = ConfigDict(extra="allow") + + task_id: str + task_dir: str + rubrics_file: str diff --git a/resources_servers/job_bench/tests/__init__.py b/resources_servers/job_bench/tests/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/resources_servers/job_bench/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/resources_servers/job_bench/tests/test_app.py b/resources_servers/job_bench/tests/test_app.py new file mode 100644 index 0000000000..902ffec473 --- /dev/null +++ b/resources_servers/job_bench/tests/test_app.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from pathlib import Path + +from resources_servers.job_bench import app + + +def test_judge_uses_official_weighted_score(monkeypatch, tmp_path: Path) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "answer.txt").write_text("answer", encoding="utf-8") + rubrics_file = tmp_path / "RUBRICS.json" + rubrics_file.write_text( + json.dumps({"rubrics": [{"rubric": "first", "weight": 3}, {"rubric": "second", "weight": 1}]}), + encoding="utf-8", + ) + monkeypatch.setattr(app.judge, "extract_all_file_contents", lambda _: "answer") + monkeypatch.setattr(app.judge, "collect_image_attachments", lambda _: []) + + def fake_judge(index, rubric, *_args): + passed = index == 0 + return { + "index": index, + "weight": rubric["weight"], + "result": {"passed": passed, "score": rubric["weight"] if passed else 0}, + }, {} + + monkeypatch.setattr(app.judge, "judge_rubric", fake_judge) + server = app.JobBenchResourcesServer.model_construct( + config=app.JobBenchConfig.model_construct( + judge_model="grok-4.3", + judge_base_url="https://api.x.ai/v1", + judge_api_key="test", + max_judge_workers=2, + ) + ) + + scorecard, results = server._judge(output_dir, rubrics_file) + + assert scorecard["normalized_score"] == 0.75 + assert scorecard["passed_count"] == 1 + assert len(results) == 2 + + +def test_empty_output_fails_without_calling_judge(monkeypatch, tmp_path: Path) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + rubrics_file = tmp_path / "RUBRICS.json" + rubrics_file.write_text(json.dumps({"rubrics": [{"rubric": "required", "weight": 5}]}), encoding="utf-8") + monkeypatch.setattr(app.judge, "judge_rubric", lambda *_args: (_ for _ in ()).throw(AssertionError())) + server = app.JobBenchResourcesServer.model_construct( + config=app.JobBenchConfig.model_construct(max_judge_workers=1) + ) + + scorecard, _ = server._judge(output_dir, rubrics_file) + + assert scorecard["normalized_score"] == 0 diff --git a/resources_servers/job_bench/vendor/LICENSE.job-bench-eval b/resources_servers/job_bench/vendor/LICENSE.job-bench-eval new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/resources_servers/job_bench/vendor/LICENSE.job-bench-eval @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/resources_servers/job_bench/vendor/README.md b/resources_servers/job_bench/vendor/README.md new file mode 100644 index 0000000000..34222f0f00 --- /dev/null +++ b/resources_servers/job_bench/vendor/README.md @@ -0,0 +1,5 @@ +# Vendored Job-Bench evaluator + +`judge.py` comes from the Apache-2.0 licensed +[`Job-Bench/job-bench-eval`](https://github.com/Job-Bench/job-bench-eval) repository. See +`LICENSE.job-bench-eval` for its license. diff --git a/resources_servers/job_bench/vendor/judge.py b/resources_servers/job_bench/vendor/judge.py new file mode 100755 index 0000000000..dc4b014d9c --- /dev/null +++ b/resources_servers/job_bench/vendor/judge.py @@ -0,0 +1,969 @@ +#!/usr/bin/env python3 +""" +Generic JobBench LLM-as-Judge. + +Reads model output files, evaluates them against task rubrics, and writes: + - a scalar reward file + - an optional detailed JSON report + +The API client is OpenAI-compatible by construction, so the same judge can be +pointed at OpenAI or any compatible proxy simply by changing +JUDGE_API_BASE / JUDGE_API_KEY / JUDGE_MODEL. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import fcntl +import hashlib +import json +import os +import re +import tempfile +import time +import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from zipfile import BadZipFile, ZipFile + + +MAX_CHARS_PER_FILE = 200_000 +SQLITE_EXTS = {"db", "sqlite", "sqlite3"} +SQLITE_ROWS_PER_TABLE = 500 +DEFAULT_JUDGE_API_BASE = "https://api.x.ai/v1" + +VISION_IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "webp"} +MAX_VISION_IMAGES = 8 +VISUAL_RUBRIC_PATTERN = re.compile( + r"\b(plot|figure|visualization|visualisation|visualize|visualise|" + r"heatmap|histogram|scatter ?plot|biplot|diagram|q[- ]?q)\b", + re.IGNORECASE, +) +VISION_MIME = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "gif": "image/gif", "webp": "image/webp"} + + +def _read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except Exception as exc: + return f"[ERROR: Failed to read text file: {path.name}: {exc}]" + + +def convert_file_to_text(path: Path) -> str: + ext = path.suffix.lower().lstrip(".") + + if ext in ( + "txt", + "md", + "csv", + "py", + "json", + "sh", + "log", + "xml", + "html", + "css", + "js", + "ts", + "yaml", + "yml", + "ini", + "cfg", + "conf", + "sql", + "rules", + "geojson", + ): + return _read_text(path) + + if ext in ("xlsx", "xls"): + try: + import pandas as pd + + xl = pd.ExcelFile(str(path)) + parts = [] + for sheet in xl.sheet_names: + df = pd.read_excel(xl, sheet_name=sheet) + parts.append(f"=== Sheet: {sheet} ===\n{df.to_csv(index=False)}") + return "\n".join(parts) + except ImportError: + return f"[ERROR: pandas/openpyxl not available for {path.name}]" + except Exception as exc: + return f"[ERROR: Failed to read Excel {path.name}: {exc}]" + + if ext == "docx": + try: + import mammoth + + def embedded_image_placeholder(image): + return { + "src": "embedded-image", + "alt": f"Embedded image: {image.content_type}", + } + + with open(str(path), "rb") as f: + result = mammoth.convert_to_markdown( + f, + convert_image=mammoth.images.img_element(embedded_image_placeholder), + ) + return result.value + except ImportError: + return f"[ERROR: mammoth not available for {path.name}]" + except Exception as exc: + return f"[ERROR: Failed to read DOCX {path.name}: {exc}]" + + if ext == "pdf": + try: + import pdfplumber + + with pdfplumber.open(str(path)) as pdf: + parts = [] + for i, page in enumerate(pdf.pages): + text = page.extract_text(layout=True) or "" + parts.append(f"=== Page {i + 1} ===\n{text}") + return "\n".join(parts) + except ImportError: + return f"[ERROR: pdfplumber not available for {path.name}]" + except Exception as exc: + return f"[ERROR: Failed to read PDF {path.name}: {exc}]" + + if ext in ("db", "sqlite", "sqlite3"): + import sqlite3 as sqlite + + try: + con = sqlite.connect(str(path)) + cur = con.cursor() + schema = con.execute("SELECT sql FROM sqlite_master WHERE sql IS NOT NULL").fetchall() + parts = ["=== Schema ==="] + parts.extend(row[0] for row in schema if row[0]) + tables = [row[0] for row in con.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()] + for table in tables: + parts.append(f"\n=== Table: {table} ===") + try: + total_rows = con.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + rows = cur.execute(f'SELECT * FROM "{table}" LIMIT {SQLITE_ROWS_PER_TABLE}').fetchall() + cols = [d[0] for d in cur.description] + parts.append(f"-- total_rows: {total_rows}; shown: {len(rows)} (LIMIT {SQLITE_ROWS_PER_TABLE})") + parts.append(",".join(cols)) + for row in rows: + parts.append(",".join("" if value is None else str(value) for value in row)) + except Exception as exc: + parts.append(f"[ERROR reading table {table}: {exc}]") + con.close() + return "\n".join(parts) + except Exception as exc: + return f"[ERROR: Failed to read SQLite {path.name}: {exc}]" + + if ext == "pptx": + try: + from pptx import Presentation + + prs = Presentation(str(path)) + parts = [] + for idx, slide in enumerate(prs.slides): + parts.append(f"=== Slide {idx + 1} ===") + for shape in slide.shapes: + if hasattr(shape, "text") and shape.text: + parts.append(shape.text) + return "\n".join(parts) + except ImportError: + return f"[ERROR: python-pptx not available for {path.name}]" + except Exception as exc: + return f"[ERROR: Failed to read PowerPoint {path.name}: {exc}]" + + if ext == "ipynb": + try: + nb = json.loads(path.read_text(encoding="utf-8")) + parts = [] + for cell in nb.get("cells", []): + parts.append(f"=== {cell['cell_type']} ===") + parts.append("".join(cell.get("source", []))) + for output in cell.get("outputs", []): + if "text" in output: + parts.append("".join(output["text"])) + return "\n".join(parts) + except Exception as exc: + return f"[ERROR: Failed to read notebook {path.name}: {exc}]" + + if ext in ("png", "jpg", "jpeg", "gif", "svg", "bmp"): + return f"[Image file: {path.name} — cannot extract text content]" + + return f"[Binary or unsupported file type: {ext} — {path.name}]" + + +def extract_all_file_contents(output_dir: Path) -> str: + parts = [] + for file_path in sorted(output_dir.rglob("*")): + if not file_path.is_file(): + continue + content = convert_file_to_text(file_path) + ext = file_path.suffix.lower().lstrip(".") + if ext not in SQLITE_EXTS and len(content) > MAX_CHARS_PER_FILE: + content = content[:MAX_CHARS_PER_FILE] + f"\n... [Content truncated at {MAX_CHARS_PER_FILE} characters]" + parts.append(f"=== FILE: {file_path.name} ===\n{content}\n") + return "\n".join(parts) + + +def rubric_needs_vision(rubric: dict) -> bool: + text = rubric.get("rubric", "") or "" + criterion = rubric.get("criterion", []) + if isinstance(criterion, list): + text = text + " " + " ".join(criterion) + elif isinstance(criterion, str): + text = text + " " + criterion + return bool(VISUAL_RUBRIC_PATTERN.search(text)) + + +def collect_image_paths(output_dir: Path, cap: int | None = MAX_VISION_IMAGES) -> list[Path]: + if not output_dir.exists(): + return [] + images = [ + p for p in sorted(output_dir.rglob("*")) if p.is_file() and p.suffix.lower().lstrip(".") in VISION_IMAGE_EXTS + ] + return images if cap is None else images[:cap] + + +def image_to_data_url(path: Path) -> str | None: + ext = path.suffix.lower().lstrip(".") + mime = VISION_MIME.get(ext) + if mime is None: + return None + try: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + except Exception: + return None + return f"data:{mime};base64,{encoded}" + + +def collect_image_attachments( + output_dir: Path, + cap: int = MAX_VISION_IMAGES, +) -> list[tuple[str, str]]: + """Collect standalone and embedded images as deduplicated data URLs.""" + if not output_dir.exists() or cap <= 0: + return [] + + attachments: list[tuple[str, str]] = [] + seen_hashes: set[str] = set() + + def add_attachment(name: str, mime: str, image_bytes: bytes) -> bool: + digest = hashlib.sha256(image_bytes).hexdigest() + if digest in seen_hashes: + return False + seen_hashes.add(digest) + encoded = base64.b64encode(image_bytes).decode("ascii") + attachments.append((name, f"data:{mime};base64,{encoded}")) + return len(attachments) >= cap + + for path in collect_image_paths(output_dir, cap=None): + ext = path.suffix.lower().lstrip(".") + mime = VISION_MIME.get(ext) + if mime is None: + continue + try: + image_bytes = path.read_bytes() + except OSError: + continue + if add_attachment(path.relative_to(output_dir).as_posix(), mime, image_bytes): + return attachments + + for docx_path in sorted(output_dir.rglob("*.docx")): + try: + with ZipFile(docx_path) as archive: + media_names = [ + name + for name in sorted(archive.namelist()) + if name.startswith("word/media/") and Path(name).suffix.lower().lstrip(".") in VISION_IMAGE_EXTS + ] + for media_name in media_names: + ext = Path(media_name).suffix.lower().lstrip(".") + mime = VISION_MIME.get(ext) + if mime is None: + continue + try: + image_bytes = archive.read(media_name) + except (KeyError, OSError): + continue + display_name = f"{docx_path.relative_to(output_dir).as_posix()}:{media_name}" + if add_attachment(display_name, mime, image_bytes): + return attachments + except (BadZipFile, OSError): + continue + + for notebook_path in sorted(output_dir.rglob("*.ipynb")): + try: + notebook = json.loads(notebook_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + for cell_index, cell in enumerate(notebook.get("cells", [])): + for output_index, output in enumerate(cell.get("outputs", [])): + data = output.get("data", {}) + if not isinstance(data, dict): + continue + for mime in ("image/png", "image/jpeg", "image/gif", "image/webp"): + encoded = data.get(mime) + if isinstance(encoded, list): + encoded = "".join(str(part) for part in encoded) + if not isinstance(encoded, str): + continue + try: + image_bytes = base64.b64decode("".join(encoded.split()), validate=True) + except (ValueError, binascii.Error): + continue + display_name = ( + f"{notebook_path.relative_to(output_dir).as_posix()}:" + f"cell-{cell_index}-output-{output_index}:{mime}" + ) + if add_attachment(display_name, mime, image_bytes): + return attachments + + return attachments + + +def normalize_criteria(rubric: dict) -> list[str]: + criterion_raw = rubric.get("criterion", []) + if isinstance(criterion_raw, str): + return [criterion_raw] + return list(criterion_raw) + + +def build_failed_rubric_result( + rubric_index: int, + rubric: dict, + overall_reasoning: str, + criteria_reasoning: str | None = None, +) -> dict: + criteria = normalize_criteria(rubric) + per_criterion_reasoning = criteria_reasoning or overall_reasoning + return { + "index": rubric_index, + "rubric": rubric.get("rubric", ""), + "weight": rubric.get("weight", 0), + "result": { + "passed": False, + "score": 0, + "criteria_count": len(criteria), + "criteria_passed": 0, + "criteria_results": [ + { + "index": idx, + "criterion": criterion, + "passed": False, + "reasoning": per_criterion_reasoning, + "evidence": "", + } + for idx, criterion in enumerate(criteria) + ], + "overall_reasoning": overall_reasoning, + }, + } + + +def build_scorecard(results: list[dict]) -> dict[str, float | int]: + total_score = sum(result["result"]["score"] for result in results) + max_score = sum(result["weight"] for result in results) + passed_count = sum(1 for result in results if result["result"]["passed"]) + total_count = len(results) + normalized = round(total_score / max_score, 4) if max_score > 0 else 0.0 + pass_rate = round(passed_count / total_count, 4) if total_count > 0 else 0.0 + + scorecard: dict[str, float | int] = { + "total_score": total_score, + "max_score": max_score, + "normalized_score": normalized, + "pass_rate": pass_rate, + "passed_count": passed_count, + "total_count": total_count, + } + for result in results: + idx = result["index"] + scorecard[f"rubric_{idx}_passed"] = 1 if result["result"]["passed"] else 0 + scorecard[f"rubric_{idx}_score"] = result["result"]["score"] + return scorecard + + +def build_reward(scorecard: dict[str, float | int]) -> dict[str, float]: + return {"reward": float(scorecard.get("normalized_score", 0.0))} + + +def build_details_report( + evaluated_model: str, + judge_model: str, + results: list[dict], + total_count: int | None = None, +) -> dict: + total_score = sum(result["result"]["score"] for result in results) + max_score = sum(result["weight"] for result in results) + passed_count = sum(1 for result in results if result["result"]["passed"]) + effective_total_count = total_count if total_count is not None else len(results) + pass_rate_value = int((passed_count / effective_total_count) * 100) if effective_total_count > 0 else 0 + + return { + "evaluated_model": evaluated_model, + "judge_model": judge_model, + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "total_score": total_score, + "max_score": max_score, + "pass_rate": f"{pass_rate_value}%", + "passed_count": passed_count, + "total_count": effective_total_count, + "rubrics": results, + } + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + handle.write("\n") + tmp_path = Path(handle.name) + os.replace(tmp_path, path) + + +def write_outputs( + result_file: Path | None, + reward: dict[str, float], + details_file: Path | None, + details: dict | None, +) -> None: + if result_file is not None: + write_json(result_file, reward) + if details_file is not None: + write_json(details_file, details or {}) + + +@contextmanager +def file_lock(lock_file: Path): + lock_file.parent.mkdir(parents=True, exist_ok=True) + with lock_file.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def load_existing_details(details_file: Path | None) -> dict | None: + if details_file is None or not details_file.exists(): + return None + try: + payload = json.loads(details_file.read_text(encoding="utf-8")) + except Exception: + return None + if isinstance(payload, dict): + return payload + return None + + +def load_existing_rubric_results(details_file: Path | None) -> dict[int, dict]: + payload = load_existing_details(details_file) + if not payload: + return {} + + results: dict[int, dict] = {} + for rubric_result in payload.get("rubrics", []): + if not isinstance(rubric_result, dict): + continue + index = rubric_result.get("index") + if isinstance(index, int): + results[index] = rubric_result + return results + + +def append_rubric_result( + details_file: Path, + lock_file: Path, + evaluated_model: str, + judge_model: str, + total_count: int, + rubric_result: dict, +) -> None: + with file_lock(lock_file): + current_payload = load_existing_details(details_file) or {} + current_rubrics = [ + item + for item in current_payload.get("rubrics", []) + if isinstance(item, dict) and item.get("index") != rubric_result.get("index") + ] + current_rubrics.append(rubric_result) + current_rubrics.sort(key=lambda item: item.get("index", -1)) + updated_payload = build_details_report( + evaluated_model, + judge_model, + current_rubrics, + total_count=total_count, + ) + write_json(details_file, updated_payload) + + +def first_non_empty(*values: str | None) -> str: + for value in values: + if value: + return value + return "" + + +def resolve_api_config( + judge_model: str, + api_base_arg: str | None, + api_key_arg: str | None, +) -> tuple[str, str]: + if not judge_model: + raise ValueError("No judge model provided. Set --judge-model or JUDGE_MODEL.") + + api_base = first_non_empty(api_base_arg, os.environ.get("JUDGE_API_BASE"), DEFAULT_JUDGE_API_BASE) + api_key = first_non_empty(api_key_arg, os.environ.get("JUDGE_API_KEY")) + return api_base, api_key + + +def get_openai_client(api_base: str, api_key: str): + from openai import OpenAI + + if not api_key: + raise ValueError("No judge API key provided. Set JUDGE_API_KEY.") + + return OpenAI(base_url=api_base, api_key=api_key) + + +def parse_judge_json(content: str) -> tuple[dict, str]: + try: + return json.loads(content), "direct_json" + except json.JSONDecodeError: + pass + + fence = re.search(r"```(?:json)?\s*\n(.*?)\n\s*```", content, re.DOTALL) + if fence: + try: + return json.loads(fence.group(1).strip()), "markdown_fence" + except json.JSONDecodeError: + pass + + first = content.find("{") + last = content.rfind("}") + if first != -1 and last > first: + try: + return json.loads(content[first : last + 1]), "first_last_brace" + except json.JSONDecodeError: + pass + + for candidate in reversed(re.findall(r"\{.*?\"criteria_results\"\s*:\s*\[.*?\].*?\}", content, re.DOTALL)): + try: + return json.loads(candidate), "regex_extract" + except json.JSONDecodeError: + continue + + raise ValueError(f"Could not extract JSON from response: {content[:500]}") + + +def judge_rubric( + rubric_index: int, + rubric: dict, + file_contents: str, + judge_model: str, + api_base: str, + api_key: str, + timeout_sec: int = 300, + max_retries: int = 3, + image_attachments: list[tuple[str, str]] | None = None, +) -> tuple[dict, dict]: + rubric_text = rubric.get("rubric", "") + weight = rubric.get("weight", 0) + criteria = normalize_criteria(rubric) + criterion_count = len(criteria) + criteria_list_text = "\n".join(f"Criterion {idx}: {criterion}" for idx, criterion in enumerate(criteria)) + + attached_images: list[tuple[str, str]] = [] + if image_attachments and rubric_needs_vision(rubric): + attached_images.extend(image_attachments) + vision_used = bool(attached_images) + + prompt = f"""You are an evaluation judge. Your task is to evaluate ALL criteria for a single rubric. + +## Rubric Description +{rubric_text} + +## Criteria to Evaluate (Judge ALL of them) +{criteria_list_text} + +## Output Files Content +The following are the contents of all output files to evaluate: + +{file_contents} + +## Evaluation Rules +- Evaluate EACH criterion listed above independently +- For each criterion: determine if it PASSES or FAILS +- Semantic matching is acceptable (you don't need exact wording match) +- Binary judgment for each criterion: PASS or FAIL only +- The rubric passes ONLY if ALL criteria pass + +## Output Format +Return your judgment as a JSON object with EXACTLY this structure (no markdown, no extra text): +{{ + "criteria_results": [ + {{"index": 0, "passed": true/false, "reasoning": "...", "evidence": "..."}}, + {{"index": 1, "passed": true/false, "reasoning": "...", "evidence": "..."}} + ], + "rubric_passed": true/false, + "overall_reasoning": "Summary of why the rubric passed or failed" +}} + +IMPORTANT: +- criteria_results array must have exactly {criterion_count} items (one for each criterion) +- rubric_passed should be true ONLY if ALL criteria passed +- Include specific evidence from the output files{" and the attached images" if vision_used else ""} +""" + + if vision_used: + user_content: list[dict] = [{"type": "text", "text": prompt}] + user_content.append( + { + "type": "text", + "text": f"\n## Attached Images ({len(attached_images)} file{'s' if len(attached_images) != 1 else ''})", + } + ) + for i, (fname, url) in enumerate(attached_images, start=1): + user_content.append({"type": "text", "text": f"Image {i}: {fname}"}) + user_content.append({"type": "image_url", "image_url": {"url": url}}) + else: + user_content = prompt + + last_error = None + raw_response = "" + parse_status = "failed" + for attempt in range(max_retries): + try: + client = get_openai_client(api_base, api_key) + response = client.chat.completions.create( + model=judge_model, + messages=[ + { + "role": "system", + "content": "You are an evaluation judge. You must return valid JSON only, with no markdown formatting or extra text.", + }, + {"role": "user", "content": user_content}, + ], + max_completion_tokens=200000, + temperature=0.0, + timeout=timeout_sec, + ) + raw_response = response.choices[0].message.content.strip() + parsed, parse_status = parse_judge_json(raw_response) + + model_criteria = parsed.get("criteria_results", []) + rubric_passed = bool(parsed.get("rubric_passed", False)) + overall_reasoning = parsed.get("overall_reasoning", "") + + enriched = [] + for idx, criterion in enumerate(criteria): + item = model_criteria[idx] if idx < len(model_criteria) else {} + enriched.append( + { + "index": idx, + "criterion": criterion, + "passed": bool(item.get("passed", False)), + "reasoning": item.get("reasoning", ""), + "evidence": item.get("evidence", ""), + } + ) + + score = weight if rubric_passed else 0 + criteria_passed = sum(1 for item in enriched if item["passed"]) + result = { + "index": rubric_index, + "rubric": rubric_text, + "weight": weight, + "result": { + "passed": rubric_passed, + "score": score, + "criteria_count": criterion_count, + "criteria_passed": criteria_passed, + "criteria_results": enriched, + "overall_reasoning": overall_reasoning, + }, + } + debug = { + "api_base": api_base, + "parse_status": parse_status, + "api_exit_code": 0, + "criterion_count": criterion_count, + "criteria_list_text": criteria_list_text, + "rubric_text": rubric_text, + "raw_response": raw_response, + "vision_used": vision_used, + "attached_images": [name for name, _ in attached_images], + } + return result, debug + except Exception as exc: + last_error = exc + if attempt < max_retries - 1: + time.sleep(2) + continue + + default_criteria = [ + { + "index": idx, + "criterion": criterion, + "passed": False, + "reasoning": f"Failed to get judge response: {last_error}", + "evidence": "", + } + for idx, criterion in enumerate(criteria) + ] + result = { + "index": rubric_index, + "rubric": rubric_text, + "weight": weight, + "result": { + "passed": False, + "score": 0, + "criteria_count": criterion_count, + "criteria_passed": 0, + "criteria_results": default_criteria, + "overall_reasoning": f"Failed after {max_retries} attempts: {last_error}", + }, + } + debug = { + "api_base": api_base, + "parse_status": parse_status, + "api_exit_code": 1 if raw_response else 2, + "criterion_count": criterion_count, + "criteria_list_text": criteria_list_text, + "rubric_text": rubric_text, + "raw_response": raw_response, + "error": str(last_error) if last_error is not None else "", + "vision_used": vision_used, + "attached_images": [name for name, _ in attached_images], + } + return result, debug + + +def write_detail_log( + detail_log_dir: Path | None, + detail_log_prefix: str, + rubric_index: int, + judge_model: str, + debug: dict, + final_result: dict, +) -> None: + if detail_log_dir is None or not detail_log_prefix: + return + + detail_log_dir.mkdir(parents=True, exist_ok=True) + detail_log_file = detail_log_dir / f"{detail_log_prefix}_rubric_{rubric_index}.log" + sections = [ + "========================================", + "Rubric Judge Detail Log", + "========================================", + f"Timestamp: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + f"Unique Key: {detail_log_prefix}", + f"Rubric Index: {rubric_index}", + f"Judge Model: {judge_model}", + f"API Base: {debug.get('api_base', '')}", + f"Parse Status: {debug.get('parse_status', 'failed')}", + f"API Exit Code: {debug.get('api_exit_code', '')}", + f"Criteria Count: {debug.get('criterion_count', '')}", + f"Vision Used: {debug.get('vision_used', False)}", + f"Attached Images: {', '.join(debug.get('attached_images', [])) or '(none)'}", + ] + if debug.get("error"): + sections.append(f"Error: {debug['error']}") + + sections.extend( + [ + "", + "========================================", + "RUBRIC TEXT", + "========================================", + debug.get("rubric_text", ""), + "", + "========================================", + "CRITERIA", + "========================================", + debug.get("criteria_list_text", ""), + "", + "========================================", + "RAW API RESPONSE", + "========================================", + debug.get("raw_response", ""), + "", + "========================================", + "FINAL RESULT", + "========================================", + json.dumps(final_result, ensure_ascii=False, indent=2), + "", + "======================================== END ========================================", + "", + ] + ) + detail_log_file.write_text("\n".join(sections), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="JobBench LLM judge") + parser.add_argument("--output-dir", required=True, help="Directory with model output files") + parser.add_argument("--rubrics-file", required=True, help="Path to RUBRICS.json") + parser.add_argument("--result-file", default=None, help="Optional path for reward JSON") + parser.add_argument("--details-file", default=None, help="Where to write detailed results JSON") + parser.add_argument("--judge-model", default=os.environ.get("JUDGE_MODEL", "")) + parser.add_argument("--api-base", default=None, help="OpenAI-compatible API base URL") + parser.add_argument("--api-key", default=None, help="OpenAI-compatible API key") + parser.add_argument("--max-workers", type=int, default=10) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument("--timeout-per-rubric", type=int, default=300) + parser.add_argument("--evaluated-model", default="", help="Name of the model output being judged") + parser.add_argument("--lock-file", default=None, help="Optional lock file path for incremental details writes") + parser.add_argument("--detail-log-dir", default=None, help="Optional directory for per-rubric detail logs") + parser.add_argument("--detail-log-prefix", default="", help="Prefix for per-rubric detail logs") + args = parser.parse_args() + + output_dir = Path(args.output_dir) + rubrics_file = Path(args.rubrics_file) + result_file = Path(args.result_file) if args.result_file else None + details_file = Path(args.details_file) if args.details_file else None + lock_file = Path(args.lock_file) if args.lock_file else None + detail_log_dir = Path(args.detail_log_dir) if args.detail_log_dir else None + if lock_file is None and details_file is not None: + lock_file = details_file.with_name(f".{details_file.stem}.lock") + + try: + if not rubrics_file.exists(): + write_outputs( + result_file, + build_reward(build_scorecard([])), + details_file, + {"error": "rubrics not found", "rubrics_file": str(rubrics_file)}, + ) + return + + rubrics_data = json.loads(rubrics_file.read_text(encoding="utf-8")) + rubrics = rubrics_data.get("rubrics") or rubrics_data.get("evaluation_rubrics") or [] + if not rubrics: + write_outputs( + result_file, + build_reward(build_scorecard([])), + details_file, + {"error": "no rubrics", "rubrics_file": str(rubrics_file)}, + ) + return + + total_rubric_count = len(rubrics) + evaluated_model = args.evaluated_model or output_dir.name + has_output_files = output_dir.exists() and any(path.is_file() for path in output_dir.rglob("*")) + if not has_output_files: + results = [ + build_failed_rubric_result(idx, rubric, "No output files found in the model output directory.") + for idx, rubric in enumerate(rubrics) + ] + else: + file_contents = extract_all_file_contents(output_dir) + if not file_contents.strip(): + results = [ + build_failed_rubric_result(idx, rubric, "Output files were unreadable or empty after conversion.") + for idx, rubric in enumerate(rubrics) + ] + else: + existing_results = load_existing_rubric_results(details_file) + api_base, api_key = resolve_api_config(args.judge_model, args.api_base, args.api_key) + image_attachments = collect_image_attachments(output_dir) + + results: list[dict | None] = [existing_results.get(idx) for idx in range(len(rubrics))] + with ThreadPoolExecutor(max_workers=args.max_workers) as executor: + futures = { + executor.submit( + judge_rubric, + idx, + rubric, + file_contents, + args.judge_model, + api_base, + api_key, + args.timeout_per_rubric, + args.max_retries, + image_attachments, + ): idx + for idx, rubric in enumerate(rubrics) + if results[idx] is None + } + for future in as_completed(futures): + idx = futures[future] + try: + results[idx], debug = future.result() + except Exception as exc: + results[idx] = build_failed_rubric_result( + idx, + rubrics[idx], + f"Judge raised an exception: {exc}", + ) + debug = { + "api_base": api_base, + "parse_status": "failed", + "api_exit_code": 2, + "criterion_count": len(normalize_criteria(rubrics[idx])), + "criteria_list_text": "\n".join( + f"Criterion {criterion_idx}: {criterion}" + for criterion_idx, criterion in enumerate(normalize_criteria(rubrics[idx])) + ), + "rubric_text": rubrics[idx].get("rubric", ""), + "raw_response": "", + "error": str(exc), + "vision_used": False, + "attached_images": [], + } + if details_file is not None and lock_file is not None and results[idx] is not None: + append_rubric_result( + details_file, + lock_file, + evaluated_model, + args.judge_model, + total_rubric_count, + results[idx], + ) + write_detail_log( + detail_log_dir, + args.detail_log_prefix, + idx, + args.judge_model, + debug, + results[idx], + ) + + results = [result for result in results if result is not None] + + scorecard = build_scorecard(results) + reward = build_reward(scorecard) + details = build_details_report( + evaluated_model, + args.judge_model, + results, + total_count=total_rubric_count, + ) + write_outputs(result_file, reward, details_file, details) + except Exception as exc: + fallback_reward = build_reward(build_scorecard([])) + write_outputs( + result_file, + fallback_reward, + details_file, + { + "error": str(exc), + "traceback": traceback.format_exc(), + "evaluated_model": args.evaluated_model or output_dir.name, + "judge_model": args.judge_model, + }, + ) + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/opencode_sandboxed_agent/app.py b/responses_api_agents/opencode_sandboxed_agent/app.py index df98760df3..551897ca10 100644 --- a/responses_api_agents/opencode_sandboxed_agent/app.py +++ b/responses_api_agents/opencode_sandboxed_agent/app.py @@ -391,6 +391,7 @@ class OpenCodeSandboxedAgentConfig(BaseResponsesAPIAgentConfig): remote_opencode_musl_binary_path: Optional[str] = None opencode_config: Dict[str, Any] = Field(default_factory=dict) opencode_max_context_window: int + opencode_max_output_tokens: int = 1_000_000_000 # Sandbox config sandbox_provider: str @@ -557,7 +558,6 @@ async def _create_opencode_config(self, request: Request) -> Dict[str, Any]: "limit": { "context": self.config.opencode_max_context_window, "input": self.config.opencode_max_context_window, - # See the OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX flag below for more information. "output": self.config.opencode_max_context_window, }, }, @@ -683,6 +683,11 @@ async def responses( f"bash {quote(self.config.remote_opencode_install_script_path)} " f"--binary {quote(self.config.remote_opencode_binary_path)}" ) + elif self.config.remote_opencode_binary_path: + install_str = ( + "mkdir -p $HOME/.opencode/bin && " + f"cp {quote(self.config.remote_opencode_binary_path)} $HOME/.opencode/bin/opencode" + ) else: print( "Downloading and installing OpenCode in the sandbox. Please consider mounting or uploading the appropriate OpenCode binary instead!", @@ -702,17 +707,12 @@ async def responses( remote_data_home = f"/tmp/nemo-gym-opencode-{uuid4().hex}" xdg_home_str = f"XDG_DATA_HOME={remote_data_home}" - # @bxyu-nvidia: Regarding `OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX=1000000000` below: - # OpenCode defaults to 32k here https://github.com/anomalyco/opencode/blob/58a99916bb96edf5cf605dc03e1be1e4bacf9ff7/packages/opencode/src/provider/transform.ts#L21 - # and there is no way to set it to null. - # Here, we set an exorbitantly high number that cannot ever be reached. - # In future versions of OpenCode, this can be directly passed via maxOutputTokens in the limit config above https://github.com/anomalyco/opencode/blob/1b18a50418f730aca32630ccfcde850f2b5fc360/packages/opencode/src/provider/transform.ts#L1418 command = f""" echo "Shell: $SHELL" \ && {install_str} \ && export PATH=$HOME/.opencode/bin:$PATH \ && echo "Installed OpenCode" \ - && OPENCODE_CONFIG_CONTENT={quote(opencode_config_content)} OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX=1000000000 {xdg_home_str} \ + && OPENCODE_CONFIG_CONTENT={quote(opencode_config_content)} OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX={self.config.opencode_max_output_tokens} {xdg_home_str} \ opencode run --title "NG dummy title" {opencode_debug_str} {opencode_thinking_str} -- {quote(query)} \ && echo "OpenCode run finished" """ diff --git a/responses_api_agents/opencode_sandboxed_agent/tests/test_app.py b/responses_api_agents/opencode_sandboxed_agent/tests/test_app.py index c3fdd5c6bb..0d8231e02c 100644 --- a/responses_api_agents/opencode_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/opencode_sandboxed_agent/tests/test_app.py @@ -184,6 +184,7 @@ def test_opencode_export_to_usages(self, opencode_export_test_data: Dict[str, An async def test_responses_sanity(self, opencode_export_test_data: Dict[str, Any], monkeypatch: MonkeyPatch) -> None: config = self._create_config() + config.opencode_max_output_tokens = 131072 server = OpenCodeSandboxedAgent(config=config, server_client=MagicMock(spec=ServerClient)) sandbox_mock = MagicMock() @@ -224,6 +225,9 @@ async def test_responses_sanity(self, opencode_export_test_data: Dict[str, Any], input=[{"role": "user", "content": "hello"}], ), ) + assert ( + "OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX=131072" in sandbox_mock.exec.await_args_list[0].kwargs["command"] + ) expected_response = NeMoGymResponse( id="resp_", created_at=0.0,