feat: jobbench environment

benchmarks/job_bench/README.md

@@ -0,0 +1,24 @@
+# Job-Bench
+
+Runs the official Job-Bench `main` split with OpenCode 1.14.18 and its weighted-rubric evaluator. Grok 4.3 is the
+default judge.
+
+```bash
+uv run gym eval prepare --benchmark job_bench
+
+export NVIDIA_API_KEY=...
+export XAI_API_KEY=...
+export RAY_ENABLE_UV_RUN_RUNTIME_ENV=0
+uv run gym eval run \
+  --benchmark job_bench \
+  --model-type inference_provider \
+  --model-url https://inference-api.nvidia.com/v1 \
+  --model nvidia/moonshotai/kimi-k3 \
+  --model-api-key "$NVIDIA_API_KEY" \
+  --split benchmark \
+  --output results/job_bench_kimi_k3.jsonl \
+  +default_host="$ROUTABLE_HOST_IP"
+```
+
+Set `OPENSANDBOX_DOMAIN`, `OPENSANDBOX_API_KEY`, and a routable `ROUTABLE_HOST_IP`. Use `JOB_BENCH_JUDGE_*` to
+override the judge. Set `JOB_BENCH_SPLIT=easy` for the smaller non-leaderboard split.

benchmarks/job_bench/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

benchmarks/job_bench/config.yaml

@@ -0,0 +1,36 @@
+config_paths:
+  - responses_api_agents/opencode_sandboxed_agent/configs/opencode_sandboxed_agent.yaml
+  - resources_servers/job_bench/configs/job_bench.yaml
+  - nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml
+
+policy_model:
+  responses_api_models:
+    inference_provider:
+      uses_reasoning_parser: true
+
+job_bench_opencode_agent:
+  _inherit_from: opencode_sandboxed_agent
+  responses_api_agents:
+    opencode_sandboxed_agent:
+      opencode_version: 1.14.18
+      opencode_max_output_tokens: 131072
+      resources_server:
+        type: resources_servers
+        name: job_bench_resources_server
+      opencode_max_context_window: 1000000
+      sandbox_timeout: 7200
+      opencode_config:
+        permission:
+          external_directory:
+            "*": deny
+            /workspace: allow
+            /workspace/**: allow
+        tools:
+          webfetch: true
+          websearch: true
+      datasets:
+        - name: job_bench
+          type: benchmark
+          jsonl_fpath: benchmarks/job_bench/data/job_bench.jsonl
+          prepare_script: benchmarks/job_bench/prepare.py
+          num_repeats: 1

benchmarks/job_bench/data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

benchmarks/job_bench/prepare.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+from pathlib import Path
+
+
+DATA_DIR = Path(__file__).parent / "data"
+OUTPUT_FPATH = DATA_DIR / "job_bench.jsonl"
+
+
+def prepare() -> Path:
+    from huggingface_hub import snapshot_download
+
+    split = os.environ.get("JOB_BENCH_SPLIT", "main")
+    source_dir = "dataset" if split == "main" else "dataset_easy"
+    root = (
+        Path(
+            snapshot_download(
+                "JobBench/job-bench",
+                repo_type="dataset",
+                allow_patterns=f"{source_dir}/**",
+            )
+        )
+        / source_dir
+    )
+    tasks = sorted(root.glob("*/task[0-9]*"))
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    with OUTPUT_FPATH.open("w", encoding="utf-8") as output:
+        for task in tasks:
+            task_id = f"{task.parent.name}/{task.name}"
+            prompt = """=== TASK FOLDER ===
+/workspace/task
+
+=== INSTRUCTIONS ===
+1. Read TASK_INSTRUCTIONS.txt in the task folder
+2. Read the files named in its Reference Files section
+3. Complete the task as specified
+4. Save only final deliverables in the output directory
+
+=== OUTPUT DIRECTORY ===
+/workspace/output
+
+All reference files are in /workspace/task. Only access /workspace or search online for needed references.
+If information conflicts, explain and justify the chosen approach. Use appropriate tools to read office files."""
+            output.write(
+                json.dumps(
+                    {
+                        "responses_create_params": {"input": [{"role": "user", "content": prompt}]},
+                        "task_id": task_id,
+                        "task_dir": str(task),
+                        "rubrics_file": str(task / "RUBRICS.json"),
+                    }
+                )
+                + "\n"
+            )
+    print(f"Wrote {len(tasks)} {split} tasks to {OUTPUT_FPATH}")
+    return OUTPUT_FPATH
+
+
+if __name__ == "__main__":
+    prepare()

resources_servers/job_bench/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

resources_servers/job_bench/app.py

@@ -0,0 +1,190 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+import tarfile
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+from fastapi import Request
+from pydantic import ConfigDict
+
+from nemo_gym.base_resources_server import (
+    BaseResourcesServerConfig,
+    BaseSeedSessionRequest,
+    BaseSeedSessionResponse,
+    BaseVerifyRequest,
+    BaseVerifyResponse,
+    SimpleResourcesServer,
+)
+from nemo_gym.global_config import get_global_config_dict
+from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec, create_provider
+from nemo_gym.sandbox.config import resolve_provider_config, resolve_provider_metadata
+from nemo_gym.server_utils import SESSION_ID_KEY, is_nemo_gym_fastapi_entrypoint
+from resources_servers.job_bench.vendor import judge
+
+
+class JobBenchConfig(BaseResourcesServerConfig):
+    judge_base_url: str
+    judge_api_key: str
+    judge_model: str
+    max_judge_workers: int = 10
+    sandbox_provider: str
+    sandbox_config: dict[str, Any]
+
+
+class JobBenchRequest(BaseSeedSessionRequest):
+    model_config = ConfigDict(extra="allow")
+    task_id: str
+    task_dir: str
+    rubrics_file: str
+
+
+class JobBenchSeedResponse(BaseSeedSessionResponse):
+    sandbox_handle: str
+
+
+class JobBenchVerifyRequest(BaseVerifyRequest):
+    model_config = ConfigDict(extra="allow")
+    task_id: str
+    task_dir: str
+    rubrics_file: str
+
+
+class JobBenchVerifyResponse(BaseVerifyResponse):
+    model_config = ConfigDict(extra="allow")
+    task_id: str
+    score: float
+    max_score: float
+    passed_count: int
+    total_count: int
+    judge_model: str
+    rubrics: list[dict[str, Any]]
+
+
+class JobBenchResourcesServer(SimpleResourcesServer):
+    config: JobBenchConfig
+
+    def model_post_init(self, context: Any, /) -> None:
+        self._sandboxes: dict[str, AsyncSandbox] = {}
+
+    async def seed_session(self, request: Request, body: JobBenchRequest) -> JobBenchSeedResponse:
+        task_dir = Path(body.task_dir)
+        if not (task_dir / "task_folder" / "TASK_INSTRUCTIONS.txt").is_file():
+            raise ValueError(f"Invalid Job-Bench task directory: {task_dir}")
+
+        global_config = get_global_config_dict()
+        provider = create_provider(resolve_provider_config(self.config.sandbox_provider, global_config))
+        sandbox = AsyncSandbox(provider)
+        resources = SandboxResources.from_mapping(self.config.sandbox_config.get("resources", {}))
+        spec = SandboxSpec(
+            image=self.config.sandbox_config["image"],
+            ttl_s=self.config.sandbox_config.get("ttl_s"),
+            ready_timeout_s=self.config.sandbox_config.get("ready_timeout_s"),
+            workdir="/workspace",
+            env={},
+            files={},
+            metadata={
+                **resolve_provider_metadata(self.config.sandbox_provider, global_config),
+                **self.config.sandbox_config.get("metadata", {}),
+                "task_id": body.task_id[:63],
+            },
+            resources=resources,
+            entrypoint=None,
+            provider_options=self.config.sandbox_config.get("provider_options", {}),
+        )
+        await sandbox.start(spec)
+
+        with tempfile.TemporaryDirectory() as temporary_dir:
+            archive = Path(temporary_dir) / "task.tar.gz"
+            with tarfile.open(archive, "w:gz", dereference=True) as tar:
+                tar.add(task_dir / "task_folder", arcname="task")
+            await sandbox.upload(archive, "/tmp/task.tar.gz")
+        result = await sandbox.exec(
+            "mkdir -p /workspace/output && tar -xzf /tmp/task.tar.gz -C /workspace",
+            cwd="/",
+        )
+        if result.return_code != 0:
+            await sandbox.stop()
+            raise RuntimeError(f"Failed to seed Job-Bench task: {result.stderr}")
+
+        session_id = request.session[SESSION_ID_KEY]
+        self._sandboxes[session_id] = sandbox
+        return JobBenchSeedResponse(sandbox_handle=sandbox._handle.sandbox_id)
+
+    def _judge(self, output_dir: Path, rubrics_file: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+        rubrics_data = json.loads(rubrics_file.read_text(encoding="utf-8"))
+        rubrics = rubrics_data.get("rubrics") or rubrics_data.get("evaluation_rubrics") or []
+        if not any(path.is_file() for path in output_dir.rglob("*")):
+            results = [
+                judge.build_failed_rubric_result(index, rubric, "No output files found in the model output directory.")
+                for index, rubric in enumerate(rubrics)
+            ]
+            return judge.build_scorecard(results), results
+        file_contents = judge.extract_all_file_contents(output_dir)
+        if not file_contents.strip():
+            results = [
+                judge.build_failed_rubric_result(
+                    index, rubric, "Output files were unreadable or empty after conversion."
+                )
+                for index, rubric in enumerate(rubrics)
+            ]
+            return judge.build_scorecard(results), results
+        images = judge.collect_image_attachments(output_dir)
+
+        with ThreadPoolExecutor(max_workers=self.config.max_judge_workers) as executor:
+            futures = [
+                executor.submit(
+                    judge.judge_rubric,
+                    index,
+                    rubric,
+                    file_contents,
+                    self.config.judge_model,
+                    self.config.judge_base_url,
+                    self.config.judge_api_key,
+                    300,
+                    3,
+                    images,
+                )
+                for index, rubric in enumerate(rubrics)
+            ]
+            results = [future.result()[0] for future in futures]
+        return judge.build_scorecard(results), results
+
+    async def verify(self, request: Request, body: JobBenchVerifyRequest) -> JobBenchVerifyResponse:
+        sandbox = self._sandboxes.pop(request.session[SESSION_ID_KEY])
+        try:
+            with tempfile.TemporaryDirectory() as temporary_dir:
+                local_dir = Path(temporary_dir)
+                archive = local_dir / "output.tar.gz"
+                result = await sandbox.exec("tar -czf /tmp/output.tar.gz -C /workspace/output .")
+                if result.return_code != 0:
+                    raise RuntimeError(f"Failed to collect Job-Bench output: {result.stderr}")
+                await sandbox.download("/tmp/output.tar.gz", archive)
+                output_dir = local_dir / "output"
+                output_dir.mkdir()
+                with tarfile.open(archive, "r:gz") as tar:
+                    tar.extractall(output_dir, filter="data")
+                scorecard, rubrics = await asyncio.to_thread(self._judge, output_dir, Path(body.rubrics_file))
+        finally:
+            await sandbox.stop()
+
+        return JobBenchVerifyResponse(
+            **body.model_dump(),
+            reward=float(scorecard["normalized_score"]),
+            score=float(scorecard["total_score"]),
+            max_score=float(scorecard["max_score"]),
+            passed_count=int(scorecard["passed_count"]),
+            total_count=int(scorecard["total_count"]),
+            judge_model=self.config.judge_model,
+            rubrics=rubrics,
+        )
+
+
+if __name__ == "__main__":
+    JobBenchResourcesServer.run_webserver()
+elif is_nemo_gym_fastapi_entrypoint(__file__):
+    app = JobBenchResourcesServer.run_webserver()  # noqa: F401

resources_servers/job_bench/configs/job_bench.yaml

@@ -0,0 +1,24 @@
+job_bench_resources_server:
+  resources_servers:
+    job_bench:
+      entrypoint: app.py
+      domain: other
+      verified: false
+      allowed_agents: [opencode_sandboxed_agent]
+      judge_base_url: ${oc.env:JOB_BENCH_JUDGE_BASE_URL,https://api.x.ai/v1}
+      judge_api_key: ${oc.env:JOB_BENCH_JUDGE_API_KEY,${oc.env:XAI_API_KEY,''}}
+      judge_model: ${oc.env:JOB_BENCH_JUDGE_MODEL,grok-4.3}
+      max_judge_workers: 10
+      sandbox_provider: sandbox
+      sandbox_config:
+        image: ${oc.env:JOB_BENCH_SANDBOX_IMAGE,python:3.13-bookworm}
+        ttl_s: 10800
+        ready_timeout_s: 1200
+        resources:
+          cpu: 1
+          memory_mib: 2048
+          disk_gib: 30
+        provider_options: {}
+        metadata:
+          benchmark: job-bench
+          harness: opencode

resources_servers/job_bench/requirements.txt

@@ -0,0 +1,9 @@
+-e nemo-gym[dev,sandbox] @ ../..
+huggingface-hub>=0.24.0
+mammoth>=1.8.0
+openai>=1.0.0
+openpyxl>=3.1.0
+pandas>=2.0.0
+pdfplumber>=0.11.0
+pyarrow>=15.0.0
+python-pptx>=0.6.0

resources_servers/job_bench/task_data.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, ConfigDict
+
+
+class TaskData(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    task_id: str
+    task_dir: str
+    rubrics_file: str

resources_servers/job_bench/tests/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0