|
42 | 42 | }, |
43 | 43 | { |
44 | 44 | "cell_type": "code", |
45 | | - "execution_count": 1, |
| 45 | + "execution_count": null, |
46 | 46 | "metadata": { |
47 | 47 | "execution": { |
48 | 48 | "iopub.execute_input": "2025-12-09T17:33:01.040097Z", |
|
51 | 51 | "shell.execute_reply": "2025-12-09T17:33:01.060081Z" |
52 | 52 | } |
53 | 53 | }, |
54 | | - "outputs": [ |
55 | | - { |
56 | | - "name": "stdout", |
57 | | - "output_type": "stream", |
58 | | - "text": [ |
59 | | - "Working directory: /Users/jtr4v/PythonProject/mcp_literature_eval\n" |
60 | | - ] |
61 | | - } |
62 | | - ], |
| 54 | + "outputs": [], |
63 | 55 | "source": [ |
64 | 56 | "import subprocess\n", |
65 | 57 | "import os\n", |
66 | 58 | "from pathlib import Path\n", |
67 | 59 | "from datetime import datetime\n", |
68 | | - "import yaml\n", |
69 | 60 | "import shutil\n", |
70 | 61 | "\n", |
71 | 62 | "# Set working directory to project root\n", |
|
117 | 108 | " print(\"✓ Verified test_cases.yaml is NOT accessible\")\n", |
118 | 109 | " \n", |
119 | 110 | " # Set output file (in original project directory)\n", |
120 | | - " output_file = project_root / f\"results/compare_agents/{agent}_gpt5_isolated_{datetime.now().strftime('%Y%m%d')}.yaml\"\n", |
| 111 | + " output_file = project_root / f\"results/compare_agents/{agent}_mcp_only_{datetime.now().strftime('%Y%m%d')}.yaml\"\n", |
121 | 112 | " \n", |
122 | | - " print(f\"=== Running evaluation ===\")\n", |
| 113 | + " print(\"=== Running evaluation ===\")\n", |
123 | 114 | " print(f\"Agent: {agent}\")\n", |
124 | 115 | " print(f\"Config: {config_basename}\")\n", |
125 | 116 | " print(f\"Run dir: {isolated_dir}\")\n", |
|
129 | 120 | " \n", |
130 | 121 | " # Set environment variables\n", |
131 | 122 | " env = os.environ.copy()\n", |
132 | | - " env[\"OPENAI_API_KEY\"] = open(Path.home() / \"openai.key\").read().strip()\n", |
| 123 | + " env[\"OPENAI_API_KEY\"] = open(Path.home() / \"openai.key.another\").read().strip()\n", |
133 | 124 | " env[\"PUBMED_EMAIL\"] = \"justinreese@lbl.gov\"\n", |
134 | 125 | " env[\"PUBMED_API_KEY\"] = \"01eec0a16472164c6d69163bd28368311808\"\n", |
135 | 126 | " \n", |
|
143 | 134 | " \n", |
144 | 135 | " # Run evaluation from isolated directory with isolated workdir\n", |
145 | 136 | " # CRITICAL: Both CWD and --workdir must be in /tmp to fully isolate the agent\n", |
| 137 | + " # NOTE: Codex config now has disable_shell_tool: true to enforce MCP-only mode\n", |
146 | 138 | " cmd = [\n", |
147 | 139 | " str(venv_python), \"-m\", \"metacoder.metacoder\", \"eval\",\n", |
148 | 140 | " str(isolated_config),\n", |
|
151 | 143 | " ]\n", |
152 | 144 | " \n", |
153 | 145 | " if background:\n", |
154 | | - " print(f\"🚀 Starting evaluation in background...\")\n", |
| 146 | + " print(\"🚀 Starting evaluation in background...\")\n", |
155 | 147 | " process = subprocess.Popen(\n", |
156 | 148 | " cmd,\n", |
157 | 149 | " cwd=isolated_dir, # Run FROM /tmp\n", |
|
162 | 154 | " )\n", |
163 | 155 | " return isolated_dir, output_file, process\n", |
164 | 156 | " else:\n", |
165 | | - " print(f\"🚀 Starting evaluation (this will take 2-3 hours)...\")\n", |
| 157 | + " print(\"🚀 Starting evaluation (this will take 2-3 hours)...\")\n", |
166 | 158 | " result = subprocess.run(\n", |
167 | 159 | " cmd,\n", |
168 | 160 | " cwd=isolated_dir, # Run FROM /tmp\n", |
|
176 | 168 | " print(f\"STDERR: {result.stderr}\")\n", |
177 | 169 | " raise RuntimeError(f\"Evaluation failed: {result.stderr}\")\n", |
178 | 170 | " \n", |
179 | | - " print(f\"✅ Evaluation complete!\")\n", |
| 171 | + " print(\"✅ Evaluation complete!\")\n", |
180 | 172 | " print(f\"Output saved to: {output_file}\")\n", |
181 | 173 | " \n", |
182 | 174 | " return isolated_dir, output_file, None" |
|
196 | 188 | ] |
197 | 189 | }, |
198 | 190 | { |
199 | | - "cell_type": "markdown", |
| 191 | + "cell_type": "code", |
| 192 | + "execution_count": null, |
200 | 193 | "metadata": {}, |
| 194 | + "outputs": [], |
201 | 195 | "source": [ |
202 | | - "# Run Codex evaluation in isolated directory\n", |
| 196 | + "# Run Codex evaluation in isolated directory with shell_tool disabled (MCP-only mode)\n", |
203 | 197 | "isolated_dir, output_file, _ = run_isolated_eval(\n", |
204 | 198 | " \"codex\",\n", |
205 | 199 | " \"project/generated/literature_mcp_eval_config_codex.yaml\",\n", |
|
265 | 259 | " Returns:\n", |
266 | 260 | " True if no project access detected, False otherwise\n", |
267 | 261 | " \"\"\"\n", |
268 | | - " print(f\"=\" * 80)\n", |
| 262 | + " print(\"=\" * 80)\n", |
269 | 263 | " print(f\"Verifying isolation for: {result_file}\")\n", |
270 | | - " print(f\"=\" * 80)\n", |
| 264 | + " print(\"=\" * 80)\n", |
271 | 265 | " \n", |
272 | 266 | " result_path = Path(result_file)\n", |
273 | 267 | " if not result_path.exists():\n", |
|
295 | 289 | " violations.append((pattern, count))\n", |
296 | 290 | " \n", |
297 | 291 | " if violations:\n", |
298 | | - " print(f\"\\n❌ PROJECT ACCESS DETECTED!\")\n", |
| 292 | + " print(\"\\n❌ PROJECT ACCESS DETECTED!\")\n", |
299 | 293 | " print(f\"\\nFound {len(violations)} suspicious patterns:\")\n", |
300 | 294 | " for pattern, count in violations:\n", |
301 | 295 | " print(f\" - '{pattern}': {count} occurrences\")\n", |
302 | 296 | " \n", |
303 | | - " print(f\"\\n⚠️ The agent accessed files in the project directory!\")\n", |
304 | | - " print(f\"This means the isolation was NOT effective.\")\n", |
| 297 | + " print(\"\\n⚠️ The agent accessed files in the project directory!\")\n", |
| 298 | + " print(\"This means the isolation was NOT effective.\")\n", |
305 | 299 | " return False\n", |
306 | 300 | " else:\n", |
307 | | - " print(f\"\\n✅ NO PROJECT ACCESS DETECTED\")\n", |
308 | | - " print(f\"The agent did NOT access any files in:\")\n", |
309 | | - " print(f\" - project/\")\n", |
310 | | - " print(f\" - results/\")\n", |
311 | | - " print(f\" - notebook/\")\n", |
312 | | - " print(f\"\\n✓ Isolation was effective!\")\n", |
| 301 | + " print(\"\\n✅ NO PROJECT ACCESS DETECTED\")\n", |
| 302 | + " print(\"The agent did NOT access any files in:\")\n", |
| 303 | + " print(\" - project/\")\n", |
| 304 | + " print(\" - results/\")\n", |
| 305 | + " print(\" - notebook/\")\n", |
| 306 | + " print(\"\\n✓ Isolation was effective!\")\n", |
313 | 307 | " return True\n", |
314 | 308 | "\n", |
315 | 309 | "# Example usage after evaluation completes:\n", |
|
0 commit comments