Skip to content

Commit 2f95260

Browse files
Merge pull request #25 from contextualizer-ai/codex-evaluation-dec2024
Codex evaluation fixes, remove local file access, update results
2 parents 9cb5cfd + a5afc99 commit 2f95260

30 files changed

Lines changed: 499410 additions & 791380 deletions

.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,3 +149,8 @@ dmypy.json
149149
.vscode
150150
.claude
151151

152+
# Large model comparison results
153+
results/compare_models/
154+
155+
# Local scripts
156+
scripts/

notebook/experiment_1_cross_agent_analysis.ipynb

Lines changed: 92 additions & 111 deletions
Large diffs are not rendered by default.

notebook/experiment_1_failure_analysis.ipynb

Lines changed: 78 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,6 @@
3333
"import pandas as pd\n",
3434
"import matplotlib.pyplot as plt\n",
3535
"import seaborn as sns\n",
36-
"import numpy as np\n",
37-
"from pathlib import Path\n",
38-
"from collections import Counter\n",
39-
"import re\n",
4036
"\n",
4137
"# Set style\n",
4238
"sns.set_style(\"whitegrid\")\n",
@@ -909,29 +905,96 @@
909905
},
910906
{
911907
"cell_type": "code",
912-
"source": "# Identify potential FALSE NEGATIVES: Factually correct but failed due to verbosity/formatting\nprint(\"POTENTIAL FALSE NEGATIVES - Correct answers marked as failures\")\nprint(\"=\"*100)\n\nfalse_negatives = []\n\nfor idx, row in retraction_cases.iterrows():\n reason_lower = str(row['reason']).lower()\n \n # Check if evaluator says it's correct but it still failed\n if (not row['passed'] and \n ('correctly states' in reason_lower or \n ('does not contradict' in reason_lower and 'correct' in reason_lower) or\n 'no factual contradictions' in reason_lower)):\n \n false_negatives.append(row)\n print(f\"\\n{row['case_name']} - {row['agent']} + {row['MCP']}\")\n print(f\" Score: {row['score']:.3f} (FAILED but appears correct)\")\n print(f\" Reason excerpt: {row['reason'][:300]}...\")\n print(\"-\"*100)\n\nprint(f\"\\n\\n{'='*100}\")\nprint(f\"SUMMARY: {len(false_negatives)} potential false negatives out of {len(retraction_cases)} total retraction cases\")\nprint(f\"This represents {len(false_negatives)/len(retraction_cases)*100:.1f}% of retraction evaluations\")\nprint(f\"\\nCurrent retraction pass rate: {retraction_cases['passed'].mean()*100:.1f}%\")\nprint(f\"If false negatives were corrected: {(retraction_cases['passed'].sum() + len(false_negatives))/len(retraction_cases)*100:.1f}%\")",
913-
"metadata": {},
914908
"execution_count": null,
915-
"outputs": []
909+
"metadata": {},
910+
"outputs": [],
911+
"source": [
912+
"# Identify potential FALSE NEGATIVES: Factually correct but failed due to verbosity/formatting\n",
913+
"print(\"POTENTIAL FALSE NEGATIVES - Correct answers marked as failures\")\n",
914+
"print(\"=\"*100)\n",
915+
"\n",
916+
"false_negatives = []\n",
917+
"\n",
918+
"for idx, row in retraction_cases.iterrows():\n",
919+
" reason_lower = str(row['reason']).lower()\n",
920+
" \n",
921+
" # Check if evaluator says it's correct but it still failed\n",
922+
" if (not row['passed'] and \n",
923+
" ('correctly states' in reason_lower or \n",
924+
" ('does not contradict' in reason_lower and 'correct' in reason_lower) or\n",
925+
" 'no factual contradictions' in reason_lower)):\n",
926+
" \n",
927+
" false_negatives.append(row)\n",
928+
" print(f\"\\n{row['case_name']} - {row['agent']} + {row['MCP']}\")\n",
929+
" print(f\" Score: {row['score']:.3f} (FAILED but appears correct)\")\n",
930+
" print(f\" Reason excerpt: {row['reason'][:300]}...\")\n",
931+
" print(\"-\"*100)\n",
932+
"\n",
933+
"print(f\"\\n\\n{'='*100}\")\n",
934+
"print(f\"SUMMARY: {len(false_negatives)} potential false negatives out of {len(retraction_cases)} total retraction cases\")\n",
935+
"print(f\"This represents {len(false_negatives)/len(retraction_cases)*100:.1f}% of retraction evaluations\")\n",
936+
"print(f\"\\nCurrent retraction pass rate: {retraction_cases['passed'].mean()*100:.1f}%\")\n",
937+
"print(f\"If false negatives were corrected: {(retraction_cases['passed'].sum() + len(false_negatives))/len(retraction_cases)*100:.1f}%\")"
938+
]
916939
},
917940
{
918941
"cell_type": "code",
919-
"source": "# Detailed analysis of each retraction case\nprint(\"DETAILED RETRACTION CASE ANALYSIS\")\nprint(\"=\"*100)\n\nfor i, (idx, row) in enumerate(retraction_cases.iterrows(), 1):\n print(f\"\\n{i}. {row['case_name']} - Agent: {row['agent']}, MCP: {row['MCP']}\")\n print(f\" Score: {row['score']:.3f} | Passed: {row['passed']}\")\n print(f\" Reason: {row['reason'][:400]}\")\n \n # Check if the answer is factually correct based on the reason\n reason_lower = str(row['reason']).lower()\n output_lower = str(row['actual_output']).lower()\n \n # Look for indicators of correctness\n if 'correctly states' in reason_lower or 'correct' in reason_lower and 'not contradict' in reason_lower:\n factually_correct = \"✓ FACTUALLY CORRECT\"\n elif 'contradict' in reason_lower or 'incorrect' in reason_lower or 'wrong' in reason_lower:\n factually_correct = \"✗ FACTUALLY WRONG\"\n elif 'omit' in reason_lower or 'missing' in reason_lower or 'not provide' in reason_lower:\n factually_correct = \"⚠ INCOMPLETE/MISSING\"\n else:\n factually_correct = \"? UNCLEAR\"\n \n print(f\" Assessment: {factually_correct}\")\n \n # Show first part of actual output\n print(f\" Output preview: {row['actual_output'][:200]}...\")\n print(\"-\"*100)",
920-
"metadata": {},
921942
"execution_count": null,
922-
"outputs": []
943+
"metadata": {},
944+
"outputs": [],
945+
"source": [
946+
"# Detailed analysis of each retraction case\n",
947+
"print(\"DETAILED RETRACTION CASE ANALYSIS\")\n",
948+
"print(\"=\"*100)\n",
949+
"\n",
950+
"for i, (idx, row) in enumerate(retraction_cases.iterrows(), 1):\n",
951+
" print(f\"\\n{i}. {row['case_name']} - Agent: {row['agent']}, MCP: {row['MCP']}\")\n",
952+
" print(f\" Score: {row['score']:.3f} | Passed: {row['passed']}\")\n",
953+
" print(f\" Reason: {row['reason'][:400]}\")\n",
954+
" \n",
955+
" # Check if the answer is factually correct based on the reason\n",
956+
" reason_lower = str(row['reason']).lower()\n",
957+
" output_lower = str(row['actual_output']).lower()\n",
958+
" \n",
959+
" # Look for indicators of correctness\n",
960+
" if 'correctly states' in reason_lower or 'correct' in reason_lower and 'not contradict' in reason_lower:\n",
961+
" factually_correct = \"✓ FACTUALLY CORRECT\"\n",
962+
" elif 'contradict' in reason_lower or 'incorrect' in reason_lower or 'wrong' in reason_lower:\n",
963+
" factually_correct = \"✗ FACTUALLY WRONG\"\n",
964+
" elif 'omit' in reason_lower or 'missing' in reason_lower or 'not provide' in reason_lower:\n",
965+
" factually_correct = \"⚠ INCOMPLETE/MISSING\"\n",
966+
" else:\n",
967+
" factually_correct = \"? UNCLEAR\"\n",
968+
" \n",
969+
" print(f\" Assessment: {factually_correct}\")\n",
970+
" \n",
971+
" # Show first part of actual output\n",
972+
" print(f\" Output preview: {row['actual_output'][:200]}...\")\n",
973+
" print(\"-\"*100)"
974+
]
923975
},
924976
{
925977
"cell_type": "code",
926-
"source": "# Get all retraction cases\nretraction_cases = df_all[df_all['case_name'].str.contains('Retraction', case=False, na=False)].copy()\n\nprint(f\"Total retraction evaluations: {len(retraction_cases)}\")\nprint(f\"Passed: {retraction_cases['passed'].sum()}\")\nprint(f\"Failed: {(~retraction_cases['passed']).sum()}\")\nprint(f\"Pass rate: {retraction_cases['passed'].mean()*100:.1f}%\")\nprint(f\"\\nMean score: {retraction_cases['score'].mean():.3f}\")\nprint(f\"Score distribution:\")\nprint(retraction_cases['score'].describe())",
927-
"metadata": {},
928978
"execution_count": null,
929-
"outputs": []
979+
"metadata": {},
980+
"outputs": [],
981+
"source": [
982+
"# Get all retraction cases\n",
983+
"retraction_cases = df_all[df_all['case_name'].str.contains('Retraction', case=False, na=False)].copy()\n",
984+
"\n",
985+
"print(f\"Total retraction evaluations: {len(retraction_cases)}\")\n",
986+
"print(f\"Passed: {retraction_cases['passed'].sum()}\")\n",
987+
"print(f\"Failed: {(~retraction_cases['passed']).sum()}\")\n",
988+
"print(f\"Pass rate: {retraction_cases['passed'].mean()*100:.1f}%\")\n",
989+
"print(f\"\\nMean score: {retraction_cases['score'].mean():.3f}\")\n",
990+
"print(\"Score distribution:\")\n",
991+
"print(retraction_cases['score'].describe())"
992+
]
930993
},
931994
{
932995
"cell_type": "markdown",
933-
"source": "## CRITICAL: Retraction Eval Analysis - False Negatives from Verbosity?\n\nAre we marking correct answers as failures just because agents provided extra detail?",
934-
"metadata": {}
996+
"metadata": {},
997+
"source": "## CRITICAL: Retraction Eval Analysis - False Negatives from Verbosity?\n\nAre we marking correct answers as failures just because agents provided extra detail?"
935998
}
936999
],
9371000
"metadata": {

notebook/experiment_1_run_evaluations.ipynb

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
},
4343
{
4444
"cell_type": "code",
45-
"execution_count": 1,
45+
"execution_count": null,
4646
"metadata": {
4747
"execution": {
4848
"iopub.execute_input": "2025-12-09T17:33:01.040097Z",
@@ -51,21 +51,12 @@
5151
"shell.execute_reply": "2025-12-09T17:33:01.060081Z"
5252
}
5353
},
54-
"outputs": [
55-
{
56-
"name": "stdout",
57-
"output_type": "stream",
58-
"text": [
59-
"Working directory: /Users/jtr4v/PythonProject/mcp_literature_eval\n"
60-
]
61-
}
62-
],
54+
"outputs": [],
6355
"source": [
6456
"import subprocess\n",
6557
"import os\n",
6658
"from pathlib import Path\n",
6759
"from datetime import datetime\n",
68-
"import yaml\n",
6960
"import shutil\n",
7061
"\n",
7162
"# Set working directory to project root\n",
@@ -117,9 +108,9 @@
117108
" print(\"✓ Verified test_cases.yaml is NOT accessible\")\n",
118109
" \n",
119110
" # Set output file (in original project directory)\n",
120-
" output_file = project_root / f\"results/compare_agents/{agent}_gpt5_isolated_{datetime.now().strftime('%Y%m%d')}.yaml\"\n",
111+
" output_file = project_root / f\"results/compare_agents/{agent}_mcp_only_{datetime.now().strftime('%Y%m%d')}.yaml\"\n",
121112
" \n",
122-
" print(f\"=== Running evaluation ===\")\n",
113+
" print(\"=== Running evaluation ===\")\n",
123114
" print(f\"Agent: {agent}\")\n",
124115
" print(f\"Config: {config_basename}\")\n",
125116
" print(f\"Run dir: {isolated_dir}\")\n",
@@ -129,7 +120,7 @@
129120
" \n",
130121
" # Set environment variables\n",
131122
" env = os.environ.copy()\n",
132-
" env[\"OPENAI_API_KEY\"] = open(Path.home() / \"openai.key\").read().strip()\n",
123+
" env[\"OPENAI_API_KEY\"] = open(Path.home() / \"openai.key.another\").read().strip()\n",
133124
" env[\"PUBMED_EMAIL\"] = \"justinreese@lbl.gov\"\n",
134125
" env[\"PUBMED_API_KEY\"] = \"01eec0a16472164c6d69163bd28368311808\"\n",
135126
" \n",
@@ -143,6 +134,7 @@
143134
" \n",
144135
" # Run evaluation from isolated directory with isolated workdir\n",
145136
" # CRITICAL: Both CWD and --workdir must be in /tmp to fully isolate the agent\n",
137+
" # NOTE: Codex config now has disable_shell_tool: true to enforce MCP-only mode\n",
146138
" cmd = [\n",
147139
" str(venv_python), \"-m\", \"metacoder.metacoder\", \"eval\",\n",
148140
" str(isolated_config),\n",
@@ -151,7 +143,7 @@
151143
" ]\n",
152144
" \n",
153145
" if background:\n",
154-
" print(f\"🚀 Starting evaluation in background...\")\n",
146+
" print(\"🚀 Starting evaluation in background...\")\n",
155147
" process = subprocess.Popen(\n",
156148
" cmd,\n",
157149
" cwd=isolated_dir, # Run FROM /tmp\n",
@@ -162,7 +154,7 @@
162154
" )\n",
163155
" return isolated_dir, output_file, process\n",
164156
" else:\n",
165-
" print(f\"🚀 Starting evaluation (this will take 2-3 hours)...\")\n",
157+
" print(\"🚀 Starting evaluation (this will take 2-3 hours)...\")\n",
166158
" result = subprocess.run(\n",
167159
" cmd,\n",
168160
" cwd=isolated_dir, # Run FROM /tmp\n",
@@ -176,7 +168,7 @@
176168
" print(f\"STDERR: {result.stderr}\")\n",
177169
" raise RuntimeError(f\"Evaluation failed: {result.stderr}\")\n",
178170
" \n",
179-
" print(f\"✅ Evaluation complete!\")\n",
171+
" print(\"✅ Evaluation complete!\")\n",
180172
" print(f\"Output saved to: {output_file}\")\n",
181173
" \n",
182174
" return isolated_dir, output_file, None"
@@ -196,10 +188,12 @@
196188
]
197189
},
198190
{
199-
"cell_type": "markdown",
191+
"cell_type": "code",
192+
"execution_count": null,
200193
"metadata": {},
194+
"outputs": [],
201195
"source": [
202-
"# Run Codex evaluation in isolated directory\n",
196+
"# Run Codex evaluation in isolated directory with shell_tool disabled (MCP-only mode)\n",
203197
"isolated_dir, output_file, _ = run_isolated_eval(\n",
204198
" \"codex\",\n",
205199
" \"project/generated/literature_mcp_eval_config_codex.yaml\",\n",
@@ -265,9 +259,9 @@
265259
" Returns:\n",
266260
" True if no project access detected, False otherwise\n",
267261
" \"\"\"\n",
268-
" print(f\"=\" * 80)\n",
262+
" print(\"=\" * 80)\n",
269263
" print(f\"Verifying isolation for: {result_file}\")\n",
270-
" print(f\"=\" * 80)\n",
264+
" print(\"=\" * 80)\n",
271265
" \n",
272266
" result_path = Path(result_file)\n",
273267
" if not result_path.exists():\n",
@@ -295,21 +289,21 @@
295289
" violations.append((pattern, count))\n",
296290
" \n",
297291
" if violations:\n",
298-
" print(f\"\\n❌ PROJECT ACCESS DETECTED!\")\n",
292+
" print(\"\\n❌ PROJECT ACCESS DETECTED!\")\n",
299293
" print(f\"\\nFound {len(violations)} suspicious patterns:\")\n",
300294
" for pattern, count in violations:\n",
301295
" print(f\" - '{pattern}': {count} occurrences\")\n",
302296
" \n",
303-
" print(f\"\\n⚠️ The agent accessed files in the project directory!\")\n",
304-
" print(f\"This means the isolation was NOT effective.\")\n",
297+
" print(\"\\n⚠️ The agent accessed files in the project directory!\")\n",
298+
" print(\"This means the isolation was NOT effective.\")\n",
305299
" return False\n",
306300
" else:\n",
307-
" print(f\"\\n✅ NO PROJECT ACCESS DETECTED\")\n",
308-
" print(f\"The agent did NOT access any files in:\")\n",
309-
" print(f\" - project/\")\n",
310-
" print(f\" - results/\")\n",
311-
" print(f\" - notebook/\")\n",
312-
" print(f\"\\n✓ Isolation was effective!\")\n",
301+
" print(\"\\n✅ NO PROJECT ACCESS DETECTED\")\n",
302+
" print(\"The agent did NOT access any files in:\")\n",
303+
" print(\" - project/\")\n",
304+
" print(\" - results/\")\n",
305+
" print(\" - notebook/\")\n",
306+
" print(\"\\n✓ Isolation was effective!\")\n",
313307
" return True\n",
314308
"\n",
315309
"# Example usage after evaluation completes:\n",

notebook/experiment_2_cross_model_analysis.ipynb

Lines changed: 170 additions & 120 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)