webvoyager runner and viewer (#73)

Author: anerli

bun.lock

@@ -0,0 +1,57 @@
+[
+    "Allrecipes--16",
+    "Allrecipes--19",
+    "Allrecipes--23",
+    "Allrecipes--3",
+    "Allrecipes--30",
+    "Allrecipes--7",
+    "Amazon--16",
+    "Amazon--19",
+    "Amazon--4",
+    "Apple--1",
+    "Apple--14",
+    "Apple--16",
+    "Apple--2",
+    "Apple--20",
+    "Apple--37",
+    "Apple--41",
+    "Apple--42",
+    "Apple--7",
+    "Apple--9",
+    "ArXiv--11",
+    "BBC News--14",
+    "BBC News--16",
+    "BBC News--18",
+    "BBC News--2",
+    "BBC News--21",
+    "BBC News--33",
+    "BBC News--37",
+    "Booking--11",
+    "Booking--13",
+    "Booking--14",
+    "Booking--6",
+    "Coursera--17",
+    "Coursera--28",
+    "ESPN--19",
+    "ESPN--2",
+    "ESPN--21",
+    "ESPN--26",
+    "GitHub--22",
+    "Google Flights--0",
+    "Google Flights--20",
+    "Google Flights--7",
+    "Google Map--13",
+    "Google Map--18",
+    "Google Map--26",
+    "Google Search--15",
+    "Google Search--16",
+    "Google Search--22",
+    "Huggingface--1",
+    "Huggingface--10",
+    "Huggingface--20",
+    "Huggingface--21",
+    "Huggingface--22",
+    "Huggingface--23",
+    "Huggingface--32",
+    "Huggingface--6"
+]

evals/webvoyager/data/buImpossible.json

@@ -0,0 +1,21 @@
+Given these web tasks that are evaluated as part of a benchmark for web LLM agents, consider any patches that need to be applied in order to adjust for recency.
+
+Keep in mind that the original benchmark was created March 2, 2024.
+
+Today is June 26, 2025.
+
+For tasks where dates in the task are affected by today's date, or require a future date, they should be adjusted so that they are affectively similar in difficulty/feasibility as if today was March 2, 2024 in the original task set.
+
+Any product information or things that have changed over time should be appropriately adjusted to have modern equivalents.
+
+Do not patch any wording or phrasing unless its directly for the purpose of adjusting for dates/time/temporal feasibility.
+
+Consider each task carefully, and output any required patches in this format:
+
+{
+    "task--id": {
+        "reason": "justification for making the patch",
+        "prev": "full original task",
+        "new": "full adjusted task",
+    }
+}

evals/webvoyager/data/createPatches.md

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import json
+
+def filter_possible_tasks(original_file, impossible_file, output_file):
+    # Load impossible task IDs
+    with open(impossible_file, 'r') as f:
+        impossible_ids = set(json.load(f))
+    
+    print(f"Loaded {len(impossible_ids)} impossible task IDs")
+    
+    # Process original tasks and filter out impossible ones
+    possible_count = 0
+    total_count = 0
+    
+    with open(original_file, 'r') as infile, open(output_file, 'w') as outfile:
+        for line in infile:
+            total_count += 1
+            task = json.loads(line.strip())
+            
+            # Check if this task ID is in the impossible list
+            if task['id'] not in impossible_ids:
+                json.dump(task, outfile)
+                outfile.write('\n')
+                possible_count += 1
+    
+    print(f"Processed {total_count} total tasks")
+    print(f"Filtered out {len(impossible_ids)} impossible tasks")
+    print(f"Wrote {possible_count} possible tasks to {output_file}")
+    
+    return possible_count, total_count
+
+if __name__ == "__main__":
+    original_file = "originalTasks.jsonl"
+    impossible_file = "impossibleTasks.json"
+    output_file = "possibleTasks.jsonl"
+    
+    possible, total = filter_possible_tasks(original_file, impossible_file, output_file)