ai-engineering-from-scratch/phases/14-agent-engineering/19-benchmarks-swebench-gaia/quiz.json at main · Amjad-AbuRmileh/ai-engineering-from-scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
  "lesson": "19-benchmarks-swebench-gaia",
  "title": "Benchmarks: SWE-bench, GAIA, AgentBench",
  "questions": [
    {
      "stage": "pre",
      "question": "What does SWE-bench's evaluator check on a candidate patch?",
      "options": [
        "BLEU score against the reference fix",
        "Previously failing tests now pass (FAIL_TO_PASS) and previously passing tests still pass (PASS_TO_PASS)",
        "Patch length under 200 lines",
        "Patch passes a separate LLM judge"
      ],
      "correct": 1,
      "explanation": "The harness gates on test transitions: bug-revealing tests must flip while regression tests must stay green."
    },
    {
      "stage": "pre",
      "question": "Why does SWE-bench Verified exist?",
      "options": [
        "It runs faster",
        "OpenAI's 500-task human-curated subset removes ambiguous issues and unreliable tests",
        "It includes more languages",
        "It uses a different patch format"
      ],
      "correct": 1,
      "explanation": "Verified is the cleaner subset for credible reporting."
    },
    {
      "stage": "check",
      "question": "What did SWE-bench+ find about successful patches?",
      "options": [
        "32.67% leaked solution text in the issue and 31.08% had suspiciously weak test coverage",
        "All patches were memorized",
        "There is no contamination",
        "Patches always exceeded 1000 lines"
      ],
      "correct": 0,
      "explanation": "SWE-bench+ flagged solution leakage and weak coverage on a large fraction of successful patches."
    },
    {
      "stage": "check",
      "question": "What is GAIA's design philosophy?",
      "options": [
        "Hard for humans, easy for AI",
        "Conceptually simple for humans (about 92%) but hard for AI (early GPT-4 with plugins: about 15%)",
        "Pure benchmark of vector retrieval",
        "Only single-turn questions"
      ],
      "correct": 1,
      "explanation": "GAIA is intentionally easy-for-humans, hard-for-AI, testing reasoning + tools + modality."
    },
    {
      "stage": "check",
      "question": "Which is NOT one of AgentBench's environment categories?",
      "options": [
        "Code (Bash, DB, KG)",
        "Games (Alfworld, LTP)",
        "Web (WebShop, Mind2Web)",
        "Gradient (RL, IRL)"
      ],
      "correct": 3,
      "explanation": "AgentBench covers code, games, web, and open-ended generation. There is no gradient category."
    },
    {
      "stage": "post",
      "question": "What does the lesson identify as the wrong way to report SWE-bench numbers?",
      "options": [
        "Reporting per-repo breakdowns",
        "Reporting one aggregate number without mentioning Verified or SWE-bench+ context",
        "Reporting step counts",
        "Reporting wall-clock"
      ],
      "correct": 1,
      "explanation": "Single-number fixation hides contamination and cost; always report Verified and per-distribution context."
    },
    {
      "stage": "post",
      "question": "Which dimension do these benchmarks NOT measure?",
      "options": [
        "Real-world operational cost (tokens, wall-clock), adversarial safety, and your own domain",
        "Test transitions",
        "Per-task success",
        "Step counts"
      ],
      "correct": 0,
      "explanation": "Benchmarks aggregate; they do not capture cost, adversarial robustness, or your domain."
    }
  ]
}