ai-engineering-from-scratch/phases/19-capstone-projects/11-llm-observability-dashboard/quiz.json at main · Amjad-AbuRmileh/ai-engineering-from-scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
  "lesson": "11-llm-observability-dashboard",
  "title": "Capstone 11 — LLM Observability & Eval Dashboard",
  "questions": [
    {
      "stage": "pre",
      "question": "Which ingest schema do Langfuse, Phoenix, and OpenLLMetry converge on?",
      "options": [
        "Proprietary JSON per vendor",
        "Prometheus exposition format",
        "OpenTelemetry GenAI semantic conventions over OTLP HTTP",
        "Plain CSV log files"
      ],
      "correct": 2,
      "explanation": ""
    },
    {
      "stage": "pre",
      "question": "Why separate ClickHouse and Postgres in the storage tier?",
      "options": [
        "They are interchangeable and one is chosen at random",
        "Postgres is faster for span ingest",
        "ClickHouse cannot store strings",
        "ClickHouse handles columnar analytics over spans while Postgres holds users, sessions, and app metadata"
      ],
      "correct": 3,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What does the tail-sampling processor in the OpenTelemetry Collector do?",
      "options": [
        "Decides whether to keep a trace after it completes, using rules like keep errors plus sample successes",
        "Replays old traces into Postgres",
        "Streams every byte unconditionally",
        "Truncates spans below 100ms"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "How does the dashboard detect drift across weeks?",
      "options": [
        "Computes PSI or KL divergence on pooled prompt embeddings and watches eval-score trends",
        "Manual eyeballing of the dashboard",
        "Reads the latest deploy timestamp",
        "Counts unique trace IDs"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What is the deliverable's MTTR target on an injected PII-leak regression?",
      "options": [
        "Under 1 hour",
        "Within 24 hours",
        "Within the next on-call shift",
        "Under 5 minutes from bug deployed to Slack alert"
      ],
      "correct": 3,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "Which SDK families must produce canonical GenAI spans to meet the trace-coverage rubric?",
      "options": [
        "Only vLLM",
        "At least six: OpenAI, Anthropic, Google GenAI, LangChain, LlamaIndex, and vLLM",
        "Any one SDK is enough",
        "OpenAI and Anthropic only"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "How are evaluation results linked back to the original LLM call?",
      "options": [
        "As a CSV emailed nightly",
        "As Slack messages only",
        "As a separate Postgres table with no trace ID",
        "As eval spans written as children of the parent trace in ClickHouse"
      ],
      "correct": 3,
      "explanation": ""
    }
  ]
}