quiz.json

{
  "lesson": "34-transformer-block",
  "title": "Transformer Block from Scratch",
  "questions": [
    {
      "stage": "pre",
      "question": "Which two sublayers live inside a single decoder transformer block?",
      "options": [
        "Convolution and recurrent gate",
        "Multi head causal attention and a position wise MLP, each wrapped in a residual connection",
        "Embedding lookup and softmax",
        "RNN encoder and decoder"
      ],
      "correct": 1,
      "explanation": "Every modern decoder LLM block is attention plus MLP, each with its own residual path and its own LayerNorm."
    },
    {
      "stage": "check",
      "question": "What is the key difference between pre-LN and post-LN at depth?",
      "options": [
        "Pre-LN uses different attention math",
        "Pre-LN keeps the residual signal unnormalized so gradients propagate cleanly to the embedding; post-LN puts the norm after the residual add, which shrinks gradients at depth and needs warmup",
        "Post-LN is faster on GPU",
        "Pre-LN works only for encoders"
      ],
      "correct": 1,
      "explanation": "Pre-LN trains stably without warmup at common learning rates; post-LN is what the 2017 paper shipped and needed warmup."
    },
    {
      "stage": "check",
      "question": "What does the causal mask in multi head attention do?",
      "options": [
        "Drops random tokens",
        "Sets the upper triangle of the attention logits to negative infinity so token i cannot attend to tokens at positions greater than i",
        "Normalizes the attention scores",
        "Picks the longest sequence in the batch"
      ],
      "correct": 1,
      "explanation": "Forgetting the causal mask trains a model that can read future tokens; the mask is the only piece that enforces causality."
    },
    {
      "stage": "check",
      "question": "Why use a fused QKV projection instead of three separate linears?",
      "options": [
        "It changes the model expressivity",
        "One linear of width 3 times d_model collapses three kernel launches and three matmuls into one, faster on every accelerator and matches the reference GPT-2 implementation",
        "It avoids a softmax",
        "It removes the residual"
      ],
      "correct": 1,
      "explanation": "Same math, fewer kernels; reference implementations of GPT-2, LLaMA, and Mistral all ship the fused projection."
    },
    {
      "stage": "post",
      "question": "Where should dropout NOT be placed in a transformer block?",
      "options": [
        "On the attention softmax output",
        "On the residual identity path itself, since that breaks the additive identity that lets the gradient flow at depth",
        "After the second MLP linear",
        "Inside the feed forward expansion"
      ],
      "correct": 1,
      "explanation": "Dropout belongs on the attention output and on the MLP residual update, never on the identity branch of the residual."
    },
    {
      "stage": "post",
      "question": "Why register the causal mask as a buffer rather than allocating it per forward call?",
      "options": [
        "Buffers are encrypted",
        "The mask depends only on the maximum context length, so allocating it once at construction and slicing the active window per call avoids an allocator hot spot at long context",
        "Buffers run on GPU only",
        "Buffers improve numerical accuracy"
      ],
      "correct": 1,
      "explanation": "Allocate once with register_buffer, slice per forward; otherwise the mask becomes a per-call allocation cost."
    }
  ]
}