quiz.json

{
  "lesson": "phase-19/39-instruction-tuning-sft",
  "title": "Instruction Tuning by Supervised Fine-Tuning",
  "questions": [
    {
      "stage": "pre",
      "question": "Before reading: you give a vanilla base model the prompt 'What is the capital of France?'. It outputs 'And what about Italy and Spain too?'. Which best diagnoses the gap?",
      "options": [
        "The model is missing the fact about Paris.",
        "The model has the language model behaviour but no instruction-following format contract.",
        "The model needs a larger vocabulary.",
        "The base model must be reset and pretrained again."
      ],
      "correct": 1,
      "explanation": "The base model can extend text but has never been shown that an instruction expects a corresponding response. SFT teaches the format contract: see instruction, emit response."
    },
    {
      "stage": "check",
      "question": "The collate function sets labels[i] = -100 on instruction positions and on padding positions. Why does PyTorch cross-entropy treat these positions correctly?",
      "options": [
        "Cross-entropy silently drops any negative target.",
        "ignore_index=-100 is honoured by F.cross_entropy: those positions contribute zero loss and zero gradient.",
        "The model output at those positions is set to NaN automatically.",
        "PyTorch normalises by sequence length so masked positions cancel out."
      ],
      "correct": 1,
      "explanation": "ignore_index is a documented argument of F.cross_entropy. Targets equal to it contribute zero to loss and zero to gradients. -100 is the conventional default value."
    },
    {
      "stage": "check",
      "question": "Why does `shifted_loss` slice `logits[:, :-1, :]` and `labels[:, 1:]` instead of using them at the same positions?",
      "options": [
        "To make the tensors the same size after the collate step.",
        "Because the model at position i predicts the token at position i+1 under the causal next-token objective.",
        "To exclude the final padding position automatically.",
        "It is an optional optimisation; using same-position labels also works."
      ],
      "correct": 1,
      "explanation": "Causal LM trains position i to predict token i+1. The slice aligns the prediction at position i with the gold label at position i+1, which is the standard next-token objective."
    },
    {
      "stage": "check",
      "question": "You train without the mask: labels are equal to input_ids with no -100. The model trains but exact-match on the response is worse than with the mask. The most likely cause?",
      "options": [
        "The optimiser overfits the response tokens.",
        "Most of the gradient signal goes to predicting the instruction (the largest masked region was holding it back).",
        "Cross-entropy is unstable when the loss includes specials.",
        "The model cannot learn the boundary token without the mask."
      ],
      "correct": 1,
      "explanation": "When instruction positions count toward the loss, they dominate the gradient because there are usually more of them than response tokens. The effective learning rate on the part you care about (the response) drops, and the model spends capacity reconstructing inputs the user always supplies."
    },
    {
      "stage": "post",
      "question": "Exact-match is 0.85 on the held-out set. A reviewer says the model paraphrases correct answers and exact-match misses those. Which metric do you add to capture this?",
      "options": [
        "Loss on the held-out set.",
        "Token-level F1 against the reference, which credits overlap even when not character-identical.",
        "Average sequence length.",
        "Cross-entropy of the reference under the model."
      ],
      "correct": 1,
      "explanation": "Token-level F1 (counted between predicted and reference tokens after normalisation) credits the model for getting most of the answer right even when surface form differs. This is lesson 41's job."
    },
    {
      "stage": "post",
      "question": "You want to use the SFT model in a chatbot that supports multi-turn dialogue. What is the minimum change required to the data and the mask?",
      "options": [
        "None: SFT models always handle multi-turn out of the box.",
        "Add a USER and ASSISTANT marker per turn, concatenate turns, and mask everything except the assistant turn currently being trained on.",
        "Switch the loss to mean-squared error.",
        "Increase the vocabulary."
      ],
      "correct": 1,
      "explanation": "Multi-turn SFT generalises the single-turn template. Each turn gets a role marker. The mask is the union of all non-assistant positions plus padding. The training objective is unchanged."
    }
  ]
}