diff --git a/CLAUDE.md b/CLAUDE.md index fe8cb65..7269f8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ Two independent packages, no top-level workspace manager: - `backend/` — FastAPI + SQLAlchemy 2 + Alembic, managed by `uv`, Python 3.14 - `frontend/` — Next.js 14 App Router, TypeScript, Tailwind CSS, Supabase SSR -- `frontend-v2/` — Next.js 16 + Tailwind v4 v2 frontend (per [ADR 0007](docs/adr/0007-frontend-v2-migration-strategy.md)). Phases 2 + 3 ship: OKLCH token system + shadcn primitives + ~17 Ascendra composites; pages for `/`, `/login`, `/signup`, `/onboarding`, `/dashboard`, `/arena`, `/arena/[mechanic]/play`, `/replay/[id]`, `/replay/[id]/share`; Supabase auth + TanStack Query; TopBar in the `(app)` layout. MVP pages stay in `frontend/` until parity is reached and the cutover happens. See `frontend-v2/CLAUDE.md` for v2 conventions (theme provider, `proxy.ts`, async cookies, design tokens, Vitest + Playwright test runner separation). +- `frontend-v2/` — Next.js 16 + Tailwind v4 v2 frontend (per [ADR 0007](docs/adr/0007-frontend-v2-migration-strategy.md)). Phases 2 + 3 (sub-plans #1 + #2) ship: OKLCH token system + shadcn primitives + ~19 Ascendra composites; pages for `/`, `/login`, `/signup`, `/onboarding`, `/dashboard`, `/arena`, `/arena/[mechanic]/play` (cv-battle + behavioural-duel + mental-maths-blitz), `/replay/[id]`, `/replay/[id]/share`; Supabase auth + TanStack Query; TopBar in the `(app)` layout. MVP pages stay in `frontend/` until parity is reached and the cutover happens. See `frontend-v2/CLAUDE.md` for v2 conventions (theme provider, `proxy.ts`, async cookies, design tokens, Vitest + Playwright test runner separation). All commands below assume you `cd` into the relevant package first. Each has its own README with more detail (`backend/README.md` is the most accurate single source of truth for backend setup). diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index d3d88fd..9af81a8 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -231,6 +231,14 @@ See `.env.example`. Required: `DATABASE_URL` (Session pooler), `SUPABASE_URL`, ` For local development against the real DB, the recommended pattern is to point `DATABASE_URL` at a Supabase project's Session pooler URL on port 5432 with the `postgresql+psycopg://` prefix. +## Public template reads sanitize answer keys + +`templates.services.challenge_templates` ships two read variants: +- `get_challenge_template_or_error` / `list_challenge_templates` — return the full ORM rubric (consumed by the scoring pipeline's `DeterministicNumericJudge` via `ctx.template.rubric_definition`). +- `get_challenge_template_or_error_for_public_api` / `list_challenge_templates_for_public_api` — detach the row and overwrite `rubric_definition` with a sanitized copy (no `correct_answers`, no `questions[i].answer`). + +The `/api/v1/challenge-templates/*` router uses the public variants. The scoring pipeline reads the template via the ORM directly so deterministic mechanics keep their answer key. When adding a new template type that carries hidden answer/hints, extend `_sanitize_rubric_for_public_read` rather than introducing a parallel surface. + ## Common Pitfalls Already Hit - Forgetting to add a new model to `app/models/__init__.py` ⇒ test suite passes locally because the import happens via another route; CI catches it via `Base.metadata.create_all` missing the table. diff --git a/backend/app/contexts/templates/api/challenge_templates.py b/backend/app/contexts/templates/api/challenge_templates.py index 34286ad..4f0d929 100644 --- a/backend/app/contexts/templates/api/challenge_templates.py +++ b/backend/app/contexts/templates/api/challenge_templates.py @@ -10,8 +10,8 @@ ChallengeTemplateRead, ) from app.contexts.templates.services.challenge_templates import ( - get_challenge_template_or_error, - list_challenge_templates, + get_challenge_template_for_public_api, + list_challenge_templates_for_public_api, ) from app.core.enums import ApplicationCycle, Arena, ChallengeMode, ChallengeType, DomainPathway @@ -29,7 +29,7 @@ def list_challenge_templates_endpoint( mode: ChallengeMode | None = None, calibration_only: bool | None = None, ) -> ChallengeTemplateListResponse: - items = list_challenge_templates( + items = list_challenge_templates_for_public_api( db, target_domain=target_domain, application_cycle=application_cycle, @@ -47,4 +47,4 @@ def get_challenge_template_endpoint( db: DatabaseSession, current_user: CurrentUser, ) -> ChallengeTemplateRead: - return get_challenge_template_or_error(db, template_id=template_id) + return get_challenge_template_for_public_api(db, template_id=template_id) diff --git a/backend/app/contexts/templates/schemas/challenge_template.py b/backend/app/contexts/templates/schemas/challenge_template.py index a2330cf..e28b7c9 100644 --- a/backend/app/contexts/templates/schemas/challenge_template.py +++ b/backend/app/contexts/templates/schemas/challenge_template.py @@ -26,12 +26,35 @@ class RubricCategory(BaseModel): focus_keywords: list[str] = Field(default_factory=list) +class RubricQuestion(BaseModel): + """One question entry on the rubric (Phase 3 sub-plan #2). + + Only the ``prompt`` is exposed publicly — the corresponding + ``answer`` (if present on the server-side rubric) is stripped by + ``_sanitize_rubric_for_public_read`` in the templates service, AND + ignored here by Pydantic ``extra="ignore"`` default behaviour. Used + by ``mental_maths_blitz`` (numeric streak). + """ + + prompt: str + + class RubricDefinition(BaseModel): categories: list[RubricCategory] = Field(default_factory=list) focus_keywords: list[str] = Field(default_factory=list) ideal_word_range_min: int | None = None ideal_word_range_max: int | None = None expected_answers: list[str] = Field(default_factory=list) + # Question battery exposed to the room composites (Phase 3 sub-plan + # #2). Two shapes are supported for backwards compatibility: + # - Plain strings (``behavioural_duel`` stores 4 STAR prompts). + # - ``RubricQuestion`` dicts (``mental_maths_blitz`` stores 30 + # ``{"prompt": ..., "answer": ...}`` entries; ``answer`` is + # stripped by ``_sanitize_rubric_for_public_read``). + questions: list[RubricQuestion | str] = Field(default_factory=list) + # Optional time-bonus window (seconds) for deterministic numeric + # judges. Used by ``mental_maths_blitz``. + time_bonus_seconds: int | None = None benchmark_anchor_score: float = 72.0 diff --git a/backend/app/contexts/templates/services/bootstrap.py b/backend/app/contexts/templates/services/bootstrap.py index 435bdff..6ac019d 100644 --- a/backend/app/contexts/templates/services/bootstrap.py +++ b/backend/app/contexts/templates/services/bootstrap.py @@ -1,5 +1,9 @@ from __future__ import annotations +import importlib.util +from pathlib import Path +from types import ModuleType + from sqlalchemy import select from sqlalchemy.orm import Session @@ -25,6 +29,38 @@ SkillNodeType, ) +# --------------------------------------------------------------------------- +# Phase 3 sub-plan #2 (Task A4): pull in seed scripts for the v2 mechanics so +# the conftest fixture + ``scripts/seed_reference_data.py`` produce a DB that +# includes ``behavioural_duel`` and ``mental_maths_blitz``. +# +# ``scripts/`` is intentionally NOT a Python package, so we use +# ``importlib.util`` to load the seed modules on first use. The modules are +# cached on the function so repeated calls (e.g. across multiple test cases) +# don't re-parse the file. Per ADR 0008 this stays inside the templates +# context — the scripts only mutate templates / mechanics / anchors rows. +# --------------------------------------------------------------------------- + +_SCRIPTS_DIR = Path(__file__).resolve().parents[4] / "scripts" + + +def _load_script_module(filename: str, alias: str) -> ModuleType: + cache = _load_script_module._cache # type: ignore[attr-defined] + if alias in cache: + return cache[alias] + path = _SCRIPTS_DIR / filename + spec = importlib.util.spec_from_file_location(alias, path) + assert spec is not None and spec.loader is not None, ( + f"Failed to locate seed script at {path}" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + cache[alias] = module + return module + + +_load_script_module._cache = {} # type: ignore[attr-defined] + def seed_reference_data(db: Session) -> None: skill_nodes = {node.slug: node for node in _upsert_skill_nodes(db)} @@ -37,9 +73,76 @@ def seed_reference_data(db: Session) -> None: pack.slug: pack for pack in _upsert_benchmark_packs(db, templates) } _upsert_quests(db, skill_nodes, benchmark_packs) + _seed_phase3_mechanics(db) db.commit() +def _seed_phase3_mechanics(db: Session) -> None: + """Hook the Phase 3 sub-plan #2 mechanic seed scripts in. Idempotent + (both scripts upsert).""" + + behavioural = _load_script_module( + "seed_phase3_behavioural_duel_mechanic.py", + "seed_phase3_behavioural_duel", + ) + behavioural.seed_behavioural_duel(db, profile="dev") + + mental_maths = _load_script_module( + "seed_phase3_mental_maths_blitz_mechanic.py", + "seed_phase3_mental_maths_blitz", + ) + mental_maths.seed_mental_maths_blitz(db) + _link_phase3_templates_to_skill_nodes(db) + + +def _link_phase3_templates_to_skill_nodes(db: Session) -> None: + """Wire the new Phase 3 sub-plan #2 templates into the existing skill + tree (Task A5). + + Existing Professional + Technical leaves cover the relevant signals + for the two new mechanics, so no new skill nodes are added in this + sub-plan. Deeper skill-tree work lands in Phase 4. + """ + + link_seeds = [ + # Behavioural Duel — STAR-anchored, drives professional-arena + # storytelling + stakeholder judgement + the conflict / + # leadership leaves it covers via the four canonical prompts. + ("behavioural-duel-phase3-default", "star-storytelling", 1.2), + ("behavioural-duel-phase3-default", "stakeholder-judgement", 1.0), + ("behavioural-duel-phase3-default", "conflict-resolution", 0.8), + ("behavioural-duel-phase3-default", "leadership-framing", 0.8), + # Mental Maths Blitz — deterministic numeric accuracy under + # heavy time pressure; also exercises problem decomposition. + ("mental-maths-blitz-phase3-default", "numerical-accuracy", 1.3), + ("mental-maths-blitz-phase3-default", "problem-decomposition", 0.7), + ] + for template_slug, node_slug, emphasis in link_seeds: + template = db.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == template_slug) + ) + node = db.scalar(select(SkillNode).where(SkillNode.slug == node_slug)) + if template is None or node is None: + # If the seed didn't run (e.g., partial DB state), skip the + # link silently — the test fixture asserts the seed ran. + continue + existing = db.scalar( + select(ChallengeTemplateSkillLink).where( + ChallengeTemplateSkillLink.challenge_template_id == template.id, + ChallengeTemplateSkillLink.skill_node_id == node.id, + ) + ) + if existing is None: + existing = ChallengeTemplateSkillLink( + challenge_template_id=template.id, + skill_node_id=node.id, + ) + existing.emphasis_weight = emphasis + existing.is_repair_anchor = True + db.add(existing) + db.flush() + + def _skill_seed( *, slug: str, diff --git a/backend/app/contexts/templates/services/challenge_templates.py b/backend/app/contexts/templates/services/challenge_templates.py index 355c43a..2d72ea9 100644 --- a/backend/app/contexts/templates/services/challenge_templates.py +++ b/backend/app/contexts/templates/services/challenge_templates.py @@ -1,6 +1,8 @@ from __future__ import annotations +import copy import uuid +from typing import Any from fastapi import status from sqlalchemy import select @@ -11,6 +13,62 @@ from app.core.errors import AppError +def _sanitize_rubric_for_public_read( + rubric: dict[str, Any] | None, +) -> dict[str, Any] | None: + """Deep-copy a rubric dict and strip fields that leak answer keys + (Phase 3 sub-plan #2 Task A3). + + The mental_maths_blitz seed embeds the deterministic answer key + into ``rubric_definition.correct_answers`` and into each entry of + ``rubric_definition.questions[i].answer``. These MUST never reach + the client — otherwise a user could read the page source and ace + the drill. This helper produces a defensive deep copy with both + fields removed so the caller can attach the sanitized dict to a + response object without mutating the original (or the + SQLAlchemy-attached ORM row). + + The pipeline / workflow / scoring helpers read ``rubric_definition`` + directly off the SQLAlchemy ORM object (see + ``DeterministicNumericJudge.score``), bypassing this helper — so + deterministic scoring still works. + """ + + if rubric is None: + return None + sanitized: dict[str, Any] = copy.deepcopy(rubric) + sanitized.pop("correct_answers", None) + questions = sanitized.get("questions") + if isinstance(questions, list): + cleaned_questions: list[Any] = [] + for entry in questions: + if isinstance(entry, dict): + cleaned_entry = {k: v for k, v in entry.items() if k != "answer"} + cleaned_questions.append(cleaned_entry) + else: + cleaned_questions.append(entry) + sanitized["questions"] = cleaned_questions + return sanitized + + +def _detach_with_sanitized_rubric( + db: Session, template: ChallengeTemplate +) -> ChallengeTemplate: + """Expunge the template from the session and overwrite + ``rubric_definition`` with a sanitized deep copy. + + Detaching avoids any risk that the in-memory mutation flushes back + to the database. Used by the public-API list/get wrappers so the + response model never sees the raw answer key. + """ + + db.expunge(template) + template.rubric_definition = _sanitize_rubric_for_public_read( + template.rubric_definition + ) or {} + return template + + def list_challenge_templates( db: Session, *, @@ -55,6 +113,35 @@ def is_visible(template: ChallengeTemplate) -> bool: return [template for template in templates if is_visible(template)] +def list_challenge_templates_for_public_api( + db: Session, + *, + target_domain: DomainPathway | None = None, + application_cycle: ApplicationCycle | None = None, + arena: Arena | None = None, + challenge_type: ChallengeType | None = None, + mode: ChallengeMode | None = None, + calibration_only: bool | None = None, +) -> list[ChallengeTemplate]: + """Wrapper around :func:`list_challenge_templates` that strips + answer-key fields out of each template's rubric (Task A3). + + The pipeline / scoring helpers continue to call + :func:`list_challenge_templates` for the full rubric. + """ + + templates = list_challenge_templates( + db, + target_domain=target_domain, + application_cycle=application_cycle, + arena=arena, + challenge_type=challenge_type, + mode=mode, + calibration_only=calibration_only, + ) + return [_detach_with_sanitized_rubric(db, t) for t in templates] + + def get_challenge_template_or_error( db: Session, *, @@ -72,3 +159,19 @@ def get_challenge_template_or_error( status_code=status.HTTP_404_NOT_FOUND, ) return template + + +def get_challenge_template_for_public_api( + db: Session, + *, + template_id: uuid.UUID, +) -> ChallengeTemplate: + """Wrapper around :func:`get_challenge_template_or_error` that strips + answer-key fields out of the rubric before returning (Task A3). + + The pipeline / scoring helpers continue to call + :func:`get_challenge_template_or_error` for the full rubric. + """ + + template = get_challenge_template_or_error(db, template_id=template_id) + return _detach_with_sanitized_rubric(db, template) diff --git a/backend/app/core/enums.py b/backend/app/core/enums.py index e9ec7b0..652844b 100644 --- a/backend/app/core/enums.py +++ b/backend/app/core/enums.py @@ -43,6 +43,11 @@ class ChallengeType(StrEnum): CV_BATTLE = "cv_battle" BEHAVIOURAL_DUEL = "behavioural_duel" TIMED_DRILL = "timed_drill" + # Phase 3 sub-plan #2: mental_maths_blitz mechanic (deterministic + # numeric judge, Technical arena). Distinct from TIMED_DRILL so + # the frontend dispatcher and analytics can branch on the mechanic + # without consulting other columns. + MENTAL_MATHS = "mental_maths" class DifficultyLevel(StrEnum): @@ -65,6 +70,13 @@ class ScoringMode(StrEnum): RUBRIC = "rubric" PAIRWISE = "pairwise" HYBRID = "hybrid" + # Phase 3 sub-plan #2: explicit semantic values used by new + # v2 mechanics. JUDGE_LLM signals an LLM-judge-only mix + # (no deterministic component); DETERMINISTIC signals a + # deterministic-judge-only mix (no LLM call). These coexist + # with HYBRID for backwards compatibility with cv_battle. + JUDGE_LLM = "judge_llm" + DETERMINISTIC = "deterministic" class InputType(StrEnum): @@ -72,12 +84,26 @@ class InputType(StrEnum): DOCUMENT = "document" STRUCTURED_FORM = "structured_form" NUMERIC = "numeric" + # Phase 3 sub-plan #2: behavioural_duel ships multi-paragraph + # text answers (one per behavioural question). LONG_TEXT lets + # the frontend pick a multi-textarea wizard vs. a single TEXT + # input. + LONG_TEXT = "long_text" + # Phase 3 sub-plan #2: mental_maths_blitz streams numeric + # answers one per question (30 sequential responses). + # Distinct from NUMERIC so the room composite can render the + # streak UI. + NUMERIC_STREAK = "numeric_streak" class OutputType(StrEnum): TEXT = "text" NUMERIC = "numeric" STRUCTURED_JSON = "structured_json" + # Phase 3 sub-plan #2: behavioural_duel emits a structured + # 4-answer payload (one per question). Distinct from + # STRUCTURED_JSON to signal a fixed Q/A array shape. + STRUCTURED = "structured" class DomainProfileStatus(StrEnum): diff --git a/backend/scripts/seed_phase3_behavioural_duel_mechanic.py b/backend/scripts/seed_phase3_behavioural_duel_mechanic.py new file mode 100644 index 0000000..a84db8c --- /dev/null +++ b/backend/scripts/seed_phase3_behavioural_duel_mechanic.py @@ -0,0 +1,674 @@ +"""Seed the ``behavioural_duel`` ChallengeMechanic + a default +ChallengeTemplate + three STRONG CalibrationAnchors (Phase 3 +sub-plan #2 Task A1). + +Mirrors ``seed_phase1_cv_battle_mechanic.py`` but for the Professional +arena. The mechanic ships with two profiles: + +- ``dev`` (default) -- Gemini-primary mix per ADR 0013: + * RubricLLMJudge on ``gemini-2.5-flash`` (cheap-tier base rubric). + * RubricLLMJudge on ``gemini-2.5-pro`` (richer second judge). + * PairwiseLLMJudge on ``gemini-2.5-pro`` vs. STRONG anchor. + +- ``production`` -- Claude-primary mix per ADR 0006: + * RubricLLMJudge on ``claude-opus-4-7``. + * PairwiseLLMJudge on ``claude-sonnet-4-6`` vs. STRONG anchor. + * RubricLLMJudge on ``claude-sonnet-4-6`` (second judge). + +The judges use ``judge_type=professional`` so MetaJudge aggregation +attributes scores to the Professional arena Elo. Each anchor's +``submission_text`` stitches the four STAR answers with a +``\\n\\n---\\n\\n`` separator so a single RubricLLMJudge prompt sees +the full set. + +Usage:: + + uv run python scripts/seed_phase3_behavioural_duel_mechanic.py \\ + [--profile dev|production] [--reset] +""" + +from __future__ import annotations + +import argparse +import logging +import sys +import uuid +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from sqlalchemy import select # noqa: E402 + +from app.contexts.calibration.models.calibration_anchor import CalibrationAnchor # noqa: E402 +from app.contexts.templates.models.challenge_mechanic import ChallengeMechanic # noqa: E402 +from app.contexts.templates.models.challenge_template import ChallengeTemplate # noqa: E402 +from app.core.enums import ( # noqa: E402 + Arena, + CalibrationSource, + CalibrationTier, + ChallengeMode, + ChallengeType, + DifficultyLevel, + InputType, + OutputType, + ScoringMode, +) +from app.db.session import configure_engine, get_session_factory # noqa: E402 + +logger = logging.getLogger("seed_phase3_behavioural_duel") +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + + +_MECHANIC_KEY = "behavioural_duel" +_TEMPLATE_SLUG = "behavioural-duel-phase3-default" + +_QUESTIONS = [ + "Tell me about a time you led a team through a difficult problem.", + "Tell me about a time you failed and what you learned.", + "Tell me about a time you disagreed with a colleague.", + "Tell me about a time you went above and beyond.", +] + + +# ────────────────────────────────────────────────────────────────────── + + +def _dev_judge_mix() -> dict: + return { + "judges": [ + { + "type": "rubric_llm", + "model": "gemini-2.5-flash", + "weight": 1.0, + "judge_type": "professional", + "judge_version": "judge.rubric_llm.v1", + }, + { + "type": "rubric_llm", + "model": "gemini-2.5-pro", + "weight": 1.0, + "judge_type": "professional", + "judge_version": "judge.rubric_llm.v1", + }, + { + "type": "pairwise_llm", + "model": "gemini-2.5-pro", + "weight": 1.0, + "anchors_required": True, + "anchors_count": 2, + "anchor_tier": "strong", + "judge_type": "professional", + "judge_version": "judge.pairwise_llm.v1", + }, + ], + "meta": {"type": "meta", "judge_version": "judge.meta.v1"}, + } + + +def _production_judge_mix() -> dict: + return { + "judges": [ + { + "type": "rubric_llm", + "model": "claude-opus-4-7", + "weight": 1.2, + "judge_type": "professional", + "judge_version": "judge.rubric_llm.v1", + }, + { + "type": "pairwise_llm", + "model": "claude-sonnet-4-6", + "weight": 1.0, + "anchors_required": True, + "anchors_count": 2, + "anchor_tier": "strong", + "judge_type": "professional", + "judge_version": "judge.pairwise_llm.v1", + }, + { + "type": "rubric_llm", + "model": "claude-sonnet-4-6", + "weight": 0.8, + "judge_type": "professional", + "judge_version": "judge.rubric_llm.v1", + }, + ], + "meta": {"type": "meta", "judge_version": "judge.meta.v1"}, + } + + +def _default_rubric() -> dict: + return { + "categories": [ + { + "name": "star_structure", + "weight": 0.25, + "focus_keywords": ["situation", "task", "action", "result"], + }, + { + "name": "specificity", + "weight": 0.25, + "focus_keywords": ["concrete", "example", "metric"], + }, + { + "name": "ownership", + "weight": 0.20, + "focus_keywords": ["I led", "my decision", "I owned"], + }, + { + "name": "learning", + "weight": 0.15, + "focus_keywords": ["learned", "next time", "improvement"], + }, + { + "name": "fit", + "weight": 0.15, + "focus_keywords": ["values", "team", "role fit"], + }, + ], + "questions": list(_QUESTIONS), + "benchmark_anchor_score": 70.0, + } + + +def _prompt_text() -> str: + """Markdown prompt listing the four behavioural questions.""" + + return ( + "Answer each of the four behavioural questions below using the " + "STAR framework (Situation, Task, Action, Result). Spend about " + "three minutes per answer. Quantify outcomes where possible and " + "own your part of the story explicitly.\n\n" + "1. " + _QUESTIONS[0] + "\n\n" + "2. " + _QUESTIONS[1] + "\n\n" + "3. " + _QUESTIONS[2] + "\n\n" + "4. " + _QUESTIONS[3] + "\n" + ) + + +def _strong_anchor_specs() -> list[tuple[str, dict, str]]: + """Three STRONG anchors. Each ``submission_text`` is the four STAR + answers stitched with ``\\n\\n---\\n\\n`` so a single rubric pass + can read the full submission. + """ + + rubric_scores_strong = { + "star_structure": 90, + "specificity": 92, + "ownership": 88, + "learning": 85, + "fit": 86, + } + + anchor_1_answers = [ + # Q1: led team + ( + "Situation: As tech-lead intern on a 5-person team owning the " + "checkout service at a Series-C fintech, our payment success " + "rate dropped from 99.2% to 96.8% over a Friday afternoon, " + "impacting roughly 12,000 transactions across the weekend. " + "Task: I owned the triage and recovery within 48 hours while " + "two senior engineers were on PTO. Action: I split the team " + "into two pairs -- one chasing the provider's idempotency " + "rollout, the other regenerating retry-buckets from our logs. " + "I ran 30-minute syncs, drafted the incident runbook updates " + "as we learned, and personally rolled back the gateway header " + "change once we confirmed it was the cause. Result: We " + "restored success rate to 99.4% within 14 hours and recovered " + "98% of failed transactions through async retries; my " + "post-mortem was adopted as the team's reference incident." + ), + # Q2: failed + ( + "Situation: In my first month I ran a load test against the " + "shared staging database believing it was an isolated replica, " + "and froze the team's environment for 90 minutes during a " + "demo day. Task: I owned naming the failure publicly and " + "preventing it from recurring. Action: I posted in the team " + "channel within 10 minutes naming the root cause, paired with " + "platform to add a tier guard on the load-test runner, and " + "wrote a one-page checklist that gets attached to every " + "load-test PR template. Result: No analogous incident has " + "happened in the four months since; my checklist now ships " + "with the team's onboarding doc. The bigger lesson was that " + "I'd been optimising for speed over checks-before-action, and " + "I now write a one-liner risk note at the top of every test " + "PR -- a habit two other engineers have adopted." + ), + # Q3: disagreed + ( + "Situation: A senior engineer on my team proposed replacing " + "our Postgres-backed feature flags with a new vendor service " + "two days before a product launch. Task: I felt the " + "migration risk outweighed the gains given the timeline and " + "needed to push back without burning trust. Action: I " + "scheduled a 20-minute one-on-one, listed three failure " + "modes I was worried about (cold-start latency, missing " + "rollback SLA, on-call training gap), and proposed a " + "compromise: ship the launch on Postgres, then pilot the " + "vendor in a low-traffic surface the following sprint. I " + "shared my notes ahead of time so it wasn't an ambush. " + "Result: The senior engineer agreed and we ran the pilot " + "two weeks later; it surfaced the cold-start issue I'd " + "flagged, which we addressed before any production user " + "saw it. The shipping path stayed on schedule." + ), + # Q4: above and beyond + ( + "Situation: Our weekly recruiter pipeline review depended on " + "a spreadsheet that a junior analyst hand-stitched from three " + "internal tools every Monday morning, taking her ~4 hours. " + "Task: It wasn't my project, but I noticed she was missing " + "team standups because of it. Action: Over two weekends I " + "wrote a 250-line Python script that pulls from the three " + "APIs, applies the dedupe logic the analyst was doing by " + "eye, and ships a CSV to a shared drive. I paired with her " + "on the validation step so she trusted the output, and wrote " + "a one-page README so she could rerun it without me. " + "Result: She got 4 hours back every week; the recruiting " + "lead asked to extend it to two adjacent pipelines, and I " + "trained her to maintain it. It earned me a spot on the " + "internal ops working group." + ), + ] + + anchor_2_answers = [ + # Q1: led team + ( + "Situation: During my consulting summer placement I led a " + "team of three analysts on a procurement-savings diagnostic " + "for a UK utilities client. Mid-week our partner flagged " + "that the supplier-master data was 6 months stale, making " + "two of our four hypotheses untestable. Task: I had 48 " + "hours to re-plan the analysis without missing Friday's " + "steering committee. Action: I called a 25-minute reset " + "with the team, redivided ownership so two analysts pulled " + "fresh data directly from the client's three legacy systems " + "while I rewrote our hypothesis tree, and negotiated with " + "the client lead to extend our data-room access by 36 " + "hours. I wrote a one-page revised plan that night so the " + "partner could redirect early. Result: We presented two " + "fresh evidence-backed hypotheses on Friday and identified " + "£3.2M of addressable savings; the client extended the " + "engagement by four weeks and named the diagnostic as a " + "lessons-learned case study internally." + ), + # Q2: failed + ( + "Situation: On a previous project I committed to a Wednesday " + "deliverable without checking that the data extract would " + "land on time; it slipped by two days and I had to tell the " + "engagement manager on Tuesday afternoon. Task: I owned " + "naming the slip early and preventing it from happening " + "again. Action: I flagged it 48 hours in advance rather " + "than the morning of, proposed a phased deliverable so the " + "client got the structure on Wednesday and the numbers on " + "Friday, and committed to a pre-commit data-readiness " + "check on every deliverable I've owned since. Result: The " + "client accepted the phased approach without complaint; " + "the EM later told me the way I handled it built more " + "trust than the original on-time deliverable would have. " + "The data-readiness check is now in my personal pre-flight " + "checklist for every commitment." + ), + # Q3: disagreed + ( + "Situation: A peer on my team wanted to use a Monte-Carlo " + "simulation to estimate procurement savings; I thought a " + "simpler bottoms-up estimate would be faster and more " + "defensible to the client. Task: I needed to make the " + "case without dismissing her preferred method. Action: I " + "asked her to walk me through the simulation in 15 " + "minutes, then showed her my bottoms-up estimate on a " + "whiteboard alongside it. We agreed to use bottoms-up for " + "the steering committee and keep the simulation as an " + "appendix sensitivity check. I made sure she got the byline " + "on the appendix. Result: The client accepted the " + "headline numbers without challenge; the sensitivity range " + "was the second-most-cited slide in the deck. The " + "experience taught me that 'simpler' isn't always " + "'simpler-for-the-audience', and to pair-check assumptions " + "before defending them in front of leadership." + ), + # Q4: above and beyond + ( + "Situation: Our team's intern cohort had no shared way of " + "tracking which clients had given guest-speaker slots, so " + "interns kept asking the same partner twice. Task: It " + "wasn't anyone's job, but I noticed three interns had been " + "embarrassed by overlapping outreach. Action: Over two " + "evenings I built a small Airtable with view-only links " + "for the cohort, scraped the past 18 months of LinkedIn " + "engagement signals to seed it, and hosted a 30-minute " + "Lunch & Learn so the other 12 interns could use it. I " + "documented the gotchas so a future cohort could extend " + "it. Result: It became the cohort's default tracker; the " + "graduate-scheme lead asked me to present it to the next " + "intake. I learned that organising other people's data is " + "often the fastest way to be helpful in a knowledge-work " + "team." + ), + ] + + anchor_3_answers = [ + # Q1: led team + ( + "Situation: As junior PM on a 6-person feature team, our " + "B2B onboarding flow had a 38% drop-off at the second step. " + "Both the staff PM and the EM were out for a week and we " + "had a board-update deadline in 8 days. Task: I owned the " + "investigation and a credible recommendation by the " + "deadline. Action: I scoped the work into three parallel " + "tracks (data, customer interviews, design audit), pulled " + "in one engineer to instrument funnel events, and personally " + "ran four 25-minute customer calls. I ran a 30-minute " + "morning sync, kept a shared Notion doc updated live, and " + "wrote a one-page recommendation by day 6 so design had " + "time to mock the fix. Result: The team shipped a 3-field " + "trimmed flow that improved step-2 conversion by 18 " + "percentage points within two release cycles; I was asked " + "to own the metric for the next quarter and was promoted " + "to PM at the end of the placement." + ), + # Q2: failed + ( + "Situation: My first user-research synthesis missed the " + "single most important quote ('we want to skip the " + "manager-approval step entirely') because I'd grouped it " + "into a generic 'admin friction' theme. The CPO caught it " + "in review. Task: I owned correcting the synthesis and " + "preventing it from repeating. Action: I rewrote the " + "synthesis the next morning with the manager-approval theme " + "broken out, added it to the priority queue, and changed " + "my own process to do a 10-minute 'top 3 verbatim' pass " + "before I cluster themes. I shared the change with the " + "two other PMs on the team. Result: The manager-approval " + "fix is now in the roadmap as a Q4 deliverable. The lesson " + "I took was to keep raw verbatims close to the surface in " + "any synthesis, not just my paraphrased themes." + ), + # Q3: disagreed + ( + "Situation: An engineer wanted to ship a new dashboard " + "behind a feature flag without the qualitative usability " + "test I'd planned; he felt the test would slow us down by " + "a week. Task: I needed to defend the test without losing " + "his goodwill, since he and I would be working closely for " + "the next two quarters. Action: I asked him what risk he " + "was trying to avoid (timeline slip), then proposed a " + "compressed format: a 3-user moderated test over 90 " + "minutes, with me running it that same day. I agreed to " + "skip the second round if the first surfaced no blockers. " + "Result: The test surfaced one wording change and one " + "missing affordance; we shipped 2 days later than his " + "plan, both fixes baked in. The engineer told me " + "afterwards that the test paid for itself and he'd push " + "for one earlier next time." + ), + # Q4: above and beyond + ( + "Situation: Our team had no shared template for product " + "review prep, so each PM spent 2-3 hours rebuilding decks " + "from scratch each cycle. Task: It wasn't part of my role, " + "but I noticed the senior PMs were skipping the review when " + "they were busy because of the cost. Action: I built a " + "reusable Figma + slide template with metric placeholders, " + "a one-pager risk register, and a 5-prompt prep checklist. " + "I ran a 30-minute walkthrough with the team, took notes " + "on the friction points, and iterated twice. I made sure " + "another PM owned the v3 update so it didn't depend on me. " + "Result: Prep time dropped to ~45 minutes; review attendance " + "went from 60% to 100%. The CPO asked me to share it with " + "the wider PM org as a benchmark; I learned that small, " + "ownerless friction is often the highest-leverage problem to " + "fix." + ), + ] + + separator = "\n\n---\n\n" + return [ + ( + separator.join(anchor_1_answers), + rubric_scores_strong.copy(), + ( + "Strong STAR structure across all four answers, " + "quantified outcomes (12,000 transactions, 90 mins " + "frozen, 4 hours/week), explicit ownership ('I led', " + "'I owned'), clear learning vignettes." + ), + ), + ( + separator.join(anchor_2_answers), + rubric_scores_strong.copy(), + ( + "Strong STAR structure with consulting-flavoured " + "specifics (£3.2M, 36-hour negotiation, two-evening " + "tool build), ownership signals throughout, and a " + "clear failure-and-recovery arc on Q2." + ), + ), + ( + separator.join(anchor_3_answers), + rubric_scores_strong.copy(), + ( + "Strong STAR with product-flavoured specifics (18pp " + "conversion lift, 3-user usability test compromise, " + "Figma+slide template adoption). Ownership voice " + "consistent across all four answers." + ), + ), + ] + + +# ────────────────────────────────────────────────────────────────────── + + +def upsert_mechanic(db, *, profile: str) -> ChallengeMechanic: + mix = _dev_judge_mix() if profile == "dev" else _production_judge_mix() + mechanic = db.scalar( + select(ChallengeMechanic).where(ChallengeMechanic.key == _MECHANIC_KEY) + ) + if mechanic is None: + mechanic = ChallengeMechanic( + key=_MECHANIC_KEY, + name="Behavioural Duel", + domain_pathway=None, + arena=Arena.PROFESSIONAL, + default_judge_mix=mix, + default_rubric=_default_rubric(), + requires_sandbox=False, + requires_video=False, + time_limit_seconds=4 * 180, # 4 questions x 3 min + ) + db.add(mechanic) + logger.info("created ChallengeMechanic key=%s", _MECHANIC_KEY) + else: + mechanic.default_judge_mix = mix + mechanic.default_rubric = _default_rubric() + mechanic.time_limit_seconds = 4 * 180 + logger.info("updated ChallengeMechanic key=%s (profile=%s)", _MECHANIC_KEY, profile) + db.flush() + return mechanic + + +def upsert_template(db, *, mechanic: ChallengeMechanic) -> ChallengeTemplate: + template = db.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == _TEMPLATE_SLUG) + ) + if template is None: + template = ChallengeTemplate( + slug=_TEMPLATE_SLUG, + title="Behavioural Duel (Phase 3 default)", + description=( + "Four behavioural interview questions answered in " + "STAR format. Multi-judge rubric scoring vs. " + "recruiter-anchored exemplars." + ), + arena=Arena.PROFESSIONAL, + domain_pathway=None, + challenge_type=ChallengeType.BEHAVIOURAL_DUEL, + difficulty=DifficultyLevel.CORE, + input_type=InputType.LONG_TEXT, + output_type=OutputType.STRUCTURED, + scoring_mode=ScoringMode.JUDGE_LLM, + prompt=_prompt_text(), + time_limit_seconds=4 * 180, + rubric_definition=_default_rubric(), + eligible_modes=[ + ChallengeMode.PRACTICE.value, + ChallengeMode.UNRANKED.value, + ChallengeMode.RANKED.value, + ], + applicable_cycles=["spring_week", "internship", "graduate_role"], + matchmaking_enabled=False, + is_calibration_template=False, + active=True, + version=1, + prompt_version=1, + mechanic_key=mechanic.key, + requires_sandbox=False, + judge_mix_definition=None, # inherit from the mechanic + ) + db.add(template) + logger.info("created ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + else: + # Keep the template pointed at the mechanic + refreshed rubric. + template.mechanic_key = mechanic.key + template.rubric_definition = _default_rubric() + template.prompt = _prompt_text() + template.time_limit_seconds = 4 * 180 + logger.info("updated ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + db.flush() + return template + + +def upsert_anchors(db, *, template: ChallengeTemplate) -> int: + """Idempotent anchor upsert. Three STRONG anchors per template.""" + + created = 0 + specs = _strong_anchor_specs() + # Look up existing STRONG anchors for this template by an + # ordinal stored in ``model_baseline_scores['anchor_index']`` so + # multiple STRONG anchors can be tracked independently. + existing_anchors = list( + db.scalars( + select(CalibrationAnchor).where( + CalibrationAnchor.template_id == template.id, + CalibrationAnchor.tier == CalibrationTier.STRONG, + ) + ) + ) + existing_by_index: dict[int, CalibrationAnchor] = {} + for anchor in existing_anchors: + idx = (anchor.model_baseline_scores or {}).get("anchor_index") + if isinstance(idx, int): + existing_by_index[idx] = anchor + + for index, (text, rubric_scores, rationale) in enumerate(specs): + existing = existing_by_index.get(index) + if existing is not None: + existing.submission_text = text + existing.rubric_scores = rubric_scores + existing.recruiter_rationale = rationale + logger.info( + "updated CalibrationAnchor tier=strong index=%d", index + ) + continue + anchor = CalibrationAnchor( + template_id=template.id, + tier=CalibrationTier.STRONG, + submission_text=text, + rubric_scores=rubric_scores, + recruiter_rationale=rationale, + # Track which spec slot this anchor came from so the + # upsert is stable across reruns (Phase 3 sub-plan #2 + # ships multiple STRONG anchors per template -- unlike + # cv_battle which had one per tier). + model_baseline_scores={"anchor_index": index}, + recruiter_id=None, + source=CalibrationSource.SYNTHETIC, + ) + db.add(anchor) + created += 1 + logger.info("created CalibrationAnchor tier=strong index=%d", index) + return created + + +def reset_existing(db) -> None: + """Drop the template (cascade-deletes its anchors) + the mechanic.""" + + template = db.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == _TEMPLATE_SLUG) + ) + if template is not None: + db.delete(template) + logger.info("deleted ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + mechanic = db.scalar( + select(ChallengeMechanic).where(ChallengeMechanic.key == _MECHANIC_KEY) + ) + if mechanic is not None: + db.delete(mechanic) + logger.info("deleted ChallengeMechanic key=%s", _MECHANIC_KEY) + db.flush() + + +def seed_behavioural_duel(db, *, profile: str = "dev") -> None: + """Reusable entry point for ``seed_reference_data`` (Phase 3 + sub-plan #2 Task A4). + + Drives the same three upserts as :func:`run` but against a caller- + supplied session. The caller owns commit/rollback. Idempotent: a + second call is a no-op for row counts but refreshes the mechanic + + template payloads (so judge-mix profile switches in place). + """ + + mechanic = upsert_mechanic(db, profile=profile) + template = upsert_template(db, mechanic=mechanic) + upsert_anchors(db, template=template) + db.flush() + + +def run(*, profile: str, reset: bool) -> None: + configure_engine() + session_factory = get_session_factory() + with session_factory() as session: + if reset: + reset_existing(session) + session.commit() + seed_behavioural_duel(session, profile=profile) + session.commit() + + logger.info( + "seed complete: mechanic=%s template=%s profile=%s", + _MECHANIC_KEY, + _TEMPLATE_SLUG, + profile, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profile", + choices=("dev", "production"), + default="dev", + help="Judge-mix profile to seed onto the behavioural_duel mechanic.", + ) + parser.add_argument( + "--reset", + action="store_true", + help="Drop existing behavioural_duel rows before seeding.", + ) + parser.add_argument( + "--user-id", # silently accepted for forward-compat; not used yet + default=str(uuid.UUID("11111111-1111-4111-8111-111111111111")), + ) + args = parser.parse_args(argv) + run(profile=args.profile, reset=args.reset) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/seed_phase3_mental_maths_blitz_mechanic.py b/backend/scripts/seed_phase3_mental_maths_blitz_mechanic.py new file mode 100644 index 0000000..0568aef --- /dev/null +++ b/backend/scripts/seed_phase3_mental_maths_blitz_mechanic.py @@ -0,0 +1,381 @@ +"""Seed the ``mental_maths_blitz`` ChallengeMechanic + a default +ChallengeTemplate (Phase 3 sub-plan #2 Task A2). + +A deterministic 30-question, 90-second numeric streak under the +Technical arena. No anchors -- the single ``DeterministicNumericJudge`` +reads the answer key directly off ``rubric_definition.correct_answers`` +and compares against ``submission_payload['answers']`` (a parallel +numeric list). + +The 30 questions are generated reproducibly with a fixed RNG seed so +re-running the script always produces the same payload. The mix: + + 10x multiplication (a x b, a,b in [2..15]) + 10x percentage of (e.g. "15% of 240") + 5x FX cross-rate (rates 1.10 / 1.25 / 0.85 / 1.40 / 0.92) + 5x clean division (cleanly divisible) + +Production cutover requires nothing -- the judge mix has a single +``deterministic_numeric`` entry with no model, so the dev/prod split +that cv_battle / behavioural_duel use doesn't apply here. + +Usage:: + + uv run python scripts/seed_phase3_mental_maths_blitz_mechanic.py [--reset] +""" + +from __future__ import annotations + +import argparse +import logging +import random +import sys +import uuid +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from sqlalchemy import select # noqa: E402 + +from app.contexts.templates.models.challenge_mechanic import ChallengeMechanic # noqa: E402 +from app.contexts.templates.models.challenge_template import ChallengeTemplate # noqa: E402 +from app.core.enums import ( # noqa: E402 + Arena, + ChallengeMode, + ChallengeType, + DifficultyLevel, + InputType, + OutputType, + ScoringMode, +) +from app.db.session import configure_engine, get_session_factory # noqa: E402 + +logger = logging.getLogger("seed_phase3_mental_maths_blitz") +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + + +_MECHANIC_KEY = "mental_maths_blitz" +_TEMPLATE_SLUG = "mental-maths-blitz-phase3-default" +_TIME_LIMIT_SECONDS = 90 +_TIME_BONUS_SECONDS = 60 + +# Reproducibility: pin the RNG seed so the answer key never drifts +# across reruns. +_QUESTION_SEED = 42 + +# FX rates used by the cross-rate questions. Picked to give clean +# integer/decimal answers for typical amounts. +_FX_RATES: list[tuple[str, str, float]] = [ + ("EUR", "USD", 1.10), + ("GBP", "USD", 1.25), + ("USD", "EUR", 0.85), + ("AUD", "USD", 0.70), # 0.70 chosen so 200 AUD -> 140 USD cleanly + ("CHF", "USD", 1.40), +] + + +def _generate_questions() -> list[dict[str, float | str]]: + """Generate the 30-question battery deterministically. + + Returns a parallel list of ``{"prompt": str, "answer": float}`` + dicts. The DeterministicNumericJudge reads ``correct_answers`` + (parallel list of just the answers); the frontend reads + ``questions[i].prompt``. + """ + + rng = random.Random(_QUESTION_SEED) + questions: list[dict[str, float | str]] = [] + + # --- 10x multiplication (a, b in [2..15]) ---------------------- + seen_mul: set[tuple[int, int]] = set() + while len([q for q in questions if q.get("kind") == "mul"]) < 10: + a = rng.randint(2, 15) + b = rng.randint(2, 15) + key = (a, b) if a <= b else (b, a) + if key in seen_mul: + continue + seen_mul.add(key) + questions.append( + { + "kind": "mul", + "prompt": f"{a} x {b}", + "answer": float(a * b), + } + ) + + # --- 10x percentage of ----------------------------------------- + # Pick percentages and bases that give clean integer products + # (e.g. 15% of 240 = 36). + percent_choices = [5, 10, 15, 20, 25, 30, 40, 50, 60, 75] + seen_pct: set[tuple[int, int]] = set() + while len([q for q in questions if q.get("kind") == "pct"]) < 10: + pct = rng.choice(percent_choices) + # base is a multiple of 20 so most percentages divide cleanly + base = rng.choice([40, 80, 120, 160, 200, 240, 320, 400, 500, 600]) + if (pct, base) in seen_pct: + continue + # Only keep if it gives an integer answer. + product = pct * base + if product % 100 != 0: + continue + seen_pct.add((pct, base)) + questions.append( + { + "kind": "pct", + "prompt": f"{pct}% of {base}", + "answer": float(product // 100), + } + ) + + # --- 5x FX cross-rate ------------------------------------------ + seen_fx: set[tuple[str, str, int]] = set() + while len([q for q in questions if q.get("kind") == "fx"]) < 5: + base_ccy, quote_ccy, rate = rng.choice(_FX_RATES) + amount = rng.choice([100, 200, 400, 500, 800, 1000]) + key_fx = (base_ccy, quote_ccy, amount) + if key_fx in seen_fx: + continue + result = round(amount * rate, 2) + # Only include if it produces a "clean" (<=2 decimal) answer. + if result != round(result, 2): + continue + seen_fx.add(key_fx) + questions.append( + { + "kind": "fx", + "prompt": ( + f"{base_ccy}/{quote_ccy} {rate:.2f} -> " + f"{amount} {base_ccy} in {quote_ccy}" + ), + "answer": float(result), + } + ) + + # --- 5x clean division ----------------------------------------- + seen_div: set[tuple[int, int]] = set() + while len([q for q in questions if q.get("kind") == "div"]) < 5: + divisor = rng.choice([2, 3, 4, 5, 6, 8, 9, 10, 12, 15]) + multiplier = rng.randint(3, 20) + dividend = divisor * multiplier + key_div = (dividend, divisor) + if key_div in seen_div: + continue + seen_div.add(key_div) + questions.append( + { + "kind": "div", + "prompt": f"{dividend} / {divisor}", + "answer": float(multiplier), + } + ) + + # Drop the "kind" tag from the public payload -- it's only there + # to make the deterministic generator easy to read. + return [{"prompt": q["prompt"], "answer": q["answer"]} for q in questions] + + +def _default_rubric() -> dict: + """Rubric for the deterministic mental maths blitz. + + The DeterministicNumericJudge expects ``correct_answers`` as a + parallel list of expected numeric values (the public read API + strips this field; see Task A3). The ``questions`` list holds + the prompt + answer for each question; the public read API + also strips the per-question ``answer`` key. + """ + + questions = _generate_questions() + return { + "categories": [ + {"name": "accuracy", "weight": 0.85, "focus_keywords": ["correct"]}, + {"name": "speed", "weight": 0.15, "focus_keywords": ["seconds"]}, + ], + "questions": questions, + "correct_answers": [q["answer"] for q in questions], + "time_bonus_seconds": _TIME_BONUS_SECONDS, + "benchmark_anchor_score": 72.0, + } + + +def _default_judge_mix() -> dict: + """Single deterministic judge, no LLM. Time bonus configured on + the judge entry so the factory passes it through to + ``DeterministicNumericJudge(time_bonus_seconds=...)``. + """ + + return { + "judges": [ + { + "type": "deterministic_numeric", + "model": "", + "weight": 1.0, + "judge_type": "technical", + "judge_version": "judge.deterministic_numeric.v1", + "time_bonus_seconds": _TIME_BONUS_SECONDS, + }, + ], + "meta": {"type": "meta", "judge_version": "judge.meta.v1"}, + } + + +def _prompt_text() -> str: + return ( + "30 numeric questions in 90 seconds. Mix of multiplication, " + "percentages, FX cross-rates, and clean divisions. Type the " + "answer (number only) and press Enter to advance. Don't dwell " + "on any single question -- the per-question time budget is " + "3 seconds. Time bonus applies if you finish under 60s with " + ">=80% accuracy." + ) + + +# ────────────────────────────────────────────────────────────────────── + + +def upsert_mechanic(db) -> ChallengeMechanic: + mix = _default_judge_mix() + rubric = _default_rubric() + mechanic = db.scalar( + select(ChallengeMechanic).where(ChallengeMechanic.key == _MECHANIC_KEY) + ) + if mechanic is None: + mechanic = ChallengeMechanic( + key=_MECHANIC_KEY, + name="Mental Maths Blitz", + domain_pathway=None, + arena=Arena.TECHNICAL, + default_judge_mix=mix, + default_rubric=rubric, + requires_sandbox=False, + requires_video=False, + time_limit_seconds=_TIME_LIMIT_SECONDS, + ) + db.add(mechanic) + logger.info("created ChallengeMechanic key=%s", _MECHANIC_KEY) + else: + mechanic.default_judge_mix = mix + mechanic.default_rubric = rubric + mechanic.time_limit_seconds = _TIME_LIMIT_SECONDS + logger.info("updated ChallengeMechanic key=%s", _MECHANIC_KEY) + db.flush() + return mechanic + + +def upsert_template(db, *, mechanic: ChallengeMechanic) -> ChallengeTemplate: + rubric = _default_rubric() + template = db.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == _TEMPLATE_SLUG) + ) + if template is None: + template = ChallengeTemplate( + slug=_TEMPLATE_SLUG, + title="Mental Maths Blitz (Phase 3 default)", + description=( + "30 numeric questions in 90 seconds. Deterministic " + "accuracy + speed bonus. No LLM." + ), + arena=Arena.TECHNICAL, + domain_pathway=None, + challenge_type=ChallengeType.MENTAL_MATHS, + difficulty=DifficultyLevel.CORE, + input_type=InputType.NUMERIC_STREAK, + output_type=OutputType.NUMERIC, + scoring_mode=ScoringMode.DETERMINISTIC, + prompt=_prompt_text(), + time_limit_seconds=_TIME_LIMIT_SECONDS, + rubric_definition=rubric, + eligible_modes=[ + ChallengeMode.PRACTICE.value, + ChallengeMode.UNRANKED.value, + ChallengeMode.RANKED.value, + ], + applicable_cycles=["spring_week", "internship", "graduate_role"], + matchmaking_enabled=False, + is_calibration_template=False, + active=True, + version=1, + prompt_version=1, + mechanic_key=mechanic.key, + requires_sandbox=False, + judge_mix_definition=None, # inherit from the mechanic + ) + db.add(template) + logger.info("created ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + else: + template.mechanic_key = mechanic.key + template.rubric_definition = rubric + template.prompt = _prompt_text() + template.time_limit_seconds = _TIME_LIMIT_SECONDS + logger.info("updated ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + db.flush() + return template + + +def reset_existing(db) -> None: + template = db.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == _TEMPLATE_SLUG) + ) + if template is not None: + db.delete(template) + logger.info("deleted ChallengeTemplate slug=%s", _TEMPLATE_SLUG) + mechanic = db.scalar( + select(ChallengeMechanic).where(ChallengeMechanic.key == _MECHANIC_KEY) + ) + if mechanic is not None: + db.delete(mechanic) + logger.info("deleted ChallengeMechanic key=%s", _MECHANIC_KEY) + db.flush() + + +def seed_mental_maths_blitz(db) -> None: + """Reusable entry point for ``seed_reference_data`` (Phase 3 + sub-plan #2 Task A4). + + Drives the same two upserts as :func:`run` but against a caller- + supplied session. The caller owns commit/rollback. Idempotent: a + second call is a no-op for row counts but refreshes the rubric so + answer-key drift propagates. + """ + + mechanic = upsert_mechanic(db) + upsert_template(db, mechanic=mechanic) + db.flush() + + +def run(*, reset: bool) -> None: + configure_engine() + session_factory = get_session_factory() + with session_factory() as session: + if reset: + reset_existing(session) + session.commit() + seed_mental_maths_blitz(session) + session.commit() + + logger.info( + "seed complete: mechanic=%s template=%s", + _MECHANIC_KEY, + _TEMPLATE_SLUG, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--reset", + action="store_true", + help="Drop existing mental_maths_blitz rows before seeding.", + ) + parser.add_argument( + "--user-id", # silently accepted; not used. + default=str(uuid.UUID("11111111-1111-4111-8111-111111111111")), + ) + args = parser.parse_args(argv) + run(reset=args.reset) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/test_seed_behavioural_duel.py b/backend/tests/test_seed_behavioural_duel.py new file mode 100644 index 0000000..31079fd --- /dev/null +++ b/backend/tests/test_seed_behavioural_duel.py @@ -0,0 +1,213 @@ +"""Tests for the Phase 3 sub-plan #2 behavioural_duel seed (Task A1). + +These mirror ``tests/test_seed_phase1_cv_battle.py`` -- imported the +script module against the SQLite session and assert the inserted +rows have the right shape, including: + +- mechanic record exists with the Professional arena + Gemini-primary + dev judge mix. +- one default template linked to the mechanic with the four STAR + questions baked into the rubric. +- three STRONG anchors, each containing four ``---``-separated answers. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from sqlalchemy import select + +from app.contexts.calibration.models.calibration_anchor import CalibrationAnchor +from app.contexts.scoring.services.judge_mix import JudgeMixDefinition +from app.contexts.templates.models.challenge_mechanic import ChallengeMechanic +from app.contexts.templates.models.challenge_template import ChallengeTemplate +from app.core.enums import ( + Arena, + CalibrationTier, + ChallengeType, + InputType, + JudgeType, + OutputType, + ScoringMode, +) + + +def _load_seed_module(): + """Load the script as a module (scripts/ is not a package).""" + + path = ( + Path(__file__).resolve().parent.parent + / "scripts" + / "seed_phase3_behavioural_duel_mechanic.py" + ) + spec = importlib.util.spec_from_file_location( + "seed_phase3_behavioural_duel", path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_seed_creates_mechanic_with_dev_judge_mix(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session, profile="dev") + db_session.commit() + + persisted = db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "behavioural_duel" + ) + ) + assert persisted is not None + assert persisted.id == mechanic.id + assert persisted.arena == Arena.PROFESSIONAL + assert persisted.time_limit_seconds == 4 * 180 + # Default rubric carries the five STAR rubric categories and the + # four behavioural questions for the room composite. + rubric = persisted.default_rubric + category_names = [c["name"] for c in rubric["categories"]] + assert category_names == [ + "star_structure", + "specificity", + "ownership", + "learning", + "fit", + ] + assert len(rubric["questions"]) == 4 + assert rubric["benchmark_anchor_score"] == 70.0 + + # Judge mix validates against the Pydantic schema. + parsed = JudgeMixDefinition.model_validate(persisted.default_judge_mix) + judge_types = [j.type for j in parsed.judges] + assert judge_types == ["rubric_llm", "rubric_llm", "pairwise_llm"] + models = [j.model for j in parsed.judges] + assert all(m.startswith("gemini-") for m in models) + # Every judge attributes to Professional arena. + assert all(j.judge_type == JudgeType.PROFESSIONAL for j in parsed.judges) + + +def test_seed_production_profile_uses_claude_models(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session, profile="production") + db_session.commit() + parsed = JudgeMixDefinition.model_validate(mechanic.default_judge_mix) + models = [j.model for j in parsed.judges] + assert all(m.startswith("claude-") for m in models) + assert all(j.judge_type == JudgeType.PROFESSIONAL for j in parsed.judges) + + +def test_seed_creates_template_linked_to_mechanic(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session, profile="dev") + template = mod.upsert_template(db_session, mechanic=mechanic) + db_session.commit() + + persisted = db_session.scalar( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "behavioural-duel-phase3-default" + ) + ) + assert persisted is not None + assert persisted.id == template.id + assert persisted.mechanic_key == "behavioural_duel" + assert persisted.arena == Arena.PROFESSIONAL + assert persisted.challenge_type == ChallengeType.BEHAVIOURAL_DUEL + assert persisted.input_type == InputType.LONG_TEXT + assert persisted.output_type == OutputType.STRUCTURED + assert persisted.scoring_mode == ScoringMode.JUDGE_LLM + # judge_mix_definition stays NULL so the factory inherits from + # the mechanic. + assert persisted.judge_mix_definition is None + # Prompt mentions all four questions (markdown list). + assert "Tell me about a time you led a team" in persisted.prompt + assert "you failed and what you learned" in persisted.prompt + assert "disagreed with a colleague" in persisted.prompt + assert "above and beyond" in persisted.prompt + + +def test_seed_creates_three_strong_anchors(db_session): + """Three STRONG anchors exist after the seed runs (the conftest + fixture pre-seeds them via ``seed_reference_data`` in Task A4, so + this test confirms the anchors have the expected shape). + """ + + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session, profile="dev") + template = mod.upsert_template(db_session, mechanic=mechanic) + # Re-running upsert_anchors is idempotent — the conftest fixture + # already inserted 3 anchors, so this call returns 0. + new_anchors = mod.upsert_anchors(db_session, template=template) + db_session.commit() + assert new_anchors == 0 + + anchors = list( + db_session.scalars( + select(CalibrationAnchor).where( + CalibrationAnchor.template_id == template.id + ) + ) + ) + assert len(anchors) == 3 + tiers = {a.tier for a in anchors} + assert tiers == {CalibrationTier.STRONG} + + # Each anchor splits into four answer blobs (one per behavioural + # question) joined by the ``---`` separator. + for anchor in anchors: + chunks = anchor.submission_text.split("\n\n---\n\n") + assert len(chunks) == 4, ( + "expected 4 STAR answers per anchor, got " + f"{len(chunks)} for index=" + f"{anchor.model_baseline_scores.get('anchor_index')}" + ) + # Rubric scores cover all five rubric categories. + assert set(anchor.rubric_scores.keys()) == { + "star_structure", + "specificity", + "ownership", + "learning", + "fit", + } + # Strong tier => high scores. + assert all(score >= 80 for score in anchor.rubric_scores.values()) + + # Indexed deterministically so reruns can match. + indices = sorted( + a.model_baseline_scores.get("anchor_index") for a in anchors + ) + assert indices == [0, 1, 2] + + +def test_seed_idempotent_on_rerun(db_session): + """Running the upsert path twice on top of the seeded fixture + leaves row counts unchanged.""" + + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session, profile="dev") + template = mod.upsert_template(db_session, mechanic=mechanic) + first_new = mod.upsert_anchors(db_session, template=template) + db_session.commit() + + mechanic_again = mod.upsert_mechanic(db_session, profile="dev") + template_again = mod.upsert_template(db_session, mechanic=mechanic_again) + second_new = mod.upsert_anchors(db_session, template=template_again) + db_session.commit() + + # Both passes are no-ops because the conftest fixture already + # seeded behavioural_duel via ``seed_reference_data``. + assert first_new == 0 + assert second_new == 0 + assert mechanic_again.id == mechanic.id + assert template_again.id == template.id + + # Still exactly three anchors after both passes. + anchors = list( + db_session.scalars( + select(CalibrationAnchor).where( + CalibrationAnchor.template_id == template.id + ) + ) + ) + assert len(anchors) == 3 diff --git a/backend/tests/test_seed_mental_maths_blitz.py b/backend/tests/test_seed_mental_maths_blitz.py new file mode 100644 index 0000000..2ef13ec --- /dev/null +++ b/backend/tests/test_seed_mental_maths_blitz.py @@ -0,0 +1,178 @@ +"""Tests for the Phase 3 sub-plan #2 mental_maths_blitz seed (Task A2). + +Mirrors ``tests/test_seed_behavioural_duel.py`` / ``test_seed_phase1_cv_battle.py``: +imports the script module against the SQLite test session and asserts the +inserted rows have the right shape. + +The mental_maths_blitz mechanic is deterministic-only: + +- Technical arena, no domain_pathway pinning. +- 30 parallel question/answer pairs baked into ``rubric_definition``. +- Single ``deterministic_numeric`` judge in ``default_judge_mix`` (no LLM). +- No anchors — the judge reads ``correct_answers`` directly off the rubric. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from sqlalchemy import select + +from app.contexts.scoring.services.judge_mix import JudgeMixDefinition +from app.contexts.templates.models.challenge_mechanic import ChallengeMechanic +from app.contexts.templates.models.challenge_template import ChallengeTemplate +from app.core.enums import ( + Arena, + ChallengeType, + InputType, + OutputType, + ScoringMode, +) + + +def _load_seed_module(): + """Load the script as a module (scripts/ is not a Python package).""" + + path = ( + Path(__file__).resolve().parent.parent + / "scripts" + / "seed_phase3_mental_maths_blitz_mechanic.py" + ) + spec = importlib.util.spec_from_file_location( + "seed_phase3_mental_maths_blitz", path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_seed_creates_mechanic_with_deterministic_judge_mix(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session) + db_session.commit() + + persisted = db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "mental_maths_blitz" + ) + ) + assert persisted is not None + assert persisted.id == mechanic.id + assert persisted.arena == Arena.TECHNICAL + assert persisted.time_limit_seconds == 90 + # No domain pinning — Technical arena drill applies broadly. + assert persisted.domain_pathway is None + + # Default rubric carries exactly 30 questions, each with a prompt + # and a numeric (float-able) answer. + rubric = persisted.default_rubric + questions = rubric["questions"] + assert len(questions) == 30 + for entry in questions: + assert isinstance(entry["prompt"], str) + assert entry["prompt"] # non-empty + # Answer is float-able (int or float). + assert isinstance(entry["answer"], (int, float)) + + # correct_answers is the parallel float-only array, exactly 30 long. + correct_answers = rubric["correct_answers"] + assert len(correct_answers) == 30 + assert all(isinstance(v, float) for v in correct_answers) + # And it lines up question-by-question with questions[i].answer. + for q, expected in zip(questions, correct_answers, strict=True): + assert float(q["answer"]) == expected + + +def test_seed_judge_mix_has_single_deterministic_judge(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session) + db_session.commit() + + parsed = JudgeMixDefinition.model_validate(mechanic.default_judge_mix) + assert len(parsed.judges) == 1 + entry = parsed.judges[0] + assert entry.type == "deterministic_numeric" + assert entry.model == "" # No LLM + assert entry.judge_version == "judge.deterministic_numeric.v1" + + +def test_seed_creates_template_linked_to_mechanic(db_session): + mod = _load_seed_module() + mechanic = mod.upsert_mechanic(db_session) + template = mod.upsert_template(db_session, mechanic=mechanic) + db_session.commit() + + persisted = db_session.scalar( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "mental-maths-blitz-phase3-default" + ) + ) + assert persisted is not None + assert persisted.id == template.id + assert persisted.mechanic_key == "mental_maths_blitz" + assert persisted.arena == Arena.TECHNICAL + assert persisted.challenge_type == ChallengeType.MENTAL_MATHS + assert persisted.input_type == InputType.NUMERIC_STREAK + assert persisted.output_type == OutputType.NUMERIC + assert persisted.scoring_mode == ScoringMode.DETERMINISTIC + # judge_mix_definition stays NULL so the factory inherits from the + # mechanic. + assert persisted.judge_mix_definition is None + # Rubric on the template is the same 30-question battery. + assert len(persisted.rubric_definition["questions"]) == 30 + assert len(persisted.rubric_definition["correct_answers"]) == 30 + + +def test_seed_idempotent_on_rerun(db_session): + """Re-running upsert paths leaves row counts unchanged.""" + + mod = _load_seed_module() + mod.upsert_mechanic(db_session) + mod.upsert_template( + db_session, + mechanic=db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "mental_maths_blitz" + ) + ), + ) + db_session.commit() + + initial_mechanic_count = db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "mental_maths_blitz" + ) + ) + initial_template_count = db_session.scalar( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "mental-maths-blitz-phase3-default" + ) + ) + + # Re-run upsert paths — should be a no-op for row counts. + mechanic_again = mod.upsert_mechanic(db_session) + template_again = mod.upsert_template(db_session, mechanic=mechanic_again) + db_session.commit() + + assert mechanic_again.id == initial_mechanic_count.id + assert template_again.id == initial_template_count.id + + # Still exactly one mechanic + one template after rerun. + mechanics = list( + db_session.scalars( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "mental_maths_blitz" + ) + ) + ) + assert len(mechanics) == 1 + templates = list( + db_session.scalars( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "mental-maths-blitz-phase3-default" + ) + ) + ) + assert len(templates) == 1 diff --git a/backend/tests/test_seed_reference_data_phase3.py b/backend/tests/test_seed_reference_data_phase3.py new file mode 100644 index 0000000..d9d78ef --- /dev/null +++ b/backend/tests/test_seed_reference_data_phase3.py @@ -0,0 +1,95 @@ +"""Tests that ``seed_reference_data`` wires in the Phase 3 sub-plan #2 +mechanics (Task A4). + +The conftest fixture already runs ``seed_reference_data`` against an +in-memory SQLite DB. So after the fixture is up, both +``behavioural_duel`` and ``mental_maths_blitz`` mechanics — plus their +default templates — must exist. +""" + +from __future__ import annotations + +from sqlalchemy import select + +from app.contexts.templates.models.challenge_mechanic import ChallengeMechanic +from app.contexts.templates.models.challenge_template import ChallengeTemplate +from app.contexts.templates.models.skill_tree import ( + ChallengeTemplateSkillLink, + SkillNode, +) + + +def test_seed_reference_data_includes_behavioural_duel(db_session): + mechanic = db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "behavioural_duel" + ) + ) + assert mechanic is not None + template = db_session.scalar( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "behavioural-duel-phase3-default" + ) + ) + assert template is not None + assert template.mechanic_key == "behavioural_duel" + + +def test_seed_reference_data_includes_mental_maths_blitz(db_session): + mechanic = db_session.scalar( + select(ChallengeMechanic).where( + ChallengeMechanic.key == "mental_maths_blitz" + ) + ) + assert mechanic is not None + template = db_session.scalar( + select(ChallengeTemplate).where( + ChallengeTemplate.slug == "mental-maths-blitz-phase3-default" + ) + ) + assert template is not None + assert template.mechanic_key == "mental_maths_blitz" + # Rubric still carries the full answer key at the ORM layer (the + # public API strips it). + assert "correct_answers" in template.rubric_definition + assert len(template.rubric_definition["correct_answers"]) == 30 + + +def _linked_skill_slugs(db_session, template_slug: str) -> set[str]: + template = db_session.scalar( + select(ChallengeTemplate).where(ChallengeTemplate.slug == template_slug) + ) + assert template is not None, f"missing template {template_slug}" + links = db_session.scalars( + select(ChallengeTemplateSkillLink).where( + ChallengeTemplateSkillLink.challenge_template_id == template.id + ) + ) + slugs: set[str] = set() + for link in links: + node = db_session.scalar( + select(SkillNode).where(SkillNode.id == link.skill_node_id) + ) + if node is not None: + slugs.add(node.slug) + return slugs + + +def test_behavioural_duel_links_to_professional_skill_nodes(db_session): + """Task A5: behavioural_duel surfaces in the Professional repair + path via the four STAR-anchored leaves.""" + + slugs = _linked_skill_slugs(db_session, "behavioural-duel-phase3-default") + assert "star-storytelling" in slugs + assert "stakeholder-judgement" in slugs + assert "conflict-resolution" in slugs + assert "leadership-framing" in slugs + + +def test_mental_maths_blitz_links_to_technical_skill_nodes(db_session): + """Task A5: mental_maths_blitz feeds numerical-accuracy + + problem-decomposition repair drills.""" + + slugs = _linked_skill_slugs(db_session, "mental-maths-blitz-phase3-default") + assert "numerical-accuracy" in slugs + assert "problem-decomposition" in slugs diff --git a/backend/tests/test_template_read_sanitization.py b/backend/tests/test_template_read_sanitization.py new file mode 100644 index 0000000..8a920d9 --- /dev/null +++ b/backend/tests/test_template_read_sanitization.py @@ -0,0 +1,211 @@ +"""Tests that the public template-read endpoints strip answer keys +(Phase 3 sub-plan #2 Task A3). + +The mental_maths_blitz mechanic embeds a 30-question deterministic +answer battery in ``rubric_definition.correct_answers`` and in each +``rubric_definition.questions[i].answer``. The API must NEVER ship +those fields to the client (cheating prevention). The scoring +pipeline, however, reads ``rubric_definition`` directly off the +SQLAlchemy ORM row via the deterministic judge — so the sanitization +must only apply at the public read boundary, not at the service +layer used by the pipeline. +""" + +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +from app.contexts.scoring.services.judges.base import JudgeContext +from app.contexts.scoring.services.judges.deterministic_numeric import ( + DeterministicNumericJudge, +) +from app.contexts.templates.models.challenge_template import ChallengeTemplate +from app.contexts.templates.services.challenge_templates import ( + _sanitize_rubric_for_public_read, + get_challenge_template_or_error, +) +from app.core.enums import JudgeType + + +def _seed_mental_maths_blitz(db_session) -> ChallengeTemplate: + """Load the seed module and run its upserts against the test DB.""" + + path = ( + Path(__file__).resolve().parent.parent + / "scripts" + / "seed_phase3_mental_maths_blitz_mechanic.py" + ) + spec = importlib.util.spec_from_file_location( + "seed_phase3_mental_maths_blitz", path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + mechanic = module.upsert_mechanic(db_session) + template = module.upsert_template(db_session, mechanic=mechanic) + db_session.commit() + return template + + +# --------------------------------------------------------------------- +# Unit tests on the helper +# --------------------------------------------------------------------- + + +def test_sanitize_rubric_strips_correct_answers_and_per_question_answer() -> None: + rubric: dict[str, Any] = { + "categories": [{"name": "accuracy", "weight": 1.0}], + "questions": [ + {"prompt": "2 x 3", "answer": 6.0}, + {"prompt": "10% of 50", "answer": 5.0}, + ], + "correct_answers": [6.0, 5.0], + "benchmark_anchor_score": 72.0, + } + sanitized = _sanitize_rubric_for_public_read(rubric) + assert sanitized is not None + assert "correct_answers" not in sanitized + for q in sanitized["questions"]: + assert "answer" not in q + assert "prompt" in q + # Untouched fields survive. + assert sanitized["benchmark_anchor_score"] == 72.0 + assert sanitized["categories"] == rubric["categories"] + # Deep copy: original is unchanged. + assert "correct_answers" in rubric + assert "answer" in rubric["questions"][0] + + +def test_sanitize_rubric_handles_none_and_missing_fields() -> None: + assert _sanitize_rubric_for_public_read(None) is None + # Rubric without questions / correct_answers is returned unchanged. + result = _sanitize_rubric_for_public_read({"categories": []}) + assert result == {"categories": []} + + +def test_sanitize_rubric_preserves_non_dict_question_entries() -> None: + """If a future caller stores questions as raw strings (unlikely + but harmless), the helper should not crash.""" + + rubric: dict[str, Any] = {"questions": ["prompt-a", "prompt-b"]} + sanitized = _sanitize_rubric_for_public_read(rubric) + assert sanitized is not None + assert sanitized["questions"] == ["prompt-a", "prompt-b"] + + +# --------------------------------------------------------------------- +# API endpoint tests +# --------------------------------------------------------------------- + + +def test_public_template_read_strips_correct_answers( + client, db_session, auth_headers +) -> None: + template = _seed_mental_maths_blitz(db_session) + + response = client.get( + f"/api/v1/challenge-templates/{template.id}", + headers=auth_headers("user-1"), + ) + assert response.status_code == 200, response.text + body = response.json() + rubric = body["rubric_definition"] + # Top-level answer key stripped. + assert "correct_answers" not in rubric + # Each question entry has prompt but no answer. + assert len(rubric["questions"]) == 30 + for q in rubric["questions"]: + assert isinstance(q["prompt"], str) and q["prompt"] + assert "answer" not in q + + +def test_public_template_list_strips_correct_answers( + client, db_session, auth_headers +) -> None: + template = _seed_mental_maths_blitz(db_session) + + response = client.get( + "/api/v1/challenge-templates", + headers=auth_headers("user-1"), + ) + assert response.status_code == 200, response.text + items = response.json()["items"] + mental_maths_item = next( + (item for item in items if item["id"] == str(template.id)), None + ) + assert mental_maths_item is not None + rubric = mental_maths_item["rubric_definition"] + assert "correct_answers" not in rubric + for q in rubric["questions"]: + assert "answer" not in q + + +def test_public_template_read_preserves_other_fields( + client, db_session, auth_headers +) -> None: + template = _seed_mental_maths_blitz(db_session) + + response = client.get( + f"/api/v1/challenge-templates/{template.id}", + headers=auth_headers("user-1"), + ) + assert response.status_code == 200, response.text + rubric = response.json()["rubric_definition"] + # Categories survive. + assert "categories" in rubric + category_names = {c["name"] for c in rubric["categories"]} + assert {"accuracy", "speed"}.issubset(category_names) + # benchmark_anchor_score survives. + assert rubric["benchmark_anchor_score"] == pytest.approx(72.0) + # time_bonus_seconds survives. + assert rubric.get("time_bonus_seconds") == 60 + + +def test_deterministic_judge_still_scores_correctly_via_orm(db_session) -> None: + """The pipeline reads templates via ORM, not the public read API, + so the deterministic judge still has the full answer key.""" + + template = _seed_mental_maths_blitz(db_session) + + # Refresh via the pipeline's read path — this is what the scoring + # pipeline calls via app.contexts.templates.ports. + full_template = get_challenge_template_or_error( + db_session, template_id=template.id + ) + assert "correct_answers" in full_template.rubric_definition + assert len(full_template.rubric_definition["correct_answers"]) == 30 + + # Build a submission with all-correct answers. + correct_answers = full_template.rubric_definition["correct_answers"] + + @dataclass + class _StubAttempt: + submission_text: str + submission_payload: dict[str, Any] + duration_seconds: int | None = None + + ctx = JudgeContext( + template=full_template, # ORM-backed; rubric is unmodified + anchors=(), + domain_profile=object(), # type: ignore[arg-type] + submission=_StubAttempt( # type: ignore[arg-type] + submission_text="", + submission_payload={"answers": list(correct_answers)}, + duration_seconds=30, # under 60 -> time bonus + ), + request_id="req-det-test", + cache_namespace="ns-det-test", + ) + judge = DeterministicNumericJudge( + judge_type=JudgeType.TECHNICAL, time_bonus_seconds=60 + ) + draft = judge.score(ctx) + # All 30 correct + time bonus -> well above 90. + assert draft.score > 90.0, f"expected high score, got {draft.score}" + assert draft.raw_payload["correct"] == 30 + assert draft.raw_payload["total"] == 30 diff --git a/docs/superpowers/plans/2026-05-14-ascendra-v2-master-plan.md b/docs/superpowers/plans/2026-05-14-ascendra-v2-master-plan.md index 993cd70..d2b5b92 100644 --- a/docs/superpowers/plans/2026-05-14-ascendra-v2-master-plan.md +++ b/docs/superpowers/plans/2026-05-14-ascendra-v2-master-plan.md @@ -1010,15 +1010,15 @@ Use **`frontend-design`** for each page redesign; use the **`vercel:shadcn`** sk **Goal:** users can play CV battle, behavioural duel, mental maths blitz, leetcode medium, and see redesigned replay. -Status: **Sub-plan #1 implementation complete 2026-05-15** ([PR #12](https://github.com/TahaKhanM/QuantiHack/pull/12)) — arena hub + cv_battle end-to-end + replay redesign + Workflow 6-step durable pipeline + dashboard real data + TopBar signOut. Sub-plans #2 (behavioural + mental_maths) and #3 (leetcode + Vercel Sandbox) still to land. +Status: **Sub-plans #1 + #2 implementation complete 2026-05-15** ([PR #12](https://github.com/TahaKhanM/QuantiHack/pull/12) + [PR #13](https://github.com/TahaKhanM/QuantiHack/pull/13)) — arena hub + cv_battle + behavioural_duel + mental_maths_blitz end-to-end + replay redesign + Workflow 6-step durable pipeline + dashboard real data + TopBar signOut + public-template-read answer-key sanitization. Sub-plan #3 (leetcode_medium + Vercel Sandbox) still to land. Deliverables: - ✅ Arena hub page (sub-plan #1). -- ⚠️ Challenge rooms for `cv_battle` (✅ sub-plan #1), `behavioural_duel` + `mental_maths_blitz` (sub-plan #2), `leetcode_medium` (sub-plan #3). +- ⚠️ Challenge rooms for `cv_battle` (✅ sub-plan #1), `behavioural_duel` + `mental_maths_blitz` (✅ sub-plan #2), `leetcode_medium` (sub-plan #3). - ✅ Replay page redesign (sub-plan #1). - ⚠️ Sandbox runner wired for `leetcode_*` — sub-plan #3 ([ADR 0005](../../adr/0005-vercel-sandbox-for-code-execution.md)). - ✅ AI Gateway-routed judges for `cv_battle` (sub-plan #1) and `behavioural_duel` (sub-plan #2) with anchors — direct provider SDKs per [ADR 0014](../../adr/0014-direct-provider-apis-no-vercel-gateway-phase1.md). -- ⚠️ Vercel Workflow runs end-to-end — 6-step durable workflow ships with sub-plan #1 ([ADR 0011](../../adr/0011-vercel-workflow-integration-topology.md)); fully wired for `cv_battle`; `behavioural_duel` / `mental_maths_blitz` / `leetcode_medium` plug in via their sub-plans. +- ⚠️ Vercel Workflow runs end-to-end — 6-step durable workflow ships with sub-plan #1 ([ADR 0011](../../adr/0011-vercel-workflow-integration-topology.md)); fully wired for `cv_battle`, `behavioural_duel`, `mental_maths_blitz`; `leetcode_medium` plugs in via sub-plan #3. ### Phase 4 — Pathway Depth, Wave 1 (4–6 weeks) diff --git a/docs/superpowers/plans/2026-05-15-phase3-behavioural-mental-maths.md b/docs/superpowers/plans/2026-05-15-phase3-behavioural-mental-maths.md new file mode 100644 index 0000000..b555dd2 --- /dev/null +++ b/docs/superpowers/plans/2026-05-15-phase3-behavioural-mental-maths.md @@ -0,0 +1,199 @@ +# Phase 3 sub-plan #2 — `behavioural_duel` + `mental_maths_blitz` (TDD) + +> **For agentic workers:** REQUIRED SUB-SKILL: `superpowers:subagent-driven-development`. Each task uses TDD discipline (failing test → run → implement → run → commit). Spec: `docs/superpowers/specs/2026-05-15-phase3-behavioural-mental-maths-design.md`. Frozen ADRs: 0006, 0008, 0011, 0012, 0013, 0014. + +**Goal:** Ship `behavioural_duel` (Pro arena, 4-Q rubric LLM) and `mental_maths_blitz` (Tech arena, deterministic 30-Q streak) end-to-end. Both reuse the Phase 3 sub-plan #1 pipeline; no scoring engine changes. + +**Architecture:** mirror `seed_phase1_cv_battle_mechanic.py` for both backend seeds. Add two new frontend-v2 composites and extend the `/arena/[mechanic]/play` dispatcher. Sanitize `correct_answers` out of the public template read. + +**Tech Stack:** unchanged from sub-plan #1. + +--- + +## Pre-flight + +### Task 0: Verify worktree + baseline + +- [ ] **Step 1:** `cd /Users/tahakhan/Documents/Work/Projects/QuantiHack/.claude/worktrees/phase-3-behavioural-mental-maths && git status && git log --oneline -3` — confirm branch `phase-3-behavioural-mental-maths` based on `phase-3-arena-hub-cv-battle`. +- [ ] **Step 2:** `cd backend && uv sync && uv run pytest -q | tail -3` — confirm baseline 249 backend tests pass. +- [ ] **Step 3:** `cd ../frontend-v2 && npm install && npm run test 2>&1 | tail -5` — confirm baseline 291 frontend-v2 tests pass. + +--- + +## Stream A — Backend mechanic seeds + sanitization + +### Task A1: `behavioural_duel` mechanic seed + +**Files:** +- Create: `backend/scripts/seed_phase3_behavioural_duel_mechanic.py` +- Create: `backend/tests/test_seed_behavioural_duel.py` + +`seed_behavioural_duel(session, *, profile="dev", reset=False) -> dict`: +- Upserts `ChallengeMechanic(key="behavioural_duel", arena=PROFESSIONAL, ...)` with the rubric + judge mix from spec §3.1. +- Upserts `ChallengeTemplate(template_slug="behavioural-duel-phase3-default", prompt_text=, judge_mix_definition=NULL)` linked to the mechanic. +- Upserts 3 STRONG `CalibrationAnchor` rows (one per "strong" exemplar — stitched 4-answer text blob). +- Mirror cv_battle seed script's CLI entrypoint. + +- [ ] **Step 1 — Failing test:** assert mechanic, template, and 3 anchors created; assert `default_judge_mix` validates as `JudgeMixDefinition` shape (rubric_llm × 2 + pairwise_llm × 1 + meta). +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(scoring): seed behavioural_duel mechanic + Gemini-primary judge mix + 3 anchors (Task A1)` with co-author trailer. + +### Task A2: `mental_maths_blitz` mechanic seed + +**Files:** +- Create: `backend/scripts/seed_phase3_mental_maths_blitz_mechanic.py` +- Create: `backend/tests/test_seed_mental_maths_blitz.py` + +`seed_mental_maths_blitz(session, *, reset=False) -> dict`: +- Upserts `ChallengeMechanic(key="mental_maths_blitz", arena=TECHNICAL, scoring_mode=DETERMINISTIC, ...)`. +- Upserts `ChallengeTemplate(template_slug="mental-maths-blitz-phase3-default", prompt_text="30 numeric questions in 90 seconds", judge_mix_definition=NULL)`. +- `rubric_definition` includes: + - `categories: [{accuracy, weight=0.85}, {speed, weight=0.15}]`. + - `questions: list[{prompt: str, answer: float | int}]` — generate exactly 30 questions with a mix of: 10 × `a × b` (a,b ∈ [2..15]), 10 × `% of ` (pct ∈ {5,10,15,20,25,30,40,50,75}, num ∈ [50..500] multiples of 10), 5 × ` × ` FX (e.g., "EUR/USD 1.10 → 500 EUR in USD"), 5 × ` / ` (clean divisions). Use a deterministic seed so the questions are stable across re-runs. + - `correct_answers: list[float]` (parallel to questions). + - `time_bonus_seconds: 60`. +- No anchors (deterministic mechanic). + +- [ ] **Step 1 — Failing test:** assert mechanic + template created; assert `rubric_definition["correct_answers"]` has exactly 30 entries and matches `questions[i].answer`; assert `default_judge_mix` validates with a single `deterministic_numeric` judge. +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(scoring): seed mental_maths_blitz mechanic + 30 deterministic questions (Task A2)`. + +### Task A3: Sanitize `correct_answers` out of public template reads + +**Files:** +- Modify: `backend/app/contexts/templates/services/challenge_templates.py` (or wherever the read goes through). Add a `_sanitize_rubric_for_public_read(rubric: dict) -> dict` helper that removes `correct_answers` and any `questions[*].answer` field. +- Modify: `backend/app/contexts/templates/api/challenge_templates.py` if the router transforms the rubric directly. +- Modify: `backend/app/contexts/templates/schemas/challenge_template.py` — add a `RubricDefinitionPublic` model that excludes the answer-key fields; use it in `ChallengeTemplateRead`. +- Create: `backend/tests/test_template_read_sanitization.py` + +The mental_maths_blitz template's `rubric_definition` carries the answer key. The cv_battle / behavioural_duel rubrics don't have answer keys (they're rubric-LLM mechanics), so this only affects deterministic-judge templates today — but the sanitization should be defense-in-depth (apply unconditionally on every public read). + +- [ ] **Step 1 — Failing test:** `test_template_read_strips_correct_answers`: seed `mental_maths_blitz` template, fetch via `GET /api/v1/challenge-templates/{id}` as a regular user, assert response's `rubric_definition` doesn't contain `correct_answers` and no `questions[*].answer`. Also assert it DOES contain `questions[*].prompt`. +- [ ] **Step 2 — Failing test:** `test_template_read_passes_through_safe_fields`: cv_battle template still returns its rubric categories, keywords, etc. +- [ ] **Step 3 — Implement.** +- [ ] **Step 4 — Commit:** `feat(templates): sanitize answer-key fields out of public template reads (Task A3)`. + +### Task A4: Wire seeds into `seed_reference_data` + +**Files:** +- Modify: `backend/scripts/seed_reference_data.py` — call `seed_behavioural_duel` and `seed_mental_maths_blitz` (in dev profile). +- Modify: `backend/tests/conftest.py` if it explicitly seeds; verify the fixture uses `seed_reference_data` (it does — per Phase 0 / Phase 1 pattern). + +- [ ] **Step 1 — Failing test:** in a fresh test DB after `seed_reference_data`, assert both mechanics exist with expected keys + arenas. +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(scoring): include behavioural_duel + mental_maths_blitz in seed_reference_data (Task A4)`. + +### Task A5: Quest + skill node hooks (light touch) + +Skip if the existing seed already has Pro / Tech arena leaves the mechanics can link to. If not, add the leaves + link the templates. Don't over-build — just enough that the skill tree page renders the leaf for the new mechanics. + +- [ ] **Step 1 — Failing test:** after seed, assert each new template has at least one linked `SkillNode` ID under its arena. +- [ ] **Step 2 — Implement** (or skip if already covered — note in commit message). +- [ ] **Step 3 — Commit:** `feat(skill): link behavioural_duel + mental_maths_blitz to existing skill leaves (Task A5)`. + +--- + +## Stream B — Frontend-v2 + +### Task B1: `BehaviouralDuelRoom` composite + +**Files:** +- Create: `frontend-v2/components/ascendra/behavioural-duel-room.tsx` +- Create: `frontend-v2/tests/components/behavioural-duel-room.test.tsx` + +Per spec §4.1: 4-question wizard, 3-min per Q countdown, force-advance on timeout, confirmation dialog before final submit. + +Patterns to reuse from `CvBattleRoom`: +- Countdown component (could extract a `` from `cv-battle-room.tsx` into `components/ascendra/_countdown.tsx` if it's not already shared). +- Submit confirmation dialog (use shadcn `Dialog`). +- Character counter (per-answer; optional for behavioural). + +- [ ] **Step 1 — Failing tests:** render with 4 questions; assert Q1 shown initially; press Next → Q2; force-advance after countdown hits 0; Submit on Q4 shows confirmation; confirm fires `onSubmit` with all 4 answers in `[{question, answer}]` shape. +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(frontend-v2): BehaviouralDuelRoom composite (Task B1)`. + +### Task B2: `MentalMathsBlitzRoom` composite + +**Files:** +- Create: `frontend-v2/components/ascendra/mental-maths-blitz-room.tsx` +- Create: `frontend-v2/tests/components/mental-maths-blitz-room.test.tsx` + +Per spec §4.2: keyboard-driven streak. Numeric input, Enter advances, 90 s total countdown, auto-submit on timeout. + +- [ ] **Step 1 — Failing tests:** render with 30 prompts; first prompt visible; type answer → press Enter → next prompt; track `answers` array; auto-submit on 0; manual submit when all 30 answered; submit fires with array of `(number | null)[]` (null for unanswered). +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(frontend-v2): MentalMathsBlitzRoom composite (Task B2)`. + +### Task B3: `/arena/[mechanic]/play` dispatcher update + +**Files:** +- Modify: `frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx` +- Modify (or extend): `frontend-v2/tests/pages/arena-mechanic-play.test.tsx` + +Extend the mechanic-key switch: + +```tsx +switch (template.mechanic_key) { + case "cv_battle": return ; + case "behavioural_duel": + return ; + case "mental_maths_blitz": + return ; + default: return ; +} +``` + +The page extracts `questions` from `template.rubric_definition.questions`. For `mental_maths_blitz`, each question is `{prompt}`; the `answer` field is stripped server-side (Task A3). + +The `onSubmit` handlers serialize the user input into a `submission_payload` matching what the judge expects: +- `behavioural_duel`: `{answers: [{question, answer}, ...]}`. +- `mental_maths_blitz`: `{answers: [number | null, ...]}`. + +- [ ] **Step 1 — Failing tests:** render with mechanic="behavioural-duel" + a 4-question template, assert BehaviouralDuelRoom renders; mechanic="mental-maths-blitz" + 30-question template renders MentalMathsBlitzRoom; submit-handler shape verified. +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(frontend-v2): /arena/[mechanic]/play dispatches behavioural_duel + mental_maths_blitz (Task B3)`. + +### Task B4: Remove from `COMING_SOON_MECHANICS` + +**Files:** +- Modify: `frontend-v2/lib/api/arena.ts` +- Modify: `frontend-v2/tests/lib/api/arena.test.ts` + +Remove `behavioural_duel` and `mental_maths_blitz` from the placeholder list. Update test expectations. + +- [ ] **Step 1 — Failing test:** `useArenaMechanics()` mock returns no placeholder for these two keys; if backend returns them as real templates, the page treats them as live. +- [ ] **Step 2 — Implement.** +- [ ] **Step 3 — Commit:** `feat(frontend-v2): mark behavioural_duel + mental_maths_blitz live on /arena (Task B4)`. + +--- + +## Stream C — Verification + PR + +### Task C1: Trifecta + lint + boundary + +- [ ] `cd backend && uv run pytest -q && uv run ruff check app tests scripts && uv run python scripts/check_context_boundaries.py`. +- [ ] `cd ../frontend-v2 && npm run test && npm run lint && npm run build`. +- [ ] Capture outputs into `docs/superpowers/plans/_verification/2026-05-15-phase3-sub2.md`. + +### Task C2: CLAUDE.md one-liners + +**Files:** +- Modify: `CLAUDE.md` (root) — note Phase 3 sub-plan #2 ships behavioural + mental-maths mechanics + answer-key sanitization. +- Modify: `frontend-v2/CLAUDE.md` — add the two new composites to the "Phase 3 sub-plan #1 additions" section (or rename section to "Phase 3 sub-plans #1 + #2"). +- Modify: `backend/CLAUDE.md` — add a note in "Common Pitfalls" or a new "Template reads sanitize answer keys" subsection. + +- [ ] **Commit:** `docs: CLAUDE.md updates for Phase 3 sub-plan #2 (Task C2)`. + +### Task C3: Open PR + +- [ ] Push branch + open PR via `gh pr create --base phase-3-arena-hub-cv-battle --head phase-3-behavioural-mental-maths` titled `Phase 3 (2/3): behavioural_duel + mental_maths_blitz mechanics`. Body lists what shipped + what's deferred. +- [ ] Note in PR description: re-target to `main` once PR #12 (sub-plan #1) merges. + +--- + +## Out of scope (next sub-plan / phase) + +- `leetcode_medium` + Vercel Sandbox runner — sub-plan #3. +- Real audio/video capture + transcript pipeline for behavioural — Phase 4. +- Live-opponent matching — Phase 5. +- Visual regression baselines for arena hub + new rooms — same follow-up as sub-plan #1. diff --git a/docs/superpowers/specs/2026-05-15-phase3-behavioural-mental-maths-design.md b/docs/superpowers/specs/2026-05-15-phase3-behavioural-mental-maths-design.md new file mode 100644 index 0000000..c70466e --- /dev/null +++ b/docs/superpowers/specs/2026-05-15-phase3-behavioural-mental-maths-design.md @@ -0,0 +1,261 @@ +# Phase 3 sub-plan #2 — `behavioural_duel` + `mental_maths_blitz` + +**Status:** Approved (autonomous /execute-v2 session 2026-05-15). +**Phase:** 3 of 10 (Ascendra v2 master plan §9). +**Stacked on:** Phase 3 sub-plan #1 (PR #12). +**Frozen ADRs:** 0006 (LLM cost / judge mix), 0008 (contexts), 0011 (Workflow topology), 0012 (Judge Protocol), 0013 (Gemini dev), 0014 (direct provider APIs). + +## 1. Goal + +Two more mechanics, both routed through the new pipeline shipped in sub-plan #1: + +1. **`behavioural_duel`** — Professional arena. 4 behavioural questions, text answers only (video/audio deferred to Phase 4 per master plan §2.1 video=stub). Judged by 2 rubric LLM judges + 1 pairwise LLM judge vs. a STRONG anchor (mirrors cv_battle but Professional rather than Profile). +2. **`mental_maths_blitz`** — Technical arena. 30 numeric questions, 90 s total. Judged deterministically (no LLM) using `DeterministicNumericJudge` already in place from Phase 1. + +Both mechanics ship end-to-end: +- Backend: mechanic + template + judge mix + anchors (where applicable). +- Frontend-v2: per-mechanic challenge room composite + dispatched from `/arena/[mechanic]/play`. +- Arena hub: mechanics promoted from `comingSoon` to live. + +## 2. Architecture (delta from sub-plan #1) + +Nothing structural changes. Everything builds on patterns already in place: + +- Mechanic + template seed pattern: mirror `backend/scripts/seed_phase1_cv_battle_mechanic.py`. +- Judge mix for `behavioural_duel`: same shape as cv_battle's dev mix (Gemini-primary) + production mix (Claude-primary). Arena is `professional`. Rubric is STAR-style. +- Judge mix for `mental_maths_blitz`: single `DeterministicNumericJudge`. Rubric `correct_answers` list seeded with 30 deterministic Q→A pairs (e.g., simple multiplication, FX cross-rates, percentages). No anchors needed. +- Pipeline path: both mechanics travel through `run_pipeline_in_process` / the durable workflow exactly like cv_battle. +- Frontend pattern: each mechanic gets a `Room` composite under `frontend-v2/components/ascendra/`, wired into `/arena/[mechanic]/play` via a dispatch switch keyed on `mechanic_key`. + +## 3. Backend changes + +### 3.1 `behavioural_duel` seed + +**Files:** +- Create `backend/scripts/seed_phase3_behavioural_duel_mechanic.py` (mirrors `seed_phase1_cv_battle_mechanic.py`). +- New: 3 STRONG anchors (sample full 4-question behavioural answers per a generic SWE-style brief). + +**Mechanic record:** +```python +ChallengeMechanic( + key="behavioural_duel", + name="Behavioural Duel", + description="Four behavioural interview questions. Type your answers using STAR; multi-judge rubric scoring vs. recruiter-anchored exemplars.", + arena=Arena.PROFESSIONAL, + challenge_type=ChallengeType.BEHAVIOURAL_DUEL, + input_type=InputType.LONG_TEXT, + output_type=OutputType.STRUCTURED, + scoring_mode=ScoringMode.JUDGE_LLM, + default_difficulty=DifficultyLevel.MEDIUM, + default_time_limit_seconds=4 * 180, # 4 questions × 3 min each = 720 s + eligible_modes=[ChallengeMode.PRACTICE, ChallengeMode.UNRANKED, ChallengeMode.RANKED], + pathway_relevance=[ + DomainPathway.SOFTWARE_ENGINEERING, + DomainPathway.INVESTMENT_BANKING, + DomainPathway.CONSULTING, + DomainPathway.QUANT, + DomainPathway.PRODUCT, + DomainPathway.LAW, + DomainPathway.GRADUATE_SCHEMES, + ], + default_judge_mix=_dev_judge_mix_behavioural(), + default_rubric_definition=_default_behavioural_rubric(), +) +``` + +**Rubric:** +```python +{ + "categories": [ + {"name": "star_structure", "weight": 0.25, "focus_keywords": ["situation", "task", "action", "result"]}, + {"name": "specificity", "weight": 0.25, "focus_keywords": ["concrete", "example", "metric"]}, + {"name": "ownership", "weight": 0.20, "focus_keywords": ["I led", "my decision", "I owned"]}, + {"name": "learning", "weight": 0.15, "focus_keywords": ["learned", "next time", "improvement"]}, + {"name": "fit", "weight": 0.15, "focus_keywords": ["values", "team", "role fit"]}, + ], + "questions": [ + "Tell me about a time you led a team through a difficult problem.", + "Tell me about a time you failed and what you learned.", + "Tell me about a time you disagreed with a colleague.", + "Tell me about a time you went above and beyond.", + ], + "benchmark_anchor_score": 70.0, +} +``` + +**Template:** one `ChallengeTemplate` with `template_slug="behavioural-duel-phase3-default"`. `prompt_text` lists the 4 questions in markdown. `judge_mix_definition` left NULL so the factory inherits from the mechanic. + +**Anchors:** 3 STRONG anchors. Each anchor's `submission_text` is a stitched 4-answer text blob (JSON-encoded list of {question, answer} pairs, or just `\n\n---\n\n` separated text — pick the simpler format that the rubric judge can read). + +### 3.2 `mental_maths_blitz` seed + +**Files:** +- Create `backend/scripts/seed_phase3_mental_maths_blitz_mechanic.py`. + +**Mechanic record:** +```python +ChallengeMechanic( + key="mental_maths_blitz", + name="Mental Maths Blitz", + description="30 numeric questions in 90 seconds. Mix of multiplication, percentages, FX cross-rates. Deterministic accuracy + time bonus.", + arena=Arena.TECHNICAL, + challenge_type=ChallengeType.MENTAL_MATHS, + input_type=InputType.NUMERIC_STREAK, + output_type=OutputType.NUMERIC, + scoring_mode=ScoringMode.DETERMINISTIC, + default_difficulty=DifficultyLevel.MEDIUM, + default_time_limit_seconds=90, + eligible_modes=[ChallengeMode.PRACTICE, ChallengeMode.UNRANKED, ChallengeMode.RANKED], + pathway_relevance=[DomainPathway.QUANT, DomainPathway.INVESTMENT_BANKING, DomainPathway.CONSULTING, DomainPathway.SOFTWARE_ENGINEERING, DomainPathway.PRODUCT, DomainPathway.GRADUATE_SCHEMES], + default_judge_mix={ + "judges": [ + { + "type": "deterministic_numeric", + "model": "", + "weight": 1.0, + "judge_type": "technical", + "judge_version": "judge.deterministic_numeric.v1", + }, + ], + "meta": {"type": "meta", "judge_version": "judge.meta.v1"}, + }, + default_rubric_definition=_default_mental_maths_rubric(), +) +``` + +**Rubric (with question + correct-answer key):** +```python +{ + "categories": [ + {"name": "accuracy", "weight": 0.85, "focus_keywords": ["correct"]}, + {"name": "speed", "weight": 0.15, "focus_keywords": ["seconds"]}, + ], + "questions": [ + # 30 generated questions + {"prompt": "12 × 7", "answer": 84.0}, + {"prompt": "15% of 240", "answer": 36.0}, + {"prompt": "EUR/USD 1.10 → 500 EUR in USD", "answer": 550.0}, + # … 27 more + ], + "correct_answers": [84.0, 36.0, 550.0, ...], # parallel to questions + "time_bonus_seconds": 60, # if user finishes under 60s, +5 bonus +} +``` + +The DeterministicNumericJudge reads `rubric.correct_answers` directly and compares against `submission_payload["answers"]` (an array of user-submitted numerics, parallel to the questions). The judge is already in place from Phase 1 — no judge code changes. + +**No anchors** — deterministic scoring doesn't use pairwise. + +**ENUM additions if missing:** the `ChallengeType.BEHAVIOURAL_DUEL`, `ChallengeType.MENTAL_MATHS`, `InputType.LONG_TEXT`, `InputType.NUMERIC_STREAK`, `OutputType.STRUCTURED`, `OutputType.NUMERIC`, `ScoringMode.JUDGE_LLM`, `ScoringMode.DETERMINISTIC` may already exist (used by cv_battle) — verify and only add what's missing. + +### 3.3 Quest + skill node seeding + +Both new mechanics should appear in: +- Default quest catalogue (one quest each: "Win a behavioural duel" / "Hit a 30/30 on mental maths blitz") — append to `backend/scripts/seed_reference_data.py` or the relevant function. +- Skill tree leaves (`SkillNode`) — `behavioural_duel` links to the existing `behavioural.communication` / `behavioural.fit` leaves under the Professional trunk; `mental_maths_blitz` links to `quant.mental_maths` / `commercial.numeracy` under the Technical trunk. Use existing leaf IDs where possible; add new ones if needed. + +If existing seed scripts already cover the necessary skill tree depth, just link the templates to existing leaves via `ChallengeTemplate.linked_skill_node_ids` (or whichever column carries the FK list). + +## 4. Frontend-v2 changes + +### 4.1 `BehaviouralDuelRoom` composite + +**File:** `frontend-v2/components/ascendra/behavioural-duel-room.tsx` + +**Spec:** four-question wizard. Each question gets a 3-min timer + text area. "Next" button advances; "Submit" appears on the last question. Optionally a per-question autosave (skip for Phase 3 since cv_battle's autosave is best-effort already). + +**Props:** +```typescript +type BehaviouralDuelRoomProps = { + attemptId: string; + questions: string[]; // pulled from template.rubric_definition.questions + timeLimitPerQuestionSeconds: number; // 180 + mode: ChallengeMode; + onSubmit: (answers: { question: string; answer: string }[]) => void; +}; +``` + +**Behaviour:** +- One question rendered at a time (step-wise). +- Top bar: "Question N of 4" + per-question countdown. +- When countdown hits 0, force-advance to next question (or submit if last). +- "Skip" and "Next" buttons; "Submit" only on the last question. +- Confirmation dialog before submit (similar to cv-battle-room). + +### 4.2 `MentalMathsBlitzRoom` composite + +**File:** `frontend-v2/components/ascendra/mental-maths-blitz-room.tsx` + +**Spec:** rapid numeric input streak. Per master plan §3.5.5: "Single big number prompt + fast numeric input + streak counter. Designed for keyboard-only." + +**Props:** +```typescript +type MentalMathsBlitzRoomProps = { + attemptId: string; + questions: { prompt: string }[]; // 30 prompts; answers checked server-side + totalTimeSeconds: number; // 90 + mode: ChallengeMode; + onSubmit: (answers: (number | null)[]) => void; +}; +``` + +**Behaviour:** +- One question at a time, full-screen-large prompt. +- `` — Enter submits + advances. +- Wrong answer doesn't pause — user moves on (no immediate feedback per blitz convention). +- 90s overall countdown at the top. +- Streak counter (correct-so-far rendered if backend exposes correctness during stream — for Phase 3 it's deferred-feedback so streak shows "X / Y answered" instead). +- At time-out, auto-submit whatever the user has entered. +- Confirmation skipped (instant submit on time-out or user pressing "Submit" once all 30 answered). + +### 4.3 `/arena/[mechanic]/play` dispatcher + +**File:** `frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx` (modify). + +Currently switches only on `cv-battle`. Extend the switch to also handle `behavioural-duel` and `mental-maths-blitz`. Pseudocode: + +```tsx +switch (mechanicKey) { + case "cv_battle": return ; + case "behavioural_duel": return ; + case "mental_maths_blitz": return ; + default: return ; +} +``` + +The page reads `template.rubric_definition.questions` to build the question list. For mental maths, also reads `template.rubric_definition.correct_answers` (no — that stays server-side; the frontend only ships prompts). + +Wait — `correct_answers` must NOT be sent to the frontend (cheating prevention). The template's read schema must filter them out for non-admin reads. **Check `ChallengeTemplateRead`** to see if `rubric_definition` is included as-is or filtered. If unfiltered, add a filter in the read service (return a sanitized view that excludes `correct_answers` and any other answer keys). + +Actually — let me read the existing template read service to see what it returns. If it returns the raw `rubric_definition`, that's a security bug I need to fix as part of this sub-plan. + +### 4.4 `lib/api/arena.ts` updates + +Update `COMING_SOON_MECHANICS`: +- Remove `behavioural_duel` and `mental_maths_blitz`. +- Leave `leetcode_medium` (still coming in sub-plan #3). + +### 4.5 Updates to existing routes + +- `/arena` should now show 3 live mechanics (cv_battle + behavioural + maths) and 1 coming-soon (leetcode). +- Visual regression baselines for `/arena` should be regenerated to reflect the new count (deferred to a follow-up alongside the Phase 3 visual baseline work — keep them out of this PR per sub-plan #1's deferred-baselines decision). + +## 5. Definition of done + +- Both mechanics seeded + tested. +- 3 STRONG anchors for behavioural_duel. +- Frontend composites + dispatcher + arena hub updates committed. +- Backend test count ≥ 257 (249 baseline + ≥ 8 new from seeds + judge wiring + template-read-filter). +- Frontend-v2 vitest ≥ 305 (291 baseline + ≥ 14 new composites + page tests). +- `correct_answers` filtered out of the public template read response (security fix bundled into this sub-plan). +- Trifecta: ruff + boundary + build + lint clean. +- CLAUDE.md updates for the new mechanics (one-liner each). +- PR opened, stacked on PR #12. + +## 6. Out of scope + +- Video / audio capture for `behavioural_duel` — Phase 4. +- Real `TranscriptAnalysisJudge` wiring — Phase 4. +- Live opponent matching for any of these mechanics — Phase 5. +- Visual regression baselines for protected routes — same follow-up as sub-plan #1. +- E2E happy paths gated on `INTEGRATION=true` — same follow-up. diff --git a/frontend-v2/CLAUDE.md b/frontend-v2/CLAUDE.md index f2442cc..ee5da2a 100644 --- a/frontend-v2/CLAUDE.md +++ b/frontend-v2/CLAUDE.md @@ -174,14 +174,14 @@ new tests. See `.env.local.example` for the current set. -## Phase 3 sub-plan #1 additions (Arena Hub + cv_battle E2E) +## Phase 3 sub-plans #1 + #2 additions (Arena Hub + cv_battle + behavioural_duel + mental_maths_blitz) ### Pages | Route | Type | Notes | |---|---|---| | `/arena` | Client | Arena hub — 3 doors (Profile / Professional / Technical) with mechanic shelves. `useArenaMechanics()` provides the data; only `cv_battle` is wired, the other 3 mechanics show "Coming soon" stubs per `lib/api/arena.ts:COMING_SOON_MECHANICS`. | -| `/arena/[mechanic]/play` | Client | Per-mechanic challenge room. Phase 3 implements `cv-battle` only. On mount: `POST /api/v1/challenge-attempts`. On submit: `POST /api/v1/challenge-attempts/{id}/submit`, redirect to `/replay/{id}`. Autosave is best-effort PUT (backend endpoint pending; errors swallowed). | +| `/arena/[mechanic]/play` | Client | Per-mechanic challenge-room dispatcher. Sub-plans #1+#2 wire `cv-battle`, `behavioural-duel`, `mental-maths-blitz`; `leetcode-medium` still shows "Coming soon" until sub-plan #3. On mount: `POST /api/v1/challenge-attempts`. On submit: `POST /api/v1/challenge-attempts/{id}/submit` with mechanic-specific `submission_payload.answers` shape (`[{question,answer}, ...]` for behavioural, `(number|null)[]` for mental maths, plain text for CV). Autosave is best-effort PUT (backend endpoint pending). Each dispatched component fetches the full `ChallengeTemplateRead` to read `rubric_definition.questions`. | | `/replay/[id]` | Client | Redesigned replay — `` + `` × N + `` (consent-gated) + `` + `` (consent-gated). | | `/replay/[id]/share` | Server | Public Server Component variant. 404 unless backend exposes a consent-gated public read (pending). | | `/dashboard` | Client | Wired to `/api/v1/me/dashboard` via `useDashboardSnapshot()`. Fixture deleted from `lib/fixtures/`; test fixture lives at `tests/fixtures/dashboard.ts`. | @@ -199,11 +199,13 @@ See `.env.local.example` for the current set. - `share-replay-dialog.tsx` — consent-gated share URL dialog. - `top-bar.tsx` — global navigation + signOut dropdown. - `sign-out-wrapper.tsx` — bridges `signOutAction` (server action) to TopBar's `onSignOut` callback. +- `behavioural-duel-room.tsx` — 4-question wizard, per-question 3-min countdown, force-advance on timeout, submit-confirm (sub-plan #2). +- `mental-maths-blitz-room.tsx` — single-prompt numeric streak, autofocus input, Enter advances, single 90s countdown, auto-submit on timeout or completion (sub-plan #2). ### API modules + hooks - `lib/api/dashboard.ts` + `lib/hooks/use-dashboard-snapshot.ts` — `/api/v1/me/dashboard`. -- `lib/api/arena.ts` + `lib/hooks/use-arena-mechanics.ts` — mechanic list (Phase 3 returns seeded cv_battle + 3 coming-soon placeholders). +- `lib/api/arena.ts` + `lib/hooks/use-arena-mechanics.ts` — mechanic list. After sub-plans #1+#2: `cv_battle`, `behavioural_duel`, `mental_maths_blitz` are live (seeded by `seed_reference_data`); `leetcode_medium` is the only remaining `COMING_SOON_MECHANICS` entry (sub-plan #3 promotes it). - `lib/api/replays.ts` + `lib/hooks/use-replay.ts` — `/api/v1/replays/{id}`. - `lib/api/client.ts` gained `apiGet`/`apiPost`/`apiPut` (client-side wrappers with explicit bearer token). - `lib/api/server.ts` — server-only `api()` helper (split out of client.ts so Client Components don't transitively import `next/headers`). diff --git a/frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx b/frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx index f32deda..16250f6 100644 --- a/frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx +++ b/frontend-v2/app/(app)/arena/[mechanic]/play/page.tsx @@ -7,6 +7,8 @@ import { useMutation, useQuery } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; import { CvBattleRoom } from "@/components/ascendra/cv-battle-room"; +import { BehaviouralDuelRoom } from "@/components/ascendra/behavioural-duel-room"; +import { MentalMathsBlitzRoom } from "@/components/ascendra/mental-maths-blitz-room"; import type { ChallengeMode } from "@/components/ascendra/mechanic-card"; import { useArenaMechanics } from "@/lib/hooks/use-arena-mechanics"; import { useSupabaseSession } from "@/lib/auth/use-session"; @@ -16,34 +18,52 @@ import type { components } from "@/lib/api/generated"; /** * `/arena/[mechanic]/play` — the challenge room. * - * Phase 3 wires `cv-battle` only (per spec §5.1). Any other mechanic - * key renders a "Coming soon" placeholder with a link back to - * `/arena`. The flow for cv-battle: + * Phase 3 sub-plan #2 extends the dispatcher to handle three live + * mechanics: `cv_battle`, `behavioural_duel`, and `mental_maths_blitz`. + * Anything else falls through to the "Coming soon" placeholder. * - * 1. Resolve the cv_battle template via `useArenaMechanics`. + * Shared flow across all three mechanics: + * + * 1. Resolve the mechanic's template metadata via `useArenaMechanics`. * 2. Resolve the user's primary domain profile via * `GET /api/v1/domain-profiles` (any pathway works for Phase 3 — * Phase 5's matchmaking layer will pick the matching pathway). * 3. POST `/api/v1/challenge-attempts` on mount to create the * attempt. Mode is `practice` by default; `?mode=ranked` upgrades * to ranked when the template allows it. - * 4. Render `` with the attempt id + the template's - * description as the role brief. + * 4. Hand off to the per-mechanic room composite. * 5. Submit → POST `/api/v1/challenge-attempts/{id}/submit` → * `router.push(/replay/{id})`. - * 6. Autosave → PUT `/api/v1/challenge-attempts/{id}` (best-effort — - * the backend autosave endpoint is wired in a later sub-plan; - * failures are swallowed silently so the user keeps typing). + * + * cv_battle additionally has best-effort autosave; the + * keystroke-driven mechanics (behavioural + mental maths) submit a + * snapshot once and skip autosave. + * + * The behavioural + mental maths rooms need the template's + * `rubric_definition.questions`, which is NOT in the + * `ChallengeTemplateSummaryRead` returned alongside the attempt — so + * those play components fetch the full `ChallengeTemplateRead` via + * `GET /api/v1/challenge-templates/{template_id}`. */ type ChallengeAttemptRead = components["schemas"]["ChallengeAttemptRead"]; type ChallengeAttemptCreate = components["schemas"]["ChallengeAttemptCreate"]; type ChallengeAttemptSubmit = components["schemas"]["ChallengeAttemptSubmit"]; +type ChallengeTemplateRead = components["schemas"]["ChallengeTemplateRead"]; type DomainProfileListResponse = components["schemas"]["DomainProfileListResponse"]; const CV_BATTLE_MECHANIC_KEY = "cv_battle"; const CV_BATTLE_URL_SEGMENT = "cv-battle"; +const BEHAVIOURAL_DUEL_MECHANIC_KEY = "behavioural_duel"; +const BEHAVIOURAL_DUEL_URL_SEGMENT = "behavioural-duel"; +const MENTAL_MATHS_BLITZ_MECHANIC_KEY = "mental_maths_blitz"; +const MENTAL_MATHS_BLITZ_URL_SEGMENT = "mental-maths-blitz"; + +/** Per-question timer ceiling for behavioural_duel (spec §4.1). */ +const BEHAVIOURAL_TIME_PER_QUESTION_S = 180; +/** Overall countdown for mental_maths_blitz (spec §4.2 fallback). */ +const MENTAL_MATHS_DEFAULT_TIME_S = 90; function readMechanicParam(value: string | string[] | undefined): string { if (typeof value === "string") return value; @@ -55,31 +75,31 @@ export default function ArenaMechanicPlayPage() { const params = useParams<{ mechanic: string }>(); const mechanic = readMechanicParam(params?.mechanic); - if (mechanic !== CV_BATTLE_URL_SEGMENT) { - return ; + if (mechanic === CV_BATTLE_URL_SEGMENT) { + return ; } - - return ; + if (mechanic === BEHAVIOURAL_DUEL_URL_SEGMENT) { + return ; + } + if (mechanic === MENTAL_MATHS_BLITZ_URL_SEGMENT) { + return ; + } + return ; } -function CvBattlePlay() { - const router = useRouter(); - const searchParams = useSearchParams(); - const session = useSupabaseSession(); - const token = session?.access_token ?? ""; - - const modeRequested: ChallengeMode = - searchParams?.get("mode") === "ranked" ? "ranked" : "practice"; - - const mechanicsQuery = useArenaMechanics(); - const cvTemplate = React.useMemo(() => { - return mechanicsQuery.data?.live.find( - (m) => m.mechanicKey === CV_BATTLE_MECHANIC_KEY, - ); - }, [mechanicsQuery.data]); +// --------------------------------------------------------------------------- +// Shared helpers for the three live mechanics. Each play component reuses the +// same domain-profile fetch + create-attempt POST + mode-derivation logic, +// only varying in the room composite + submission_payload shape. +// --------------------------------------------------------------------------- - const profilesQuery = useQuery({ - queryKey: ["domain-profiles", session?.user?.id ?? null], +/** + * Loads the user's primary domain profile via React Query. Returns + * the query so callers can surface its loading + error states. + */ +function useDomainProfileQuery(token: string, userId: string | null) { + return useQuery({ + queryKey: ["domain-profiles", userId], queryFn: async ({ signal }) => { if (!token) throw new Error("no_access_token"); const { data, error } = await apiGet( @@ -94,19 +114,33 @@ function CvBattlePlay() { enabled: token.length > 0, staleTime: 60_000, }); +} - // Effective mode honours the template's eligibility — a user - // visiting `?mode=ranked` against a practice-only template falls - // back to practice instead of failing the create call. - const effectiveMode: ChallengeMode = React.useMemo(() => { - if (!cvTemplate) return modeRequested; - if (modeRequested === "ranked" && cvTemplate.eligibleModes.includes("ranked")) { - return "ranked"; - } - return cvTemplate.eligibleModes.includes("practice") ? "practice" : "unranked"; - }, [cvTemplate, modeRequested]); +/** + * Resolves the requested mode against the template's + * `eligibleModes`. `?mode=ranked` only sticks when the template + * permits ranked play — otherwise we degrade gracefully to + * practice / unranked rather than failing the create call. + */ +function deriveEffectiveMode( + template: { eligibleModes: ReadonlyArray } | undefined, + requested: ChallengeMode, +): ChallengeMode { + if (!template) return requested; + if (requested === "ranked" && template.eligibleModes.includes("ranked")) { + return "ranked"; + } + if (template.eligibleModes.includes("practice")) return "practice"; + return "unranked"; +} - const createAttempt = useMutation< +/** + * Build the `useMutation` that creates a new ChallengeAttempt. Same + * call site as the legacy cv_battle path — the new mechanics differ + * only in what they do after the attempt resolves. + */ +function useCreateAttemptMutation(token: string) { + return useMutation< ChallengeAttemptRead, Error, { templateId: string; domainProfileId: string; mode: ChallengeMode } @@ -129,6 +163,63 @@ function CvBattlePlay() { return data; }, }); +} + +/** + * Fetches the full `ChallengeTemplateRead` (which includes + * `rubric_definition`) for a given template id. Used by the + * behavioural + mental maths play components — they need the + * question list, which the summary returned alongside the attempt + * does not expose. + */ +function useFullTemplateQuery(token: string, templateId: string | null) { + return useQuery({ + queryKey: ["challenge-template", templateId], + queryFn: async ({ signal }) => { + if (!token) throw new Error("no_access_token"); + if (!templateId) throw new Error("no_template_id"); + const { data, error } = await apiGet( + `/api/v1/challenge-templates/${templateId}`, + { token, signal }, + ); + if (error || !data) { + throw new Error(error?.code ?? error?.message ?? "template_fetch_failed"); + } + return data; + }, + enabled: token.length > 0 && Boolean(templateId), + staleTime: 5 * 60_000, + }); +} + +// --------------------------------------------------------------------------- +// CV Battle — unchanged behaviour from sub-plan #1. Kept inline here so +// the dispatcher remains a single file. +// --------------------------------------------------------------------------- + +function CvBattlePlay() { + const router = useRouter(); + const searchParams = useSearchParams(); + const session = useSupabaseSession(); + const token = session?.access_token ?? ""; + + const modeRequested: ChallengeMode = + searchParams?.get("mode") === "ranked" ? "ranked" : "practice"; + + const mechanicsQuery = useArenaMechanics(); + const cvTemplate = React.useMemo(() => { + return mechanicsQuery.data?.live.find( + (m) => m.mechanicKey === CV_BATTLE_MECHANIC_KEY, + ); + }, [mechanicsQuery.data]); + + const profilesQuery = useDomainProfileQuery(token, session?.user?.id ?? null); + const effectiveMode = React.useMemo( + () => deriveEffectiveMode(cvTemplate, modeRequested), + [cvTemplate, modeRequested], + ); + + const createAttempt = useCreateAttemptMutation(token); const submitAttempt = useMutation< ChallengeAttemptRead, @@ -241,24 +332,7 @@ function CvBattlePlay() { // No domain profile yet — user needs to finish onboarding. if (!profilesQuery.data?.items?.length) { - return ( -
-

- Finish onboarding first -

-

- You need a domain profile before you can start a CV Battle. Pick - a pathway from onboarding to get going. -

- - - -
- ); + return ; } // Awaiting the create-attempt POST (in-flight or not yet fired). @@ -288,6 +362,412 @@ function CvBattlePlay() { ); } +// --------------------------------------------------------------------------- +// Behavioural Duel — 4-question STAR wizard. Reads questions from the full +// template's rubric_definition; posts back `{answers: [{question, answer}, ...]}`. +// --------------------------------------------------------------------------- + +function BehaviouralDuelPlay() { + const router = useRouter(); + const searchParams = useSearchParams(); + const session = useSupabaseSession(); + const token = session?.access_token ?? ""; + + const modeRequested: ChallengeMode = + searchParams?.get("mode") === "ranked" ? "ranked" : "practice"; + + const mechanicsQuery = useArenaMechanics(); + const mechanicMeta = React.useMemo( + () => + mechanicsQuery.data?.live.find( + (m) => m.mechanicKey === BEHAVIOURAL_DUEL_MECHANIC_KEY, + ), + [mechanicsQuery.data], + ); + + const profilesQuery = useDomainProfileQuery(token, session?.user?.id ?? null); + const effectiveMode = React.useMemo( + () => deriveEffectiveMode(mechanicMeta, modeRequested), + [mechanicMeta, modeRequested], + ); + + const createAttempt = useCreateAttemptMutation(token); + const templateQuery = useFullTemplateQuery( + token, + mechanicMeta?.templateId ?? null, + ); + + const submitAttempt = useMutation< + ChallengeAttemptRead, + Error, + { + attemptId: string; + answers: { question: string; answer: string }[]; + } + >({ + mutationFn: async ({ attemptId, answers }) => { + if (!token) throw new Error("no_access_token"); + const payload: ChallengeAttemptSubmit = { + submission_payload: { answers }, + }; + const { data, error } = await apiPost( + `/api/v1/challenge-attempts/${attemptId}/submit`, + payload, + { token }, + ); + if (error || !data) { + throw new Error(error?.code ?? error?.message ?? "attempt_submit_failed"); + } + return data; + }, + onSuccess: (data) => { + router.push(`/replay/${data.id}`); + }, + }); + + const createTriggered = React.useRef(false); + React.useEffect(() => { + if (createTriggered.current) return; + if (!mechanicMeta) return; + const profile = profilesQuery.data?.items?.[0]; + if (!profile) return; + createTriggered.current = true; + createAttempt.mutate({ + templateId: mechanicMeta.templateId, + domainProfileId: profile.id, + mode: effectiveMode, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [mechanicMeta, profilesQuery.data, effectiveMode]); + + const handleSubmit = React.useCallback( + (answers: { question: string; answer: string }[]) => { + const attempt = createAttempt.data; + if (!attempt) return; + submitAttempt.mutate({ attemptId: attempt.id, answers }); + }, + [createAttempt.data, submitAttempt], + ); + + // Errors surfaced first. + if (mechanicsQuery.isError) { + return ( + void mechanicsQuery.refetch()} + /> + ); + } + if (profilesQuery.isError) { + return ( + void profilesQuery.refetch()} + /> + ); + } + if (templateQuery.isError) { + return ( + void templateQuery.refetch()} + /> + ); + } + if (createAttempt.isError) { + return ( + { + createTriggered.current = false; + createAttempt.reset(); + }} + /> + ); + } + + if ( + mechanicsQuery.isLoading || + profilesQuery.isLoading || + templateQuery.isLoading + ) { + return ; + } + + if (!mechanicMeta) { + return ( + + ); + } + + if (!profilesQuery.data?.items?.length) { + return ; + } + + const questions = extractBehaviouralQuestions( + templateQuery.data?.rubric_definition, + ); + if (!questions.length) { + return ( + + ); + } + + const attempt = createAttempt.data; + if (!attempt) { + return ; + } + + return ( + + ); +} + +// --------------------------------------------------------------------------- +// Mental Maths Blitz — 30-prompt streak with a single overall countdown. +// Posts back `{answers: (number | null)[]}`. +// --------------------------------------------------------------------------- + +function MentalMathsBlitzPlay() { + const router = useRouter(); + const searchParams = useSearchParams(); + const session = useSupabaseSession(); + const token = session?.access_token ?? ""; + + const modeRequested: ChallengeMode = + searchParams?.get("mode") === "ranked" ? "ranked" : "practice"; + + const mechanicsQuery = useArenaMechanics(); + const mechanicMeta = React.useMemo( + () => + mechanicsQuery.data?.live.find( + (m) => m.mechanicKey === MENTAL_MATHS_BLITZ_MECHANIC_KEY, + ), + [mechanicsQuery.data], + ); + + const profilesQuery = useDomainProfileQuery(token, session?.user?.id ?? null); + const effectiveMode = React.useMemo( + () => deriveEffectiveMode(mechanicMeta, modeRequested), + [mechanicMeta, modeRequested], + ); + + const createAttempt = useCreateAttemptMutation(token); + const templateQuery = useFullTemplateQuery( + token, + mechanicMeta?.templateId ?? null, + ); + + const submitAttempt = useMutation< + ChallengeAttemptRead, + Error, + { attemptId: string; answers: (number | null)[] } + >({ + mutationFn: async ({ attemptId, answers }) => { + if (!token) throw new Error("no_access_token"); + const payload: ChallengeAttemptSubmit = { + submission_payload: { answers }, + }; + const { data, error } = await apiPost( + `/api/v1/challenge-attempts/${attemptId}/submit`, + payload, + { token }, + ); + if (error || !data) { + throw new Error(error?.code ?? error?.message ?? "attempt_submit_failed"); + } + return data; + }, + onSuccess: (data) => { + router.push(`/replay/${data.id}`); + }, + }); + + const createTriggered = React.useRef(false); + React.useEffect(() => { + if (createTriggered.current) return; + if (!mechanicMeta) return; + const profile = profilesQuery.data?.items?.[0]; + if (!profile) return; + createTriggered.current = true; + createAttempt.mutate({ + templateId: mechanicMeta.templateId, + domainProfileId: profile.id, + mode: effectiveMode, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [mechanicMeta, profilesQuery.data, effectiveMode]); + + const handleSubmit = React.useCallback( + (answers: (number | null)[]) => { + const attempt = createAttempt.data; + if (!attempt) return; + submitAttempt.mutate({ attemptId: attempt.id, answers }); + }, + [createAttempt.data, submitAttempt], + ); + + if (mechanicsQuery.isError) { + return ( + void mechanicsQuery.refetch()} + /> + ); + } + if (profilesQuery.isError) { + return ( + void profilesQuery.refetch()} + /> + ); + } + if (templateQuery.isError) { + return ( + void templateQuery.refetch()} + /> + ); + } + if (createAttempt.isError) { + return ( + { + createTriggered.current = false; + createAttempt.reset(); + }} + /> + ); + } + + if ( + mechanicsQuery.isLoading || + profilesQuery.isLoading || + templateQuery.isLoading + ) { + return ; + } + + if (!mechanicMeta) { + return ( + + ); + } + + if (!profilesQuery.data?.items?.length) { + return ; + } + + const questions = extractMentalMathsPrompts( + templateQuery.data?.rubric_definition, + ); + if (!questions.length) { + return ( + + ); + } + + const attempt = createAttempt.data; + if (!attempt) { + return ; + } + + const totalTimeSeconds = + attempt.template?.time_limit_seconds ?? + mechanicMeta.timeLimitSeconds ?? + MENTAL_MATHS_DEFAULT_TIME_S; + + return ( + + ); +} + +// --------------------------------------------------------------------------- +// Rubric extraction. `rubric_definition` is typed as a closed +// `RubricDefinition` in the generated OpenAPI client, but the backend +// stores per-mechanic question shapes underneath. The page consumes +// those via a loose-cast helper so a backend schema drift logs a +// warning rather than a Type error wall. +// --------------------------------------------------------------------------- + +function extractBehaviouralQuestions( + rubric: ChallengeTemplateRead["rubric_definition"] | undefined, +): string[] { + if (!rubric) return []; + const raw = (rubric as { questions?: unknown }).questions; + if (!Array.isArray(raw)) return []; + return raw.filter((q): q is string => typeof q === "string"); +} + +function extractMentalMathsPrompts( + rubric: ChallengeTemplateRead["rubric_definition"] | undefined, +): { prompt: string }[] { + if (!rubric) return []; + const raw = (rubric as { questions?: unknown }).questions; + if (!Array.isArray(raw)) return []; + return raw + .map((q) => { + if (typeof q !== "object" || q === null) return null; + const prompt = (q as { prompt?: unknown }).prompt; + return typeof prompt === "string" ? { prompt } : null; + }) + .filter((q): q is { prompt: string } => q !== null); +} + +// --------------------------------------------------------------------------- +// Shared UI fragments for the placeholder / loading / error / no-profile +// states. Same component the cv_battle path used in sub-plan #1; lifted out +// so all three mechanics render identical chrome. +// --------------------------------------------------------------------------- + +function NoDomainProfile({ mechanicLabel }: { mechanicLabel: string }) { + return ( +
+

+ Finish onboarding first +

+

+ You need a domain profile before you can start a {mechanicLabel}. + Pick a pathway from onboarding to get going. +

+ + + +
+ ); +} + function ComingSoonPlaceholder({ mechanic, reason, @@ -295,6 +775,18 @@ function ComingSoonPlaceholder({ mechanic: string; reason?: "not_seeded"; }) { + const seedMessage = (() => { + if (mechanic === CV_BATTLE_URL_SEGMENT) { + return "The CV Battle template isn’t seeded in this environment yet — ask backend to run seed_reference_data."; + } + if (mechanic === BEHAVIOURAL_DUEL_URL_SEGMENT) { + return "The Behavioural Duel template isn’t seeded in this environment yet — ask backend to run seed_reference_data."; + } + if (mechanic === MENTAL_MATHS_BLITZ_URL_SEGMENT) { + return "The Mental Maths Blitz template isn’t seeded in this environment yet — ask backend to run seed_reference_data."; + } + return "This mechanic isn’t seeded in this environment yet — ask backend to run seed_reference_data."; + })(); return (

{reason === "not_seeded" - ? "The CV Battle template isn’t seeded in this environment yet — ask backend to run seed_reference_data." - : `The ${mechanic || "selected"} mechanic isn’t live yet. Phase 3 wires CV Battle; the other arenas ship in later sub-plans.`} + ? seedMessage + : `The ${mechanic || "selected"} mechanic isn’t live yet. Phase 3 wires CV Battle, Behavioural Duel and Mental Maths Blitz; the other arenas ship in later sub-plans.`}

diff --git a/frontend-v2/components/ascendra/_countdown.tsx b/frontend-v2/components/ascendra/_countdown.tsx new file mode 100644 index 0000000..027b295 --- /dev/null +++ b/frontend-v2/components/ascendra/_countdown.tsx @@ -0,0 +1,70 @@ +"use client"; + +import * as React from "react"; +import { cn } from "@/lib/utils"; + +const DEFAULT_WARN_THRESHOLD_S = 30; + +/** + * Formats a seconds-only countdown as `mm:ss`. Clamps negatives to 0. + * + * Lives next to `Countdown` (rather than `lib/`) because the only + * consumers are challenge rooms in `components/ascendra/`. + */ +export function formatCountdown(seconds: number): string { + const safe = Math.max(0, seconds); + const mm = String(Math.floor(safe / 60)).padStart(2, "0"); + const ss = String(safe % 60).padStart(2, "0"); + return `${mm}:${ss}`; +} + +export interface CountdownProps { + /** Remaining time, in whole seconds. Negatives render as 00:00. */ + seconds: number; + /** Threshold (in seconds) at which to flip into the warning palette. */ + warnThresholdSeconds?: number; + /** + * Optional aria-label override — defaults to "Time remaining mm:ss" + * which is the right shape for both `BehaviouralDuelRoom`'s + * per-question timer and `CvBattleRoom`'s overall timer. + */ + ariaLabel?: string; + className?: string; +} + +/** + * Pill-shaped countdown with warning styling once `seconds` drops to + * `warnThresholdSeconds` or below. Surfaces `data-testid="countdown"` + * + `data-warn` so tests can assert state without depending on the + * specific styling. + * + * The component is purely presentational — it doesn't drive its own + * tick interval. Rooms own the timer state so they can also fire + * side-effects (force-advance, force-submit) when it hits zero. + */ +export function Countdown({ + seconds, + warnThresholdSeconds = DEFAULT_WARN_THRESHOLD_S, + ariaLabel, + className, +}: CountdownProps) { + const warn = seconds <= warnThresholdSeconds; + const label = ariaLabel ?? `Time remaining ${formatCountdown(seconds)}`; + return ( + + {formatCountdown(seconds)} + + ); +} diff --git a/frontend-v2/components/ascendra/behavioural-duel-room.tsx b/frontend-v2/components/ascendra/behavioural-duel-room.tsx new file mode 100644 index 0000000..a60ae4f --- /dev/null +++ b/frontend-v2/components/ascendra/behavioural-duel-room.tsx @@ -0,0 +1,250 @@ +"use client"; + +import * as React from "react"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Countdown } from "@/components/ascendra/_countdown"; +import { cn } from "@/lib/utils"; + +import type { ChallengeMode } from "@/components/ascendra/mechanic-card"; + +export interface BehaviouralDuelRoomProps { + attemptId: string; + /** Behavioural prompts pulled from `template.rubric_definition.questions`. */ + questions: string[]; + /** Per-question countdown ceiling. Spec §4.1 ships 180s. */ + timeLimitPerQuestionSeconds: number; + mode: ChallengeMode; + /** Fires with one `{question, answer}` per prompt (empty string for skipped). */ + onSubmit: (answers: { question: string; answer: string }[]) => void; +} + +/** + * Behavioural duel — a 4-question STAR-style wizard with a per-question + * timer + final confirmation dialog. The room owns the step index, + * the per-step answer state, and a single setInterval that resets on + * step changes. The timer drives `force-advance` (or `force-submit` + * on the last step) when it hits zero. + * + * Patterns mirror `CvBattleRoom`: shared `Countdown` for warning + * styling, shadcn `Dialog` for the confirm step. + */ +export function BehaviouralDuelRoom({ + attemptId, + questions, + timeLimitPerQuestionSeconds, + mode, + onSubmit, +}: BehaviouralDuelRoomProps) { + const total = questions.length; + const [step, setStep] = React.useState(0); + const [answers, setAnswers] = React.useState(() => + questions.map(() => ""), + ); + const [draft, setDraft] = React.useState(""); + const [secondsRemaining, setSecondsRemaining] = React.useState( + timeLimitPerQuestionSeconds, + ); + const [confirmOpen, setConfirmOpen] = React.useState(false); + + // Refs so the timer's tick callback always sees the freshest state + // without re-subscribing every keystroke. + const draftRef = React.useRef(draft); + const answersRef = React.useRef(answers); + const stepRef = React.useRef(step); + const onSubmitRef = React.useRef(onSubmit); + const submittedRef = React.useRef(false); + React.useLayoutEffect(() => { + draftRef.current = draft; + answersRef.current = answers; + stepRef.current = step; + onSubmitRef.current = onSubmit; + }); + + const isLast = step === total - 1; + + // Build the final {question, answer}[] payload from a snapshot of + // the answers array — used both on user-confirm + on time-expiry + // auto-submit so the shape is identical regardless of trigger. + const buildPayload = React.useCallback( + (final: string[]) => + questions.map((q, i) => ({ question: q, answer: final[i] ?? "" })), + [questions], + ); + + // Save current draft, advance the step, reset the timer. Splits the + // "save" vs "skip" path so callers don't pass a magic flag. + const advance = React.useCallback( + (saveDraft: boolean) => { + const currentStep = stepRef.current; + const nextAnswers = saveDraft + ? answersRef.current.map((a, i) => + i === currentStep ? draftRef.current : a, + ) + : answersRef.current; + if (currentStep + 1 >= total) { + // Last step reached — auto-submit immediately (time-expiry path). + if (submittedRef.current) return; + submittedRef.current = true; + onSubmitRef.current(buildPayload(nextAnswers)); + return; + } + setAnswers(nextAnswers); + setStep(currentStep + 1); + setDraft(""); + setSecondsRemaining(timeLimitPerQuestionSeconds); + }, + [buildPayload, timeLimitPerQuestionSeconds, total], + ); + + // Countdown — restart whenever the step changes. Forces save+advance + // on timeout; on the last step force-save then auto-submit. + React.useEffect(() => { + if (submittedRef.current) return; + const handle = setInterval(() => { + setSecondsRemaining((prev) => { + if (prev <= 1) { + clearInterval(handle); + // Save the in-progress draft on timeout so the user doesn't + // lose their work just because the timer ran out. + advance(true); + return 0; + } + return prev - 1; + }); + }, 1_000); + return () => clearInterval(handle); + }, [step, advance]); + + function handleNext() { + advance(true); + } + function handleSkip() { + advance(false); + } + function handleSubmitClick() { + setConfirmOpen(true); + } + function handleConfirmSubmit() { + if (submittedRef.current) return; + submittedRef.current = true; + setConfirmOpen(false); + const final = answersRef.current.map((a, i) => + i === stepRef.current ? draftRef.current : a, + ); + onSubmit(buildPayload(final)); + } + + return ( +
+
+
+

+ Behavioural Duel +

+

+ {mode} · Question {step + 1} of {total} +

+
+ +
+ +
+
+

+ {questions[step]} +

+
+ + +