feat(api): add scorecard extraction and persistence

Add helprs-scorecard JSON block extraction from Claude session output.
New scorecard.py module parses the fenced code block, validates structure
(3 dimensions, 0-10 range). Scorecard is persisted to container_sessions
table after mark_completed. New GET /sessions/{id}/scorecard endpoint.

Author: mariuspruvot

apps/api/alembic/versions/a2b3c4d5e6f8_add_scorecard_columns.py

@@ -0,0 +1,26 @@
+"""Add scorecard and xp_earned columns to container_sessions.
+
+Revision ID: a2b3c4d5e6f8
+Revises: a1b2c3d4e5f7
+Create Date: 2026-04-20
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "a2b3c4d5e6f8"
+down_revision: str | None = "a1b2c3d4e5f7"
+branch_labels: tuple[str, ...] | None = None
+depends_on: tuple[str, ...] | None = None
+
+
+def upgrade() -> None:
+    op.add_column("container_sessions", sa.Column("scorecard", postgresql.JSONB(), nullable=True))
+    op.add_column("container_sessions", sa.Column("xp_earned", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("container_sessions", "xp_earned")
+    op.drop_column("container_sessions", "scorecard")

apps/api/src/helprs/modules/container/models.py

@@ -48,6 +48,8 @@ class ContainerSession(Base):
     )
     started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    scorecard: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    xp_earned: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
 
 class SessionEvent(Base):

apps/api/src/helprs/modules/container/router.py

@@ -1,13 +1,17 @@
 """Container session API routes."""
 
 import json
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 import structlog
 from fastapi import APIRouter, Depends, Request
 from fastapi.responses import StreamingResponse
 from sqlalchemy import select
 
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
 from helprs.core.config import get_settings
 from helprs.core.database import get_db_context
 from helprs.core.dependencies import DbSession, GetSettings, get_current_user
@@ -19,6 +23,7 @@
 from helprs.modules.container.schemas import (
     ContainerSessionResponse,
     CreateSessionRequest,
+    ScorecardResponse,
     SendMessageRequest,
     SendMessageResponse,
     SessionEventResponse,
@@ -181,6 +186,7 @@ async def _event_stream():
                     if completed.status == ContainerStatus.FAILED:
                         msg = "Session failed."
                     elif completed.status == ContainerStatus.COMPLETED:
+                        await _persist_scorecard(db_ctx, completed)
                         await _post_results_comment(session_id, cs)
             except Exception:
                 pass  # Best effort; cleanup task handles stragglers
@@ -224,6 +230,25 @@ async def get_session_events_endpoint(
     )
 
 
+@router.get("/sessions/{session_id}/scorecard", response_model=ScorecardResponse)
+@limiter.limit("30/minute")
+async def get_session_scorecard(
+    session_id: UUID,
+    request: Request,
+    db: DbSession,
+    settings: GetSettings,
+    user=Depends(get_current_user),  # noqa: B008
+):
+    """Get the parsed scorecard for a completed session."""
+    cs = await get_session_or_404(db, session_id)
+    await verify_session_access(user, cs, db, settings)
+    return ScorecardResponse(
+        session_id=session_id,
+        scorecard=cs.scorecard,
+        xp_earned=cs.xp_earned,
+    )
+
+
 @router.post("/sessions/{session_id}/message", response_model=SendMessageResponse)
 @limiter.limit("30/minute")
 async def send_session_message(
@@ -281,6 +306,33 @@ async def stop_container_session(
     )
 
 
+async def _persist_scorecard(db: "AsyncSession", cs: ContainerSession) -> None:  # noqa: UP037
+    """Extract and persist the helprs-scorecard from session events (best-effort)."""
+    from helprs.modules.container.scorecard import extract_scorecard
+
+    events = await get_session_events(db, cs.id)
+    # Walk events in reverse to find the last assistant text block
+    for event in reversed(events):
+        data = event.data
+        if data.get("type") == "assistant" and isinstance(data.get("message"), dict):
+            for block in reversed(data["message"].get("content", [])):
+                if block.get("type") == "text":
+                    scorecard = extract_scorecard(block.get("text", ""))
+                    if scorecard:
+                        cs.scorecard = scorecard
+                        await db.flush()
+                        return
+    # Also check result events which carry the final text
+    for event in reversed(events):
+        data = event.data
+        if data.get("type") == "result" and isinstance(data.get("result"), str):
+            scorecard = extract_scorecard(data["result"])
+            if scorecard:
+                cs.scorecard = scorecard
+                await db.flush()
+                return
+
+
 async def _post_results_comment(session_id: UUID, cs: ContainerSession) -> None:
     """Post challenge-me results to the PR as a GitHub comment (best-effort).
 

apps/api/src/helprs/modules/container/schemas.py

@@ -54,6 +54,8 @@ class ContainerSessionResponse(BaseModel):
     status: str
     started_at: datetime | None
     completed_at: datetime | None
+    scorecard: dict | None = None
+    xp_earned: int | None = None
     created_at: datetime
     updated_at: datetime
 
@@ -103,3 +105,11 @@ class SessionEventsListResponse(BaseModel):
     session_id: uuid.UUID
     events: list[SessionEventResponse]
     total: int
+
+
+class ScorecardResponse(BaseModel):
+    """Parsed scorecard for a completed session."""
+
+    session_id: uuid.UUID
+    scorecard: dict | None
+    xp_earned: int | None

apps/api/src/helprs/modules/container/scorecard.py

@@ -0,0 +1,47 @@
+"""Scorecard extraction from Claude session output.
+
+Skills emit a structured JSON scorecard wrapped in a fenced code block
+tagged `helprs-scorecard`. This module parses it from the last assistant
+message text.
+"""
+
+import json
+import re
+
+SCORECARD_PATTERN = re.compile(
+    r"```helprs-scorecard\s*\n(.*?)\n```",
+    re.DOTALL,
+)
+
+REQUIRED_FIELDS = {"skill", "version", "dimensions", "summary"}
+
+
+def extract_scorecard(text: str) -> dict | None:
+    """Extract the helprs-scorecard JSON block from text.
+
+    Returns the parsed dict if found and valid, None otherwise.
+    """
+    match = SCORECARD_PATTERN.search(text)
+    if not match:
+        return None
+
+    try:
+        data = json.loads(match.group(1))
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+    if not isinstance(data, dict):
+        return None
+
+    if not REQUIRED_FIELDS.issubset(data.keys()):
+        return None
+
+    dims = data.get("dimensions")
+    if not isinstance(dims, dict) or len(dims) != 3:
+        return None
+
+    for value in dims.values():
+        if not isinstance(value, (int, float)) or value < 0 or value > 10:
+            return None
+
+    return data

apps/api/tests/modules/container/test_scorecard.py

@@ -0,0 +1,115 @@
+"""Tests for scorecard extraction from Claude session output."""
+
+from helprs.modules.container.scorecard import extract_scorecard
+
+
+VALID_SCORECARD = """\
+Some assistant text before the scorecard.
+
+```helprs-scorecard
+{
+  "skill": "challenge-me",
+  "version": 1,
+  "questions_asked": 3,
+  "questions_answered": 3,
+  "dimensions": {
+    "depth": 8,
+    "clarity": 7,
+    "rigor": 6
+  },
+  "summary": "Strong understanding of failure modes.",
+  "highlights": ["Good instinct on idempotency"]
+}
+```
+
+Some text after.
+"""
+
+
+def test_extract_valid_scorecard():
+    result = extract_scorecard(VALID_SCORECARD)
+    assert result is not None
+    assert result["skill"] == "challenge-me"
+    assert result["version"] == 1
+    assert result["dimensions"] == {"depth": 8, "clarity": 7, "rigor": 6}
+    assert result["summary"] == "Strong understanding of failure modes."
+    assert result["highlights"] == ["Good instinct on idempotency"]
+
+
+def test_extract_returns_none_when_no_block():
+    assert extract_scorecard("Just some regular text without a scorecard") is None
+
+
+def test_extract_returns_none_for_invalid_json():
+    text = "```helprs-scorecard\n{invalid json}\n```"
+    assert extract_scorecard(text) is None
+
+
+def test_extract_returns_none_for_missing_required_fields():
+    text = '```helprs-scorecard\n{"skill": "challenge-me"}\n```'
+    assert extract_scorecard(text) is None
+
+
+def test_extract_returns_none_for_wrong_dimension_count():
+    text = """```helprs-scorecard
+{
+  "skill": "challenge-me",
+  "version": 1,
+  "dimensions": {"depth": 8, "clarity": 7},
+  "summary": "Missing one dimension"
+}
+```"""
+    assert extract_scorecard(text) is None
+
+
+def test_extract_returns_none_for_out_of_range_scores():
+    text = """```helprs-scorecard
+{
+  "skill": "challenge-me",
+  "version": 1,
+  "dimensions": {"depth": 11, "clarity": 7, "rigor": 6},
+  "summary": "Score out of range"
+}
+```"""
+    assert extract_scorecard(text) is None
+
+
+def test_extract_returns_none_for_negative_scores():
+    text = """```helprs-scorecard
+{
+  "skill": "challenge-me",
+  "version": 1,
+  "dimensions": {"depth": -1, "clarity": 7, "rigor": 6},
+  "summary": "Negative score"
+}
+```"""
+    assert extract_scorecard(text) is None
+
+
+def test_extract_handles_float_scores():
+    text = """```helprs-scorecard
+{
+  "skill": "eli5",
+  "version": 1,
+  "dimensions": {"accuracy": 7.5, "simplicity": 8.0, "completeness": 6.5},
+  "summary": "Good explanation"
+}
+```"""
+    result = extract_scorecard(text)
+    assert result is not None
+    assert result["dimensions"]["accuracy"] == 7.5
+
+
+def test_extract_handles_different_skill_dimensions():
+    text = """```helprs-scorecard
+{
+  "skill": "pair-debug",
+  "version": 1,
+  "dimensions": {"detection": 9, "methodology": 7, "speed": 8},
+  "summary": "Found the bug quickly"
+}
+```"""
+    result = extract_scorecard(text)
+    assert result is not None
+    assert result["skill"] == "pair-debug"
+    assert set(result["dimensions"].keys()) == {"detection", "methodology", "speed"}