Skip to content

Commit a25691e

Browse files
authored
feat: data lineage view (#107)
Implements GET /api/lineage, lineage dialog, and wiring from KPIs, risk-alert chart, and student roster. Row-level source data respects roster RBAC. Closes #107
1 parent ba5b924 commit a25691e

11 files changed

Lines changed: 1135 additions & 24 deletions

File tree

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
import { type NextRequest, NextResponse } from "next/server"
2+
import { getPool } from "@/lib/db"
3+
import { canAccess, type Role } from "@/lib/roles"
4+
import {
5+
buildStudentLevelDashboardConditionsAliased,
6+
optionalDashboardFilterRecord,
7+
type DashboardFilterParams,
8+
} from "@/lib/dashboard-filters"
9+
import {
10+
LINEAGE_STUDENT_LEVEL_SCHEMA_IDS,
11+
isLineageMetricId,
12+
lineageStepsForMetric,
13+
isRosterLineageField,
14+
rosterFieldLineageLabel,
15+
type LineageMetricId,
16+
} from "@/lib/lineage-config"
17+
import { SCHEMAS } from "@/lib/upload-schemas"
18+
19+
const MAX_PAGE_SIZE = 100
20+
21+
type LineageUploadHistoryRow = {
22+
id: string
23+
filename: string
24+
file_type: string
25+
uploaded_at: Date
26+
status: string
27+
user_email: string | null
28+
rows_inserted: number
29+
rows_skipped: number
30+
error_count: number
31+
has_validation_report: boolean
32+
}
33+
34+
const METRIC_LABEL: Record<LineageMetricId, string> = {
35+
overall_retention: "Overall retention rate",
36+
avg_predicted_retention: "Average predicted retention",
37+
high_critical_risk_count: "Students at high / critical risk",
38+
avg_course_completion: "Average course completion",
39+
risk_alert_segment: "Risk alert segment",
40+
retention_risk_segment: "Retention risk segment",
41+
roster_cell: "Roster value",
42+
}
43+
44+
function metricDescription(metric: LineageMetricId, category: string | null): string {
45+
switch (metric) {
46+
case "risk_alert_segment":
47+
return category
48+
? `Students in the “${category}” slice of the risk alert distribution.`
49+
: "A slice of the risk alert distribution."
50+
case "retention_risk_segment":
51+
return category
52+
? `Students in the “${category}” retention-probability band.`
53+
: "A slice of the retention risk distribution."
54+
case "roster_cell":
55+
return "Single student field as shown on the roster."
56+
default:
57+
return METRIC_LABEL[metric]
58+
}
59+
}
60+
61+
function appendMetricPredicate(
62+
metric: LineageMetricId,
63+
category: string | null,
64+
studentGuid: string | null,
65+
conditions: string[],
66+
values: unknown[]
67+
): { error?: string } {
68+
switch (metric) {
69+
case "high_critical_risk_count":
70+
conditions.push(`s.at_risk_alert IN ('HIGH', 'URGENT')`)
71+
return {}
72+
case "risk_alert_segment":
73+
if (!category?.trim()) return { error: "category is required for risk_alert_segment" }
74+
values.push(category.trim())
75+
conditions.push(`s.at_risk_alert = $${values.length}`)
76+
return {}
77+
case "retention_risk_segment":
78+
if (!category?.trim()) return { error: "category is required for retention_risk_segment" }
79+
values.push(category.trim())
80+
conditions.push(`s.retention_risk_category = $${values.length}`)
81+
return {}
82+
case "roster_cell":
83+
if (!studentGuid?.trim()) return { error: "studentGuid is required for roster_cell" }
84+
values.push(studentGuid.trim())
85+
conditions.push(`s."Student_GUID" = $${values.length}`)
86+
return {}
87+
default:
88+
return {}
89+
}
90+
}
91+
92+
export async function GET(request: NextRequest) {
93+
const role = request.headers.get("x-user-role") as Role | null
94+
if (!role) {
95+
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
96+
}
97+
98+
const { searchParams } = new URL(request.url)
99+
const metricRaw = searchParams.get("metric") || ""
100+
if (!isLineageMetricId(metricRaw)) {
101+
return NextResponse.json(
102+
{ error: "Invalid or missing metric", allowed: "overall_retention | avg_predicted_retention | …" },
103+
{ status: 400 }
104+
)
105+
}
106+
const metric = metricRaw
107+
108+
const cohort = searchParams.get("cohort") ?? ""
109+
const enrollmentType = searchParams.get("enrollmentType") ?? ""
110+
const credentialType = searchParams.get("credentialType") ?? ""
111+
const category = searchParams.get("category") ?? ""
112+
const studentGuid = searchParams.get("studentGuid") ?? ""
113+
const fieldRaw = searchParams.get("field") ?? ""
114+
const categoryForApi = category || null
115+
116+
if (metric === "roster_cell") {
117+
if (!isRosterLineageField(fieldRaw)) {
118+
return NextResponse.json({ error: "Invalid or missing field for roster_cell" }, { status: 400 })
119+
}
120+
}
121+
122+
const page = Math.max(1, Number(searchParams.get("page") || 1))
123+
const pageSize = Math.min(MAX_PAGE_SIZE, Math.max(1, Number(searchParams.get("pageSize") || 50)))
124+
const offset = (page - 1) * pageSize
125+
126+
const showIdentifiers = canAccess("/api/students", role)
127+
128+
const filterParams: DashboardFilterParams = { cohort, enrollmentType, credentialType }
129+
const { conditions: filterConds, values: filterVals } =
130+
metric === "roster_cell"
131+
? { conditions: [] as string[], values: [] as unknown[] }
132+
: buildStudentLevelDashboardConditionsAliased(filterParams, "s")
133+
134+
const conditions = [...filterConds]
135+
const values = [...filterVals]
136+
137+
const predErr = appendMetricPredicate(metric, categoryForApi, studentGuid || null, conditions, values)
138+
if (predErr.error) {
139+
return NextResponse.json({ error: predErr.error }, { status: 400 })
140+
}
141+
142+
const whereSql = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""
143+
144+
const pool = getPool()
145+
146+
try {
147+
const [countRes, uploadRes] = await Promise.all([
148+
pool.query<{ c: number }>(
149+
`SELECT COUNT(*)::int AS c FROM student_level_with_predictions s ${whereSql}`,
150+
values
151+
),
152+
pool.query<LineageUploadHistoryRow>(
153+
`SELECT id, filename, file_type, uploaded_at, status, user_email, rows_inserted, rows_skipped, error_count,
154+
(validation_report IS NOT NULL) AS has_validation_report
155+
FROM upload_history
156+
WHERE file_type = ANY($1::text[])
157+
ORDER BY uploaded_at DESC
158+
LIMIT 1`,
159+
[LINEAGE_STUDENT_LEVEL_SCHEMA_IDS]
160+
),
161+
])
162+
163+
const rowCount = Number(countRes.rows[0]?.c ?? 0)
164+
const uploadRow = uploadRes.rows[0]
165+
166+
const schemaMeta = uploadRow ? SCHEMAS.find((x) => x.id === uploadRow.file_type) : undefined
167+
168+
let sourceRows: Record<string, unknown>[] | undefined
169+
if (showIdentifiers && rowCount > 0) {
170+
const dataSql = `
171+
SELECT
172+
s."Student_GUID" AS student_guid,
173+
s."Cohort" AS cohort,
174+
s."Enrollment_Intensity_First_Term" AS enrollment_intensity,
175+
s."Retention" AS retention,
176+
ROUND((s.retention_probability * 100)::numeric, 1) AS retention_pct,
177+
s.at_risk_alert,
178+
s.retention_risk_category,
179+
ROUND((s.course_completion_rate * 100)::numeric, 1) AS course_completion_pct
180+
FROM student_level_with_predictions s
181+
${whereSql}
182+
ORDER BY s."Student_GUID"
183+
LIMIT $${values.length + 1} OFFSET $${values.length + 2}
184+
`
185+
const dataRes = await pool.query(dataSql, [...values, pageSize, offset])
186+
sourceRows = dataRes.rows as Record<string, unknown>[]
187+
}
188+
189+
const steps = lineageStepsForMetric(metric)
190+
if (metric === "roster_cell" && isRosterLineageField(fieldRaw)) {
191+
const rf = rosterFieldLineageLabel(fieldRaw)
192+
steps.push({
193+
order: 4,
194+
title: rf.label,
195+
detail: rf.detail,
196+
})
197+
}
198+
199+
const uploadEvent = uploadRow
200+
? {
201+
id: Number(uploadRow.id),
202+
filename: uploadRow.filename,
203+
fileType: uploadRow.file_type,
204+
schemaLabel: schemaMeta?.label ?? uploadRow.file_type,
205+
uploadedAt: uploadRow.uploaded_at.toISOString(),
206+
status: uploadRow.status,
207+
userEmail: uploadRow.user_email,
208+
rowsInserted: Number(uploadRow.rows_inserted ?? 0),
209+
rowsSkipped: Number(uploadRow.rows_skipped ?? 0),
210+
errorCount: Number(uploadRow.error_count ?? 0),
211+
hasValidationReport: Boolean(uploadRow.has_validation_report),
212+
}
213+
: null
214+
215+
return NextResponse.json({
216+
metricId: metric,
217+
metricLabel: METRIC_LABEL[metric],
218+
metricDescription: metricDescription(metric, categoryForApi),
219+
field: metric === "roster_cell" ? fieldRaw : undefined,
220+
filters: metric === "roster_cell" ? {} : optionalDashboardFilterRecord(filterParams),
221+
dimension: category || undefined,
222+
aggregate: {
223+
rowCount,
224+
summary:
225+
metric === "roster_cell"
226+
? "One student row."
227+
: `${rowCount.toLocaleString()} student(s) match the current filters and metric scope.`,
228+
},
229+
sourceRowsVisible: showIdentifiers,
230+
sourceRowsRestrictedMessage: showIdentifiers
231+
? undefined
232+
: "Row-level identifiers are available to Admin, Advisor, and IR roles (same access as the student roster).",
233+
sourceRows:
234+
showIdentifiers && sourceRows
235+
? { page, pageSize, total: rowCount, rows: sourceRows }
236+
: undefined,
237+
uploadEvent,
238+
transformationSteps: steps,
239+
})
240+
} catch (error) {
241+
console.error("Lineage API error:", error)
242+
return NextResponse.json(
243+
{
244+
error: "Failed to load lineage",
245+
details: error instanceof Error ? error.message : String(error),
246+
},
247+
{ status: 500 }
248+
)
249+
}
250+
}

codebenders-dashboard/app/page.tsx

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ import {
1717
import { TrendingUp, Users, AlertTriangle, BookOpen, Search, Table2, X } from "lucide-react"
1818
import Link from "next/link"
1919
import { GlossaryMetricEntryLink } from "@/components/glossary-metric-entry-link"
20+
import { useDataLineage } from "@/components/data-lineage-drawer"
21+
import { optionalDashboardFilterRecord } from "@/lib/dashboard-filters"
22+
import type { LineageMetricId } from "@/lib/lineage-config"
2023

2124
interface KPIData {
2225
overallRetentionRate: string
@@ -60,6 +63,7 @@ const ENROLLMENT_TYPES = ["Full-Time", "Part-Time"]
6063
const CREDENTIAL_TYPES = ["Certificate", "Associate", "Bachelor"]
6164

6265
export default function DashboardPage() {
66+
const { drawer: lineageDrawer, openLineage } = useDataLineage()
6367
const [kpis, setKpis] = useState<KPIData | null>(null)
6468
const [riskAlerts, setRiskAlerts] = useState<RiskAlertData[]>([])
6569
const [retentionRisk, setRetentionRisk] = useState<RetentionRiskData[]>([])
@@ -85,6 +89,13 @@ export default function DashboardPage() {
8589
return qs ? `?${qs}` : ""
8690
}
8791

92+
function openKpiLineage(metric: LineageMetricId) {
93+
openLineage({
94+
metric,
95+
...optionalDashboardFilterRecord({ cohort, enrollmentType, credentialType }),
96+
})
97+
}
98+
8899
useEffect(() => {
89100
const qs = buildFilterParams()
90101

@@ -271,6 +282,7 @@ export default function DashboardPage() {
271282
icon={TrendingUp}
272283
subtitle={kpis ? `${kpis.totalStudents.toLocaleString()} total students` : undefined}
273284
loading={loading}
285+
onLineageClick={loading ? undefined : () => openKpiLineage("overall_retention")}
274286
info={
275287
<>
276288
<p><strong>What it shows:</strong> Percentage of students retained year-to-year based on historical data.</p>
@@ -286,6 +298,7 @@ export default function DashboardPage() {
286298
icon={Users}
287299
subtitle="ML model prediction"
288300
loading={loading}
301+
onLineageClick={loading ? undefined : () => openKpiLineage("avg_predicted_retention")}
289302
info={
290303
<>
291304
<p><strong>Model:</strong> XGBoost Classifier trained on 31 features including demographics, academic prep, and course performance.</p>
@@ -307,6 +320,7 @@ export default function DashboardPage() {
307320
icon={AlertTriangle}
308321
subtitle="Require immediate intervention"
309322
loading={loading}
323+
onLineageClick={loading ? undefined : () => openKpiLineage("high_critical_risk_count")}
310324
info={
311325
<>
312326
<p><strong>How it's calculated:</strong> Composite risk score combining:</p>
@@ -332,6 +346,7 @@ export default function DashboardPage() {
332346
icon={BookOpen}
333347
subtitle="Credits earned / attempted"
334348
loading={loading}
349+
onLineageClick={loading ? undefined : () => openKpiLineage("avg_course_completion")}
335350
info={
336351
<>
337352
<p><strong>Formula:</strong> (Total credits earned ÷ Total credits attempted) × 100</p>
@@ -354,6 +369,13 @@ export default function DashboardPage() {
354369
<RiskAlertChart
355370
data={riskAlerts}
356371
loading={loading}
372+
onSegmentLineage={(category) =>
373+
openLineage({
374+
metric: "risk_alert_segment",
375+
category,
376+
...optionalDashboardFilterRecord({ cohort, enrollmentType, credentialType }),
377+
})
378+
}
357379
info={
358380
<>
359381
<p><strong>What it shows:</strong> Distribution of students across risk alert levels (URGENT, HIGH, MODERATE, LOW).</p>
@@ -427,6 +449,7 @@ export default function DashboardPage() {
427449
</div>
428450
</div>
429451
</div>
452+
{lineageDrawer}
430453
</div>
431454
)
432455
}

0 commit comments

Comments
 (0)