refactor(utilities): unify list/detail serializer + ?fields= support (ALL-733)

app/api/v1/utilities/[slug]/route.ts

@@ -19,6 +19,7 @@ async function handleGet(req: Request, ctx: RouteContext) {
 
   const url = new URL(req.url);
   const include = url.searchParams.get("include");
+  const fields = url.searchParams.get("fields");
 
   const db = getDb();
   const [utility] = await db.select().from(utilities).where(eq(utilities.slug, slug)).limit(1);
@@ -71,10 +72,10 @@ async function handleGet(req: Request, ctx: RouteContext) {
     }
 
     await Promise.all(fetches);
-    return publicJsonResponse(result, 200);
+    return publicJsonResponse(result, 200, {}, { fields });
   }
 
-  return publicJsonResponse(utility, 200);
+  return publicJsonResponse(utility, 200, {}, { fields });
 }
 
 const handler = withRequestId(withErrorHandling(withTiming(handleGet)));

app/api/v1/utilities/route.ts

@@ -3,28 +3,12 @@ import type { NextRequest } from "next/server";
 import { corsHeaders } from "@/lib/api/cors";
 import { generateRequestId, withErrorHandling, withRequestId, withTiming } from "@/lib/api/middleware";
 import { encodeCursor, parsePaginationParams } from "@/lib/api/pagination";
-import { stripInternal } from "@/lib/api/public-response";
+import { parseFieldsParam, selectFields, stripInternal } from "@/lib/api/public-response";
 import { jsonResponse, paginatedResponse } from "@/lib/api/response";
 import type { RouteContext } from "@/lib/api/types";
 import { getDb } from "@/lib/db/client";
 import { balancingAuthorities, isos, rtos, utilities } from "@/lib/db/schema";
 
-// ---------------------------------------------------------------------------
-// Sparse field selection helper
-// ---------------------------------------------------------------------------
-
-function selectFields(items: Record<string, unknown>[], fields: string[]): Record<string, unknown>[] {
-  return items.map((item) => {
-    const result: Record<string, unknown> = {};
-    for (const field of fields) {
-      if (field in item) {
-        result[field] = item[field];
-      }
-    }
-    return result;
-  });
-}
-
 // ---------------------------------------------------------------------------
 // Filter params interface
 // ---------------------------------------------------------------------------
@@ -218,13 +202,18 @@ async function handleDatabaseMode(params: DbFilterParams) {
     resultData = await resolveIncludes(db, data, includes);
   }
 
-  // Sparse fields
-  if (params.fields) {
-    const fieldList = params.fields.split(",").map((f) => f.trim());
-    resultData = selectFields(resultData, fieldList);
+  // Sanitize internal fields first, then apply sparse-fieldset projection.
+  // Order matters: stripping must happen before projection so a field like
+  // `searchVector` can't be resurrected by an explicit `?fields=searchVector`
+  // request.
+  resultData = stripInternal(resultData) as Record<string, unknown>[];
+
+  const fieldList = parseFieldsParam(params.fields);
+  if (fieldList) {
+    resultData = resultData.map((item) => selectFields(item, fieldList));
   }
 
-  return jsonResponse(paginatedResponse(stripInternal(resultData), Number(count), nextCursor, limit), 200, {
+  return jsonResponse(paginatedResponse(resultData, Number(count), nextCursor, limit), 200, {
     ...corsHeaders(),
     "Cache-Control": "public, s-maxage=300, stale-while-revalidate=600",
   });

lib/api/__tests__/public-response.test.ts

@@ -13,7 +13,13 @@
 import { describe, expect, it } from "vitest";
 
 import { INTERNAL_FIELDS } from "../internal-fields";
-import { publicJsonResponse, publicPaginatedResponse, stripInternal } from "../public-response";
+import {
+  parseFieldsParam,
+  publicJsonResponse,
+  publicPaginatedResponse,
+  selectFields,
+  stripInternal,
+} from "../public-response";
 
 describe("stripInternal", () => {
   it("removes every field listed in INTERNAL_FIELDS from a single object", () => {
@@ -169,3 +175,220 @@ describe("publicPaginatedResponse", () => {
     expect(body.meta).toEqual({ total: 2, cursor: null, limit: 50, hasMore: false });
   });
 });
+
+// ---------------------------------------------------------------------------
+// Sparse fieldsets (ALL-733)
+//
+// Origin: Morgan's Relay bug report (2026-05-06), bug #2.
+// The list endpoint was dropping numeric fields and `?fields=` was ignored,
+// forcing list-then-detail access patterns (~3,150 API calls for a full
+// sync). These tests guard the fix.
+// ---------------------------------------------------------------------------
+
+describe("parseFieldsParam", () => {
+  it("returns null for null/undefined/empty input", () => {
+    expect(parseFieldsParam(null)).toBeNull();
+    expect(parseFieldsParam(undefined)).toBeNull();
+    expect(parseFieldsParam("")).toBeNull();
+    expect(parseFieldsParam("   ")).toBeNull();
+    expect(parseFieldsParam(",,, ,")).toBeNull();
+  });
+
+  it("splits on comma, trims whitespace, drops empties", () => {
+    expect(parseFieldsParam("id, slug ,name")).toEqual(["id", "slug", "name"]);
+    expect(parseFieldsParam("a,,b")).toEqual(["a", "b"]);
+  });
+
+  it("de-dupes while preserving the first-seen order", () => {
+    expect(parseFieldsParam("id,slug,id,name,slug")).toEqual(["id", "slug", "name"]);
+  });
+});
+
+describe("selectFields", () => {
+  it("returns only the requested keys that exist on the object", () => {
+    const input = {
+      id: "util_abc",
+      slug: "green-mountain-power",
+      name: "Green Mountain Power",
+      customerCount: 267602,
+      totalMeterCount: 288142,
+    };
+    expect(selectFields(input, ["id", "slug", "customerCount"])).toEqual({
+      id: "util_abc",
+      slug: "green-mountain-power",
+      customerCount: 267602,
+    });
+  });
+
+  it("silently drops unknown field names", () => {
+    const input = { id: "x", name: "X" };
+    expect(selectFields(input, ["id", "nope"])).toEqual({ id: "x" });
+  });
+
+  it("preserves null and zero values (does not confuse them with missing keys)", () => {
+    const input = { id: "x", customerCount: null, totalMeterCount: 0 };
+    expect(selectFields(input, ["customerCount", "totalMeterCount"])).toEqual({
+      customerCount: null,
+      totalMeterCount: 0,
+    });
+  });
+
+  it("returns non-object inputs unchanged", () => {
+    expect(selectFields(null, ["id"])).toBeNull();
+    expect(selectFields(undefined, ["id"])).toBeUndefined();
+    expect(selectFields("hi", ["id"])).toBe("hi");
+    expect(selectFields(42, ["id"])).toBe(42);
+  });
+});
+
+describe("publicJsonResponse with sparse fieldsets", () => {
+  it("applies ?fields= projection and still strips internal fields", async () => {
+    const res = publicJsonResponse(
+      {
+        id: "util_abc",
+        slug: "green-mountain-power",
+        name: "Green Mountain Power",
+        customerCount: 267602,
+        totalMeterCount: 288142,
+        submittedBy: "u1",
+        searchVector: "'green':1",
+      },
+      200,
+      {},
+      { fields: "id,slug,name,customerCount,totalMeterCount" }
+    );
+
+    const body = (await res.json()) as { data: Record<string, unknown> };
+    expect(body.data).toEqual({
+      id: "util_abc",
+      slug: "green-mountain-power",
+      name: "Green Mountain Power",
+      customerCount: 267602,
+      totalMeterCount: 288142,
+    });
+  });
+
+  it("refuses to resurrect internal fields via ?fields= (e.g. searchVector)", async () => {
+    const res = publicJsonResponse(
+      { id: "x", name: "X", searchVector: "'x':1", submittedBy: "u1" },
+      200,
+      {},
+      { fields: "id,searchVector,submittedBy" }
+    );
+    const body = (await res.json()) as { data: Record<string, unknown> };
+    expect(body.data).toEqual({ id: "x" });
+  });
+
+  it("returns the full public shape when ?fields= is omitted", async () => {
+    const res = publicJsonResponse({
+      id: "x",
+      name: "X",
+      customerCount: 100,
+      submittedBy: "u1",
+    });
+    const body = (await res.json()) as { data: Record<string, unknown> };
+    expect(body.data).toEqual({ id: "x", name: "X", customerCount: 100 });
+  });
+
+  it("accepts pre-parsed string[] fields", async () => {
+    const res = publicJsonResponse({ id: "x", name: "X", extra: "e" }, 200, {}, { fields: ["id", "name"] });
+    const body = (await res.json()) as { data: Record<string, unknown> };
+    expect(body.data).toEqual({ id: "x", name: "X" });
+  });
+});
+
+describe("publicPaginatedResponse with sparse fieldsets", () => {
+  it("applies ?fields= projection to every item", async () => {
+    const res = publicPaginatedResponse(
+      [
+        { id: "a", name: "A", customerCount: 100, submittedBy: "u1" },
+        { id: "b", name: "B", customerCount: 200, submittedBy: "u2" },
+      ],
+      { total: 2, cursor: null, limit: 50, hasMore: false },
+      200,
+      {},
+      { fields: "id,customerCount" }
+    );
+
+    const body = (await res.json()) as { data: Array<Record<string, unknown>> };
+    expect(body.data).toEqual([
+      { id: "a", customerCount: 100 },
+      { id: "b", customerCount: 200 },
+    ]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// List/detail shape parity guard (ALL-733)
+//
+// The core bug: list-endpoint rows had a narrower shape than detail-endpoint
+// rows, forcing clients into a 1+N access pattern. These tests are route-less
+// but guard the invariant that `stripInternal` + `selectFields` produce the
+// same shape for the same input, regardless of which envelope wraps it.
+// ---------------------------------------------------------------------------
+
+describe("list/detail shape parity", () => {
+  const row = {
+    id: "2da5b7fc-9f3d-8198-9c0e-d09ded80b4ad",
+    slug: "green-mountain-power",
+    name: "Green Mountain Power",
+    segment: "DISTRIBUTION",
+    customerCount: 267602,
+    totalMeterCount: 288142,
+    amiMeterCount: 279560,
+    peakDemandMw: 612.3,
+    // Internal fields that must be stripped from both.
+    submittedBy: "u1",
+    reviewedAt: "2026-01-01T00:00:00Z",
+    searchVector: "'green':1 'mountain':2 'power':3",
+    notionPageId: "2da5b7fc-9f3d-8198-9c0e-d09ded80b4ad",
+  };
+
+  it("publicJsonResponse(row) and publicPaginatedResponse([row]) produce the same per-record shape", async () => {
+    const detailRes = publicJsonResponse(row);
+    const listRes = publicPaginatedResponse([row], { total: 1, cursor: null, limit: 50, hasMore: false });
+
+    const detailBody = (await detailRes.json()) as { data: Record<string, unknown> };
+    const listBody = (await listRes.json()) as { data: Array<Record<string, unknown>> };
+
+    expect(listBody.data).toHaveLength(1);
+    const listRecord = listBody.data[0];
+    if (!listRecord) throw new Error("expected at least one list record");
+    expect(Object.keys(detailBody.data).sort()).toEqual(Object.keys(listRecord).sort());
+    expect(listRecord).toEqual(detailBody.data);
+  });
+
+  it("numeric fields (customerCount, totalMeterCount, amiMeterCount) survive both envelopes", async () => {
+    const detailRes = publicJsonResponse(row);
+    const listRes = publicPaginatedResponse([row], { total: 1, cursor: null, limit: 50, hasMore: false });
+    const detailBody = (await detailRes.json()) as { data: Record<string, unknown> };
+    const listBody = (await listRes.json()) as { data: Array<Record<string, unknown>> };
+
+    for (const field of ["customerCount", "totalMeterCount", "amiMeterCount", "peakDemandMw"]) {
+      expect(detailBody.data[field]).toBe((row as Record<string, unknown>)[field]);
+      expect(listBody.data[0]?.[field]).toBe((row as Record<string, unknown>)[field]);
+    }
+  });
+
+  it("?fields= yields identical projections on both envelopes", async () => {
+    const opts = { fields: "id,slug,name,customerCount" };
+    const detailRes = publicJsonResponse(row, 200, {}, opts);
+    const listRes = publicPaginatedResponse(
+      [row],
+      { total: 1, cursor: null, limit: 50, hasMore: false },
+      200,
+      {},
+      opts
+    );
+    const detailBody = (await detailRes.json()) as { data: Record<string, unknown> };
+    const listBody = (await listRes.json()) as { data: Array<Record<string, unknown>> };
+
+    expect(detailBody.data).toEqual({
+      id: row.id,
+      slug: row.slug,
+      name: row.name,
+      customerCount: row.customerCount,
+    });
+    expect(listBody.data[0]).toEqual(detailBody.data);
+  });
+});