Merge pull request #100 from deariary/fix/paginate-commits-and-jsdom

feat: paginate commit fetching and remove per-repo/total caps

Author: unhappychoice

src/cli/commands/fetch.ts

@@ -10,6 +10,7 @@ import { fetchEvents, dedupeEvents } from "../../collector/fetch-events.js";
 import { fetchContributions } from "../../collector/fetch-contributions.js";
 import { fetchPRsByRefs, type PRRef } from "../../collector/fetch-repo-prs.js";
 import { fetchCommitMessages } from "../../collector/fetch-commits.js";
+import { fetchReleases } from "../../collector/fetch-releases.js";
 import { aggregateRepositories } from "../../collector/aggregate.js";
 import { getWeekId, getCurrentWeekId } from "../../deployer/week.js";
 import type { GitHubEvent } from "../../types.js";
@@ -238,6 +239,11 @@ const runWeeklyFetch = async (options: BaseOptions): Promise<void> => {
   const totalMsgs = commitMessages.reduce((sum, r) => sum + r.messages.length, 0);
   console.log(`Collected ${totalMsgs} commit messages from ${commitMessages.length} repositories.`);
 
+  // Fetch releases per repository
+  console.log(`Fetching releases for ${repoNames.length} repositories...`);
+  const releases = await fetchReleases(options.token, repoNames, plan.range);
+  console.log(`Collected ${releases.length} releases.`);
+
   const githubData = {
     username: contributions.username,
     avatarUrl: contributions.avatarUrl,
@@ -257,8 +263,9 @@ const runWeeklyFetch = async (options: BaseOptions): Promise<void> => {
     repositories,
     pullRequests,
     issues: [],
-    events,
+    events: events.filter((e) => e.payload.kind === "review"),
     commitMessages,
+    releases,
     externalContributions: [],
   };
   const dataPath = join(plan.reportDir, "github-data.yaml");

src/cli/commands/generate.test.ts

@@ -178,6 +178,7 @@ pullRequests: []
 issues: []
 events: []
 commitMessages: []
+releases: []
 externalContributions: []
 `;
 

src/cli/commands/render.test.ts

@@ -73,6 +73,7 @@ pullRequests: []
 issues: []
 events: []
 commitMessages: []
+releases: []
 externalContributions: []
 `;
 

src/collector/fetch-commits.test.ts

@@ -12,24 +12,27 @@ const makeRawCommit = (message: string) => ({
   commit: { message, author: { date: "2026-04-01T12:00:00Z" } },
 });
 
+// Helper to create a Response with a Link header for pagination
+const pagedResponse = (commits: unknown[], nextUrl?: string) => {
+  const headers: Record<string, string> = {};
+  if (nextUrl) headers["link"] = `<${nextUrl}>; rel="next"`;
+  return new Response(JSON.stringify(commits), { status: 200, headers });
+};
+
 describe("fetchCommitMessages", () => {
   beforeEach(() => {
     vi.restoreAllMocks();
   });
 
   it("fetches commit messages for multiple repos", async () => {
     vi.spyOn(globalThis, "fetch")
-      .mockResolvedValueOnce(
-        new Response(JSON.stringify([
-          makeRawCommit("feat: add login"),
-          makeRawCommit("fix: typo in header"),
-        ]), { status: 200 }),
-      )
-      .mockResolvedValueOnce(
-        new Response(JSON.stringify([
-          makeRawCommit("chore: update deps"),
-        ]), { status: 200 }),
-      );
+      .mockResolvedValueOnce(pagedResponse([
+        makeRawCommit("feat: add login"),
+        makeRawCommit("fix: typo in header"),
+      ]))
+      .mockResolvedValueOnce(pagedResponse([
+        makeRawCommit("chore: update deps"),
+      ]));
 
     const result = await fetchCommitMessages("token", "user", ["org/repo-a", "org/repo-b"], range);
 
@@ -38,11 +41,24 @@ describe("fetchCommitMessages", () => {
     expect(result[1]).toEqual({ repo: "org/repo-b", messages: ["chore: update deps"] });
   });
 
+  it("paginates through multiple pages", async () => {
+    const page1 = Array.from({ length: 100 }, (_, i) => makeRawCommit(`page1-${i}`));
+    const page2 = Array.from({ length: 50 }, (_, i) => makeRawCommit(`page2-${i}`));
+
+    vi.spyOn(globalThis, "fetch")
+      .mockResolvedValueOnce(pagedResponse(page1, "https://api.github.com/repos/org/repo/commits?page=2"))
+      .mockResolvedValueOnce(pagedResponse(page2));
+
+    const result = await fetchCommitMessages("token", "user", ["org/repo"], range);
+
+    expect(result[0].messages).toHaveLength(150);
+    expect(result[0].messages[0]).toBe("page1-0");
+    expect(result[0].messages[100]).toBe("page2-0");
+  });
+
   it("extracts only the first line of multi-line commit messages", async () => {
     vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
-      new Response(JSON.stringify([
-        makeRawCommit("feat: new feature\n\nLong description here\nMore details"),
-      ]), { status: 200 }),
+      pagedResponse([makeRawCommit("feat: new feature\n\nLong description here\nMore details")]),
     );
 
     const result = await fetchCommitMessages("token", "user", ["org/repo"], range);
@@ -53,7 +69,7 @@ describe("fetchCommitMessages", () => {
   it("truncates long commit messages to 200 characters", async () => {
     const longMessage = "a".repeat(300);
     vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
-      new Response(JSON.stringify([makeRawCommit(longMessage)]), { status: 200 }),
+      pagedResponse([makeRawCommit(longMessage)]),
     );
 
     const result = await fetchCommitMessages("token", "user", ["org/repo"], range);
@@ -63,9 +79,7 @@ describe("fetchCommitMessages", () => {
   });
 
   it("skips repos with no commits", async () => {
-    vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
-      new Response(JSON.stringify([]), { status: 200 }),
-    );
+    vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(pagedResponse([]));
 
     const result = await fetchCommitMessages("token", "user", ["org/empty"], range);
 
@@ -92,32 +106,6 @@ describe("fetchCommitMessages", () => {
     expect(result).toHaveLength(0);
   });
 
-  it("respects per_page limit from API (10 per repo)", async () => {
-    // API honors per_page=10, so even if repo has more commits, only 10 are returned
-    const commits = Array.from({ length: 10 }, (_, i) => makeRawCommit(`commit ${i}`));
-    vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
-      new Response(JSON.stringify(commits), { status: 200 }),
-    );
-
-    const result = await fetchCommitMessages("token", "user", ["org/busy"], range);
-
-    expect(result[0].messages.length).toBe(10);
-  });
-
-  it("caps total messages at 50 across all repos", async () => {
-    // 6 repos each returning 10 commits = 60 total, should be capped at 50
-    const commits = Array.from({ length: 10 }, (_, i) => makeRawCommit(`msg ${i}`));
-    vi.spyOn(globalThis, "fetch").mockImplementation(
-      () => Promise.resolve(new Response(JSON.stringify(commits), { status: 200 })),
-    );
-
-    const repos = Array.from({ length: 6 }, (_, i) => `org/repo-${i}`);
-    const result = await fetchCommitMessages("token", "user", repos, range);
-
-    const totalMsgs = result.reduce((sum, r) => sum + r.messages.length, 0);
-    expect(totalMsgs).toBeLessThanOrEqual(50);
-  });
-
   it("returns empty array for empty repos list", async () => {
     const result = await fetchCommitMessages("token", "user", [], range);
 
@@ -129,9 +117,7 @@ describe("fetchCommitMessages", () => {
       .mockResolvedValueOnce(
         new Response("", { status: 429, headers: { "retry-after": "0" } }),
       )
-      .mockResolvedValueOnce(
-        new Response(JSON.stringify([makeRawCommit("after retry")]), { status: 200 }),
-      );
+      .mockResolvedValueOnce(pagedResponse([makeRawCommit("after retry")]));
 
     const result = await fetchCommitMessages("token", "user", ["org/repo"], range);
 

src/collector/fetch-commits.ts

@@ -16,8 +16,7 @@ type RawCommit = {
   };
 };
 
-const MAX_MESSAGES_PER_REPO = 10;
-const MAX_TOTAL_MESSAGES = 50;
+const PER_PAGE = 100;
 const MAX_MESSAGE_LENGTH = 200;
 const MAX_RETRIES = 3;
 const DEFAULT_RETRY_DELAY_MS = 5_000;
@@ -40,6 +39,14 @@ const parseRetryDelay = (response: Response): number => {
   return DEFAULT_RETRY_DELAY_MS;
 };
 
+// Parse Link header to find next page URL
+const parseNextUrl = (response: Response): string | null => {
+  const link = response.headers.get("link");
+  if (!link) return null;
+  const match = link.match(/<([^>]+)>;\s*rel="next"/);
+  return match?.[1] ?? null;
+};
+
 // Extract the first line of a commit message and truncate to MAX_MESSAGE_LENGTH
 const firstLine = (message: string): string => {
   const subject = message.split("\n")[0]?.trim() ?? message.trim();
@@ -48,50 +55,58 @@ const firstLine = (message: string): string => {
     : subject;
 };
 
-const fetchRepoCommits = async (
+const fetchPage = async (
   token: string,
-  repo: string,
-  author: string,
-  range: DateRange,
-): Promise<string[]> => {
-  const params = new URLSearchParams({
-    author,
-    since: range.from.toISOString(),
-    until: range.to.toISOString(),
-    per_page: String(MAX_MESSAGES_PER_REPO),
-  });
-  const url = `https://api.github.com/repos/${repo}/commits?${params}`;
-
+  url: string,
+): Promise<{ commits: RawCommit[]; nextUrl: string | null } | null> => {
   for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
     const response = await fetch(url, { headers: GITHUB_HEADERS(token) });
 
     if (response.ok) {
       const commits = (await response.json()) as RawCommit[];
-      return commits.map((c) => firstLine(c.commit.message));
+      return { commits, nextUrl: parseNextUrl(response) };
     }
 
-    if (response.status === 409) {
-      // Empty repository
-      return [];
-    }
+    if (response.status === 409) return null; // Empty repository
+    if (response.status === 403 || response.status === 404) return null;
 
     if (response.status === 429 && attempt < MAX_RETRIES) {
       const delay = parseRetryDelay(response);
-      console.warn(`  ${repo}: 429, retrying in ${Math.round(delay / 1000)}s (attempt ${attempt + 1}/${MAX_RETRIES})`);
+      console.warn(`  429, retrying in ${Math.round(delay / 1000)}s (attempt ${attempt + 1}/${MAX_RETRIES})`);
       await sleep(delay);
       continue;
     }
 
-    if (response.status === 403 || response.status === 404) {
-      // Permission denied or repo not found (private, deleted, etc.)
-      return [];
-    }
+    console.warn(`  Failed to fetch commits: ${response.status} ${response.statusText}`);
+    return null;
+  }
 
-    console.warn(`  Failed to fetch commits for ${repo}: ${response.status} ${response.statusText}`);
-    return [];
+  return null;
+};
+
+const fetchRepoCommits = async (
+  token: string,
+  repo: string,
+  author: string,
+  range: DateRange,
+): Promise<string[]> => {
+  const params = new URLSearchParams({
+    author,
+    since: range.from.toISOString(),
+    until: range.to.toISOString(),
+    per_page: String(PER_PAGE),
+  });
+  let url: string | null = `https://api.github.com/repos/${repo}/commits?${params}`;
+  const messages: string[] = [];
+
+  while (url) {
+    const result = await fetchPage(token, url);
+    if (!result) break;
+    messages.push(...result.commits.map((c) => firstLine(c.commit.message)));
+    url = result.nextUrl;
   }
 
-  return [];
+  return messages;
 };
 
 const CONCURRENCY = 5;
@@ -129,16 +144,5 @@ export const fetchCommitMessages = async (
     }
   });
 
-  // Cap total messages after all fetches complete
-  const capped: RepoCommits[] = [];
-  let total = 0;
-  for (const entry of results) {
-    if (total >= MAX_TOTAL_MESSAGES) break;
-    const remaining = MAX_TOTAL_MESSAGES - total;
-    const trimmed = entry.messages.slice(0, remaining);
-    capped.push({ repo: entry.repo, messages: trimmed });
-    total += trimmed.length;
-  }
-
-  return capped;
+  return results;
 };