fix(corpus): skip oversized files instead of aborting the whole build

Review found that returning bytelimit's error straight from collectFile
made it fatal: collectFromRoot's WalkDir callback treats any error as
abort-the-walk, and Collect returns on the first source error, so one
big file discarded every record already collected from every source in
the run. Stat the file first and skip it (matching the existing
too-small-word/char-count skip pattern) instead of reading and failing;
a real cloned repository can have one oversized file (a big CHANGELOG,
a vendored spec) without that being reason to fail the whole corpus
build.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCh2j2oeSw6UgNsmpjRonL

Author: claude

internal/corpus/collect.go

@@ -176,6 +176,24 @@ func collectFile(
 		return Record{}, false, nil
 	}
 
+	// Skip files over the shared byte-limit cap rather than failing the
+	// whole source: collectFromRoot's caller aborts the entire walk (and
+	// every record already collected from every source before it, since
+	// Collect returns on the first error) on any error from this
+	// function. A cloned third-party repository can legitimately contain
+	// one oversized file (a large CHANGELOG, a vendored spec) without
+	// that file being reason to discard the rest of the corpus build.
+	info, err := os.Stat(fullPath)
+	if err != nil {
+		return Record{}, false, fmt.Errorf("stat file %s: %w", fullPath, err)
+	}
+	if info.Size() > bytelimit.DefaultMaxInputBytes {
+		reportProgress(cfg, fmt.Sprintf(
+			"skipping %s: %d bytes exceeds the %d byte limit",
+			relPath, info.Size(), bytelimit.DefaultMaxInputBytes))
+		return Record{}, false, nil
+	}
+
 	content, err := bytelimit.ReadFileLimited(fullPath, bytelimit.DefaultMaxInputBytes)
 	if err != nil {
 		return Record{}, false, fmt.Errorf("read file %s: %w", fullPath, err)

internal/corpus/collect_test.go

@@ -210,24 +210,31 @@ func TestCollect_ErrorPath(t *testing.T) {
 	}
 }
 
-// TestCollect_OversizedFile_ReturnsError guards against an unbounded
+// TestCollect_OversizedFile_SkippedNotFatal guards against an unbounded
 // os.ReadFile on a corpus source: collectFile ingests markdown from
 // cloned third-party repositories, which are untrusted input
 // (docs/development/high-performance-go.md — "os.ReadFile on huge
 // inputs: one giant alloc, all resident"). A file over the shared
-// bytelimit.DefaultMaxInputBytes cap must fail the walk rather than
-// being read into memory in full.
-func TestCollect_OversizedFile_ReturnsError(t *testing.T) {
+// bytelimit.DefaultMaxInputBytes cap must be skipped rather than read
+// into memory in full — and, since a real source repository can contain
+// one large file among many good ones (a big CHANGELOG, a vendored
+// spec), skipping it must not abort collection of the rest of that
+// source or of sources collected earlier in the same run.
+func TestCollect_OversizedFile_SkippedNotFatal(t *testing.T) {
 	t.Parallel()
 
 	root := filepath.Join(t.TempDir(), "docs")
 	if err := os.MkdirAll(root, 0o755); err != nil {
 		t.Fatalf("mkdir: %v", err)
 	}
-	oversized := bytes.Repeat([]byte("a"), int(bytelimit.DefaultMaxInputBytes)+1)
+	oversized := bytes.Repeat([]byte("a "), int(bytelimit.DefaultMaxInputBytes)/2+1)
 	if err := os.WriteFile(filepath.Join(root, "huge.md"), oversized, 0o644); err != nil {
 		t.Fatalf("write oversized markdown: %v", err)
 	}
+	normalContent := []byte("# Title\n\nword word word word word word\n")
+	if err := os.WriteFile(filepath.Join(root, "normal.md"), normalContent, 0o644); err != nil {
+		t.Fatalf("write normal markdown: %v", err)
+	}
 
 	cfg := &Config{
 		CollectedAt:      "2026-02-16",
@@ -243,9 +250,15 @@ func TestCollect_OversizedFile_ReturnsError(t *testing.T) {
 		}},
 	}
 
-	_, err := Collect(cfg, t.TempDir())
-	if err == nil {
-		t.Fatal("Collect: expected an error for a file over the byte-limit cap, got nil")
+	records, err := Collect(cfg, t.TempDir())
+	if err != nil {
+		t.Fatalf("Collect: unexpected error, oversized file should be skipped: %v", err)
+	}
+	if len(records) != 1 {
+		t.Fatalf("record count = %d, want 1 (only normal.md; huge.md must be skipped)", len(records))
+	}
+	if records[0].Path != "normal.md" {
+		t.Fatalf("Path = %q, want normal.md", records[0].Path)
 	}
 }