Skip to content

Commit 306d458

Browse files
committed
fix(corpus): cap collectFile's read at bytelimit.DefaultMaxInputBytes
collectFile used a bare os.ReadFile while walking cloned third-party repositories for the training corpus — the one file-ingestion loop in the codebase reading genuinely untrusted external content without the byte-cap every other read site already applies (bytelimit, internal/rules/githooksync, internal/schema). A single oversized file in a source repo used to be read into memory in full; it now fails the walk instead. Ref: docs/development/high-performance-go.md ("os.ReadFile on huge inputs: one giant alloc, all resident"). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCh2j2oeSw6UgNsmpjRonL
1 parent f5842c2 commit 306d458

2 files changed

Lines changed: 45 additions & 1 deletion

File tree

internal/corpus/collect.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"path/filepath"
1010
"strings"
1111
"unicode/utf8"
12+
13+
"github.com/jeduden/mdsmith/internal/bytelimit"
1214
)
1315

1416
// Collect gathers markdown records from configured sources.
@@ -174,7 +176,7 @@ func collectFile(
174176
return Record{}, false, nil
175177
}
176178

177-
content, err := os.ReadFile(fullPath)
179+
content, err := bytelimit.ReadFileLimited(fullPath, bytelimit.DefaultMaxInputBytes)
178180
if err != nil {
179181
return Record{}, false, fmt.Errorf("read file %s: %w", fullPath, err)
180182
}

internal/corpus/collect_test.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package corpus
22

33
import (
4+
"bytes"
45
"os"
56
"path/filepath"
67
"testing"
8+
9+
"github.com/jeduden/mdsmith/internal/bytelimit"
710
)
811

912
func TestCollect_HappyPath(t *testing.T) {
@@ -207,6 +210,45 @@ func TestCollect_ErrorPath(t *testing.T) {
207210
}
208211
}
209212

213+
// TestCollect_OversizedFile_ReturnsError guards against an unbounded
214+
// os.ReadFile on a corpus source: collectFile ingests markdown from
215+
// cloned third-party repositories, which are untrusted input
216+
// (docs/development/high-performance-go.md — "os.ReadFile on huge
217+
// inputs: one giant alloc, all resident"). A file over the shared
218+
// bytelimit.DefaultMaxInputBytes cap must fail the walk rather than
219+
// being read into memory in full.
220+
func TestCollect_OversizedFile_ReturnsError(t *testing.T) {
221+
t.Parallel()
222+
223+
root := filepath.Join(t.TempDir(), "docs")
224+
if err := os.MkdirAll(root, 0o755); err != nil {
225+
t.Fatalf("mkdir: %v", err)
226+
}
227+
oversized := bytes.Repeat([]byte("a"), int(bytelimit.DefaultMaxInputBytes)+1)
228+
if err := os.WriteFile(filepath.Join(root, "huge.md"), oversized, 0o644); err != nil {
229+
t.Fatalf("write oversized markdown: %v", err)
230+
}
231+
232+
cfg := &Config{
233+
CollectedAt: "2026-02-16",
234+
MinWords: 1,
235+
MinChars: 1,
236+
LicenseAllowlist: []string{"MIT"},
237+
Sources: []SourceConfig{{
238+
Name: "seed",
239+
Repository: "github.com/acme/seed",
240+
Root: root,
241+
CommitSHA: "abc123",
242+
License: "MIT",
243+
}},
244+
}
245+
246+
_, err := Collect(cfg, t.TempDir())
247+
if err == nil {
248+
t.Fatal("Collect: expected an error for a file over the byte-limit cap, got nil")
249+
}
250+
}
251+
210252
// --- reportProgress ---
211253

212254
// TestReportProgress pins all three branches: nil cfg is a no-op,

0 commit comments

Comments
 (0)