Skip to content

Commit fe7d1a9

Browse files
authored
fix(llmcontext): report unresolved/external edges, and stop truncating the quality preface (#87)
The Extraction Quality block guarded the unresolved-edge count on CoverageGaps > 0, so a snapshot with zero gaps and many unresolved outbound call sites reported neither. Gaps, unresolved edges and external edges are independent signals and now render under independent guards. External edges are reported without a warning and excluded from the "extraction limits" footer — a hardcoded third-party host is expected, not a blind spot. Render() also broke out of the section loop on the first section to overrun the token budget, and Repository Map emits one uncapped row per module. On a multi-repo snapshot the map consumed the whole budget and the Extraction Quality preface was dropped entirely, from exactly the snapshots whose extraction is least complete. Sections can now reserve budget ahead of layout; reserving buys survival, not precedence. Rewriting that loop exposed a third bug: the cut sliced bytes, not runes, so truncation could emit invalid UTF-8. cutAt backs the cut off to the nearest rune boundary. Renderer-only — facts.jsonl and insights.json are byte-identical across the change, so no cacheVersion bump and no golden regeneration. Adds the first tests for the Extraction Quality block and for truncation.
1 parent f998f65 commit fe7d1a9

2 files changed

Lines changed: 273 additions & 34 deletions

File tree

internal/renderers/llmcontext/llm.go

Lines changed: 89 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"sort"
77
"strings"
8+
"unicode/utf8"
89

910
"github.com/enola-labs/enola/internal/facts"
1011
)
@@ -30,6 +31,26 @@ func (r *LLMContextRenderer) Name() string {
3031
type section struct {
3132
name string
3233
content string
34+
// reserve sets this section's budget aside before layout, so an oversized
35+
// earlier section cannot starve it.
36+
reserve bool
37+
}
38+
39+
// cutAt returns the first n bytes of s, backed off to the nearest rune boundary.
40+
// The token budget is counted in bytes but the content is UTF-8, so a naive slice
41+
// can split a multi-byte character (a warning glyph, a non-ASCII identifier) and
42+
// emit invalid UTF-8.
43+
func cutAt(s string, n int) string {
44+
if n >= len(s) {
45+
return s
46+
}
47+
if n < 0 {
48+
n = 0
49+
}
50+
for n > 0 && !utf8.RuneStart(s[n]) {
51+
n--
52+
}
53+
return s[:n]
3354
}
3455

3556
// Render produces the llm_context.md artifact using progressive summarization.
@@ -38,59 +59,77 @@ type section struct {
3859
func (r *LLMContextRenderer) Render(ctx context.Context, snapshot *facts.Snapshot) ([]facts.Artifact, error) {
3960
// Sections ordered by priority (most important first)
4061
sections := []section{
41-
{"Repository Map", r.renderRepoMap(snapshot)},
62+
{name: "Repository Map", content: r.renderRepoMap(snapshot)},
4263
// Extraction Quality is a compact trust-preface: it states how complete this
43-
// extraction was (thin extraction, parse errors, coverage gaps) BEFORE the
44-
// reader relies on the map below. It sits high on purpose — it is small, and
45-
// it is the in-loop signal an agent uses to spot when enola's own coverage is
46-
// the problem, so it must survive the token budget rather than being the first
47-
// thing truncated on a large repo.
48-
{"Extraction Quality", r.renderExtractionQuality(snapshot)},
49-
{"Architecture Pattern", r.renderArchPattern(snapshot)},
50-
{"Cross-Repo Dependencies", r.renderCrossRepo(snapshot)},
51-
{"Entry Points", r.renderEntryPoints(snapshot)},
52-
{"Routes", r.renderRoutes(snapshot)},
53-
{"Storage", r.renderStorage(snapshot)},
54-
{"Dependency Rules", r.renderDependencyRules(snapshot)},
55-
{"Critical Modules", r.renderCriticalModules(snapshot)},
56-
{"Risk Zones", r.renderRiskZones(snapshot)},
57-
{"How to Add a Feature", r.renderFeatureGuide(snapshot)},
58-
{"Meta", r.renderMeta(snapshot)},
64+
// extraction was (thin extraction, parse errors, unresolved cross-repo edges)
65+
// before the reader relies on the graph. It is reserved rather than merely
66+
// placed high: Repository Map grows with the repo and, on a large multi-repo
67+
// snapshot, overran the whole budget — which truncated this section away on
68+
// exactly the clusters whose extraction is least complete.
69+
{name: "Extraction Quality", content: r.renderExtractionQuality(snapshot), reserve: true},
70+
{name: "Architecture Pattern", content: r.renderArchPattern(snapshot)},
71+
{name: "Cross-Repo Dependencies", content: r.renderCrossRepo(snapshot)},
72+
{name: "Entry Points", content: r.renderEntryPoints(snapshot)},
73+
{name: "Routes", content: r.renderRoutes(snapshot)},
74+
{name: "Storage", content: r.renderStorage(snapshot)},
75+
{name: "Dependency Rules", content: r.renderDependencyRules(snapshot)},
76+
{name: "Critical Modules", content: r.renderCriticalModules(snapshot)},
77+
{name: "Risk Zones", content: r.renderRiskZones(snapshot)},
78+
{name: "How to Add a Feature", content: r.renderFeatureGuide(snapshot)},
79+
{name: "Meta", content: r.renderMeta(snapshot)},
5980
}
6081

6182
header := "# Architecture Snapshot\n\n"
6283
maxChars := r.maxTokens * 4 // rough estimate: 1 token ~= 4 chars
6384
remaining := maxChars - len(header)
6485

86+
// Set the reserved sections' budget aside before laying anything out, so the
87+
// running total the loop spends can never encroach on them. Sections are still
88+
// emitted in priority order; reserving buys survival, not precedence.
89+
for _, sec := range sections {
90+
if sec.reserve {
91+
remaining -= len(sec.content)
92+
}
93+
}
94+
6595
var sb strings.Builder
6696
sb.WriteString(header)
6797

98+
// Once the budget is spent, keep scanning: a later reserved section is still owed
99+
// the bytes held back for it.
100+
spent := false
68101
for i, sec := range sections {
69102
if sec.content == "" {
70103
continue
71104
}
72-
if len(sec.content) <= remaining {
105+
if sec.reserve {
106+
sb.WriteString(sec.content)
107+
continue
108+
}
109+
if spent {
110+
continue
111+
}
112+
113+
switch {
114+
case len(sec.content) <= remaining:
73115
sb.WriteString(sec.content)
74116
remaining -= len(sec.content)
75-
} else if remaining > 200 {
117+
case remaining > 200:
76118
// Partially include this section
77-
cutpoint := remaining - 100
78-
if cutpoint < 0 {
79-
cutpoint = 0
80-
}
81-
sb.WriteString(sec.content[:cutpoint])
119+
sb.WriteString(cutAt(sec.content, remaining-100))
82120
fmt.Fprintf(&sb, "\n\n---\n*[Truncated in: %s]*\n", sec.name)
83-
break
84-
} else {
85-
// List omitted sections
121+
spent = true
122+
default:
123+
// List omitted sections. Reserved ones are not omitted, so naming them
124+
// here would be a lie.
86125
var omitted []string
87126
for _, s := range sections[i:] {
88-
if s.content != "" {
127+
if s.content != "" && !s.reserve {
89128
omitted = append(omitted, s.name)
90129
}
91130
}
92131
fmt.Fprintf(&sb, "\n\n---\n*[Omitted: %s]*\n", strings.Join(omitted, ", "))
93-
break
132+
spent = true
94133
}
95134
}
96135

@@ -570,7 +609,8 @@ func (r *LLMContextRenderer) renderFeatureGuide(snapshot *facts.Snapshot) string
570609
// reading the snapshot can SEE thin extraction (a bad ignore glob, a failing
571610
// extractor, unresolved cross-repo edges) without calling snapshot_receipt — the
572611
// in-loop signal for improving enola's own coverage. It emphasizes genuine
573-
// signals (parse errors, coverage gaps) and stays quiet when extraction is clean.
612+
// signals (parse errors, coverage gaps, unresolved edges) and stays quiet when
613+
// extraction is clean.
574614
func (r *LLMContextRenderer) renderExtractionQuality(snapshot *facts.Snapshot) string {
575615
m := snapshot.Meta
576616
// Nothing meaningful to report on an old/auto-loaded snapshot with no receipt.
@@ -589,12 +629,27 @@ func (r *LLMContextRenderer) renderExtractionQuality(snapshot *facts.Snapshot) s
589629
sb.WriteString("- Parse errors: 0\n")
590630
}
591631

592-
if m.Coverage != nil && m.Coverage.CoverageGaps > 0 {
593-
fmt.Fprintf(&sb, "- ⚠️ Cross-repo coverage gaps: **%d** service(s), %d unresolved outbound edge(s) — some links could not be resolved to a loaded repo\n",
594-
m.Coverage.CoverageGaps, m.Coverage.UnresolvedEdges)
632+
// Gaps and unresolved edges are independent: a connected service has resolved an
633+
// outbound edge by definition, yet may still have call sites that resolved to no
634+
// loaded repo. Guarding the count on the gap total hid it on exactly the healthy
635+
// multi-repo snapshot an agent is most likely to trust. External edges are a third
636+
// signal — expected, not a blind spot — so they are reported without a warning.
637+
if m.Coverage != nil {
638+
if m.Coverage.CoverageGaps > 0 {
639+
fmt.Fprintf(&sb, "- ⚠️ Cross-repo coverage gaps: **%d** service(s) with no resolved outbound edges — their outbound links could not be resolved to a loaded repo\n",
640+
m.Coverage.CoverageGaps)
641+
}
642+
if m.Coverage.UnresolvedEdges > 0 {
643+
fmt.Fprintf(&sb, "- ⚠️ Unresolved outbound edges: **%d** — call site(s) that did not resolve to a loaded repo (an unloaded repo, or an extractor blind spot)\n",
644+
m.Coverage.UnresolvedEdges)
645+
}
646+
if m.Coverage.ExternalEdges > 0 {
647+
fmt.Fprintf(&sb, "- Outbound edges to external hosts: **%d** — third-party APIs, expected (not a coverage blind spot)\n",
648+
m.Coverage.ExternalEdges)
649+
}
595650
}
596651

597-
if m.ParseErrors > 0 || (m.Coverage != nil && m.Coverage.CoverageGaps > 0) {
652+
if m.ParseErrors > 0 || (m.Coverage != nil && (m.Coverage.CoverageGaps > 0 || m.Coverage.UnresolvedEdges > 0)) {
598653
sb.WriteString("\n_These are extraction limits, not code defects — verify against source, and consider whether an extractor, detection, or ignore glob needs improving._\n")
599654
}
600655

internal/renderers/llmcontext/llm_test.go

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package llmcontext
22

33
import (
44
"context"
5+
"fmt"
56
"strings"
67
"testing"
8+
"unicode/utf8"
79

810
"github.com/enola-labs/enola/internal/facts"
911
)
@@ -249,6 +251,188 @@ func TestCriticalModules_FanInFanOut(t *testing.T) {
249251
}
250252
}
251253

254+
// renderWithCoverage renders a minimal snapshot whose only extraction-quality
255+
// signal is the given cross-repo coverage summary. A nil summary is the
256+
// single-repo shape: coverageSummary returns nil when there are no service nodes.
257+
func renderWithCoverage(t *testing.T, cov *facts.CoverageSummary) string {
258+
t.Helper()
259+
snapshot := makeSnapshot(nil, nil)
260+
snapshot.Meta.Coverage = cov
261+
return string(mustRender(t, snapshot))
262+
}
263+
264+
// lineWith returns the first rendered line containing substr, or "" if none does.
265+
func lineWith(content, substr string) string {
266+
for _, line := range strings.Split(content, "\n") {
267+
if strings.Contains(line, substr) {
268+
return line
269+
}
270+
}
271+
return ""
272+
}
273+
274+
// A connected service resolves at least one outbound edge, so it is not a coverage
275+
// gap — yet any call site it could not resolve still lands in UnresolvedEdges. That
276+
// is the normal healthy multi-repo shape, and it must not hide the count.
277+
func TestExtractionQuality_UnresolvedEdgesWithoutGaps(t *testing.T) {
278+
content := renderWithCoverage(t, &facts.CoverageSummary{
279+
ServicesTotal: 4, CoverageGaps: 0, UnresolvedEdges: 23, ExternalEdges: 2,
280+
})
281+
282+
unresolved := lineWith(content, "Unresolved outbound edges")
283+
if !strings.Contains(unresolved, "**23**") {
284+
t.Errorf("unresolved-edge count not rendered; got line %q in:\n%s", unresolved, content)
285+
}
286+
if !strings.Contains(unresolved, "⚠️") {
287+
t.Errorf("unresolved edges are an internal blind spot and must be flagged; got %q", unresolved)
288+
}
289+
290+
external := lineWith(content, "external hosts")
291+
if !strings.Contains(external, "**2**") {
292+
t.Errorf("external-edge count not rendered; got line %q in:\n%s", external, content)
293+
}
294+
// External call sites are expected, not a blind spot: flagging them would invite
295+
// exactly the misreading this section exists to prevent.
296+
if strings.Contains(external, "⚠️") {
297+
t.Errorf("external edges must not be flagged as a warning; got %q", external)
298+
}
299+
300+
if strings.Contains(content, "coverage gaps") {
301+
t.Errorf("no gaps in this snapshot, yet a gaps line rendered:\n%s", content)
302+
}
303+
if !strings.Contains(content, "These are extraction limits") {
304+
t.Errorf("unresolved edges are an extraction limit; footer missing in:\n%s", content)
305+
}
306+
}
307+
308+
func TestExtractionQuality_GapsStillRender(t *testing.T) {
309+
content := renderWithCoverage(t, &facts.CoverageSummary{
310+
ServicesTotal: 2, CoverageGaps: 1, UnresolvedEdges: 348,
311+
})
312+
313+
if gaps := lineWith(content, "Cross-repo coverage gaps"); !strings.Contains(gaps, "**1**") {
314+
t.Errorf("gap count not rendered; got line %q in:\n%s", gaps, content)
315+
}
316+
if unresolved := lineWith(content, "Unresolved outbound edges"); !strings.Contains(unresolved, "**348**") {
317+
t.Errorf("unresolved count not rendered alongside gaps; got line %q in:\n%s", unresolved, content)
318+
}
319+
if strings.Contains(content, "external hosts") {
320+
t.Errorf("no external edges in this snapshot, yet a line rendered:\n%s", content)
321+
}
322+
if !strings.Contains(content, "These are extraction limits") {
323+
t.Errorf("footer missing in:\n%s", content)
324+
}
325+
}
326+
327+
// Clean cross-repo coverage stays quiet: the section header still renders (Coverage
328+
// is non-nil), but nothing warns.
329+
func TestExtractionQuality_CleanCoverageQuiet(t *testing.T) {
330+
content := renderWithCoverage(t, &facts.CoverageSummary{ServicesTotal: 4})
331+
332+
if !strings.Contains(content, "## Extraction Quality") {
333+
t.Fatalf("section should render whenever Coverage is non-nil:\n%s", content)
334+
}
335+
for _, unwanted := range []string{
336+
"coverage gaps", "Unresolved outbound edges", "external hosts", "These are extraction limits",
337+
} {
338+
if strings.Contains(content, unwanted) {
339+
t.Errorf("clean coverage must not render %q in:\n%s", unwanted, content)
340+
}
341+
}
342+
}
343+
344+
// A single-repo snapshot has no service nodes, so Coverage is nil and there is no
345+
// receipt to report on at all.
346+
func TestExtractionQuality_SingleRepoNoCoverage(t *testing.T) {
347+
content := renderWithCoverage(t, nil)
348+
if strings.Contains(content, "## Extraction Quality") {
349+
t.Errorf("section should be absent with no receipt and no coverage:\n%s", content)
350+
}
351+
}
352+
353+
// bigRepoMapSnapshot returns a snapshot whose Repository Map alone overruns any
354+
// small token budget — the shape of a real multi-repo cluster.
355+
func bigRepoMapSnapshot(modules int, cov *facts.CoverageSummary) *facts.Snapshot {
356+
var ff []facts.Fact
357+
for i := 0; i < modules; i++ {
358+
ff = append(ff, facts.Fact{
359+
Kind: facts.KindModule,
360+
Name: fmt.Sprintf("service/%03d/%s", i, strings.Repeat("segment_", 4)),
361+
Props: map[string]any{"language": "go"},
362+
})
363+
}
364+
snapshot := makeSnapshot(ff, nil)
365+
snapshot.Meta.FilesSeen = 5552
366+
snapshot.Meta.FilesParsed = 2870
367+
snapshot.Meta.Coverage = cov
368+
return snapshot
369+
}
370+
371+
// The Extraction Quality preface is the signal an agent uses to calibrate how far to
372+
// trust the graph, and it is worth more than the tail of a module table. An earlier
373+
// oversized section must not starve it — which is exactly what happened on a 4-repo
374+
// snapshot, where the whole section vanished behind "[Truncated in: Repository Map]".
375+
func TestRender_ExtractionQualitySurvivesTruncatedRepoMap(t *testing.T) {
376+
const maxTokens = 1000
377+
snapshot := bigRepoMapSnapshot(400, &facts.CoverageSummary{
378+
ServicesTotal: 4, CoverageGaps: 0, UnresolvedEdges: 23, ExternalEdges: 2,
379+
})
380+
381+
artifacts, err := New(maxTokens).Render(context.Background(), snapshot)
382+
if err != nil {
383+
t.Fatalf("Render: %v", err)
384+
}
385+
content := string(artifacts[0].Content)
386+
387+
if !strings.Contains(content, "[Truncated in: Repository Map]") {
388+
t.Fatalf("test is not exercising truncation; Repository Map fit the budget:\n%s", content)
389+
}
390+
if !strings.Contains(content, "## Extraction Quality") {
391+
t.Errorf("Extraction Quality starved by an earlier oversized section:\n%s", content)
392+
}
393+
if !strings.Contains(content, "**23**") {
394+
t.Errorf("unresolved-edge count lost to truncation:\n%s", content)
395+
}
396+
// The reservation must come out of the budget, not be added on top of it.
397+
if limit := maxTokens*4 + 100; len(content) > limit {
398+
t.Errorf("content length %d exceeds budget %d", len(content), limit)
399+
}
400+
}
401+
402+
// The budget is a byte count, but the content is UTF-8: a warning glyph or a
403+
// non-ASCII identifier must never be cut in half into invalid UTF-8.
404+
func TestCutAt_NeverSplitsRune(t *testing.T) {
405+
s := "map: `ünïcode/módule` ⚠️ tail"
406+
for n := 0; n <= len(s); n++ {
407+
got := cutAt(s, n)
408+
if !utf8.ValidString(got) {
409+
t.Errorf("cutAt(%q, %d) = %q, which is not valid UTF-8", s, n, got)
410+
}
411+
if len(got) > n {
412+
t.Errorf("cutAt(%q, %d) returned %d bytes, over the limit", s, n, len(got))
413+
}
414+
}
415+
}
416+
417+
// The cut point moves with the budget, so sweep it: some budget lands mid-rune in a
418+
// non-ASCII module name, and the artifact must stay valid UTF-8 at every one.
419+
func TestRender_TruncatedOutputIsValidUTF8(t *testing.T) {
420+
snapshot := bigRepoMapSnapshot(400, nil)
421+
for i := range snapshot.Facts {
422+
snapshot.Facts[i].Name = fmt.Sprintf("sérvice/%03d/%s", i, strings.Repeat("ø", 20))
423+
}
424+
425+
for tokens := 200; tokens <= 400; tokens++ {
426+
artifacts, err := New(tokens).Render(context.Background(), snapshot)
427+
if err != nil {
428+
t.Fatalf("Render(%d): %v", tokens, err)
429+
}
430+
if !utf8.Valid(artifacts[0].Content) {
431+
t.Fatalf("truncation at maxTokens=%d produced invalid UTF-8", tokens)
432+
}
433+
}
434+
}
435+
252436
func TestFileDir(t *testing.T) {
253437
tests := []struct {
254438
input string

0 commit comments

Comments
 (0)