Skip to content

Commit c7297b1

Browse files
theRebelliousNerdtheRebelliousNerd
committed
fix(stats): deterministic /stats and atom normalization
- Handle /stats deterministically in chat and nerd run - Normalize shard type/verb atoms to avoid invalid //… symbols - nerd run no longer executes unary next_action; waits for routing/execution results
1 parent f58ebf4 commit c7297b1

6 files changed

Lines changed: 406 additions & 12 deletions

File tree

‎cmd/nerd/chat/helpers.go‎

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,108 @@ func countFileLines(workspace, path string) (int64, error) {
384384
return lines, nil
385385
}
386386

387+
func (m *Model) handleStatsIntent(ctx context.Context, intent perception.Intent) (string, error) {
388+
target := strings.TrimSpace(intent.Target)
389+
if target == "" || strings.EqualFold(target, "none") {
390+
return "", fmt.Errorf("stats requires a file or directory target")
391+
}
392+
393+
full := resolvePath(m.workspace, target)
394+
info, err := os.Stat(full)
395+
if err != nil {
396+
// If the target looks like a path, surface a clear error; otherwise the /stats key isn't wired yet.
397+
if strings.ContainsAny(target, `/\`) || filepath.Ext(target) != "" {
398+
return "", fmt.Errorf("file not found: %s", full)
399+
}
400+
return "", fmt.Errorf("unsupported stats target %q (try a file path)", target)
401+
}
402+
403+
if info.IsDir() {
404+
// Directory LOC: count lines across code-ish files under the directory.
405+
allowedExt := map[string]bool{
406+
".go": true,
407+
".mg": true,
408+
".mangle": true,
409+
".dl": true,
410+
".py": true,
411+
".js": true,
412+
".jsx": true,
413+
".ts": true,
414+
".tsx": true,
415+
".rs": true,
416+
".java": true,
417+
".kt": true,
418+
".c": true,
419+
".cc": true,
420+
".cpp": true,
421+
".h": true,
422+
".hpp": true,
423+
".cs": true,
424+
".sh": true,
425+
".ps1": true,
426+
}
427+
428+
var totalLines int64
429+
var countedFiles int64
430+
var skippedFiles int64
431+
const maxFileSize = 5 * 1024 * 1024 // 5MB
432+
433+
walkErr := filepath.WalkDir(full, func(path string, d fs.DirEntry, err error) error {
434+
if err != nil {
435+
return err
436+
}
437+
if ctx.Err() != nil {
438+
return ctx.Err()
439+
}
440+
name := d.Name()
441+
if d.IsDir() {
442+
// Skip hidden and dependency/cache directories.
443+
if strings.HasPrefix(name, ".") || name == "node_modules" || name == "vendor" {
444+
return filepath.SkipDir
445+
}
446+
if name == "bin" || name == "build" || name == "tmp" {
447+
return filepath.SkipDir
448+
}
449+
return nil
450+
}
451+
452+
ext := strings.ToLower(filepath.Ext(name))
453+
if !allowedExt[ext] {
454+
return nil
455+
}
456+
457+
if st, statErr := os.Stat(path); statErr == nil && st.Size() > maxFileSize {
458+
skippedFiles++
459+
return nil
460+
}
461+
462+
lines, err := countFileLines(m.workspace, path)
463+
if err != nil {
464+
skippedFiles++
465+
return nil
466+
}
467+
totalLines += lines
468+
countedFiles++
469+
return nil
470+
})
471+
if walkErr != nil {
472+
return "", walkErr
473+
}
474+
475+
resp := fmt.Sprintf("%s: %d total lines across %d files", target, totalLines, countedFiles)
476+
if skippedFiles > 0 {
477+
resp += fmt.Sprintf(" (%d skipped)", skippedFiles)
478+
}
479+
return resp, nil
480+
}
481+
482+
lines, err := countFileLines(m.workspace, target)
483+
if err != nil {
484+
return "", err
485+
}
486+
return fmt.Sprintf("%s: %d lines", target, lines), nil
487+
}
488+
387489
func writeFileContent(workspace, path, content string) error {
388490
full := resolvePath(workspace, path)
389491
dir := filepath.Dir(full)
@@ -1128,7 +1230,6 @@ func isConversationalIntent(intent perception.Intent) bool {
11281230
alwaysConversational := map[string]bool{
11291231
"/greet": true, // Greetings: hello, hi, hey
11301232
"/help": true, // Capability questions: what can you do?
1131-
"/stats": true, // Codebase statistics: how many files? breakdown?
11321233
"/knowledge": true, // Memory queries: what do you remember?
11331234
"/shadow": true, // What-if queries: what would happen if?
11341235
"/git": true, // Git operations: commit, push, status

‎cmd/nerd/chat/process.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,19 @@ OUTPUT:
387387
}
388388
}
389389

390+
// 1.7 STATS: Deterministic codebase/file statistics (no LLM follow-up).
391+
if shardType == "" && intent.Verb == "/stats" {
392+
m.ReportStatus("Stats: computing...")
393+
statsResp, err := m.handleStatsIntent(ctx, intent)
394+
if err != nil {
395+
return responseMsg(m.appendSystemSummary(
396+
fmt.Sprintf("Stats error: %v", err),
397+
m.collectSystemSummary(ctx, baseRoutingCount, baseExecCount),
398+
))
399+
}
400+
return responseMsg(m.appendSystemSummary(statsResp, m.collectSystemSummary(ctx, baseRoutingCount, baseExecCount)))
401+
}
402+
390403
// 1.7 DIRECT RESPONSE: For non-actionable verbs (/explain, /read, etc.) with
391404
// no shard and a valid perception response, return the perception response
392405
// directly. This handles greetings, capability questions, and general queries

‎cmd/nerd/main.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1482,6 +1482,8 @@ func runInstruction(cmd *cobra.Command, args []string) error {
14821482
ctx = usage.NewContext(ctx, cortex.UsageTracker)
14831483
}
14841484

1485+
baseRouting, baseExec := systemResultBaselines(cortex.Kernel)
1486+
14851487
emitter := articulation.NewEmitter()
14861488

14871489
// 2. Perception Layer: Transduce Input -> Intent
@@ -1494,6 +1496,28 @@ func runInstruction(cmd *cobra.Command, args []string) error {
14941496
zap.String("verb", intent.Verb),
14951497
zap.String("target", intent.Target))
14961498

1499+
// /stats is deterministic and should not require running shards or policy.
1500+
if intent.Verb == "/stats" {
1501+
stats, err := computeStats(ctx, cortex.Workspace, intent.Target)
1502+
if err != nil {
1503+
stats = fmt.Sprintf("Stats error: %v", err)
1504+
}
1505+
emitter.Emit(articulation.PiggybackEnvelope{
1506+
Surface: stats,
1507+
Control: articulation.ControlPacket{
1508+
IntentClassification: articulation.IntentClassification{
1509+
Category: intent.Category,
1510+
Verb: intent.Verb,
1511+
Target: intent.Target,
1512+
Constraint: intent.Constraint,
1513+
Confidence: intent.Confidence,
1514+
},
1515+
MangleUpdates: []string{fmt.Sprintf("observation(/stats, %q)", stats)},
1516+
},
1517+
})
1518+
return nil
1519+
}
1520+
14971521
// 3. World Model: Incremental Scan Workspace (fast)
14981522
logger.Debug("Scanning workspace incrementally", zap.String("path", cortex.Workspace))
14991523
scanRes, err := cortex.Scanner.ScanWorkspaceIncremental(ctx, cortex.Workspace, cortex.LocalDB, world.IncrementalOptions{SkipWhenUnchanged: true})
@@ -1556,18 +1580,18 @@ func runInstruction(cmd *cobra.Command, args []string) error {
15561580
actionFacts, _ := cortex.Kernel.Query("next_action")
15571581
if len(actionFacts) > 0 {
15581582
fact := actionFacts[0]
1559-
logger.Info("Executing action", zap.Any("action", fact))
1560-
result, err := cortex.VirtualStore.RouteAction(ctx, fact)
1561-
if err != nil {
1562-
output = fmt.Sprintf("Action failed: %v", err)
1563-
} else {
1564-
output = fmt.Sprintf("Action result: %v", result)
1565-
}
1583+
logger.Info("Derived next_action (unary; executed by system shards if enabled)", zap.Any("action", fact))
1584+
output = fmt.Sprintf("Next action: %v", fact.Args[0])
15661585
} else {
15671586
output = "No action derived from policy"
15681587
}
15691588
}
15701589

1590+
routingNew, execNew := waitForSystemResults(ctx, cortex.Kernel, baseRouting, baseExec, 3*time.Second)
1591+
if summary := formatSystemResults(routingNew, execNew); summary != "" {
1592+
output = output + "\n\n" + summary
1593+
}
1594+
15711595
// 6. Articulation Layer: Report
15721596
payload := articulation.PiggybackEnvelope{
15731597
Surface: fmt.Sprintf("Processed: %s\nResult: %s", userInput, output),

‎cmd/nerd/stats.go‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"context"
6+
"fmt"
7+
"io"
8+
"io/fs"
9+
"os"
10+
"path/filepath"
11+
"strings"
12+
)
13+
14+
func computeStats(ctx context.Context, workspace, target string) (string, error) {
15+
target = strings.TrimSpace(target)
16+
if target == "" || strings.EqualFold(target, "none") {
17+
return "", fmt.Errorf("stats requires a file or directory target")
18+
}
19+
20+
full := target
21+
if !filepath.IsAbs(full) {
22+
full = filepath.Join(workspace, target)
23+
}
24+
25+
info, err := os.Stat(full)
26+
if err != nil {
27+
return "", fmt.Errorf("file not found: %s", full)
28+
}
29+
30+
if !info.IsDir() {
31+
lines, err := countFileLines(full)
32+
if err != nil {
33+
return "", err
34+
}
35+
return fmt.Sprintf("%s: %d lines", target, lines), nil
36+
}
37+
38+
allowedExt := map[string]bool{
39+
".go": true,
40+
".mg": true,
41+
".mangle": true,
42+
".dl": true,
43+
".py": true,
44+
".js": true,
45+
".jsx": true,
46+
".ts": true,
47+
".tsx": true,
48+
".rs": true,
49+
".java": true,
50+
".kt": true,
51+
".c": true,
52+
".cc": true,
53+
".cpp": true,
54+
".h": true,
55+
".hpp": true,
56+
".cs": true,
57+
".sh": true,
58+
".ps1": true,
59+
}
60+
61+
var totalLines int64
62+
var countedFiles int64
63+
var skippedFiles int64
64+
const maxFileSize = 5 * 1024 * 1024 // 5MB
65+
66+
walkErr := filepath.WalkDir(full, func(path string, d fs.DirEntry, err error) error {
67+
if err != nil {
68+
return err
69+
}
70+
if ctx.Err() != nil {
71+
return ctx.Err()
72+
}
73+
74+
name := d.Name()
75+
if d.IsDir() {
76+
if strings.HasPrefix(name, ".") || name == "node_modules" || name == "vendor" {
77+
return filepath.SkipDir
78+
}
79+
if name == "bin" || name == "build" || name == "tmp" {
80+
return filepath.SkipDir
81+
}
82+
return nil
83+
}
84+
85+
ext := strings.ToLower(filepath.Ext(name))
86+
if !allowedExt[ext] {
87+
return nil
88+
}
89+
90+
if st, statErr := os.Stat(path); statErr == nil && st.Size() > maxFileSize {
91+
skippedFiles++
92+
return nil
93+
}
94+
95+
lines, err := countFileLines(path)
96+
if err != nil {
97+
skippedFiles++
98+
return nil
99+
}
100+
101+
totalLines += lines
102+
countedFiles++
103+
return nil
104+
})
105+
if walkErr != nil {
106+
return "", walkErr
107+
}
108+
109+
resp := fmt.Sprintf("%s: %d total lines across %d files", target, totalLines, countedFiles)
110+
if skippedFiles > 0 {
111+
resp += fmt.Sprintf(" (%d skipped)", skippedFiles)
112+
}
113+
return resp, nil
114+
}
115+
116+
func countFileLines(path string) (int64, error) {
117+
f, err := os.Open(path)
118+
if err != nil {
119+
return 0, err
120+
}
121+
defer f.Close()
122+
123+
reader := bufio.NewReaderSize(f, 256*1024)
124+
var lines int64
125+
var sawAnyByte bool
126+
lastByteWasNewline := false
127+
128+
for {
129+
chunk, err := reader.ReadBytes('\n')
130+
if len(chunk) > 0 {
131+
sawAnyByte = true
132+
lastByteWasNewline = chunk[len(chunk)-1] == '\n'
133+
if lastByteWasNewline {
134+
lines++
135+
}
136+
}
137+
if err != nil {
138+
if err == io.EOF {
139+
break
140+
}
141+
return 0, err
142+
}
143+
}
144+
145+
if !sawAnyByte {
146+
return 0, nil
147+
}
148+
if !lastByteWasNewline {
149+
lines++
150+
}
151+
return lines, nil
152+
}

0 commit comments

Comments
 (0)