Skip to content

Commit 74030f3

Browse files
authored
feat(activity): size-based retention cap for the activity log (spec 073, MCP-1002) (#585)
* feat(activity): size-based retention cap for the activity log (spec 073) Bounds config.db growth: the activity log was pruned only by age (90d) and count (100k); large per-record payloads let it reach hundreds of MB while under both caps (~438MB/93k rows observed). Add a configurable total-size cap. - config: activity_max_size_mb (default 256, 0=disabled) - storage: Manager.PruneActivitiesToSize(maxBytes) — deletes oldest-first until the activity_records bucket data is within budget; always keeps the newest record; <=0 is a no-op - runtime: plumb the byte budget into ActivityService; run the size prune in runRetentionCleanup after the age/count prunes (and at startup) - docs: document activity_max_size_mb + the compaction caveat TDD: 4 storage tests (size, oldest-first, keep-newest, disabled) + 2 runtime wiring tests; go test ./internal/storage ./internal/config + activity runtime tests green; go vet + gofmt clean. Refs MCP-1002. * chore(oas): regenerate swagger for activity_max_size_mb config field
1 parent 3b9a477 commit 74030f3

15 files changed

Lines changed: 621 additions & 4 deletions

File tree

docs/features/activity-log.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,7 @@ Activity logging is enabled by default. Configure via `mcp_config.json`:
405405
{
406406
"activity_retention_days": 90,
407407
"activity_max_records": 100000,
408+
"activity_max_size_mb": 256,
408409
"activity_max_response_size": 65536,
409410
"activity_cleanup_interval_min": 60
410411
}
@@ -414,9 +415,12 @@ Activity logging is enabled by default. Configure via `mcp_config.json`:
414415
|---------|---------|-------------|
415416
| `activity_retention_days` | 90 | Days to retain activity records |
416417
| `activity_max_records` | 100000 | Maximum records before pruning oldest |
418+
| `activity_max_size_mb` | 256 | Maximum total activity-log size in MB before pruning oldest (`0` disables). Runs alongside the age and count caps to bound `config.db` growth when records carry large payloads. |
417419
| `activity_max_response_size` | 65536 | Max response size stored (bytes) |
418420
| `activity_cleanup_interval_min` | 60 | Background cleanup interval (minutes) |
419421

422+
> **Why the size cap?** The age and count caps alone do not bound disk: with large per-record payloads the log can reach hundreds of MB while still under 100k records / 90 days. `activity_max_size_mb` removes the oldest records (always keeping the newest) until the log is within the byte budget. Note: pruning frees pages for reuse but does not shrink the database file on disk (BBolt does not return freed pages to the OS).
423+
420424
## Use Cases
421425

422426
### Debugging Tool Calls

internal/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ type Config struct {
138138
// Activity logging settings (RFC-003)
139139
ActivityRetentionDays int `json:"activity_retention_days,omitempty" mapstructure:"activity-retention-days"` // Max age before pruning (default: 90)
140140
ActivityMaxRecords int `json:"activity_max_records,omitempty" mapstructure:"activity-max-records"` // Max records before pruning (default: 100000)
141+
ActivityMaxSizeMB int `json:"activity_max_size_mb,omitempty" mapstructure:"activity-max-size-mb"` // Max total activity-log size in MB before pruning oldest (default: 256, 0=disabled)
141142
ActivityMaxResponseSize int `json:"activity_max_response_size,omitempty" mapstructure:"activity-max-response-size"` // Response truncation limit in bytes (default: 65536)
142143
ActivityCleanupIntervalMin int `json:"activity_cleanup_interval_min,omitempty" mapstructure:"activity-cleanup-interval-min"` // Background cleanup interval in minutes (default: 60)
143144

@@ -1030,6 +1031,7 @@ func DefaultConfig() *Config {
10301031
// Activity logging defaults (RFC-003)
10311032
ActivityRetentionDays: 90, // 90 days retention
10321033
ActivityMaxRecords: 100000, // 100K records max
1034+
ActivityMaxSizeMB: 256, // 256MB total activity-log size cap (0 = disabled)
10331035
ActivityMaxResponseSize: 65536, // 64KB response truncation
10341036
ActivityCleanupIntervalMin: 60, // 1 hour cleanup interval
10351037

internal/runtime/activity_service.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ const (
2020
DefaultRetentionMaxRecords = 10000
2121
// DefaultRetentionCheckInterval is the default interval between retention checks (1 hour)
2222
DefaultRetentionCheckInterval = 1 * time.Hour
23+
// DefaultRetentionMaxSizeBytes is the default total activity-log size cap (256MB)
24+
DefaultRetentionMaxSizeBytes int64 = 256 * 1024 * 1024
2325
)
2426

2527
// SensitiveDataEventEmitter provides the ability to emit sensitive data detection events.
@@ -43,6 +45,7 @@ type ActivityService struct {
4345
// Retention configuration
4446
maxAge time.Duration
4547
maxRecords int
48+
maxSizeBytes int64 // total activity-log size cap in bytes (0 = disabled)
4649
checkInterval time.Duration
4750

4851
// Sensitive data detector (Spec 026)
@@ -68,6 +71,7 @@ func NewActivityService(storage *storage.Manager, logger *zap.Logger) *ActivityS
6871
done: make(chan struct{}),
6972
maxAge: DefaultRetentionMaxAge,
7073
maxRecords: DefaultRetentionMaxRecords,
74+
maxSizeBytes: DefaultRetentionMaxSizeBytes,
7175
checkInterval: DefaultRetentionCheckInterval,
7276
detector: nil, // Detector is optional, set via SetDetector
7377
usage: newUsageStore(),
@@ -92,7 +96,7 @@ func (s *ActivityService) SetEventEmitter(emitter SensitiveDataEventEmitter) {
9296
// maxAge: maximum age for records (0 = no age limit)
9397
// maxRecords: maximum number of records (0 = no count limit)
9498
// checkInterval: how often to run retention cleanup
95-
func (s *ActivityService) SetRetentionConfig(maxAge time.Duration, maxRecords int, checkInterval time.Duration) {
99+
func (s *ActivityService) SetRetentionConfig(maxAge time.Duration, maxRecords int, checkInterval time.Duration, maxSizeBytes int64) {
96100
if maxAge > 0 {
97101
s.maxAge = maxAge
98102
}
@@ -102,6 +106,11 @@ func (s *ActivityService) SetRetentionConfig(maxAge time.Duration, maxRecords in
102106
if checkInterval > 0 {
103107
s.checkInterval = checkInterval
104108
}
109+
// maxSizeBytes may be explicitly set to 0 to DISABLE the size cap, so a
110+
// negative sentinel (-1) means "leave unchanged"; >= 0 is applied verbatim.
111+
if maxSizeBytes >= 0 {
112+
s.maxSizeBytes = maxSizeBytes
113+
}
105114
}
106115

107116
// Start begins listening for activity events and persisting them.
@@ -189,6 +198,19 @@ func (s *ActivityService) runRetentionCleanup() {
189198
zap.Int("max_records", s.maxRecords))
190199
}
191200
}
201+
202+
// Prune by total size (runs after age+count; 0 disables). Bounds config.db
203+
// growth from large per-record payloads that stay under the count/age caps.
204+
if s.maxSizeBytes > 0 {
205+
deleted, err := s.storage.PruneActivitiesToSize(s.maxSizeBytes)
206+
if err != nil {
207+
s.logger.Error("Failed to prune activities to size budget", zap.Error(err))
208+
} else if deleted > 0 {
209+
s.logger.Info("Pruned activity records to size budget",
210+
zap.Int("deleted", deleted),
211+
zap.Int64("max_size_mb", s.maxSizeBytes/(1024*1024)))
212+
}
213+
}
192214
}
193215

194216
// Stop gracefully shuts down the activity service.
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package runtime
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
"go.uber.org/zap"
12+
13+
"github.com/smart-mcp-proxy/mcpproxy-go/internal/storage"
14+
)
15+
16+
func seedRuntimeActivities(t *testing.T, store *storage.Manager, prefix string, n, payload int) {
17+
t.Helper()
18+
now := time.Now().UTC()
19+
for i := 0; i < n; i++ {
20+
require.NoError(t, store.SaveActivity(&storage.ActivityRecord{
21+
ID: fmt.Sprintf("%s-%02d", prefix, i),
22+
Type: storage.ActivityTypeToolCall,
23+
Status: "success",
24+
Response: strings.Repeat("x", payload),
25+
Timestamp: now.Add(time.Duration(i) * time.Second), // all recent; last is newest
26+
}))
27+
}
28+
}
29+
30+
// The size cap runs inside the existing retention cleanup and removes oldest
31+
// records until the log is within budget, leaving the age/count caps intact.
32+
func TestActivityRetention_SizeCapRemovesOldest(t *testing.T) {
33+
store, cleanup := setupTestStorage(t)
34+
defer cleanup()
35+
svc := NewActivityService(store, zap.NewNop())
36+
37+
// Only the size cap should act: keep default age/count caps (7d / 10000),
38+
// seed recent records well under those, and set a small size budget.
39+
svc.SetRetentionConfig(0, 0, 0, 30*1024) // 30KB; age/count left unchanged
40+
seedRuntimeActivities(t, store, "rt", 10, 10*1024)
41+
42+
svc.runRetentionCleanup()
43+
44+
newest, err := store.GetActivity("rt-09")
45+
require.NoError(t, err)
46+
assert.NotNil(t, newest, "newest record must survive the size cap")
47+
oldest, err := store.GetActivity("rt-00")
48+
require.NoError(t, err)
49+
assert.Nil(t, oldest, "oldest record should be pruned by the size cap")
50+
51+
// Log is now within budget: a direct size prune deletes nothing more.
52+
again, err := store.PruneActivitiesToSize(30 * 1024)
53+
require.NoError(t, err)
54+
assert.Equal(t, 0, again, "log should already be within the size budget")
55+
}
56+
57+
func TestActivityRetention_SizeCapDisabled(t *testing.T) {
58+
store, cleanup := setupTestStorage(t)
59+
defer cleanup()
60+
svc := NewActivityService(store, zap.NewNop())
61+
62+
svc.SetRetentionConfig(0, 0, 0, 0) // disable size cap (age/count stay default)
63+
seedRuntimeActivities(t, store, "d", 5, 10*1024)
64+
65+
svc.runRetentionCleanup()
66+
67+
for i := 0; i < 5; i++ {
68+
rec, err := store.GetActivity(fmt.Sprintf("d-%02d", i))
69+
require.NoError(t, err)
70+
assert.NotNilf(t, rec, "d-%02d must survive when the size cap is disabled", i)
71+
}
72+
}

internal/runtime/runtime.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,13 +212,17 @@ func New(cfg *config.Config, cfgPath string, logger *zap.Logger) (*Runtime, erro
212212
}
213213

214214
// Wire activity retention config from config file
215-
if cfg.ActivityRetentionDays > 0 || cfg.ActivityMaxRecords > 0 || cfg.ActivityCleanupIntervalMin > 0 {
215+
if cfg.ActivityRetentionDays > 0 || cfg.ActivityMaxRecords > 0 || cfg.ActivityCleanupIntervalMin > 0 || cfg.ActivityMaxSizeMB >= 0 {
216216
maxAge := time.Duration(cfg.ActivityRetentionDays) * 24 * time.Hour
217217
checkInterval := time.Duration(cfg.ActivityCleanupIntervalMin) * time.Minute
218-
activityService.SetRetentionConfig(maxAge, cfg.ActivityMaxRecords, checkInterval)
218+
// ActivityMaxSizeMB: 0 disables the size cap, so pass the explicit byte
219+
// value (>= 0 is applied; -1 would mean "unchanged").
220+
maxSizeBytes := int64(cfg.ActivityMaxSizeMB) * 1024 * 1024
221+
activityService.SetRetentionConfig(maxAge, cfg.ActivityMaxRecords, checkInterval, maxSizeBytes)
219222
logger.Info("Activity retention config applied",
220223
zap.Int("retention_days", cfg.ActivityRetentionDays),
221224
zap.Int("max_records", cfg.ActivityMaxRecords),
225+
zap.Int("max_size_mb", cfg.ActivityMaxSizeMB),
222226
zap.Int("cleanup_interval_min", cfg.ActivityCleanupIntervalMin))
223227
}
224228

internal/storage/activity.go

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,76 @@ func (m *Manager) PruneExcessActivities(maxRecords int, targetPercent float64) (
454454
return deleted, nil
455455
}
456456

457+
// PruneActivitiesToSize deletes the oldest activity records until the activity
458+
// log's stored data (sum of key+value bytes) is at or below maxBytes. Activity
459+
// keys are timestamp-ordered, so a single forward cursor pass removes
460+
// oldest-first. The newest record is ALWAYS retained — the log is never emptied
461+
// while any record exists, even if that newest record alone exceeds the budget.
462+
// maxBytes <= 0 disables size pruning (no-op). Returns the number deleted.
463+
func (m *Manager) PruneActivitiesToSize(maxBytes int64) (int, error) {
464+
if maxBytes <= 0 {
465+
return 0, nil
466+
}
467+
468+
var deleted int
469+
470+
err := m.db.db.Update(func(tx *bbolt.Tx) error {
471+
bucket := tx.Bucket([]byte(ActivityRecordsBucket))
472+
if bucket == nil {
473+
return nil
474+
}
475+
476+
keyCount := bucket.Stats().KeyN
477+
if keyCount <= 1 {
478+
return nil // never empty the log (always keep the newest record)
479+
}
480+
481+
// Total stored bytes for the bucket.
482+
var total int64
483+
_ = bucket.ForEach(func(k, v []byte) error {
484+
total += int64(len(k) + len(v))
485+
return nil
486+
})
487+
if total <= maxBytes {
488+
return nil
489+
}
490+
491+
// Delete oldest-first (smallest keys) until within budget, but NEVER the
492+
// last (newest) record — stop before processing it.
493+
var keysToDelete [][]byte
494+
cursor := bucket.Cursor()
495+
processed := 0
496+
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
497+
if total <= maxBytes || processed == keyCount-1 {
498+
break
499+
}
500+
keysToDelete = append(keysToDelete, append([]byte{}, k...))
501+
total -= int64(len(k) + len(v))
502+
processed++
503+
}
504+
505+
for _, key := range keysToDelete {
506+
if err := bucket.Delete(key); err != nil {
507+
return fmt.Errorf("failed to delete activity for size cap: %w", err)
508+
}
509+
deleted++
510+
}
511+
return nil
512+
})
513+
514+
if err != nil {
515+
return deleted, err
516+
}
517+
518+
if deleted > 0 {
519+
m.logger.Infow("Pruned activity records to size budget",
520+
"deleted", deleted,
521+
"max_bytes", maxBytes)
522+
}
523+
524+
return deleted, nil
525+
}
526+
457527
// SaveActivityAsync saves an activity record asynchronously.
458528
// This is non-blocking and suitable for recording tool calls without impacting latency.
459529
func (m *Manager) SaveActivityAsync(record *ActivityRecord) {
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package storage
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
// saveSizedActivities saves n records (id-00 oldest → id-(n-1) newest), each
14+
// padded so its stored value is ~payload bytes.
15+
func saveSizedActivities(t *testing.T, m *Manager, n, payload int) {
16+
t.Helper()
17+
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
18+
for i := 0; i < n; i++ {
19+
require.NoError(t, m.SaveActivity(&ActivityRecord{
20+
ID: fmt.Sprintf("id-%02d", i),
21+
Type: ActivityTypeToolCall,
22+
Status: "success",
23+
Response: strings.Repeat("x", payload),
24+
Timestamp: base.Add(time.Duration(i) * time.Minute),
25+
}))
26+
}
27+
}
28+
29+
// exists reports whether an activity record with the given id is present.
30+
// GetActivity returns (nil, nil) — not an error — when a record is absent.
31+
func exists(t *testing.T, m *Manager, id string) bool {
32+
t.Helper()
33+
rec, err := m.GetActivity(id)
34+
require.NoError(t, err)
35+
return rec != nil
36+
}
37+
38+
func TestPruneActivitiesToSize_RemovesOldestUntilUnderBudget(t *testing.T) {
39+
m, cleanup := setupTestStorageForActivity(t)
40+
defer cleanup()
41+
42+
// 10 records × ~10KB ≈ 100KB total.
43+
saveSizedActivities(t, m, 10, 10*1024)
44+
budget := int64(45 * 1024) // only the newest few fit
45+
46+
deleted, err := m.PruneActivitiesToSize(budget)
47+
require.NoError(t, err)
48+
assert.Greater(t, deleted, 0)
49+
50+
// Oldest pruned, newest retained.
51+
assert.False(t, exists(t, m, "id-00"), "oldest record should be pruned")
52+
assert.True(t, exists(t, m, "id-09"), "newest record must be retained")
53+
54+
// Idempotent: a second pass deletes nothing more.
55+
again, err := m.PruneActivitiesToSize(budget)
56+
require.NoError(t, err)
57+
assert.Equal(t, 0, again, "second pass should be a no-op")
58+
}
59+
60+
func TestPruneActivitiesToSize_AlreadyUnderBudget_NoOp(t *testing.T) {
61+
m, cleanup := setupTestStorageForActivity(t)
62+
defer cleanup()
63+
saveSizedActivities(t, m, 5, 1024) // ~5KB total
64+
65+
deleted, err := m.PruneActivitiesToSize(10 * 1024 * 1024) // 10MB budget
66+
require.NoError(t, err)
67+
assert.Equal(t, 0, deleted)
68+
for i := 0; i < 5; i++ {
69+
assert.True(t, exists(t, m, fmt.Sprintf("id-%02d", i)))
70+
}
71+
}
72+
73+
func TestPruneActivitiesToSize_KeepsNewestEvenIfOverBudget(t *testing.T) {
74+
m, cleanup := setupTestStorageForActivity(t)
75+
defer cleanup()
76+
// 5 records × 10KB; budget smaller than a single record.
77+
saveSizedActivities(t, m, 5, 10*1024)
78+
79+
deleted, err := m.PruneActivitiesToSize(1024) // 1KB < one record
80+
require.NoError(t, err)
81+
assert.Equal(t, 4, deleted, "all but the newest should be deleted")
82+
83+
// Only the newest survives — the log is never emptied.
84+
assert.True(t, exists(t, m, "id-04"), "newest must survive")
85+
for i := 0; i < 4; i++ {
86+
assert.Falsef(t, exists(t, m, fmt.Sprintf("id-%02d", i)), "id-%02d should be pruned", i)
87+
}
88+
}
89+
90+
func TestPruneActivitiesToSize_DisabledWhenZeroOrNegative(t *testing.T) {
91+
m, cleanup := setupTestStorageForActivity(t)
92+
defer cleanup()
93+
saveSizedActivities(t, m, 4, 10*1024)
94+
95+
for _, budget := range []int64{0, -1} {
96+
deleted, err := m.PruneActivitiesToSize(budget)
97+
require.NoError(t, err)
98+
assert.Equalf(t, 0, deleted, "budget %d disables size pruning", budget)
99+
}
100+
for i := 0; i < 4; i++ {
101+
assert.True(t, exists(t, m, fmt.Sprintf("id-%02d", i)))
102+
}
103+
}

oas/docs.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

oas/swagger.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ components:
1111
activity_max_response_size:
1212
description: 'Response truncation limit in bytes (default: 65536)'
1313
type: integer
14+
activity_max_size_mb:
15+
description: 'Max total activity-log size in MB before pruning oldest (default:
16+
256, 0=disabled)'
17+
type: integer
1418
activity_retention_days:
1519
description: Activity logging settings (RFC-003)
1620
type: integer

0 commit comments

Comments
 (0)