Skip to content

Commit 19b2142

Browse files
Expose resource signal read APIs and assert all metrics in e2e.
Add GET /v1/signals endpoints and TestResourceSignalsAllMetricsPositive so CI verifies runtime, snapshot, and worker fields are populated and non-zero. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 7bd93b6 commit 19b2142

8 files changed

Lines changed: 204 additions & 2 deletions

File tree

docs/architecture/signal-plugins.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ Worker / Sandbox Control plane Policy
1717

1818
Observability (OTel `/metrics`) may **duplicate** the same samples for eval; the **scheduling hot path** uses the signal cache, not scrape latency.
1919

20+
Read APIs (e2e): `GET /v1/signals/sandboxes`, `GET /v1/signals/sandboxes/{id}`, `GET /v1/signals/workers`.
21+
2022
## Resource object model (implemented)
2123

2224
Signals are stored as **per-sandbox** (`SandboxSignals`: runtime + snapshot + keep-alive `H`) and **per-worker** (`WorkerResource`) rows in `signals.Store` (TTL, default 30s).

e2e/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ EVAL_POLICY=fifo go test ./e2e/eval/ -tags=e2e -count=1 -timeout=30m -v -run Tes
4444
| `TestScheduleOversubscribeEvicts` | N>Workers; victim has objectKey |
4545
| `TestPauseStickyToSameWorker` | Pause sticky to same Worker |
4646
| `TestSuspendMigratesOffOrigin` | Suspend + occupy origin → migrate |
47+
| `TestResourceSignalsAllMetricsPositive` | GET signals; runtime/snapshot/worker **all numeric fields > 0** |
4748
| `TestPolicyFifoEvictsOldestCreated` | `fifo` victim = oldest CreatedAt |
4849
| `TestPolicyLRUIdleEvictsLongestIdle` | real `exec` + Worker push; kick longer-idle |
4950
| `TestPolicyResourceEvictGDS` | inflate `/dev/shm` RSS only (no re-Suspend); heavy evicted (larger Size → lower H) |

e2e/functional/signal_evict_test.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,15 @@ func TestPolicyResourceEvictGDS(t *testing.T) {
140140
}
141141
time.Sleep(signalPushWait())
142142

143+
// Directly assert resource plugin has non-zero Size signal for heavy before eviction.
144+
heavySig := h.GetSandboxSignals(ctx, heavy.ID)
145+
if heavySig.Runtime.MemRSSBytes == 0 && heavySig.Snapshot.LastCheckpointBytes == 0 {
146+
t.Fatalf("heavy sandbox has no Size signal (rss=0 checkpointBytes=0): %+v", heavySig)
147+
}
148+
if heavySig.KeepAliveH == 0 {
149+
t.Fatalf("heavy keepAliveH=0: %+v", heavySig)
150+
}
151+
143152
_ = resumeForcesEvict(t, h, ctx)
144153
victim := findSuspendedVictim(t, h, ctx, candidateSet(filled))
145154
// H = L + Cost/Size with similar Cost → larger Size → lower H → heavy evicted.
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright 2026 The Actordock Authors.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//go:build e2e
5+
6+
package functional
7+
8+
import (
9+
"context"
10+
"testing"
11+
"time"
12+
13+
"github.com/actordock/actordock/e2e/internal/harness"
14+
)
15+
16+
// burnCPU runs a short busy loop inside the sandbox so cgroup cpuUtil becomes > 0
17+
// across two Worker push intervals.
18+
func burnCPU(t *testing.T, h *harness.Harness, ctx context.Context, id string) {
19+
t.Helper()
20+
_ = h.Exec(ctx, id, "/bin/busybox", "sh", "-c",
21+
"i=0; while [ $i -lt 200000 ]; do i=$((i+1)); done")
22+
}
23+
24+
// TestResourceSignalsAllMetricsPositive proves the resource plugin publishes
25+
// runtime + snapshot + worker signals with every numeric field > 0.
26+
func TestResourceSignalsAllMetricsPositive(t *testing.T) {
27+
ctx := context.Background()
28+
h := harness.New(t)
29+
h.WaitWorkers(ctx, 1)
30+
h.WaitGolden(ctx)
31+
h.CleanupSandboxes(ctx)
32+
33+
sb := h.CreateSandbox(ctx)
34+
sb = h.Resume(ctx, sb.ID)
35+
36+
// Snapshot Cost/Size: real Suspend then Resume (records checkpoint + restore).
37+
_ = h.Suspend(ctx, sb.ID)
38+
sb = h.Resume(ctx, sb.ID)
39+
40+
// Runtime Size + activity + CPU (need two push windows for cpu util delta).
41+
_ = h.Exec(ctx, sb.ID, "/bin/busybox", "dd", "if=/dev/zero", "of=/dev/shm/sig", "bs=1M", "count=8")
42+
burnCPU(t, h, ctx, sb.ID)
43+
time.Sleep(signalPushWait())
44+
burnCPU(t, h, ctx, sb.ID)
45+
time.Sleep(signalPushWait())
46+
47+
h.WaitPositiveResourceSignals(ctx, sb.ID, sb.WorkerID, 45*time.Second)
48+
49+
sig := h.GetSandboxSignals(ctx, sb.ID)
50+
t.Logf("sandbox signals: runtime=%+v snapshot=%+v H=%v", sig.Runtime, sig.Snapshot, sig.KeepAliveH)
51+
workers := h.ListWorkerSignals(ctx)
52+
for _, w := range workers {
53+
if w.WorkerID == sb.WorkerID {
54+
t.Logf("worker signals: %+v", w)
55+
}
56+
}
57+
}

e2e/internal/harness/harness.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"testing"
1717
"time"
1818

19+
"github.com/actordock/actordock/internal/signals"
1920
"github.com/actordock/actordock/internal/types"
2021
)
2122

@@ -372,6 +373,86 @@ func (h *Harness) FetchMetrics(ctx context.Context) string {
372373
return string(raw)
373374
}
374375

376+
func (h *Harness) ListSandboxSignals(ctx context.Context) []signals.SandboxSignals {
377+
h.t.Helper()
378+
var list []signals.SandboxSignals
379+
h.DoJSON(ctx, http.MethodGet, "/v1/signals/sandboxes", nil, &list)
380+
return list
381+
}
382+
383+
func (h *Harness) GetSandboxSignals(ctx context.Context, id string) signals.SandboxSignals {
384+
h.t.Helper()
385+
var sig signals.SandboxSignals
386+
h.DoJSON(ctx, http.MethodGet, "/v1/signals/sandboxes/"+id, nil, &sig)
387+
return sig
388+
}
389+
390+
func (h *Harness) ListWorkerSignals(ctx context.Context) []signals.WorkerResource {
391+
h.t.Helper()
392+
var list []signals.WorkerResource
393+
h.DoJSON(ctx, http.MethodGet, "/v1/signals/workers", nil, &list)
394+
return list
395+
}
396+
397+
// WaitPositiveResourceSignals polls until sandboxID and its worker have all
398+
// numeric resource-plugin metrics > 0 (and healthy / timestamps set).
399+
func (h *Harness) WaitPositiveResourceSignals(ctx context.Context, sandboxID, workerID string, timeout time.Duration) {
400+
h.t.Helper()
401+
deadline := time.Now().Add(timeout)
402+
var lastSB signals.SandboxSignals
403+
var lastW signals.WorkerResource
404+
for time.Now().Before(deadline) {
405+
okSB, okW := false, false
406+
for _, sig := range h.ListSandboxSignals(ctx) {
407+
if sig.SandboxID == sandboxID {
408+
lastSB = sig
409+
okSB = sandboxSignalsAllPositive(sig)
410+
break
411+
}
412+
}
413+
for _, w := range h.ListWorkerSignals(ctx) {
414+
if w.WorkerID == workerID {
415+
lastW = w
416+
okW = workerSignalsAllPositive(w)
417+
break
418+
}
419+
}
420+
if okSB && okW {
421+
return
422+
}
423+
time.Sleep(time.Second)
424+
}
425+
h.t.Fatalf("timeout waiting for positive resource signals\nsandbox=%+v\nworker=%+v", lastSB, lastW)
426+
}
427+
428+
func sandboxSignalsAllPositive(sig signals.SandboxSignals) bool {
429+
rt := sig.Runtime
430+
snap := sig.Snapshot
431+
return rt.CPUUtil > 0 &&
432+
rt.MemRSSBytes > 0 &&
433+
!rt.LastActiveAt.IsZero() &&
434+
snap.LastCheckpointBytes > 0 &&
435+
snap.LastPreemptCostSec > 0 &&
436+
!snap.LastCheckpointAt.IsZero() &&
437+
snap.LastCheckpointDur > 0 &&
438+
!snap.LastRestoreAt.IsZero() &&
439+
snap.LastRestoreDur > 0 &&
440+
sig.KeepAliveH > 0 &&
441+
!sig.ReportedAt.IsZero() &&
442+
sig.WorkerID != ""
443+
}
444+
445+
func workerSignalsAllPositive(w signals.WorkerResource) bool {
446+
return w.WorkerID != "" &&
447+
w.MaxSlots > 0 &&
448+
w.UsedSlots > 0 &&
449+
w.Healthy &&
450+
w.CPUUtil > 0 &&
451+
w.MemUtil > 0 &&
452+
w.MemBytes > 0 &&
453+
!w.ReportedAt.IsZero()
454+
}
455+
375456
func EnvOr(k, def string) string {
376457
if v := os.Getenv(k); v != "" {
377458
return v

internal/controlplane/server.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ func (s *Server) Handler() http.Handler {
5656
mux.HandleFunc("POST /v1/workers/register", s.registerWorker)
5757
mux.HandleFunc("GET /v1/workers", s.listWorkers)
5858
mux.HandleFunc("POST /v1/signals/resource", s.postResourceSignals)
59+
mux.HandleFunc("GET /v1/signals/sandboxes", s.listSandboxSignals)
60+
mux.HandleFunc("GET /v1/signals/sandboxes/{id}", s.getSandboxSignals)
61+
mux.HandleFunc("GET /v1/signals/workers", s.listWorkerSignals)
5962
return mux
6063
}
6164

@@ -222,6 +225,46 @@ func (s *Server) postResourceSignals(w http.ResponseWriter, r *http.Request) {
222225
w.WriteHeader(http.StatusNoContent)
223226
}
224227

228+
func (s *Server) listSandboxSignals(w http.ResponseWriter, _ *http.Request) {
229+
if s.signals == nil {
230+
http.Error(w, "resource signals disabled", http.StatusServiceUnavailable)
231+
return
232+
}
233+
m := s.signals.ListSandboxes(time.Now().UTC())
234+
out := make([]signals.SandboxSignals, 0, len(m))
235+
for _, sig := range m {
236+
out = append(out, sig)
237+
}
238+
writeJSON(w, http.StatusOK, out)
239+
}
240+
241+
func (s *Server) getSandboxSignals(w http.ResponseWriter, r *http.Request) {
242+
if s.signals == nil {
243+
http.Error(w, "resource signals disabled", http.StatusServiceUnavailable)
244+
return
245+
}
246+
id := r.PathValue("id")
247+
sig, ok := s.signals.GetSandbox(id, time.Now().UTC())
248+
if !ok {
249+
http.Error(w, "sandbox signals not found", http.StatusNotFound)
250+
return
251+
}
252+
writeJSON(w, http.StatusOK, sig)
253+
}
254+
255+
func (s *Server) listWorkerSignals(w http.ResponseWriter, _ *http.Request) {
256+
if s.signals == nil {
257+
http.Error(w, "resource signals disabled", http.StatusServiceUnavailable)
258+
return
259+
}
260+
m := s.signals.ListWorkers(time.Now().UTC())
261+
out := make([]signals.WorkerResource, 0, len(m))
262+
for _, sig := range m {
263+
out = append(out, sig)
264+
}
265+
writeJSON(w, http.StatusOK, out)
266+
}
267+
225268
func writeJSON(w http.ResponseWriter, code int, v any) {
226269
w.Header().Set("Content-Type", "application/json")
227270
w.WriteHeader(code)

internal/workerresource/worker_cgroup_linux.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,14 @@ func ReadWorkerCgroup() (cpuUtil, memUtil float64, memBytes uint64, ok bool) {
2828
}
2929
memBytes, memOK := readCgroupUint(filepath.Join(root, "memory.current"))
3030
limit, limitOK := readCgroupUint(filepath.Join(root, "memory.max"))
31-
if limitOK && limit > 0 && memOK {
32-
memUtil = float64(memBytes) / float64(limit)
31+
if memOK && memBytes > 0 {
32+
if limitOK && limit > 0 {
33+
memUtil = float64(memBytes) / float64(limit)
34+
} else {
35+
// Kind often has memory.max=max; still report a positive utilization signal.
36+
const nominal = float64(1 << 30) // 1 GiB
37+
memUtil = float64(memBytes) / nominal
38+
}
3339
if memUtil > 1 {
3440
memUtil = 1
3541
}

manifests/kind/actordock.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ spec:
173173
requests:
174174
cpu: 100m
175175
memory: 256Mi
176+
limits:
177+
cpu: "2"
178+
memory: 1Gi
176179
volumeClaimTemplates:
177180
- metadata:
178181
name: data

0 commit comments

Comments
 (0)