diff --git a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main.go b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main.go index 8fea0dcd8..6cba4a18c 100644 --- a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main.go +++ b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main.go @@ -63,7 +63,7 @@ func loadConfig(getenv func(string) string, nextGeneration generationSource) (co } journalPath := strings.TrimSpace(getenv("CPAMP_RUNTIME_JOURNAL_PATH")) executable := strings.TrimSpace(getenv("CPAMP_CPA_EXECUTABLE")) - if err := validateStartConfig(journalPath, executable); err != nil { + if err := validateLifecycleConfig(journalPath, executable); err != nil { return config{}, err } var generation uint64 @@ -110,6 +110,10 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler) err return serveWithShutdownTimeout(ctx, listener, handler, shutdownTimeout) } +type lifecycleAdmissionCloser interface { + CloseAdmission() +} + func serveWithShutdownTimeout(ctx context.Context, listener net.Listener, handler http.Handler, timeout time.Duration) error { server := newHTTPServer(handler) serveResult := make(chan error, 1) @@ -127,6 +131,9 @@ func serveWithShutdownTimeout(ctx context.Context, listener net.Listener, handle case <-ctx.Done(): } + if lifecycle, ok := handler.(lifecycleAdmissionCloser); ok { + lifecycle.CloseAdmission() + } shutdownCtx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() shutdownErr := server.Shutdown(shutdownCtx) diff --git a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main_test.go b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main_test.go index fde59ae38..e67bd6cd9 100644 --- a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main_test.go +++ b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/main_test.go @@ -4,13 +4,21 @@ import ( "context" "encoding/json" "errors" + "io" "net" "net/http" "net/http/httptest" + "path/filepath" "reflect" "strings" + "sync" "testing" "time" + + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/cpaprocess" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/journal" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/lifecycle" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/protocol" ) func TestLoadConfig(t *testing.T) { @@ -176,6 +184,185 @@ func TestServeStopsCleanlyWhenContextIsCanceled(t *testing.T) { } } +func TestSupervisorShutdownClosesLifecycleAdmissionBeforeDraining(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseAccepted := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(func() { + cancel() + releaseAccepted() + _ = listener.Close() + }) + + journalPath := filepath.Join(t.TempDir(), "operations.sqlite") + store, err := journal.Open(t.Context(), journalPath, journal.Options{}) + if err != nil { + t.Fatal(err) + } + trackedJournal := &shutdownJournal{Store: store} + child := &shutdownProcess{entered: make(chan struct{}), release: release} + executor, err := lifecycle.NewExecutor( + journal.Authority{RuntimeIdentity: "runtime-01", RuntimeGeneration: 41}, + trackedJournal, + child, + "supervisor-local-cpa", + ) + if err != nil { + t.Fatal(err) + } + queued := &queuedStartExecutor{ + next: executor, + operationID: "queued-start", + waiting: make(chan struct{}), + } + protocolHandler, err := protocol.NewHandler(protocol.Config{ + RuntimeIdentity: "runtime-01", + RuntimeGeneration: 41, + Token: "runtime-token", + Start: queued, + Stop: executor, + }) + if err != nil { + t.Fatal(err) + } + runtime := &runtimeHandler{Handler: protocolHandler, executor: executor} + shutdown := &shutdownAdmissionObserver{runtimeHandler: runtime, closed: make(chan struct{})} + serveResult := make(chan error, 1) + go func() { + serveErr := serve(ctx, listener, shutdown) + serveResult <- errors.Join(serveErr, runtime.Close()) + }() + + client := &http.Client{Timeout: 5 * time.Second} + submit := func(operationID string) <-chan shutdownHTTPResult { + result := make(chan shutdownHTTPResult, 1) + go func() { + req, err := http.NewRequest( + http.MethodPost, + "http://"+listener.Addr().String()+"/v1/runtime/operations/start", + strings.NewReader(startBody(operationID, 41)), + ) + if err != nil { + result <- shutdownHTTPResult{err: err} + return + } + req.Header.Set("Authorization", "Bearer runtime-token") + response, err := client.Do(req) + if err != nil { + result <- shutdownHTTPResult{err: err} + return + } + body, readErr := io.ReadAll(response.Body) + result <- shutdownHTTPResult{ + status: response.StatusCode, + body: string(body), + err: errors.Join(readErr, response.Body.Close()), + } + }() + return result + } + + acceptedResult := submit("accepted-start") + select { + case <-child.entered: + case <-time.After(time.Second): + t.Fatal("accepted Start did not reach its process side effect") + } + queuedResult := submit("queued-start") + select { + case <-queued.waiting: + case <-time.After(time.Second): + t.Fatal("second Start did not enter the handler and wait on the lifecycle gate") + } + + cancel() + select { + case <-shutdown.closed: + case <-time.After(time.Second): + t.Fatal("Supervisor shutdown did not close lifecycle admission") + } + if child.startCount() != 1 { + t.Fatalf("process starts before drain = %d, want 1", child.startCount()) + } + if trackedJournal.closeCount() != 0 { + t.Fatalf("journal closed while accepted Start was still executing") + } + if resolves, begins := trackedJournal.submissionCounts(); resolves != 1 || begins != 1 { + t.Fatalf("journal submissions before drain = Resolve %d, Begin %d; want 1, 1", resolves, begins) + } + select { + case err := <-serveResult: + t.Fatalf("server shutdown completed before accepted Start drained: %v", err) + case <-time.After(20 * time.Millisecond): + } + + reader, err := journal.Open(t.Context(), journalPath, journal.Options{}) + if err != nil { + t.Fatalf("open journal during accepted execution: %v", err) + } + accepted, acceptedErr := reader.Get(t.Context(), "runtime-01", "accepted-start") + _, queuedErr := reader.Get(t.Context(), "runtime-01", "queued-start") + if closeErr := reader.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if acceptedErr != nil || accepted.State != journal.StateRunning { + t.Fatalf("accepted durable execution = %+v, %v", accepted, acceptedErr) + } + if !errors.Is(queuedErr, journal.ErrOperationNotFound) { + t.Fatalf("queued Start wrote durable intent before drain: %v", queuedErr) + } + + releaseAccepted() + acceptedHTTP := <-acceptedResult + if acceptedHTTP.err != nil || acceptedHTTP.status != http.StatusOK || !strings.Contains(acceptedHTTP.body, `"state":"succeeded"`) { + t.Fatalf("accepted Start response = %d, %s, %v", acceptedHTTP.status, acceptedHTTP.body, acceptedHTTP.err) + } + queuedHTTP := <-queuedResult + if queuedHTTP.err != nil || queuedHTTP.status != http.StatusServiceUnavailable || + !strings.Contains(queuedHTTP.body, "operation_persistence_unavailable") { + t.Fatalf("queued Start response = %d, %s, %v", queuedHTTP.status, queuedHTTP.body, queuedHTTP.err) + } + select { + case err := <-serveResult: + if err != nil { + t.Fatalf("Supervisor shutdown: %v", err) + } + case <-time.After(time.Second): + t.Fatal("server shutdown did not finish after accepted Start drained") + } + if child.startCount() != 1 { + t.Fatalf("shutdown admitted an additional process Start: %d", child.startCount()) + } + if trackedJournal.closeCount() != 1 { + t.Fatalf("journal close calls = %d, want 1", trackedJournal.closeCount()) + } + if resolves, begins := trackedJournal.submissionCounts(); resolves != 1 || begins != 1 { + t.Fatalf("queued Start reached journal = Resolve %d, Begin %d; want 1, 1", resolves, begins) + } + if err := runtime.Close(); err != nil || trackedJournal.closeCount() != 1 { + t.Fatalf("repeated runtime Close = %v, journal close calls %d", err, trackedJournal.closeCount()) + } + + reopened, err := journal.Open(t.Context(), journalPath, journal.Options{}) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + accepted, acceptedErr = reopened.Get(t.Context(), "runtime-01", "accepted-start") + _, queuedErr = reopened.Get(t.Context(), "runtime-01", "queued-start") + if acceptedErr != nil || accepted.State != journal.StateSucceeded { + t.Fatalf("drained durable execution = %+v, %v", accepted, acceptedErr) + } + if !errors.Is(queuedErr, journal.ErrOperationNotFound) { + t.Fatalf("queued Start wrote durable intent during shutdown: %v", queuedErr) + } +} + func TestServeReportsListenerFailure(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -284,3 +471,137 @@ func requestRuntime(t *testing.T, handler http.Handler, path string) runtimeResp } return response } + +type shutdownAdmissionObserver struct { + *runtimeHandler + once sync.Once + closed chan struct{} +} + +func (h *shutdownAdmissionObserver) CloseAdmission() { + h.runtimeHandler.CloseAdmission() + h.once.Do(func() { close(h.closed) }) +} + +type queuedStartExecutor struct { + next protocol.StartExecutor + operationID string + waiting chan struct{} + once sync.Once +} + +func (e *queuedStartExecutor) Start(ctx context.Context, request lifecycle.StartRequest) (journal.Operation, error) { + if request.OperationID != e.operationID { + return e.next.Start(ctx, request) + } + type result struct { + operation journal.Operation + err error + } + completed := make(chan result, 1) + go func() { + operation, err := e.next.Start(ctx, request) + completed <- result{operation: operation, err: err} + }() + timer := time.NewTimer(20 * time.Millisecond) + defer timer.Stop() + select { + case got := <-completed: + return got.operation, got.err + case <-timer.C: + e.once.Do(func() { close(e.waiting) }) + got := <-completed + return got.operation, got.err + } +} + +type shutdownJournal struct { + *journal.Store + mu sync.Mutex + resolveCalls int + beginCalls int + closeCalls int +} + +func (j *shutdownJournal) Resolve(ctx context.Context, authority journal.Authority, intent journal.Intent) (journal.Operation, bool, error) { + j.mu.Lock() + j.resolveCalls++ + j.mu.Unlock() + return j.Store.Resolve(ctx, authority, intent) +} + +func (j *shutdownJournal) Begin(ctx context.Context, authority journal.Authority, intent journal.Intent) (journal.Operation, bool, error) { + j.mu.Lock() + j.beginCalls++ + j.mu.Unlock() + return j.Store.Begin(ctx, authority, intent) +} + +func (j *shutdownJournal) Close() error { + j.mu.Lock() + j.closeCalls++ + j.mu.Unlock() + return j.Store.Close() +} + +func (j *shutdownJournal) closeCount() int { + j.mu.Lock() + defer j.mu.Unlock() + return j.closeCalls +} + +func (j *shutdownJournal) submissionCounts() (int, int) { + j.mu.Lock() + defer j.mu.Unlock() + return j.resolveCalls, j.beginCalls +} + +type shutdownProcess struct { + mu sync.Mutex + observation cpaprocess.Observation + starts int + entered chan struct{} + release <-chan struct{} + once sync.Once +} + +func (p *shutdownProcess) Observe() cpaprocess.Observation { + p.mu.Lock() + defer p.mu.Unlock() + if p.observation.State == "" { + return cpaprocess.Observation{State: cpaprocess.StateNotStarted} + } + return p.observation +} + +func (p *shutdownProcess) Start(ctx context.Context, _ cpaprocess.StartSpec) (cpaprocess.Observation, error) { + p.mu.Lock() + p.starts++ + p.mu.Unlock() + p.once.Do(func() { close(p.entered) }) + select { + case <-p.release: + case <-ctx.Done(): + return p.Observe(), ctx.Err() + } + p.mu.Lock() + defer p.mu.Unlock() + p.observation = cpaprocess.Observation{State: cpaprocess.StateRunning, PID: 123} + return p.observation, nil +} + +func (p *shutdownProcess) PrepareStop() (cpaprocess.StopTarget, error) { + return nil, cpaprocess.ErrStateConflict +} + +func (p *shutdownProcess) startCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.starts +} + +type shutdownHTTPResult struct { + status int + body string + err error +} diff --git a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime.go b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime.go index c6bbb507d..8ab50efa0 100644 --- a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime.go +++ b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime.go @@ -15,10 +15,10 @@ import ( type runtimeHandler struct { http.Handler - starter *lifecycle.Starter + executor *lifecycle.Executor } -func validateStartConfig(journalPath, executable string) error { +func validateLifecycleConfig(journalPath, executable string) error { if (journalPath == "") != (executable == "") { return errors.New("CPAMP_RUNTIME_JOURNAL_PATH and CPAMP_CPA_EXECUTABLE must be configured together") } @@ -31,7 +31,7 @@ func validateStartConfig(journalPath, executable string) error { // newRuntimeHandler opens the private journal once at Supervisor startup. HTTP // submissions share this resource and the same child ownership/serialization. func newRuntimeHandler(ctx context.Context, cfg config) (*runtimeHandler, error) { - if err := validateStartConfig(cfg.journalPath, cfg.cpaExecutable); err != nil { + if err := validateLifecycleConfig(cfg.journalPath, cfg.cpaExecutable); err != nil { return nil, err } settings := protocol.Config{ @@ -45,14 +45,15 @@ func newRuntimeHandler(ctx context.Context, cfg config) (*runtimeHandler, error) if err != nil { return nil, fmt.Errorf("open operation journal: %w", err) } - runtime.starter, err = lifecycle.NewStarter(journal.Authority{ + runtime.executor, err = lifecycle.NewExecutor(journal.Authority{ RuntimeIdentity: strings.TrimSpace(cfg.runtimeIdentity), RuntimeGeneration: cfg.runtimeGeneration, }, store, &cpaprocess.Manager{}, cfg.cpaExecutable) if err != nil { return nil, errors.Join(err, store.Close()) } - settings.Start = runtime.starter + settings.Start = runtime.executor + settings.Stop = runtime.executor } handler, err := protocol.NewHandler(settings) if err != nil { @@ -63,8 +64,14 @@ func newRuntimeHandler(ctx context.Context, cfg config) (*runtimeHandler, error) } func (r *runtimeHandler) Close() error { - if r.starter == nil { + if r.executor == nil { return nil } - return r.starter.Close() + return r.executor.Close() +} + +func (r *runtimeHandler) CloseAdmission() { + if r.executor != nil { + r.executor.CloseAdmission() + } } diff --git a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime_test.go b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime_test.go index 87fb1ea7e..745bdc6c4 100644 --- a/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime_test.go +++ b/apps/runtime-supervisor/cmd/cpamp-runtime-supervisor/runtime_test.go @@ -52,7 +52,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestStartConfigurationIsAllOrNothing(t *testing.T) { +func TestLifecycleConfigurationIsAllOrNothing(t *testing.T) { tests := []struct { name, journal, executable string wantError bool @@ -73,7 +73,7 @@ func TestStartConfigurationIsAllOrNothing(t *testing.T) { t.Fatalf("loadConfig = %+v, %v", cfg, err) } if err == nil && (cfg.journalPath != strings.TrimSpace(test.journal) || cfg.cpaExecutable != strings.TrimSpace(test.executable)) { - t.Fatalf("Start configuration = %+v", cfg) + t.Fatalf("lifecycle configuration = %+v", cfg) } }) } @@ -119,7 +119,7 @@ func TestRuntimeStartEndToEnd(t *testing.T) { t.Helper() for _, path := range []string{"/v1/runtime/handshake", "/v1/runtime/status"} { got := requestRuntime(t, h, path) - if got.RuntimeGeneration != generation || !reflect.DeepEqual(got.Capabilities, []string{"start"}) || + if got.RuntimeGeneration != generation || !reflect.DeepEqual(got.Capabilities, []string{"start", "stop"}) || got.CPAObservedVersion != "" || (path == "/v1/runtime/status" && got.State != "unknown") { t.Fatalf("Start changed authority or invented readiness: %+v", got) } @@ -218,6 +218,114 @@ func TestRuntimeStartEndToEnd(t *testing.T) { awaitSpawnCount(t, directory, 2) } +func TestRuntimeStopEndToEndKeepsReplayAwayFromReplacementChild(t *testing.T) { + directory := t.TempDir() + values := validConfigValues() + values["CPAMP_RUNTIME_JOURNAL_PATH"] = filepath.Join(directory, "runtime", "operations.sqlite") + values["CPAMP_CPA_EXECUTABLE"] = copyStartHelper(t, directory) + values["NORMAL_SENTINEL"] = "test-only-ordinary-value" + values["CPAMP_DEPLOYMENT_SENTINEL"] = "test-only-deployment-value" + values["HTTP_PROXY"] = "http://http-proxy.invalid:18080" + values["HTTPS_PROXY"] = "http://https-proxy.invalid:18443" + values["NO_PROXY"] = "bypass.invalid" + for key, value := range values { + t.Setenv(key, value) + } + cfg, err := loadConfig(os.Getenv, fixedGeneration(41)) + if err != nil { + t.Fatal(err) + } + handler, err := newRuntimeHandler(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = handler.Close() }) + exitFile := filepath.Join(directory, "exit") + t.Cleanup(func() { _ = os.WriteFile(exitFile, nil, 0o600) }) + + for _, path := range []string{"/v1/runtime/handshake", "/v1/runtime/status"} { + got := requestRuntime(t, handler, path) + if got.RuntimeGeneration != 41 || !reflect.DeepEqual(got.Capabilities, []string{"start", "stop"}) || + got.CPAObservedVersion != "" || (path == "/v1/runtime/status" && got.State != "unknown") { + t.Fatalf("configured lifecycle metadata = %+v", got) + } + } + + if started := runtimeStart(handler, t.Context(), "first-start", 41); started.Code != http.StatusOK { + t.Fatalf("Start = %d, %s", started.Code, started.Body.String()) + } + awaitSpawnCount(t, directory, 1) + + // Authentication fails before decode/journal/process access. The existing + // child must remain running and the operation ID must remain unseen. + req := httptest.NewRequest(http.MethodPost, "/v1/runtime/operations/stop", strings.NewReader(startBody("unauthorized-stop", 41))) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("unauthorized Stop = %d, %s", w.Code, w.Body.String()) + } + if proof := runtimeStart(handler, t.Context(), "still-running", 41); proof.Code != http.StatusConflict || + !strings.Contains(proof.Body.String(), "operation_state_conflict") { + t.Fatalf("unauthorized Stop changed child ownership = %d, %s", proof.Code, proof.Body.String()) + } + + stopped := runtimeStop(handler, t.Context(), "first-stop", 41) + if stopped.Code != http.StatusOK { + t.Fatalf("Stop = %d, %s", stopped.Code, stopped.Body.String()) + } + var stoppedOperation struct { + RuntimeGeneration uint64 `json:"runtimeGeneration"` + State journal.State `json:"state"` + } + if err := json.Unmarshal(stopped.Body.Bytes(), &stoppedOperation); err != nil { + t.Fatal(err) + } + if stoppedOperation.RuntimeGeneration != 41 || stoppedOperation.State != journal.StateSucceeded { + t.Fatalf("Stop result = %s", stopped.Body.String()) + } + if replay := runtimeStop(handler, t.Context(), "first-stop", 41); replay.Code != http.StatusOK || replay.Body.String() != stopped.Body.String() { + t.Fatalf("same-ID Stop replay = %d, %s", replay.Code, replay.Body.String()) + } + if absent := runtimeStop(handler, t.Context(), "already-exited", 41); absent.Code != http.StatusConflict || + !strings.Contains(absent.Body.String(), "operation_state_conflict") { + t.Fatalf("Stop without running child = %d, %s", absent.Code, absent.Body.String()) + } + + if replacement := runtimeStart(handler, t.Context(), "replacement-start", 41); replacement.Code != http.StatusOK { + t.Fatalf("replacement Start after confirmed reap = %d, %s", replacement.Code, replacement.Body.String()) + } + awaitSpawnCount(t, directory, 2) + if replay := runtimeStop(handler, t.Context(), "first-stop", 41); replay.Code != http.StatusOK || replay.Body.String() != stopped.Body.String() { + t.Fatalf("old Stop replay with replacement child = %d, %s", replay.Code, replay.Body.String()) + } + if proof := runtimeStart(handler, t.Context(), "replacement-still-running", 41); proof.Code != http.StatusConflict || + !strings.Contains(proof.Body.String(), "operation_state_conflict") { + t.Fatalf("old Stop replay affected replacement child = %d, %s", proof.Code, proof.Body.String()) + } + + reader, err := journal.Open(t.Context(), cfg.journalPath, journal.Options{}) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + for _, id := range []string{"unauthorized-stop", "still-running", "already-exited", "replacement-still-running"} { + if _, err := reader.Get(t.Context(), cfg.runtimeIdentity, id); !errors.Is(err, journal.ErrOperationNotFound) { + t.Fatalf("rejected operation %q wrote intent: %v", id, err) + } + } + stored, err := reader.Get(t.Context(), cfg.runtimeIdentity, "first-stop") + if err != nil || stored.State != journal.StateSucceeded || stored.OperationType != "stop" || stored.RuntimeGeneration != 41 { + t.Fatalf("durable Stop result = %+v, %v", stored, err) + } + + if err := handler.Close(); err != nil { + t.Fatal(err) + } + if closed := runtimeStop(handler, t.Context(), "closed-stop", 41); closed.Code != http.StatusServiceUnavailable { + t.Fatalf("Stop after journal shutdown = %d, %s", closed.Code, closed.Body.String()) + } +} + func copyStartHelper(t *testing.T, directory string) string { t.Helper() executable, err := os.Executable() @@ -268,6 +376,14 @@ func runtimeStart(handler http.Handler, ctx context.Context, id string, generati return w } +func runtimeStop(handler http.Handler, ctx context.Context, id string, generation uint64) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/v1/runtime/operations/stop", strings.NewReader(startBody(id, generation))).WithContext(ctx) + req.Header.Set("Authorization", "Bearer runtime-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w +} + func awaitSpawnCount(t *testing.T, directory string, count int) { t.Helper() for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); { diff --git a/apps/runtime-supervisor/internal/cpaprocess/process.go b/apps/runtime-supervisor/internal/cpaprocess/process.go index dcb649e20..55d6b1b3d 100644 --- a/apps/runtime-supervisor/internal/cpaprocess/process.go +++ b/apps/runtime-supervisor/internal/cpaprocess/process.go @@ -1,6 +1,7 @@ // Package cpaprocess owns the Supervisor's in-memory CPA child process. -// It exposes no product entry point. Future lifecycle callers must durably -// record intent before calling Start; this primitive does not own that policy. +// It exposes no product entry point. Lifecycle callers must durably record +// intent before Start or prepared Stop termination; this primitive does not +// own that policy. package cpaprocess import ( @@ -18,6 +19,7 @@ var ( ErrStateConflict = errors.New("CPA child is already owned") ErrInvalidSpec = errors.New("invalid CPA start spec") ErrSpawnFailed = errors.New("CPA child spawn failed") + ErrStopFailed = errors.New("CPA child stop failed") ) // StartSpec supplies an executable and literal arguments, never a shell command. @@ -57,9 +59,24 @@ type Observation struct { type Manager struct { mu sync.Mutex cmd *exec.Cmd + waitDone chan struct{} + stopTarget *exec.Cmd observation Observation } +// StopTarget is an opaque reservation for the exact child owned when Stop was +// prepared. It carries no PID or caller-selected termination policy. +type StopTarget interface { + Terminate(context.Context) (Observation, error) + Release() +} + +type ownedStopTarget struct { + manager *Manager + cmd *exec.Cmd + done <-chan struct{} +} + // Start serializes validation, spawn and ownership publication. Context is // checked before spawn, including after executable lookup. It does not bound // the OS spawn call or the child's lifetime: once spawn succeeds, cancellation @@ -73,7 +90,7 @@ func (m *Manager) Start(ctx context.Context, spec StartSpec) (Observation, error if err := ctx.Err(); err != nil { return m.observeLocked(), err } - if m.cmd != nil { + if m.cmd != nil || m.stopTarget != nil { return m.observeLocked(), ErrStateConflict } if strings.TrimSpace(spec.Executable) == "" || strings.ContainsRune(spec.Executable, '\x00') { @@ -94,11 +111,93 @@ func (m *Manager) Start(ctx context.Context, spec StartSpec) (Observation, error return m.observeLocked(), fmt.Errorf("%w: %w", ErrSpawnFailed, err) } m.cmd = cmd + m.waitDone = make(chan struct{}) m.observation = Observation{State: StateRunning, PID: cmd.Process.Pid} - go m.wait(cmd) + go m.wait(cmd, m.waitDone) return m.observation, nil } +// PrepareStop reserves the exact currently owned running child before durable +// Stop intent is recorded. Start remains fenced even if that child naturally +// exits before termination begins, preventing a later child from being used as +// the target of an older Stop operation. Release abandons the reservation +// without sending a termination request. +func (m *Manager) PrepareStop() (StopTarget, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.cmd == nil || m.observation.State != StateRunning || m.stopTarget != nil || m.waitDone == nil { + return nil, ErrStateConflict + } + m.stopTarget = m.cmd + return &ownedStopTarget{manager: m, cmd: m.cmd, done: m.waitDone}, nil +} + +// Terminate uses the reserved child handle only. Success requires the same +// child's Wait path to publish a confirmed exit and release ownership. +func (target *ownedStopTarget) Terminate(ctx context.Context) (Observation, error) { + if target == nil || target.manager == nil || target.cmd == nil || target.done == nil { + return Observation{}, ErrStateConflict + } + manager := target.manager + defer target.Release() + + manager.mu.Lock() + if manager.stopTarget != target.cmd { + observation := manager.observeLocked() + manager.mu.Unlock() + return observation, ErrStateConflict + } + if err := ctx.Err(); err != nil { + observation := manager.observeLocked() + manager.mu.Unlock() + return observation, err + } + if manager.cmd == nil && manager.observation.State == StateExited { + observation := manager.observeLocked() + manager.mu.Unlock() + return observation, nil + } + if manager.cmd != target.cmd || manager.observation.State != StateRunning { + observation := manager.observeLocked() + manager.mu.Unlock() + return observation, ErrStopFailed + } + manager.mu.Unlock() + + killErr := target.cmd.Process.Kill() + if killErr != nil { + select { + case <-target.done: + // A natural exit may race the hard termination request. Confirm the + // exact child's Wait/reap result below before declaring success. + default: + return manager.Observe(), fmt.Errorf("%w: %w", ErrStopFailed, killErr) + } + } else { + <-target.done + } + + manager.mu.Lock() + defer manager.mu.Unlock() + observation := manager.observeLocked() + if manager.cmd == nil && observation.State == StateExited { + return observation, nil + } + return observation, ErrStopFailed +} + +// Release removes only this target's reservation. It never terminates a child. +func (target *ownedStopTarget) Release() { + if target == nil || target.manager == nil || target.cmd == nil { + return + } + target.manager.mu.Lock() + defer target.manager.mu.Unlock() + if target.manager.stopTarget == target.cmd { + target.manager.stopTarget = nil + } +} + // childEnvironment strips the Supervisor-private namespace and executable // setting. Other entries retain their names, values and order. func childEnvironment(parent []string, windows bool) []string { @@ -132,10 +231,15 @@ func (m *Manager) observeLocked() Observation { return m.observation } -func (m *Manager) wait(cmd *exec.Cmd) { +func (m *Manager) wait(cmd *exec.Cmd, done chan struct{}) { err := cmd.Wait() m.mu.Lock() defer m.mu.Unlock() + defer close(done) + + if m.cmd != cmd || m.waitDone != done { + return + } if cmd.ProcessState == nil { // An OS wait failure is not proof the child exited. Retain the handle @@ -145,6 +249,7 @@ func (m *Manager) wait(cmd *exec.Cmd) { return } m.cmd = nil + m.waitDone = nil m.observation = Observation{State: StateExited} if code := cmd.ProcessState.ExitCode(); code >= 0 { m.observation.ExitCode = code diff --git a/apps/runtime-supervisor/internal/cpaprocess/process_test.go b/apps/runtime-supervisor/internal/cpaprocess/process_test.go index c5a349d94..c1982e2aa 100644 --- a/apps/runtime-supervisor/internal/cpaprocess/process_test.go +++ b/apps/runtime-supervisor/internal/cpaprocess/process_test.go @@ -359,12 +359,123 @@ func TestSignalExitHasNoInventedExitCode(t *testing.T) { } } +func TestStopTerminatesExactOwnedChildAfterConfirmedReap(t *testing.T) { + t.Parallel() + var manager Manager + child := newHelper(t, &manager, 0) + startHelper(t, &manager, child) + waitFor(t, func() bool { return len(child.starts(t)) == 1 }) + manager.mu.Lock() + cmd := manager.cmd + manager.mu.Unlock() + target, err := manager.PrepareStop() + if err != nil { + t.Fatal(err) + } + got, err := target.Terminate(t.Context()) + if err != nil || got.State != StateExited || got.PID != 0 || got.WaitError != nil { + t.Fatalf("Terminate() = %+v, %v", got, err) + } + if cmd.ProcessState == nil { + t.Fatal("Stop succeeded before Cmd.Wait reaped the exact child") + } + + next := newHelper(t, &manager, 0) + startHelper(t, &manager, next) + next.release(t) + assertExit(t, waitForExit(t, &manager), 0) +} + +func TestStopTargetPreventsABAKillOfReplacementChild(t *testing.T) { + t.Parallel() + var manager Manager + first := newHelper(t, &manager, 0) + startHelper(t, &manager, first) + waitFor(t, func() bool { return len(first.starts(t)) == 1 }) + target, err := manager.PrepareStop() + if err != nil { + t.Fatal(err) + } + + // The target exits naturally after it is reserved. Its Wait path may clear + // process ownership, but Start remains fenced until this exact target is + // resolved or released. + first.release(t) + assertExit(t, waitForExit(t, &manager), 0) + second := newHelper(t, &manager, 0) + if _, err := manager.Start(t.Context(), second.spec); !errors.Is(err, ErrStateConflict) { + t.Fatalf("replacement Start while old Stop target is reserved = %v", err) + } + if got, err := target.Terminate(t.Context()); err != nil || got.State != StateExited { + t.Fatalf("natural-exit Stop = %+v, %v", got, err) + } + + secondStarted := startHelper(t, &manager, second) + waitFor(t, func() bool { return len(second.starts(t)) == 1 }) + if _, err := target.Terminate(t.Context()); !errors.Is(err, ErrStateConflict) { + t.Fatalf("old target reuse = %v", err) + } + if got := manager.Observe(); got != secondStarted { + t.Fatalf("old Stop target changed replacement child: %+v, want %+v", got, secondStarted) + } + second.release(t) + assertExit(t, waitForExit(t, &manager), 0) +} + +func TestPrepareStopFailsClosedWithoutConfirmedRunningOwnership(t *testing.T) { + for _, test := range []struct { + name string + state State + owned bool + }{ + {name: "not started", state: StateNotStarted}, + {name: "exited", state: StateExited}, + {name: "unknown", state: StateUnknown, owned: true}, + } { + t.Run(test.name, func(t *testing.T) { + manager := Manager{observation: Observation{State: test.state}} + if test.owned { + manager.cmd = exec.Command("unused") + manager.waitDone = make(chan struct{}) + manager.observation.PID = 123 + } + if target, err := manager.PrepareStop(); target != nil || !errors.Is(err, ErrStateConflict) { + t.Fatalf("PrepareStop() = %#v, %v", target, err) + } + }) + } +} + +func TestOnlyOneStopTargetCanReserveOwnedChild(t *testing.T) { + t.Parallel() + var manager Manager + child := newHelper(t, &manager, 0) + startHelper(t, &manager, child) + waitFor(t, func() bool { return len(child.starts(t)) == 1 }) + target, err := manager.PrepareStop() + if err != nil { + t.Fatal(err) + } + if duplicate, err := manager.PrepareStop(); duplicate != nil || !errors.Is(err, ErrStateConflict) { + t.Fatalf("duplicate PrepareStop() = %#v, %v", duplicate, err) + } + target.Release() + retry, err := manager.PrepareStop() + if err != nil { + t.Fatal(err) + } + if _, err := retry.Terminate(t.Context()); err != nil { + t.Fatal(err) + } +} + func TestWaitFailureRetainsOwnership(t *testing.T) { // Fault-inject a Wait error with no ProcessState using an unstarted Cmd. // No real child is created or abandoned by this error-path test. cmd := exec.Command("unused-test-command") - manager := Manager{cmd: cmd, observation: Observation{State: StateRunning, PID: 123}} - manager.wait(cmd) + done := make(chan struct{}) + manager := Manager{cmd: cmd, waitDone: done, observation: Observation{State: StateRunning, PID: 123}} + manager.wait(cmd, done) got := manager.Observe() if got.State != StateUnknown || got.PID != 123 || got.WaitError == nil || got.ExitCodeKnown { t.Fatalf("unconfirmed wait = %+v", got) diff --git a/apps/runtime-supervisor/internal/lifecycle/start.go b/apps/runtime-supervisor/internal/lifecycle/start.go index b92686803..451dd4a24 100644 --- a/apps/runtime-supervisor/internal/lifecycle/start.go +++ b/apps/runtime-supervisor/internal/lifecycle/start.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "unicode/utf8" "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/cpaprocess" @@ -15,9 +16,9 @@ import ( ) var ( - ErrInvalidRequest = errors.New("invalid Start request") + ErrInvalidRequest = errors.New("invalid lifecycle request") ErrPersistenceUnavailable = errors.New("operation persistence unavailable") - ErrExecutionFailed = errors.New("Start execution failed") + ErrExecutionFailed = errors.New("lifecycle execution failed") ) // StartRequest has an empty typed payload. Execution inputs belong exclusively @@ -29,9 +30,24 @@ type StartRequest struct { } func (r StartRequest) Validate() error { - if r.OperationID == "" || !utf8.ValidString(r.OperationID) || len(r.OperationID) > 128 || - strings.TrimSpace(r.ExpectedRuntimeIdentity) == "" || !utf8.ValidString(r.ExpectedRuntimeIdentity) || - r.ExpectedRuntimeGeneration == 0 { + return validateRequest(r.OperationID, r.ExpectedRuntimeIdentity, r.ExpectedRuntimeGeneration) +} + +// StopRequest has an empty typed payload. The owned child target and hard +// termination policy belong exclusively to the Supervisor. +type StopRequest struct { + OperationID string `json:"operationId"` + ExpectedRuntimeIdentity string `json:"expectedRuntimeIdentity"` + ExpectedRuntimeGeneration uint64 `json:"expectedRuntimeGeneration"` +} + +func (r StopRequest) Validate() error { + return validateRequest(r.OperationID, r.ExpectedRuntimeIdentity, r.ExpectedRuntimeGeneration) +} + +func validateRequest(operationID, expectedIdentity string, expectedGeneration uint64) error { + if operationID == "" || !utf8.ValidString(operationID) || len(operationID) > 128 || + strings.TrimSpace(expectedIdentity) == "" || !utf8.ValidString(expectedIdentity) || expectedGeneration == 0 { return ErrInvalidRequest } return nil @@ -48,40 +64,46 @@ type operationJournal interface { type process interface { Observe() cpaprocess.Observation Start(context.Context, cpaprocess.StartSpec) (cpaprocess.Observation, error) + PrepareStop() (cpaprocess.StopTarget, error) } -// Starter serializes submissions for one Supervisor incarnation. Keep one -// Starter and one child manager: the lock covers resolve, precondition and all +// Executor owns the shared Start/Stop mutation serialization and resources for +// one Supervisor incarnation. The lock covers resolve, precondition and all // execution evidence, while cpaprocess remains the final ownership fence. -type Starter struct { +type Executor struct { mu sync.Mutex authority journal.Authority journal operationJournal process process executable string - closed bool + closed atomic.Bool + closeOnce sync.Once + closeErr error } -func NewStarter(authority journal.Authority, store operationJournal, child process, executable string) (*Starter, error) { +func NewExecutor(authority journal.Authority, store operationJournal, child process, executable string) (*Executor, error) { if strings.TrimSpace(authority.RuntimeIdentity) == "" || authority.RuntimeGeneration == 0 || store == nil || child == nil { - return nil, errors.New("Start requires Supervisor authority, journal and child manager") + return nil, errors.New("lifecycle mutations require Supervisor authority, journal and child manager") } if strings.TrimSpace(executable) == "" || strings.ContainsRune(executable, '\x00') { - return nil, errors.New("Start requires a local CPA executable without NUL") + return nil, errors.New("lifecycle mutations require a local CPA executable without NUL") } - return &Starter{authority: authority, journal: store, process: child, executable: executable}, nil + return &Executor{authority: authority, journal: store, process: child, executable: executable}, nil } -func (s *Starter) Start(ctx context.Context, request StartRequest) (journal.Operation, error) { +func (e *Executor) Start(ctx context.Context, request StartRequest) (journal.Operation, error) { if err := request.Validate(); err != nil { return journal.Operation{}, err } - s.mu.Lock() - defer s.mu.Unlock() + if e.closed.Load() { + return journal.Operation{}, ErrPersistenceUnavailable + } + e.mu.Lock() + defer e.mu.Unlock() if err := ctx.Err(); err != nil { return journal.Operation{}, err } - if s.closed { + if e.closed.Load() { return journal.Operation{}, ErrPersistenceUnavailable } intent := journal.Intent{ @@ -91,7 +113,7 @@ func (s *Starter) Start(ctx context.Context, request StartRequest) (journal.Oper ExpectedRuntimeGeneration: request.ExpectedRuntimeGeneration, RequestFingerprint: sha256.Sum256([]byte("runtime.start/v1:{}")), } - operation, found, err := s.journal.Resolve(ctx, s.authority, intent) + operation, found, err := e.journal.Resolve(ctx, e.authority, intent) if err != nil { return journal.Operation{}, submissionError(err) } @@ -99,13 +121,13 @@ func (s *Starter) Start(ctx context.Context, request StartRequest) (journal.Oper // Retained accepted/running evidence is replayed, never resumed here. return operation, nil } - switch s.process.Observe().State { + switch e.process.Observe().State { case cpaprocess.StateNotStarted, cpaprocess.StateExited: // cpaprocess only publishes exited after a confirmed Wait/reap. default: return journal.Operation{}, journal.ErrOperationStateConflict } - operation, created, err := s.journal.Begin(ctx, s.authority, intent) + operation, created, err := e.journal.Begin(ctx, e.authority, intent) if err != nil { return journal.Operation{}, submissionError(err) } @@ -117,16 +139,16 @@ func (s *Starter) Start(ctx context.Context, request StartRequest) (journal.Oper // cancellation/deadlines/values no longer control the operation or child. // This remains synchronous; each journal write has its own bounded retry. executionCtx := context.Background() - operation, err = s.journal.MarkRunning(executionCtx, s.authority.RuntimeIdentity, intent.OperationID) + operation, err = e.journal.MarkRunning(executionCtx, e.authority.RuntimeIdentity, intent.OperationID) if err != nil { return journal.Operation{}, fmt.Errorf("%w: %w", ErrPersistenceUnavailable, err) } - _, spawnErr := s.process.Start(executionCtx, cpaprocess.StartSpec{Executable: s.executable}) + _, spawnErr := e.process.Start(executionCtx, cpaprocess.StartSpec{Executable: e.executable}) state, failureCode := journal.StateSucceeded, "" if spawnErr != nil { state, failureCode = journal.StateFailed, "process_start_failed" } - result, err := s.journal.Complete(executionCtx, s.authority.RuntimeIdentity, intent.OperationID, state, failureCode) + result, err := e.journal.Complete(executionCtx, e.authority.RuntimeIdentity, intent.OperationID, state, failureCode) if err != nil { // Spawn may already have succeeded. Never retry it or kill the child // to compensate for missing terminal evidence. @@ -150,15 +172,22 @@ func submissionError(err error) error { } } +// CloseAdmission prevents new lifecycle mutations from entering the shared +// execution gate. Submissions already queued on the gate recheck this state +// before resolving or recording durable intent. +func (e *Executor) CloseAdmission() { + e.closed.Store(true) +} + // Close drains any synchronous execution before closing the startup-owned // journal, including after HTTP shutdown forcibly disconnects a caller. It // rejects later submissions and does not stop the CPA child. -func (s *Starter) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return nil - } - s.closed = true - return s.journal.Close() +func (e *Executor) Close() error { + e.CloseAdmission() + e.closeOnce.Do(func() { + e.mu.Lock() + defer e.mu.Unlock() + e.closeErr = e.journal.Close() + }) + return e.closeErr } diff --git a/apps/runtime-supervisor/internal/lifecycle/start_test.go b/apps/runtime-supervisor/internal/lifecycle/start_test.go index e41ef8af7..634c3d590 100644 --- a/apps/runtime-supervisor/internal/lifecycle/start_test.go +++ b/apps/runtime-supervisor/internal/lifecycle/start_test.go @@ -338,7 +338,7 @@ type startFixture struct { path string journal *recordingJournal child *fakeProcess - starter *Starter + starter *Executor } func newStartFixture(t *testing.T) *startFixture { @@ -350,7 +350,7 @@ func newStartFixture(t *testing.T) *startFixture { } f.journal = &recordingJournal{Store: store, events: &f.events} f.child = &fakeProcess{events: &f.events, observation: cpaprocess.Observation{State: cpaprocess.StateNotStarted}} - f.starter, err = NewStarter(journal.Authority{RuntimeIdentity: "runtime-01", RuntimeGeneration: 41}, f.journal, f.child, "supervisor-local-cpa") + f.starter, err = NewExecutor(journal.Authority{RuntimeIdentity: "runtime-01", RuntimeGeneration: 41}, f.journal, f.child, "supervisor-local-cpa") if err != nil { t.Fatal(err) } @@ -425,7 +425,10 @@ type fakeProcess struct { events *[]string observation cpaprocess.Observation starts int + stops int onStart func(context.Context, cpaprocess.StartSpec) error + onPrepare func() + onStop func(context.Context) error } func (p *fakeProcess) Observe() cpaprocess.Observation { @@ -444,3 +447,41 @@ func (p *fakeProcess) Start(ctx context.Context, spec cpaprocess.StartSpec) (cpa p.observation = cpaprocess.Observation{State: cpaprocess.StateRunning, PID: 123} return p.observation, nil } + +func (p *fakeProcess) PrepareStop() (cpaprocess.StopTarget, error) { + *p.events = append(*p.events, "prepare-stop") + if p.onPrepare != nil { + p.onPrepare() + } + if p.observation.State != cpaprocess.StateRunning { + return nil, cpaprocess.ErrStateConflict + } + return &fakeStopTarget{process: p}, nil +} + +type fakeStopTarget struct { + process *fakeProcess + released bool +} + +func (target *fakeStopTarget) Terminate(ctx context.Context) (cpaprocess.Observation, error) { + *target.process.events = append(*target.process.events, "terminate") + if target.process.observation.State == cpaprocess.StateExited { + return target.process.observation, nil + } + target.process.stops++ + if target.process.onStop != nil { + if err := target.process.onStop(ctx); err != nil { + return target.process.observation, err + } + } + target.process.observation = cpaprocess.Observation{State: cpaprocess.StateExited} + return target.process.observation, nil +} + +func (target *fakeStopTarget) Release() { + if target.released { + return + } + target.released = true +} diff --git a/apps/runtime-supervisor/internal/lifecycle/stop.go b/apps/runtime-supervisor/internal/lifecycle/stop.go new file mode 100644 index 000000000..bf0db3187 --- /dev/null +++ b/apps/runtime-supervisor/internal/lifecycle/stop.go @@ -0,0 +1,87 @@ +package lifecycle + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/cpaprocess" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/journal" +) + +// Stop serializes with Start through the Executor gate. PrepareStop reserves +// the exact owned child before durable intent; no replacement child can cross +// that reservation while persistence is committed and termination completes. +func (e *Executor) Stop(ctx context.Context, request StopRequest) (journal.Operation, error) { + if err := request.Validate(); err != nil { + return journal.Operation{}, err + } + if e.closed.Load() { + return journal.Operation{}, ErrPersistenceUnavailable + } + e.mu.Lock() + defer e.mu.Unlock() + if err := ctx.Err(); err != nil { + return journal.Operation{}, err + } + if e.closed.Load() { + return journal.Operation{}, ErrPersistenceUnavailable + } + intent := journal.Intent{ + OperationID: request.OperationID, + OperationType: "stop", + ExpectedRuntimeIdentity: request.ExpectedRuntimeIdentity, + ExpectedRuntimeGeneration: request.ExpectedRuntimeGeneration, + RequestFingerprint: sha256.Sum256([]byte("runtime.stop/v1:{}")), + } + operation, found, err := e.journal.Resolve(ctx, e.authority, intent) + if err != nil { + return journal.Operation{}, submissionError(err) + } + if found { + // Retained evidence owns replay. Never reserve or terminate a current + // child for an accepted, running, succeeded or failed Stop retry. + return operation, nil + } + target, err := e.process.PrepareStop() + if err != nil { + if errors.Is(err, cpaprocess.ErrStateConflict) { + return journal.Operation{}, journal.ErrOperationStateConflict + } + return journal.Operation{}, fmt.Errorf("%w: prepare Stop: %w", ErrExecutionFailed, err) + } + defer target.Release() + + operation, created, err := e.journal.Begin(ctx, e.authority, intent) + if err != nil { + return journal.Operation{}, submissionError(err) + } + if !created { + return operation, nil + } + + // Durable acceptance transfers execution to the Supervisor. The target is + // already bound to one exact child; caller cancellation cannot replace it + // or interrupt termination and confirmed Wait/reap. + executionCtx := context.Background() + operation, err = e.journal.MarkRunning(executionCtx, e.authority.RuntimeIdentity, intent.OperationID) + if err != nil { + return journal.Operation{}, fmt.Errorf("%w: %w", ErrPersistenceUnavailable, err) + } + _, stopErr := target.Terminate(executionCtx) + state, failureCode := journal.StateSucceeded, "" + if stopErr != nil { + state, failureCode = journal.StateFailed, "process_stop_failed" + } + result, err := e.journal.Complete(executionCtx, e.authority.RuntimeIdentity, intent.OperationID, state, failureCode) + if err != nil { + // The exact target may already be reaped. Retained running evidence + // prevents replay from terminating a current or later child. + return operation, fmt.Errorf("%w: record result: %w", ErrExecutionFailed, err) + } + if stopErr != nil { + return result, fmt.Errorf("%w: %w", ErrExecutionFailed, stopErr) + } + return result, nil +} diff --git a/apps/runtime-supervisor/internal/lifecycle/stop_test.go b/apps/runtime-supervisor/internal/lifecycle/stop_test.go new file mode 100644 index 000000000..0037b3653 --- /dev/null +++ b/apps/runtime-supervisor/internal/lifecycle/stop_test.go @@ -0,0 +1,443 @@ +package lifecycle + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "reflect" + "sync" + "testing" + "time" + + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/cpaprocess" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/journal" +) + +func TestStopOrdersDurableEvidenceBeforeTerminationAndConfirmedReap(t *testing.T) { + f := newStopFixture(t) + f.child.onStop = func(ctx context.Context) error { + reader, err := journal.Open(ctx, f.path, journal.Options{}) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + operation, err := reader.Get(ctx, "runtime-01", "op") + if err != nil || operation.State != journal.StateRunning || operation.RuntimeGeneration != 41 || + operation.RequestFingerprint != sha256.Sum256([]byte("runtime.stop/v1:{}")) { + t.Fatalf("pre-termination durable evidence = %+v, %v", operation, err) + } + return nil + } + result, err := f.starter.Stop(t.Context(), stopRequest("op")) + if err != nil || result.State != journal.StateSucceeded || f.child.observation.State != cpaprocess.StateExited { + t.Fatalf("Stop = %+v, %v; process %+v", result, err, f.child.observation) + } + want := []string{"resolve", "prepare-stop", "begin", "running", "terminate", "complete:succeeded"} + if !reflect.DeepEqual(f.events, want) { + t.Fatalf("ordering = %v, want %v", f.events, want) + } + if f.starter.authority.RuntimeGeneration != 41 { + t.Fatal("Stop changed Supervisor generation") + } +} + +func TestStopFailureWindowsNeverResumeOnReplay(t *testing.T) { + tests := []struct { + name string + failAt string + stopError bool + wantState journal.State + wantStops int + wantError error + }{ + {"lookup", "resolve", false, "", 0, ErrPersistenceUnavailable}, + {"intent", "begin", false, "", 0, ErrPersistenceUnavailable}, + {"running", "running", false, journal.StateAccepted, 0, ErrPersistenceUnavailable}, + {"termination", "", true, journal.StateFailed, 1, ErrExecutionFailed}, + {"success result", "complete", false, journal.StateRunning, 1, ErrExecutionFailed}, + {"failure result", "complete", true, journal.StateRunning, 1, ErrExecutionFailed}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := newStopFixture(t) + f.journal.failAt = test.failAt + if test.stopError { + f.child.onStop = func(context.Context) error { return cpaprocess.ErrStopFailed } + } + _, err := f.starter.Stop(t.Context(), stopRequest("op")) + if !errors.Is(err, test.wantError) || f.child.stops != test.wantStops { + t.Fatalf("Stop error %v, terminations %d", err, f.child.stops) + } + stored, err := f.journal.Store.Get(t.Context(), "runtime-01", "op") + if test.wantState == "" { + if !errors.Is(err, journal.ErrOperationNotFound) { + t.Fatalf("unexpected intent: %+v, %v", stored, err) + } + return + } + if err != nil || stored.State != test.wantState { + t.Fatalf("retained state = %+v, %v", stored, err) + } + if test.wantState == journal.StateFailed && stored.FailureCode != "process_stop_failed" { + t.Fatalf("failure evidence = %+v", stored) + } + f.journal.failAt = "" + f.events = nil + replay, err := f.starter.Stop(t.Context(), stopRequest("op")) + if err != nil || !reflect.DeepEqual(replay, stored) || f.child.stops != test.wantStops || + !reflect.DeepEqual(f.events, []string{"resolve"}) { + t.Fatalf("replay = %+v, %v; terminations %d; events %v", replay, err, f.child.stops, f.events) + } + }) + } +} + +func TestStopPreconditionRejectsUnownedOrUnconfirmedChildWithoutIntent(t *testing.T) { + for _, state := range []cpaprocess.State{cpaprocess.StateNotStarted, cpaprocess.StateExited, cpaprocess.StateUnknown, "unrecognized"} { + t.Run(string(state), func(t *testing.T) { + f := newStopFixture(t) + f.child.observation = cpaprocess.Observation{State: state, PID: 123} + _, err := f.starter.Stop(t.Context(), stopRequest("op")) + if !errors.Is(err, journal.ErrOperationStateConflict) || f.child.stops != 0 || + !reflect.DeepEqual(f.events, []string{"resolve", "prepare-stop"}) { + t.Fatalf("Stop error %v; terminations %d; events %v", err, f.child.stops, f.events) + } + if _, err := f.journal.Store.Get(t.Context(), "runtime-01", "op"); !errors.Is(err, journal.ErrOperationNotFound) { + t.Fatalf("precondition failure wrote intent: %v", err) + } + }) + } +} + +func TestStopFencingPrecedesProcessPrecondition(t *testing.T) { + for _, identityMismatch := range []bool{true, false} { + f := newStopFixture(t) + f.child.observation.State = cpaprocess.StateUnknown + request := stopRequest("op") + request.ExpectedRuntimeGeneration++ + want := journal.ErrStaleRuntimeGeneration + if identityMismatch { + request.ExpectedRuntimeIdentity = "another-runtime" + want = journal.ErrRuntimeIdentityMismatch + } + _, err := f.starter.Stop(t.Context(), request) + if !errors.Is(err, want) || !reflect.DeepEqual(f.events, []string{"resolve"}) { + t.Fatalf("fenced Stop error = %v, events %v", err, f.events) + } + } +} + +func TestStopReplaysAllRetainedStatesAndRejectsDifferentType(t *testing.T) { + for _, state := range []journal.State{journal.StateAccepted, journal.StateRunning, journal.StateSucceeded, journal.StateFailed} { + t.Run(string(state), func(t *testing.T) { + f := newStopFixture(t) + intent := stopIntent("op") + if _, _, err := f.journal.Store.Begin(t.Context(), f.starter.authority, intent); err != nil { + t.Fatal(err) + } + var err error + if state == journal.StateRunning { + _, err = f.journal.Store.MarkRunning(t.Context(), "runtime-01", "op") + } else if state != journal.StateAccepted { + code := "" + if state == journal.StateFailed { + code = "process_stop_failed" + } + _, err = f.journal.Store.Complete(t.Context(), "runtime-01", "op", state, code) + } + if err != nil { + t.Fatal(err) + } + f.child.observation.State = cpaprocess.StateUnknown + f.starter.authority.RuntimeGeneration = 42 + request := stopRequest("op") + request.ExpectedRuntimeGeneration = 42 + got, err := f.starter.Stop(t.Context(), request) + if err != nil || got.State != state || got.RuntimeGeneration != 41 || f.child.stops != 0 || + !reflect.DeepEqual(f.events, []string{"resolve"}) { + t.Fatalf("retained replay = %+v, %v; events %v", got, err, f.events) + } + }) + } + + for _, test := range []struct { + name string + intent journal.Intent + }{ + {name: "type", intent: startIntent("shared-id")}, + {name: "fingerprint", intent: func() journal.Intent { + intent := stopIntent("shared-id") + intent.RequestFingerprint = sha256.Sum256([]byte("runtime.stop/v2:{}")) + return intent + }()}, + } { + t.Run("conflicting "+test.name, func(t *testing.T) { + f := newStopFixture(t) + if _, _, err := f.journal.Store.Begin(t.Context(), f.starter.authority, test.intent); err != nil { + t.Fatal(err) + } + if _, err := f.starter.Stop(t.Context(), stopRequest("shared-id")); !errors.Is(err, journal.ErrOperationIDConflict) || + f.child.stops != 0 || !reflect.DeepEqual(f.events, []string{"resolve"}) { + t.Fatalf("conflicting Stop replay = %v; events %v", err, f.events) + } + }) + } +} + +func TestStopCallerCancellationBeforeAndAfterDurableIntent(t *testing.T) { + for _, when := range []string{"before submission", "before commit", "after commit"} { + t.Run(when, func(t *testing.T) { + f := newStopFixture(t) + type requestKey struct{} + ctx, cancel := context.WithCancel(context.WithValue(t.Context(), requestKey{}, "caller-value")) + defer cancel() + switch when { + case "before submission": + cancel() + case "before commit": + f.journal.beforeBegin = cancel + case "after commit": + f.journal.afterBegin = cancel + } + f.child.onStop = func(execution context.Context) error { + if ctx.Err() == nil || execution.Err() != nil || execution.Value(requestKey{}) != nil { + t.Fatal("accepted Stop execution still belongs to caller context") + } + return nil + } + got, err := f.starter.Stop(ctx, stopRequest("op")) + if when == "after commit" { + if err != nil || got.State != journal.StateSucceeded || f.child.stops != 1 { + t.Fatalf("accepted Stop = %+v, %v", got, err) + } + } else { + if !errors.Is(err, context.Canceled) || f.child.stops != 0 { + t.Fatalf("cancelled Stop = %+v, %v; terminations %d", got, err, f.child.stops) + } + if _, err := f.journal.Store.Get(t.Context(), "runtime-01", "op"); !errors.Is(err, journal.ErrOperationNotFound) { + t.Fatalf("cancelled request left intent: %v", err) + } + } + }) + } +} + +func TestStopDoesNotSucceedUntilTargetReportsConfirmedReap(t *testing.T) { + f := newStopFixture(t) + entered, reaped := make(chan struct{}), make(chan struct{}) + f.child.onStop = func(context.Context) error { + close(entered) + <-reaped + return nil + } + result := make(chan struct { + operation journal.Operation + err error + }, 1) + go func() { + operation, err := f.starter.Stop(t.Context(), stopRequest("op")) + result <- struct { + operation journal.Operation + err error + }{operation, err} + }() + <-entered + reader, err := journal.Open(t.Context(), f.path, journal.Options{}) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + stored, err := reader.Get(t.Context(), "runtime-01", "op") + if err != nil || stored.State != journal.StateRunning { + t.Fatalf("Stop before reap = %+v, %v", stored, err) + } + select { + case early := <-result: + t.Fatalf("Stop completed before confirmed reap: %+v, %v", early.operation, early.err) + case <-time.After(10 * time.Millisecond): + } + close(reaped) + completed := <-result + if completed.err != nil || completed.operation.State != journal.StateSucceeded { + t.Fatalf("Stop after reap = %+v, %v", completed.operation, completed.err) + } +} + +func TestStopNaturalExitAfterExactTargetReservationSucceedsWithoutTermination(t *testing.T) { + f := newStopFixture(t) + f.journal.beforeBegin = func() { + f.child.observation = cpaprocess.Observation{State: cpaprocess.StateExited} + } + got, err := f.starter.Stop(t.Context(), stopRequest("op")) + if err != nil || got.State != journal.StateSucceeded || f.child.stops != 0 { + t.Fatalf("natural-exit Stop = %+v, %v; terminations %d", got, err, f.child.stops) + } +} + +func TestStopTerminalPersistenceFailureReplayNeverTargetsReplacementChild(t *testing.T) { + f := newStopFixture(t) + f.journal.failAt = "complete" + _, err := f.starter.Stop(t.Context(), stopRequest("old-stop")) + if !errors.Is(err, ErrExecutionFailed) || f.child.stops != 1 || f.child.observation.State != cpaprocess.StateExited { + t.Fatalf("Stop with terminal persistence failure = %v; terminations %d; state %+v", err, f.child.stops, f.child.observation) + } + retained, err := f.journal.Store.Get(t.Context(), "runtime-01", "old-stop") + if err != nil || retained.State != journal.StateRunning { + t.Fatalf("retained Stop evidence = %+v, %v", retained, err) + } + + f.journal.failAt = "" + started, err := f.starter.Start(t.Context(), startRequest("replacement-start")) + if err != nil || started.State != journal.StateSucceeded || f.child.observation.State != cpaprocess.StateRunning { + t.Fatalf("replacement Start = %+v, %v", started, err) + } + replay, err := f.starter.Stop(t.Context(), stopRequest("old-stop")) + if err != nil || !reflect.DeepEqual(replay, retained) || f.child.stops != 1 || f.child.observation.State != cpaprocess.StateRunning { + t.Fatalf("old Stop replay = %+v, %v; terminations %d; state %+v", replay, err, f.child.stops, f.child.observation) + } +} + +func TestConcurrentStopIDsTerminateExactlyOnce(t *testing.T) { + f := newStopFixture(t) + gate := make(chan struct{}) + results := make(chan error, 32) + var callers sync.WaitGroup + for i := range 32 { + callers.Add(1) + go func() { + defer callers.Done() + <-gate + _, err := f.starter.Stop(t.Context(), stopRequest(fmt.Sprintf("op-%d", i))) + results <- err + }() + } + close(gate) + callers.Wait() + close(results) + var successes, conflicts int + for err := range results { + switch { + case err == nil: + successes++ + case errors.Is(err, journal.ErrOperationStateConflict): + conflicts++ + default: + t.Fatalf("concurrent Stop = %v", err) + } + } + if successes != 1 || conflicts != 31 || f.child.stops != 1 { + t.Fatalf("successes %d, conflicts %d, terminations %d", successes, conflicts, f.child.stops) + } + var intents int + for _, event := range f.events { + if event == "begin" { + intents++ + } + } + if intents != 1 { + t.Fatalf("concurrent Stop submissions created %d intents", intents) + } +} + +func TestStartAndStopShareLifecycleSerialization(t *testing.T) { + f := newStopFixture(t) + entered, release := make(chan struct{}), make(chan struct{}) + f.child.onStop = func(context.Context) error { + close(entered) + <-release + return nil + } + stopResult, startResult := make(chan error, 1), make(chan error, 1) + go func() { _, err := f.starter.Stop(t.Context(), stopRequest("stop")); stopResult <- err }() + <-entered + go func() { _, err := f.starter.Start(t.Context(), startRequest("replacement")); startResult <- err }() + select { + case err := <-startResult: + t.Fatalf("Start crossed in-flight Stop serialization: %v", err) + case <-time.After(10 * time.Millisecond): + } + close(release) + if err := <-stopResult; err != nil { + t.Fatal(err) + } + if err := <-startResult; err != nil { + t.Fatal(err) + } + if f.child.stops != 1 || f.child.starts != 1 || f.child.observation.State != cpaprocess.StateRunning { + t.Fatalf("serialized lifecycle = terminations %d, starts %d, state %+v", f.child.stops, f.child.starts, f.child.observation) + } + var completed, secondResolve int = -1, -1 + for index, event := range f.events { + if event == "complete:succeeded" && completed == -1 { + completed = index + } + if event == "resolve" && completed != -1 { + secondResolve = index + break + } + } + if completed == -1 || secondResolve <= completed { + t.Fatalf("Start/Stop operations interleaved: %v", f.events) + } +} + +func TestCloseDrainsStopAndClosesSharedJournalOnce(t *testing.T) { + f := newStopFixture(t) + entered, release := make(chan struct{}), make(chan struct{}) + f.child.onStop = func(context.Context) error { + close(entered) + <-release + return nil + } + stopped, closed := make(chan error, 1), make(chan error, 1) + go func() { _, err := f.starter.Stop(t.Context(), stopRequest("op")); stopped <- err }() + <-entered + go func() { closed <- f.starter.Close() }() + for !f.starter.closed.Load() { + time.Sleep(time.Millisecond) + } + if _, err := f.starter.Start(t.Context(), startRequest("during-close-start")); !errors.Is(err, ErrPersistenceUnavailable) { + t.Fatalf("Start admitted after shutdown began = %v", err) + } + if _, err := f.starter.Stop(t.Context(), stopRequest("during-close-stop")); !errors.Is(err, ErrPersistenceUnavailable) { + t.Fatalf("Stop admitted after shutdown began = %v", err) + } + select { + case err := <-closed: + t.Fatalf("journal closed during accepted Stop: %v", err) + case <-time.After(10 * time.Millisecond): + } + close(release) + if err := <-stopped; err != nil { + t.Fatal(err) + } + if err := <-closed; err != nil { + t.Fatal(err) + } + if err := f.starter.Close(); err != nil || f.journal.closeCalls != 1 { + t.Fatalf("Close = %v, calls %d", err, f.journal.closeCalls) + } + if _, err := f.starter.Start(t.Context(), startRequest("later-start")); !errors.Is(err, ErrPersistenceUnavailable) { + t.Fatalf("Start after Close = %v", err) + } + if _, err := f.starter.Stop(t.Context(), stopRequest("later-stop")); !errors.Is(err, ErrPersistenceUnavailable) { + t.Fatalf("Stop after Close = %v", err) + } +} + +func newStopFixture(t *testing.T) *startFixture { + t.Helper() + f := newStartFixture(t) + f.child.observation = cpaprocess.Observation{State: cpaprocess.StateRunning, PID: 123} + return f +} + +func stopRequest(id string) StopRequest { + return StopRequest{OperationID: id, ExpectedRuntimeIdentity: "runtime-01", ExpectedRuntimeGeneration: 41} +} + +func stopIntent(id string) journal.Intent { + return journal.Intent{OperationID: id, OperationType: "stop", ExpectedRuntimeIdentity: "runtime-01", + ExpectedRuntimeGeneration: 41, RequestFingerprint: sha256.Sum256([]byte("runtime.stop/v1:{}"))} +} diff --git a/apps/runtime-supervisor/internal/protocol/handler.go b/apps/runtime-supervisor/internal/protocol/handler.go index 5345773e1..88fbd660f 100644 --- a/apps/runtime-supervisor/internal/protocol/handler.go +++ b/apps/runtime-supervisor/internal/protocol/handler.go @@ -22,6 +22,7 @@ type Config struct { RuntimeGeneration uint64 Token string Start StartExecutor + Stop StopExecutor } type handler struct { @@ -29,6 +30,7 @@ type handler struct { runtimeGeneration uint64 tokenDigest [sha256.Size]byte start StartExecutor + stop StopExecutor } type handshakeResponse struct { @@ -75,6 +77,7 @@ func NewHandler(config Config) (http.Handler, error) { runtimeGeneration: config.RuntimeGeneration, tokenDigest: sha256.Sum256([]byte(config.Token)), start: config.Start, + stop: config.Stop, }, nil } @@ -82,7 +85,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { method := http.MethodGet switch r.URL.Path { case handshakePath, statusPath: - case startPath: + case startPath, stopPath: method = http.MethodPost default: writeError(w, http.StatusNotFound, "not_found", "runtime endpoint not found") @@ -118,14 +121,20 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }) case startPath: h.submitStart(w, r) + case stopPath: + h.submitStop(w, r) } } func (h *handler) capabilities() []string { + capabilities := make([]string, 0, 2) if h.start != nil { - return []string{CapabilityStart} + capabilities = append(capabilities, CapabilityStart) + } + if h.stop != nil { + capabilities = append(capabilities, CapabilityStop) } - return []string{} + return capabilities } func (h *handler) authorized(authorization string) bool { diff --git a/apps/runtime-supervisor/internal/protocol/start.go b/apps/runtime-supervisor/internal/protocol/start.go index 3a5d2f2d5..b06e5aa18 100644 --- a/apps/runtime-supervisor/internal/protocol/start.go +++ b/apps/runtime-supervisor/internal/protocol/start.go @@ -18,7 +18,7 @@ const ( // current child state permits it or that the Runtime is ready. CapabilityStart = "start" startPath = "/v1/runtime/operations/start" - maxStartBody = 16 << 10 + maxMutationBody = 16 << 10 ) type StartExecutor interface { @@ -35,7 +35,7 @@ type operationResponse struct { } func (h *handler) submitStart(w http.ResponseWriter, r *http.Request) { - request, err := decodeStartRequest(w, r) + envelope, err := decodeMutationRequest(w, r) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request", "invalid Start request") return @@ -44,11 +44,25 @@ func (h *handler) submitStart(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "unsupported_operation", "Start is not configured") return } - operation, err := h.start.Start(r.Context(), request) + operation, err := h.start.Start(r.Context(), lifecycle.StartRequest(envelope)) if err != nil { - writeStartError(w, err) + writeMutationError(w, err, "Start") return } + writeOperationResponse(w, operation, "Start") +} + +type mutationRequest struct { + OperationID string `json:"operationId"` + ExpectedRuntimeIdentity string `json:"expectedRuntimeIdentity"` + ExpectedRuntimeGeneration uint64 `json:"expectedRuntimeGeneration"` +} + +func (r mutationRequest) Validate() error { + return lifecycle.StartRequest(r).Validate() +} + +func writeOperationResponse(w http.ResponseWriter, operation journal.Operation, operationName string) { response := operationResponse{ OperationID: operation.OperationID, OperationType: operation.OperationType, @@ -57,14 +71,14 @@ func (h *handler) submitStart(w http.ResponseWriter, r *http.Request) { State: operation.State, } if operation.FailureCode != "" { - response.Error = &protocolError{Code: operation.FailureCode, Message: "Start execution failed"} + response.Error = &protocolError{Code: operation.FailureCode, Message: operationName + " execution failed"} } writeJSON(w, http.StatusOK, response) } -func decodeStartRequest(w http.ResponseWriter, r *http.Request) (lifecycle.StartRequest, error) { - var request lifecycle.StartRequest - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxStartBody)) +func decodeMutationRequest(w http.ResponseWriter, r *http.Request) (mutationRequest, error) { + var request mutationRequest + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxMutationBody)) if err != nil || !utf8.Valid(body) { return request, lifecycle.ErrInvalidRequest } @@ -108,8 +122,8 @@ func decodeStartRequest(w http.ResponseWriter, r *http.Request) (lifecycle.Start return request, request.Validate() } -func writeStartError(w http.ResponseWriter, err error) { - code, message, status := "internal_error", "Start execution failed", http.StatusInternalServerError +func writeMutationError(w http.ResponseWriter, err error, operationName string) { + code, message, status := "internal_error", operationName+" execution failed", http.StatusInternalServerError switch { case errors.Is(err, lifecycle.ErrExecutionFailed): // Post-spawn/result failures remain internal errors even when their @@ -117,7 +131,7 @@ func writeStartError(w http.ResponseWriter, err error) { case errors.Is(err, lifecycle.ErrPersistenceUnavailable): code, message, status = "operation_persistence_unavailable", "operation persistence is unavailable", http.StatusServiceUnavailable case errors.Is(err, lifecycle.ErrInvalidRequest): - code, message, status = "invalid_request", "invalid Start request", http.StatusBadRequest + code, message, status = "invalid_request", "invalid "+operationName+" request", http.StatusBadRequest case errors.Is(err, journal.ErrRuntimeIdentityMismatch): code, message, status = "runtime_identity_mismatch", "runtime identity does not match", http.StatusConflict case errors.Is(err, journal.ErrStaleRuntimeGeneration): @@ -125,7 +139,7 @@ func writeStartError(w http.ResponseWriter, err error) { case errors.Is(err, journal.ErrOperationIDConflict): code, message, status = "operation_id_conflict", "operation ID names a different request", http.StatusConflict case errors.Is(err, journal.ErrOperationStateConflict): - code, message, status = "operation_state_conflict", "CPA process ownership does not permit Start", http.StatusConflict + code, message, status = "operation_state_conflict", "CPA process ownership does not permit "+operationName, http.StatusConflict } writeError(w, status, code, message) } diff --git a/apps/runtime-supervisor/internal/protocol/start_test.go b/apps/runtime-supervisor/internal/protocol/start_test.go index d3f2fe4c0..27b0d5544 100644 --- a/apps/runtime-supervisor/internal/protocol/start_test.go +++ b/apps/runtime-supervisor/internal/protocol/start_test.go @@ -50,7 +50,7 @@ func TestStartStrictRequestValidation(t *testing.T) { "missing generation": `{"operationId":"op","expectedRuntimeIdentity":"runtime-01"}`, "trailing object": validStartBody + `{}`, "trailing null": validStartBody + `null`, "trailing garbage": validStartBody + `x`, - "oversized body": strings.Repeat(" ", maxStartBody) + validStartBody, + "oversized body": strings.Repeat(" ", maxMutationBody) + validStartBody, "case alias": strings.Replace(validStartBody, "operationId", "OperationId", 1), "duplicate ID": strings.Replace(validStartBody, `"operationId":"op"`, `"operationId":"op","operationId":"other"`, 1), } @@ -96,7 +96,7 @@ func TestStartReadOnlyModeIsUnsupported(t *testing.T) { if method.Code != http.StatusMethodNotAllowed || method.Header().Get("Allow") != http.MethodPost { t.Fatalf("Start method = %d, Allow %q", method.Code, method.Header().Get("Allow")) } - for _, path := range []string{"/v1/runtime/operations/stop", "/v1/runtime/operations/restart", "/v1/runtime/operations/op"} { + for _, path := range []string{"/v1/runtime/operations/restart", "/v1/runtime/operations/op"} { if w := request(t, h, http.MethodPost, path, testRuntimeToken); w.Code != http.StatusNotFound { t.Fatalf("out-of-scope endpoint %s is exposed: %d", path, w.Code) } diff --git a/apps/runtime-supervisor/internal/protocol/stop.go b/apps/runtime-supervisor/internal/protocol/stop.go new file mode 100644 index 000000000..53079b6ef --- /dev/null +++ b/apps/runtime-supervisor/internal/protocol/stop.go @@ -0,0 +1,38 @@ +package protocol + +import ( + "context" + "net/http" + + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/journal" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/lifecycle" +) + +const ( + // CapabilityStop means typed Stop submission is supported, not that a + // running child is currently available to terminate. + CapabilityStop = "stop" + stopPath = "/v1/runtime/operations/stop" +) + +type StopExecutor interface { + Stop(context.Context, lifecycle.StopRequest) (journal.Operation, error) +} + +func (h *handler) submitStop(w http.ResponseWriter, r *http.Request) { + envelope, err := decodeMutationRequest(w, r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "invalid Stop request") + return + } + if h.stop == nil { + writeError(w, http.StatusBadRequest, "unsupported_operation", "Stop is not configured") + return + } + operation, err := h.stop.Stop(r.Context(), lifecycle.StopRequest(envelope)) + if err != nil { + writeMutationError(w, err, "Stop") + return + } + writeOperationResponse(w, operation, "Stop") +} diff --git a/apps/runtime-supervisor/internal/protocol/stop_test.go b/apps/runtime-supervisor/internal/protocol/stop_test.go new file mode 100644 index 000000000..18ce45cea --- /dev/null +++ b/apps/runtime-supervisor/internal/protocol/stop_test.go @@ -0,0 +1,224 @@ +package protocol + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/journal" + "github.com/seakee/cpa-manager-plus/apps/runtime-supervisor/internal/lifecycle" +) + +const validStopBody = `{"operationId":"op","expectedRuntimeIdentity":"runtime-01","expectedRuntimeGeneration":7}` + +func TestStopAuthenticatesBeforeReadingBodyOrCallingExecutor(t *testing.T) { + h := stopHandler(t, stopFunc(func(context.Context, lifecycle.StopRequest) (journal.Operation, error) { + t.Fatal("unauthorized request reached Stop execution boundary") + return journal.Operation{}, nil + })) + req := httptest.NewRequest(http.MethodPost, stopPath, nil) + req.Body = unreadableBody{t} + req.Header.Set("Authorization", "Bearer wrong-token") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized || w.Header().Get("WWW-Authenticate") != "Bearer" { + t.Fatalf("unauthorized response: %d, %s", w.Code, w.Body.String()) + } + assertErrorCode(t, w, "unauthorized") +} + +func TestStopStrictRequestValidationRejectsCallerTerminationAuthority(t *testing.T) { + h := stopHandler(t, stopFunc(func(context.Context, lifecycle.StopRequest) (journal.Operation, error) { + t.Fatal("invalid request reached journal/process executor") + return journal.Operation{}, nil + })) + tests := map[string]string{ + "empty": "", "null": "null", "array": "[]", "string": `"request"`, + "empty object": `{}`, "truncated": `{"operationId":`, + "empty ID": strings.Replace(validStopBody, `"op"`, `""`, 1), + "long ID": strings.Replace(validStopBody, `"op"`, `"`+strings.Repeat("x", 129)+`"`, 1), + "UTF-8 byte limit": strings.Replace(validStopBody, `"op"`, `"`+strings.Repeat("字", 43)+`"`, 1), + "invalid UTF-8": strings.Replace(validStopBody, "op\"", string([]byte{0xff})+"\"", 1), + "missing identity": `{"operationId":"op","expectedRuntimeGeneration":7}`, + "blank identity": strings.Replace(validStopBody, `"runtime-01"`, `" "`, 1), + "null identity": strings.Replace(validStopBody, `"runtime-01"`, `null`, 1), + "missing generation": `{"operationId":"op","expectedRuntimeIdentity":"runtime-01"}`, + "trailing object": validStopBody + `{}`, "trailing null": validStopBody + `null`, + "trailing garbage": validStopBody + `x`, + "oversized body": strings.Repeat(" ", maxMutationBody) + validStopBody, + "case alias": strings.Replace(validStopBody, "operationId", "OperationId", 1), + "duplicate ID": strings.Replace(validStopBody, `"operationId":"op"`, `"operationId":"op","operationId":"other"`, 1), + } + for _, generation := range []string{"0", "-1", "1.5", "18446744073709551616", `"7"`, "null", "true"} { + tests["generation "+generation] = strings.Replace(validStopBody, ":7", ":"+generation, 1) + } + for _, field := range []string{ + "pid", "signal", "force", "timeout", "grace", "executable", "argv", "args", "env", "cwd", + "workingDirectory", "shell", "shellCommand", "action", "params", + } { + tests[field] = strings.TrimSuffix(validStopBody, "}") + `,"` + field + `":"not-allowed"}` + } + for name, body := range tests { + t.Run(name, func(t *testing.T) { + w := submitStop(t, h, body) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + assertErrorCode(t, w, "invalid_request") + }) + } +} + +func TestStopPreservesOpaqueIDAndFullWidthGeneration(t *testing.T) { + id := strings.Repeat("字", 42) + " a" + h := stopHandler(t, stopFunc(func(_ context.Context, request lifecycle.StopRequest) (journal.Operation, error) { + if request.OperationID != id || request.ExpectedRuntimeGeneration != ^uint64(0) { + t.Fatalf("typed request = %+v", request) + } + return journal.Operation{OperationID: id}, nil + })) + body := fmt.Sprintf(`{"operationId":%q,"expectedRuntimeIdentity":"runtime-01","expectedRuntimeGeneration":18446744073709551615}`, id) + if w := submitStop(t, h, body); w.Code != http.StatusOK { + t.Fatalf("valid opaque request = %d, %s", w.Code, w.Body.String()) + } +} + +func TestStopReadOnlyModeIsUnsupported(t *testing.T) { + h := newTestHandler(t) + w := submitStop(t, h, validStopBody) + if w.Code != http.StatusBadRequest { + t.Fatalf("read-only Stop = %d", w.Code) + } + assertErrorCode(t, w, "unsupported_operation") + method := request(t, h, http.MethodGet, stopPath, testRuntimeToken) + if method.Code != http.StatusMethodNotAllowed || method.Header().Get("Allow") != http.MethodPost { + t.Fatalf("Stop method = %d, Allow %q", method.Code, method.Header().Get("Allow")) + } +} + +func TestLifecycleCapabilitiesAreDeterministicAndDoNotClaimReadiness(t *testing.T) { + executor := lifecycleFunc{ + start: func(_ context.Context, request lifecycle.StartRequest) (journal.Operation, error) { + return journal.Operation{OperationID: request.OperationID, OperationType: "start", RuntimeIdentity: "runtime-01", RuntimeGeneration: 7, State: journal.StateSucceeded}, nil + }, + stop: func(_ context.Context, request lifecycle.StopRequest) (journal.Operation, error) { + return journal.Operation{OperationID: request.OperationID, OperationType: "stop", RuntimeIdentity: "runtime-01", RuntimeGeneration: 7, State: journal.StateSucceeded}, nil + }, + } + h, err := NewHandler(Config{RuntimeIdentity: "runtime-01", RuntimeGeneration: 7, Token: testRuntimeToken, Start: executor, Stop: executor}) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{handshakePath, statusPath} { + var status statusResponse + decodeResponse(t, request(t, h, http.MethodGet, path, testRuntimeToken), &status) + if status.RuntimeGeneration != 7 || status.CPAObservedVersion != "" || + !reflect.DeepEqual(status.Capabilities, []string{"start", "stop"}) || + (path == statusPath && status.State != "unknown") { + t.Fatalf("%s invented observation or lost lifecycle capability: %+v", path, status) + } + } +} + +func TestStopResponseAndErrorMapping(t *testing.T) { + for _, state := range []journal.State{journal.StateAccepted, journal.StateRunning, journal.StateSucceeded, journal.StateFailed} { + t.Run(string(state), func(t *testing.T) { + h := stopHandler(t, stopFunc(func(_ context.Context, request lifecycle.StopRequest) (journal.Operation, error) { + operation := journal.Operation{OperationID: request.OperationID, OperationType: "stop", RuntimeIdentity: "runtime-01", RuntimeGeneration: 6, State: state} + if state == journal.StateFailed { + operation.FailureCode = "process_stop_failed" + } + return operation, nil + })) + w := submitStop(t, h, validStopBody) + if w.Code != http.StatusOK { + t.Fatalf("Stop response = %d, %s", w.Code, w.Body.String()) + } + var got operationResponse + decodeResponse(t, w, &got) + if got.OperationID != "op" || got.OperationType != "stop" || got.RuntimeIdentity != "runtime-01" || + got.RuntimeGeneration != 6 || got.State != state { + t.Fatalf("operation response = %+v", got) + } + if state == journal.StateFailed && (got.Error == nil || got.Error.Code != "process_stop_failed") { + t.Fatalf("replay lost Stop failure evidence: %+v", got) + } + if strings.Contains(w.Body.String(), "Fingerprint") || strings.Contains(w.Body.String(), "ready") { + t.Fatalf("Stop response leaked journal fields or invented readiness: %s", w.Body.String()) + } + }) + } + + tests := []struct { + err error + status int + code string + }{ + {lifecycle.ErrInvalidRequest, 400, "invalid_request"}, + {journal.ErrRuntimeIdentityMismatch, 409, "runtime_identity_mismatch"}, + {journal.ErrStaleRuntimeGeneration, 409, "stale_runtime_generation"}, + {journal.ErrOperationIDConflict, 409, "operation_id_conflict"}, + {journal.ErrOperationStateConflict, 409, "operation_state_conflict"}, + {lifecycle.ErrPersistenceUnavailable, 503, "operation_persistence_unavailable"}, + {errors.Join(lifecycle.ErrPersistenceUnavailable, journal.ErrOperationStateConflict), 503, "operation_persistence_unavailable"}, + {errors.Join(lifecycle.ErrExecutionFailed, journal.ErrOperationStateConflict), 500, "internal_error"}, + {errors.New("secret-pid-or-signal"), 500, "internal_error"}, + } + for _, test := range tests { + t.Run(test.err.Error(), func(t *testing.T) { + h := stopHandler(t, stopFunc(func(context.Context, lifecycle.StopRequest) (journal.Operation, error) { + return journal.Operation{}, test.err + })) + w := submitStop(t, h, validStopBody) + if w.Code != test.status { + t.Fatalf("error response = %d, %s", w.Code, w.Body.String()) + } + assertErrorCode(t, w, test.code) + if strings.Contains(w.Body.String(), "secret-pid-or-signal") { + t.Fatal("raw Stop execution error escaped to HTTP") + } + }) + } +} + +type stopFunc func(context.Context, lifecycle.StopRequest) (journal.Operation, error) + +func (f stopFunc) Stop(ctx context.Context, request lifecycle.StopRequest) (journal.Operation, error) { + return f(ctx, request) +} + +type lifecycleFunc struct { + start startFunc + stop stopFunc +} + +func (f lifecycleFunc) Start(ctx context.Context, request lifecycle.StartRequest) (journal.Operation, error) { + return f.start(ctx, request) +} + +func (f lifecycleFunc) Stop(ctx context.Context, request lifecycle.StopRequest) (journal.Operation, error) { + return f.stop(ctx, request) +} + +func stopHandler(t *testing.T, executor StopExecutor) http.Handler { + t.Helper() + h, err := NewHandler(Config{RuntimeIdentity: "runtime-01", RuntimeGeneration: 7, Token: testRuntimeToken, Stop: executor}) + if err != nil { + t.Fatal(err) + } + return h +} + +func submitStop(t *testing.T, handler http.Handler, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, stopPath, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+testRuntimeToken) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w +} diff --git a/docs/architecture/adr/0001-v2-runtime-foundation.md b/docs/architecture/adr/0001-v2-runtime-foundation.md index 27fb91c4c..4525b2c9c 100644 --- a/docs/architecture/adr/0001-v2-runtime-foundation.md +++ b/docs/architecture/adr/0001-v2-runtime-foundation.md @@ -74,7 +74,7 @@ Transport placement: - Docker Embedded: private Docker network only; no host publication by default. - Native Linux/macOS/Windows: loopback by default. -The protocol MUST be authenticated per installation and MUST support explicit timeouts. The implemented surface contains handshake, protocol version, runtime identity/generation, capabilities, status, running CPA version, and an optional typed Start mutation. Additional lifecycle mutations are separate later slices. +The protocol MUST be authenticated per installation and MUST support explicit timeouts. The implemented surface contains handshake, protocol version, runtime identity/generation, capabilities, status, running CPA version, and typed Start and Stop mutations. Additional lifecycle mutations are separate later slices. #### Runtime generation @@ -152,18 +152,28 @@ A mutation result contains at least `operationId`, `operationType`, `runtimeIden The endpoint fixes the operation type to `start`; its typed payload is empty. The request body is limited to 16 KiB and MUST reject unknown or duplicate fields and trailing JSON values. HTTP callers MUST NOT supply executable paths, arguments, environment, working directories, shell commands, or generic action parameters. Logical request identity is the secret-free, versioned canonical value `runtime.start/v1:{}`; the journal stores its SHA-256 fingerprint and compares the operation type separately. -Start requires the Supervisor-local settings `CPAMP_RUNTIME_JOURNAL_PATH` and `CPAMP_CPA_EXECUTABLE`. Both absent preserves read-only operation and returns `400 unsupported_operation` for a valid authenticated Start request; configuring only one fails startup. With both configured, Supervisor opens its private journal once at startup, retains one Start executor and child manager, and drains accepted execution before closing the journal at shutdown. It does not terminate the child to compensate for a request or persistence failure. +Start and Stop require the Supervisor-local settings `CPAMP_RUNTIME_JOURNAL_PATH` and `CPAMP_CPA_EXECUTABLE`. Both absent preserves read-only operation and returns `400 unsupported_operation` for a valid authenticated lifecycle request; configuring only one fails startup. With both configured, Supervisor opens its private journal once at startup, retains one shared lifecycle executor and child manager, and drains accepted execution before closing the journal exactly once at shutdown. It does not terminate the child to compensate for a request or persistence failure. At spawn, Supervisor MUST explicitly build the CPA child environment by removing the Supervisor-private namespace `CPAMP_RUNTIME_*` and the exact variable `CPAMP_CPA_EXECUTABLE`. Name matching is case sensitive on Unix and case insensitive on Windows. All other parent entries retain their names, values and order, including proxy, certificate, timezone, locale, XDG and non-private `CPAMP_*` settings. An empty result MUST NOT fall back to unfiltered parent environment inheritance. New Supervisor-private settings must use that namespace or be explicitly added to the filter. The working directory remains inherited; this boundary does not introduce a general environment override API or OS privilege sandbox. -Handshake and status advertise the exact capability `start` only when the Start executor is configured. This capability means typed Start submission is supported; it does not promise that current process preconditions permit Start. Read-only operation MUST NOT advertise it. +Handshake and status advertise the exact capabilities `start` and `stop`, in that order, only when the shared lifecycle executor is configured. `start` means typed Start submission is supported; it does not promise that current process preconditions permit Start. Read-only operation MUST NOT advertise either capability. -Start submission is serialized. After authentication, strict validation, identity/generation fencing and durable operation-ID lookup, an existing operation is returned with HTTP `200`, its retained state and its creation generation. Replay never resumes or repeats execution, including for `accepted` or `running` evidence. A new Start requires `not_started` or confirmed, reaped `exited` process state; `running` or unknown ownership returns `409 operation_state_conflict` before writing intent. +Start and Stop submissions share one serialization boundary. After authentication, strict validation, identity/generation fencing and durable operation-ID lookup, an existing operation is returned with HTTP `200`, its retained state and its creation generation. Replay never resumes or repeats execution, including for `accepted` or `running` evidence. A new Start requires `not_started` or confirmed, reaped `exited` process state; `running` or unknown ownership returns `409 operation_state_conflict` before writing intent. For a new operation, Supervisor commits accepted intent, commits running evidence, spawns the locally configured executable, and commits a terminal result. Required pre-spawn persistence failure returns `503 operation_persistence_unavailable` and MUST NOT spawn. After durable intent commits, the HTTP caller's cancellation or deadline no longer owns execution or child lifetime. Execution remains synchronous without a queue or recovery worker. Start `succeeded` means only successful OS spawn and child ownership publication. It does not establish readiness, probe listeners or the CPA Management API, populate observed CPA version, or change RuntimeGeneration. Spawn failure attempts to retain a `failed` result with stable code `process_start_failed` and returns `500 internal_error`. Terminal persistence failure also returns `500 internal_error`; the child may already be running, and retained accepted/running evidence MUST prevent a repeated spawn for the same operation ID. Operation observation and recovery are separate capabilities. +#### Typed Stop + +`POST /v1/runtime/operations/stop` accepts only the same common mutation envelope as Start. The endpoint fixes the operation type to `stop`; its typed payload is empty, and its logical request identity is `runtime.stop/v1:{}` with a SHA-256 journal fingerprint. The caller cannot provide a PID, signal, force flag, grace or timeout policy, executable, arguments, environment, working directory, shell command, or generic action parameters. + +A new Stop requires a confirmed running child that the current `cpaprocess.Manager` owns. The Supervisor reserves that exact owned child handle during the operation precondition and retains the reservation across durable accepted/running evidence and termination. `not_started`, confirmed `exited`, unknown state, unconfirmed ownership, or another active Stop reservation returns `409 operation_state_conflict` before writing intent. The implementation MUST NOT recover a target from an observed PID, search for an OS process, or redirect an older Stop to a replacement child. + +After accepted intent and running evidence are durably committed, Stop uses the Supervisor-owned execution context and a cross-platform hard termination primitive on the reserved child handle. Stop `succeeded` requires the same child's `Wait` path to confirm exit, reap it, and release Manager ownership; a successful termination request alone is insufficient. A natural exit racing the termination request may succeed only when that same reserved child is confirmed reaped. Termination failure without confirmed exit attempts to retain `failed` with stable code `process_stop_failed`. + +Retained `accepted`, `running`, `succeeded`, or `failed` Stop evidence is replayed without resuming or repeating termination. If the child is already confirmed stopped but terminal persistence fails, the side effect is not rolled back and the retained operation ID MUST NOT terminate a current or later replacement child. The `stop` capability means typed Stop submission is supported; it does not promise that a running child is currently available. Stop does not change RuntimeGeneration or establish readiness. + Unix Domain Sockets and Windows Named Pipes are deferred. They may later be introduced as transport adapters without changing protocol semantics. ### 5. Supervisor operation journal