fix: exempt /metrics, /healthz, /readyz from auth; numeric UID in Dockerfile

- server: register /metrics, /healthz, and /readyz on a top-level mux so
  Kubernetes probes and Prometheus scrapers bypass API key auth entirely.
  All other routes fall through to the auth/rate-limit-wrapped apiMux.
- Dockerfile: create the mantle user with explicit -u 10001 -g 10001 and
  switch to USER 10001:10001 so kubelet can verify runAsNonRoot: true
  without a numeric runAsUser override in the Helm chart.

Closes #139
Closes #137

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: 2 people

packages/engine/Dockerfile

@@ -26,8 +26,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build \
 FROM alpine:3.21
 
 RUN apk add --no-cache ca-certificates tzdata \
-    && addgroup -S mantle \
-    && adduser -S -G mantle -h /home/mantle -s /bin/sh mantle
+    && addgroup -S -g 10001 mantle \
+    && adduser -S -u 10001 -G mantle -h /home/mantle -s /bin/sh mantle
 
 COPY --from=builder /mantle /usr/local/bin/mantle
 
@@ -39,7 +39,7 @@ EXPOSE 8080
 HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
     CMD wget -qO- http://localhost:8080/healthz || exit 1
 
-USER mantle
+USER 10001:10001
 WORKDIR /home/mantle
 
 ENTRYPOINT ["mantle"]

packages/engine/internal/server/server.go

@@ -108,43 +108,43 @@ func metricsMiddleware(next http.Handler) http.Handler {
 // Start starts the HTTP server, cron scheduler, and webhook handler.
 // It blocks until the context is cancelled.
 func (s *Server) Start(ctx context.Context) error {
-	mux := http.NewServeMux()
-
-	// Prometheus metrics endpoint.
-	mux.Handle("/metrics", promhttp.Handler())
+	// apiMux holds all routes that require authentication (and rate limiting).
+	// Health and metrics endpoints are registered on the top-level mux below
+	// so that Kubernetes probes and Prometheus scrapes bypass auth entirely.
+	apiMux := http.NewServeMux()
 
 	// Git push webhook endpoint — registered before the generic /hooks/ prefix
 	// so the more-specific path is explicit (Go's ServeMux uses longest-prefix
 	// match regardless of registration order, but explicit ordering is clearer).
-	mux.Handle("/hooks/git/", &GitWebhookHandler{
+	apiMux.Handle("/hooks/git/", &GitWebhookHandler{
 		DB:     s.DB,
 		Store:  s.RepoStore,
 		Driver: s.GitDriver,
 	})
 
 	// Webhook endpoints.
-	mux.HandleFunc("/hooks/", s.webhooks.ServeHTTP)
+	apiMux.HandleFunc("/hooks/", s.webhooks.ServeHTTP)
 
 	// API endpoints.
-	mux.HandleFunc("POST /api/v1/run/{workflow}", s.handleRun)
-	mux.HandleFunc("POST /api/v1/cancel/{execution}", s.handleCancel)
-	mux.HandleFunc("GET /api/v1/executions", s.handleListExecutions)
-	mux.HandleFunc("GET /api/v1/executions/{id}", s.handleGetExecution)
+	apiMux.HandleFunc("POST /api/v1/run/{workflow}", s.handleRun)
+	apiMux.HandleFunc("POST /api/v1/cancel/{execution}", s.handleCancel)
+	apiMux.HandleFunc("GET /api/v1/executions", s.handleListExecutions)
+	apiMux.HandleFunc("GET /api/v1/executions/{id}", s.handleGetExecution)
 
 	// Workflow definition endpoints.
-	mux.HandleFunc("GET /api/v1/workflows", s.handleListWorkflows)
-	mux.HandleFunc("GET /api/v1/workflows/{name}", s.handleGetWorkflow)
-	mux.HandleFunc("GET /api/v1/workflows/{name}/versions", s.handleListWorkflowVersions)
-	mux.HandleFunc("GET /api/v1/workflows/{name}/versions/{version}", s.handleGetWorkflowVersion)
+	apiMux.HandleFunc("GET /api/v1/workflows", s.handleListWorkflows)
+	apiMux.HandleFunc("GET /api/v1/workflows/{name}", s.handleGetWorkflow)
+	apiMux.HandleFunc("GET /api/v1/workflows/{name}/versions", s.handleListWorkflowVersions)
+	apiMux.HandleFunc("GET /api/v1/workflows/{name}/versions/{version}", s.handleGetWorkflowVersion)
 
 	// Budget endpoints.
-	mux.HandleFunc("GET /api/v1/budgets", s.handleListBudgets)
-	mux.HandleFunc("PUT /api/v1/budgets/{provider}", s.handleSetBudget)
-	mux.HandleFunc("DELETE /api/v1/budgets/{provider}", s.handleDeleteBudget)
-	mux.HandleFunc("GET /api/v1/budgets/usage", s.handleGetUsage)
+	apiMux.HandleFunc("GET /api/v1/budgets", s.handleListBudgets)
+	apiMux.HandleFunc("PUT /api/v1/budgets/{provider}", s.handleSetBudget)
+	apiMux.HandleFunc("DELETE /api/v1/budgets/{provider}", s.handleDeleteBudget)
+	apiMux.HandleFunc("GET /api/v1/budgets/usage", s.handleGetUsage)
 
 	// OpenAPI spec endpoint.
-	mux.HandleFunc("GET /api/v1/openapi.json", handleOpenAPISpec)
+	apiMux.HandleFunc("GET /api/v1/openapi.json", handleOpenAPISpec)
 
 	// Start distributed engine components (worker + reaper).
 	if cfg := config.FromContext(ctx); cfg != nil {
@@ -238,24 +238,30 @@ func (s *Server) Start(ctx context.Context) error {
 		go s.pollQueueDepth(ctx, cfg.Engine.ReaperInterval)
 	}
 
-	// Health endpoints. /readyz checks DB connectivity only; worker/reaper
-	// liveness is no longer gated here to avoid flapping between poll cycles.
-	mux.Handle("/healthz", health.HealthzHandler())
-	mux.Handle("/readyz", health.ReadyzHandler(s.DB))
-
-	// Wrap with metrics middleware (innermost, runs for every request).
-	handler := metricsMiddleware(mux)
+	// Wrap the API mux with metrics middleware, auth, and rate limiting.
+	apiHandler := metricsMiddleware(apiMux)
 	if s.AuthStore != nil {
 		if s.Auditor != nil {
-			handler = auth.AuthMiddleware(s.AuthStore, s.OIDCValidator, handler, s.Auditor)
+			apiHandler = auth.AuthMiddleware(s.AuthStore, s.OIDCValidator, apiHandler, s.Auditor)
 		} else {
-			handler = auth.AuthMiddleware(s.AuthStore, s.OIDCValidator, handler)
+			apiHandler = auth.AuthMiddleware(s.AuthStore, s.OIDCValidator, apiHandler)
 		}
 	}
-
-	// Apply rate limiting (after auth so rate limit keys can use API key prefix).
 	rl := NewRateLimiter(100, 200)
-	handler = rl.Middleware(handler)
+	apiHandler = rl.Middleware(apiHandler)
+
+	// Top-level mux: health and metrics endpoints bypass auth so that Kubernetes
+	// probes and Prometheus scrapers work without an API key. All other requests
+	// fall through to the auth-wrapped apiHandler.
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle("/healthz", health.HealthzHandler())
+	// /readyz checks DB connectivity only; worker/reaper liveness is intentionally
+	// excluded to prevent flapping between poll cycles.
+	mux.Handle("/readyz", health.ReadyzHandler(s.DB))
+	mux.Handle("/", apiHandler)
+
+	handler := mux
 
 	s.httpServer = &http.Server{
 		Addr:              s.Address,