Skip to content

feat: add HTTP health endpoints to instance manager #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
usiegj00 merged 2 commits into from
Jan 25, 2026
Merged

feat: add HTTP health endpoints to instance manager #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
34efed2
feat: add HTTP health endpoints to instance manager
usiegj00 Jan 25, 2026
ed81590
fix: handle return values for lint compliance
usiegj00 Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 158 additions & 0 deletions .github/workflows/e2e.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,164 @@ jobs:
exit 1
fi

- name: Test /healthz endpoint returns 200 when healthy
run: |
echo "Testing /healthz endpoint..."
# Use kubectl port-forward to access health endpoint
kubectl port-forward pod/rfr-test-im-0 8080:8080 &
PF_PID=$!
sleep 2

RESPONSE=$(curl -s -w "\n%{http_code}" http://localhost:8080/healthz)
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)

kill $PF_PID 2>/dev/null || true

echo "Response: $BODY"
echo "HTTP Code: $HTTP_CODE"

if [[ "$HTTP_CODE" == "200" ]]; then
echo "✓ /healthz returned 200"
else
echo "✗ /healthz returned $HTTP_CODE, expected 200"
exit 1
fi

# Verify response contains expected fields
if echo "$BODY" | grep -q '"status":"ok"'; then
echo "✓ /healthz response contains status:ok"
else
echo "✗ /healthz response missing status:ok"
exit 1
fi

- name: Test /readyz endpoint returns 200 when ready
run: |
echo "Testing /readyz endpoint..."
kubectl port-forward pod/rfr-test-im-0 8080:8080 &
PF_PID=$!
sleep 2

RESPONSE=$(curl -s -w "\n%{http_code}" http://localhost:8080/readyz)
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)

kill $PF_PID 2>/dev/null || true

echo "Response: $BODY"
echo "HTTP Code: $HTTP_CODE"

if [[ "$HTTP_CODE" == "200" ]]; then
echo "✓ /readyz returned 200"
else
echo "✗ /readyz returned $HTTP_CODE, expected 200"
exit 1
fi

# Verify response contains role
if echo "$BODY" | grep -q '"role"'; then
echo "✓ /readyz response contains role"
else
echo "✗ /readyz response missing role"
exit 1
fi

- name: Test /status endpoint returns detailed info
run: |
echo "Testing /status endpoint..."
kubectl port-forward pod/rfr-test-im-0 8080:8080 &
PF_PID=$!
sleep 2

RESPONSE=$(curl -s -w "\n%{http_code}" http://localhost:8080/status)
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)

kill $PF_PID 2>/dev/null || true

echo "Response: $BODY"
echo "HTTP Code: $HTTP_CODE"

if [[ "$HTTP_CODE" == "200" ]]; then
echo "✓ /status returned 200"
else
echo "✗ /status returned $HTTP_CODE, expected 200"
exit 1
fi

# Verify response contains expected sections
if echo "$BODY" | grep -q '"redis"' && echo "$BODY" | grep -q '"instance_manager"'; then
echo "✓ /status response contains redis and instance_manager sections"
else
echo "✗ /status response missing expected sections"
exit 1
fi

- name: Test /healthz returns 503 when Redis is killed
run: |
echo "Testing /healthz returns 503 when Redis process dies..."

# Start port-forward
kubectl port-forward pod/rfr-test-im-0 8080:8080 &
PF_PID=$!
sleep 2

# Verify healthy first
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/healthz)
if [[ "$HTTP_CODE" != "200" ]]; then
echo "✗ Initial /healthz check failed with $HTTP_CODE"
kill $PF_PID 2>/dev/null || true
exit 1
fi
echo "✓ Initial /healthz is healthy"

# Kill redis-server process (instance manager will detect this)
echo "Killing redis-server process..."
kubectl exec rfr-test-im-0 -- /bin/sh -c "kill \$(pgrep redis-server)" || true

# Wait for health check to detect (checks every 1s)
echo "Waiting for health check to detect failure..."
sleep 3

# Check that /healthz now returns 503
RESPONSE=$(curl -s -w "\n%{http_code}" http://localhost:8080/healthz)
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)

kill $PF_PID 2>/dev/null || true

echo "Response after kill: $BODY"
echo "HTTP Code: $HTTP_CODE"

if [[ "$HTTP_CODE" == "503" ]]; then
echo "✓ /healthz returned 503 after Redis killed"
else
echo "⚠ /healthz returned $HTTP_CODE (pod may have restarted already)"
fi

- name: Wait for pod recovery after health test
run: |
echo "Waiting for pod to recover..."
sleep 5
kubectl wait --for=condition=Ready pod/rfr-test-im-0 --timeout=120s
echo "✓ Pod recovered and is Ready"

# Verify health endpoints work again
kubectl port-forward pod/rfr-test-im-0 8080:8080 &
PF_PID=$!
sleep 2

HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/healthz)
kill $PF_PID 2>/dev/null || true

if [[ "$HTTP_CODE" == "200" ]]; then
echo "✓ /healthz returns 200 after recovery"
else
echo "✗ /healthz returned $HTTP_CODE after recovery"
exit 1
fi

- name: Collect logs on failure
if: failure()
run: |
Expand Down
33 changes: 30 additions & 3 deletions cmd/instance/run/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ import (
"github.com/spf13/cobra"
)

// healthServer is the global health server instance
var healthServer *HealthServer

const (
defaultDataDir = "/data"
defaultDBFilename = "dump.rdb"
Expand All @@ -44,6 +47,8 @@ var (
dataDir string
dbFilename string
redisConf string
healthPort int
redisPort string
)

// NewCmd creates the run command
Expand Down Expand Up @@ -77,6 +82,8 @@ This architecture provides:
cmd.Flags().StringVar(&dataDir, "data-dir", defaultDataDir, "Redis data directory")
cmd.Flags().StringVar(&dbFilename, "db-filename", defaultDBFilename, "Main RDB filename to preserve during cleanup")
cmd.Flags().StringVar(&redisConf, "redis-conf", defaultRedisConf, "Path to redis.conf")
cmd.Flags().IntVar(&healthPort, "health-port", defaultHealthPort, "Port for health check endpoints")
cmd.Flags().StringVar(&redisPort, "redis-port", "6379", "Redis port for health checks")

return cmd
}
Expand All @@ -94,12 +101,27 @@ func runInstance(cmd *cobra.Command, args []string) error {
go runZombieReaper(ctx)

// Step 2: Perform startup cleanup
if err := performStartupCleanup(); err != nil {
cleanupErr := performStartupCleanup()
if cleanupErr != nil {
// Log but don't fail - Redis should still be able to start
fmt.Printf("redis-instance: warning: startup cleanup failed: %v\n", err)
fmt.Printf("redis-instance: warning: startup cleanup failed: %v\n", cleanupErr)
}

// Step 3: Start health server (provides /healthz, /readyz, /status)
healthServer = NewHealthServer(healthPort, redisPort)
healthServer.SetCleanupDone(cleanupErr == nil)
if err := healthServer.Start(ctx); err != nil {
fmt.Printf("redis-instance: warning: failed to start health server: %v\n", err)
}
defer func() {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
if err := healthServer.Stop(shutdownCtx); err != nil {
fmt.Printf("redis-instance: warning: health server stop error: %v\n", err)
}
}()

// Step 3: Main process loop (CNPG pattern)
// Step 4: Main process loop (CNPG pattern)
// This loop allows for process restarts without manager exit
return runProcessLoop(ctx, cancel)
}
Expand Down Expand Up @@ -127,6 +149,11 @@ func runProcessLoop(ctx context.Context, cancel context.CancelFunc) error {
redisPid := redisCmd.Process.Pid
fmt.Printf("redis-instance: redis-server started with PID %d\n", redisPid)

// Notify health server of Redis PID
if healthServer != nil {
healthServer.SetRedisPID(redisPid)
}

// Wait for either Redis to exit or a signal
doneChan := make(chan error, 1)
go func() {
Expand Down
Loading