feat: production-ready horizontal scaling and multi-replica support

1. Configurable replicas: add spec.availability.replicas field
   (default 1, ignored when HPA is enabled). Controller preserves
   HPA-managed replica count to avoid fighting the autoscaler.

2. Heartbeat leader election: when replicas > 1, only pod-0 runs
   the heartbeat scheduler. Uses a shell wrapper that checks the
   StatefulSet ordinal in $HOSTNAME before exec-ing the entrypoint.

3. Auto TCP probes: in authenticated/single-tenant mode, /api/health
   returns 403 without credentials, breaking HTTP probes. New
   spec.probes.type field (auto/http/tcp) defaults to "auto" which
   uses TCP for authenticated modes and HTTP for open mode.

4. Managed DB improvements: add default resource requests/limits
   (250m/256Mi requests, 1/1Gi limits), enable --data-checksums
   for storage corruption detection, add preStop hook for graceful
   PostgreSQL shutdown. Document that external DB is recommended
   for HA production deployments.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: stubbi

api/v1alpha1/paperclipinstance_types.go

@@ -177,6 +177,9 @@ type DeploymentSpec struct {
 }
 
 // DatabaseSpec configures PostgreSQL.
+// For high-availability production deployments, use mode "external" with a managed
+// PostgreSQL service (e.g., Amazon RDS, Cloud SQL). The "managed" mode provides a
+// single-instance PostgreSQL suitable for development and small deployments.
 type DatabaseSpec struct {
 	// Mode selects the database mode: "embedded" (PGlite), "external" (connection string), or "managed" (operator-managed StatefulSet).
 	// +kubebuilder:default="managed"
@@ -474,6 +477,13 @@ type LoggingSpec struct {
 
 // AvailabilitySpec configures scaling and pod scheduling.
 type AvailabilitySpec struct {
+	// Replicas is the desired number of Paperclip server pods.
+	// Ignored when autoScaling is enabled (the HPA manages replicas).
+	// +kubebuilder:default=1
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty"`
+
 	// PodDisruptionBudget configures the PDB.
 	// +optional
 	PodDisruptionBudget *PDBSpec `json:"podDisruptionBudget,omitempty"`
@@ -540,6 +550,14 @@ type AutoScalingSpec struct {
 
 // ProbesSpec configures health probes.
 type ProbesSpec struct {
+	// Type specifies the probe mechanism: "auto" (default), "http", or "tcp".
+	// "auto" uses HTTP probes in open mode and TCP probes in authenticated/single-tenant mode
+	// (where /api/health returns 403 without credentials).
+	// +kubebuilder:default="auto"
+	// +kubebuilder:validation:Enum=auto;http;tcp
+	// +optional
+	Type string `json:"type,omitempty"`
+
 	// Liveness configures the liveness probe against /api/health.
 	// +optional
 	Liveness *ProbeSpec `json:"liveness,omitempty"`

api/v1alpha1/zz_generated.deepcopy.go

@@ -1083,6 +1083,14 @@ spec:
                     required:
                     - enabled
                     type: object
+                  replicas:
+                    default: 1
+                    description: |-
+                      Replicas is the desired number of Paperclip server pods.
+                      Ignored when autoScaling is enabled (the HPA manages replicas).
+                    format: int32
+                    minimum: 1
+                    type: integer
                   tolerations:
                     description: Tolerations specifies pod tolerations.
                     items:
@@ -5271,6 +5279,17 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  type:
+                    default: auto
+                    description: |-
+                      Type specifies the probe mechanism: "auto" (default), "http", or "tcp".
+                      "auto" uses HTTP probes in open mode and TCP probes in authenticated/single-tenant mode
+                      (where /api/health returns 403 without credentials).
+                    enum:
+                    - auto
+                    - http
+                    - tcp
+                    type: string
                 type: object
               resources:
                 description: Resources specifies the compute resources for the Paperclip

charts/paperclip-operator/templates/crds/paperclip.inc_instances.yaml

@@ -1077,6 +1077,14 @@ spec:
                     required:
                     - enabled
                     type: object
+                  replicas:
+                    default: 1
+                    description: |-
+                      Replicas is the desired number of Paperclip server pods.
+                      Ignored when autoScaling is enabled (the HPA manages replicas).
+                    format: int32
+                    minimum: 1
+                    type: integer
                   tolerations:
                     description: Tolerations specifies pod tolerations.
                     items:
@@ -5265,6 +5273,17 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  type:
+                    default: auto
+                    description: |-
+                      Type specifies the probe mechanism: "auto" (default), "http", or "tcp".
+                      "auto" uses HTTP probes in open mode and TCP probes in authenticated/single-tenant mode
+                      (where /api/health returns 403 without credentials).
+                    enum:
+                    - auto
+                    - http
+                    - tcp
+                    type: string
                 type: object
               resources:
                 description: Resources specifies the compute resources for the Paperclip

config/crd/bases/paperclip.inc_instances.yaml

@@ -372,6 +372,11 @@ func (r *InstanceReconciler) reconcileStatefulSet(ctx context.Context, instance
 
 	_, err := controllerutil.CreateOrUpdate(ctx, r.Client, obj, func() error {
 		obj.Labels = desired.Labels
+		// When HPA is enabled, preserve the current replica count to avoid
+		// fighting the autoscaler on every reconcile.
+		if as := instance.Spec.Availability.AutoScaling; as != nil && as.Enabled && obj.Spec.Replicas != nil {
+			desired.Spec.Replicas = obj.Spec.Replicas
+		}
 		obj.Spec = desired.Spec
 		return controllerutil.SetControllerReference(instance, obj, r.Scheme)
 	})

internal/controller/instance_controller.go

@@ -42,13 +42,40 @@ const (
 
 	// HealthPath is the HTTP health check path.
 	HealthPath = "/api/health"
+
+	// DefaultPaperclipEntrypoint is the default Paperclip container entrypoint.
+	// Used when the operator needs to inject a shell wrapper (e.g., heartbeat leader election).
+	DefaultPaperclipEntrypoint = `node --import ./server/node_modules/tsx/dist/loader.mjs server/dist/index.js`
 )
 
 // Ptr returns a pointer to the given value.
 func Ptr[T any](v T) *T {
 	return &v
 }
 
+// EffectiveReplicas returns the configured replica count, defaulting to 1.
+func EffectiveReplicas(instance *paperclipv1alpha1.Instance) int32 {
+	if instance.Spec.Availability.Replicas != nil {
+		return *instance.Spec.Availability.Replicas
+	}
+	return 1
+}
+
+// UseTCPProbes returns true when probes should use TCP instead of HTTP.
+// This is needed in authenticated/single-tenant mode where /api/health returns 403.
+func UseTCPProbes(instance *paperclipv1alpha1.Instance) bool {
+	probeType := instance.Spec.Probes.Type
+	if probeType == "tcp" {
+		return true
+	}
+	if probeType == "http" {
+		return false
+	}
+	// "auto" or empty: use TCP for authenticated/single-tenant modes
+	mode := instance.Spec.Deployment.Mode
+	return mode == "authenticated" || mode == "single-tenant"
+}
+
 // Labels returns the standard labels for a Instance resource.
 func Labels(instance *paperclipv1alpha1.Instance) map[string]string {
 	return map[string]string{

internal/resources/common.go

@@ -49,8 +49,16 @@ func BuildDatabaseStatefulSet(instance *paperclipv1alpha1.Instance) *appsv1.Stat
 				},
 			},
 			{Name: "PGDATA", Value: DatabaseMountPath + "/pgdata"},
+			{Name: "POSTGRES_INITDB_ARGS", Value: "--data-checksums"},
+		},
+		Resources:                databaseResources(instance),
+		Lifecycle: &corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{"/bin/sh", "-c", "pg_ctl stop -m fast -D $PGDATA"},
+				},
+			},
 		},
-		Resources:                instance.Spec.Database.Managed.Resources,
 		ImagePullPolicy:          corev1.PullIfNotPresent,
 		TerminationMessagePath:   "/dev/termination-log",
 		TerminationMessagePolicy: corev1.TerminationMessageReadFile,
@@ -144,6 +152,23 @@ func BuildDatabaseStatefulSet(instance *paperclipv1alpha1.Instance) *appsv1.Stat
 	return sts
 }
 
+func databaseResources(instance *paperclipv1alpha1.Instance) corev1.ResourceRequirements {
+	r := instance.Spec.Database.Managed.Resources
+	if len(r.Requests) == 0 && len(r.Limits) == 0 {
+		return corev1.ResourceRequirements{
+			Requests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("250m"),
+				corev1.ResourceMemory: resource.MustParse("256Mi"),
+			},
+			Limits: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("1"),
+				corev1.ResourceMemory: resource.MustParse("1Gi"),
+			},
+		}
+	}
+	return r
+}
+
 // BuildDatabaseSecret constructs the auto-generated database credentials Secret.
 func BuildDatabaseSecret(instance *paperclipv1alpha1.Instance, password string) *corev1.Secret {
 	return &corev1.Secret{

internal/resources/database.go

@@ -95,9 +95,12 @@ func TestBuildStatefulSet(t *testing.T) {
 		t.Error("expected startup probe")
 	}
 
-	// Verify health check path
-	if container.LivenessProbe.HTTPGet.Path != HealthPath {
-		t.Errorf("expected liveness probe path %q, got %q", HealthPath, container.LivenessProbe.HTTPGet.Path)
+	// Verify probe type: authenticated mode should use TCP probes
+	if container.LivenessProbe.TCPSocket == nil {
+		t.Error("expected TCP liveness probe for authenticated mode")
+	}
+	if container.ReadinessProbe.TCPSocket == nil {
+		t.Error("expected TCP readiness probe for authenticated mode")
 	}
 
 	// Verify volume mounts

internal/resources/resources_test.go

@@ -16,7 +16,7 @@ func BuildStatefulSet(instance *paperclipv1alpha1.Instance) *appsv1.StatefulSet
 	labels := LabelsWithComponent(instance, "server")
 	selectorLabels := SelectorLabels(instance)
 
-	replicas := int32(1)
+	replicas := EffectiveReplicas(instance)
 
 	container := buildMainContainer(instance)
 	volumes := buildVolumes(instance)
@@ -136,6 +136,15 @@ func buildMainContainer(instance *paperclipv1alpha1.Instance) corev1.Container {
 		}
 	}
 
+	// Multi-replica heartbeat gating: only pod-0 runs the scheduler.
+	// Uses a shell wrapper that checks the StatefulSet ordinal in $HOSTNAME.
+	if instance.Spec.Heartbeat.Enabled && EffectiveReplicas(instance) > 1 {
+		container.Command = []string{"/bin/sh", "-c"}
+		container.Args = []string{
+			`case "$HOSTNAME" in *-0) export HEARTBEAT_SCHEDULER_ENABLED=true ;; *) export HEARTBEAT_SCHEDULER_ENABLED=false ;; esac; exec ` + DefaultPaperclipEntrypoint,
+		}
+	}
+
 	// Probes
 	container.LivenessProbe = buildLivenessProbe(instance, port)
 	container.ReadinessProbe = buildReadinessProbe(instance, port)
@@ -231,6 +240,9 @@ func buildEnvVars(instance *paperclipv1alpha1.Instance) []corev1.EnvVar {
 	}
 
 	// Heartbeat scheduler
+	// When heartbeat is disabled, explicitly disable it on all pods.
+	// When enabled with multiple replicas, the command wrapper handles per-pod gating
+	// (only pod-0 runs the scheduler), so we skip the static env var here.
 	if !instance.Spec.Heartbeat.Enabled {
 		vars = append(vars, corev1.EnvVar{Name: "HEARTBEAT_SCHEDULER_ENABLED", Value: "false"})
 	}
@@ -349,15 +361,26 @@ func buildVolumeMounts(instance *paperclipv1alpha1.Instance) []corev1.VolumeMoun
 	return mounts
 }
 
-func buildLivenessProbe(instance *paperclipv1alpha1.Instance, port int32) *corev1.Probe {
-	probe := &corev1.Probe{
-		ProbeHandler: corev1.ProbeHandler{
-			HTTPGet: &corev1.HTTPGetAction{
-				Path:   HealthPath,
-				Port:   intstr.FromInt32(port),
-				Scheme: corev1.URISchemeHTTP,
+func probeHandler(instance *paperclipv1alpha1.Instance, port int32) corev1.ProbeHandler {
+	if UseTCPProbes(instance) {
+		return corev1.ProbeHandler{
+			TCPSocket: &corev1.TCPSocketAction{
+				Port: intstr.FromInt32(port),
 			},
+		}
+	}
+	return corev1.ProbeHandler{
+		HTTPGet: &corev1.HTTPGetAction{
+			Path:   HealthPath,
+			Port:   intstr.FromInt32(port),
+			Scheme: corev1.URISchemeHTTP,
 		},
+	}
+}
+
+func buildLivenessProbe(instance *paperclipv1alpha1.Instance, port int32) *corev1.Probe {
+	probe := &corev1.Probe{
+		ProbeHandler:        probeHandler(instance, port),
 		InitialDelaySeconds: 15,
 		PeriodSeconds:       20,
 		TimeoutSeconds:      5,
@@ -388,13 +411,7 @@ func buildLivenessProbe(instance *paperclipv1alpha1.Instance, port int32) *corev
 
 func buildReadinessProbe(instance *paperclipv1alpha1.Instance, port int32) *corev1.Probe {
 	probe := &corev1.Probe{
-		ProbeHandler: corev1.ProbeHandler{
-			HTTPGet: &corev1.HTTPGetAction{
-				Path:   HealthPath,
-				Port:   intstr.FromInt32(port),
-				Scheme: corev1.URISchemeHTTP,
-			},
-		},
+		ProbeHandler:        probeHandler(instance, port),
 		InitialDelaySeconds: 5,
 		PeriodSeconds:       10,
 		TimeoutSeconds:      3,
@@ -425,13 +442,7 @@ func buildReadinessProbe(instance *paperclipv1alpha1.Instance, port int32) *core
 
 func buildStartupProbe(instance *paperclipv1alpha1.Instance, port int32) *corev1.Probe {
 	probe := &corev1.Probe{
-		ProbeHandler: corev1.ProbeHandler{
-			HTTPGet: &corev1.HTTPGetAction{
-				Path:   HealthPath,
-				Port:   intstr.FromInt32(port),
-				Scheme: corev1.URISchemeHTTP,
-			},
-		},
+		ProbeHandler:        probeHandler(instance, port),
 		InitialDelaySeconds: 0,
 		PeriodSeconds:       5,
 		TimeoutSeconds:      3,