fix(kotsadm): add wait-for-rqlite init container before schemahero migrations (#5925)

* fix(kotsadm): add wait-for-rqlite init container before schemahero

When kotsadm and rqlite restart simultaneously (e.g., during EC
upgrades), schemahero-plan runs before rqlite accepts connections,
causing CrashLoopBackOff with "tried all peers unsuccessfully".

Insert a wait-for-rqlite init container at position 0 in both the
Deployment and StatefulSet that polls http://kotsadm-rqlite:4001/readyz
until rqlite reports ready. This prevents the race between schemahero
and rqlite startup.

Affects both KotsadmDeployment() and KotsadmStatefulSet() init
container lists.

Closes replicated-collab/netbox-replicated#149
Ref: replicated-collab/netbox-replicated#148

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(kotsadm): add 5-min timeout, use lighter image, add unit tests

Address review findings:
- Add 5-minute timeout to wait loop so rqlite failures surface as a
  clear init error rather than an indefinite hang
- Use kotsadm-migrations image (lighter, already pulled for schemahero)
- Replace private repo link with Shortcut story reference
- Add unit tests for waitForRqliteInitContainer() and verify init
  container ordering in KotsadmDeployment()

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* refactor(kotsadm): extract wait-for-rqlite script to embedded .sh file

Per review feedback: move the inline shell script out of the Go source
and load it via go:embed, matching the existing pattern used by
minio_objects.go for scripts/{copy,export,import}-minio-data.sh.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: kriscoleman

pkg/kotsadm/objects/kotsadm_objects.go

@@ -1,8 +1,10 @@
 package kotsadm
 
 import (
+	_ "embed"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/kots/pkg/ingress"
@@ -20,6 +22,9 @@ import (
 	"k8s.io/utils/pointer"
 )
 
+//go:embed scripts/wait-for-rqlite.sh
+var waitForRqliteScript string
+
 func KotsadmClusterRole() *rbacv1.ClusterRole {
 	clusterRole := &rbacv1.ClusterRole{
 		TypeMeta: metav1.TypeMeta{
@@ -195,6 +200,32 @@ func updateKotsadmDeploymentScriptsPath(existing *appsv1.Deployment) {
 	}
 }
 
+// waitForRqliteInitContainer returns an init container that polls the rqlite
+// readiness endpoint before schemahero-plan runs. This prevents CrashLoopBackOff
+// when kotsadm and rqlite restart simultaneously (e.g., during EC upgrades).
+// Times out after 5 minutes so rqlite failures surface as a clear init error
+// rather than an indefinite hang.
+// Ref: https://app.shortcut.com/replicated/story/138103
+func waitForRqliteInitContainer(deployOptions types.DeployOptions) corev1.Container {
+	return corev1.Container{
+		Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Name:            "wait-for-rqlite",
+		Command:         []string{"sh", "-c"},
+		Args:            []string{strings.TrimSpace(waitForRqliteScript)},
+		Resources: corev1.ResourceRequirements{
+			Limits: corev1.ResourceList{
+				"memory": resource.MustParse("50Mi"),
+			},
+			Requests: corev1.ResourceList{
+				"cpu":    resource.MustParse("10m"),
+				"memory": resource.MustParse("10Mi"),
+			},
+		},
+		SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
+	}
+}
+
 func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, error) {
 	securityContext := k8sutil.SecurePodContext(1001, 1001, deployOptions.StrictSecurityContext)
 	if deployOptions.IsOpenShift {
@@ -493,6 +524,7 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e
 					RestartPolicy:      corev1.RestartPolicyAlways,
 					ImagePullSecrets:   pullSecrets,
 					InitContainers: []corev1.Container{
+						waitForRqliteInitContainer(deployOptions),
 						{
 							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
 							ImagePullPolicy: corev1.PullIfNotPresent,
@@ -1086,6 +1118,7 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit
 					RestartPolicy:      corev1.RestartPolicyAlways,
 					ImagePullSecrets:   pullSecrets,
 					InitContainers: []corev1.Container{
+						waitForRqliteInitContainer(deployOptions),
 						{
 							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
 							ImagePullPolicy: corev1.PullIfNotPresent,

pkg/kotsadm/objects/kotsadm_objects_test.go

@@ -1,9 +1,12 @@
 package kotsadm
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/replicatedhq/kots/pkg/kotsadm/types"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -226,3 +229,44 @@ func Test_updateKotsadmDeploymentScriptsPath(t *testing.T) {
 		})
 	}
 }
+
+func Test_waitForRqliteInitContainer(t *testing.T) {
+	opts := types.DeployOptions{
+		Namespace:              "default",
+		StrictSecurityContext:  true,
+	}
+	c := waitForRqliteInitContainer(opts)
+
+	assert.Equal(t, "wait-for-rqlite", c.Name)
+	assert.Equal(t, corev1.PullIfNotPresent, c.ImagePullPolicy)
+	assert.Equal(t, []string{"sh", "-c"}, c.Command)
+	require.Len(t, c.Args, 1)
+
+	// Polls /readyz
+	assert.Contains(t, c.Args[0], "kotsadm-rqlite:4001/readyz")
+	// Has a timeout (not an infinite loop)
+	assert.Contains(t, c.Args[0], "timeout=300")
+	// Exits non-zero on timeout
+	assert.True(t, strings.HasSuffix(c.Args[0], "exit 1"))
+
+	// Resource requests are set
+	assert.NotNil(t, c.Resources.Requests.Cpu())
+	assert.NotNil(t, c.Resources.Requests.Memory())
+	assert.NotNil(t, c.Resources.Limits.Memory())
+
+	// Security context is set
+	assert.NotNil(t, c.SecurityContext)
+}
+
+func Test_kotsadmDeploymentHasWaitForRqlite(t *testing.T) {
+	opts := types.DeployOptions{
+		Namespace: "default",
+	}
+	dep, err := KotsadmDeployment(opts)
+	require.NoError(t, err)
+
+	initContainers := dep.Spec.Template.Spec.InitContainers
+	require.True(t, len(initContainers) >= 4, "expected at least 4 init containers, got %d", len(initContainers))
+	assert.Equal(t, "wait-for-rqlite", initContainers[0].Name, "wait-for-rqlite should be the first init container")
+	assert.Equal(t, "schemahero-plan", initContainers[1].Name, "schemahero-plan should follow wait-for-rqlite")
+}

pkg/kotsadm/objects/scripts/wait-for-rqlite.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+# Polls the rqlite readiness endpoint before schemahero-plan runs.
+# Prevents CrashLoopBackOff when kotsadm and rqlite restart simultaneously
+# (e.g., during Embedded Cluster upgrades).
+# Times out after 5 minutes so rqlite failures surface as a clear init error
+# rather than an indefinite hang.
+
+timeout=300
+elapsed=0
+
+while [ $elapsed -lt $timeout ]; do
+  if wget -qO- http://kotsadm-rqlite:4001/readyz 2>/dev/null | grep -q "ok"; then
+    echo "rqlite is ready (${elapsed}s)"
+    exit 0
+  fi
+  echo "Waiting for rqlite... (${elapsed}s/${timeout}s)"
+  sleep 2
+  elapsed=$((elapsed+2))
+done
+
+echo "ERROR: rqlite not ready after ${timeout}s"
+exit 1