K8SPS-866: stop pt-heartbeat from starting during a clone

api/v1/perconaservermysql_types.go

@@ -1253,7 +1253,16 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion
 		cr.Spec.MySQL.StartupProbe.SuccessThreshold = 1
 	}
 	if cr.Spec.MySQL.StartupProbe.TimeoutSeconds == 0 {
-		cr.Spec.MySQL.StartupProbe.TimeoutSeconds = 12 * 60 * 60
+		// The startup probe runs the bootstrap (including the clone), so its
+		// timeout caps how long a clone may take. From 1.3.0 a clone may run for
+		// as long as it keeps making progress (the stall watchdog aborts only a
+		// frozen clone), so we give it a much larger cap: 7 days. Older clusters
+		// keep the previous 12 hours.
+		if cr.Spec.CRVersion != "" && cr.CompareVersion("1.3.0") >= 0 {
+			cr.Spec.MySQL.StartupProbe.TimeoutSeconds = 7 * 24 * 60 * 60
+		} else {
+			cr.Spec.MySQL.StartupProbe.TimeoutSeconds = 12 * 60 * 60
+		}
 	}
 
 	if cr.Spec.MySQL.LivenessProbe.InitialDelaySeconds == 0 {

build/heartbeat-entrypoint.sh

@@ -1,10 +1,15 @@
 #!/bin/bash
 
 shutdown_requested=0
+hb_pid=''
 
 handle_shutdown() {
 	shutdown_requested=1
 	echo '[INFO] Shutdown requested, exiting heartbeat entrypoint'
+	# Forward the signal to a running pt-heartbeat so the container stops promptly.
+	if [ -n "$hb_pid" ] && kill -0 "$hb_pid" 2>/dev/null; then
+		kill -TERM "$hb_pid" 2>/dev/null
+	fi
 }
 
 trap handle_shutdown SIGTERM SIGINT
@@ -18,38 +23,42 @@ until [ ! -f "$DATA_DIR/bootstrap.lock" ] && [ ! -f "$DATA_DIR/clone.lock" ] &&
 	sleep 10
 done
 
-# wait until bootstrap start clone process
-# we can get situation when ps-entrypoint removed bootstrap.lock but bootstrap has not created clone.lock yet
 if [ "$shutdown_requested" -eq 1 ]; then
 	exit 0
 fi
-sleep 10
-if [ -f /var/lib/mysql/clone.lock ]; then
-	CLONE_IN_PROGRESS='yes'
-fi
 
 MYSQL_ADMIN_PORT='33062'
 MYSQL_USER="${MYSQL_USERNAME:-monitor}"
 MYSQL_PASSWORD=$(cat /etc/mysql/mysql-users-secret/monitor || :)
-TIMEOUT="${CLONE_TIMEOUT_SECONDS:-3600}"
 MYSQL_CMDLINE="/usr/bin/timeout 10 /usr/bin/mysql -nNE -u$MYSQL_USER"
 
-# Check clone status every 5 seconds up to TIMEOUT
+# Gate the start on authoritative DB state, not on the clone.lock file: native
+# InnoDB clone (CLONE INSTANCE) does not reliably create clone.lock, so a
+# missing lock is not proof the datadir is ready. Start pt-heartbeat only once
+# no clone is 'In Progress' and the sys_operator schema it needs is present -
+# a partial clone leaves clone_status='In Progress' with no sys_operator, which
+# otherwise crash-loops pt-heartbeat on "Unknown database 'sys_operator'".
+#
+# Wait for as long as it takes: a clone of a large dataset can run for hours,
+# and waiting costs nothing. A genuinely stuck replica is surfaced by the mysql
+# container's own readiness, not by this sidecar. We only stop on SIGTERM.
 CHECK_INTERVAL=5
-ELAPSED=0
 
-while [ "$ELAPSED" -lt "$TIMEOUT" ]; do
+while true; do
 	if [ "$shutdown_requested" -eq 1 ]; then
 		exit 0
 	fi
 
 	CLONE_STATUS=$(MYSQL_PWD=${MYSQL_PASSWORD} $MYSQL_CMDLINE -P$MYSQL_ADMIN_PORT -e 'SELECT STATE FROM performance_schema.clone_status;' | sed -n -e '2p' | tr -d '\n')
-	if [[ $CLONE_STATUS == "Completed" || -z $CLONE_IN_PROGRESS ]]; then
-		echo '[INFO] Clone completed, starting pt-heartbeat'
-		break
+	if [[ $CLONE_STATUS != "In Progress" ]]; then
+		HAS_SYS_OPERATOR=$(MYSQL_PWD=${MYSQL_PASSWORD} $MYSQL_CMDLINE -P$MYSQL_ADMIN_PORT -e "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME='sys_operator';" | sed -n -e '2p' | tr -d '\n')
+		if [[ $HAS_SYS_OPERATOR == "sys_operator" ]]; then
+			echo '[INFO] Clone finished and sys_operator present, starting pt-heartbeat'
+			break
+		fi
 	fi
 
-	echo "[INFO] Waiting for MySQL initialization (${ELAPSED}s/${TIMEOUT}s)"
+	echo "[INFO] Waiting for clone to finish and sys_operator to appear, clone_status='${CLONE_STATUS:-none}'"
 
 	# Sleep in 1-second intervals to allow signal handling
 	for ((j = 0; j < CHECK_INTERVAL; j++)); do
@@ -58,25 +67,72 @@ while [ "$ELAPSED" -lt "$TIMEOUT" ]; do
 		fi
 		sleep 1
 	done
-
-	ELAPSED=$((ELAPSED + CHECK_INTERVAL))
 done
 
 # If password contains commas they must be escaped with a backslash: “exam,ple” according https://docs.percona.com/percona-toolkit/pt-heartbeat.html
 ESCAPED_HEARTBEAT_PASSWORD="${HEARTBEAT_PASSWORD//,/\\,}"
 
 HEARTBEAT_USER='heartbeat'
-echo "[INFO] pt-heartbeat --update --replace --fail-successive-errors 20 --check-read-only --create-table --database sys_operator \
-	--table heartbeat --user ${HEARTBEAT_USER} --password XXXX --port ${MYSQL_ADMIN_PORT}"
-
-exec pt-heartbeat \
-	--update \
-	--replace \
-	--fail-successive-errors 20 \
-	--check-read-only \
-	--create-table \
-	--database sys_operator \
-	--table heartbeat \
-	--user "${HEARTBEAT_USER}" \
-	--password "${ESCAPED_HEARTBEAT_PASSWORD}" \
-	--port "${MYSQL_ADMIN_PORT}"
+
+# A clone finishes with a mandatory mysqld restart to finalize the data. The
+# datadir already looks ready (clone done, sys_operator present) several seconds
+# before that restart, so pt-heartbeat can be started just in time to hit it and
+# exit with "Server shutdown in progress". Run pt-heartbeat under a bounded retry
+# loop: if it exits shortly after starting, wait for MySQL to come back and relaunch
+# in-place, so a fresh replica does not show a container restart. A pt-heartbeat that
+# ran for a while before exiting is treated as a real failure and surfaced by exiting,
+# which lets the container restart as usual.
+RETRY_GRACE_SECONDS=60
+MAX_QUICK_RETRIES=10
+quick_retries=0
+
+while true; do
+	if [ "$shutdown_requested" -eq 1 ]; then
+		exit 0
+	fi
+
+	echo "[INFO] pt-heartbeat --update --replace --fail-successive-errors 20 --check-read-only --create-table --database sys_operator \
+		--table heartbeat --user ${HEARTBEAT_USER} --password XXXX --port ${MYSQL_ADMIN_PORT}"
+
+	start_ts=$SECONDS
+	pt-heartbeat \
+		--update \
+		--replace \
+		--fail-successive-errors 20 \
+		--check-read-only \
+		--create-table \
+		--database sys_operator \
+		--table heartbeat \
+		--user "${HEARTBEAT_USER}" \
+		--password "${ESCAPED_HEARTBEAT_PASSWORD}" \
+		--port "${MYSQL_ADMIN_PORT}" &
+	hb_pid=$!
+	wait "$hb_pid"
+	rc=$?
+	hb_pid=''
+	ran_for=$((SECONDS - start_ts))
+
+	if [ "$shutdown_requested" -eq 1 ]; then
+		exit 0
+	fi
+
+	if [ "$ran_for" -ge "$RETRY_GRACE_SECONDS" ]; then
+		# Ran long enough to be considered healthy before exiting: a real failure.
+		# Exit so the container restarts and the problem is visible.
+		echo "[ERROR] pt-heartbeat exited after ${ran_for}s (rc=${rc}); exiting so the container restarts"
+		exit "$rc"
+	fi
+
+	quick_retries=$((quick_retries + 1))
+	if [ "$quick_retries" -gt "$MAX_QUICK_RETRIES" ]; then
+		echo "[ERROR] pt-heartbeat kept exiting quickly (${quick_retries} times, last rc=${rc}); giving up so the container restarts"
+		exit "$rc"
+	fi
+
+	echo "[WARN] pt-heartbeat exited after ${ran_for}s (rc=${rc}); expected around the post-clone MySQL restart - waiting for MySQL and retrying (${quick_retries}/${MAX_QUICK_RETRIES})"
+
+	# Wait for the admin interface to accept connections again before retrying.
+	until [ "$shutdown_requested" -eq 1 ] || MYSQL_PWD=${MYSQL_PASSWORD} $MYSQL_CMDLINE -P$MYSQL_ADMIN_PORT -e 'SELECT 1;' >/dev/null 2>&1; do
+		sleep 2
+	done
+done

cmd/bootstrap/async/async_replication.go

@@ -96,6 +96,17 @@ func Bootstrap(ctx context.Context) error {
 	}
 	params.CloneTimeoutSeconds = cloneTimeout
 
+	// From crVersion 1.3.0 the operator sets BOOTSTRAP_CLONE_STALL_TIMEOUT, which
+	// switches on a progress watchdog instead of a fixed clone timeout: the clone
+	// is aborted only if it stops transferring bytes for this long (0 = disabled).
+	cloneStallTimeout, stallSet, err := utils.GetCloneStallTimeout()
+	if err != nil {
+		return errors.Wrap(err, "get clone stall timeout")
+	}
+	if stallSet {
+		log.Printf("Clone progress watchdog stall timeout: %ds (0 = disabled)", cloneStallTimeout)
+	}
+
 	sourceRetryCount, err := utils.GetSourceRetryCount()
 	if err != nil {
 		return errors.Wrap(err, "get source retry count")
@@ -167,7 +178,7 @@ func Bootstrap(ctx context.Context) error {
 
 		timer.Start("clone")
 		log.Printf("Cloning from %s", donor)
-		err = db.Clone(ctx, donor, string(apiv1.UserOperator), operatorPass, mysql.DefaultAdminPort, params.CloneTimeoutSeconds)
+		err = db.Clone(ctx, donor, string(apiv1.UserOperator), operatorPass, mysql.DefaultAdminPort, params.CloneTimeoutSeconds, cloneStallTimeout)
 		timer.Stop("clone")
 		if err != nil && !errors.Is(err, database.ErrRestartAfterClone) {
 			return errors.Wrapf(err, "clone from donor %s", donor)

cmd/bootstrap/utils/utils.go

@@ -65,6 +65,26 @@ func GetCloneTimeout() (uint32, error) {
 	return uint32(cloneTimeout), nil
 }
 
+// GetCloneStallTimeout reads BOOTSTRAP_CLONE_STALL_TIMEOUT. The bool return
+// reports whether the env is set at all: the operator sets it only from
+// crVersion 1.3.0, so its presence is what switches on the progress watchdog.
+// A value of 0 means "watchdog disabled" (clone runs with no progress check).
+func GetCloneStallTimeout() (uint32, bool, error) {
+	s, ok := os.LookupEnv(naming.EnvBootstrapCloneStallTimeout)
+	if !ok {
+		return 0, false, nil
+	}
+	stall, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, true, errors.Wrap(err, "failed to parse BOOTSTRAP_CLONE_STALL_TIMEOUT")
+	}
+	if stall < 0 {
+		return 0, true, errors.New("BOOTSTRAP_CLONE_STALL_TIMEOUT should be a non-negative value")
+	}
+
+	return uint32(stall), true, nil
+}
+
 func GetSourceRetryCount() (uint32, error) {
 	s, ok := os.LookupEnv(naming.EnvAsyncSourceRetryCount)
 	if !ok {

cmd/bootstrap/utils/utils_test.go

@@ -132,6 +132,65 @@ func TestGetCloneTimeout(t *testing.T) {
 	}
 }
 
+func TestGetCloneStallTimeout(t *testing.T) {
+	tests := map[string]struct {
+		set           bool
+		envValue      string
+		expectedValue uint32
+		expectedSet   bool
+		expectedError string
+	}{
+		"unset -> not present": {
+			set:         false,
+			expectedSet: false,
+		},
+		"valid positive": {
+			set:           true,
+			envValue:      "900",
+			expectedValue: 900,
+			expectedSet:   true,
+		},
+		"zero disables but is present": {
+			set:           true,
+			envValue:      "0",
+			expectedValue: 0,
+			expectedSet:   true,
+		},
+		"negative is an error": {
+			set:           true,
+			envValue:      "-1",
+			expectedSet:   true,
+			expectedError: "BOOTSTRAP_CLONE_STALL_TIMEOUT should be a non-negative value",
+		},
+		"non-numeric is an error": {
+			set:           true,
+			envValue:      "abc",
+			expectedSet:   true,
+			expectedError: "failed to parse BOOTSTRAP_CLONE_STALL_TIMEOUT: strconv.Atoi: parsing \"abc\": invalid syntax",
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			require.NoError(t, os.Unsetenv("BOOTSTRAP_CLONE_STALL_TIMEOUT"))
+			if tt.set {
+				require.NoError(t, os.Setenv("BOOTSTRAP_CLONE_STALL_TIMEOUT", tt.envValue))
+				defer func() { require.NoError(t, os.Unsetenv("BOOTSTRAP_CLONE_STALL_TIMEOUT")) }()
+			}
+
+			value, present, err := GetCloneStallTimeout()
+
+			assert.Equal(t, tt.expectedSet, present)
+			if tt.expectedError != "" {
+				assert.EqualError(t, err, tt.expectedError)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedValue, value)
+			}
+		})
+	}
+}
+
 func TestGetSourceRetryCount(t *testing.T) {
 	tests := map[string]struct {
 		envValue       string

cmd/example-gen/pkg/defaults/manual.go

@@ -31,7 +31,7 @@ func ManualCluster(cr *apiv1.PerconaServerMySQL) {
 }
 
 func mysqlDefaults(spec *apiv1.MySQLSpec) {
-	podSpecDefaults(&spec.PodSpec, ImageMySQL, resources("2Gi", "", "4Gi", ""), configurationMySQL, 600, envList("BOOTSTRAP_READ_TIMEOUT", "600", "ASYNC_SOURCE_RETRY_COUNT", "3", "ASYNC_SOURCE_CONNECT_RETRY", "60"), envFromList("mysql-env-secret"))
+	podSpecDefaults(&spec.PodSpec, ImageMySQL, resources("2Gi", "", "4Gi", ""), configurationMySQL, 600, envList("BOOTSTRAP_READ_TIMEOUT", "600", "ASYNC_SOURCE_RETRY_COUNT", "3", "ASYNC_SOURCE_CONNECT_RETRY", "60", "BOOTSTRAP_CLONE_STALL_TIMEOUT", "900"), envFromList("mysql-env-secret"))
 
 	spec.AutoRecovery = true
 	spec.VolumeSpec = nil