ci: Improve Couchbase K8s resilience (#3527)

* ci: Improve Couchbase K8s resilience with health probes and password reset on restart

Move admin password reset outside the one-time guard file check so it
runs on every container start, preventing auth failures when the sandbox
image reinitializes after a restart. Add liveness and readiness probes
to auto-detect and recover from degraded Couchbase state.

* fix: Restrict envsubst to known deploy variables only

envsubst was replacing $COUCHBASE_ADMINISTRATOR_PASSWORD in the probe
commands with an empty string, since that variable is a K8s secret not
a CI environment variable. Restrict envsubst to only substitute the
three variables used in Service annotations: RESOURCE_GROUP,
PUBLIC_IP_NAME, and PUBLIC_IP.

Author: tippmar-nr

tests/Agent/IntegrationTests/UnboundedServices/couchbase/configure-server.sh

@@ -1,27 +1,30 @@
 #!/bin/sh
-set -e 
+set -e
 /entrypoint.sh couchbase-server &
 
-if [ ! -f /nr-container-configured ]; then
+# wait for the server to be up and running
+# when the file /opt/couchbase/var/lib/couchbase/container-configured exists, the server is ready
+while [ ! -f /opt/couchbase/var/lib/couchbase/container-configured ]; do
+  sleep 1
+done
 
-  # wait for the server to be up and running
-  # when the file /opt/couchbase/var/lib/couchbase/container-configured exists, the server is ready
-  while [ ! -f /opt/couchbase/var/lib/couchbase/container-configured ]; do
-    sleep 1
-  done
+echo "Waiting a bit longer for the server to be ready..."
+sleep 15s
 
-  echo "Waiting a bit longer for the server to be ready..."
-  sleep 15s
+# Always reset the administrator password on startup to ensure it matches
+# the expected value, even after container restarts where the sandbox image
+# may have re-initialized with its default password.
+echo "Resetting administrator password"
+/opt/couchbase/bin/couchbase-cli reset-admin-password --new-password ${COUCHBASE_ADMINISTRATOR_PASSWORD} || { echo "Error: Failed to reset administrator password"; exit 1; }
 
-  # use the couchbase cli to change the administrator password
-  echo "Changing administrator password"
-  /opt/couchbase/bin/couchbase-cli reset-admin-password --new-password ${COUCHBASE_ADMINISTRATOR_PASSWORD} || { echo "Error: Failed to reset administrator password"; exit 1; }
+# Only create the FTS index on first run (idempotent but avoids noisy errors on restart)
+if [ ! -f /nr-container-configured ]; then
 
   # Get UUID of travel-sample bucket
   uuid=$(curl -u Administrator:${COUCHBASE_ADMINISTRATOR_PASSWORD} http://127.0.0.1:8091/pools/default/buckets/travel-sample | jq '. | .uuid') || { echo "Error: Failed to retrieve UUID"; exit 1; }
 
   echo "Creating a full text search index on the hotel collection in the inventory scope"
-                      
+
   curl -XPUT -H "Content-Type: application/json" -u Administrator:${COUCHBASE_ADMINISTRATOR_PASSWORD} \
   http://localhost:8094/api/bucket/travel-sample/scope/inventory/index/index-hotel-description \
   -d '{

tests/Agent/IntegrationTests/UnboundedServices/deploy_to_aks.sh

@@ -33,7 +33,7 @@ for manifest in k8s/*.yaml; do
     fi
 
     echo "Applying $manifest..."
-    envsubst < $manifest | kubectl apply -f -
+    envsubst '${RESOURCE_GROUP} ${PUBLIC_IP_NAME} ${PUBLIC_IP}' < $manifest | kubectl apply -f -
 done
 
 echo "Deployment to AKS completed successfully!"

tests/Agent/IntegrationTests/UnboundedServices/k8s/couchbase.yaml

@@ -32,21 +32,26 @@ spec:
             secretKeyRef:
               name: unboundedservices-secrets
               key: COUCHBASE_ADMINISTRATOR_PASSWORD
-        # livenessProbe:
-        #   exec:
-        #     command:
-        #     - /bin/sh
-        #     - -c
-        #     - "couchbase-cli server-info -c localhost:8091 -u Administrator -p $COUCHBASE_ADMINISTRATOR_PASSWORD | grep -q '\"status\": \"healthy\"' || exit 1"
-        #   initialDelaySeconds: 120 # wait a while because couchbase takes time to start
-        #   periodSeconds: 60
-        #   timeoutSeconds: 5
-        #   failureThreshold: 3
-        # resources:
-        #   requests:
-        #     memory: "8Gi"
-        #   limits:
-        #     memory: "8Gi"
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - "couchbase-cli server-info -c localhost:8091 -u Administrator -p $COUCHBASE_ADMINISTRATOR_PASSWORD | grep -q '\"status\": \"healthy\"' || exit 1"
+          initialDelaySeconds: 90
+          periodSeconds: 15
+          timeoutSeconds: 10
+          failureThreshold: 3
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - "couchbase-cli server-info -c localhost:8091 -u Administrator -p $COUCHBASE_ADMINISTRATOR_PASSWORD | grep -q '\"status\": \"healthy\"' || exit 1"
+          initialDelaySeconds: 120
+          periodSeconds: 60
+          timeoutSeconds: 10
+          failureThreshold: 3
 ---
 apiVersion: v1
 kind: Service