ensure all metadata components are started properly

Metadata components need to be started in the order
repd -> bucketd -> cloudserver
as they are not able to recover on their own when a
component they depend on is not available at startup.

Issue: ZENKO-4414

Author: Kerkesni

.github/scripts/end2end/common.sh

@@ -9,3 +9,55 @@ get_token() {
         jq -cr '.id_token'
 }
 
+wait_for_endpoint() {
+    local host=$1
+    local port=$2
+    local timeout_s=$3
+
+    kubectl run wait-for-port \
+        --image=busybox \
+        --attach=True \
+        --rm \
+        --restart=Never  \
+        --pod-running-timeout=5m \
+        --image-pull-policy=IfNotPresent \
+        --env="HOST=${host}" \
+        --env="PORT=${port}" \
+        --env="TIMEOUT_S=${timeout_s}" \
+        -- sh -c '
+wait_for_endpoint() {
+    local count=0
+    echo "waiting for $HOST:$PORT to be available"
+    while ! nc -z -w 1 $HOST "$PORT"; do
+        count=$((count + 1))
+        [ "$count" -ge "$TIMEOUT_S" ] && echo "Error: timedout waiting for $HOST:$PORT after $TIMEOUT_S seconds" && return 1
+        sleep 1
+    done
+    echo "$HOST:$PORT is now available."
+}
+wait_for_endpoint
+'
+}
+
+wait_for_all_pods_behind_services() {
+    local service=$1
+    local namespace=$2
+    local port_regex=$3
+    local timeout_s=$4
+    kubectl get pods -n $namespace -l app=$service -o jsonpath='{range .items[*]}{.metadata.deletionTimestamp}:{.status.podIP}:{.spec.containers[*].ports[*].containerPort}{"\n"}{end}' | while read -r output; do
+        deletion_timestamp=$(echo $output | cut -d':' -f1)
+        ip=$(echo $output | cut -d':' -f2)
+        ports=$(echo $output | cut -d':' -f3)
+        # skip pods that are terminating
+        if [ -n "$deletion_timestamp" ] || [ -z "$ip" ] || [ -z "$ports" ]; then
+            continue
+        fi
+        # waiting for all ports that match the port prefix in cases where
+        # multiple containers are running within the same pod
+        for port in $ports; do
+            if [[ $port == $port_regex ]]; then
+                wait_for_endpoint $ip $port $timeout_s
+            fi
+        done
+    done
+}

.github/scripts/end2end/deploy-metadata.sh

@@ -2,6 +2,8 @@
 
 set -exu
 
+. "$(dirname $0)/common.sh"
+
 # create a separate namespace for metadata
 kubectl create namespace metadata
 
@@ -19,18 +21,27 @@ helm install -n metadata \
     --set metadata.sproxyd.persistentVolume.storageClass='' \
     s3c cloudserver/
 
-# wait for the repds to be ready
+# wait for the repds to be created
 kubectl -n metadata rollout status --watch --timeout=300s statefulset/s3c-metadata-repd
+# wait for all repd pods to start serving admin API ports
+wait_for_all_pods_behind_services metadata-repd metadata "91*" 60
 
 # current chart uses an old version of bucketd that has issues reconnecting to the repd
 # when bucketd is started first. Restarting bucketd after repd is ready.
 kubectl -n metadata rollout restart deployment/s3c-metadata-bucketd
+# wait for the bucketd pods to be created
 kubectl -n metadata rollout status --watch --timeout=300s deploy/s3c-metadata-bucketd
+# wait for all bucketd pods to start serving port 9000
+wait_for_all_pods_behind_services metadata-bucketd metadata 9000 60
 
-# manually add "s3c.local" to the rest endpoint list as it is not configurable in the chart
+# manually add "s3c.local" to the rest endpoints list as it's not configurable in the chart
 current_config=$(kubectl get configmap/s3c-cloudserver-config-json -n metadata -o jsonpath='{.data.config\.json}')
 updated_config=$(echo "$current_config" | jq '.restEndpoints["s3c.local"] = "us-east-1"')
 kubectl patch configmap/s3c-cloudserver-config-json -n metadata --type='merge' -p="$(jq -n --arg v "$updated_config" '{"data": {"config.json": $v}}')"
 
 # restarting cloudserver to take the new configmap changes into account
 kubectl -n metadata rollout restart deployment/s3c-cloudserver
+# wait for the cloudserver pods to be created
+kubectl -n metadata rollout status --watch --timeout=300s deployment/s3c-cloudserver
+# wait for the cloudserver pods to start serving port 8000
+wait_for_all_pods_behind_services cloudserver metadata 8000 60