diff --git a/e2e-tests/conf/client.yaml b/e2e-tests/conf/client.yaml index 251e3ef0cc..1765af1c5b 100644 --- a/e2e-tests/conf/client.yaml +++ b/e2e-tests/conf/client.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: pg-client - image: perconalab/percona-distribution-postgresql:16 + image: docker.io/perconalab/percona-distribution-postgresql:16 imagePullPolicy: Always command: - sleep diff --git a/e2e-tests/conf/cmctl.yml b/e2e-tests/conf/cmctl.yml index 53698fe133..43a14072e4 100644 --- a/e2e-tests/conf/cmctl.yml +++ b/e2e-tests/conf/cmctl.yml @@ -42,13 +42,14 @@ spec: serviceAccountName: cmctl containers: - name: cmctl - image: curlimages/curl + image: docker.io/alpine/curl:latest imagePullPolicy: Always command: - /bin/sh - -c - | - curl -fsSL -o /tmp/cmctl https://github.com/cert-manager/cmctl/releases/download/v2.4.1/cmctl_linux_amd64 \ + arch="$(uname -m)"; case "$arch" in aarch64|arm64) arch=arm64 ;; *) arch=amd64 ;; esac + curl -fsSL -o /tmp/cmctl "https://github.com/cert-manager/cmctl/releases/download/v2.4.1/cmctl_linux_${arch}" \ && chmod +x /tmp/cmctl \ && sleep 100500 restartPolicy: Always diff --git a/e2e-tests/functions b/e2e-tests/functions index 2f9c012775..84f41af01f 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -5,7 +5,6 @@ ROOT_REPO=${ROOT_REPO:-$(realpath ../../..)} CERT_MANAGER_VER="1.20.2" CERT_MANAGER_TEST_LABEL="e2e.percona.com/managed-by=percona-postgresql-operator-tests" CHAOS_MESH_VER="2.5.1" -BUSYBOX_VER="1.36" test_name=$(basename "$(pwd)") source "${ROOT_REPO}/e2e-tests/vars.sh" @@ -14,6 +13,150 @@ function init_temp_dir() { mkdir -p "$TEMP_DIR" } +arch_scheduling_enabled() { + [[ -n ${ARCH} ]] +} + +# S3 object is -pg-.tar.gz. +# ARM packages use a -arm64 version suffix so amd64 objects stay unchanged. +custom_ext_version() { + local version=$1 + if [[ ${ARCH} == arm64 ]]; then + echo "${version}-arm64" + else + echo "${version}" + fi +} + +if arch_scheduling_enabled; then + echo "ARCH=${ARCH}: adding architecture nodeSelector/affinity and toleration to e2e workloads" +fi + +add_arch_to_podspecs() { + if ! arch_scheduling_enabled; then + cat "$@" + return + fi + + ARCH="${ARCH}" yq eval ' + (select(.kind == "Pod").spec) |= ( + .nodeSelector."kubernetes.io/arch" = strenv(ARCH) | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "Deployment" or .kind == "StatefulSet" or .kind == "DaemonSet" or .kind == "ReplicaSet").spec.template.spec) |= ( + .nodeSelector."kubernetes.io/arch" = strenv(ARCH) | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "Job").spec.template.spec) |= ( + .nodeSelector."kubernetes.io/arch" = strenv(ARCH) | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "CronJob").spec.jobTemplate.spec.template.spec) |= ( + .nodeSelector."kubernetes.io/arch" = strenv(ARCH) | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) + ' "$@" +} + +add_arch_to_pg() { + if ! arch_scheduling_enabled; then + cat "$@" + return + fi + + ARCH="${ARCH}" yq eval ' + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("instances")) | .spec.instances[]?) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("logicalReplicas")).spec.logicalReplicas[]?) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("proxy")) | select(.spec.proxy | has("pgBouncer")) | .spec.proxy.pgBouncer) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("backups")) | select(.spec.backups | has("pgbackrest")) | select(.spec.backups.pgbackrest | has("repoHost")) | .spec.backups.pgbackrest.repoHost) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("backups")) | select(.spec.backups | has("pgbackrest")) | .spec.backups.pgbackrest).jobs |= ( + . = (. // {}) | + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("backups")) | select(.spec.backups | has("pgbackrest")) | select(.spec.backups.pgbackrest | has("restore")) | .spec.backups.pgbackrest.restore) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster") | select(.spec | has("backups")) | select(.spec.backups | has("volumeSnapshots")) | .spec.backups.volumeSnapshots).jobs |= ( + . = (. // {}) | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("dataSource")) | select(.spec.dataSource | has("postgresCluster")) | .spec.dataSource.postgresCluster) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("dataSource")) | select(.spec.dataSource | has("pgbackrest")) | .spec.dataSource.pgbackrest) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("dataSource")) | select(.spec.dataSource | has("volumes")) | select(.spec.dataSource.volumes | has("pgDataVolume")) | .spec.dataSource.volumes.pgDataVolume) |= ( + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("dataSource")) | select(.spec.dataSource | has("volumes")) | select(.spec.dataSource.volumes | has("pgWALVolume")) | .spec.dataSource.volumes.pgWALVolume) |= ( + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) | + (select(.kind == "PerconaPGCluster" or .kind == "PostgresCluster") | select(.spec | has("dataSource")) | select(.spec.dataSource | has("volumes")) | select(.spec.dataSource.volumes | has("pgBackRestVolume")) | .spec.dataSource.volumes.pgBackRestVolume) |= ( + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) + ' "$@" +} + +add_arch_to_pg_upgrade() { + if ! arch_scheduling_enabled; then + cat "$@" + return + fi + + ARCH="${ARCH}" yq eval ' + (select(.kind == "PerconaPGUpgrade").spec) |= ( + .affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "kubernetes.io/arch", "operator": "In", "values": [strenv(ARCH)]}]}] | + .tolerations = ((.tolerations // []) + [{"key": "kubernetes.io/arch", "operator": "Equal", "value": strenv(ARCH), "effect": "NoSchedule"}] | unique_by(.key + "|" + .operator + "|" + (.value // "") + "|" + .effect)) + ) + ' "$@" +} + +helm_arch_args() { + local arr_name=$1 + local prefix="${2:-}" + + if ! arch_scheduling_enabled; then + return + fi + + # Append into the caller array by name (avoid `local -n` — unsupported on macOS bash 3.2). + eval "$arr_name"'+=( + --set "'"${prefix}"'nodeSelector.kubernetes\.io/arch='"${ARCH}"'" + --set "'"${prefix}"'tolerations[0].key=kubernetes.io/arch" + --set "'"${prefix}"'tolerations[0].operator=Equal" + --set "'"${prefix}"'tolerations[0].value='"${ARCH}"'" + --set "'"${prefix}"'tolerations[0].effect=NoSchedule" + )' +} + +arch_run_overrides() { + printf '{"spec":{"nodeSelector":{"kubernetes.io/arch":"%s"},"tolerations":[{"key":"kubernetes.io/arch","operator":"Equal","value":"%s","effect":"NoSchedule"}]}}' "${ARCH}" "${ARCH}" +} + +merge_patch_pg() { + local name=$1 + local patch + patch=$(add_arch_to_pg - | yq -o=json '{"spec": .spec}') + kubectl -n "${NAMESPACE}" patch pg "${name}" --type=merge --patch "${patch}" +} + function create_namespace() { local namespace=$1 @@ -79,6 +222,7 @@ function deploy_operator() { | yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "'${disable_telemetry}'"' - \ | yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - \ | yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = "'${PGO_FEATURE_GATES}'"' - \ + | add_arch_to_podspecs - \ | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - } @@ -99,6 +243,11 @@ function update_operator() { kubectl -n "${OPERATOR_NS:-$NAMESPACE}" patch deployment percona-postgresql-operator -p \ '{"spec":{"template":{"spec":{"containers":[{"name":"operator","image":"'${IMAGE}'"}]}}}}' + + if arch_scheduling_enabled; then + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" patch deployment percona-postgresql-operator -p \ + '{"spec":{"template":{"spec":{"nodeSelector":{"kubernetes.io/arch":"'${ARCH}'"},"tolerations":[{"key":"kubernetes.io/arch","operator":"Equal","value":"'${ARCH}'","effect":"NoSchedule"}]}}}}' + fi } function deploy_operator_gh() { @@ -119,6 +268,7 @@ function deploy_operator_gh() { local docker_hub_org=$(echo "$IMAGE" | $sed -E 's|(.*/)?([^/]+)/[^/]+(:.*)?|\2|') yq eval '.spec.template.spec.containers[0].image = "'${REGISTRY_NAME_FULL}''$docker_hub_org'/percona-postgresql-operator:'${git_tag#v}'"' \ "${TEMP_DIR}/${cw_prefix}operator_${git_tag}.yaml" \ + | add_arch_to_podspecs - \ | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - } @@ -262,6 +412,9 @@ function deploy_minio() { aws_extra_args="--no-verify-ssl" fi + helm_arch_args additional_helm_args + helm_arch_args additional_helm_args "postJob." + helm uninstall -n "${NAMESPACE}" "${name}" || : helm repo remove minio || : helm repo add minio https://charts.min.io/ @@ -291,9 +444,52 @@ function deploy_minio() { wait_pod $MINIO_POD # create bucket - kubectl -n "${NAMESPACE}" run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \ - bash -c "AWS_ACCESS_KEY_ID='$access_key' AWS_SECRET_ACCESS_KEY='$secret_key' AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url ${endpoint_url} ${aws_extra_args} s3 mb s3://operator-testing" + aws_cli "$access_key" "$secret_key" us-east-1 \ + --endpoint-url "${endpoint_url}" ${aws_extra_args} s3 mb s3://operator-testing +} + +aws_cli() { + local access_key=$1 + local secret_key=$2 + local region=$3 + shift 3 + + local -a run_args=( + -n "${NAMESPACE}" run -i --rm "aws-cli-${RANDOM}" + --image="${IMAGE_AWS_CLI}" + --restart=Never + --env="AWS_ACCESS_KEY_ID=${access_key}" + --env="AWS_SECRET_ACCESS_KEY=${secret_key}" + --env="AWS_DEFAULT_REGION=${region}" + --env="AWS_EC2_METADATA_DISABLED=true" + ) + if arch_scheduling_enabled; then + run_args+=(--overrides="$(arch_run_overrides)") + fi + + if [[ ${1:-} == bash || ${1:-} == /bin/bash ]]; then + [[ $1 == bash ]] && shift && set -- /bin/bash "$@" + kubectl "${run_args[@]}" --command -- "$@" + else + kubectl "${run_args[@]}" -- "$@" + fi +} + +copy_custom_extensions_from_aws() { + set +o xtrace + + local minio_key minio_secret + minio_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" + minio_secret="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" + + retry 2 20 aws_cli "$(get_aws_access_key_id aws-s3-secret)" "$(get_aws_access_key aws-s3-secret)" us-east-1 \ + bash -c " + aws s3 cp s3://pg-extensions/ /tmp/pg-ext/ --recursive && + AWS_ACCESS_KEY_ID='${minio_key}' AWS_SECRET_ACCESS_KEY='${minio_secret}' AWS_DEFAULT_REGION=us-east-1 \ + aws --endpoint-url http://minio-service:9000 s3 cp /tmp/pg-ext/ s3://operator-testing/ --recursive + " + + set -o xtrace } function get_repo_auth() { @@ -368,11 +564,11 @@ function deploy_s3_secrets() { } function deploy_client() { - kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/client.yaml" + add_arch_to_podspecs "${TESTS_CONFIG_DIR}/client.yaml" | kubectl -n "${NAMESPACE}" apply -f - } function deploy_cmctl() { - envsubst <"${TESTS_CONFIG_DIR}/cmctl.yml" | kubectl -n "${NAMESPACE}" apply -f - + envsubst '${NAMESPACE}' < "${TESTS_CONFIG_DIR}/cmctl.yml" | add_arch_to_podspecs - | kubectl -n "${NAMESPACE}" apply -f - kubectl -n "${NAMESPACE}" wait deployment cmctl --for=condition=available --timeout=60s } @@ -427,6 +623,10 @@ function get_cr() { yq eval -i '.spec.openshift = true' "$TEMP_DIR/cr.yaml" fi + if arch_scheduling_enabled; then + add_arch_to_pg "$TEMP_DIR/cr.yaml" >"$TEMP_DIR/cr.yaml.arch" && mv "$TEMP_DIR/cr.yaml.arch" "$TEMP_DIR/cr.yaml" + fi + cat "$TEMP_DIR/cr.yaml" } @@ -562,41 +762,12 @@ function get_aws_access_key_id() { kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.AWS_ACCESS_KEY_ID | base64decode }}' } -function get_psql_user_host() { - local secret_name=${1} - - kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.host | base64decode }}' -} - function get_instance_set_pods() { local instance=${1:-instance1} kubectl get pods -n ${NAMESPACE} --selector postgres-operator.crunchydata.com/instance-set=${instance} -o custom-columns='NAME:.metadata.name' --no-headers } -function copy_custom_extensions_form_aws() { - set +o xtrace - - access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" - secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" - - kubectl -n "${NAMESPACE}" run -i --rm aws-cli \ - --image=perconalab/awscli \ - --restart=Never -- \ - bash -c " - AWS_ACCESS_KEY_ID=$(get_aws_access_key_id aws-s3-secret) \ - AWS_SECRET_ACCESS_KEY=$(get_aws_access_key aws-s3-secret) \ - AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url https://s3.amazonaws.com s3 cp s3://pg-extensions/ /tmp/ --recursive && - - AWS_ACCESS_KEY_ID='${access_key}' \ - AWS_SECRET_ACCESS_KEY='${secret_key}' \ - AWS_DEFAULT_REGION=us-east-1 \ - /usr/bin/aws --endpoint-url http://minio-service:9000 s3 cp /tmp/ s3://operator-testing/ --recursive - " - set -o xtrace -} - function get_psql_pod_host() { local pod=${1} @@ -889,11 +1060,15 @@ function deploy_pmm_server() { local additional_params="--set platform=openshift --set supresshttp2=false --set serviceAccount.create=false --set serviceAccount.name=pmm-server" fi + local -a pmm_arch_args + helm_arch_args pmm_arch_args + retry 10 60 helm install monitoring percona/pmm -n "${NAMESPACE}" \ --set fullnameOverride=monitoring \ --set image.tag=${IMAGE_PMM_SERVER#*:} \ --set image.repository=${IMAGE_PMM_SERVER%:*} \ --set service.type=LoadBalancer \ + "${pmm_arch_args[@]}" \ $additional_params \ --force } @@ -1103,13 +1278,17 @@ function deploy_chaos_mesh() { helm repo add chaos-mesh https://charts.chaos-mesh.org \ --force-update - helm install chaos-mesh chaos-mesh/chaos-mesh \ - --namespace="${NAMESPACE}" \ - --create-namespace \ - --version="${CHAOS_MESH_VER}" \ - --set chaosDaemon.runtime="${runtime}" \ - --set chaosDaemon.socketPath="${socket_path}" \ + local -a chaos_args=( + --namespace="${NAMESPACE}" + --create-namespace + --version="${CHAOS_MESH_VER}" + --set chaosDaemon.runtime="${runtime}" + --set chaosDaemon.socketPath="${socket_path}" --set dashboard.create=false + ) + helm_arch_args chaos_args "controllerManager." + helm_arch_args chaos_args "chaosDaemon." + helm install chaos-mesh chaos-mesh/chaos-mesh "${chaos_args[@]}" if [[ ${PLATFORM} == "openshift" ]]; then kubectl wait --for=create daemonset/chaos-daemon --namespace="${NAMESPACE}" --timeout=60s @@ -1558,12 +1737,18 @@ function deploy_cert_manager() { echo 'deploy cert manager' helm repo add jetstack https://charts.jetstack.io --force-update || : helm repo update || : + local -a cert_manager_arch_args + helm_arch_args cert_manager_arch_args + helm_arch_args cert_manager_arch_args "webhook." + helm_arch_args cert_manager_arch_args "cainjector." + helm_arch_args cert_manager_arch_args "startupapicheck." retry 2 20 helm upgrade --install cert-manager jetstack/cert-manager \ --namespace cert-manager \ --create-namespace \ --version "${CERT_MANAGER_VER}" \ --set installCRDs=true \ - --set "extraArgs={--enable-certificate-owner-ref=true}" + --set "extraArgs={--enable-certificate-owner-ref=true}" \ + "${cert_manager_arch_args[@]}" kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true || : kubectl label namespace cert-manager "${CERT_MANAGER_TEST_LABEL}" --overwrite || : @@ -1894,6 +2079,7 @@ function apply_hugepages_daemonset() { HUGEPAGE_COUNT="${hugepage_count}" NODE_SERVICE="${node_service}" \ yq "${expression}" "${manifest}" \ + | add_arch_to_podspecs - \ | kubectl apply -f - } @@ -2351,16 +2537,21 @@ function create_k8s_cluster() { local platform="$3" local region="$4" local ttl="${5:-"1"}" + local gke_machine_type="${K8S_UPGRADE_MACHINE_TYPE:-n1-standard-4}" + local doks_node_size="${K8S_UPGRADE_NODE_SIZE:-s-4vcpu-16gb-amd}" + if [[ ${ARCH} == arm64 ]]; then + gke_machine_type="${K8S_UPGRADE_MACHINE_TYPE:-t2a-standard-4}" + fi case "$platform" in doks) - echo "Creating DOKS cluster: $name ($version)" + echo "Creating DOKS cluster: $name ($version, size=$doks_node_size)" creation_time=$(date +%s) doctl kubernetes cluster create "$name" \ --region "$region" \ --version "$version" \ - --size "s-4vcpu-16gb-amd" \ + --size "$doks_node_size" \ --count "3" \ --tag "delete-cluster-after-hours:$ttl" \ --tag "creation-time:$creation_time" \ @@ -2377,13 +2568,13 @@ function create_k8s_cluster() { ;; gke) - echo "Creating GKE cluster: $name ($version)" + echo "Creating GKE cluster: $name ($version, machine-type=$gke_machine_type)" gcloud container clusters create "$name" \ --zone "$region" \ --cluster-ipv4-cidr=/21 \ --cluster-version "$version" \ --num-nodes "3" \ - --machine-type "n1-standard-4" \ + --machine-type "$gke_machine_type" \ --labels="delete-cluster-after-hours=$ttl,team=cloud,product=pg-operator" \ --no-enable-autoupgrade \ --monitoring=NONE \ @@ -2739,6 +2930,10 @@ function start_vault() { local values_file="${tmp_dir}/vault.values.yaml" local values_template="${TESTS_CONFIG_DIR}/vault-values-no-tls.yaml" + local -a vault_arch_args + helm_arch_args vault_arch_args "server." + helm_arch_args vault_arch_args "injector." + if [[ ${protocol} == "https" ]]; then vault_tls "${name}" "${tmp_dir}" values_template="${TESTS_CONFIG_DIR}/vault-values-tls.yaml" @@ -2753,7 +2948,8 @@ function start_vault() { --disable-openapi-validation \ --version $VAULT_VER \ --namespace "$vault_namespace" \ - --values "${values_file}" + --values "${values_file}" \ + "${vault_arch_args[@]}" if [[ ${PLATFORM} == "openshift" ]]; then oc patch clusterrole $name-agent-injector-clusterrole --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]' diff --git a/e2e-tests/release_versions b/e2e-tests/release_versions index cb0fc42a3a..b5a3ac5626 100644 --- a/e2e-tests/release_versions +++ b/e2e-tests/release_versions @@ -63,4 +63,4 @@ RANCHER=2.15.1 CERT_MANAGER=1.21.1 OPENSHIFT_MIN=4.18.40 OPENSHIFT_MAX=4.21.14 -MINIKUBE_MAX=1.35.1 +MINIKUBE_MAX=1.35.1 \ No newline at end of file diff --git a/e2e-tests/tests/backup-enable-disable/03-create-cluster-datasource.yaml b/e2e-tests/tests/backup-enable-disable/03-create-cluster-datasource.yaml index b218126e91..be89bf5ea4 100644 --- a/e2e-tests/tests/backup-enable-disable/03-create-cluster-datasource.yaml +++ b/e2e-tests/tests/backup-enable-disable/03-create-cluster-datasource.yaml @@ -17,4 +17,5 @@ commands: } } ' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml index 555dcf591f..df500ad4c1 100644 --- a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml @@ -10,6 +10,9 @@ commands: kubectl -n "$NAMESPACE" delete perconapgcluster cert-manager-tls --wait=true || true kubectl -n "$NAMESPACE" delete postgrescluster cert-manager-tls --wait=true || true + kubectl -n "$NAMESPACE" delete perconapgbackup --all --wait=true || true + kubectl -n "$NAMESPACE" delete job -l postgres-operator.crunchydata.com/cluster=cert-manager-tls --wait=true || true + kubectl -n "$NAMESPACE" delete pvc -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true kubectl -n "$NAMESPACE" delete secret -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true kubectl -n "$NAMESPACE" delete configmap internal-pki-cert-serial || true @@ -17,5 +20,10 @@ commands: cluster_deleted() { ! kubectl -n "$NAMESPACE" get postgrescluster cert-manager-tls 2>/dev/null } + backups_deleted() { + [[ -z "$(kubectl -n "$NAMESPACE" get perconapgbackup -o name 2>/dev/null)" ]] && \ + [[ -z "$(kubectl -n "$NAMESPACE" get job -l postgres-operator.crunchydata.com/cluster=cert-manager-tls -o name 2>/dev/null)" ]] + } retry 12 5 cluster_deleted + retry 12 5 backups_deleted timeout: 120 diff --git a/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml b/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml index 8c8f8b0b0e..c104770537 100644 --- a/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml +++ b/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml @@ -1,7 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: |- + - timeout: 200 + script: |- set -o errexit set -o xtrace @@ -12,5 +13,4 @@ commands: deploy_client deploy_s3_secrets deploy_minio - copy_custom_extensions_form_aws - timeout: 200 + copy_custom_extensions_from_aws diff --git a/e2e-tests/tests/custom-extensions/01-create-cluster.yaml b/e2e-tests/tests/custom-extensions/01-create-cluster.yaml index 084dd3e4ea..5d058e70f8 100644 --- a/e2e-tests/tests/custom-extensions/01-create-cluster.yaml +++ b/e2e-tests/tests/custom-extensions/01-create-cluster.yaml @@ -1,7 +1,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - timeout: 60 + - timeout: 120 script: |- set -o errexit set -o xtrace diff --git a/e2e-tests/tests/custom-extensions/02-install-pg_stat_kcache.yaml b/e2e-tests/tests/custom-extensions/02-install-pg_stat_kcache.yaml index 4422049b4f..4f24b6126c 100644 --- a/e2e-tests/tests/custom-extensions/02-install-pg_stat_kcache.yaml +++ b/e2e-tests/tests/custom-extensions/02-install-pg_stat_kcache.yaml @@ -1,14 +1,16 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -spec: - extensions: - custom: - - name: pg_stat_kcache - version: 2.3.1 - patroni: - dynamicConfiguration: - postgresql: - parameters: - shared_preload_libraries: pg_stat_statements,pg_stat_kcache +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kcache_ver=$(custom_ext_version 2.3.1) + kubectl -n "${NAMESPACE}" patch pg custom-extensions --type merge -p "{ + \"spec\": { + \"extensions\": {\"custom\": [{\"name\": \"pg_stat_kcache\", \"version\": \"${kcache_ver}\"}]}, + \"patroni\": {\"dynamicConfiguration\": {\"postgresql\": {\"parameters\": {\"shared_preload_libraries\": \"pg_stat_statements,pg_stat_kcache\"}}}} + } + }" diff --git a/e2e-tests/tests/custom-extensions/11-install-pg_stat_kcache.yaml b/e2e-tests/tests/custom-extensions/11-install-pg_stat_kcache.yaml index 4422049b4f..4f24b6126c 100644 --- a/e2e-tests/tests/custom-extensions/11-install-pg_stat_kcache.yaml +++ b/e2e-tests/tests/custom-extensions/11-install-pg_stat_kcache.yaml @@ -1,14 +1,16 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -spec: - extensions: - custom: - - name: pg_stat_kcache - version: 2.3.1 - patroni: - dynamicConfiguration: - postgresql: - parameters: - shared_preload_libraries: pg_stat_statements,pg_stat_kcache +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kcache_ver=$(custom_ext_version 2.3.1) + kubectl -n "${NAMESPACE}" patch pg custom-extensions --type merge -p "{ + \"spec\": { + \"extensions\": {\"custom\": [{\"name\": \"pg_stat_kcache\", \"version\": \"${kcache_ver}\"}]}, + \"patroni\": {\"dynamicConfiguration\": {\"postgresql\": {\"parameters\": {\"shared_preload_libraries\": \"pg_stat_statements,pg_stat_kcache\"}}}} + } + }" diff --git a/e2e-tests/tests/custom-extensions/14-update-pg_stat_kcache.yaml b/e2e-tests/tests/custom-extensions/14-update-pg_stat_kcache.yaml index 8f32f45f8c..66ea981b66 100644 --- a/e2e-tests/tests/custom-extensions/14-update-pg_stat_kcache.yaml +++ b/e2e-tests/tests/custom-extensions/14-update-pg_stat_kcache.yaml @@ -1,14 +1,16 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -spec: - extensions: - custom: - - name: pg_stat_kcache - version: 2.3.2 - patroni: - dynamicConfiguration: - postgresql: - parameters: - shared_preload_libraries: pg_stat_statements,pg_stat_kcache +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kcache_ver=$(custom_ext_version 2.3.2) + kubectl -n "${NAMESPACE}" patch pg custom-extensions --type merge -p "{ + \"spec\": { + \"extensions\": {\"custom\": [{\"name\": \"pg_stat_kcache\", \"version\": \"${kcache_ver}\"}]}, + \"patroni\": {\"dynamicConfiguration\": {\"postgresql\": {\"parameters\": {\"shared_preload_libraries\": \"pg_stat_statements,pg_stat_kcache\"}}}} + } + }" diff --git a/e2e-tests/tests/custom-extensions/17-assert.yaml b/e2e-tests/tests/custom-extensions/17-assert.yaml index 7b4e037224..84dbf31b3f 100644 --- a/e2e-tests/tests/custom-extensions/17-assert.yaml +++ b/e2e-tests/tests/custom-extensions/17-assert.yaml @@ -3,23 +3,20 @@ kind: TestAssert # adding pg_cron to the custom list and to shared_preload_libraries restarts # every instance pod timeout: 420 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -status: - installedCustomExtensions: - - pg_stat_kcache - - pg_cron - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping custom pg_cron assertion on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + + wait_cluster_consistency custom-extensions + + installed=$(kubectl -n "${NAMESPACE}" get pg custom-extensions -o jsonpath='{.status.installedCustomExtensions}') + echo "${installed}" | grep -q pg_cron \ + || { echo "ERROR: pg_cron was not recorded as an installed custom extension: ${installed}"; exit 1; } diff --git a/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml b/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml index debc2efd72..1485ad04cc 100644 --- a/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml +++ b/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml @@ -1,20 +1,31 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -spec: - # pg_cron is installed as a custom extension here so the next steps can move - # it to spec.extensions.pg_cron: it is the only extension that is both - # available as a custom archive and managed as a builtin one, and it owns - # data (the cron.job rows) that a wrong drop would destroy - extensions: - custom: - - name: pg_stat_kcache - version: 2.3.2 - - name: pg_cron - version: 1.6.7 - patroni: - dynamicConfiguration: - postgresql: - parameters: - shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping custom pg_cron on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + + # pg_cron is installed as a custom extension here so the next steps can move + # it to spec.extensions.pg_cron: it is the only extension that is both + # available as a custom archive and managed as a builtin one, and it owns + # data (the cron.job rows) that a wrong drop would destroy + kcache_ver=$(custom_ext_version 2.3.2) + cron_ver=$(custom_ext_version 1.6.7) + kubectl -n "${NAMESPACE}" patch pg custom-extensions --type merge -p "{ + \"spec\": { + \"extensions\": {\"custom\": [ + {\"name\": \"pg_stat_kcache\", \"version\": \"${kcache_ver}\"}, + {\"name\": \"pg_cron\", \"version\": \"${cron_ver}\"} + ]}, + \"patroni\": {\"dynamicConfiguration\": {\"postgresql\": {\"parameters\": {\"shared_preload_libraries\": \"pg_stat_statements,pg_stat_kcache,pg_cron\"}}}} + } + }" + + wait_cluster_consistency custom-extensions diff --git a/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml b/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml index c71fff4342..b982d5c139 100644 --- a/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml +++ b/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml @@ -7,6 +7,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping pg_cron restart on arm64: custom pg_cron was not installed" + exit 0 + fi + # pg_cron was added to shared_preload_libraries in the previous step and # postgres only loads it on start, so the pods have to be restarted before # the extension can be created diff --git a/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml b/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml index 26cee3cc53..9cb023c0b2 100644 --- a/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml +++ b/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping pg_cron enable on arm64: custom pg_cron was not installed" + exit 0 + fi + uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)" # cron.database_name only exists once the pg_cron library is loaded, so it diff --git a/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml b/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml index 68ad48930e..721ef4149c 100644 --- a/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml +++ b/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml @@ -1,20 +1,28 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: custom-extensions -spec: - # pg_cron moves from a custom extension to a builtin one: the operator takes - # over its lifecycle, but the extension and its data must survive the move. - # Dropping it here used to be the default behavior, because the extension - # disappears from the custom list. - extensions: - pg_cron: - enabled: true - custom: - - name: pg_stat_kcache - version: 2.3.2 - patroni: - dynamicConfiguration: - postgresql: - parameters: - shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping custom-to-builtin pg_cron migration on arm64: custom pg_cron was not installed" + exit 0 + fi + + # pg_cron moves from a custom extension to a builtin one: the operator takes + # over its lifecycle, but the extension and its data must survive the move. + # Dropping it here used to be the default behavior, because the extension + # disappears from the custom list. + kcache_ver=$(custom_ext_version 2.3.2) + kubectl -n "${NAMESPACE}" patch pg custom-extensions --type merge -p "{ + \"spec\": { + \"extensions\": { + \"pg_cron\": {\"enabled\": true}, + \"custom\": [{\"name\": \"pg_stat_kcache\", \"version\": \"${kcache_ver}\"}] + }, + \"patroni\": {\"dynamicConfiguration\": {\"postgresql\": {\"parameters\": {\"shared_preload_libraries\": \"pg_stat_statements,pg_stat_kcache,pg_cron\"}}}} + } + }" diff --git a/e2e-tests/tests/custom-extensions/21-check-migration.yaml b/e2e-tests/tests/custom-extensions/21-check-migration.yaml index b2e751301a..3d7cd998ad 100644 --- a/e2e-tests/tests/custom-extensions/21-check-migration.yaml +++ b/e2e-tests/tests/custom-extensions/21-check-migration.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping custom-to-builtin pg_cron check on arm64: custom pg_cron was not installed" + exit 0 + fi + uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)" # moving an extension from spec.extensions.custom to a builtin flag must diff --git a/e2e-tests/tests/demand-backup-offline-snapshot/01-create-cluster.yaml b/e2e-tests/tests/demand-backup-offline-snapshot/01-create-cluster.yaml index ebd9f57526..4eb10863e6 100644 --- a/e2e-tests/tests/demand-backup-offline-snapshot/01-create-cluster.yaml +++ b/e2e-tests/tests/demand-backup-offline-snapshot/01-create-cluster.yaml @@ -24,4 +24,5 @@ commands: ) // . ' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/demand-backup-offline-snapshot/11-create-cluster-from-snapshot.yaml b/e2e-tests/tests/demand-backup-offline-snapshot/11-create-cluster-from-snapshot.yaml index 23af72d792..5c26da4a40 100644 --- a/e2e-tests/tests/demand-backup-offline-snapshot/11-create-cluster-from-snapshot.yaml +++ b/e2e-tests/tests/demand-backup-offline-snapshot/11-create-cluster-from-snapshot.yaml @@ -24,4 +24,5 @@ commands: ) // . ' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/ldap-tls/02-openldap-tls.yaml b/e2e-tests/tests/ldap-tls/02-openldap-tls.yaml index 787d6d62c3..98ff19daad 100644 --- a/e2e-tests/tests/ldap-tls/02-openldap-tls.yaml +++ b/e2e-tests/tests/ldap-tls/02-openldap-tls.yaml @@ -1,4 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -apply: - - files/openldap-tls-deploy.yaml \ No newline at end of file +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + add_arch_to_podspecs files/openldap-tls-deploy.yaml | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/ldap-tls/files/openldap-tls-deploy.yaml b/e2e-tests/tests/ldap-tls/files/openldap-tls-deploy.yaml index 8468ff5387..a355a5c906 100644 --- a/e2e-tests/tests/ldap-tls/files/openldap-tls-deploy.yaml +++ b/e2e-tests/tests/ldap-tls/files/openldap-tls-deploy.yaml @@ -18,7 +18,7 @@ spec: serviceAccountName: openldap-tls containers: - name: openldap - image: osixia/openldap:latest + image: docker.io/osixia/openldap:latest imagePullPolicy: Always args: ["--copy-service"] env: @@ -82,7 +82,7 @@ spec: timeoutSeconds: 5 failureThreshold: 3 - name: ldap-setup - image: osixia/openldap:latest + image: docker.io/osixia/openldap:latest imagePullPolicy: Always command: ["/bin/bash"] args: diff --git a/e2e-tests/tests/ldap/01-openldap.yaml b/e2e-tests/tests/ldap/01-openldap.yaml index ce6c527629..cefae7d60a 100644 --- a/e2e-tests/tests/ldap/01-openldap.yaml +++ b/e2e-tests/tests/ldap/01-openldap.yaml @@ -14,5 +14,5 @@ commands: oc adm policy add-scc-to-user anyuid -z openldap -n "${NAMESPACE}" sleep 5 fi -apply: - - files/openldap.yaml + + add_arch_to_podspecs files/openldap.yaml | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/ldap/files/openldap.yaml b/e2e-tests/tests/ldap/files/openldap.yaml index 70925e7764..f455c2892b 100644 --- a/e2e-tests/tests/ldap/files/openldap.yaml +++ b/e2e-tests/tests/ldap/files/openldap.yaml @@ -26,7 +26,7 @@ spec: serviceAccountName: openldap containers: - name: openldap - image: osixia/openldap:latest + image: docker.io/osixia/openldap:latest imagePullPolicy: Always args: ["--copy-service"] env: @@ -70,7 +70,7 @@ spec: timeoutSeconds: 5 failureThreshold: 3 - name: ldap-setup - image: osixia/openldap:latest + image: docker.io/osixia/openldap:latest imagePullPolicy: Always command: ["/bin/bash"] args: diff --git a/e2e-tests/tests/logcollection/06-check-s3-logs.yaml b/e2e-tests/tests/logcollection/06-check-s3-logs.yaml index 2cc920ec70..d1a0403d11 100644 --- a/e2e-tests/tests/logcollection/06-check-s3-logs.yaml +++ b/e2e-tests/tests/logcollection/06-check-s3-logs.yaml @@ -21,14 +21,10 @@ commands: dump_s3_logs() { ( set +o xtrace - kubectl -n "${NAMESPACE}" run -i --rm "aws-cli-${RANDOM}" \ - --image=perconalab/awscli --restart=Never \ - --env="AWS_ACCESS_KEY_ID=${access_key}" \ - --env="AWS_SECRET_ACCESS_KEY=${secret_key}" \ - --env="AWS_DEFAULT_REGION=us-east-1" \ - --command -- bash -c ' + aws_cli "${access_key}" "${secret_key}" us-east-1 \ + bash -c ' set -e - aws_cmd="/usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl" + aws_cmd="aws --endpoint-url https://minio-service:9000 --no-verify-ssl" key=$($aws_cmd s3 ls s3://operator-testing/ --recursive 2>/dev/null \ | grep postgres | head -1 | awk "{print \$NF}") [ -n "$key" ] || exit 1 diff --git a/e2e-tests/tests/logical-replica/04-add-logical-replica.yaml b/e2e-tests/tests/logical-replica/04-add-logical-replica.yaml index e86f68465e..a998416e8e 100644 --- a/e2e-tests/tests/logical-replica/04-add-logical-replica.yaml +++ b/e2e-tests/tests/logical-replica/04-add-logical-replica.yaml @@ -15,27 +15,25 @@ commands: # Both seeding methods against the same primary and the same database. # The slot, publication and subscription names all carry the replica name, # so the two never collide. - kubectl -n "${NAMESPACE}" patch pg logical-replica --type=merge --patch '{ - "spec": { - "logicalReplicas": [ - { - "name": "analytics", - "databases": ["logical-replica"], - "bootstrapMethod": "pgbackrest", - "dataVolumeClaimSpec": { - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "1Gi"}} - } - }, - { - "name": "reporting", - "databases": ["logical-replica"], - "bootstrapMethod": "pg_basebackup", - "dataVolumeClaimSpec": { - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "1Gi"}} - } - } - ] - } - }' + merge_patch_pg logical-replica <|${CRUNCHY_REPO_PATH}|g" | \ $sed "s||${MINIO_ENDPOINT}|g" | \ + add_arch_to_pg - | \ kubectl -n "${CRUNCHY_NS}" apply -f - timeout: 20 diff --git a/e2e-tests/tests/migration-from-crunchy-backup-restore/03-trigger-backup.yaml b/e2e-tests/tests/migration-from-crunchy-backup-restore/03-trigger-backup.yaml index 7cb2034a97..af8e80396a 100644 --- a/e2e-tests/tests/migration-from-crunchy-backup-restore/03-trigger-backup.yaml +++ b/e2e-tests/tests/migration-from-crunchy-backup-restore/03-trigger-backup.yaml @@ -8,6 +8,6 @@ commands: CRUNCHY_NS="${NAMESPACE}-crunchy" # Trigger a full backup so the data written in the previous step is captured - kubectl -n "${CRUNCHY_NS}" annotate postgrescluster crunchy-source \ + kubectl -n "${CRUNCHY_NS}" annotate postgrescluster.postgres-operator.crunchydata.com crunchy-source \ postgres-operator.crunchydata.com/pgbackrest-backup="$(date +%s)" timeout: 30 diff --git a/e2e-tests/tests/migration-from-crunchy-pv/00-deploy-operators.yaml b/e2e-tests/tests/migration-from-crunchy-pv/00-deploy-operators.yaml index 1289a68822..55d25bfa2d 100644 --- a/e2e-tests/tests/migration-from-crunchy-pv/00-deploy-operators.yaml +++ b/e2e-tests/tests/migration-from-crunchy-pv/00-deploy-operators.yaml @@ -11,11 +11,14 @@ commands: # The Crunchy source cluster and the Percona target cluster never coexist: # Crunchy is deleted in step 03, Percona is created in step 04. # A single namespace is therefore sufficient for this test. + pgo_helm_args=() + helm_arch_args pgo_helm_args helm install pgo \ oci://registry.developers.crunchydata.com/crunchydata/pgo \ --namespace "${NAMESPACE}" \ --version "${CPGO_VERSION}" \ --set singleNamespace=true \ + "${pgo_helm_args[@]}" \ --wait # Deploy MinIO using the shared helper so this test follows the same diff --git a/e2e-tests/tests/migration-from-crunchy-pv/01-assert.yaml b/e2e-tests/tests/migration-from-crunchy-pv/01-assert.yaml index 5c1ed85717..0b5b60b01b 100644 --- a/e2e-tests/tests/migration-from-crunchy-pv/01-assert.yaml +++ b/e2e-tests/tests/migration-from-crunchy-pv/01-assert.yaml @@ -5,7 +5,7 @@ commands: - script: |- set -o errexit - kubectl wait postgrescluster/crunchy-source \ + kubectl wait postgrescluster.postgres-operator.crunchydata.com/crunchy-source \ -n "${NAMESPACE}" \ --for=condition=PGBackRestReplicaRepoReady \ --timeout=240s diff --git a/e2e-tests/tests/migration-from-crunchy-pv/01-create-crunchy-cluster.yaml b/e2e-tests/tests/migration-from-crunchy-pv/01-create-crunchy-cluster.yaml index 5ae6ce1128..04a11c288b 100644 --- a/e2e-tests/tests/migration-from-crunchy-pv/01-create-crunchy-cluster.yaml +++ b/e2e-tests/tests/migration-from-crunchy-pv/01-create-crunchy-cluster.yaml @@ -22,5 +22,6 @@ commands: yq eval ".spec.postgresVersion = ${PG_VER}" | \ $sed "s||${CRUNCHY_REPO_PATH}|g" | \ $sed "s||${MINIO_ENDPOINT}|g" | \ + add_arch_to_pg - | \ kubectl -n "${NAMESPACE}" apply -f - timeout: 20 diff --git a/e2e-tests/tests/migration-from-crunchy-pv/03-migrate-pv.yaml b/e2e-tests/tests/migration-from-crunchy-pv/03-migrate-pv.yaml index 2d00ea1b20..a4ec78a169 100644 --- a/e2e-tests/tests/migration-from-crunchy-pv/03-migrate-pv.yaml +++ b/e2e-tests/tests/migration-from-crunchy-pv/03-migrate-pv.yaml @@ -52,7 +52,7 @@ commands: kubectl patch postgrescluster crunchy-source -n "${NAMESPACE}" \ --type=json -p='[{"op":"replace","path":"/metadata/finalizers","value":[]}]' 2>/dev/null || true - kubectl delete postgrescluster crunchy-source -n "${NAMESPACE}" --ignore-not-found + kubectl delete postgrescluster.postgres-operator.crunchydata.com crunchy-source -n "${NAMESPACE}" --ignore-not-found # Uninstall pgo now — it has nothing left to reconcile and would # otherwise keep running alongside the Percona operator unnecessarily. diff --git a/e2e-tests/tests/migration-from-crunchy-standby/00-deploy-operators.yaml b/e2e-tests/tests/migration-from-crunchy-standby/00-deploy-operators.yaml index ab06269c20..aec178f831 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/00-deploy-operators.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/00-deploy-operators.yaml @@ -14,11 +14,14 @@ commands: kubectl create namespace "${CRUNCHY_NS}" || true # singleNamespace=true scopes pgo to its own namespace only + pgo_helm_args=() + helm_arch_args pgo_helm_args helm install pgo \ oci://registry.developers.crunchydata.com/crunchydata/pgo \ --namespace "${CRUNCHY_NS}" \ --version "${CPGO_VERSION}" \ --set singleNamespace=true \ + "${pgo_helm_args[@]}" \ --wait # Deploy minio WITH TLS — pgBackRest requires HTTPS; repo1-s3-verify-tls=n skips validation. diff --git a/e2e-tests/tests/migration-from-crunchy-standby/01-assert.yaml b/e2e-tests/tests/migration-from-crunchy-standby/01-assert.yaml index 96c4c51a8d..8901a7216b 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/01-assert.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/01-assert.yaml @@ -24,7 +24,7 @@ commands: # Wait for pgbackrest stanza to be created. # The backup annotation (triggering the full backup) is applied in the # next step only after stanza creation, so we don't wait for a job here. - kubectl wait postgrescluster/crunchy-source \ + kubectl wait postgrescluster.postgres-operator.crunchydata.com/crunchy-source \ -n "${CRUNCHY_NS}" \ --for=jsonpath='{.status.pgbackrest.repos[0].stanzaCreated}'=true \ --timeout=300s diff --git a/e2e-tests/tests/migration-from-crunchy-standby/01-create-crunchy-cluster.yaml b/e2e-tests/tests/migration-from-crunchy-standby/01-create-crunchy-cluster.yaml index 819f2a343d..34916b139f 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/01-create-crunchy-cluster.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/01-create-crunchy-cluster.yaml @@ -21,4 +21,5 @@ commands: yq eval ".spec.postgresVersion = ${PG_VER}" | \ $sed "s||${REPO_PATH}|g" | \ $sed "s||${MINIO_ENDPOINT}|g" | \ + add_arch_to_pg - | \ kubectl -n "${CRUNCHY_NS}" apply -f - diff --git a/e2e-tests/tests/migration-from-crunchy-standby/03-trigger-backup.yaml b/e2e-tests/tests/migration-from-crunchy-standby/03-trigger-backup.yaml index 34fe7245d2..1a2b45df9d 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/03-trigger-backup.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/03-trigger-backup.yaml @@ -12,5 +12,5 @@ commands: # Doing the backup AFTER writing data means the Percona standby will # restore close to the current state and replay only a small amount of WAL, # which is the realistic production migration pattern. - kubectl -n "${CRUNCHY_NS}" annotate postgrescluster crunchy-source \ + kubectl -n "${CRUNCHY_NS}" annotate postgrescluster.postgres-operator.crunchydata.com crunchy-source \ postgres-operator.crunchydata.com/pgbackrest-backup="$(date +%s)" diff --git a/e2e-tests/tests/migration-from-crunchy-standby/08-cutover.yaml b/e2e-tests/tests/migration-from-crunchy-standby/08-cutover.yaml index 4cd95b8d9c..bd9830153b 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/08-cutover.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/08-cutover.yaml @@ -46,7 +46,7 @@ commands: # Convert Crunchy to standby mode: Patroni demotes the leader, archives # the final WAL segments to S3, then enters recovery. # ----------------------------------------------------------------------- - kubectl patch postgrescluster crunchy-source \ + kubectl patch postgrescluster.postgres-operator.crunchydata.com crunchy-source \ -n "${CRUNCHY_NS}" \ --type=merge \ -p '{"spec": {"standby": {"enabled": true, "repoName": "repo1"}}}' diff --git a/e2e-tests/tests/migration-from-crunchy-standby/09-shutdown-source.yaml b/e2e-tests/tests/migration-from-crunchy-standby/09-shutdown-source.yaml index 8abe92b206..d680709977 100644 --- a/e2e-tests/tests/migration-from-crunchy-standby/09-shutdown-source.yaml +++ b/e2e-tests/tests/migration-from-crunchy-standby/09-shutdown-source.yaml @@ -12,7 +12,7 @@ commands: # Shut down the Crunchy source cluster (optional step — frees resources # and ensures no split-brain once Percona is promoted). - kubectl patch postgrescluster crunchy-source \ + kubectl patch postgrescluster.postgres-operator.crunchydata.com crunchy-source \ -n "${CRUNCHY_NS}" \ --type=merge \ -p '{"spec": {"shutdown": true}}' diff --git a/e2e-tests/tests/monitoring/00-assert.yaml b/e2e-tests/tests/monitoring/00-assert.yaml index ae5a062d84..6aa291a661 100644 --- a/e2e-tests/tests/monitoring/00-assert.yaml +++ b/e2e-tests/tests/monitoring/00-assert.yaml @@ -1,24 +1,27 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 120 ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: perconapgclusters.pgv2.percona.com -spec: - group: pgv2.percona.com - names: - kind: PerconaPGCluster - listKind: PerconaPGClusterList - plural: perconapgclusters - singular: perconapgcluster - scope: Namespaced ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert metadata: name: check-operator-deploy-status timeout: 120 commands: - - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 + - script: | + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + until kubectl assert exist-enhanced crd perconapgclusters.pgv2.percona.com; do + echo "Waiting for PerconaPGCluster CRD" + sleep 5 + done + + until kubectl assert exist-enhanced deployment percona-postgresql-operator \ + -n "${OPERATOR_NS:-$NAMESPACE}" --field-selector status.readyReplicas=1; do + echo "Waiting for percona-postgresql-operator deployment" + sleep 5 + done diff --git a/e2e-tests/tests/monitoring/00-deploy-operator.yaml b/e2e-tests/tests/monitoring/00-deploy-operator.yaml index 1aaca58be2..6c380c0d16 100644 --- a/e2e-tests/tests/monitoring/00-deploy-operator.yaml +++ b/e2e-tests/tests/monitoring/00-deploy-operator.yaml @@ -7,6 +7,12 @@ commands: set -o xtrace source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + init_temp_dir # do this only in the first TestStep deploy_operator diff --git a/e2e-tests/tests/monitoring/01-assert.yaml b/e2e-tests/tests/monitoring/01-assert.yaml index c6d14b6c9f..242ce9f73f 100644 --- a/e2e-tests/tests/monitoring/01-assert.yaml +++ b/e2e-tests/tests/monitoring/01-assert.yaml @@ -3,6 +3,17 @@ kind: TestAssert timeout: 520 commands: - script: | + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + set +o errexit set +o xtrace load_balancer="" @@ -11,20 +22,11 @@ commands: | jq -r '.status.loadBalancer.ingress[0] | .ip // .hostname // empty') [ -n "${load_balancer}" ] || sleep 5 done ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: monitoring -status: - collisionCount: 0 - currentReplicas: 1 - observedGeneration: 1 - readyReplicas: 1 - replicas: 1 - updatedReplicas: 1 ---- -kind: Service -apiVersion: v1 -metadata: - name: monitoring-service + + set -o errexit + + until kubectl assert exist-enhanced statefulset monitoring -n "${NAMESPACE}" \ + --field-selector status.readyReplicas=1; do + echo "Waiting for PMM server statefulset" + sleep 5 + done diff --git a/e2e-tests/tests/monitoring/01-deploy-pmm-server.yaml b/e2e-tests/tests/monitoring/01-deploy-pmm-server.yaml index 5e54943d7e..24272f389f 100644 --- a/e2e-tests/tests/monitoring/01-deploy-pmm-server.yaml +++ b/e2e-tests/tests/monitoring/01-deploy-pmm-server.yaml @@ -7,5 +7,10 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + deploy_pmm_server timeout: 300 diff --git a/e2e-tests/tests/monitoring/02-assert.yaml b/e2e-tests/tests/monitoring/02-assert.yaml index 0044a0a64b..d82adb8f5c 100644 --- a/e2e-tests/tests/monitoring/02-assert.yaml +++ b/e2e-tests/tests/monitoring/02-assert.yaml @@ -1,8 +1,15 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - name: monitoring-pmm-secret +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + kubectl assert exist-enhanced secret monitoring-pmm-secret -n "${NAMESPACE}" diff --git a/e2e-tests/tests/monitoring/02-create-pmm-secret.yaml b/e2e-tests/tests/monitoring/02-create-pmm-secret.yaml index 72dcc4264f..c3b84702de 100644 --- a/e2e-tests/tests/monitoring/02-create-pmm-secret.yaml +++ b/e2e-tests/tests/monitoring/02-create-pmm-secret.yaml @@ -6,4 +6,10 @@ commands: set -o xtrace source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + kubectl create -n "${NAMESPACE}" secret generic monitoring-pmm-secret --from-literal=PMM_SERVER_TOKEN="" || true diff --git a/e2e-tests/tests/monitoring/03-assert.yaml b/e2e-tests/tests/monitoring/03-assert.yaml index 87a3a2bdcf..15f55ae0f1 100644 --- a/e2e-tests/tests/monitoring/03-assert.yaml +++ b/e2e-tests/tests/monitoring/03-assert.yaml @@ -1,101 +1,28 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 420 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - labels: - e2e: monitoring - name: monitoring -spec: - backups: - pgbackrest: - manual: - options: - - --type=full - repoName: repo1 - repos: - - name: repo1 - schedules: - full: 0 0 * * 6 - volume: - volumeClaimSpec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - instances: - - dataVolumeClaimSpec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: instance1 - replicas: 3 - pmm: - enabled: true - secret: monitoring-pmm-secret - serverHost: monitoring-service - customClusterName: monitoring-pmm-custom-name - postgresParams: "--environment=dev-postgres" - port: 5432 - proxy: - pgBouncer: - port: 5432 - replicas: 3 -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready ---- -kind: Job -apiVersion: batch/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: monitoring - postgres-operator.crunchydata.com/pgbackrest: '' - postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true -status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - annotations: - pgv2.percona.com/pgbackrest-backup-job-type: replica-create - generation: 1 -spec: - pgCluster: monitoring - repoName: repo1 -status: - backupType: full - repo: - name: repo1 - schedules: - full: 0 0 * * 6 - volume: - volumeClaimSpec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - state: Succeeded - storageType: filesystem +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + wait_cluster_consistency monitoring 42 + + pmm=$(kubectl -n "${NAMESPACE}" get pg monitoring \ + -o jsonpath='{.spec.pmm.enabled},{.spec.pmm.secret},{.spec.pmm.serverHost},{.spec.pmm.customClusterName},{.spec.pmm.postgresParams}') + if [[ ${pmm} != "true,monitoring-pmm-secret,monitoring-service,monitoring-pmm-custom-name,--environment=dev-postgres" ]]; then + echo "Unexpected PMM spec: ${pmm}" + exit 1 + fi + + retry 30 10 bash -c ' + kubectl -n "'"${NAMESPACE}"'" get pg-backup \ + -o jsonpath="{.items[*].status.state}" | grep -qw Succeeded + ' diff --git a/e2e-tests/tests/monitoring/03-create-cluster.yaml b/e2e-tests/tests/monitoring/03-create-cluster.yaml index d3ccb03b57..04f5c1ad4f 100644 --- a/e2e-tests/tests/monitoring/03-create-cluster.yaml +++ b/e2e-tests/tests/monitoring/03-create-cluster.yaml @@ -1,13 +1,18 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - - script: |- + - timeout: 120 + script: |- set -o errexit set -o xtrace source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + get_cr \ | yq eval '.spec.pmm.enabled=true' - \ | yq eval ".spec.pmm.image=\"${IMAGE_PMM_CLIENT}\"" - \ diff --git a/e2e-tests/tests/monitoring/04-assert.yaml b/e2e-tests/tests/monitoring/04-assert.yaml index 0328431558..42340fe970 100644 --- a/e2e-tests/tests/monitoring/04-assert.yaml +++ b/e2e-tests/tests/monitoring/04-assert.yaml @@ -1,71 +1,28 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 420 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - labels: - e2e: monitoring - name: monitoring -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready ---- -apiVersion: upstream.pgv2.percona.com/v1beta1 -kind: PostgresCluster -metadata: - name: monitoring - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster - name: monitoring -status: - instances: - - name: instance1 - readyReplicas: 3 - replicas: 3 - updatedReplicas: 3 ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: monitoring - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 - ownerReferences: - - apiVersion: upstream.pgv2.percona.com/v1beta1 - kind: PostgresCluster - name: monitoring - controller: true - blockOwnerDeletion: true -status: - observedGeneration: 2 - replicas: 1 - readyReplicas: 1 - updatedReplicas: 1 - collisionCount: 0 ---- -kind: Pod -apiVersion: v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: monitoring - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 - postgres-operator.crunchydata.com/patroni: monitoring-ha - ownerReferences: - - apiVersion: apps/v1 - kind: StatefulSet -status: - phase: Running +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + wait_cluster_consistency monitoring 42 + + kubectl assert exist-enhanced perconapgcluster monitoring -n "${NAMESPACE}" \ + --field-selector status.state=ready + + sts=$(kubectl -n "${NAMESPACE}" get sts \ + --selector=postgres-operator.crunchydata.com/instance-set=instance1 \ + -o jsonpath='{.items[*].metadata.name}') + for st in $sts; do + kubectl assert exist-enhanced statefulset "${st}" -n "${NAMESPACE}" \ + --field-selector status.observedGeneration=2 \ + --field-selector status.readyReplicas=1 + done diff --git a/e2e-tests/tests/monitoring/04-update-pmm-server-token.yaml b/e2e-tests/tests/monitoring/04-update-pmm-server-token.yaml index 0b9a9c6e88..f49f42e527 100644 --- a/e2e-tests/tests/monitoring/04-update-pmm-server-token.yaml +++ b/e2e-tests/tests/monitoring/04-update-pmm-server-token.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + sts=$(kubectl -n "${NAMESPACE}" get sts --selector=postgres-operator.crunchydata.com/instance-set=instance1 -o jsonpath='{.items[*].metadata.name}') for st in $sts; do wait_for_generation "sts/$st" 1 diff --git a/e2e-tests/tests/monitoring/05-check-qan.yaml b/e2e-tests/tests/monitoring/05-check-qan.yaml index 5980350fb7..08cf7bc250 100644 --- a/e2e-tests/tests/monitoring/05-check-qan.yaml +++ b/e2e-tests/tests/monitoring/05-check-qan.yaml @@ -7,6 +7,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + token=$(kubectl get -n "${NAMESPACE}" secret monitoring-pmm-secret --template='{{.data.PMM_SERVER_TOKEN | base64decode}}') instance=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/instance-set=instance1 -o 'jsonpath={.items[].metadata.name}') diff --git a/e2e-tests/tests/monitoring/06-check-pgstatstatements-query-source.yaml b/e2e-tests/tests/monitoring/06-check-pgstatstatements-query-source.yaml index 43c0f55fa9..ec4d491da8 100644 --- a/e2e-tests/tests/monitoring/06-check-pgstatstatements-query-source.yaml +++ b/e2e-tests/tests/monitoring/06-check-pgstatstatements-query-source.yaml @@ -7,6 +7,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + primary=$(get_pod_by_role monitoring primary name) kubectl -n ${NAMESPACE} exec ${primary} -- bash -c 'psql -c "SELECT * FROM pg_extension;"' if ! kubectl -n ${NAMESPACE} exec ${primary} -- bash -c 'psql -c "SELECT extname FROM pg_extension;"' | grep pg_stat_statements; then diff --git a/e2e-tests/tests/monitoring/07-enable-pgstatmonitor-query-source.yaml b/e2e-tests/tests/monitoring/07-enable-pgstatmonitor-query-source.yaml index faf25f2e95..b03f245e0d 100644 --- a/e2e-tests/tests/monitoring/07-enable-pgstatmonitor-query-source.yaml +++ b/e2e-tests/tests/monitoring/07-enable-pgstatmonitor-query-source.yaml @@ -5,13 +5,18 @@ commands: set -o errexit set -o xtrace + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + if [[ "${PG_DISTRIBUTION}" == "community" ]]; then echo "Skipping pg_stat_monitor query-source test: not available in community images" exit 0 fi - source ../../functions - kubectl -n ${NAMESPACE} patch perconapgcluster/monitoring --type=merge -p '{ "spec":{ "pmm":{"querySource":"pgstatmonitor"} diff --git a/e2e-tests/tests/monitoring/08-check-mountpoint-expose.yaml b/e2e-tests/tests/monitoring/08-check-mountpoint-expose.yaml index b14ef5c73f..75b8e5fae4 100644 --- a/e2e-tests/tests/monitoring/08-check-mountpoint-expose.yaml +++ b/e2e-tests/tests/monitoring/08-check-mountpoint-expose.yaml @@ -7,6 +7,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + series_fetched=$(execute_curl --insecure -G "https://admin:admin@$(get_service_ip monitoring-service)/prometheus/api/v1/query_range" \ --data-urlencode "query=node_filesystem_free_bytes{mountpoint=\"/pgdata\"}" \ --data-urlencode "start=$(($(date +%s) - 300))" \ diff --git a/e2e-tests/tests/monitoring/09-check-password-leak.yaml b/e2e-tests/tests/monitoring/09-check-password-leak.yaml index f82477d7df..4a4365e0af 100644 --- a/e2e-tests/tests/monitoring/09-check-password-leak.yaml +++ b/e2e-tests/tests/monitoring/09-check-password-leak.yaml @@ -7,5 +7,10 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + # Temporarily skipping this check # check_passwords_leak diff --git a/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml index 06abda6aee..d9d871d017 100644 --- a/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml @@ -1,14 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -delete: -- apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster - metadata: - name: monitoring -- apiVersion: upstream.pgv2.percona.com/v1beta1 - kind: PostgresCluster - metadata: - name: monitoring commands: - script: |- set -o errexit @@ -16,7 +7,13 @@ commands: source ../../functions - remove_all_finalizers + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping monitoring test on arm64: PMM server does not support ARM" + exit 0 + fi + + kubectl -n "${NAMESPACE}" delete pg monitoring --ignore-not-found --wait=false + wait_for_delete pg/monitoring check_operator_panic destroy_operator - timeout: 60 + timeout: 180 diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml index b7e8b5249f..04688c2759 100644 --- a/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml +++ b/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml @@ -46,4 +46,4 @@ commands: $sed -i "s||${MINIO_ENDPOINT}|g" "${TEMP_DIR}/cr_lower.yaml" $sed -i 's|repo2-s3-verify-tls: n$|repo2-s3-verify-tls: "n"|' "${TEMP_DIR}/cr_lower.yaml" - kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr_lower.yaml" + add_arch_to_pg "${TEMP_DIR}/cr_lower.yaml" | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml b/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml index 098d743e4c..5145dcd3f2 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml @@ -4,5 +4,25 @@ metadata: name: check-operator-and-minio timeout: 180 commands: - - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 - - script: kubectl assert exist-enhanced deployment minio-service -n ${NAMESPACE} --field-selector status.readyReplicas=1 + - script: | + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + + until kubectl assert exist-enhanced deployment percona-postgresql-operator \ + -n "${OPERATOR_NS:-$NAMESPACE}" --field-selector status.readyReplicas=1; do + echo "Waiting for percona-postgresql-operator deployment" + sleep 5 + done + + until kubectl assert exist-enhanced deployment minio-service \ + -n "${NAMESPACE}" --field-selector status.readyReplicas=1; do + echo "Waiting for minio-service deployment" + sleep 5 + done diff --git a/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml b/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml index 440d5143c6..26691e4d5d 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml @@ -7,6 +7,12 @@ commands: set -o xtrace source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + init_temp_dir # do this only in the first TestStep destroy_cert_manager @@ -18,4 +24,4 @@ commands: # extension artifacts storage (same flow as the custom-extensions suite) deploy_s3_secrets deploy_minio - copy_custom_extensions_form_aws + copy_custom_extensions_from_aws diff --git a/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml b/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml index afaa68357c..3102c1c01e 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml @@ -1,13 +1,16 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: operator-upgrade-extensions -status: - state: ready - postgres: - ready: 3 - size: 3 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + + wait_cluster_consistency operator-upgrade-extensions 60 diff --git a/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml b/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml index 7c959cbb9c..463e84a85a 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + lower_ver="$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" curl -s "https://raw.githubusercontent.com/percona/percona-postgresql-operator/v${lower_ver}/deploy/cr.yaml" >"${TEMP_DIR}/cr_lower.yaml" @@ -38,6 +43,7 @@ commands: # extensions installer runs from the operator image of the RELEASED version $sed -i "s||${REGISTRY_NAME_FULL}${operator_image}|g" "${TEMP_DIR}/cr.yaml" + add_arch_to_pg "${TEMP_DIR}/cr.yaml" >"${TEMP_DIR}/cr.yaml.arch" && mv "${TEMP_DIR}/cr.yaml.arch" "${TEMP_DIR}/cr.yaml" kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr.yaml" # released-CRD canary: unknown fields are silently pruned — fail loudly instead diff --git a/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml b/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml index 84981087cd..70eb461e88 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + kubectl -n "${NAMESPACE}" patch perconapgcluster ${test_name} --type merge -p '{ "spec": { "extensions": {"custom": [{"name": "pg_cron", "version": "1.6.6"}]}, diff --git a/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml b/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml index 7b8c773986..d5146be41c 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" for ext in pg_stat_statements pgaudit vector pg_repack pg_cron; do diff --git a/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml b/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml index b00e4ff7d4..0011167949 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + snapshot_workload_revisions ${test_name} ${TEMP_DIR}/revisions.before date -u +%Y-%m-%dT%H:%M:%SZ > ${TEMP_DIR}/update_ts.txt diff --git a/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml b/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml index 2c48ee7516..4c9e26e0d5 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + wait_cluster_consistency ${test_name} assert_no_operator_rollout ${test_name} ${TEMP_DIR}/revisions.before diff --git a/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml b/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml index b4f17d2112..9f69917f0e 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + pg_ver_lower="$(cat ${TEMP_DIR}/pg_ver.txt)" target_image_postgresql=$IMAGE_POSTGRESQL target_image_pgbouncer=$IMAGE_PGBOUNCER diff --git a/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml b/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml index bf7d6d7add..ccada0bb81 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml @@ -8,6 +8,11 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" run_psql_local 'SELECT extname||$$=$$||extversion FROM pg_extension ORDER BY 1' "$uri" \ diff --git a/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml b/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml index 80f67980ee..1b048e2d39 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml @@ -8,4 +8,9 @@ commands: source ../../functions + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + check_logs_for_errors ${test_name} "$(cat ${TEMP_DIR}/update_ts.txt)" ../../tests/operator-upgrade-extensions/conf/log-allowlist.txt diff --git a/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml index 9b24d3ca80..6d7a4c39b1 100644 --- a/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml @@ -1,10 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -delete: -- apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster -- apiVersion: upstream.pgv2.percona.com/v1beta1 - kind: PostgresCluster commands: - script: |- set -o errexit @@ -12,9 +7,13 @@ commands: source ../../functions - kubectl -n ${NAMESPACE} patch postgrescluster operator-upgrade-extensions --type=merge -p='{"metadata":{"finalizers":[]}}' 2>/dev/null || true - kubectl -n ${NAMESPACE} delete postgrescluster operator-upgrade-extensions --ignore-not-found - remove_all_finalizers + if [[ ${ARCH} == arm64 ]]; then + echo "Skipping operator-upgrade-extensions on arm64: pg_cron-*-arm64.tar.gz is not in extension storage" + exit 0 + fi + + kubectl -n "${NAMESPACE}" delete pg operator-upgrade-extensions --ignore-not-found --wait=false + wait_for_delete pg/operator-upgrade-extensions check_operator_panic destroy_operator - timeout: 60 + timeout: 180 diff --git a/e2e-tests/tests/pg-tde-wal-encrypt/02-create-cluster.yaml b/e2e-tests/tests/pg-tde-wal-encrypt/02-create-cluster.yaml index 743fd75b8c..c59959b0fb 100644 --- a/e2e-tests/tests/pg-tde-wal-encrypt/02-create-cluster.yaml +++ b/e2e-tests/tests/pg-tde-wal-encrypt/02-create-cluster.yaml @@ -17,4 +17,5 @@ commands: | yq '.spec.extensions.pg_tde.vault.tokenSecret.key = "token"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde-wal-encrypt/03-enable-wal-encryption.yaml b/e2e-tests/tests/pg-tde-wal-encrypt/03-enable-wal-encryption.yaml index 70a26a4c86..26ebc53fec 100644 --- a/e2e-tests/tests/pg-tde-wal-encrypt/03-enable-wal-encryption.yaml +++ b/e2e-tests/tests/pg-tde-wal-encrypt/03-enable-wal-encryption.yaml @@ -17,4 +17,5 @@ commands: | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ | yq '.spec.extensions.pg_tde.walEncryption = true' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde-wal-encrypt/10-change-vault-provider.yaml b/e2e-tests/tests/pg-tde-wal-encrypt/10-change-vault-provider.yaml index 202973c240..4257e031ac 100644 --- a/e2e-tests/tests/pg-tde-wal-encrypt/10-change-vault-provider.yaml +++ b/e2e-tests/tests/pg-tde-wal-encrypt/10-change-vault-provider.yaml @@ -31,4 +31,5 @@ commands: | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret-rotated"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ | yq '.spec.extensions.pg_tde.walEncryption = true' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde-wal-encrypt/15-disable-pgtde.yaml b/e2e-tests/tests/pg-tde-wal-encrypt/15-disable-pgtde.yaml index 225ea48c0b..20767f9e17 100644 --- a/e2e-tests/tests/pg-tde-wal-encrypt/15-disable-pgtde.yaml +++ b/e2e-tests/tests/pg-tde-wal-encrypt/15-disable-pgtde.yaml @@ -25,4 +25,5 @@ commands: | yq '.spec.extensions.pg_tde.vault.tokenSecret.key = "token"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret-rotated"' \ | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde-wal-encrypt/16-remove-pgtde-config.yaml b/e2e-tests/tests/pg-tde-wal-encrypt/16-remove-pgtde-config.yaml index 160dfa00c4..b48d2ec3c8 100644 --- a/e2e-tests/tests/pg-tde-wal-encrypt/16-remove-pgtde-config.yaml +++ b/e2e-tests/tests/pg-tde-wal-encrypt/16-remove-pgtde-config.yaml @@ -9,4 +9,5 @@ commands: get_cr \ | yq 'del(.spec.extensions.pg_tde)' \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/sidecars/01-assert.yaml b/e2e-tests/tests/sidecars/01-assert.yaml index 9467d0a230..d33a8e2acb 100644 --- a/e2e-tests/tests/sidecars/01-assert.yaml +++ b/e2e-tests/tests/sidecars/01-assert.yaml @@ -146,12 +146,12 @@ spec: - command: - sleep - 30d - image: busybox + image: docker.io/busybox:latest name: sidecar1 - command: - sleep - 30d - image: busybox + image: docker.io/busybox:latest name: sidecar2 proxy: pgBouncer: @@ -169,7 +169,7 @@ spec: - command: - sleep - 30d - image: busybox + image: docker.io/busybox:latest name: sidecar1 status: pgbouncer: diff --git a/e2e-tests/tests/sidecars/01-create-cluster.yaml b/e2e-tests/tests/sidecars/01-create-cluster.yaml index ac7edd935d..2dd8fa5521 100644 --- a/e2e-tests/tests/sidecars/01-create-cluster.yaml +++ b/e2e-tests/tests/sidecars/01-create-cluster.yaml @@ -9,14 +9,14 @@ commands: source ../../functions get_cr \ - | yq eval '.spec.instances[0].sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"]}]' - \ - | yq eval '.spec.instances[0].sidecars += [{"name": "sidecar2", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.instances[0].sidecars = [{"name": "sidecar1", "image": "docker.io/busybox:latest", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.instances[0].sidecars += [{"name": "sidecar2", "image": "docker.io/busybox:latest", "command": ["sleep", "30d"]}]' - \ | yq eval '.spec.instances[0].sidecarVolumes = [{"name": "sidecar-secret", "secret": {"secretName": "mysecret"}}]' - \ | yq eval '.spec.instances[0].sidecarPVCs = [{"name": "sidecar-instance-claim", "spec": {"resources": {"requests": {"storage": "1Gi"}}, "volumeMode": "Filesystem", "accessModes": ["ReadWriteOnce"]}}]' - \ - | yq eval '.spec.proxy.pgBouncer.sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.proxy.pgBouncer.sidecars = [{"name": "sidecar1", "image": "docker.io/busybox:latest", "command": ["sleep", "30d"]}]' - \ | yq eval '.spec.proxy.pgBouncer.sidecarVolumes = [{"name": "sidecar-secret", "secret": {"secretName": "mysecret"}}]' - \ | yq eval '.spec.proxy.pgBouncer.sidecarPVCs = [{"name": "sidecar-pgbouncer-claim", "spec": {"resources": {"requests": {"storage": "1Gi"}}, "volumeMode": "Filesystem", "accessModes": ["ReadWriteOnce"]}}]' - \ - | yq eval '.spec.backups.pgbackrest.repoHost.sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.backups.pgbackrest.repoHost.sidecars = [{"name": "sidecar1", "image": "docker.io/busybox:latest", "command": ["sleep", "30d"]}]' - \ | yq eval '.spec.backups.pgbackrest.repoHost.sidecarVolumes = [{"name": "sidecar-secret", "secret": {"secretName": "mysecret"}}]' - \ | yq eval '.spec.backups.pgbackrest.repoHost.sidecarPVCs = [{"name": "sidecar-repohost-claim", "spec": {"resources": {"requests": {"storage": "1Gi"}}, "volumeMode": "Filesystem", "accessModes": ["ReadWriteOnce"]}}]' - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/start-from-volumes/04-create-cluster-from-volumes.yaml b/e2e-tests/tests/start-from-volumes/04-create-cluster-from-volumes.yaml index 41fc9665ee..2c07589979 100644 --- a/e2e-tests/tests/start-from-volumes/04-create-cluster-from-volumes.yaml +++ b/e2e-tests/tests/start-from-volumes/04-create-cluster-from-volumes.yaml @@ -18,4 +18,5 @@ commands: .spec.instances[].replicas=1 | .spec.dataSource.volumes.pgDataVolume.pvcName="'"${PVC}"'" | .spec.dataSource.volumes.pgDataVolume.directory="'"${DIR}"'"' - \ + | add_arch_to_pg - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/telemetry-transfer/00-deploy-version-service.yaml b/e2e-tests/tests/telemetry-transfer/00-deploy-version-service.yaml index 46bfb1deb1..ab6daa11e3 100644 --- a/e2e-tests/tests/telemetry-transfer/00-deploy-version-service.yaml +++ b/e2e-tests/tests/telemetry-transfer/00-deploy-version-service.yaml @@ -1,4 +1,3 @@ - apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 10 @@ -13,54 +12,56 @@ commands: kubectl -n ${NAMESPACE} create configmap versions \ --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.pg-operator.dep.json" \ --from-file "${TESTS_CONFIG_DIR}/operator.9.9.9.pg-operator.json" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - run: version-service - name: version-service -spec: - replicas: 1 - selector: - matchLabels: - run: version-service - template: - metadata: - labels: - run: version-service - spec: - containers: - - env: - - name: SERVE_HTTP - value: "true" - image: perconalab/version-service:main-latest - imagePullPolicy: IfNotPresent + + add_arch_to_podspecs - <<'EOF' | kubectl -n ${NAMESPACE} apply -f - + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + run: version-service + name: version-service + spec: + replicas: 1 + selector: + matchLabels: + run: version-service + template: + metadata: + labels: + run: version-service + spec: + containers: + - env: + - name: SERVE_HTTP + value: "true" + image: docker.io/perconalab/version-service:test-multi + imagePullPolicy: IfNotPresent + name: version-service + ports: + - containerPort: 11000 + protocol: TCP + volumeMounts: + - name: versions + mountPath: /sources/operator.9.9.9.pg-operator.dep.json + subPath: operator.9.9.9.pg-operator.dep.json + - name: versions + mountPath: /sources/operator.9.9.9.pg-operator.json + subPath: operator.9.9.9.pg-operator.json + volumes: + - name: versions + configMap: + name: versions + --- + apiVersion: v1 + kind: Service + metadata: name: version-service + spec: ports: - - containerPort: 11000 + - port: 11000 protocol: TCP - volumeMounts: - - name: versions - mountPath: /sources/operator.9.9.9.pg-operator.dep.json - subPath: operator.9.9.9.pg-operator.dep.json - - name: versions - mountPath: /sources/operator.9.9.9.pg-operator.json - subPath: operator.9.9.9.pg-operator.json - volumes: - - name: versions - configMap: - name: versions ---- -apiVersion: v1 -kind: Service -metadata: - name: version-service -spec: - ports: - - port: 11000 - protocol: TCP - targetPort: 11000 - selector: - run: version-service - type: ClusterIP + targetPort: 11000 + selector: + run: version-service + type: ClusterIP + EOF diff --git a/e2e-tests/tests/telemetry-transfer/06-verify-transferred-sidecars.yaml b/e2e-tests/tests/telemetry-transfer/06-verify-transferred-sidecars.yaml index 8aeec9f576..9d43a5502e 100644 --- a/e2e-tests/tests/telemetry-transfer/06-verify-transferred-sidecars.yaml +++ b/e2e-tests/tests/telemetry-transfer/06-verify-transferred-sidecars.yaml @@ -8,7 +8,7 @@ commands: source ../../functions - kubectl -n ${NAMESPACE} patch perconapgcluster/${test_name} --type=json -p '[{"op":"add","path":"/spec/instances/0/sidecars","value":[{"name":"testcontainer","image":"busybox","command":["sleep","infinity"]}]}]' + kubectl -n ${NAMESPACE} patch perconapgcluster/${test_name} --type=json -p '[{"op":"add","path":"/spec/instances/0/sidecars","value":[{"name":"testcontainer","image":"docker.io/busybox:latest","command":["sleep","infinity"]}]}]' sleep 7 CR_ID=$(kubectl -n ${NAMESPACE} get perconapgcluster/${test_name} --template='{{.metadata.uid}}') TRANSFERRED_KEYS=$(kubectl -n ${NAMESPACE} logs -l run=version-service| grep ${CR_ID} | tail -n1 | grep -Eo '\{.*\}$' | jq '."grpc.request.content".msg | keys') diff --git a/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml b/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml index 53538bb1dd..d92ede677e 100644 --- a/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml +++ b/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml @@ -19,7 +19,7 @@ commands: # if the lower version of operator doesn't have new PG version we will use lower PG version # but in that case not the intended target PG database image will be tested # (this should happen once a year on major PG version release, but only for this latest version) - if [ $pg_exists -eq 0 ]; then + if [[ ${pg_exists:-0} =~ ^[0-9]+$ && $pg_exists -eq 0 && $pg_ver =~ ^[0-9]+$ ]]; then pg_ver=$((pg_ver - 1)) fi echo $pg_ver >"${TEMP_DIR}/pg_ver.txt" @@ -53,4 +53,5 @@ commands: yq '.spec.crVersion' "${TEMP_DIR}/cr_lower.yaml" >"${TEMP_DIR}/cr_ver_released.txt" + add_arch_to_pg "${TEMP_DIR}/cr.yaml" >"${TEMP_DIR}/cr.yaml.arch" && mv "${TEMP_DIR}/cr.yaml.arch" "${TEMP_DIR}/cr.yaml" kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr.yaml" diff --git a/e2e-tests/vars.sh b/e2e-tests/vars.sh index cfb0a1829b..f197a69b21 100755 --- a/e2e-tests/vars.sh +++ b/e2e-tests/vars.sh @@ -77,6 +77,13 @@ else export PG_VER="${PG_VER:-18}" fi +export ARCH="${ARCH:-}" +if [[ -z ${ARCH} ]]; then + node_arches=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' 2>/dev/null | sort -u) + if [[ ${node_arches} == "arm64" ]]; then + export ARCH=arm64 + fi +fi export PG_DISTRIBUTION="${PG_DISTRIBUTION:-}" if [[ $PG_DISTRIBUTION == "community" ]]; then @@ -100,6 +107,7 @@ export PGOV1_VER=${PGOV1_VER:-"14"} export CPGO_VERSION=${CPGO_VERSION:-"5.8.7"} export MINIO_VER="5.4.0" export VAULT_VER="0.32.0" +export IMAGE_AWS_CLI=${IMAGE_AWS_CLI:-"docker.io/amazon/aws-cli:2.34.60"} # Add 'docker.io' for images that are provided without registry export REGISTRY_NAME="docker.io"