integration

#!/bin/bash

# !! Deprecated, but kept for reference and for manual testing

set -ex
function echo_with_time {
    echo "$(date --utc +%Y-%m-%dT%H:%M:%SZ) $*"
}

source ./scripts/version

echo_with_time "INFO - Running $0"

ls -la ./dist/artifacts

helm install rancher-backup-crd ./dist/artifacts/rancher-backup-crd-$HELM_CHART_VERSION.tgz -n cattle-resources-system --create-namespace --wait
helm install rancher-backup ./dist/artifacts/rancher-backup-$HELM_CHART_VERSION.tgz -n cattle-resources-system --set image.tag=$TAG --set imagePullPolicy=IfNotPresent

time timeout 300 bash -c 'while ! (kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done'

kubectl get pods -n cattle-resources-system

time timeout 300 bash -c 'while ! (kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done'

kubectl get pods -n cattle-resources-system

# Minio not available for s390x, only test on amd64 and arm64
if [ "$ARCH" = "s390x" ]; then
    echo_with_time "Minio part of integration test only run on amd64 and arm64"
    exit 0
fi

# Deploy Minio
./scripts/deploy minio

# Deploy EcryptionConfig resources to be used by encrypted Backups
./scripts/deploy encryption_config

export POD_NAME=$(kubectl get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}")
kubectl port-forward $POD_NAME 9000 --namespace minio &
sleep 10

mkdir -p $HOME/.mc/certs/CAs
PUBLIC_CRT=$(cat <<EOF
-----BEGIN CERTIFICATE-----
MIICSTCCAe+gAwIBAgIQWgUVWCiZdyOGruNe6m4iWjAKBggqhkjOPQQDAjBMMRww
GgYDVQQKExNDZXJ0Z2VuIERldmVsb3BtZW50MSwwKgYDVQQLDCNlbGl5YW1sZXZ5
QEVsaXlhbXMtTUJQLmF0dGxvY2FsLm5ldDAeFw0yMjA1MTExNDAxMjBaFw0zMjA1
MTEwMjAxMjBaMEwxHDAaBgNVBAoTE0NlcnRnZW4gRGV2ZWxvcG1lbnQxLDAqBgNV
BAsMI2VsaXlhbWxldnlARWxpeWFtcy1NQlAuYXR0bG9jYWwubmV0MFkwEwYHKoZI
zj0CAQYIKoZIzj0DAQcDQgAEFH8UPCl/vAHkMbTF3E8yhSdLNH2XueKUHns+O4FR
hn096OJKnGZFb/HiW9iJWhj4CJ4LubSvsiZJZ7YuDlM9faOBsjCBrzAOBgNVHQ8B
Af8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAd
BgNVHQ4EFgQUImjresql78fBpwSV7lp4fT4+NnwwWAYDVR0RBFEwT4IFbWluaW+C
C21pbmlvLm1pbmlvgg9taW5pby5taW5pby5zdmOCHW1pbmlvLm1pbmlvLnN2Yy5j
bHVzdGVyLmxvY2Fsgglsb2NhbGhvc3QwCgYIKoZIzj0EAwIDSAAwRQIgWT4CU5ib
LNeXJmh2lnqEvaeKgqLHPFgMOQg+4TyO+uQCIQCI5WX1E84B+z6yX7WKIBYJIjto
RjQi75QniF10pi2jKA==
-----END CERTIFICATE-----
EOF
)

echo "$PUBLIC_CRT" > $HOME/.mc/certs/CAs/public.crt
export MC_HOST_miniolocal=https://inspectorgadget:gogadgetgo@localhost:9000

# Backup resource with encryption enabled
./scripts/deploy create-backup-encrypted

time timeout 180 bash -c 'while ! (kubectl wait --for condition=ready backup.resources.cattle.io/s3-backup-encrypted 2>/dev/null); do kubectl get backup.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=-1; sleep 2; done'

kubectl delete "backup.resources.cattle.io/s3-backup-encrypted"

# Backup resources with a recurring schedule
for BACKUP in rancherbackups-insecure rancherbackups; do

    if [[ $BACKUP = "rancherbackups-insecure" ]]; then
        # Backup without CA / insecure TLS configured
        ./scripts/deploy create-backup-insecure

        BACKUP_NAME="s3-recurring-backup-insecure"
    else
        ./scripts/deploy create-backup
        BACKUP_NAME="s3-recurring-backup"
    fi

    time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready backup.resources.cattle.io/'"${BACKUP_NAME}"' 2>/dev/null); do kubectl get backup.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=-1; sleep 2; done'

    mc ls --quiet --no-color "miniolocal/${BACKUP}"
    FIRST_BACKUP=$(mc ls --quiet --no-color miniolocal/${BACKUP} | awk '{ print $NF }')
    if [[ $FIRST_BACKUP != ${BACKUP_NAME}* ]]; then
        echo_with_time "$FIRST_BACKUP does not start with [${BACKUP_NAME}]"
        exit 1
    fi
    sleep 90
    for BACKUP_FILE in $(mc ls --quiet --no-color miniolocal/${BACKUP} | awk '{ print $NF }'); do
        echo_with_time $BACKUP_FILE
        if [[ $BACKUP_FILE != ${BACKUP_NAME}* ]]; then
            echo_with_time "$BACKUP_FILE does not start with [${BACKUP_NAME}]"
            exit 1
        fi

        if [ "${BACKUP_FILE}" = "${FIRST_BACKUP}" ]; then
            echo_with_time "First was not removed!"
            exit 1
        fi
    done
    # Disable the recurring back-ups by deleting the backup CRD
    kubectl delete "backup.resources.cattle.io/${BACKUP_NAME}"
done

# Restore resource with spec.preserveUnknownFields
# https://github.com/rancher/backup-restore-operator/issues/186

cd ./tests/files/preserve-unknown-fields
tar cvzf /tmp/preserve-unknown-fields.tar.gz -- *
cd -
mc cp --quiet --no-color /tmp/preserve-unknown-fields.tar.gz miniolocal/rancherbackups

mc ls --quiet --no-color miniolocal/rancherbackups

kubectl create -f - <<EOF
apiVersion: resources.cattle.io/v1
kind: Restore
metadata:
  name: restore-preserve-unknown-fields
spec:
  backupFilename: preserve-unknown-fields.tar.gz
  prune: false
  storageLocation:
    s3:
      credentialSecretName: miniocreds
      credentialSecretNamespace: default
      bucketName: rancherbackups
      endpoint: minio.minio.svc.cluster.local:9000
      endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTVENDQWUrZ0F3SUJBZ0lRV2dVVldDaVpkeU9HcnVOZTZtNGlXakFLQmdncWhrak9QUVFEQWpCTU1Sd3cKR2dZRFZRUUtFeE5EWlhKMFoyVnVJRVJsZG1Wc2IzQnRaVzUwTVN3d0tnWURWUVFMRENObGJHbDVZVzFzWlhaNQpRRVZzYVhsaGJYTXRUVUpRTG1GMGRHeHZZMkZzTG01bGREQWVGdzB5TWpBMU1URXhOREF4TWpCYUZ3MHpNakExCk1URXdNakF4TWpCYU1Fd3hIREFhQmdOVkJBb1RFME5sY25SblpXNGdSR1YyWld4dmNHMWxiblF4TERBcUJnTlYKQkFzTUkyVnNhWGxoYld4bGRubEFSV3hwZVdGdGN5MU5RbEF1WVhSMGJHOWpZV3d1Ym1WME1Ga3dFd1lIS29aSQp6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVGSDhVUENsL3ZBSGtNYlRGM0U4eWhTZExOSDJYdWVLVUhucytPNEZSCmhuMDk2T0pLbkdaRmIvSGlXOWlKV2hqNENKNEx1YlN2c2laSlo3WXVEbE05ZmFPQnNqQ0JyekFPQmdOVkhROEIKQWY4RUJBTUNBcVF3RXdZRFZSMGxCQXd3Q2dZSUt3WUJCUVVIQXdFd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZApCZ05WSFE0RUZnUVVJbWpyZXNxbDc4ZkJwd1NWN2xwNGZUNCtObnd3V0FZRFZSMFJCRkV3VDRJRmJXbHVhVytDCkMyMXBibWx2TG0xcGJtbHZnZzl0YVc1cGJ5NXRhVzVwYnk1emRtT0NIVzFwYm1sdkxtMXBibWx2TG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc2dnbHNiMk5oYkdodmMzUXdDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWdXVDRDVTVpYgpMTmVYSm1oMmxucUV2YWVLZ3FMSFBGZ01PUWcrNFR5Tyt1UUNJUUNJNVdYMUU4NEIrejZ5WDdXS0lCWUpJanRvClJqUWk3NVFuaUYxMHBpMmpLQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
EOF

time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready restore.resources.cattle.io/restore-preserve-unknown-fields 2>/dev/null); do kubectl get restore.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done'

# Restore resource with metadata.deletionGracePeriodSeconds
# https://github.com/rancher/backup-restore-operator/issues/188

cd ./tests/files/deletion-grace-period-seconds
tar cvzf /tmp/deletion-grace-period-seconds.tar.gz -- *
cd -
mc cp --quiet --no-color /tmp/deletion-grace-period-seconds.tar.gz miniolocal/rancherbackups

mc ls --quiet --no-color miniolocal/rancherbackups

# Run this twice as the error happens when the to be restored resource already exists
for i in $(seq 1 2); do
    echo "Running restore #${i} with resource having metadata.deletionGracePeriodSeconds"

    kubectl create -f - <<EOF
apiVersion: resources.cattle.io/v1
kind: Restore
metadata:
  name: restore-deletion-grace-period-seconds
spec:
  backupFilename: deletion-grace-period-seconds.tar.gz
  prune: false
  storageLocation:
    s3:
      credentialSecretName: miniocreds
      credentialSecretNamespace: default
      bucketName: rancherbackups
      endpoint: minio.minio.svc.cluster.local:9000
      endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTVENDQWUrZ0F3SUJBZ0lRV2dVVldDaVpkeU9HcnVOZTZtNGlXakFLQmdncWhrak9QUVFEQWpCTU1Sd3cKR2dZRFZRUUtFeE5EWlhKMFoyVnVJRVJsZG1Wc2IzQnRaVzUwTVN3d0tnWURWUVFMRENObGJHbDVZVzFzWlhaNQpRRVZzYVhsaGJYTXRUVUpRTG1GMGRHeHZZMkZzTG01bGREQWVGdzB5TWpBMU1URXhOREF4TWpCYUZ3MHpNakExCk1URXdNakF4TWpCYU1Fd3hIREFhQmdOVkJBb1RFME5sY25SblpXNGdSR1YyWld4dmNHMWxiblF4TERBcUJnTlYKQkFzTUkyVnNhWGxoYld4bGRubEFSV3hwZVdGdGN5MU5RbEF1WVhSMGJHOWpZV3d1Ym1WME1Ga3dFd1lIS29aSQp6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVGSDhVUENsL3ZBSGtNYlRGM0U4eWhTZExOSDJYdWVLVUhucytPNEZSCmhuMDk2T0pLbkdaRmIvSGlXOWlKV2hqNENKNEx1YlN2c2laSlo3WXVEbE05ZmFPQnNqQ0JyekFPQmdOVkhROEIKQWY4RUJBTUNBcVF3RXdZRFZSMGxCQXd3Q2dZSUt3WUJCUVVIQXdFd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZApCZ05WSFE0RUZnUVVJbWpyZXNxbDc4ZkJwd1NWN2xwNGZUNCtObnd3V0FZRFZSMFJCRkV3VDRJRmJXbHVhVytDCkMyMXBibWx2TG0xcGJtbHZnZzl0YVc1cGJ5NXRhVzVwYnk1emRtT0NIVzFwYm1sdkxtMXBibWx2TG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc2dnbHNiMk5oYkdodmMzUXdDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWdXVDRDVTVpYgpMTmVYSm1oMmxucUV2YWVLZ3FMSFBGZ01PUWcrNFR5Tyt1UUNJUUNJNVdYMUU4NEIrejZ5WDdXS0lCWUpJanRvClJqUWk3NVFuaUYxMHBpMmpLQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
EOF

    time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready restore.resources.cattle.io/restore-deletion-grace-period-seconds 2>/dev/null); do kubectl get restore.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done'
    kubectl delete restore.resources.cattle.io/restore-deletion-grace-period-seconds
done

# Restore encrypted resource with wildcard setting
# https://github.com/rancher/backup-restore-operator/issues/602
cd ./tests/files/encrypted-resources
tar cvzf /tmp/encrypted-resources.tar.gz -- *
cd -
mc cp --quiet --no-color /tmp/encrypted-resources.tar.gz miniolocal/rancherbackups

mc ls --quiet --no-color miniolocal/rancherbackups

kubectl create -f - <<EOF
apiVersion: resources.cattle.io/v1
kind: Restore
metadata:
  name: restore-encrypted-resources
spec:
  backupFilename: encrypted-resources.tar.gz
  encryptionConfigSecretName: encryptionconfig
  prune: false
  storageLocation:
    s3:
      credentialSecretName: miniocreds
      credentialSecretNamespace: default
      bucketName: rancherbackups
      endpoint: minio.minio.svc.cluster.local:9000
      endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNTVENDQWUrZ0F3SUJBZ0lRV2dVVldDaVpkeU9HcnVOZTZtNGlXakFLQmdncWhrak9QUVFEQWpCTU1Sd3cKR2dZRFZRUUtFeE5EWlhKMFoyVnVJRVJsZG1Wc2IzQnRaVzUwTVN3d0tnWURWUVFMRENObGJHbDVZVzFzWlhaNQpRRVZzYVhsaGJYTXRUVUpRTG1GMGRHeHZZMkZzTG01bGREQWVGdzB5TWpBMU1URXhOREF4TWpCYUZ3MHpNakExCk1URXdNakF4TWpCYU1Fd3hIREFhQmdOVkJBb1RFME5sY25SblpXNGdSR1YyWld4dmNHMWxiblF4TERBcUJnTlYKQkFzTUkyVnNhWGxoYld4bGRubEFSV3hwZVdGdGN5MU5RbEF1WVhSMGJHOWpZV3d1Ym1WME1Ga3dFd1lIS29aSQp6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVGSDhVUENsL3ZBSGtNYlRGM0U4eWhTZExOSDJYdWVLVUhucytPNEZSCmhuMDk2T0pLbkdaRmIvSGlXOWlKV2hqNENKNEx1YlN2c2laSlo3WXVEbE05ZmFPQnNqQ0JyekFPQmdOVkhROEIKQWY4RUJBTUNBcVF3RXdZRFZSMGxCQXd3Q2dZSUt3WUJCUVVIQXdFd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZApCZ05WSFE0RUZnUVVJbWpyZXNxbDc4ZkJwd1NWN2xwNGZUNCtObnd3V0FZRFZSMFJCRkV3VDRJRmJXbHVhVytDCkMyMXBibWx2TG0xcGJtbHZnZzl0YVc1cGJ5NXRhVzVwYnk1emRtT0NIVzFwYm1sdkxtMXBibWx2TG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc2dnbHNiMk5oYkdodmMzUXdDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWdXVDRDVTVpYgpMTmVYSm1oMmxucUV2YWVLZ3FMSFBGZ01PUWcrNFR5Tyt1UUNJUUNJNVdYMUU4NEIrejZ5WDdXS0lCWUpJanRvClJqUWk3NVFuaUYxMHBpMmpLQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
EOF

time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready restore.resources.cattle.io/restore-encrypted-resources 2>/dev/null); do kubectl get restore.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done'