Increase backup wait time and delete timeouts in node-backup (#3662)

---------

Signed-off-by: Oriol Muñoz <oriol.munoz@digitalasset.com>

Author: OriolMunoz-da

cluster/scripts/node-backup.sh

@@ -11,6 +11,7 @@ source "${TOOLS_LIB}/libcli.source"
 source "${SPLICE_ROOT}/cluster/scripts/utils.source"
 
 RUN_ID=$(date +%s)
+WAIT_FOR_BACKUP_RETRIES=450
 
 ##### PVC Backup & Restore
 
@@ -54,7 +55,10 @@ function wait_for_pvc_backup() {
       _info "Backup of $description PVC ready!"
       break
     else
-      (( i++ )) && (( i > 300 )) && _error "Timed out waiting for backup of $description PVC"
+      (( i++ )) && (( i > WAIT_FOR_BACKUP_RETRIES )) && {
+        # kubectl delete volumesnapshot -n "$namespace" "$backupName";
+        _error "Timed out waiting for backup of $description PVC";
+      }
       sleep 5
       _info "still waiting..."
     fi
@@ -148,7 +152,10 @@ function wait_for_cloudsql_backup() {
       _info "Backup of $description ready! Backup ID: $id "
       break
     else
-      (( i++ ))&& (( i > 300 )) &&_error "Timed out waiting for backup of $description db"
+      (( i++ ))&& (( i > WAIT_FOR_BACKUP_RETRIES )) && {
+        # gcloud sql backups delete "$id" --instance "$db_id" --quiet;
+        _error "Timed out waiting for backup of $description db";
+      }
       sleep 5
       _info "still waiting..."
     fi