Merge remote-tracking branch 'upstream/develop' into develop

Author: adskyiproger

infrastructure/backups/backup.sh

@@ -177,6 +177,13 @@ elasticsearch_host() {
   fi
 }
 
+get_target_indices() {
+  docker run --rm --network=$NETWORK appropriate/curl curl -s "http://$(elasticsearch_host)/_cat/indices?h=index" \
+    | grep -E '^(ocrvs-|events_)' \
+    | paste -sd, - \
+    | sed 's/\,$//'
+}
+
 # Today's date is used for filenames if LABEL is not provided
 #-----------------------------------
 BACKUP_DATE=$(date +%Y-%m-%d)
@@ -201,17 +208,6 @@ done
 # Backup PostgreSQL
 # -----------------
 
-echo "Backing up PostgreSQL 'events' database"
-docker run --rm \
-  -e PGPASSWORD=$POSTGRES_PASSWORD \
-  -v $ROOT_PATH/backups/postgres:/backups \
-  --network=$NETWORK \
-  postgres:17 \
-  bash -c "pg_dump -h postgres -U $POSTGRES_USER -d events -F c -f /backups/events-${LABEL:-$BACKUP_DATE}.dump"
-
-# Backup PostgreSQL
-# -----------------
-
 echo "Backing up PostgreSQL 'events' database"
 docker run --rm \
   -e PGPASSWORD=$POSTGRES_PASSWORD \
@@ -261,8 +257,12 @@ echo "Backup Elasticsearch as a set of snapshot files into an elasticsearch sub
 echo ""
 
 create_elasticsearch_backup() {
+  indices=$(get_target_indices)
+  echo "List indices for backup: $indices"
   OUTPUT=""
-  OUTPUT=$(docker run --rm --network=$NETWORK appropriate/curl curl -sS -X PUT -H "Content-Type: application/json;charset=UTF-8" "http://$(elasticsearch_host)/_snapshot/ocrvs/snapshot_${LABEL:-$BACKUP_DATE}?wait_for_completion=true&pretty" -d '{ "indices": "ocrvs" }' 2>/dev/null)
+  json_payload="{\"indices\": \"${indices}\"}"
+  OUTPUT=$(docker run --rm --network=$NETWORK appropriate/curl curl -sS -X PUT -H "Content-Type: application/json;charset=UTF-8" "http://$(elasticsearch_host)/_snapshot/ocrvs/snapshot_${LABEL:-$BACKUP_DATE}?wait_for_completion=true&pretty" -d "$json_payload" 2>/dev/null) || true
+
   if echo $OUTPUT | jq -e '.snapshot.state == "SUCCESS"' > /dev/null; then
     echo "Snapshot state is SUCCESS"
   else
@@ -343,4 +343,3 @@ fi
 rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc
 rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz
 rm -r $BACKUP_RAW_FILES_DIR
-

infrastructure/backups/restore.sh

@@ -139,7 +139,25 @@ elasticsearch_host() {
 
 echo "delete any previously created snapshot if any.  This may error on a fresh install with a repository_missing_exception error.  Just ignore it."
 docker run --rm --network=$NETWORK appropriate/curl curl -X DELETE "http://$(elasticsearch_host)/_snapshot/ocrvs"
-docker run --rm --network=$NETWORK appropriate/curl curl -X DELETE "http://$(elasticsearch_host)/*" -v
+
+# Delete all data from elasticsearch
+#-----------------------------------
+approved_words=${ES_INDEX_PREFIXES:-"events_ ocrvs-"}
+indices=$(docker run --rm --network=$NETWORK appropriate/curl curl -sS -XGET "http://$(elasticsearch_host)/_cat/indices?h=index")
+echo "--------------------------"
+echo "🧹 cleanup for indices: $approved_words from $indices"
+echo "--------------------------"
+for index in ${indices[@]}; do
+  for approved in $approved_words; do
+    case "$index" in
+    "$approved_words"*) 
+        echo "Delete index $index..."
+        docker run --rm --network=$NETWORK appropriate/curl curl -sS -XDELETE "http://$(elasticsearch_host)/$index"
+        break
+        ;;
+    esac
+  done
+done
 
 echo "Waiting for elasticsearch to restart so that the restore script can find the updated volume."
 docker service update --force --update-parallelism 1 --update-delay 30s opencrvs_elasticsearch
@@ -239,7 +257,8 @@ sleep 10
 
 # Restore all data from a backup into search
 #-------------------------------------------
-docker run --rm --network=$NETWORK appropriate/curl curl -X POST -H "Content-Type: application/json;charset=UTF-8" "http://$(elasticsearch_host)/_snapshot/ocrvs/snapshot_$LABEL/_restore?pretty" -d '{ "indices": "ocrvs" }'
+json_payload="{\"indices\": \"ocrvs-*,events_*\", \"include_global_state\": false}"
+docker run --rm --network=$NETWORK appropriate/curl curl -X POST -H "Content-Type: application/json;charset=UTF-8" "http://$(elasticsearch_host)/_snapshot/ocrvs/snapshot_$LABEL/_restore?pretty" -d "$json_payload"
 sleep 10
 echo "Waiting 1 minute to rotate elasticsearch passwords"
 echo