Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 9 additions & 17 deletions build-tools/cncluster
Original file line number Diff line number Diff line change
Expand Up @@ -1735,23 +1735,16 @@ subcommand_whitelist[backup_nodes]='Backup one or more CN nodes in the cluster'
function subcmd_backup_nodes() {
_cluster_must_exist

if [ "$#" -lt 2 ]; then
echo "Usage: $0 <migration_id> <internal_stack> [node...]"
if [ "$#" -lt 1 ]; then
echo "Usage: $0 <migration_id> [node...]"
exit 1
fi

if [[ ! "$1" =~ ^[0-9]+$ ]]; then
echo "Usage: $0 <migration_id> [node...]"
_error "<migration_id> must be a positive integer"
fi

if [[ "$2" != "true" && "$2" != "false" ]]; then
echo "Usage: $0 <internal_stack> [node...]"
_error "<internal_stack> must be either \"true\" or \"false\""
fi

migration_id=$1
internal_stack=$2

if [ $# -eq 2 ]; then
nodes="sv-1 sv-2 sv-3 sv-da-1 validator1 splitwell"
Expand All @@ -1764,11 +1757,11 @@ function subcmd_backup_nodes() {
case "$node" in
sv-1|sv-2|sv-3|sv-4|sv-da-1)
_info "Backing up sv node $node"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-backup.sh sv "$node" "$migration_id" "$internal_stack"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-backup.sh sv "$node" "$migration_id"
;;
validator1|splitwell)
_info "Backing up validator node $node"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-backup.sh validator "$node" "$migration_id" "$internal_stack"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-backup.sh validator "$node" "$migration_id"
;;
*)
_error "Unknown node $node"
Expand Down Expand Up @@ -1805,16 +1798,15 @@ function subcmd_restore_node() {

node="${1-}"
migration_id="${2-}"
internal_stack="${3-}"
backup_run_id="${4-}"
backup_run_id="${3-}"

if [ -z "$node" ] || [ -z "$migration_id" ]; then
_error "Usage: $SCRIPTNAME restore_node <namespace_name> <migration_id> <internal_stack> [<backup_run_id>]"
_error "Usage: $SCRIPTNAME restore_node <namespace_name> <migration_id> [<backup_run_id>]"
fi

if [ -z "$backup_run_id" ]; then
_info "No backup run_id given, looking for the latest full backup"
backup_run_id=$(SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/find-recent-backup.sh "$node" "$migration_id" "$internal_stack")
backup_run_id=$(SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/find-recent-backup.sh "$node" "$migration_id")
if [[ -z "$backup_run_id" || "$backup_run_id" == "null" ]]; then
_error "No recent backup found for $node"
fi
Expand All @@ -1831,11 +1823,11 @@ function subcmd_restore_node() {
_info "Silencing SV report creation alerts for $sv_name for 1 hour"
subcmd_silence_grafana_alerts "1 hour" "alertname=Report Creation Time Lag" "report_publisher=$sv_name"
_info "Restoring sv node $node"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-restore.sh $force "$node" "$migration_id" "$backup_run_id" "$internal_stack" cometbft sequencer participant mediator cn-apps
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-restore.sh $force "$node" "$migration_id" "$backup_run_id" cometbft sequencer participant mediator cn-apps
;;
validator1|splitwell)
_info "Restoring validator node $node"
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-restore.sh $force "$node" "$migration_id" "$backup_run_id" "$internal_stack" participant validator
SPLICE_SV=$node SPLICE_MIGRATION_ID=$migration_id "$SPLICE_ROOT"/cluster/scripts/node-restore.sh $force "$node" "$migration_id" "$backup_run_id" participant validator
;;
*)
_error "Unknown node $node"
Expand Down
16 changes: 7 additions & 9 deletions cluster/scripts/find-recent-backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ source "${TOOLS_LIB}/libcli.source"
source "${SPLICE_ROOT}/cluster/scripts/utils.source"

function usage() {
_info "Usage: $0 <namespace> <migration_id> <internal (true|false)>"
_info "Usage: $0 <namespace> <migration_id>"
}

function is_full_backup_kube() {
Expand Down Expand Up @@ -64,16 +64,15 @@ function latest_full_backup_run_id_gcloud() {
local migration_id=$2
local is_sv=$3
local expected_components=$4
local internal=$5
local num_components
num_components=$(echo "$expected_components" | wc -w)
local stack

declare -A run_ids_dict

for component in $expected_components; do
stack=$(get_stack_for_namespace_component "$namespace" "$component" "$internal")
instance="$(create_component_instance "$component" "$migration_id" "$namespace" "$internal")"
stack=$(get_stack_for_namespace_component "$namespace" "$component")
instance="$(create_component_instance "$component" "$migration_id" "$namespace")"
local full_component_instance="$namespace-$instance-pg"

local cloudsql_id
Expand Down Expand Up @@ -110,27 +109,26 @@ function latest_full_backup_run_id_gcloud() {
}

function main() {
if [ "$#" -lt 3 ]; then
if [ "$#" -lt 2 ]; then
usage
exit 1
fi

local namespace=$1
local migration_id=$2
local internal=$3

case "$namespace" in
sv-1|sv-2|sv-3|sv-4|sv-da-1)
is_sv=true
full_instance="$namespace-cn-apps-pg"
expected_components="cn-apps sequencer participant mediator"
stack=$(get_stack_for_namespace_component "$namespace" "cn-apps" "$internal")
stack=$(get_stack_for_namespace_component "$namespace" "cn-apps")
;;
*)
is_sv=false
full_instance="$namespace-validator-pg"
expected_components="validator participant"
stack=$(get_stack_for_namespace_component "$namespace" "participant" "$internal")
stack=$(get_stack_for_namespace_component "$namespace" "participant")
;;
esac

Expand All @@ -140,7 +138,7 @@ function main() {
backup_run_id=$(latest_full_backup_run_id_kube "$namespace" "$migration_id" "$is_sv" "$expected_components")
echo "$backup_run_id"
elif [ "$type" == "canton:cloud:postgres" ]; then
backup_run_id=$(latest_full_backup_run_id_gcloud "$namespace" "$migration_id" "$is_sv" "$expected_components" "$internal")
backup_run_id=$(latest_full_backup_run_id_gcloud "$namespace" "$migration_id" "$is_sv" "$expected_components")
echo "$backup_run_id"
elif [ -z "$type" ]; then
_error "No postgres instance $full_instance found in stack ${stack}. Is the cluster deployed with split DB instances?"
Expand Down
45 changes: 21 additions & 24 deletions cluster/scripts/node-backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -208,17 +208,16 @@ function backup_component() {
local component=$2
local requested_component=$3
local migration_id=$4
local internal=$5

local stack
stack=$(get_stack_for_namespace_component "$namespace" "$component" "$internal")
stack=$(get_stack_for_namespace_component "$namespace" "$component")

if [ "$component" == "$requested_component" ] || [ -z "$requested_component" ]; then
if [ "$component" == "cometbft-$migration_id" ]; then
backup_pvc "cometBFT" "$namespace" "global-domain-$migration_id-cometbft-cometbft-data" "$migration_id"
else
local db_name
db_name=$(create_component_instance "$component" "$migration_id" "$namespace" "$internal")
db_name=$(create_component_instance "$component" "$migration_id" "$namespace")
SPLICE_SV=$namespace SPLICE_MIGRATION_ID=$migration_id backup_postgres "$component" "$namespace" "$db_name-pg" "$migration_id" "$stack"
fi
else
Expand All @@ -231,16 +230,15 @@ function wait_for_backup() {
local component=$2
local requested_component=$3
local migration_id=$4
local internal=$5

local stack
stack=$(get_stack_for_namespace_component "$namespace" "$component" "$internal")
stack=$(get_stack_for_namespace_component "$namespace" "$component")

if [ "$component" == "$requested_component" ] || [ -z "$requested_component" ]; then
if [ "$component" == "cometbft-$migration_id" ]; then
wait_for_pvc_backup "cometBFT" "$namespace" "global-domain-$migration_id-cometbft-cometbft-data"
else
instance=$(create_component_instance "$component" "$migration_id" "$namespace" "$internal")
instance=$(create_component_instance "$component" "$migration_id" "$namespace")
wait_for_postgres_backup "$component" "$namespace" "$instance-pg" "$migration_id" "$stack"
fi
else
Expand All @@ -249,45 +247,44 @@ function wait_for_backup() {
}

function usage() {
echo "Usage: $0 <sv|validator> <namespace> <migration id> <internal (true|false)> [<component_name>]"
echo "Usage: $0 <sv|validator> <namespace> <migration id> [<component_name>]"
}

function main() {
if [ "$#" -lt 4 ]; then
if [ "$#" -lt 3 ]; then
usage
exit 1
fi

local namespace=$2
local migration_id=$3
local internal=$4 # "true" for internal stack, "false" for external stack
local requested_component="${5:-}"
local requested_component="${4:-}"

# TODO(#9361): support multiple domains / non-default-ID'd ones
if [ "$1" == "validator" ]; then
_info "Backing up validator $namespace"
backup_component "$namespace" "validator" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "validator" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "validator" "$requested_component" "$migration_id"
wait_for_backup "$namespace" "validator" "$requested_component" "$migration_id"
# CN apps must be strictly before participant, so we sync on apps before starting the participant backup
backup_component "$namespace" "participant" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "participant" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "participant" "$requested_component" "$migration_id"
wait_for_backup "$namespace" "participant" "$requested_component" "$migration_id"
elif [ "$1" == "sv" ]; then
_info "Backing up SV node $namespace"

backup_component "$namespace" "cn-apps" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "mediator" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "sequencer" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "cometbft-$migration_id" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "cn-apps" "$requested_component" "$migration_id"
backup_component "$namespace" "mediator" "$requested_component" "$migration_id"
backup_component "$namespace" "sequencer" "$requested_component" "$migration_id"
backup_component "$namespace" "cometbft-$migration_id" "$requested_component" "$migration_id"

wait_for_backup "$namespace" "cn-apps" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "cn-apps" "$requested_component" "$migration_id"

# CN apps must be strictly before participant, so we sync on apps before starting the participant backup
backup_component "$namespace" "participant" "$requested_component" "$migration_id" "$internal"
backup_component "$namespace" "participant" "$requested_component" "$migration_id"

wait_for_backup "$namespace" "participant" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "mediator" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "sequencer" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "cometbft-$migration_id" "$requested_component" "$migration_id" "$internal"
wait_for_backup "$namespace" "participant" "$requested_component" "$migration_id"
wait_for_backup "$namespace" "mediator" "$requested_component" "$migration_id"
wait_for_backup "$namespace" "sequencer" "$requested_component" "$migration_id"
wait_for_backup "$namespace" "cometbft-$migration_id" "$requested_component" "$migration_id"
else
usage
exit 1
Expand Down
11 changes: 4 additions & 7 deletions cluster/scripts/utils.source
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,14 @@ function get_cloudsql_id() {
function get_stack_for_namespace_component() {
local namespace=$1
local component=$2
local internal=$3

local stack=""
if [[ "${namespace}" =~ sv.* ]]; then
if [[ "${component}" == "participant" && "${internal}" != "true" ]]; then
if [[ "${component}" == "participant" ]]; then
stack="sv-canton"
elif [[ "${component}" == "sequencer" && "${internal}" != "true" ]]; then
elif [[ "${component}" == "sequencer" ]]; then
stack="sv-canton"
elif [[ "${component}" == "mediator" && "${internal}" != "true" ]]; then
elif [[ "${component}" == "mediator" ]]; then
stack="sv-canton"
else
stack="canton-network"
Expand All @@ -51,11 +50,9 @@ create_component_instance() {
local component="$1"
local migration_id="$2"
local namespace="$3"
local internal="$4"

if [[ ("$component" == "sequencer" || "$component" == "mediator" || "$component" == "participant")
&& ("$namespace" != "splitwell" && "$namespace" != "validator1" && "$namespace" != "sv")
&& ("$internal" != "true") ]]; then
&& ("$namespace" != "splitwell" && "$namespace" != "validator1" && "$namespace" != "sv"); then
component_instance="${component}-${migration_id}"
else
component_instance="${component}"
Expand Down