Skip to content

Commit b1ae09b

Browse files
feat(scripts): add dev-cluster node upgrade tooling
1 parent e76b083 commit b1ae09b

6 files changed

Lines changed: 507 additions & 0 deletions

File tree

RELEASES.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,27 @@ and `:mainnet-release`. Promote with the retag workflows:
161161
Use `source-tag = 3.11.0` and `release-tag = testnet-release` or
162162
`mainnet-release`.
163163

164+
## Ops tooling
165+
166+
[`scripts/ops/menu.sh`](./scripts/ops/menu.sh) is the entry point for the
167+
scripted parts of a release. It offers two things:
168+
169+
1. **release github code** — runs `prepare-github-release.sh` (step 1 above).
170+
2. **migrate devnet cluster** — rolls a published release out to a NEAR One dev
171+
cluster via [`scripts/ops/dev-cluster/dev-menu.sh`](./scripts/ops/dev-cluster/dev-menu.sh).
172+
173+
The dev-cluster flow asks for the network (testnet first, then mainnet), the
174+
version, and the cluster's Nomad IP and credentials, then swaps each
175+
`mpc-node-*` Nomad job to the release image (plan, confirm, run) and checks the
176+
nodes report the new `release=` in their build info.
177+
178+
Every command is printed before it runs and every write is behind a
179+
confirmation prompt, so a run can be stopped at any step. Nothing
180+
cluster-specific is stored in this repo; addresses and credentials are typed in
181+
per run, or supplied through the per-network `NOMAD_ADDR_DEV_*`,
182+
`NOMAD_HTTP_AUTH_DEV_*`, and `MPC_NODE_ADDRS_DEV_*` environment variables to
183+
skip the matching prompt.
184+
164185
## Re-running after a failure
165186

166187
The workflow refuses to start if a release for the version already

scripts/ops/common.sh

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/usr/bin/env bash
2+
#
3+
# common.sh — generic helpers shared by the ops scripts (source, don't run).
4+
# Dev-cluster-specific helpers live in dev-cluster/dev-common.sh.
5+
#
6+
7+
# Only when both streams are terminals, so redirected output stays clean.
8+
# NO_COLOR is honoured (https://no-color.org).
9+
if [[ -t 1 && -t 2 && -z "${NO_COLOR:-}" ]]; then
10+
C_RESET=$'\033[0m' C_CMD=$'\033[36m' C_OUT=$'\033[2m'
11+
C_STEP=$'\033[1;34m' C_OK=$'\033[32m' C_WARN=$'\033[33m' C_ERR=$'\033[1;31m'
12+
else
13+
C_RESET="" C_CMD="" C_OUT="" C_STEP="" C_OK="" C_WARN="" C_ERR=""
14+
fi
15+
16+
die() {
17+
printf '%sError: %s%s\n' "$C_ERR" "$1" "$C_RESET" >&2
18+
exit 1
19+
}
20+
21+
step() { printf '\n%s%s%s\n' "$C_STEP" "$*" "$C_RESET"; }
22+
ok() { printf '%s%s%s\n' "$C_OK" "$*" "$C_RESET"; }
23+
warn() { printf '%s%s%s\n' "$C_WARN" "$*" "$C_RESET" >&2; }
24+
25+
require_cmds() {
26+
local missing=0
27+
for cmd in "$@"; do
28+
command -v "$cmd" >/dev/null 2>&1 || {
29+
printf 'Missing dependency: %s\n' "$cmd" >&2
30+
missing=1
31+
}
32+
done
33+
[[ "$missing" -eq 0 ]] || die "Please install the missing dependencies above."
34+
}
35+
36+
# Mirrors .github/workflows/release.yml, so release candidates work too.
37+
check_version() {
38+
[[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]] \
39+
|| die "'$1' is not valid semver (expected MAJOR.MINOR.PATCH[-SUFFIX])."
40+
}
41+
42+
# sha256sum is GNU-only; macOS ships shasum.
43+
sha256_of() {
44+
if command -v sha256sum >/dev/null 2>&1; then
45+
sha256sum "$1" | cut -d' ' -f1
46+
elif command -v shasum >/dev/null 2>&1; then
47+
shasum -a 256 "$1" | cut -d' ' -f1
48+
else
49+
die "Need sha256sum or shasum to hash ${1}."
50+
fi
51+
}
52+
53+
confirm() {
54+
local reply
55+
read -rp "$1 [y/N] " reply
56+
[[ "$reply" == y || "$reply" == Y ]]
57+
}
58+
59+
# Keeps the printed line copy-pasteable: quote anything not shell-safe.
60+
fmt_cmd() {
61+
local out="" arg
62+
for arg in "$@"; do
63+
case "$arg" in
64+
''|*[!A-Za-z0-9_/.:=@%+,-]*) out+=" '${arg//\'/\'\\\'\'}'" ;;
65+
*) out+=" $arg" ;;
66+
esac
67+
done
68+
printf '%s' "${out# }"
69+
}
70+
71+
# To stderr, so it stays visible when the caller captures stdout.
72+
show_cmd() {
73+
printf '\n%s $ %s%s\n' "$C_CMD" "$(fmt_cmd "$@")" "$C_RESET" >&2
74+
}
75+
76+
# Echoes a captured response, truncated — job definitions run to several KB.
77+
show_output() {
78+
local text=$1 limit=${2:-1500}
79+
if (( ${#text} > limit )); then
80+
printf '%s%s\n … (%d more characters)%s\n' \
81+
"$C_OUT" "${text:0:limit}" "$(( ${#text} - limit ))" "$C_RESET" >&2
82+
else
83+
printf '%s%s%s\n' "$C_OUT" "$text" "$C_RESET" >&2
84+
fi
85+
}
86+
87+
# Print a command, run it, let its output through.
88+
run_cmd() {
89+
show_cmd "$@"
90+
"$@"
91+
}
92+
93+
# Subshell, so a die() inside ends the step rather than the menu around it.
94+
run_step() {
95+
( "$@" )
96+
}
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env bash
2+
#
3+
# dev-common.sh — helpers specific to the NEAR One dev clusters (source, don't
4+
# run). Generic helpers live in ../common.sh.
5+
#
6+
# MPC_SIGN_WITH: use sign-with-legacy-keychain when the keychain can't find
7+
# a key written to ~/.near-credentials.
8+
#
9+
10+
SIGN_WITH="${MPC_SIGN_WITH:-sign-with-keychain}"
11+
12+
# Sets CONTRACT, NEAR_NET, MEMBER_ACCOUNTS, SIGN_DEPOSIT, and
13+
# re-points the endpoint vars from any exported per-cluster ones
14+
# (NOMAD_ADDR_DEV_TESTNET, ...), so the network choice drives every step.
15+
# Addresses themselves stay out of this repo.
16+
resolve_dev_cluster() {
17+
local suffix var
18+
case "$1" in
19+
testnet)
20+
CONTRACT="mpc-dev-contract.testnet" NEAR_NET="testnet" SIGN_DEPOSIT="1 NEAR"
21+
MEMBER_ACCOUNTS="mpc-node-0-mpc-dev.testnet mpc-node-1-mpc-dev.testnet"
22+
suffix="TESTNET" ;;
23+
mainnet)
24+
CONTRACT="dev-contract.near" NEAR_NET="mainnet" SIGN_DEPOSIT="0.1 NEAR"
25+
MEMBER_ACCOUNTS="mpc-0-dev-mainnet.dev-signer.near mpc-1-dev-mainnet.dev-signer.near"
26+
suffix="MAINNET" ;;
27+
*) die "Unknown dev cluster '$1' (expected testnet|mainnet)." ;;
28+
esac
29+
var="NOMAD_ADDR_DEV_${suffix}"; [[ -z "${!var:-}" ]] || export NOMAD_ADDR="${!var}"
30+
var="MPC_NODE_ADDRS_DEV_${suffix}"; [[ -z "${!var:-}" ]] || export MPC_NODE_ADDRS="${!var}"
31+
# +set: an intentionally empty value still disables the prompt.
32+
var="NOMAD_HTTP_AUTH_DEV_${suffix}"; [[ -z "${!var+set}" ]] || export NOMAD_HTTP_AUTH="${!var}"
33+
}
34+
35+
# Typed in per run; the matching NOMAD_*_DEV_<NET> export skips the prompt.
36+
# Takes the bare IP — scheme and API path are the script's business.
37+
prompt_nomad_ip() {
38+
local label=${1:-target} input
39+
while [[ -z "${NOMAD_ADDR:-}" ]]; do
40+
read -rp "Nomad IP address for the ${label} dev cluster: " input
41+
# Tolerate a pasted URL.
42+
input="${input#http://}"; input="${input#https://}"; input="${input%%/*}"
43+
if [[ ! "$input" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?$ ]]; then
44+
echo " Expected an IPv4 address, optionally with a port (e.g. 10.0.0.1 or 10.0.0.1:4646)."
45+
continue
46+
fi
47+
NOMAD_ADDR="http://${input}"
48+
done
49+
export NOMAD_ADDR
50+
}
51+
52+
prompt_http_auth() {
53+
local user pass
54+
read -rp "Nomad user for ${NOMAD_ADDR} (or user:password, blank for none): " user
55+
if [[ -z "$user" ]]; then
56+
NOMAD_HTTP_AUTH=""
57+
elif [[ "$user" == *:* ]]; then
58+
# Already joined — this form echoes the password to the terminal.
59+
NOMAD_HTTP_AUTH="$user"
60+
else
61+
read -rsp "Nomad password: " pass
62+
echo
63+
NOMAD_HTTP_AUTH="${user}:${pass}"
64+
fi
65+
export NOMAD_HTTP_AUTH
66+
}
67+
68+
prompt_node_addrs() {
69+
local input
70+
[[ -z "${MPC_NODE_ADDRS+set}" ]] || return 0
71+
read -rp "Node metrics addresses, space-separated (blank to skip verification): " input
72+
export MPC_NODE_ADDRS="$input"
73+
}
74+
75+
# Whether a credential is configured — never the credential itself.
76+
nomad_auth_state() {
77+
if [[ -z "${NOMAD_HTTP_AUTH+set}" ]]; then echo "(will prompt)"
78+
elif [[ -n "$NOMAD_HTTP_AUTH" ]]; then echo "(set)"
79+
else echo "(none)"; fi
80+
}
81+
82+
# Check every MPC_NODE_ADDRS node reports release="<version>".
83+
verify_nodes() {
84+
local version=$1
85+
require_cmds curl
86+
[[ -n "${MPC_NODE_ADDRS:-}" ]] || die "MPC_NODE_ADDRS is not set (e.g. \"host:8080 host:8080\")."
87+
88+
local addr info ok=0 fail=0
89+
for addr in ${MPC_NODE_ADDRS}; do
90+
# The metrics listener is plain HTTP, internal-only; no TLS endpoint
91+
# exists to point at.
92+
# nosemgrep: trailofbits.generic.curl-unencrypted-url.curl-unencrypted-url
93+
show_cmd curl -sf "http://${addr}/metrics" '|' grep mpc_node_build_info
94+
# nosemgrep: trailofbits.generic.curl-unencrypted-url.curl-unencrypted-url
95+
info=$(curl -sf --max-time 5 "http://${addr}/metrics" \
96+
| grep -o 'mpc_node_build_info{[^}]*}') || { echo " (unreachable)"; fail=1; continue; }
97+
echo " $info"
98+
if [[ "$info" == *"release=\"${version}\""* ]]; then ok=1; else fail=1; fi
99+
done
100+
if [[ "$fail" -eq 0 && "$ok" -eq 1 ]]; then
101+
ok "All nodes report release=\"${version}\"."
102+
else
103+
warn "Not all nodes are on ${version} yet."
104+
fi
105+
}
106+
107+
# Test signature request against the cluster contract (on-chain txn).
108+
test_sign() {
109+
resolve_dev_cluster "$1"
110+
require_cmds near
111+
112+
local signer=${MEMBER_ACCOUNTS%% *}
113+
local payload='[12,1,2,0,4,5,6,8,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,44]'
114+
local cmd=(near contract call-function as-transaction "$CONTRACT" sign
115+
json-args "{\"request\": {\"payload\": ${payload}, \"path\": \"test\", \"key_version\": 0}}"
116+
prepaid-gas '300.0 Tgas' attached-deposit "$SIGN_DEPOSIT"
117+
sign-as "$signer" network-config "$NEAR_NET" "$SIGN_WITH" send)
118+
119+
echo "Test sign on ${CONTRACT} as ${signer} (deposit ${SIGN_DEPOSIT})."
120+
show_cmd "${cmd[@]}"
121+
confirm "Send it?" || return 0
122+
if "${cmd[@]}"; then
123+
ok "Signature returned — the cluster is signing."
124+
else
125+
warn "Test sign failed — investigate before proceeding."
126+
fi
127+
}
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/usr/bin/env bash
2+
#
3+
# dev-menu.sh — entry point for dev-cluster work. Picks the network and
4+
# version, then upgrades the cluster nodes and verifies them.
5+
#
6+
# Usage: ./scripts/ops/dev-cluster/dev-menu.sh [testnet|mainnet] [VERSION]
7+
# The Nomad IP address, its credentials, and the node metrics addresses are prompted
8+
# for. Exporting the per-network NOMAD_ADDR_DEV_{TESTNET,MAINNET},
9+
# NOMAD_HTTP_AUTH_DEV_{TESTNET,MAINNET}, MPC_NODE_ADDRS_DEV_{TESTNET,MAINNET}
10+
# skips the matching prompt.
11+
#
12+
13+
set -euo pipefail
14+
15+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16+
# shellcheck source=../common.sh
17+
source "${SCRIPT_DIR}/../common.sh"
18+
# shellcheck source=dev-common.sh
19+
source "${SCRIPT_DIR}/dev-common.sh"
20+
21+
# Validated here so a typo re-prompts instead of hitting die().
22+
ask_network() {
23+
local choice
24+
while true; do
25+
read -rp "Network (testnet|mainnet) [testnet]: " choice
26+
NETWORK="${choice:-testnet}"
27+
case "$NETWORK" in
28+
testnet|mainnet) return 0 ;;
29+
*) echo "Unknown network '${NETWORK}'." ;;
30+
esac
31+
done
32+
}
33+
34+
NETWORK="${1:-}"
35+
VERSION="${2:-}"
36+
case "$NETWORK" in
37+
testnet|mainnet) ;;
38+
"") ask_network ;;
39+
*) die "Unknown network '${NETWORK}' (expected testnet|mainnet)." ;;
40+
esac
41+
if [[ -z "$VERSION" ]]; then
42+
read -rp "Version (e.g. 3.14.0): " VERSION
43+
fi
44+
check_version "$VERSION"
45+
46+
resolve_dev_cluster "$NETWORK"
47+
prompt_nomad_ip "$NETWORK"
48+
[[ -n "${NOMAD_HTTP_AUTH+set}" ]] || prompt_http_auth
49+
prompt_node_addrs
50+
51+
cat <<EOF
52+
53+
Upgrading the ${NETWORK} dev cluster to ${VERSION}
54+
contract: ${CONTRACT}
55+
Nomad: ${NOMAD_ADDR}
56+
Nomad auth: $(nomad_auth_state)
57+
node metrics: ${MPC_NODE_ADDRS:-(none — verification will be skipped)}
58+
EOF
59+
confirm "Proceed?" || { echo "Aborted."; exit 0; }
60+
61+
step "### Step 1 — nodes"
62+
run_cmd "${SCRIPT_DIR}/migrate-dev-nodes.sh" "$VERSION" \
63+
|| die "Node upgrade did not complete."
64+
65+
step "### Verify"
66+
if [[ -n "${MPC_NODE_ADDRS:-}" ]]; then
67+
run_step verify_nodes "$VERSION" || true
68+
else
69+
echo "No node addresses given — skipping the build-info check."
70+
fi
71+
run_step test_sign "$NETWORK" || true
72+
73+
echo
74+
ok "Done. Testnet first — upgrade the mainnet dev cluster only once this one is healthy."

0 commit comments

Comments
 (0)