KilianTrunk
diff --git a/‎scripts/devnet-comprehensive.sh‎
Lines changed: 331 additions & 0 deletions b/‎scripts/devnet-comprehensive.sh‎
Lines changed: 331 additions & 0 deletions
@@ -0,0 +1,331 @@
+#!/usr/bin/env bash
+#
+# Comprehensive devnet test orchestrator.
+#
+# Runs (in order, on top of an already-started 6-node devnet):
+#   1. v10-rc-validation.sh                (15-section API smoke)
+#   2. _devnet-full-sweep.sh               (baseline harnesses)
+#   3. rc11 recovery tests (promote-crash + shutdown-mid-publish)
+#   4. rfc38-all aggregator (lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
+#   5. rc.12 feature probes (hub-rotation, multi-RPC failover, libp2p tunables,
+#      CG-phonebook agent discovery, structured ACK rejection reasons)
+#   6. node-ui smoke
+#   7. soak suite (libp2p / SWM / RS)
+#
+# Bash 3.2 compatible (uses parallel indexed arrays instead of
+# `declare -A`).
+#
+# IMPORTANT: do NOT edit this file while an instance is running — bash 3.2
+# re-reads the script as it executes, and a mid-run byte shift will derail
+# the parser (we've observed double-emitted PASS/FAIL log lines under that
+# race). Wait for the run to finish, or duplicate the script before
+# editing.
+#
+# Env knobs:
+#   RESULTS_DIR         override the output directory
+#                       (default: $REPO_ROOT/.devnet/comprehensive-results/<ts>)
+#   SKIP_SOAK=1         skip the long soak suite
+#   SOAK_ONLY=1         run only the soak suite
+#   SKIP_PROBES=1       skip the rc.12-specific probes
+#   SKIP_RFC38_EXTRAS=1 skip the rfc38-all suite
+#   SKIP_UI=1           skip the node-ui smoke
+#   FAIL_FAST=1         stop on first FAIL
+#   SOAK_RS_SECONDS     length of the devnet-soak-rs run (default 1800)
+#   SOAK_LIBP2P_CYCLES  libp2p-soak cycle count (default 5; each cycle ~60s)
+#   SOAK_SWM_CYCLES     swm-soak cycle count (default 10)
+
+set -u
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+TS=$(date -u +'%Y%m%dT%H%M%SZ')
+RESULTS="${RESULTS_DIR:-$REPO_ROOT/.devnet/comprehensive-results/$TS}"
+mkdir -p "$RESULTS"
+# `latest` symlink for convenience. Use -sfn so re-runs in the same dir
+# atomically replace any prior link without leaving a "latest/latest" trail.
+ln -sfn "$TS" "$(dirname "$RESULTS")/latest" 2>/dev/null || true
+
+log() { echo "[orch $(date -u +'%H:%M:%S')] $*" | tee -a "$RESULTS/orchestrator.log"; }
+
+# ── Pre-flight ───────────────────────────────────────────────────
+log "Pre-flight: devnet status"
+HARDHAT_PORT="${HARDHAT_PORT:-8545}"
+API_PORT_BASE="${API_PORT_BASE:-9201}"
+DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}"
+
+if ! curl -sf "http://127.0.0.1:$HARDHAT_PORT" -X POST -H "Content-Type: application/json" \
+   -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' > /dev/null 2>&1; then
+  log "FATAL: Hardhat not responding on :$HARDHAT_PORT — start the devnet first ($REPO_ROOT/scripts/devnet.sh start)"
+  exit 2
+fi
+AUTH=$(grep -v '^#' "$DEVNET_DIR/node1/auth.token" 2>/dev/null | head -1 || echo "")
+if [ -z "$AUTH" ]; then
+  log "FATAL: no auth token at $DEVNET_DIR/node1/auth.token"
+  exit 2
+fi
+if ! curl -sf -H "Authorization: Bearer $AUTH" "http://127.0.0.1:$API_PORT_BASE/api/status" > /dev/null 2>&1; then
+  log "FATAL: node 1 not responding on :$API_PORT_BASE"
+  exit 2
+fi
+export DKG_AUTH="$AUTH"
+log "Devnet is up. Results dir: $RESULTS"
+
+# ── Suite registry (parallel arrays; bash 3.2 compatible) ───────
+SUITE_IDS=()
+SUITE_CMDS=()
+SUITE_GROUPS=()
+SUITE_RESULTS=()
+SUITE_LOGS=()
+SUITE_ELAPSEDS=()
+
+register() {
+  SUITE_IDS+=("$1")
+  SUITE_GROUPS+=("$2")
+  SUITE_CMDS+=("$3")
+  SUITE_RESULTS+=("PENDING")
+  SUITE_LOGS+=("")
+  SUITE_ELAPSEDS+=("0")
+}
+
+# Group: smoke
+register "v10-rc-validation"   "smoke" "$REPO_ROOT/scripts/v10-rc-validation.sh"
+
+# Group: sweep
+register "devnet-full-sweep"   "sweep" "$REPO_ROOT/scripts/_devnet-full-sweep.sh"
+
+# Group: rc11 recovery
+register "rc11-promote-crash"  "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-promote-crash-recovery.sh"
+register "rc11-shutdown-mid"   "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-shutdown-mid-publish.sh"
+
+# Group: rfc38 extras (the all-aggregator runs lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
+if [ "${SKIP_RFC38_EXTRAS:-0}" != "1" ]; then
+  register "rfc38-all" "rfc38-extras" "$REPO_ROOT/scripts/devnet-test-rfc38-all.sh"
+fi
+
+# Group: rc.12 feature probes
+if [ "${SKIP_PROBES:-0}" != "1" ]; then
+  for p in hub-rotation multi-rpc-failover libp2p-tunables cg-phonebook ack-rejection-reasons; do
+    register "probe-${p}" "rc12-probes" "$REPO_ROOT/scripts/devnet-probe-${p}.sh"
+  done
+fi
+
+# Group: node-ui smoke
+if [ "${SKIP_UI:-0}" != "1" ]; then
+  register "node-ui-smoke" "node-ui" "$REPO_ROOT/scripts/devnet-test-node-ui-smoke.sh"
+fi
+
+# Group: soak (LONG)
+if [ "${SKIP_SOAK:-0}" != "1" ]; then
+  SOAK_RECIPIENT_PEER=$(curl -sf -H "Authorization: Bearer $AUTH" \
+    "http://127.0.0.1:$((API_PORT_BASE + 1))/api/status" 2>/dev/null \
+    | python3 -c "import sys,json;print(json.load(sys.stdin).get('peerId',''))" 2>/dev/null || echo "")
+  SOAK_LIBP2P_CYCLES="${SOAK_LIBP2P_CYCLES:-5}"
+  SOAK_SWM_CYCLES="${SOAK_SWM_CYCLES:-10}"
+  SOAK_RS_SECONDS="${SOAK_RS_SECONDS:-1800}"
+
+  register "libp2p-soak-short" "soak" \
+    "env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE RECIPIENT_PEER_ID=$SOAK_RECIPIENT_PEER RECIPIENT=devnet-node-2 SENDER_TAG=rc12 TOTAL_CYCLES=$SOAK_LIBP2P_CYCLES INTERVAL_S=60 $REPO_ROOT/scripts/libp2p-soak-test.sh"
+
+  # SWM soak — solo mode (PEERS_EXPECTED unset). Confirms write-tag
+  # rate on local SWM survives N × 30s cycles; cross-peer delivery
+  # is already exercised by rfc38-multi-member + rfc38-cross-cg.
+  # PEERS_EXPECTED is a comma-separated TAG list (not a count); set
+  # only if running concurrent operators sharing a SOAK_COHORT_ID.
+  register "swm-soak-short" "soak" \
+    "env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE SWM_CG_PUBLIC=devnet-test SWM_CG_CURATED=devnet-isolation SWM_INTERVAL_S=30 SWM_TOTAL_CYCLES=$SOAK_SWM_CYCLES SENDER_TAG=rc12 $REPO_ROOT/scripts/swm-soak-test.sh"
+
+  register "devnet-soak-rs" "soak" \
+    "$REPO_ROOT/scripts/devnet-soak-rs.sh 1 $SOAK_RS_SECONDS"
+fi
+
+# Apply SOAK_ONLY filter
+if [ "${SOAK_ONLY:-0}" = "1" ]; then
+  NEW_IDS=()
+  NEW_CMDS=()
+  NEW_GROUPS=()
+  NEW_RESULTS=()
+  NEW_LOGS=()
+  NEW_ELAPSEDS=()
+  i=0
+  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+    if [ "${SUITE_GROUPS[$i]}" = "soak" ]; then
+      NEW_IDS+=("${SUITE_IDS[$i]}")
+      NEW_CMDS+=("${SUITE_CMDS[$i]}")
+      NEW_GROUPS+=("${SUITE_GROUPS[$i]}")
+      NEW_RESULTS+=("PENDING")
+      NEW_LOGS+=("")
+      NEW_ELAPSEDS+=("0")
+    fi
+    i=$((i + 1))
+  done
+  SUITE_IDS=("${NEW_IDS[@]}")
+  SUITE_CMDS=("${NEW_CMDS[@]}")
+  SUITE_GROUPS=("${NEW_GROUPS[@]}")
+  SUITE_RESULTS=("${NEW_RESULTS[@]}")
+  SUITE_LOGS=("${NEW_LOGS[@]}")
+  SUITE_ELAPSEDS=("${NEW_ELAPSEDS[@]}")
+fi
+
+log "Registered ${#SUITE_IDS[@]} suite(s):"
+i=0
+while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+  log "  - ${SUITE_IDS[$i]}  [${SUITE_GROUPS[$i]}]"
+  i=$((i + 1))
+done
+
+# ── Run loop ────────────────────────────────────────────────────
+START=$(date +%s)
+TOTAL_PASS=0
+TOTAL_FAIL=0
+TOTAL_MISSING=0
+
+i=0
+while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+  id="${SUITE_IDS[$i]}"
+  cmd="${SUITE_CMDS[$i]}"
+  group="${SUITE_GROUPS[$i]}"
+  logfile="$RESULTS/${id}.log"
+  SUITE_LOGS[$i]="$logfile"
+
+  # Extract the script path (last token in the command, possibly after env vars)
+  bare_path=""
+  for tok in $cmd; do
+    case "$tok" in
+      *.sh) bare_path="$tok" ;;
+    esac
+  done
+  [ -z "$bare_path" ] && bare_path=$(echo "$cmd" | awk '{print $NF}')
+
+  if [ ! -e "$bare_path" ]; then
+    log "MISSING: $id ($bare_path)"
+    SUITE_RESULTS[$i]="MISSING"
+    TOTAL_MISSING=$((TOTAL_MISSING + 1))
+    i=$((i + 1))
+    if [ "${FAIL_FAST:-0}" = "1" ]; then
+      log "FAIL_FAST=1 — aborting"
+      break
+    fi
+    continue
+  fi
+
+  log "============================================================"
+  log "RUN  $id  [$group]"
+  log "============================================================"
+  suite_start=$(date +%s)
+  ( cd "$REPO_ROOT" && bash -c "$cmd" ) > "$logfile" 2>&1
+  ec=$?
+  suite_end=$(date +%s)
+  elapsed=$((suite_end - suite_start))
+  SUITE_ELAPSEDS[$i]="$elapsed"
+
+  if [ "$ec" -eq 0 ]; then
+    SUITE_RESULTS[$i]="PASS"
+    TOTAL_PASS=$((TOTAL_PASS + 1))
+    log "PASS $id  (${elapsed}s)"
+  else
+    SUITE_RESULTS[$i]="FAIL:$ec"
+    TOTAL_FAIL=$((TOTAL_FAIL + 1))
+    log "FAIL $id  (exit=$ec, ${elapsed}s)"
+    log "  last 12 lines of $logfile:"
+    tail -n 12 "$logfile" 2>/dev/null | sed 's/^/      /' | tee -a "$RESULTS/orchestrator.log"
+    if [ "${FAIL_FAST:-0}" = "1" ]; then
+      log "FAIL_FAST=1 — aborting"
+      break
+    fi
+  fi
+  i=$((i + 1))
+done
+
+END=$(date +%s)
+WALL=$((END - START))
+
+# ── Reports ─────────────────────────────────────────────────────
+log ""
+log "============================================================"
+log "DONE — ${WALL}s wall (~$((WALL/60))m)"
+log "PASS=$TOTAL_PASS FAIL=$TOTAL_FAIL MISSING=$TOTAL_MISSING TOTAL=${#SUITE_IDS[@]}"
+log "============================================================"
+
+# Markdown report
+MD="$RESULTS/REPORT.md"
+{
+  echo "# Comprehensive devnet test report"
+  echo
+  echo "- **Started**: $(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')"
+  echo "- **Ended**: $(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')"
+  echo "- **Wall**: ${WALL}s (~$((WALL/60))m)"
+  echo "- **Branch**: $(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD) @ $(cd "$REPO_ROOT" && git rev-parse --short HEAD)"
+  echo "- **Results dir**: \`$RESULTS\`"
+  echo
+  echo "## Summary"
+  echo
+  echo "| | count |"
+  echo "|---|---|"
+  echo "| PASS | $TOTAL_PASS |"
+  echo "| FAIL | $TOTAL_FAIL |"
+  echo "| MISSING | $TOTAL_MISSING |"
+  echo "| Total registered | ${#SUITE_IDS[@]} |"
+  echo
+  echo "## Suites"
+  echo
+  echo "| id | group | result | elapsed | log |"
+  echo "|---|---|---|---:|---|"
+  i=0
+  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+    logf=$(basename "${SUITE_LOGS[$i]}")
+    echo "| \`${SUITE_IDS[$i]}\` | ${SUITE_GROUPS[$i]} | ${SUITE_RESULTS[$i]} | ${SUITE_ELAPSEDS[$i]}s | \`$logf\` |"
+    i=$((i + 1))
+  done
+  echo
+  if [ "$TOTAL_FAIL" -gt 0 ]; then
+    echo "## Failures — last 25 lines of each failing log"
+    echo
+    i=0
+    while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+      case "${SUITE_RESULTS[$i]}" in
+        FAIL:*)
+          echo "### ${SUITE_IDS[$i]}"
+          echo
+          echo '```'
+          tail -n 25 "${SUITE_LOGS[$i]}" 2>/dev/null || echo "(no log)"
+          echo '```'
+          echo
+          ;;
+      esac
+      i=$((i + 1))
+    done
+  fi
+} > "$MD"
+
+# JSON report
+JSON="$RESULTS/REPORT.json"
+{
+  echo "{"
+  echo "  \"startedAt\": \"$(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')\","
+  echo "  \"endedAt\": \"$(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')\","
+  echo "  \"wallSeconds\": $WALL,"
+  echo "  \"branch\": \"$(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD)\","
+  echo "  \"commit\": \"$(cd "$REPO_ROOT" && git rev-parse HEAD)\","
+  echo "  \"totals\": { \"pass\": $TOTAL_PASS, \"fail\": $TOTAL_FAIL, \"missing\": $TOTAL_MISSING, \"registered\": ${#SUITE_IDS[@]} },"
+  echo "  \"suites\": ["
+  first=1
+  i=0
+  while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
+    [ "$first" -eq 0 ] && echo ","
+    first=0
+    printf '    { "id": "%s", "group": "%s", "result": "%s", "elapsedSeconds": %s, "log": "%s" }' \
+      "${SUITE_IDS[$i]}" "${SUITE_GROUPS[$i]}" "${SUITE_RESULTS[$i]}" "${SUITE_ELAPSEDS[$i]}" \
+      "$(basename "${SUITE_LOGS[$i]}")"
+    i=$((i + 1))
+  done
+  echo
+  echo "  ]"
+  echo "}"
+} > "$JSON"
+
+log "Report: $MD"
+log "JSON:   $JSON"
+
+if [ "$TOTAL_FAIL" -gt 0 ]; then
+  exit 1
+fi
+exit 0