Skip to content

Merge pull request #125 from paritytech/feat/remove-gh-auto-create-mo… #215

Merge pull request #125 from paritytech/feat/remove-gh-auto-create-mo…

Merge pull request #125 from paritytech/feat/remove-gh-auto-create-mo… #215

Workflow file for this run

name: E2E Tests
on:
push:
branches: [main]
pull_request:
# Daily at 06:00 UTC — catches testnet regressions
schedule:
- cron: "0 6 * * *"
workflow_dispatch:
# Fires the init-cold-smoke job after Dev Release publishes the per-PR
# `dev/<branch>` tag, so the smoke can install via the same one-liner
# the dev-release bot posts on the PR.
workflow_run:
workflows: ["Dev Release"]
types: [completed]
permissions:
pull-requests: write # sticky PR comment posting
issues: write # auto-issue on schedule fail
actions: read # /runs/{id}/jobs API for per-leg fetch
concurrency:
group: e2e-${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test-no-publish:
name: "E2E · ${{ matrix.cell }}"
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 25
strategy:
fail-fast: false
max-parallel: 5
matrix:
include:
- cell: pr-install
# source: e2e/cli/install.test.ts → describe("dot install")
pattern: "dot install"
- cell: pr-preflight
# sources: e2e/cli/build.test.ts → describe("dot build")
# e2e/cli/deploy.test.ts → describe("dot deploy — preflight and validation")
pattern: "dot build|preflight and validation"
- cell: pr-mod
# source: e2e/cli/mod.test.ts → describe("dot mod — clone")
pattern: "dot mod — clone"
- cell: pr-init-session
# sources: e2e/cli/init.test.ts → describe("dot init …")
# e2e/cli/session.test.ts → describe("session management")
pattern: "dot init|session management"
outputs:
tag: ${{ steps.setup.outputs.tag }}
steps:
# checkout must run before the local composite action can be loaded.
- uses: actions/checkout@v4
- id: setup
uses: ./.github/actions/setup-e2e
- name: Run E2E cell (one retry on transient testnet failures)
uses: nick-fields/retry@v3
with:
timeout_minutes: 20
max_attempts: 2
retry_wait_seconds: 30
command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
env:
TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
DOT_DEPLOY_VERBOSE: "1"
DOT_TAG: ${{ steps.setup.outputs.tag }}
DOT_TELEMETRY: "1"
- name: Surface failure detail
if: failure()
uses: ./.github/actions/surface-e2e-failure
- name: Upload forensic artefacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: e2e-reports-${{ matrix.cell }}
path: e2e-reports/
retention-days: 7
if-no-files-found: ignore
test-publish:
name: "E2E · ${{ matrix.cell }}"
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 55
strategy:
fail-fast: false
max-parallel: 1 # serial — share SIGNER + registry domains
matrix:
include:
- cell: pr-deploy-cdm
# source: e2e/cli/deploy.test.ts → describe("dot deploy — cdm …")
pattern: "deploy — cdm"
- cell: pr-deploy-frontend
# source: e2e/cli/deploy.test.ts → describe("dot deploy --playground — full pipeline …")
pattern: "full pipeline"
- cell: pr-deploy-foundry
# source: e2e/cli/deploy.test.ts → describe("dot deploy — foundry …")
pattern: "deploy — foundry"
steps:
# checkout must run before the local composite action can be loaded.
- uses: actions/checkout@v4
- id: setup
uses: ./.github/actions/setup-e2e
- name: Install Rust/CDM toolchain
if: matrix.cell == 'pr-deploy-cdm'
shell: bash
run: |
sudo apt-get update -q
sudo apt-get install -y -q --no-install-recommends build-essential pkg-config
if ! command -v rustup >/dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
source "$HOME/.cargo/env"
fi
rustup toolchain install nightly --profile minimal --component rust-src
rustup default nightly
curl -fsSL https://raw.githubusercontent.com/paritytech/contract-dependency-manager/main/install.sh | bash
echo "$HOME/.cdm/bin" >> "$GITHUB_PATH"
export PATH="$HOME/.cdm/bin:$PATH"
command -v cdm
cargo pvm-contract --help
- name: Run E2E cell (one retry on transient testnet failures)
uses: nick-fields/retry@v3
with:
timeout_minutes: 25
max_attempts: 2
retry_wait_seconds: 30
command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
env:
TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
DOT_DEPLOY_VERBOSE: "1"
DOT_TAG: ${{ steps.setup.outputs.tag }}
DOT_TELEMETRY: "1"
- name: Surface failure detail
if: failure()
uses: ./.github/actions/surface-e2e-failure
- name: Upload forensic artefacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: e2e-reports-${{ matrix.cell }}
path: e2e-reports/
retention-days: 7
if-no-files-found: ignore
test-nightly-publish:
name: "E2E · ${{ matrix.cell }}"
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 55
strategy:
fail-fast: false
max-parallel: 1 # serial — share SIGNER + registry domains
matrix:
include:
- cell: nightly-deploy-hardhat
# source: e2e/cli/deploy.test.ts → describe("dot deploy — hardhat …")
pattern: "deploy — hardhat"
- cell: nightly-deploy-multi
# source: e2e/cli/deploy.test.ts → describe("dot deploy — multi …")
pattern: "deploy — multi"
- cell: nightly-chaos-rpc
# source: e2e/cli/chaos.test.ts → describe("dot deploy — chaos RPC failover")
pattern: "chaos RPC failover"
testFile: e2e/cli/chaos.test.ts
steps:
- uses: actions/checkout@v4
- id: setup
uses: ./.github/actions/setup-e2e
- name: Run E2E cell (one retry on transient testnet failures)
uses: nick-fields/retry@v3
with:
timeout_minutes: 25
max_attempts: 2
retry_wait_seconds: 30
command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
env:
TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
DOT_DEPLOY_VERBOSE: "1"
DOT_TAG: ${{ steps.setup.outputs.tag }}
DOT_TELEMETRY: "1"
- name: Surface failure detail
if: failure()
uses: ./.github/actions/surface-e2e-failure
- name: Upload forensic artefacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: e2e-reports-${{ matrix.cell }}
path: e2e-reports/
retention-days: 7
if-no-files-found: ignore
test-nightly-no-publish:
name: "E2E · ${{ matrix.cell }}"
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 25
strategy:
fail-fast: false
max-parallel: 5
matrix:
include:
- cell: nightly-mod-miss
# source: e2e/cli/mod.test.ts → describe("dot mod — registry miss")
pattern: "dot mod — registry miss"
- cell: nightly-diagnostic
# source: e2e/cli/diagnostic.test.ts → describe("diagnostic mode")
# testFile scoping skips globalSetup chain-RPC calls (no chain access needed for diagnostic flag tests).
pattern: "diagnostic mode"
testFile: e2e/cli/diagnostic.test.ts
- cell: nightly-rejections
# source: e2e/cli/deploy.test.ts → describe("dot deploy — rejects --no-contract-build with no artefacts")
pattern: "rejects --no-contract-build"
- cell: nightly-chaos-sigint
# source: e2e/cli/chaos.test.ts → describe("dot deploy — chaos")
pattern: "dot deploy — chaos"
testFile: e2e/cli/chaos.test.ts
steps:
# checkout must run before the local composite action can be loaded.
- uses: actions/checkout@v4
- id: setup
uses: ./.github/actions/setup-e2e
- name: Run E2E cell (one retry on transient testnet failures)
uses: nick-fields/retry@v3
with:
timeout_minutes: 20
max_attempts: 2
retry_wait_seconds: 30
command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
env:
TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
DOT_DEPLOY_VERBOSE: "1"
DOT_TAG: ${{ steps.setup.outputs.tag }}
DOT_TELEMETRY: "1"
- name: Surface failure detail
if: failure()
uses: ./.github/actions/surface-e2e-failure
- name: Upload forensic artefacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: e2e-reports-${{ matrix.cell }}
path: e2e-reports/
retention-days: 7
if-no-files-found: ignore
report:
name: E2E Report
needs: [test-no-publish, test-publish, test-nightly-no-publish, test-nightly-publish]
if: always()
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 5
steps:
- name: Download JUnit + forensic artefacts
# Phase 1 has one upload (e2e-reports-current). When Phase 4 adds
# the matrix, drop `merge-multiple: true` and parse per-cell sub-dirs
# so per-leg junit.xml don't collide on the same flat path.
uses: actions/download-artifact@v4
with:
pattern: e2e-reports-*
path: artefacts/
merge-multiple: true
- name: Install xmlstarlet (for JUnit parsing)
run: sudo apt-get update -q && sudo apt-get install -y -q xmlstarlet
- name: Fetch per-leg job conclusions
id: legs
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: /usr/bin/bash -euo pipefail {0}
run: |
curl -fsSL \
-H "Authorization: bearer $GITHUB_TOKEN" \
-H "Accept: application/vnd.github+json" \
"${{ github.api_url }}/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
| jq -r '.jobs[] | select(.name | startswith("E2E · ")) | "\(.name)\t\(.conclusion)\t\((.completed_at // now | fromdateiso8601) - (.started_at // now | fromdateiso8601))"' \
> /tmp/legs.tsv
if [ ! -s /tmp/legs.tsv ]; then
echo "::error::No 'E2E · …' jobs found in run ${{ github.run_id }}; matrix-job naming may have changed."
exit 1
fi
echo "Per-leg conclusions:"
cat /tmp/legs.tsv
- name: Parse JUnit and render report body
env:
AGGREGATE_PUBLISH: ${{ needs.test-publish.result }}
AGGREGATE_NO_PUBLISH: ${{ needs.test-no-publish.result }}
AGGREGATE_NIGHTLY_NO_PUBLISH: ${{ needs.test-nightly-no-publish.result }}
AGGREGATE_NIGHTLY_PUBLISH: ${{ needs.test-nightly-publish.result }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
TAG: ${{ needs.test-no-publish.outputs.tag }}
shell: /usr/bin/bash -euo pipefail {0}
run: |
# Gather results across all matrices. Add each new matrix job
# here when introducing one (Phase 5b+). 'skipped' (a matrix
# that didn't run because of an event-name gate) doesn't
# count as failure.
all_results="$AGGREGATE_PUBLISH $AGGREGATE_NO_PUBLISH $AGGREGATE_NIGHTLY_NO_PUBLISH $AGGREGATE_NIGHTLY_PUBLISH"
if echo "$all_results" | grep -q failure; then
AGGREGATE=failure
elif echo "$all_results" | grep -q cancelled; then
AGGREGATE=cancelled
else
AGGREGATE=success
fi
emoji() { case "$1" in
success) echo "✅ PASS";;
failure) echo "❌ FAIL";;
skipped) echo "⏭️ SKIP";;
cancelled) echo "🚫 CANCEL";;
*) echo "❓ $1";;
esac; }
BRANCH="${{ github.head_ref || github.ref_name }}"
SHA_SHORT="${GITHUB_SHA:0:7}"
TAG_LABEL="${TAG:-e2e-ci-pr}"
# ---- Part 1: per-cell summary table ----
{
echo "<!-- e2e-pr-report -->"
echo "## E2E Test Pass · $(emoji "$AGGREGATE")"
echo ""
echo "Tag: \`$TAG_LABEL\` · Branch: \`$BRANCH\` · Commit: \`$SHA_SHORT\` · [Run logs]($RUN_URL)"
echo ""
echo "| Cell | Result | Time |"
echo "|------|--------|------|"
while IFS=$'\t' read -r name conclusion duration; do
dur_min=$(awk -v d="$duration" 'BEGIN{ printf "%dm%02ds", int(d/60), int(d%60) }')
echo "| \`${name#E2E · }\` | $(emoji "$conclusion") | $dur_min |"
done < /tmp/legs.tsv
echo ""
} > /tmp/body.md
# ---- Part 2: failures detail block (only if any test failed) ----
FAILS_FILE=/tmp/failures.tsv
: > "$FAILS_FILE"
# nullglob so non-matching globs expand to nothing.
shopt -s nullglob
XML_FILES=( artefacts/junit*.xml artefacts/*/junit*.xml )
shopt -u nullglob
if command -v xmlstarlet >/dev/null 2>&1; then
for xml in "${XML_FILES[@]}"; do
xmlstarlet sel -t -m "//testcase[failure or error]" \
-v "concat(@classname, ' › ', @name)" -o $'\t' \
-v "failure/@message | error/@message" -n "$xml" \
>> "$FAILS_FILE" || true
done
else
for xml in "${XML_FILES[@]}"; do
grep -E '<testcase|<failure|<error' "$xml" \
| awk '/<testcase/{name=$0} /<failure|<error/{print name "\t" $0}' \
>> "$FAILS_FILE" || true
done
fi
if [ -s "$FAILS_FILE" ]; then
FAIL_COUNT=$(wc -l < "$FAILS_FILE")
MAX_LINES=100
TRUNCATED=$([ "$FAIL_COUNT" -gt "$MAX_LINES" ] && echo "1" || echo "0")
{
echo "<details><summary>❌ Failed tests ($FAIL_COUNT)</summary>"
echo ""
head -n "$MAX_LINES" "$FAILS_FILE" | while IFS=$'\t' read -r name msg; do
msg_clean=$(echo "$msg" | sed -E 's/<[^>]+>//g; s/&quot;/"/g; s/&lt;/</g; s/&gt;/>/g; s/&amp;/\&/g')
msg_first3=$(echo "$msg_clean" | head -3 | sed 's/^/ /')
echo "- \`$name\`"
echo "$msg_first3"
done
if [ "$TRUNCATED" = "1" ]; then
REMAINING=$((FAIL_COUNT - MAX_LINES))
echo ""
echo "... and $REMAINING more failures (see [run logs]($RUN_URL))"
fi
echo ""
echo "</details>"
echo ""
} >> /tmp/body.md
fi
# ---- Part 3: Sentry traces link ----
# Project ID 4511298552135760 mirrors src/telemetry-config.ts (PLAYGROUND_SENTRY_DSN
# → o4511059872841728 / 4511298552135760) and sentry/dashboards/2143100.json.
# If the project is ever rotated, update all three together.
# Only the traces URL is emitted — it works against any cli.tag value.
# The "E2E Health dashboard" link is intentionally NOT emitted in
# Phase 1 (requires manual one-time dashboard creation per spec §9a).
{
echo "**Sentry traces:** [view spans for this run](https://paritytech.sentry.io/explore/traces/?project=4511298552135760&query=cli.tag%3A$TAG_LABEL)"
} >> /tmp/body.md
echo "Rendered body:"
cat /tmp/body.md
echo ""
echo "---- writing to step summary ----"
cat /tmp/body.md >> "$GITHUB_STEP_SUMMARY"
- name: Post or update PR comment
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const body = fs.readFileSync('/tmp/body.md', 'utf8');
const marker = '<!-- e2e-pr-report -->';
const prNumber = context.issue.number;
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
});
const existing = comments.find(c => c.body && c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
console.log(`Updated existing comment ${existing.id}`);
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body,
});
console.log("Created new comment");
}
- name: Open failure issue
if: >
(github.event_name == 'schedule' || github.event_name == 'release') &&
(needs.test-no-publish.result != 'success' || needs.test-publish.result != 'success' || needs.test-nightly-no-publish.result != 'success' || needs.test-nightly-publish.result != 'success')
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ needs.test-no-publish.outputs.tag }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
API_URL: ${{ github.api_url }}
REPO: ${{ github.repository }}
shell: /usr/bin/bash -euo pipefail {0}
run: |
TODAY=$(date -u +%Y-%m-%d)
TITLE="Nightly E2E failure: $TODAY"
CONTEXT="Nightly E2E failed on $TODAY (tag \`$TAG\`)."
# Release-trigger branches will be added in Phase 7 (post-Phase-4).
{
echo "$CONTEXT"
echo ""
cat /tmp/body.md
echo ""
echo "- Run: $RUN_URL"
} > /tmp/issue-body.md
BODY=$(cat /tmp/issue-body.md)
curl -fsSL -X POST \
-H "Authorization: bearer $GITHUB_TOKEN" \
-H "Accept: application/vnd.github+json" \
"$API_URL/repos/$REPO/issues" \
-d "$(jq -n --arg title "$TITLE" --arg body "$BODY" '{title: $title, body: $body}')"
init-cold-smoke:
# Runs `dot init` inside a fresh ubuntu:22.04 container — no rustup,
# no IPFS, no foundry, no cdm pre-installed. The cell-based jobs
# above run on a GitHub Actions runner that already has most of
# those tools on PATH, so they can't catch regressions in the
# install / post-install path-config logic (e.g. paritytech/
# playground-app#118 — newly-installed rustup unreachable from the
# same init process). This job is the cold-start defence: every
# dependency must be installed AND become reachable to subsequent
# init steps in the same process.
#
# Install path: the same `curl … install.sh | VERSION=… bash`
# one-liner the dev-release bot posts on every PR — i.e. exactly
# what an end user runs. install.sh downloads the SEA binary,
# adds it to PATH, then runs `dot init` as its final step (errors
# surface via exit code; see CLAUDE.md "Non-obvious invariants").
# Going through the published binary catches Bun-compile-only
# regressions that `bun run src/index.ts` masks.
#
# Triggers:
# - workflow_run (Dev Release succeeded) → install dev/<branch>
# for that PR; this is the per-PR cold-start gate.
# - schedule / workflow_dispatch on main → install latest
# stable release (no VERSION).
name: Init cold-start smoke test
runs-on: ${{ github.repository_owner == 'paritytech' && 'parity-default' || 'ubuntu-latest' }}
timeout-minutes: 30
if: |
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
steps:
- name: Run dot init via install.sh in fresh ubuntu container
env:
# Empty for schedule/dispatch → install.sh resolves the
# latest stable tag. Set to dev/<head-branch> after a
# successful Dev Release so we test the PR's binary.
VERSION_OVERRIDE: ${{ github.event_name == 'workflow_run' && format('dev/{0}', github.event.workflow_run.head_branch) || '' }}
run: |
docker run --rm \
-e CI=true \
-e VERSION="$VERSION_OVERRIDE" \
ubuntu:22.04 \
bash -eux -c '
# install.sh needs only curl + ca-certs to fetch the
# SEA binary. Anything beyond this is what `dot init`
# (called from install.sh) installs for itself.
apt-get update -qq
DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \
curl ca-certificates bash
# ── The actual smoke test ──────────────────────────────
# End-user install path. install.sh exits non-zero if
# the embedded `dot init` fails, so the pipeline alone
# catches the #118-class "installed but unreachable"
# regression. We additionally grep the captured output
# for failed-dependency markers because some failure
# modes still let init exit 0 (e.g. an optional row
# rendered with the failure glyph).
curl -fsSL https://raw.githubusercontent.com/paritytech/playground-cli/main/install.sh \
| bash 2>&1 | tee /tmp/init.log
grep -q "setup complete" /tmp/init.log || {
echo "::error::dot init did not reach '\''setup complete'\''"
exit 1
}
# Reject failed-row markers rendered by the dot TUI.
# Fail glyph is U+2715 ✕ (see src/utils/ui/theme/
# tokens.ts); we also accept U+2717 ✗ for terminal-
# font fallback. Word-anchored "failed" only — bare
# " error" would also match " errors" (e.g. apt's
# "0 errors") and apt warnings unrelated to the smoke.
if grep -E "✗ |✕ |\bFAILED\b|\bfailed dependency\b" /tmp/init.log; then
echo "::error::dot init reported a failed dependency"
exit 1
fi
'