chore: And some more tests #22
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run QNS (QUIC Network Simulator) on PRs to test interop between the | |
| # changes in the PR and the main branch. | |
| name: QNS PR | |
| on: | |
| pull_request: | |
| branches: ["main"] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| DELIM: 'vs.' | |
| TIMEOUT: 20 | |
| jobs: | |
| docker-image: | |
| name: Build Docker images for QNS | |
| if: github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| include: | |
| - name: pr | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: main | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| steps: | |
| - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 | |
| with: | |
| cache-binary: true | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ matrix.ref }} | |
| - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 | |
| id: meta | |
| with: | |
| images: ghcr.io/${{ github.repository }}-qns-${{ matrix.name }} | |
| tags: type=ref,event=pr | |
| - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 | |
| with: | |
| push: false | |
| tags: ${{ steps.meta.outputs.tags }} | |
| file: qns/Dockerfile | |
| build-args: RUST_VERSION=stable | |
| platforms: linux/amd64 | |
| outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 | |
| with: | |
| name: ${{ matrix.name }} Docker image | |
| path: /tmp/${{ matrix.name }}.tar | |
| implementations: | |
| name: Determine interop pairs | |
| needs: docker-image | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| pairs: ${{ steps.config.outputs.pairs }} | |
| implementations: ${{ steps.config.outputs.implementations }} | |
| env: | |
| URL: https://github.com/mozilla/neqo | |
| ROLE: both | |
| steps: | |
| - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| pattern: '* Docker image' | |
| path: /tmp | |
| merge-multiple: true | |
| - env: | |
| REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| for IMAGE in pr main; do | |
| docker load --input "/tmp/$IMAGE.tar" | |
| echo "${IMAGE^^}_IMAGE=ghcr.io/$REPO-qns-$IMAGE:pr-$PR_NUMBER" >> "$GITHUB_ENV" | |
| done | |
| - id: config | |
| run: | | |
| # Add neqo-pr, neqo-main, and some non-default implementations | |
| # tquic: https://github.com/quic-interop/quic-interop-runner/pull/385/files | |
| # openssl: https://github.com/quic-interop/quic-interop-runner/pull/402/files | |
| # linuxquic: https://github.com/quic-interop/quic-interop-runner/pull/428 | |
| cat <<EOF > additional.json | |
| { | |
| "neqo-pr": { | |
| "image": "$PR_IMAGE", | |
| "url": "$URL", | |
| "role": "$ROLE" | |
| }, | |
| "neqo-main": { | |
| "image": "$MAIN_IMAGE", | |
| "url": "$URL", | |
| "role": "$ROLE" | |
| }, | |
| "tquic": { | |
| "image": "tquicgroup/tquic:latest", | |
| "url": "https://github.com/Tencent/tquic/", | |
| "role": "both" | |
| }, | |
| "openssl": { | |
| "image": "quay.io/openssl-ci/openssl-quic-interop", | |
| "url": "https://github.com/openssl/openssl", | |
| "role": "client" | |
| }, | |
| "linuxquic": { | |
| "image": "quay.io/lxin/linuxquic-interop:latest", | |
| "url": "https://github.com/lxin/quic", | |
| "role": "both" | |
| } | |
| } | |
| EOF | |
| curl -o runner.json https://raw.githubusercontent.com/quic-interop/quic-interop-runner/master/implementations.json | |
| jq '. += input' runner.json additional.json > implementations.json | |
| { | |
| echo "implementations<<EOF" | |
| cat implementations.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| # Determine valid interop pairs: all combinations involving neqo-pr or neqo-main | |
| # shellcheck disable=SC2016 | |
| jq < implementations.json '[ | |
| [to_entries[] | select(.value.role=="server" or .value.role=="both").key] as $servers | | |
| [to_entries[] | select(.value.role=="client" or .value.role=="both").key] as $clients | | |
| $clients[] as $client | | |
| $servers[] as $server | | |
| $client + " '"$DELIM"' " + $server | | |
| select((contains("neqo-pr") or contains("neqo-main")) and ($client != $server)) | |
| ]' > pairs.json | |
| { | |
| echo "pairs<<EOF" | |
| cat pairs.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| run-qns: | |
| name: Run QNS | |
| needs: implementations | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| pair: ${{ fromJson(needs.implementations.outputs.pairs) }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| # quic-interop-runner requires Docker v28.1. GitHub's ubuntu-24.04 comes | |
| # with v28.0.4. Install newer version. | |
| - name: Set up Docker | |
| uses: docker/setup-docker-action@e61617a16c407a86262fb923c35a616ddbe070b3 # v4.6.0 | |
| with: | |
| version: 'v28.1.0' | |
| - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| name: 'pr Docker image' | |
| path: /tmp | |
| - run: docker load --input "/tmp/pr.tar" | |
| - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| if: contains(matrix.pair, 'neqo-main') | |
| with: | |
| name: 'main Docker image' | |
| path: /tmp | |
| - if: contains(matrix.pair, 'neqo-main') | |
| run: docker load --input /tmp/main.tar | |
| - id: depair | |
| env: | |
| PAIR: ${{ matrix.pair }} | |
| run: | | |
| PAIR="${PAIR/ $DELIM /%}" | |
| echo "client=$(echo "$PAIR" | cut -d% -f1)" >> "$GITHUB_OUTPUT" | |
| echo "server=$(echo "$PAIR" | cut -d% -f2)" >> "$GITHUB_OUTPUT" | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| persist-credentials: false | |
| # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. | |
| - uses: ./.github/actions/quic-interop-runner | |
| timeout-minutes: ${{ fromJSON(env.TIMEOUT) }} | |
| continue-on-error: true | |
| id: run | |
| with: | |
| client: ${{ steps.depair.outputs.client }} | |
| server: ${{ steps.depair.outputs.server }} | |
| implementations: ${{ needs.implementations.outputs.implementations }} | |
| - if: always() | |
| env: | |
| RESULT: ${{ steps.run.outputs.result }} | |
| run: test "$RESULT" != "error" | |
| report: | |
| name: Report results | |
| if: ${{ !cancelled() }} | |
| needs: [docker-image, run-qns, implementations] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| pattern: '*results' | |
| path: results | |
| - run: sudo apt-get install -y --no-install-recommends wdiff cmark-gfm | |
| - env: | |
| PAIRS: ${{ needs.implementations.outputs.pairs }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| PR: 'neqo-pr' | |
| run: | | |
| # shellcheck disable=SC2153 | |
| mapfile -t LIST < <(echo "$PAIRS" | jq '.[]' | sort) | |
| for PREFIX in "${LIST[@]}"; do | |
| PREFIX=$(echo "$PREFIX" | tr -d '"') | |
| # Skip neqo-main results as they're only used as baseline | |
| [[ "$PREFIX" == *"neqo-main"* ]] && continue | |
| CLIENT=$(echo "$PREFIX" | cut -f1 -d " ") | |
| SERVER=$(echo "$PREFIX" | cut -f3 -d " ") | |
| if [ "$CLIENT" == "$PR" ]; then | |
| ROLE=client | |
| else | |
| ROLE=server | |
| fi | |
| RUN="results/${PREFIX} results" | |
| PAIR="$CLIENT $DELIM $SERVER" | |
| if [ ! -e "$RUN/result.json" ]; then | |
| echo "* $PAIR: run cancelled after $TIMEOUT min" >> "$ROLE.failed.md" | |
| continue | |
| fi | |
| MAIN_RUN="${RUN/$PR/neqo-main}" | |
| if [ ! -s "$MAIN_RUN/result.json" ]; then | |
| echo "* $PAIR: baseline result missing" >> "$ROLE.failed.md" | |
| continue | |
| fi | |
| for BRANCH in "$RUN" "$MAIN_RUN"; do | |
| if [ ! -s "$BRANCH/result.json" ]; then | |
| echo "* $PAIR: $BRANCH result missing" >> "$ROLE.failed.md" | |
| continue 2 | |
| fi | |
| jq < "$BRANCH/result.json" ' | |
| . as $data | | |
| .results[][].result //= "failed" | | |
| { | |
| results: [.results[] | group_by(.result)[] | {(.[0].result): [.[] | .abbr]}] | | |
| add | |
| } | | |
| . + {log_url: $data.log_url} | |
| ' > "$BRANCH/$ROLE.grouped.json" | |
| done | |
| for ROLE in client server; do | |
| [ ! -e "$RUN/$ROLE.grouped.json" ] && continue | |
| for GROUP in $(jq -r < "$RUN/$ROLE.grouped.json" '.results | keys[]'); do | |
| PR_RESULT=$(jq < "$RUN/$ROLE.grouped.json" -r '.results.'"$GROUP"'[]' | fmt -w 1000) | |
| LOG=$(jq -r < "$RUN/$ROLE.grouped.json" -r '.log_url') | |
| # Get baseline from neqo-main if available | |
| BASELINE=$(mktemp) | |
| MAIN_GROUPED="$MAIN_RUN/$ROLE.grouped.json" | |
| if [ -s "$MAIN_GROUPED" ]; then | |
| jq < "$MAIN_GROUPED" -r '.results.'"$GROUP"'[]' | fmt -w 1000 > "$BASELINE" | |
| fi | |
| [ -n "$PR_RESULT" ] || continue | |
| DIFF=$(wdiff -n "$BASELINE" - <<< "$PR_RESULT" || true) | |
| ADD="" | |
| ADD_DELIM="" | |
| DEL="" | |
| DEL_DELIM="" | |
| if [ "$GROUP" == "failed" ]; then | |
| ADD=":warning:" | |
| ADD_DELIM="\*\*" | |
| DEL=":rocket:" | |
| DEL_DELIM="~~" | |
| elif [ "$GROUP" == "succeeded" ]; then | |
| ADD=":rocket:" | |
| ADD_DELIM="~~" | |
| DEL=":warning:" | |
| DEL_DELIM="\*\*" | |
| fi | |
| RESULT=$(echo "$DIFF" | sed -E "s/\[-/ $DEL$DEL_DELIM/g; s/-\]/$DEL_DELIM /g; s/\{\+/ $ADD$ADD_DELIM/g; s/\+\}/$ADD_DELIM /g") | |
| echo "* [$PAIR]($LOG): $RESULT" >> "$ROLE.$GROUP.md" | |
| done | |
| done | |
| done | |
| { | |
| echo "### Failed Interop Tests" | |
| if [ -e client.failed.md ] || [ -e server.failed.md ]; then | |
| echo -n "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*" | |
| if [ "$BASE_SHA" ]; then | |
| echo -n ", differences relative to \`main\` at ${BASE_SHA}" | |
| fi | |
| echo "." | |
| echo | |
| echo "<table><tr><th align=\"left\">$PR as client</th><th align=\"left\">$PR as server</th></tr>" | |
| echo -n "<tr><td valign=\"top\">" | |
| if [ -e client.failed.md ]; then | |
| cmark-gfm -e strikethrough -e autolink client.failed.md | tr -d '\n' | sed 's/<\/*ul>//g; s/<li>//g; s/<\/li>/<br>/g' | |
| fi | |
| echo -n "</td><td valign=\"top\">" | |
| if [ -e server.failed.md ]; then | |
| cmark-gfm -e strikethrough -e autolink server.failed.md | tr -d '\n' | sed 's/<\/*ul>//g; s/<li>//g; s/<\/li>/<br>/g' | |
| fi | |
| echo "</td></tr></table>" | |
| else | |
| echo -n "None " | |
| if [ -e "client.succeeded.md" ] || [ -e "server.succeeded.md" ] || [ -e "client.unsupported.md" ] || [ -e "server.unsupported.md" ]; then | |
| echo ":tada:" | |
| else | |
| echo ":question:" | |
| fi | |
| fi | |
| echo "<details><summary>All results</summary>" | |
| echo | |
| for GROUP in succeeded unsupported; do | |
| echo "### ${GROUP^} Interop Tests" | |
| if [ -e "client.$GROUP.md" ] || [ -e "server.$GROUP.md" ]; then | |
| echo "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*" | |
| echo "#### $PR as client" | |
| cat "client.$GROUP.md" | |
| echo "#### $PR as server" | |
| cat "server.$GROUP.md" | |
| else | |
| echo "None :question:" | |
| fi | |
| done | |
| echo | |
| echo "</details>" | |
| } >> comment.md | |
| - uses: ./.github/actions/pr-comment-data-export | |
| with: | |
| name: ${{ github.workflow }} | |
| contents: comment.md |