autoparity-7c8094477a315cdb3ff75a8592a6d1f095001147 · mi350 #1156
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parity Report | |
| # Generates the ROCm-vs-CUDA (or commit-vs-commit) test parity report for one | |
| # or more architectures. Three jobs run in sequence: | |
| # 1. setup-matrix - turn the "arch" input into a job matrix + filename prefix | |
| # 2. generate-parity - per arch: download_testlogs -> per-arch CSV artifact | |
| # 3. summarize - merge the per-arch CSVs into one report + step summary | |
| # | |
| # The run-name below is just a human-friendly title shown in the Actions list. | |
| # It reads as: [autoparity-][<csv_name>-]<label> · <archs>, where <label> is the | |
| # first of: "<sha> vs <baseline>", "PR <id>", or <sha> (which defaults to | |
| # "latest"). "autoparity-" only marks auto-trigger dispatches (parity-auto.yml) | |
| # and csv_name, when given, is prepended so a custom run is easy to spot. | |
| run-name: "${{ inputs.auto_triggered && 'autoparity-' || '' }}${{ inputs.csv_name && format('{0}-', inputs.csv_name) || '' }}${{ inputs.baseline_sha && format('{0} vs {1}', inputs.sha, inputs.baseline_sha) || inputs.pr_id && format('PR {0}', inputs.pr_id) || inputs.sha }} · ${{ inputs.arch }}" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # download_testlogs flags | |
| sha: | |
| description: 'Commit SHA to pull test results for. Example: 67f1ccf46a966e75f37facd497a03f7d1bd72982.' | |
| required: false | |
| default: 'latest' | |
| type: string | |
| baseline_sha: | |
| description: 'Baseline commit SHA to compare against (same workflow/arch). Produces a commit-vs-commit report instead of ROCm-vs-CUDA.' | |
| required: false | |
| type: string | |
| pr_id: | |
| description: 'Pull request number (alternative to SHA, uses latest commit). Example: 176306' | |
| required: false | |
| type: string | |
| arch: | |
| description: 'ROCm architectures, comma or space separated. Options: mi350, mi300, mi200, nightly, navi31. Example: "nightly, mi350" or "mi300"' | |
| required: false | |
| default: 'mi350, mi300, mi200' | |
| type: string | |
| auto_triggered: | |
| description: 'Internal: set by parity-auto.yml for runs dispatched by the auto-trigger. Only prefixes the run name with "autoparity-"; does not affect output/CSV filenames.' | |
| required: false | |
| default: false | |
| type: boolean | |
| exclude_distributed: | |
| description: 'Exclude distributed tests (auto-excluded for navi31)' | |
| required: false | |
| default: false | |
| type: boolean | |
| exclude_inductor: | |
| description: 'Exclude inductor tests (auto-excluded for navi31)' | |
| required: false | |
| default: false | |
| type: boolean | |
| exclude_default: | |
| description: 'Exclude default tests' | |
| required: false | |
| default: false | |
| type: boolean | |
| include_logs: | |
| description: 'Download and include CI log files (.txt) in artifact zip' | |
| required: false | |
| default: true | |
| type: boolean | |
| skip_rocm: | |
| description: 'Skip downloading ROCm test results (generate CUDA-only report)' | |
| required: false | |
| default: false | |
| type: boolean | |
| skip_cuda: | |
| description: 'Skip downloading CUDA test results (generate ROCm-only report)' | |
| required: false | |
| default: false | |
| type: boolean | |
| # summarize_xml_testreports flags | |
| set1_name: | |
| description: 'Label for ROCm columns in output CSV. Examples: rocm, nightly, mi300. Default: rocm' | |
| required: false | |
| default: 'rocm' | |
| type: string | |
| set2_name: | |
| description: 'Label for CUDA columns in output CSV. Examples: cuda, trunk. Default: cuda' | |
| required: false | |
| default: 'cuda' | |
| type: string | |
| csv_name: | |
| description: 'Custom prefix for output filenames and artifacts. Default: YYYYMMDD_all_tests_status' | |
| required: false | |
| type: string | |
| include_inductor_periodic: | |
| description: 'Download inductor-periodic benchmark artifacts (separate from parity CSV)' | |
| required: false | |
| default: false | |
| type: boolean | |
| include_xml: | |
| description: 'Include raw XML test reports in artifact zip (WARNING: drastically increases artifact size ~10x)' | |
| required: false | |
| default: false | |
| type: boolean | |
| auto_classify: | |
| description: 'Auto-classify skip reasons for SKIPPED/MISSED tests in the output CSV' | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| arch-matrix: ${{ steps.parse.outputs.matrix }} | |
| prefix: ${{ steps.parse.outputs.prefix }} | |
| steps: | |
| - name: Parse arch input into matrix | |
| id: parse | |
| run: | | |
| ARCHS="${{ inputs.arch }}" | |
| ARCHS=$(echo "$ARCHS" | tr ',[:space:]' '\n' | sed '/^$/d' | tr '\n' ' ') | |
| JSON=$(echo "$ARCHS" | tr ' ' '\n' | sed '/^$/d' | sed 's/^/"/;s/$/"/' | paste -sd',' | sed 's/^/[/;s/$/]/') | |
| echo "matrix=$JSON" >> "$GITHUB_OUTPUT" | |
| echo "Architectures: $JSON" | |
| # Prefix for artifact names so the per-arch jobs and the summarize job | |
| # agree on where to find each other's outputs. Prefer the most | |
| # specific identifier the dispatcher gave us. | |
| if [ -n "${{ inputs.csv_name }}" ]; then | |
| PREFIX="${{ inputs.csv_name }}" | |
| elif [ -n "${{ inputs.sha }}" ] && [ "${{ inputs.sha }}" != "latest" ]; then | |
| PREFIX="${{ inputs.sha }}" | |
| elif [ -n "${{ inputs.pr_id }}" ]; then | |
| PREFIX="${{ inputs.pr_id }}" | |
| else | |
| PREFIX="parity" | |
| fi | |
| PREFIX=$(echo "$PREFIX" | xargs) | |
| echo "prefix=$PREFIX" >> "$GITHUB_OUTPUT" | |
| echo "Artifact prefix: $PREFIX" | |
| generate-parity: | |
| needs: setup-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: ${{ fromJson(needs.setup-matrix.outputs.arch-matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: pip install -r requirements.txt | |
| - name: Download artifacts | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PARITY_GITHUB_TOKEN || github.token }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| # Forward run options to download_testlogs. | |
| # | |
| # pipefail: without it, tee's 0 exit code hides a download_testlogs | |
| # failure and we'd publish a partial report as a green run. | |
| set -o pipefail | |
| ARGS="--arch ${{ matrix.arch }}" | |
| if [ -n "${{ inputs.sha }}" ] && [ "${{ inputs.sha }}" != "latest" ]; then | |
| ARGS="$ARGS --sha1 ${{ inputs.sha }}" | |
| fi | |
| if [ -n "${{ inputs.pr_id }}" ]; then | |
| ARGS="$ARGS --pr_id ${{ inputs.pr_id }}" | |
| fi | |
| if [ "${{ inputs.exclude_distributed }}" = "true" ]; then | |
| ARGS="$ARGS --exclude_distributed" | |
| fi | |
| if [ "${{ inputs.exclude_inductor }}" = "true" ]; then | |
| ARGS="$ARGS --exclude_inductor" | |
| fi | |
| if [ "${{ inputs.exclude_default }}" = "true" ]; then | |
| ARGS="$ARGS --exclude_default" | |
| fi | |
| ARGS="$ARGS --ignore_status" | |
| if [ "${{ inputs.include_logs }}" != "true" ]; then | |
| ARGS="$ARGS --artifacts_only" | |
| fi | |
| if [ "${{ inputs.skip_rocm }}" = "true" ]; then | |
| ARGS="$ARGS --no_rocm" | |
| fi | |
| if [ "${{ inputs.skip_cuda }}" = "true" ]; then | |
| ARGS="$ARGS --no_cuda" | |
| fi | |
| if [ "${{ inputs.include_inductor_periodic }}" = "true" ]; then | |
| ARGS="$ARGS --include_inductor_periodic" | |
| fi | |
| if [ -n "${{ inputs.baseline_sha }}" ]; then | |
| ARGS="$ARGS --baseline_sha ${{ inputs.baseline_sha }}" | |
| fi | |
| echo "Running: python3 ./download_testlogs $ARGS" | |
| python3 ./download_testlogs $ARGS 2>&1 | tee download_${{ matrix.arch }}.log | |
| - name: Identify output folder | |
| id: folder | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: | | |
| FOLDER=$(ls -dt [0-9]*_[0-9a-f]*/ 2>/dev/null | head -1 | sed 's:/$::') | |
| if [ -z "$FOLDER" ]; then | |
| echo "ERROR: No output folder found" | |
| exit 1 | |
| fi | |
| echo "folder=$FOLDER" >> "$GITHUB_OUTPUT" | |
| SHA=$(echo "$FOLDER" | grep -oP '[0-9a-f]{40}') | |
| echo "sha=$SHA" >> "$GITHUB_OUTPUT" | |
| DATE=$(TZ='America/Los_Angeles' date '+%Y%m%d') | |
| echo "date=$DATE" >> "$GITHUB_OUTPUT" | |
| mv download_${{ matrix.arch }}.log "$FOLDER/" 2>/dev/null || true | |
| echo "Output folder: $FOLDER, SHA: $SHA, Date: $DATE" | |
| - name: Generate CSV | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: | | |
| FOLDER="${{ steps.folder.outputs.folder }}" | |
| DATE="${{ steps.folder.outputs.date }}" | |
| ARCH="${{ matrix.arch }}" | |
| if [ -n "${{ inputs.csv_name }}" ]; then | |
| CSV_NAME="${{ inputs.csv_name }}_${ARCH}" | |
| else | |
| CSV_NAME="${DATE}_all_tests_status_${ARCH}" | |
| fi | |
| ARGS="--set1 $FOLDER/rocm_xml" | |
| if [ -n "${{ inputs.baseline_sha }}" ]; then | |
| ARGS="$ARGS --set2 $FOLDER/baseline_xml" | |
| CURRENT_SHORT=$(echo "${{ steps.folder.outputs.sha }}" | cut -c1-8) | |
| BASELINE_SHORT=$(echo "${{ inputs.baseline_sha }}" | cut -c1-8) | |
| ARGS="$ARGS --set1_name ${CURRENT_SHORT}" | |
| ARGS="$ARGS --set2_name ${BASELINE_SHORT}" | |
| else | |
| if [ "${{ inputs.skip_cuda }}" != "true" ]; then | |
| ARGS="$ARGS --set2 $FOLDER/cuda_xml" | |
| fi | |
| ARGS="$ARGS --set1_name ${{ inputs.set1_name }}" | |
| ARGS="$ARGS --set2_name ${{ inputs.set2_name }}" | |
| fi | |
| ARGS="$ARGS --output_csv $FOLDER/${CSV_NAME}.csv" | |
| SHORT_ARCH=$(echo "$ARCH" | sed 's/^mi//') | |
| if [ -n "${{ inputs.csv_name }}" ]; then | |
| RT_NAME="${{ inputs.csv_name }}_running_time_${SHORT_ARCH}" | |
| else | |
| RT_NAME="${DATE}_running_time_${SHORT_ARCH}" | |
| fi | |
| ARGS="$ARGS --test_file_running_time_output_csv $FOLDER/${RT_NAME}.csv" | |
| echo "Running: python3 -u summarize_xml_testreports.py $ARGS" | |
| python3 -u summarize_xml_testreports.py $ARGS 2>&1 | tee "$FOLDER/xml_processing_${DATE}.log" | |
| - name: Auto-classify skip reasons | |
| if: ${{ inputs.auto_classify }} | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: | | |
| FOLDER="${{ steps.folder.outputs.folder }}" | |
| CSV=$(find "$FOLDER" -maxdepth 1 -name "*.csv" ! -name "*_running_time*" | head -1) | |
| if [ -n "$CSV" ]; then | |
| echo "Auto-classifying skip reasons in $CSV" | |
| python3 auto_classify_skip_reasons.py -i "$CSV" -o "$CSV" --report 2>&1 | |
| else | |
| echo "No parity CSV found in $FOLDER, skipping auto-classify" | |
| fi | |
| - name: Detect log-based failures (timeouts, crashes) | |
| if: ${{ inputs.include_logs }} | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: | | |
| FOLDER="${{ steps.folder.outputs.folder }}" | |
| if ls "$FOLDER"/*.txt 1>/dev/null 2>&1; then | |
| python3 detect_log_failures.py --logs-dir "$FOLDER" --output "$FOLDER/log_failures_${{ matrix.arch }}.csv" 2>&1 || true | |
| else | |
| echo "No log files found in $FOLDER, skipping log failure detection" | |
| fi | |
| - name: Collect upload paths | |
| id: upload-paths | |
| run: | | |
| FOLDER=".automation_scripts/pytorch-unit-test-scripts/${{ steps.folder.outputs.folder }}" | |
| PATHS="${FOLDER}/*.csv | |
| ${FOLDER}/*.log | |
| ${FOLDER}/*.txt | |
| ${FOLDER}/inductor_periodic_rocm_dir/ | |
| ${FOLDER}/inductor_periodic_cuda_dir/" | |
| if [ "${{ inputs.include_xml }}" = "true" ]; then | |
| PATHS="${PATHS} | |
| ${FOLDER}/rocm_xml/ | |
| ${FOLDER}/cuda_xml/ | |
| ${FOLDER}/baseline_xml/" | |
| fi | |
| echo "paths<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "$PATHS" >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ needs.setup-matrix.outputs.prefix }}-results-${{ matrix.arch }} | |
| path: ${{ steps.upload-paths.outputs.paths }} | |
| summarize: | |
| needs: [setup-matrix, generate-parity] | |
| if: ${{ !cancelled() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Download all per-arch CSV artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: ${{ needs.setup-matrix.outputs.prefix }}-results-* | |
| path: artifacts | |
| - name: Build parity report | |
| working-directory: .automation_scripts/pytorch-unit-test-scripts | |
| run: | | |
| ARCHS="${{ inputs.arch }}" | |
| SHA="${{ inputs.sha }}" | |
| [ "$SHA" = "latest" ] && SHA="" | |
| PR_ID="${{ inputs.pr_id }}" | |
| BASELINE_SHA="${{ inputs.baseline_sha }}" | |
| if [ -n "$BASELINE_SHA" ]; then | |
| SET1=$(echo "$SHA" | cut -c1-8) | |
| SET2=$(echo "$BASELINE_SHA" | cut -c1-8) | |
| else | |
| SET1="${{ inputs.set1_name }}" | |
| SET2="${{ inputs.set2_name }}" | |
| fi | |
| ARCHS=$(echo "$ARCHS" | tr ',[:space:]' ' ') | |
| PREFIX=$(echo "${{ needs.setup-matrix.outputs.prefix }}" | xargs) | |
| CSV_ARGS=() | |
| ARCH_ARGS=() | |
| for ARCH in $ARCHS; do | |
| ARTIFACT_DIR="../../artifacts/${PREFIX}-results-${ARCH}" | |
| CSV=$(find "$ARTIFACT_DIR"/ -maxdepth 2 -name "*.csv" ! -name "*_running_time*" ! -name "*_summary*" ! -name "log_failures_*" ! -name "log_shards_*" ! -name "flaky_tests_*" 2>/dev/null | head -1) | |
| if [ -z "$CSV" ]; then | |
| echo "WARNING: No CSV found for $ARCH, skipping" | |
| continue | |
| fi | |
| echo "Found CSV for $ARCH: $CSV" | |
| CSV_ARGS+=("$CSV") | |
| ARCH_ARGS+=("$ARCH") | |
| done | |
| if [ ${#CSV_ARGS[@]} -eq 0 ]; then | |
| echo "::warning::No CSVs found for any architecture — some or all generate-parity jobs may have failed" | |
| echo "## ⚠ No CSVs produced" >> "$GITHUB_STEP_SUMMARY" | |
| echo "No parity CSVs were found. Check the generate-parity job logs for errors." >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| ARGS=(--csv "${CSV_ARGS[@]}" --arch "${ARCH_ARGS[@]}") | |
| ARGS+=(--set1_name "$SET1" --set2_name "$SET2") | |
| if [ -n "$SHA" ]; then | |
| ARGS+=(--sha "$SHA") | |
| else | |
| DETECTED_SHA=$(basename "$(find ../../artifacts/ -name '*.csv' | head -1)" | grep -oP '[0-9a-f]{40}' || true) | |
| if [ -n "$DETECTED_SHA" ]; then | |
| ARGS+=(--sha "$DETECTED_SHA") | |
| fi | |
| fi | |
| if [ -n "$PR_ID" ]; then | |
| ARGS+=(--pr_id "$PR_ID") | |
| fi | |
| # Collect log failure CSVs if they exist | |
| LOG_FAIL_ARGS=() | |
| for ARCH in $ARCHS; do | |
| LF="../../artifacts/${PREFIX}-results-${ARCH}" | |
| LF_CSV=$(find "$LF"/ -maxdepth 2 -name "log_failures_*.csv" 2>/dev/null | head -1) | |
| if [ -n "$LF_CSV" ]; then | |
| LOG_FAIL_ARGS+=("$LF_CSV") | |
| echo "Found log failures for $ARCH: $LF_CSV" | |
| fi | |
| done | |
| if [ ${#LOG_FAIL_ARGS[@]} -gt 0 ]; then | |
| ARGS+=(--log-failures "${LOG_FAIL_ARGS[@]}") | |
| fi | |
| OUTPUT="${PREFIX}_summary" | |
| ARGS+=(--output "$OUTPUT") | |
| echo "Running: python3 generate_summary.py ${ARGS[*]}" | |
| python3 generate_summary.py "${ARGS[@]}" | |
| cat "${OUTPUT}.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload parity summary markdown | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: parity-summary-md | |
| path: .automation_scripts/pytorch-unit-test-scripts/*_summary.md | |
| if-no-files-found: ignore | |
| retention-days: 120 | |
| - name: Add artifact links to summary | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| ARTIFACTS_JSON=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --paginate -q '.artifacts[] | {name, id}') | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" | |
| { | |
| echo "" | |
| echo "### ARTIFACTS" | |
| echo "" | |
| echo "| Artifact | Link |" | |
| echo "| --- | --- |" | |
| echo "$ARTIFACTS_JSON" | jq -r '"| \(.name) | [Download]('"${RUN_URL}"'/\(.id)) |"' | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" |