[CI] Add CI gating based on E2E performance test results check

.ci/benchmarks/timm_models_list.txt

@@ -59,3 +59,7 @@ visformer_small 128
 vit_base_patch16_224 128
 volo_d1_224 128
 xcit_large_24_p8_224 16
+convnextv2_nano.fcmae_ft_in22k_in1k
+deit_tiny_patch16_224.fb_in1k
+vit_base_patch14_dinov2.lvd142m
+vit_base_patch16_siglip_256

.github/actions/linux-testenv/action.yml

@@ -140,6 +140,7 @@ runs:
           # for dlrm
           pip install pyre-extensions
           curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          pip uninstall -y fbgemm_gpu_nightly-cpu
           # for soft_actor_critic, temp fix
           pip install git+https://github.com/nocoding03/gym@fix-np
           cd ../pytorch
@@ -159,6 +160,7 @@ runs:
           # for dlrm
           pip install pyre-extensions
           curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          pip uninstall -y fbgemm_gpu_nightly-cpu
           cd ../pytorch
         else
           pip install -r ./.ci/docker/requirements-ci.txt

.github/ci_expected_accuracy/rolling/inductor_timm_models_inference.csv

@@ -61,3 +61,7 @@ visformer_small,pass,pass,pass,pass,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
 xcit_large_24_p8_224,pass,pass,pass,pass,pass
+convnextv2_nano.fcmae_ft_in22k_in1k,pass,pass,pass,pass,pass
+deit_tiny_patch16_224.fb_in1k,pass,pass,pass,pass,pass
+vit_base_patch14_dinov2.lvd142m,pass,pass,pass,pass,pass
+vit_base_patch16_siglip_256,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/rolling/inductor_timm_models_training.csv

@@ -63,3 +63,7 @@ visformer_small,pass,pass,pass,pass,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
 xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
+convnextv2_nano.fcmae_ft_in22k_in1k,pass,pass,pass,pass,pass
+deit_tiny_patch16_224.fb_in1k,pass,pass,pass,pass,pass
+vit_base_patch14_dinov2.lvd142m,pass,pass,pass,pass,pass
+vit_base_patch16_siglip_256,pass,pass,pass,pass,pass

.github/scripts/e2e_summary.sh

@@ -35,24 +35,24 @@ function get_acc_details() {
                         exit_label = 0;
                     }{
                         if ($0 ~/Real failed/){
-                            color="red";
+                            color="🔴";
                             exit_label++;
                         }else if ($0 ~/Expected failed/){
-                            color="blue";
+                            color="🔵";
                         }else if ($0 ~/Warning timeout/){
-                            color="orange";
+                            color="🟡";
                         }else if ($0 ~/New models/){
-                            color="blue";
+                            color="🔵";
                         }else if ($0 ~/Failed to passed/){
-                            color="green";
+                            color="🟢";
                             exit_label++;
                         }
                     }END{print color, exit_label}')
                     echo "${colorful}" >> /tmp/tmp-result.txt
                     context=$(find "${results_dir}" -name "*.csv" |\
                         grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4 |\
-                        awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}')
-                    eval "export ${mode}_${dtype}=${context}"
+                        awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("%s%s", c, $1)}}')
+                    eval "export ${mode}_${dtype}=\${context}"
                 done
             done
             accuracy_row="$(echo -e "<tr>
@@ -68,7 +68,7 @@ function get_acc_details() {
                     <td>${inference_float16}</td>
                     <td>${inference_amp_bf16}</td>
                     <td>${inference_amp_fp16}</td>
-                </tr>" |sed '/__color__/{s/__color__/\\color/g;s/_/\\_/g}'
+                </tr>"
             )"
             if [[ "${accuracy_row}" =~ "red" ]];then
                 echo "${accuracy_row}" |tee -a accuracy.details.html >> accuracy.regression.html
@@ -99,7 +99,7 @@ if [ "${accuracy}" -gt 0 ];then
 
 #### accuracy
 
-| Category | Total | Passed | Pass Rate | \$\${\\color{red}Failed}\$\$ |\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ |\$\${\\color{green}New Passed}\$\$ | \$\${\\color{blue}New Enabled}\$\$ | Not Run |
+| Category | Total | Passed | Pass Rate | Failed | Xfailed | Timeout | New Passed | New Enabled | Not Run |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
 EOF
 
@@ -124,18 +124,30 @@ EOF
             if($0 ~/Real failed/){
                 failed = $4;
                 failed_models = $5;
+                if(failed > 0){
+                    failed = "🔴"$4;
+                }
             }
             if($0 ~/Expected failed/){
                 xfail = $4;
                 xfail_models = $5;
+                if(xfail > 0){
+                    failed = "🔵"$4;
+                }
             }
             if($0 ~/timeout/){
                 timeout = $4;
                 timeout_models = $5;
+                if(timeout > 0){
+                    failed = "🟡"$4;
+                }
             }
             if($0 ~/Failed to passed/){
                 new_passed = $5;
                 new_passed_models = $6;
+                if(new_passed > 0){
+                    failed = "🟢"$4;
+                }
             }
             if($0 ~/Not run/){
                 not_run = $4;
@@ -144,9 +156,12 @@ EOF
             if($0 ~/New models/){
                 new_enabled = $3;
                 new_enabled_models = $4;
+                if(new_enabled > 0){
+                    failed = "🔵"$4;
+                }
             }
         }END {
-            printf(" %d | %d | %s | %d | %d | %d | %d | %d | %d\n",
+            printf(" %s | %s | %s | %s | %s | %s | %s | %s | %s\n",
                 total, passed, pass_rate, failed, xfail, timeout, new_passed, new_enabled, not_run);
         }')"
         echo "| ${category} | ${test_result} |" >> accuracy.summary.html
@@ -183,31 +198,33 @@ summary_file="e2e-test-result.html"
 cat > ${summary_file} << EOF
 
 #### Note:
-\$\${\\color{red}Red}\$\$: the failed cases which need look into
-\$\${\\color{green}Green}\$\$: the new passed cases which need update reference
-\$\${\\color{blue}Blue}\$\$: the expected failed or new enabled cases
-\$\${\\color{orange}Orange}\$\$: the warning cases
+🔴: the failed cases which need look into
+🟢: the new passed cases which need update reference
+🔵: the expected failed or new enabled cases
+🟡: the warning cases
 Empty means the cases NOT run
 
 $(
     if ((accuracy_regression + performance_regression > 0));then
-        echo -e "\n### Highlight regressions\n"
+        echo -e "\n### 🎯 Highlight regressions\n"
         if (( accuracy_regression > 0 ));then
             cat accuracy.regression.html
+            echo "accuracy 1" >> /tmp/tmp-result.txt
         fi
         if (( performance_regression > 0 ));then
             cat performance.regression.html
+            echo "performance 1" >> /tmp/tmp-result.txt
         fi
     fi
 )
 
-### Summary
+### 📊 Summary
 
 $(cat accuracy.summary.html)
 
 $(cat performance.summary.html)
 
-### Details
+### 📖 Details
 
 <details><summary>View detailed result</summary>
 

.github/scripts/lintrunner.sh

@@ -24,7 +24,7 @@ if ! command -v lintrunner &> /dev/null; then
 fi
 
 # Ignoring errors in one specific run
-export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046 -e SC2076"
+export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046 -e SC2076 -e SC2034"
 
 # This has already been cached in the docker image
 lintrunner init 2> /dev/null

.github/scripts/perf_comparison.py

@@ -36,15 +36,14 @@ def find_files(pattern, path):
 
 def color_result(criteria, input):
     if input == -1:
-        return input
+        output = input
     elif input < criteria:
-        output = r"$${\color{red}" + f"{input}" + "}$$"
-        return output
+        output = f"🔴{input}"
     elif input > 1 - criteria + 1:
-        output = r"$${\color{green}" + f"{input}" + "}$$"
-        return output
+        output = f"🟢{input}"
     else:
-        return input
+        output = input
+    return output
 
 # comparison result output
 output_header = ["Category", "Model",
@@ -111,11 +110,11 @@ def color_result(criteria, input):
 for column_name in ["Target vs. Baseline [Inductor]", "Target vs. Baseline [Eager]", "Inductor vs. Eager [Target]"]:
     data = [row[column_name] for index, row in output_data.iterrows() if row[column_name] > 0]
     if len(data) > 0:
-        geomean_sum["all"].append(geometric_mean(data))
+        geomean_sum["all"].append(color_result(args.criteria, geometric_mean(data)))
     for model_name in ["huggingface", "timm_models", "torchbench"]:
         data = [row[column_name] for index, row in output_data.iterrows() if row[column_name] > 0 and re.match(model_name, row["Category"])]
         if len(data) > 0:
-            geomean_sum[model_name].append(geometric_mean(data))
+            geomean_sum[model_name].append(color_result(args.criteria, geometric_mean(data)))
 geomean_sum = {k: v for k, v in geomean_sum.items() if v}
 output_sum = pd.DataFrame(geomean_sum, index=["Target vs. Baseline [Inductor]", "Target vs. Baseline [Eager]", "Inductor vs. Eager [Target]"]).T
 output = output_sum.to_html(header=True)

.github/workflows/_linux_e2e.yml

@@ -133,15 +133,15 @@ jobs:
         uses: ./.github/actions/linux-e2etest
         with:
           suite: timm_models
-          dt: float16
+          dt: float16,bfloat16
           mode: training
           scenario: accuracy,performance
       - name: Nightly Torchbench BF16 Training Test
         if: ${{ contains(inputs.test_type, 'nightly') && contains(inputs.suite, 'torchbench') }}
         uses: ./.github/actions/linux-e2etest
         with:
           suite: torchbench
-          dt: bfloat16
+          dt: float16,bfloat16
           mode: training
           scenario: accuracy,performance
       - name: Nightly PT2E Full Test

.github/workflows/_linux_e2e_summary.yml

@@ -52,7 +52,7 @@ jobs:
           mkdir baseline/
           cd baseline/
           if [[ "${{ inputs.test_type }}" != *"ly" ]];then
-            artifact_type="$(echo ${{ inputs.test_type }} |awk -F '-' '{print $1}')-weekly"
+            artifact_type="$(echo ${{ inputs.test_type }} |awk -F '-' '{print $1}')-nightly"
           else
             artifact_type="${{ inputs.test_type }}"
           fi

.github/workflows/pull.yml

@@ -145,14 +145,17 @@ jobs:
     uses: ./.github/workflows/_linux_e2e.yml
     with:
       runner: pvc_rolling
+      test_type: ${{ needs.conditions-filter.outputs.pytorch == 'nightly_wheel' && 'wheel-cicd' || 'build-cicd' }}
       pytorch: ${{ needs.conditions-filter.outputs.pytorch }}
       suite: ${{ matrix.suite }}
   linux-e2e-summary:
     name: linux-e2e-summary
-    needs: [linux-e2e]
+    needs: [conditions-filter, linux-e2e]
     if: ${{ ! cancelled() && ! endsWith(needs.linux-e2e.result, 'ed') }}
     permissions: write-all
     uses: ./.github/workflows/_linux_e2e_summary.yml
+    with:
+      test_type: ${{ needs.conditions-filter.outputs.pytorch == 'nightly_wheel' && 'wheel-cicd' || 'build-cicd' }}
 
   windows:
     name: windows