[CI] Add CI gating based on E2E performance test results check

.ci/benchmarks/timm_models_list.txt

@@ -59,3 +59,7 @@ visformer_small 128
 vit_base_patch16_224 128
 volo_d1_224 128
 xcit_large_24_p8_224 16
+convnextv2_nano.fcmae_ft_in22k_in1k
+deit_tiny_patch16_224.fb_in1k
+vit_base_patch14_dinov2.lvd142m
+vit_base_patch16_siglip_256

.github/actions/add-comment/action.yml

@@ -0,0 +1,39 @@
+name: 'Comment from File'
+description: 'Add a PR comment from a file'
+inputs:
+  file_path:
+    description: 'Path to the comment file'
+    required: true
+  github_token:
+    description: 'GitHub token'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Read and post comment
+      shell: bash
+      run: |
+        if [ -f "${{ inputs.file_path }}" ]; then
+          BODY=$(cat "${{ inputs.file_path }}")
+          BODY="${BODY//'%'/'%25'}"
+          BODY="${BODY//$'\n'/'%0A'}"
+          BODY="${BODY//$'\r'/'%0D'}"
+          echo "BODY<<EOF" >> $GITHUB_ENV
+          echo "$BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        else
+          echo "BODY=Comment file not found: ${{ inputs.file_path }}" >> $GITHUB_ENV
+        fi
+
+    - name: Create comment
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ inputs.github_token }}
+        script: |
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: process.env.BODY
+          })

.github/actions/linux-testenv/action.yml

@@ -140,6 +140,7 @@ runs:
           # for dlrm
           pip install pyre-extensions
           curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          pip uninstall -y fbgemm_gpu_nightly-cpu
           # for soft_actor_critic, temp fix
           pip install git+https://github.com/nocoding03/gym@fix-np
           cd ../pytorch
@@ -159,6 +160,7 @@ runs:
           # for dlrm
           pip install pyre-extensions
           curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          pip uninstall -y fbgemm_gpu_nightly-cpu
           cd ../pytorch
         else
           pip install -r ./.ci/docker/requirements-ci.txt

.github/ci_expected_accuracy/rolling/inductor_timm_models_inference.csv

@@ -61,3 +61,7 @@ visformer_small,pass,pass,pass,pass,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
 xcit_large_24_p8_224,pass,pass,pass,pass,pass
+convnextv2_nano.fcmae_ft_in22k_in1k,pass,pass,pass,pass,pass
+deit_tiny_patch16_224.fb_in1k,pass,pass,pass,pass,pass
+vit_base_patch14_dinov2.lvd142m,pass,pass,pass,pass,pass
+vit_base_patch16_siglip_256,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/rolling/inductor_timm_models_training.csv

@@ -63,3 +63,7 @@ visformer_small,pass,pass,pass,pass,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
 xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
+convnextv2_nano.fcmae_ft_in22k_in1k,pass,pass,pass,pass,pass
+deit_tiny_patch16_224.fb_in1k,pass,pass,pass,pass,pass
+vit_base_patch14_dinov2.lvd142m,pass,pass,pass,pass,pass
+vit_base_patch16_siglip_256,pass,pass,pass,pass,pass

.github/scripts/e2e_summary.sh

@@ -35,24 +35,24 @@ function get_acc_details() {
                         exit_label = 0;
                     }{
                         if ($0 ~/Real failed/){
-                            color="red";
+                            color="🔴";
                             exit_label++;
                         }else if ($0 ~/Expected failed/){
-                            color="blue";
+                            color="🔵";
                         }else if ($0 ~/Warning timeout/){
-                            color="orange";
+                            color="🟡";
                         }else if ($0 ~/New models/){
-                            color="blue";
+                            color="🔵";
                         }else if ($0 ~/Failed to passed/){
-                            color="green";
+                            color="🟢";
                             exit_label++;
                         }
                     }END{print color, exit_label}')
                     echo "${colorful}" >> /tmp/tmp-result.txt
                     context=$(find "${results_dir}" -name "*.csv" |\
                         grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4 |\
-                        awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}')
-                    eval "export ${mode}_${dtype}=${context}"
+                        awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("%s%s", c, $1)}}')
+                    eval "export ${mode}_${dtype}=\${context}"
                 done
             done
             accuracy_row="$(echo -e "<tr>
@@ -68,7 +68,7 @@ function get_acc_details() {
                     <td>${inference_float16}</td>
                     <td>${inference_amp_bf16}</td>
                     <td>${inference_amp_fp16}</td>
-                </tr>" |sed '/__color__/{s/__color__/\\color/g;s/_/\\_/g}'
+                </tr>"
             )"
             if [[ "${accuracy_row}" =~ "red" ]];then
                 echo "${accuracy_row}" |tee -a accuracy.details.html >> accuracy.regression.html
@@ -99,7 +99,7 @@ if [ "${accuracy}" -gt 0 ];then
 
 #### accuracy
 
-| Category | Total | Passed | Pass Rate | \$\${\\color{red}Failed}\$\$ |\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ |\$\${\\color{green}New Passed}\$\$ | \$\${\\color{blue}New Enabled}\$\$ | Not Run |
+| Category | Total | Passed | Pass Rate | Failed | Xfailed | Timeout | New Passed | New Enabled | Not Run |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
 EOF
 
@@ -124,18 +124,30 @@ EOF
             if($0 ~/Real failed/){
                 failed = $4;
                 failed_models = $5;
+                if(failed > 0){
+                    failed = "🔴"$4;
+                }
             }
             if($0 ~/Expected failed/){
                 xfail = $4;
                 xfail_models = $5;
+                if(xfail > 0){
+                    xfail = "🔵"$4;
+                }
             }
             if($0 ~/timeout/){
                 timeout = $4;
                 timeout_models = $5;
+                if(timeout > 0){
+                    timeout = "🟡"$4;
+                }
             }
             if($0 ~/Failed to passed/){
                 new_passed = $5;
                 new_passed_models = $6;
+                if(new_passed > 0){
+                    new_passed = "🟢"$4;
+                }
             }
             if($0 ~/Not run/){
                 not_run = $4;
@@ -144,9 +156,12 @@ EOF
             if($0 ~/New models/){
                 new_enabled = $3;
                 new_enabled_models = $4;
+                if(new_enabled > 0){
+                    new_enabled = "🔵"$4;
+                }
             }
         }END {
-            printf(" %d | %d | %s | %d | %d | %d | %d | %d | %d\n",
+            printf(" %s | %s | %s | %s | %s | %s | %s | %s | %s\n",
                 total, passed, pass_rate, failed, xfail, timeout, new_passed, new_enabled, not_run);
         }')"
         echo "| ${category} | ${test_result} |" >> accuracy.summary.html
@@ -177,21 +192,22 @@ if [ "${performance}" -gt 0 ];then
         --gcc "${GCC_VERSION}" --python "${python}" \
         --pytorch "${TORCH_BRANCH_ID}/${TORCH_COMMIT_ID}" --torch-xpu-ops "${TORCH_XPU_OPS_COMMIT:-"${GITHUB_SHA}"}"
 fi
+echo "performance_regression=${performance_regression}" >> ${GITHUB_OUTPUT}
 
 # Show result
 summary_file="e2e-test-result.html"
 cat > ${summary_file} << EOF
 
 #### Note:
-\$\${\\color{red}Red}\$\$: the failed cases which need look into
-\$\${\\color{green}Green}\$\$: the new passed cases which need update reference
-\$\${\\color{blue}Blue}\$\$: the expected failed or new enabled cases
-\$\${\\color{orange}Orange}\$\$: the warning cases
+🔴: the failed cases which need look into
+🟢: the new passed cases which need update reference
+🔵: the expected failed or new enabled cases
+🟡: the warning cases
 Empty means the cases NOT run
 
 $(
     if ((accuracy_regression + performance_regression > 0));then
-        echo -e "\n### Highlight regressions\n"
+        echo -e "\n### 🎯 Highlight regressions\n"
         if (( accuracy_regression > 0 ));then
             cat accuracy.regression.html
         fi
@@ -201,13 +217,13 @@ $(
     fi
 )
 
-### Summary
+### 📊 Summary
 
 $(cat accuracy.summary.html)
 
 $(cat performance.summary.html)
 
-### Details
+### 📖 Details
 
 <details><summary>View detailed result</summary>
 

.github/scripts/lintrunner.sh

@@ -24,7 +24,7 @@ if ! command -v lintrunner &> /dev/null; then
 fi
 
 # Ignoring errors in one specific run
-export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046 -e SC2076"
+export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046 -e SC2076 -e SC2034"
 
 # This has already been cached in the docker image
 lintrunner init 2> /dev/null