pin test-infra-ref (#1402)

Co-authored-by: Andrew Ho <[email protected]>

Author: andrewkho

.github/workflows/build_wheels_linux.yml

@@ -24,7 +24,7 @@ jobs:
       package-type: wheel
       os: linux
       test-infra-repository: pytorch/test-infra
-      test-infra-ref: main
+      test-infra-ref: release/2.5
       with-cuda: disable
       with-rocm: disable
   build: