Fix vision build for 3.14 and 3.14t and 3.13t build (#7036)

I see 3.14 and 3.14t and 3.13t builds failing for vision:
https://github.com/pytorch/vision/actions/runs/17097200564/job/48484314516
This should fix it, prior to this PR:
#7022

Please see successful builds here:
https://github.com/pytorch/test-infra/actions/runs/17102204277/job/48501597067?pr=7036

Author: atalman

.github/actions/setup-binary-builds/action.yml

@@ -127,15 +127,21 @@ runs:
           CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
           export CONDA_EXTRA_PARAM=""
 
-          if [[ ( "${PYTHON_VERSION:-}" == *t || "${PYTHON_VERSION:-}" == 3.14* ) && "$(uname)" == Darwin ]]; then
-            # downgrade conda version for python 3.13t/3.14t install.
-            # TODO: remove this once python 3.13t/3.14t is fully supported on conda
+          if [[ "${PYTHON_VERSION:-}" == *t || "${PYTHON_VERSION:-}" == 3.14* ]]; then
+
+            # downgrade conda version for python 3.13t, 3.14t, 3.14 install.
+            # TODO: remove this once python  3.13t, 3.14t, 3.14 is fully supported on conda
             # Please see : https://github.com/conda/conda/issues/14554
-            conda install -y python=3.9
-            if [[ -n "$(conda list | grep conda-anaconda-telemetry)" ]]; then
-              conda uninstall -y conda-anaconda-telemetry conda-anaconda-tos
+            if [[ "$(uname)" == Darwin ]]; then
+              # required to be able to downgrade on MacOS arm64
+              conda install -y python=3.9
+              if [[ -n "$(conda list | grep conda-anaconda-telemetry)" ]]; then
+                conda uninstall -y conda-anaconda-telemetry conda-anaconda-tos
+              fi
+              conda install -y conda=24.7.1 conda-libmamba-solver=24.1.0
+            else
+              conda install -y conda=24.7.1
             fi
-            conda install -y conda=24.7.1 conda-libmamba-solver=24.1.0
           fi
 
           # shellcheck disable=SC2153