Remove PyPy from CI/build matrices (#47286)

danieljurek · web-flow · commit a99f5ae486bb · 2026-06-04T09:50:34.000-07:00
* First cut: Remove pypy from engsys, there are still references in dev_requirements.txt that could be cleaned up but those intersect with local development scenarios as well

* Consolidate ML matrix (it's the same as the default matrix now)
diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
@@ -25,17 +25,6 @@
         }
       }
     },
-    {
-      "PyPyConfig": {
-        "ubuntu2404_pypy311": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "pypy3.11",
-          "CoverageArg": "--disablecov",
-          "TestSamples": "false"
-        }
-      }
-    },
     {
       "CoverageConfig": {
         "ubuntu2404_310_coverage": {
diff --git a/eng/pipelines/templates/steps/use-python-version.yml b/eng/pipelines/templates/steps/use-python-version.yml
@@ -12,60 +12,6 @@ steps:
       }
     displayName: 'Use Public Feed'
 
-  # Optional override for the hosted toolcache PyPy build. Activated when the
-  # matrix entry sets:
-  #   OverridePyPyVersion: "true"
-  #   PyPyBuildVersion: "v7.3.22"   (or any release from downloads.python.org/pypy)
-  #   PyPyLangVersion:  "3.11.15"   (the Python language version directory in the toolcache)
-  # Rationale: PyPy 7.3.21 (currently in the hosted toolcache for Python 3.11.15)
-  # crashes with `Fatal RPython error: AssertionError` from
-  # `PythonCodeMaker_duplicate_exits_without_lineno` while compiling
-  # `azure.ai.ml`. PyPy 7.3.22+ does not exhibit this bug.
-  - bash: |
-      set -euo pipefail
-      PYPY_BUILD="${PYPY_BUILD_VERSION:?PyPyBuildVersion must be set in the matrix when OverridePyPyVersion=true}"
-      PY_LANG="${PYPY_LANG_VERSION:?PyPyLangVersion must be set in the matrix when OverridePyPyVersion=true}"
-      TARGET="/opt/hostedtoolcache/PyPy/${PY_LANG}/x64"
-      WANT_MICRO="${PYPY_BUILD#v}"
-      if [ -x "${TARGET}/bin/python3" ]; then
-        CURRENT=$("${TARGET}/bin/python3" -c 'import sys; v=sys.pypy_version_info; print(f"{v.major}.{v.minor}.{v.micro}")') || CURRENT="unknown"
-        echo "Existing toolcache PyPy: ${CURRENT}"
-        if [ "$CURRENT" = "$WANT_MICRO" ]; then
-          echo "PyPy ${WANT_MICRO} already installed; nothing to do."
-          exit 0
-        fi
-      fi
-      echo "Installing PyPy ${PYPY_BUILD} for Python ${PY_LANG} into ${TARGET}..."
-      TARBALL="pypy3.11-${PYPY_BUILD}-linux64.tar.bz2"
-      cd /tmp
-      curl -sSLO "https://downloads.python.org/pypy/${TARBALL}"
-      tar xjf "${TARBALL}"
-      sudo rm -rf "${TARGET}"
-      sudo mkdir -p "$(dirname "${TARGET}")"
-      sudo mv "pypy3.11-${PYPY_BUILD}-linux64" "${TARGET}"
-      # Recreate the python3/python symlinks that the upstream toolcache layout provides.
-      if [ ! -e "${TARGET}/bin/python3" ]; then
-        sudo ln -sf "${TARGET}/bin/pypy3.11" "${TARGET}/bin/python3"
-      fi
-      if [ ! -e "${TARGET}/bin/python" ]; then
-        sudo ln -sf "${TARGET}/bin/pypy3.11" "${TARGET}/bin/python"
-      fi
-      sudo touch "${TARGET}.complete"
-      "${TARGET}/bin/python3" --version
-      # The plain pypy tarball does not include pip; the Azure hosted
-      # toolcache build does. Bootstrap pip via ensurepip so downstream
-      # steps (e.g. `python -m pip install uv`) keep working.
-      if ! "${TARGET}/bin/python3" -c "import pip" >/dev/null 2>&1; then
-        echo "Bootstrapping pip via ensurepip..."
-        sudo "${TARGET}/bin/python3" -m ensurepip --upgrade
-      fi
-      "${TARGET}/bin/python3" -m pip --version
-    env:
-      PYPY_BUILD_VERSION: $(PyPyBuildVersion)
-      PYPY_LANG_VERSION: $(PyPyLangVersion)
-    displayName: 'Override hosted toolcache PyPy build'
-    condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['OverridePyPyVersion'], 'true'))
-
   - task: UsePythonVersion@0
     displayName: "Use Python ${{ parameters.versionSpec }}"
     inputs:
diff --git a/eng/tools/azure-sdk-tools/ci_tools/functions.py b/eng/tools/azure-sdk-tools/ci_tools/functions.py
@@ -68,18 +68,7 @@
 PATHS_EXCLUDED_FROM_DISCOVERY = []
 
 TEST_COMPATIBILITY_MAP = {"azure-ai-ml": ">=3.7"}
-TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {
-    "azure-storage-blob": "pypy",
-    "azure-storage-queue": "pypy",
-    "azure-storage-file-datalake": "pypy",
-    "azure-storage-file-share": "pypy",
-    "azure-eventhub": "pypy",
-    "azure-servicebus": "pypy",
-    "azure-ai-projects": "pypy",
-    "azure-ai-agents": "pypy",
-    "azure-identity-broker": "pypy",
-    "azure-storage-extensions": "pypy",
-}
+TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {}
 
 omit_regression = (
     lambda x: "nspkg" not in x
diff --git a/sdk/agentserver/ci.yml b/sdk/agentserver/ci.yml
@@ -32,8 +32,6 @@ extends:
     TestProxy: true
     BuildDocs: true
     TestTimeoutInMinutes: 60
-    # The job "Test ubuntu2404_pypy311" in the "python - agentserver" pipeline hangs and eventually times out.
-    # Disable it until the issue is understood.
     MatrixConfigs:
       - Name: agentserver
         Path: sdk/agentserver/platform-matrix.json
diff --git a/sdk/agentserver/tests.yml b/sdk/agentserver/tests.yml
@@ -12,9 +12,6 @@ extends:
       ServiceDirectory: agentserver
       TestMarkArgument: tracing_e2e
       TestTimeoutInMinutes: 60
-      MatrixFilters:
-        - PythonVersion=^(?!pypy3).*
       EnvVars:
         AZURE_SKIP_LIVE_RECORDING: 'true'
         AZURE_TEST_RUN_LIVE: 'true'
-
diff --git a/sdk/ai/ci.yml b/sdk/ai/ci.yml
@@ -32,8 +32,6 @@ extends:
     TestProxy: true
     BuildDocs: true
     TestTimeoutInMinutes: 60
-    # The job "Test ubuntu2404_pypy39" in the "python - ai" pipeline hangs and eventually times out.
-    # Disable it until the issue is understood.
     Artifacts:
     - name: azure-ai-projects
       safeName: azureaiprojects
diff --git a/sdk/communication/azure-communication-phonenumbers/phonenumbers-livetest-matrix.json b/sdk/communication/azure-communication-phonenumbers/phonenumbers-livetest-matrix.json
@@ -24,7 +24,7 @@
         "COMMUNICATION_SKIP_CAPABILITIES_LIVE_TEST": "false"
       }
     },
-    "PythonVersion": ["pypy3.11", "3.11", "3.12"],
+    "PythonVersion": ["3.11", "3.12"],
     "CoverageArg": "--disablecov",
     "TestSamples": "false"
   },
diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json
@@ -88,18 +88,6 @@
         }
       }
     },
-    {
-      "PyPyConfig": {
-        "ubuntu2404_pypy311_query": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "pypy3.11",
-          "CoverageArg": "--disablecov",
-          "TestSamples": "false",
-          "TestMarkArgument": "cosmosQuery"
-        }
-      }
-    },
     {
       "CoverageConfig": {
         "ubuntu2404_310_coverage_query": {
diff --git a/sdk/eventhub/ci.yml b/sdk/eventhub/ci.yml
@@ -29,8 +29,6 @@ extends:
   template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
   parameters:
     ServiceDirectory: eventhub
-    MatrixFilters:
-      - PythonVersion=^(?!pypy3).*
     Artifacts:
     - name: azure-eventhub
       safeName: azureeventhub
diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml
@@ -6,8 +6,6 @@ extends:
       ServiceDirectory: eventhub
       TestTimeoutInMinutes: 240
       BuildTargetingString: azure-eventhub*
-      MatrixFilters:
-        - PythonVersion=^(?!pypy3).*
       EnvVars:
         AZURE_TEST_RUN_LIVE: 'true'
         AZURE_SUBSCRIPTION_ID: $(EVENTHUB_SUBSCRIPTION_ID)
diff --git a/sdk/ml/ci.yml b/sdk/ml/ci.yml
@@ -32,11 +32,6 @@ extends:
     ServiceDirectory: ml
     TestTimeoutInMinutes: 75
     TestProxy: true
-    MatrixConfigs:
-      - Name: Python_ml_ci_matrix
-        Path: sdk/ml/platform-matrix.json
-        Selection: sparse
-        GenerateVMJobs: true
     # This is a short term solution to create API review for python azure-ml package only when running pipeline manually
     # Long term solution should be to have different versions on main branch and release branch for python package so APIView can have different revisions for each version.
     # Tracking issue: https://github.com/Azure/azure-sdk-for-python/issues/29196
diff --git a/sdk/ml/platform-matrix.json b/sdk/ml/platform-matrix.json
diff --git a/sdk/planetarycomputer/ci.yml b/sdk/planetarycomputer/ci.yml
@@ -29,17 +29,11 @@ extends:
   parameters:
     ServiceDirectory: planetarycomputer
     TestProxy: true
-    # The job "Test ubuntu2404_pypy39" fails due to nh3 (required by readme_renderer==43.0) 
-    # not being compatible with PyPy 3.9. nh3 requires PyPy 3.11+.
-    # Using custom matrix to filter out PyPy tests until PyPy is upgraded to 3.11+.
-    # Tracking issue: #43708
     MatrixConfigs:
       - Name: planetarycomputer_ci_matrix
         Path: sdk/planetarycomputer/platform-matrix.json
         Selection: sparse
         GenerateVMJobs: true
-    MatrixFilters:
-      - PythonVersion=^(?!pypy3).*
     Artifacts:
     - name: azure-mgmt-planetarycomputer
       safeName: azuremgmtplanetarycomputer
diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml
@@ -9,5 +9,3 @@ extends:
       EnvVars:
         AZURE_SKIP_LIVE_RECORDING: 'True'
         AZURE_TEST_RUN_LIVE: 'true'
-      MatrixFilters:
-        - '"PythonVersion=^(?!pypy3).*"'
diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml
@@ -30,11 +30,8 @@ extends:
   parameters:
     ServiceDirectory: servicebus
     TestProxy: true
-    MatrixFilters:
-      - PythonVersion=^(?!pypy3).*
     Artifacts:
     - name: azure-mgmt-servicebus
       safeName: azuremgmtservicebus
     - name: azure-servicebus
       safeName: azureservicebus
-
diff --git a/sdk/servicebus/tests.yml b/sdk/servicebus/tests.yml
@@ -10,5 +10,3 @@ extends:
         AZURE_SUBSCRIPTION_ID: $(SERVICEBUS_SUBSCRIPTION_ID)
         AZURE_TEST_RUN_LIVE: 'true'
         AZURE_SKIP_LIVE_RECORDING: 'True'
-      MatrixFilters:
-        - PythonVersion=^(?!pypy3).*
diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml
@@ -39,8 +39,6 @@ extends:
     ServiceDirectory: storage
     TestProxy: true
     TestTimeoutInMinutes: 120
-    MatrixFilters:
-      - PythonVersion=^(?!pypy3).*
     Artifacts:
     - name: azure-storage-blob
       safeName: azurestorageblob

Original file line number	Diff line number	Diff line change
`@@ -25,17 +25,6 @@`
`25`	`25`	`}`
`26`	`26`	`}`
`27`	`27`	`},`
`28`		`- {`
`29`		`- "PyPyConfig": {`
`30`		`- "ubuntu2404_pypy311": {`
`31`		`- "OSVmImage": "env:LINUXVMIMAGE",`
`32`		`- "Pool": "env:LINUXPOOL",`
`33`		`- "PythonVersion": "pypy3.11",`
`34`		`- "CoverageArg": "--disablecov",`
`35`		`- "TestSamples": "false"`
`36`		`- }`
`37`		`- }`
`38`		`- },`
`39`	`28`	`{`
`40`	`29`	`"CoverageConfig": {`
`41`	`30`	`"ubuntu2404_310_coverage": {`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@`
`24`	`24`	`"COMMUNICATION_SKIP_CAPABILITIES_LIVE_TEST": "false"`
`25`	`25`	`}`
`26`	`26`	`},`
`27`		`- "PythonVersion": ["pypy3.11", "3.11", "3.12"],`
	`27`	`+ "PythonVersion": ["3.11", "3.12"],`
`28`	`28`	`"CoverageArg": "--disablecov",`
`29`	`29`	`"TestSamples": "false"`
`30`	`30`	`},`
Original file line number	Diff line number	Diff line change
`@@ -88,18 +88,6 @@`
`88`	`88`	`}`
`89`	`89`	`}`
`90`	`90`	`},`
`91`		`- {`
`92`		`- "PyPyConfig": {`
`93`		`- "ubuntu2404_pypy311_query": {`
`94`		`- "OSVmImage": "env:LINUXVMIMAGE",`
`95`		`- "Pool": "env:LINUXPOOL",`
`96`		`- "PythonVersion": "pypy3.11",`
`97`		`- "CoverageArg": "--disablecov",`
`98`		`- "TestSamples": "false",`
`99`		`- "TestMarkArgument": "cosmosQuery"`
`100`		`- }`
`101`		`- }`
`102`		`- },`
`103`	`91`	`{`
`104`	`92`	`"CoverageConfig": {`
`105`	`93`	`"ubuntu2404_310_coverage_query": {`