CI Testing current MacOS failures

.github/workflows/tests.yml

@@ -42,7 +42,7 @@ jobs:
       # TODO: remove 'fail-fast' line once timeout issue from the Hub is solved
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9"]
         os: ["ubuntu-latest", "macos-13", "windows-latest"]
     runs-on: ${{ matrix.os }}
     steps:

tests/conftest.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import platform
+import re
+
 import pytest
 
 
@@ -31,3 +34,41 @@ def pytest_collection_modifyitems(config, items):
     for item in items:
         if "regression" in item.keywords:
             item.add_marker(skip_regression)
+
+
+# TODO: remove this once support for PyTorch 2.2 (the latest one still supported by GitHub MacOS x86_64 runners) is
+# dropped, see https://github.com/huggingface/peft/issues/2431.
+# note: the function name is fixed by the pytest plugin system, don't change it
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    """
+    Plug into the pytest test report generation to skip a specific MacOS failure caused by transformers.
+
+    Failure 0 was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load
+    failing when using torch < 2.6.
+
+    Failure 1 was introduced by https://github.com/huggingface/transformers/pull/37919, which results in a DTensor
+    import being triggered in the save_pretrained, which fails with MacOS and torch 2.2 (possibly also later MacOS
+    versions).
+
+    Since the MacOS x86 runners need to use an older torch version, those steps are necessary to get the CI green.
+    """
+    outcome = yield
+    rep = outcome.get_result()
+    # ref:
+    # https://github.com/huggingface/transformers/blob/858ce6879a4aa7fa76a7c4e2ac20388e087ace26/src/transformers/utils/import_utils.py#L1418
+    error_msg_0 = re.compile(r"Due to a serious vulnerability issue in `torch.load`")
+    error_msg_1 = re.compile(r"cannot import name 'DTensor' from 'torch.distributed.tensor'")
+
+    # notes:
+    # - pytest uses hard-coded strings, we cannot import and use constants
+    #   https://docs.pytest.org/en/stable/reference/reference.html#pytest.TestReport
+    # - errors can happen during call (running the test) but also setup (e.g. in fixtures)
+    if rep.failed and (rep.when in ("setup", "call")) and (platform.system() == "Darwin"):
+        exc_msg = str(call.excinfo.value)
+        if error_msg_0.search(exc_msg) or error_msg_1.search(exc_msg):
+            # turn this failure into an xfail:
+            rep.outcome = "skipped"
+            # for this attribute, see:
+            # https://github.com/pytest-dev/pytest/blob/bd6877e5874b50ee57d0f63b342a67298ee9a1c3/src/_pytest/reports.py#L266C5-L266C13
+            rep.wasxfail = "Errors known to occur on MacOS with older torch versions, won't be fixed"