From 67a959ceb7a3bfa205324eb66b59ee28bde10c1d Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 30 May 2025 17:02:31 +0200 Subject: [PATCH 1/2] CI Handle 2 errors with MacOS and transformers Changes in transformers introduced 2 errors in the MacOS CI, which are handled in this PR. Context For context on why we use torch 2.2 for MacOS, check #2431. Unfortunately, as of today, the available GH workers for MacOS still haven't improved. Description The 1st error was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load failing when using torch < 2.6. The 2nd error was introduced by https://github.com/huggingface/transformers/pull/37919, which results in a DTensor import being triggered when calling save_pretrained, which fails with MacOS and torch 2.2 (possibly also later MacOS versions, I haven't checked). The proposed solution is to plug into pytest, intercept the test report, check for these specific errors, and turn them into skips. Alternative solutions The proposed solution is obviously an ugly hack. However, these are errors we cannot fix directly, as they're caused by a dependency and are caused by the old torch version we're forced to use (thus fixing them in transformers is probably not an option). Instead of altering the test report, the individual tests that fail could get an explicit skip marker when MacOS is detected. However, since the amount of affected tests are several hundreds, this is very impractical and leads to a lot of noise in the tests. Alternatively, we could move forward with the proposal in #2431 and remove MacOS completely from the CI. I do, however, still have the faint hope that GH will provide arm64 workers with more RAM in the future, allowing us to switch. --- tests/conftest.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5c2d14cbeb..5db10b7ed7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import platform +import re + import pytest @@ -31,3 +34,41 @@ def pytest_collection_modifyitems(config, items): for item in items: if "regression" in item.keywords: item.add_marker(skip_regression) + + +# TODO: remove this once support for PyTorch 2.2 (the latest one still supported by GitHub MacOS x86_64 runners) is +# dropped, or if MacOS is removed from the test matrix, see https://github.com/huggingface/peft/issues/2431. +# Note: the function name is fixed by the pytest plugin system, don't change it +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Plug into the pytest test report generation to skip a specific MacOS failure caused by transformers. + + Failure 0 was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load + failing when using torch < 2.6. + + Failure 1 was introduced by https://github.com/huggingface/transformers/pull/37919, which results in a DTensor + import being triggered in the save_pretrained, which fails with MacOS and torch 2.2 (possibly also later MacOS + versions). + + Since the MacOS x86 runners need to use an older torch version, those steps are necessary to get the CI green. + """ + outcome = yield + rep = outcome.get_result() + # ref: + # https://github.com/huggingface/transformers/blob/858ce6879a4aa7fa76a7c4e2ac20388e087ace26/src/transformers/utils/import_utils.py#L1418 + error_msg_0 = re.compile(r"Due to a serious vulnerability issue in `torch.load`") + error_msg_1 = re.compile(r"cannot import name 'DTensor' from 'torch.distributed.tensor'") + + # notes: + # - pytest uses hard-coded strings, we cannot import and use constants + # https://docs.pytest.org/en/stable/reference/reference.html#pytest.TestReport + # - errors can happen during call (running the test) but also setup (e.g. in fixtures) + if rep.failed and (rep.when in ("setup", "call")) and (platform.system() == "Darwin"): + exc_msg = str(call.excinfo.value) + if error_msg_0.search(exc_msg) or error_msg_1.search(exc_msg): + # turn this failure into an xfail: + rep.outcome = "skipped" + # for this attribute, see: + # https://github.com/pytest-dev/pytest/blob/bd6877e5874b50ee57d0f63b342a67298ee9a1c3/src/_pytest/reports.py#L266C5-L266C13 + rep.wasxfail = "Errors known to occur on MacOS with older torch versions, won't be fixed" From aea21951b63a3f7d52184f584139ec538edf483f Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 30 May 2025 17:56:24 +0200 Subject: [PATCH 2/2] Remove workaround for error 2, fix in transformers --- tests/conftest.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5db10b7ed7..e4ee39b69e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,21 +44,16 @@ def pytest_runtest_makereport(item, call): """ Plug into the pytest test report generation to skip a specific MacOS failure caused by transformers. - Failure 0 was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load + The error was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load failing when using torch < 2.6. - Failure 1 was introduced by https://github.com/huggingface/transformers/pull/37919, which results in a DTensor - import being triggered in the save_pretrained, which fails with MacOS and torch 2.2 (possibly also later MacOS - versions). - Since the MacOS x86 runners need to use an older torch version, those steps are necessary to get the CI green. """ outcome = yield rep = outcome.get_result() # ref: # https://github.com/huggingface/transformers/blob/858ce6879a4aa7fa76a7c4e2ac20388e087ace26/src/transformers/utils/import_utils.py#L1418 - error_msg_0 = re.compile(r"Due to a serious vulnerability issue in `torch.load`") - error_msg_1 = re.compile(r"cannot import name 'DTensor' from 'torch.distributed.tensor'") + error_msg = re.compile(r"Due to a serious vulnerability issue in `torch.load`") # notes: # - pytest uses hard-coded strings, we cannot import and use constants @@ -66,9 +61,9 @@ def pytest_runtest_makereport(item, call): # - errors can happen during call (running the test) but also setup (e.g. in fixtures) if rep.failed and (rep.when in ("setup", "call")) and (platform.system() == "Darwin"): exc_msg = str(call.excinfo.value) - if error_msg_0.search(exc_msg) or error_msg_1.search(exc_msg): + if error_msg.search(exc_msg): # turn this failure into an xfail: rep.outcome = "skipped" # for this attribute, see: # https://github.com/pytest-dev/pytest/blob/bd6877e5874b50ee57d0f63b342a67298ee9a1c3/src/_pytest/reports.py#L266C5-L266C13 - rep.wasxfail = "Errors known to occur on MacOS with older torch versions, won't be fixed" + rep.wasxfail = "Error known to occur on MacOS with older torch versions, won't be fixed"