diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 752870abe36b..5431f1d5163f 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -71,7 +71,8 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py && - pytest -v -s v1/test_oracle.py v1/test_request.py v1/test_outputs.py && + pytest -v -s v1/test_oracle.py v1/test_request.py && + pytest -v -s v1/test_outputs.py && pytest -v -s v1/sample' retry: automatic: diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh index 0c463b0eb005..1dc3537e9280 100755 --- a/.buildkite/scripts/hardware_ci/run-intel-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh @@ -378,7 +378,32 @@ export HF_TOKEN ZE_AFFINITY_MASK VLLM_DISABLE_COMPILE_CACHE -e CMDS \ --name "${container_name}" \ "${IMAGE}" \ - bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \ + bash -c 'set -e +echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}" +if [[ -n "${PYTEST_ADDOPTS:-}" ]]; then + # Tolerate exit 5 only when pytest-shard reports 0 items for this shard while the pre-shard selection (collected minus deselected) was non-empty; + # -k/-m filters that deselect everything still fail normally. + pytest() { + local ec=0 log collected deselected shard_count selected + log="$(mktemp)" + command pytest "$@" 2>&1 | tee "$log" + ec=${PIPESTATUS[0]} + if [[ $ec -eq 5 ]]; then + collected=$(grep -oE "collected [0-9]+ item" "$log" | grep -oE "[0-9]+" | tail -1) + deselected=$(grep -oE "[0-9]+ deselected" "$log" | grep -oE "[0-9]+" | tail -1) + shard_count=$(grep -oE "Running [0-9]+ items? in this shard" "$log" | grep -oE "[0-9]+" | tail -1) + selected=$(( ${collected:-0} - ${deselected:-0} )) + if [[ "${shard_count:-}" == "0" && $selected -gt 0 ]]; then + echo "pytest exited with status 5 but ${selected} item(s) were selected (0 assigned to this shard); treating as success." + rm -f "$log" + return 0 + fi + fi + rm -f "$log" + return $ec + } +fi +eval "$CMDS"' \ >/dev/null } 9>/tmp/docker-pull.lock