Merge pull request #170 from shanejbrown/scan-test

Move and retry flaky scan test

Author: shanejbrown

test_requirements.in

@@ -7,3 +7,4 @@ ruff>=0.1.7
 ruff-lsp>=0.0.45
 # Python 3.8 only supports up to 3.5.0
 pre-commit<3.6
+pytest-rerunfailures>=14.0

test_requirements.txt

@@ -35,6 +35,7 @@ nodeenv==1.8.0
 packaging==23.1
     # via
     #   pytest
+    #   pytest-rerunfailures
     #   ruff-lsp
 platformdirs==3.5.1
     # via virtualenv
@@ -49,11 +50,14 @@ pytest==7.3.1
     #   -r test_requirements.in
     #   pytest-cov
     #   pytest-randomly
+    #   pytest-rerunfailures
     #   pytest-xdist
 pytest-cov==4.1.0
     # via -r test_requirements.in
 pytest-randomly==3.12.0
     # via -r test_requirements.in
+pytest-rerunfailures==14.0
+    # via -r test_requirements.in
 pytest-xdist==3.3.1
     # via -r test_requirements.in
 pyyaml==6.0.1

tests/test-files/test-security-scan.yaml renamed to tests/test-files/scan/test-security-scan.yaml

@@ -19,7 +19,6 @@
     "test-general-buildx.yaml",
     "test-general.yaml",
     "test-push-artifact-buildx.yaml",
-    "test-security-scan.yaml",
 ]
 
 
@@ -186,14 +185,11 @@ def test_buildrunner_arm_dir(test_dir: str, file_name, args, exit_code):
     _test_buildrunner_file(test_dir, file_name, args, exit_code)
 
 
-@pytest.mark.serial
-@pytest.mark.skipif(
-    "arm64" not in platform.uname().machine,
-    reason="This test should only be run on arm64 architecture",
-)
+@pytest.mark.flaky(reruns=2, reruns_delay=1)
 @pytest.mark.parametrize(
     "test_dir, file_name, args, exit_code",
-    _get_test_runs(test_dir=f"{TEST_DIR}/test-files/arm-arch", serial_tests=True),
+    _get_test_runs(test_dir=f"{TEST_DIR}/test-files/scan", serial_tests=False),
 )
-def test_serial_buildrunner_arm_dir(test_dir: str, file_name, args, exit_code):
+def test_buildrunner_scan_dir(test_dir: str, file_name, args, exit_code):
+    # The scan tests can be flaky, with errors like "TOOMANYREQUESTS: retry-after: 804.543µs, allowed: 44000/minute"
     _test_buildrunner_file(test_dir, file_name, args, exit_code)