test: capture all screenshot

Author: jourdain

tests/conftest.py

@@ -18,7 +18,7 @@
 
 class Utils:
     @staticmethod
-    async def assert_screenshot(page, baseline_image, result_directory, threshold=0.1):
+    async def compare_screenshot(page, baseline_image, result_directory, threshold=0.1):
         test_image = result_directory / baseline_image.name
         await page.screenshot(path=test_image)
 
@@ -29,7 +29,7 @@ async def assert_screenshot(page, baseline_image, result_directory, threshold=0.
         file_diff = (test_image.parent / test_image.stem).with_suffix(".diff.png")
         mismatch = pixelmatch(img_ref, img_test, img_diff, threshold=threshold)
         img_diff.save(file_diff)
-        assert mismatch < threshold
+        return mismatch < threshold
 
 
 def create_vtk_pipeline():

tests/test_cone.py

@@ -34,6 +34,7 @@ async def test_cone(ConeApp, utils, config):
     task = app.server.start(exec_mode="task", port=0)
     await app.server.ready
     RESULT_BASE = Path(__file__).with_name("results") / "cone" / conf_key
+    valid_image_comparisons = []
 
     async with async_playwright() as p:
         browser = await p.chromium.launch(headless=True)
@@ -42,20 +43,38 @@ async def test_cone(ConeApp, utils, config):
 
         await page.goto(f"http://localhost:{app.server.port}/")
         await expect(page.locator(".readyCount")).to_have_text("1")
-        await utils.assert_screenshot(page, BASELINES[0], RESULT_BASE, threshold=0.1)
+        valid_image_comparisons.append(
+            await utils.compare_screenshot(
+                page, BASELINES[0], RESULT_BASE, threshold=0.1
+            )
+        )
 
         app.resolution = 60
         await expect(page.locator(".readyCount")).to_have_text("2")
-        await utils.assert_screenshot(page, BASELINES[1], RESULT_BASE, threshold=0.1)
+        valid_image_comparisons.append(
+            await utils.compare_screenshot(
+                page, BASELINES[1], RESULT_BASE, threshold=0.1
+            )
+        )
 
         app.mounted = False
         app.resolution = 4
-        await utils.assert_screenshot(page, BASELINES[2], RESULT_BASE, threshold=0.1)
+        valid_image_comparisons.append(
+            await utils.compare_screenshot(
+                page, BASELINES[2], RESULT_BASE, threshold=0.1
+            )
+        )
 
         app.mounted = True
         await asyncio.sleep(0.1)  # Debounced resize needs complete
         await expect(page.locator(".readyCount")).to_have_text("3")
-        await utils.assert_screenshot(page, BASELINES[3], RESULT_BASE, threshold=0.1)
+        valid_image_comparisons.append(
+            await utils.compare_screenshot(
+                page, BASELINES[3], RESULT_BASE, threshold=0.1
+            )
+        )
+
+        assert all(valid_image_comparisons), "Some images don't match"
 
         # Clean up resource
         await browser.close()