[visual-testing] Add extended failure message for visual checks (#6445)

Author: uarlouski

docs/modules/plugins/pages/plugin-visual.adoc

@@ -30,6 +30,12 @@ include::partial$plugin-installation.adoc[]
 |`<empty>`
 |Used to append scenario based index to a baseline name
 
+|`ui.visual.extended-assertion-format-enabled`
+a|`true`
+`false`
+|`false`
+|Extends assertion message for baseline validation by including baseline name
+
 |`ui.visual.override-baselines`
 a|`true`
 `false`

vividus-extension-visual-testing/src/main/java/org/vividus/visual/steps/AbstractVisualSteps.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2026 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,6 +37,8 @@ public abstract class AbstractVisualSteps
     private final IAttachmentPublisher attachmentPublisher;
     private final ISoftAssert softAssert;
 
+    private boolean extendedAssertionFormatEnabled;
+
     protected AbstractVisualSteps(IUiContext uiContext, IAttachmentPublisher attachmentPublisher,
             ISoftAssert softAssert)
     {
@@ -90,7 +92,22 @@ private <T extends AbstractVisualCheck, R extends VisualCheckResult> Optional<R>
     protected void verifyResult(VisualCheckResult result)
     {
         boolean passed = result.isPassed() ^ result.getActionType() == VisualActionType.CHECK_INEQUALITY_AGAINST;
-        softAssert.assertTrue("Visual check passed", passed);
+        if (extendedAssertionFormatEnabled)
+        {
+            String format = "Visual check for '%s' baseline is %s";
+            if (passed)
+            {
+                softAssert.recordPassedAssertion(format.formatted(result.getBaselineName(), "passed"));
+            }
+            else
+            {
+                softAssert.recordFailedAssertion(format.formatted(result.getBaselineName(), "failed"));
+            }
+        }
+        else
+        {
+            softAssert.assertTrue("Visual check passed", passed);
+        }
     }
 
     protected abstract String getTemplateName();
@@ -99,4 +116,9 @@ protected ISoftAssert getSoftAssert()
     {
         return softAssert;
     }
+
+    public void setExtendedAssertionFormatEnabled(boolean extendedAssertionFormatEnabled)
+    {
+        this.extendedAssertionFormatEnabled = extendedAssertionFormatEnabled;
+    }
 }

vividus-extension-visual-testing/src/main/resources/properties/defaults/default.properties

@@ -1 +1,2 @@
 ui.visual.indexer=
+ui.visual.extended-assertion-format-enabled=false

vividus-extension-visual-testing/src/main/resources/vividus-extension/spring.xml

@@ -6,6 +6,10 @@
         http://www.springframework.org/schema/util https://www.springframework.org/schema/util/spring-util.xsd"
        default-lazy-init="true">
 
+    <bean name="abstractVisualSteps" class="org.vividus.visual.steps.AbstractVisualSteps" abstract="true">
+        <property name="extendedAssertionFormatEnabled" value="${ui.visual.extended-assertion-format-enabled}" />
+    </bean>
+
     <beans profile="web_selenium,mobile_app">
         <bean id="scenario" class="org.vividus.visual.screenshot.ScenarioBasedIndexer" />
 

vividus-extension-visual-testing/src/test/java/org/vividus/visual/steps/VisualStepsTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2026 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ class VisualStepsTests
     private static final String VISUAL_CHECK_PASSED = "Visual check passed";
     private static final String COMPARE_AGAINST = "true, COMPARE_AGAINST";
     private static final String CHECK_INEQUALITY_AGAINST = "false, CHECK_INEQUALITY_AGAINST";
+    private static final String EXTENDED_MESSAGE_FORMAT = "Visual check for '%s' baseline is %s";
 
     @Mock private IUiContext uiContext;
     @Mock private IAttachmentPublisher attachmentPublisher;
@@ -150,6 +151,32 @@ void shouldPerformVisualCheckAndPublishAttachment(boolean passed, VisualActionTy
         assertNull(visualCheck.getSearchContext());
     }
 
+    @Test
+    void shouldUseExtendedAssertionFormatPassed()
+    {
+        var visualCheck = new AbstractVisualCheck(BASELINE_NAME, VisualActionType.COMPARE_AGAINST) { };
+        var visualCheckResult = new VisualCheckResult(visualCheck);
+        visualCheckResult.setPassed(true);
+
+        visualSteps.setExtendedAssertionFormatEnabled(true);
+        visualSteps.verifyResult(visualCheckResult);
+
+        verify(softAssert).recordPassedAssertion(EXTENDED_MESSAGE_FORMAT.formatted(BASELINE_NAME, "passed"));
+    }
+
+    @Test
+    void shouldUseExtendedAssertionFormatFailed()
+    {
+        var visualCheck = new AbstractVisualCheck(BASELINE_NAME, VisualActionType.COMPARE_AGAINST) { };
+        var visualCheckResult = new VisualCheckResult(visualCheck);
+        visualCheckResult.setPassed(false);
+
+        visualSteps.setExtendedAssertionFormatEnabled(true);
+        visualSteps.verifyResult(visualCheckResult);
+
+        verify(softAssert).recordFailedAssertion(EXTENDED_MESSAGE_FORMAT.formatted(BASELINE_NAME, "failed"));
+    }
+
     private static final class TestVisualSteps extends AbstractVisualSteps
     {
         private TestVisualSteps(IUiContext uiContext, IAttachmentPublisher attachmentPublisher, ISoftAssert softAssert)

vividus-plugin-applitools/src/main/resources/vividus-plugin/spring.xml

@@ -25,11 +25,11 @@
     <bean class="org.vividus.visual.eyes.converter.ExamplesTableToRenderingBrowserInfoConverter" />
     <bean class="org.vividus.visual.eyes.converter.StringToAccessibilitySettingsConverter" />
 
-    <bean id="visualTestingSteps" class="org.vividus.visual.eyes.VisualTestingSteps" >
+    <bean id="visualTestingSteps" class="org.vividus.visual.eyes.VisualTestingSteps" parent="abstractVisualSteps">
         <constructor-arg index="2" ref="screenshotParametersFactory" />
     </bean>
 
-    <bean id="ufgSteps" class="org.vividus.visual.eyes.UfgSteps" />
+    <bean id="ufgSteps" class="org.vividus.visual.eyes.UfgSteps" parent="abstractVisualSteps"/>
 
     <bean id="eyesLogHandler" class="org.vividus.visual.eyes.logger.EyesLogHandler">
         <constructor-arg value="org.vividus.visual.eyes.service.ImageVisualTestingService" />

vividus-plugin-visual/src/main/resources/vividus-plugin/spring.xml

@@ -8,7 +8,7 @@
 
     <context:component-scan base-package="org.vividus.visual.storage" />
 
-    <bean id="visualSteps" class="org.vividus.visual.VisualSteps">
+    <bean id="visualSteps" class="org.vividus.visual.VisualSteps" parent="abstractVisualSteps">
         <constructor-arg ref="screenshotParametersFactory" />
     </bean>