Skip to content

Commit 9ef9165

Browse files
mxschmittmxschmitt
authored
test: allow updating test snapshots via env (#2927)
1 parent c14ebe5 commit 9ef9165

13 files changed

+101
-113
lines changed

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,6 @@ MigrationBackup/
353353
.idea
354354
src/Playwright.sln.DotSettings
355355
src/Playwright/.drivers
356-
src/Playwright.Tests/Screenshots/**/test.png
357356
src/Playwright/Microsoft.Playwright.xml
358357

359358
testCert.cer

‎src/Playwright.Tests/BaseTests/PlaywrightAssert.cs

+53
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
using System.Text.Json;
2626
using NUnit.Framework.Constraints;
27+
using SixLabors.ImageSharp;
28+
using SixLabors.ImageSharp.PixelFormats;
2729

2830
namespace Microsoft.Playwright.Tests;
2931

@@ -50,4 +52,55 @@ internal static void AreJsonEqual(object expected, object actual)
5052
{
5153
Assert.AreEqual(JsonSerializer.Serialize(expected), JsonSerializer.Serialize(actual));
5254
}
55+
56+
internal static void ToMatchSnapshot(string expectedImageName, string actual)
57+
=> ToMatchSnapshot(expectedImageName, File.ReadAllBytes(actual));
58+
59+
internal static void ToMatchSnapshot(string expectedImageName, byte[] actual)
60+
{
61+
const int pixelThreshold = 10;
62+
const decimal totalTolerance = 0.05m;
63+
64+
var goldenPathDir = Path.Combine(TestUtils.FindParentDirectory("Screenshots"), TestConstants.BrowserName);
65+
var expectedImagePath = Path.Combine(goldenPathDir, expectedImageName);
66+
if (Environment.GetEnvironmentVariable("UPDATE_SNAPSHOTS") == "1")
67+
{
68+
File.WriteAllBytes(expectedImagePath, actual);
69+
}
70+
71+
var expectedImage = Image.Load<Rgb24>(expectedImagePath);
72+
var actualImage = Image.Load<Rgb24>(actual);
73+
74+
if (expectedImage.Width != actualImage.Width || expectedImage.Height != actualImage.Height)
75+
{
76+
Assert.Fail("Expected image dimensions do not match actual image dimensions.\n" +
77+
$"Expected: {expectedImage.Width}x{expectedImage.Height}\n" +
78+
$"Actual: {actualImage.Width}x{actualImage.Height}");
79+
return;
80+
}
81+
82+
int invalidPixelsCount = 0;
83+
for (int y = 0; y < expectedImage.Height; y++)
84+
{
85+
for (int x = 0; x < expectedImage.Width; x++)
86+
{
87+
var pixelA = expectedImage[x, y];
88+
var pixelB = actualImage[x, y];
89+
90+
91+
if (Math.Abs(pixelA.R - pixelB.R) > pixelThreshold ||
92+
Math.Abs(pixelA.G - pixelB.G) > pixelThreshold ||
93+
Math.Abs(pixelA.B - pixelB.B) > pixelThreshold)
94+
{
95+
invalidPixelsCount++;
96+
}
97+
}
98+
}
99+
// cast invalidPixelsCount to decimal to avoid integer division
100+
if (((decimal)invalidPixelsCount / (expectedImage.Height * expectedImage.Width)) > totalTolerance)
101+
{
102+
Assert.Fail($"Expected image to match snapshot but it did not. {invalidPixelsCount} pixels do not match.\n" +
103+
$"Set the environment variable 'UPDATE_SNAPSHOTS' to '1' to update the snapshot.");
104+
}
105+
}
53106
}

‎src/Playwright.Tests/ElementHandleScreenshotTests.cs

+13-13
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public async Task ShouldWork()
3737
await Page.EvaluateAsync("window.scrollBy(50, 100)");
3838
var elementHandle = await Page.QuerySelectorAsync(".box:nth-of-type(3)");
3939
byte[] screenshot = await elementHandle.ScreenshotAsync();
40-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", screenshot));
40+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", screenshot);
4141
}
4242

4343
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should take into account padding and border")]
@@ -56,7 +56,7 @@ await Page.SetContentAsync(@"
5656
<div id=""d""></div>");
5757
var elementHandle = await Page.QuerySelectorAsync("div#d");
5858
byte[] screenshot = await elementHandle.ScreenshotAsync();
59-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-padding-border.png", screenshot));
59+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-padding-border.png", screenshot);
6060
}
6161

6262
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should capture full element when larger than viewport in parallel")]
@@ -84,7 +84,7 @@ await Page.SetContentAsync(@"
8484
var screenshotTasks = elementHandles.Select(e => e.ScreenshotAsync()).ToArray();
8585
await TaskUtils.WhenAll(screenshotTasks);
8686

87-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-larger-than-viewport.png", screenshotTasks.ElementAt(2).Result));
87+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-larger-than-viewport.png", screenshotTasks.ElementAt(2).Result);
8888
}
8989

9090
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should capture full element when larger than viewport")]
@@ -110,7 +110,7 @@ await Page.SetContentAsync(@"
110110

111111
var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
112112
byte[] screenshot = await elementHandle.ScreenshotAsync();
113-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-larger-than-viewport.png", screenshot));
113+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-larger-than-viewport.png", screenshot);
114114
await TestUtils.VerifyViewportAsync(Page, 500, 500);
115115
}
116116

@@ -136,7 +136,7 @@ await Page.SetContentAsync(@"
136136
<div class=""to-screenshot""></div>");
137137
var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
138138
byte[] screenshot = await elementHandle.ScreenshotAsync();
139-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-scrolled-into-view.png", screenshot));
139+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-scrolled-into-view.png", screenshot);
140140
}
141141

142142
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should scroll 15000px into view")]
@@ -161,7 +161,7 @@ await Page.SetContentAsync(@"
161161
<div class=""to-screenshot""></div>");
162162
var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
163163
byte[] screenshot = await elementHandle.ScreenshotAsync();
164-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-scrolled-into-view.png", screenshot));
164+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-scrolled-into-view.png", screenshot);
165165
}
166166

167167
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with a rotated element")]
@@ -179,7 +179,7 @@ await Page.SetContentAsync(@"
179179
");
180180
var elementHandle = await Page.QuerySelectorAsync("div");
181181
byte[] screenshot = await elementHandle.ScreenshotAsync();
182-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-rotate.png", screenshot));
182+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-rotate.png", screenshot);
183183
}
184184

185185
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should fail to screenshot a detached element")]
@@ -221,7 +221,7 @@ public async Task ShouldWaitForVisible()
221221
await elementHandle.EvaluateAsync("e => e.style.visibility = 'visible'");
222222

223223
byte[] screenshot = await task;
224-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", screenshot));
224+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", screenshot);
225225
}
226226

227227
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work for an element with fractional dimensions")]
@@ -230,7 +230,7 @@ public async Task ShouldWorkForAnElementWithFractionalDimensions()
230230
await Page.SetContentAsync("<div style=\"width:48.51px;height:19.8px;border:1px solid black;\"></div>");
231231
var elementHandle = await Page.QuerySelectorAsync("div");
232232
byte[] screenshot = await elementHandle.ScreenshotAsync();
233-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-fractional.png", screenshot));
233+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-fractional.png", screenshot);
234234
}
235235

236236
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with a mobile viewport")]
@@ -252,7 +252,7 @@ public async Task ShouldWorkWithAMobileViewport()
252252
var elementHandle = await page.QuerySelectorAsync(".box:nth-of-type(3)");
253253
byte[] screenshot = await elementHandle.ScreenshotAsync();
254254

255-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-mobile.png", screenshot));
255+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-mobile.png", screenshot);
256256
}
257257

258258
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with device scale factor")]
@@ -274,7 +274,7 @@ public async Task ShouldWorkWithDeviceScaleFactor()
274274
var elementHandle = await page.QuerySelectorAsync(".box:nth-of-type(3)");
275275
byte[] screenshot = await elementHandle.ScreenshotAsync();
276276

277-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-mobile-dsf.png", screenshot));
277+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-mobile-dsf.png", screenshot);
278278
}
279279

280280
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work for an element with an offset")]
@@ -283,7 +283,7 @@ public async Task ShouldWorkForAnElementWithAnOffset()
283283
await Page.SetContentAsync("<div style=\"position:absolute; top: 10.3px; left: 20.4px;width:50.3px;height:20.2px;border:1px solid black;\"></div>");
284284
var elementHandle = await Page.QuerySelectorAsync("div");
285285
byte[] screenshot = await elementHandle.ScreenshotAsync();
286-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-fractional-offset.png", screenshot));
286+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-fractional-offset.png", screenshot);
287287
}
288288

289289
[PlaywrightTest("elementhandle-screenshot.spec.ts", "should take screenshots when default viewport is null")]
@@ -399,6 +399,6 @@ public async Task PathOptionShouldCreateSubdirectories()
399399
using var tmpDir = new TempDirectory();
400400
string outputPath = Path.Combine(tmpDir.Path, "these", "are", "directories", "screenshot.png");
401401
await elementHandle.ScreenshotAsync(new() { Path = outputPath });
402-
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", outputPath));
402+
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", outputPath);
403403
}
404404
}

‎src/Playwright.Tests/Helpers/ScreenshotHelper.cs

-72
This file was deleted.

‎src/Playwright.Tests/PageRequestFulfillTests.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
namespace Microsoft.Playwright.Tests;
2929

30-
public class RequestFulfillTests : PageTestEx
30+
public class PageRequestFulfillTests : PageTestEx
3131
{
3232
[PlaywrightTest("page-request-fulfill.spec.ts", "should work")]
3333
public async Task ShouldWork()
@@ -88,7 +88,7 @@ await Page.EvaluateAsync(@"PREFIX => {
8888
return new Promise(fulfill => img.onload = fulfill);
8989
}", Server.Prefix);
9090
var img = await Page.QuerySelectorAsync("img");
91-
Assert.True(ScreenshotHelper.PixelMatch("mock-binary-response.png", await img.ScreenshotAsync()));
91+
PlaywrightAssert.ToMatchSnapshot("mock-binary-response.png", await img.ScreenshotAsync());
9292
}
9393

9494
[PlaywrightTest("page-request-fulfill.spec.ts", "should allow mocking svg with charset")]
@@ -116,7 +116,7 @@ await Page.EvaluateAsync(@"PREFIX => {
116116
return new Promise(fulfill => img.onload = fulfill);
117117
}", Server.Prefix);
118118
var img = await Page.QuerySelectorAsync("img");
119-
Assert.True(ScreenshotHelper.PixelMatch("mock-binary-response.png", await img.ScreenshotAsync()));
119+
PlaywrightAssert.ToMatchSnapshot("mock-binary-response.png", await img.ScreenshotAsync());
120120
}
121121

122122
[PlaywrightTest("page-request-fulfill.spec.ts", "should stringify intercepted request response headers")]

0 commit comments

Comments
 (0)