Skip to content

Commit 5c6da6c

Browse files
authored
[Testing] Add Material3 test category and separate pipeline stage for android material3 UI tests (#33409)
This pull request adds support for running UI tests specifically targeting Material3 components on Android. It introduces new pipeline parameters and stages to enable Material3 builds and tests, updates artifact publishing logic to handle Material3-specific outputs, and adds a sample test case for a Material3 CheckBox. The changes ensure that Material3 tests are built, executed, and their results published separately from standard tests. **Pipeline and Build Enhancements:** * Added a new `useMaterial3` parameter to `ui-tests-build-sample.yml` and updated the build script to enable Material3 builds for Android when this parameter is set. [[1]](diffhunk://#diff-5d0763fa79fb5e8f8d535acf38d8796d9450d36e0e320f41f99c38f686641840R14) [[2]](diffhunk://#diff-5d0763fa79fb5e8f8d535acf38d8796d9450d36e0e320f41f99c38f686641840L58-R69) * Modified artifact publishing logic in `ui-tests-build-sample.yml` to handle Material3-specific build and failure artifacts, ensuring separation from standard artifacts. **Pipeline Stage and Test Execution Updates:** * Added dedicated build and test stages for Material3 in `ui-tests.yml`, including a new build stage (`build_ui_tests_material3`) for Android and a new test stage (`android_ui_tests_material3`) that downloads and runs Material3-specific tests. [[1]](diffhunk://#diff-8f76930e2126f6e5f60cae836de1df54dd41263baac4d4baf04789be312b257aR119-R137) [[2]](diffhunk://#diff-8f76930e2126f6e5f60cae836de1df54dd41263baac4d4baf04789be312b257aR228-R271) * Updated comments and category handling to clarify that Material3 has its own dedicated build and test stages. **Test Infrastructure and Sample Test:** * Added a new `Material3` test category to `UITestCategories.cs` for organizing Material3-specific tests. * Introduced a new sample XAML page (`IssueMaterial3CheckBox.xaml` and code-behind) and a corresponding UI test (`IssueMaterial3CheckBoxTest.cs`) to validate the default appearance of a Material3 CheckBox on Android. [[1]](diffhunk://#diff-dc0d79ae720ef99010bc997a04bb1fcf0deb3ee6655a2ce26deca504a9de3cb1R1-R22) [[2]](diffhunk://#diff-9edaec9a356096db6815ea7942b3ffba9c4ffe149d9ad953a05502d8e4ff072aR1-R13) [[3]](diffhunk://#diff-de155ddb12dcb2baa0d2f946517dfe0ad6fb19ff6832d0beadbff290421046bdR1-R25) **Pipeline stage:** <img width="668" height="121" alt="image" src="https://github.com/user-attachments/assets/a222fcdf-5c4b-46eb-bcb5-f3c623a084df" /> <img width="553" height="258" alt="image" src="https://github.com/user-attachments/assets/d5cb48ea-7cb0-4da0-99df-f24e085dc490" /> **Test execution result:** <img width="910" height="463" alt="image" src="https://github.com/user-attachments/assets/d5362e1a-b24d-4482-98d5-1730b1ee42ba" />
2 parents 2f64d2c + e1a8cc0 commit 5c6da6c

File tree

10 files changed

+192
-11
lines changed

10 files changed

+192
-11
lines changed

eng/cake/dotnet.cake

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,13 @@ Task("uitests-apphost")
218218
properties.Add("_UseNativeAot", "true");
219219
properties.Add("RuntimeIdentifier", "iossimulator-x64");
220220
}
221+
222+
var useMaterial3 = Argument("usematerial3", false);
223+
if (useMaterial3)
224+
{
225+
Information("Building with Material3 enabled");
226+
properties.Add("UseMaterial3", "true");
227+
}
221228

222229
if (useNuget)
223230
{

eng/pipelines/common/ui-tests-build-sample.yml

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ parameters:
1111
testFilter: ''
1212
headless: true
1313
runtimeVariant: 'Mono' #Mono, CoreCLR, NativeAOT
14+
useMaterial3: false # NEW PARAMETER - Enable Material3 build
1415

1516
steps:
1617
- ${{ if eq(parameters.platform, 'ios')}}:
@@ -52,9 +53,29 @@ steps:
5253
- pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)"
5354
displayName: 'Add .NET to PATH'
5455

56+
# Clean artifacts for Material3 build to ensure clean state
57+
# Reference: Controls.TestCases.HostApp.csproj line 17-19 warning comment
58+
- pwsh: |
59+
if (("${{ parameters.useMaterial3 }}" -eq "true") -and ("${{ parameters.platform }}" -eq "android")) {
60+
Write-Host "Cleaning artifacts folder for Material3 clean build"
61+
Remove-Item -Path "$(System.DefaultWorkingDirectory)/artifacts" -Recurse -Force -ErrorAction SilentlyContinue
62+
Write-Host "Artifacts folder cleaned successfully"
63+
}
64+
displayName: 'Clean artifacts for Material3'
65+
5566
- pwsh: |
5667
Get-Content $PSCommandPath
57-
./build.ps1 --target=uitests-apphost --configuration="${{ parameters.configuration }}" --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false --runtimevariant="${{ parameters.runtimeVariant }}"
68+
$buildCommand = "./build.ps1 --target=uitests-apphost --configuration=${{ parameters.configuration }} --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false --runtimevariant=${{ parameters.runtimeVariant }}"
69+
70+
# Add Material3 build argument if enabled
71+
if (("${{ parameters.useMaterial3 }}" -eq "true") -and ("${{ parameters.platform }}" -eq "android")) {
72+
$buildCommand += " --usematerial3=true"
73+
Write-Host "Building with Material3 enabled"
74+
} elseif ("${{ parameters.useMaterial3 }}" -eq "true") {
75+
Write-Host "Material3 is only supported for Android platform, ignoring useMaterial3 parameter"
76+
}
77+
78+
Invoke-Expression $buildCommand
5879
displayName: 'Build the samples'
5980

6081
- bash: |
@@ -65,33 +86,51 @@ steps:
6586
continueOnError: true
6687

6788
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
68-
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'NativeAOT'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), succeeded())
89+
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'NativeAOT'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'false'), succeeded())
6990
artifact: ui-tests-samples
7091

7192
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
7293
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'NativeAOT'), succeeded())
7394
artifact: ui-tests-samples-nativeaot
7495

7596
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
76-
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), succeeded())
97+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'false'), succeeded())
7798
artifact: ui-tests-samples-coreclr
7899

100+
# Material3 artifacts
101+
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
102+
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'true'), succeeded())
103+
artifact: ui-tests-samples-material3
104+
105+
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
106+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'true'), succeeded())
107+
artifact: ui-tests-samples-material3-coreclr
108+
79109
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
80110
condition: and(eq('${{ parameters.platform }}' , 'windows'), succeeded())
81111
artifact: ui-tests-samples-windows
82112

83113
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
84-
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'NativeAOT'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), failed())
114+
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'NativeAOT'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'false'), failed())
85115
artifact: ui-tests-samples_failed_$(System.JobAttempt)
86116

87117
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
88118
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'NativeAOT'), failed())
89119
artifact: ui-tests-samples-nativeaot_failed_$(System.JobAttempt)
90120

91121
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
92-
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), failed())
122+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'false'), failed())
93123
artifact: ui-tests-samples-coreclr_failed_$(System.JobAttempt)
94124

125+
# Material3 failure artifacts
126+
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
127+
condition: and(ne('${{ parameters.platform }}' , 'windows'), ne('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'true'), failed())
128+
artifact: ui-tests-samples-material3_failed_$(System.JobAttempt)
129+
130+
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
131+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'true'), failed())
132+
artifact: ui-tests-samples-material3-coreclr_failed_$(System.JobAttempt)
133+
95134
- publish: $(System.DefaultWorkingDirectory)/artifacts/bin
96135
condition: and(eq('${{ parameters.platform }}' , 'windows'), failed())
97136
artifact: ui-tests-samples-windows_failed_$(System.JobAttempt)

eng/pipelines/common/ui-tests-steps.yml

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@ parameters:
22
platform: '' # [ android, ios, windows, catalyst ]
33
path: '' # path to csproj
44
device: '' # the xharness device to use
5+
deviceType: '' # the device type/skin for Android emulator (e.g., pixel_3XL, Nexus 5X)
56
cakeArgs: '' # additional cake args
67
app: '' #path to app to test
78
version: '' #the iOS version'
89
provisionatorChannel: 'latest'
910
configuration: "Release"
1011
runtimeVariant: "Mono"
12+
useMaterial3: false # NEW PARAMETER - Enable Material3 build
1113
testFilter: ''
1214
headless: true
1315
testConfigurationArgs: ''
@@ -16,17 +18,22 @@ parameters:
1618

1719
steps:
1820
- task: DownloadPipelineArtifact@2
19-
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'Mono'))
21+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'Mono'), eq('${{ parameters.useMaterial3 }}', 'false'))
2022
inputs:
2123
artifact: ui-tests-samples
2224

25+
- task: DownloadPipelineArtifact@2
26+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'Mono'), eq('${{ parameters.useMaterial3 }}', 'true'))
27+
inputs:
28+
artifact: ui-tests-samples-material3
29+
2330
- task: DownloadPipelineArtifact@2
2431
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'NativeAOT'))
2532
inputs:
2633
artifact: ui-tests-samples-nativeaot
2734

2835
- task: DownloadPipelineArtifact@2
29-
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'))
36+
condition: and(ne('${{ parameters.platform }}' , 'windows'), eq('${{ parameters.runtimeVariant }}' , 'CoreCLR'), eq('${{ parameters.useMaterial3 }}', 'false'))
3037
inputs:
3138
artifact: ui-tests-samples-coreclr
3239

@@ -129,6 +136,12 @@ steps:
129136
$command += " --runtimevariant=""${{ parameters.runtimeVariant }}"""
130137
$command += " --results=""$(TestResultsDirectory)"" --binlog=""$(LogDirectory)"" ${{ parameters.cakeArgs }} --verbosity=diagnostic"
131138
139+
# Add device skin parameter if specified (for Android emulator hardware profile)
140+
$deviceType = "${{ parameters.deviceType }}"
141+
if ($deviceType) {
142+
$command += " --skin=""$deviceType"""
143+
}
144+
132145
$testFilter = ""
133146
$testConfigrationArgs = "${{ parameters.testConfigurationArgs }}"
134147

eng/pipelines/common/ui-tests.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ parameters:
66
windowsBuildPool: { }
77
macosPool: { }
88
androidApiLevels: [ 30 ]
9+
androidApiLevelsExtended: [ 36 ] # API 36 for Material3 tests with Pixel 3 XL
910
iosVersions: [ 'latest' ]
1011
provisionatorChannel: 'latest'
1112
timeoutInMinutes: 180
@@ -15,6 +16,7 @@ parameters:
1516
categoryGroupsToTest:
1617
# Make sure that this list is always up-to-date with src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs
1718
# we might want to improve this somehow depending on how much the categories change over time
19+
# Note: Material3 is intentionally excluded from this list and has its own dedicated build and test stages below with a separate test filter
1820
- 'Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks'
1921
- 'Border,BoxView,Brush,Button'
2022
- 'CarouselView'
@@ -115,6 +117,27 @@ stages:
115117
platform: windows
116118
skipProvisioning: ${{ parameters.skipProvisioning }}
117119

120+
# Material3 Build Stage - Separate build with UseMaterial3=true (Android-only)
121+
- stage: build_ui_tests_material3
122+
displayName: Build UITests Material3 Sample App
123+
dependsOn: []
124+
jobs:
125+
- job: build_ui_tests_material3
126+
displayName: Build Sample App (Material3)
127+
workspace:
128+
clean: all
129+
pool: ${{ parameters.androidPool }}
130+
variables:
131+
REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE)
132+
APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/
133+
steps:
134+
- template: ui-tests-build-sample.yml
135+
parameters:
136+
runtimeVariant: "Mono"
137+
platform: android
138+
useMaterial3: true
139+
skipProvisioning: ${{ parameters.skipProvisioning }}
140+
118141
- stage: android_ui_tests
119142
displayName: Android UITests
120143
dependsOn: build_ui_tests
@@ -205,6 +228,46 @@ stages:
205228
platform: 'Android'
206229
artifactName: 'uitest-snapshot-results-android-$(System.StageName)-$(System.JobName)-$(System.JobAttempt)'
207230

231+
- stage: android_ui_tests_material3
232+
displayName: Android UITests Material3
233+
dependsOn: build_ui_tests_material3
234+
jobs:
235+
- ${{ each project in parameters.projects }}:
236+
- ${{ if ne(project.android, '') }}:
237+
- ${{ each api in parameters.androidApiLevelsExtended }}:
238+
- ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}:
239+
- job: M3_android_ui_tests_${{ project.name }}_${{ api }}
240+
timeoutInMinutes: 240
241+
workspace:
242+
clean: all
243+
displayName: ${{ coalesce(project.desc, project.name) }} Material3 (API ${{ api }})
244+
pool: ${{ parameters.androidLinuxPool }}
245+
variables:
246+
REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE)
247+
APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/
248+
steps:
249+
- template: ui-tests-steps.yml
250+
parameters:
251+
platform: android
252+
version: ${{ api }}
253+
path: ${{ project.android }}
254+
app: ${{ project.app }}
255+
${{ if eq(api, 27) }}:
256+
device: android-emulator-32_${{ api }}
257+
${{ if not(eq(api, 27)) }}:
258+
device: android-emulator-64_${{ api }}
259+
deviceType: "pixel_3_xl"
260+
provisionatorChannel: ${{ parameters.provisionatorChannel }}
261+
useMaterial3: true
262+
testFilter: "Material3"
263+
skipProvisioning: ${{ parameters.skipProvisioning }}
264+
265+
# Collect and publish Android Material3 snapshot diffs
266+
- template: ui-tests-collect-snapshot-diffs.yml
267+
parameters:
268+
platform: 'Android Material3'
269+
artifactName: 'uitest-snapshot-results-android-material3-$(System.StageName)-$(System.JobName)-$(System.JobAttempt)'
270+
208271
- stage: ios_ui_tests_mono
209272
displayName: iOS UITests Mono
210273
dependsOn: build_ui_tests
45.3 KB
Loading
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<ContentPage
3+
xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
4+
xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
5+
x:Class="Maui.Controls.Sample.Issues.Material3CheckBoxDefaultAppearance">
6+
7+
<ScrollView>
8+
<VerticalStackLayout Padding="20"
9+
Spacing="16">
10+
<Label Text="Material3 CheckBox Test"
11+
FontSize="18"
12+
FontAttributes="Bold"/>
13+
<CheckBox
14+
x:Name="DefaultCheckBox"
15+
AutomationId="DefaultCheckBox"
16+
IsChecked="True"/>
17+
<Button
18+
Text="Sample Button"/>
19+
</VerticalStackLayout>
20+
</ScrollView>
21+
</ContentPage>
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
namespace Maui.Controls.Sample.Issues
3+
{
4+
[Issue(IssueTracker.None, 0, "Material3 CheckBox Testing", PlatformAffected.Android)]
5+
public partial class Material3CheckBoxDefaultAppearance : ContentPage
6+
{
7+
public Material3CheckBoxDefaultAppearance()
8+
{
9+
InitializeComponent();
10+
}
11+
}
12+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#if ANDROID
2+
using NUnit.Framework;
3+
using UITest.Appium;
4+
using UITest.Core;
5+
6+
namespace Microsoft.Maui.TestCases.Tests.Issues
7+
{
8+
public class Material3CheckBoxDefaultAppearanceTest : _IssuesUITest
9+
{
10+
public Material3CheckBoxDefaultAppearanceTest(TestDevice testDevice) : base(testDevice)
11+
{
12+
}
13+
14+
public override string Issue => "Material3 CheckBox Testing";
15+
16+
[Test]
17+
[Category(UITestCategories.Material3)]
18+
public void Material3CheckBox_DefaultAppearance()
19+
{
20+
App.WaitForElement("DefaultCheckBox");
21+
VerifyScreenshot();
22+
}
23+
}
24+
}
25+
#endif

src/Controls/tests/TestCases.Shared.Tests/UITest.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -339,9 +339,9 @@ but both can happen.
339339
}
340340

341341
if (!((deviceApiLevel == 30 && (deviceScreenSize.Equals("1080x1920", StringComparison.OrdinalIgnoreCase) || deviceScreenSize.Equals("1920x1080", StringComparison.OrdinalIgnoreCase)) && deviceScreenDensity == 420) ||
342-
(deviceApiLevel == 36 && (deviceScreenSize.Equals("1080x2424", StringComparison.OrdinalIgnoreCase) || deviceScreenSize.Equals("2424x1080", StringComparison.OrdinalIgnoreCase)) && deviceScreenDensity == 420)))
342+
(deviceApiLevel == 36 && (deviceScreenSize.Equals("1440x2960", StringComparison.OrdinalIgnoreCase) || deviceScreenSize.Equals("2960x1440", StringComparison.OrdinalIgnoreCase)) && deviceScreenDensity == 560)))
343343
{
344-
Assert.Fail($"Android visual tests should be run on an API30 emulator image with 1080x1920 420dpi screen or API36 emulator image with 1080x2424 420dpi screen, but the current device is API {deviceApiLevel} with a {deviceScreenSize} {deviceScreenDensity}dpi screen. Follow the steps on the MAUI UI testing wiki to launch the Android emulator with the right image.");
344+
Assert.Fail($"Android visual tests should be run on an API30 emulator image with 1080x1920 420dpi screen or API36 emulator image with 1440x2960 560dpi screen, but the current device is API {deviceApiLevel} with a {deviceScreenSize} {deviceScreenDensity}dpi screen. Follow the steps on the MAUI UI testing wiki to launch the Android emulator with the right image.");
345345
}
346346
break;
347347

@@ -412,7 +412,7 @@ but both can happen.
412412
// bar at the top as it varies slightly based on OS theme and is also not part of the app.
413413
int cropFromTop = _testDevice switch
414414
{
415-
TestDevice.Android => environmentName == "android-notch-36" ? 95 : 60,
415+
TestDevice.Android => environmentName == "android-notch-36" ? 112 : 60,
416416
TestDevice.iOS => environmentName == "ios-iphonex" ? 90 : 110,
417417
TestDevice.Windows => 32,
418418
TestDevice.Mac => 29,
@@ -431,7 +431,7 @@ but both can happen.
431431
// For iOS, crop the home indicator at the bottom.
432432
int cropFromBottom = _testDevice switch
433433
{
434-
TestDevice.Android => environmentName == "android-notch-36" ? 40 : 125,
434+
TestDevice.Android => environmentName == "android-notch-36" ? 52 : 125,
435435
TestDevice.iOS => 40,
436436
_ => 0,
437437
};

src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,6 @@ internal static class UITestCategories
7575
public const string GraphicsView = "GraphicsView";
7676
public const string Fonts = "Fonts";
7777
public const string SafeAreaEdges = "SafeAreaEdges";
78+
public const string Material3 = "Material3";
7879
}
7980
}

0 commit comments

Comments
 (0)