-
Notifications
You must be signed in to change notification settings - Fork 301
Add MSTest.Windows.AppTesting thin-layer for Windows desktop E2E testing #7810
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
e4f54a4
Add MSTest.DesktopTesting thin-layer for Windows desktop E2E testing
Evangelink 1b6a641
Address PR review comments
Evangelink 0678b77
Rename
Evangelink e952ac5
Merge branch 'main' into dev/amauryleve/desktop-testing-e2e
Evangelink f53c770
Address PR review comments for MSTest.Windows.AppTesting
Evangelink 05d5efb
Merge branch 'main' into dev/amauryleve/desktop-testing-e2e
Evangelink 262d444
Address second round of PR review comments
Evangelink ae139d7
Merge branch 'dev/amauryleve/desktop-testing-e2e' of https://github.c…
Evangelink c9ac13f
Address third round of PR review comments
Evangelink ba4ba97
Update verify-nupkgs.ps1 with MSTest.Windows.AppTesting package valid…
Evangelink 76e6f0b
Address PR review: narrow FrameworkReference to WPF, DynamicData for …
Evangelink 66d35e0
Add TestFramework.Windows.AppTesting to TestFx.slnx
Evangelink b2e124d
Fix IDE0005, IDE0007, IDE0022 analyzer errors
Evangelink File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 62 additions & 0 deletions
62
samples/public/DemoMSTestSdk/ProjectUsingWindowsAppTesting/CalculatorTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System.Windows.Automation; | ||
|
|
||
| namespace ProjectUsingWindowsAppTesting; | ||
|
|
||
| /// <summary> | ||
| /// Sample end-to-end tests for the Windows Calculator application. | ||
| /// Demonstrates the WindowTest base class which manages app launch, main window | ||
| /// discovery, and teardown — analogous to how PageTest works for Playwright. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// <para> | ||
| /// On modern Windows 10/11, <c>calc.exe</c> launches the Store/UWP Calculator, whose | ||
| /// UI Automation tree uses different AutomationIds than the classic Win32 Calculator. | ||
| /// The <c>Calculator_NumberButtons_AreVisible</c> test below is illustrative — you will | ||
| /// need to replace the AutomationId values with the actual IDs for your target application. | ||
| /// </para> | ||
| /// </remarks> | ||
| [STATestClass] | ||
| public class CalculatorTests : WindowTest | ||
| { | ||
| /// <summary> | ||
| /// Path to the application under test. | ||
| /// Override this to point to your own application executable. | ||
| /// </summary> | ||
| public override string ApplicationPath => "calc.exe"; | ||
|
|
||
| [TestMethod] | ||
| public void Calculator_MainWindow_HasExpectedTitle() | ||
| { | ||
| // The MainWindow property is automatically populated by WindowTest | ||
| // after launching the application specified by ApplicationPath. | ||
| Assert.AreEqual(ControlType.Window, MainWindow.Current.ControlType, | ||
| "Expected the main window element to be of control type Window."); | ||
|
|
||
| string title = MainWindow.Current.Name; | ||
| Assert.IsFalse( | ||
| string.IsNullOrWhiteSpace(title), | ||
| "Expected the main window to have a non-empty title."); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| [Ignore("AutomationIds are app-version-specific. Replace 'num1Button'/'num2Button' with actual IDs for your target application.")] | ||
| public void Calculator_NumberButtons_AreVisible() | ||
| { | ||
| // NOTE: Modern Windows Calculator (UWP) uses different AutomationIds than the | ||
| // classic Win32 Calculator. Replace "num1Button" / "num2Button" with the actual | ||
| // AutomationIds for your target application version. | ||
| AutomationElement? button1 = MainWindow.FindFirst( | ||
| TreeScope.Descendants, | ||
| new PropertyCondition(AutomationElement.AutomationIdProperty, "num1Button")); | ||
|
|
||
| AutomationElement? button2 = MainWindow.FindFirst( | ||
| TreeScope.Descendants, | ||
| new PropertyCondition(AutomationElement.AutomationIdProperty, "num2Button")); | ||
|
|
||
| Assert.IsNotNull(button1, "Could not find the button with AutomationId 'num1Button'."); | ||
| Assert.IsNotNull(button2, "Could not find the button with AutomationId 'num2Button'."); | ||
| } | ||
|
Evangelink marked this conversation as resolved.
|
||
| } | ||
43 changes: 43 additions & 0 deletions
43
...s/public/DemoMSTestSdk/ProjectUsingWindowsAppTesting/ProjectUsingWindowsAppTesting.csproj
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| <Project Sdk="MSTest.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net8.0-windows</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <!-- Enables the MSTest Windows App Testing feature --> | ||
| <EnableWindowsAppTesting>true</EnableWindowsAppTesting> | ||
| </PropertyGroup> | ||
|
|
||
|
Evangelink marked this conversation as resolved.
|
||
| </Project> | ||
|
|
||
|
|
||
| <!-- | ||
| Below is the equivalent project configuration when not using MSTest.Sdk | ||
|
|
||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net8.0-windows</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
|
|
||
| <IsPackable>false</IsPackable> | ||
| <IsTestProject>true</IsTestProject> | ||
| <EnableMSTestRunner>true</EnableMSTestRunner> | ||
| <OutputType>Exe</OutputType> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="MSTest.Windows.AppTesting" Version="$(MSTestWindowsAppTestingVersion)" /> | ||
| <PackageReference Include="MSTest.Analyzers" Version="$(MSTestVersion)" /> | ||
| <PackageReference Include="MSTest.TestAdapter" Version="$(MSTestVersion)" /> | ||
| <PackageReference Include="MSTest.TestFramework" Version="$(MSTestVersion)" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Using Include="Microsoft.VisualStudio.TestTools.UnitTesting" /> | ||
| <Using Include="Microsoft.MSTest.Windows.AppTesting" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> | ||
| --> | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
src/Package/MSTest.Sdk/Sdk/Features/WindowsAppTesting.targets
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <?xml version="1.0" encoding="utf-8" ?> | ||
| <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> | ||
|
|
||
| <PropertyGroup> | ||
| <_IsWindowsTfm>false</_IsWindowsTfm> | ||
| <_IsWindowsTfm Condition="$(TargetFramework.Contains('-windows'))">true</_IsWindowsTfm> | ||
| </PropertyGroup> | ||
|
|
||
| <Target Name="_ValidateWindowsAppTestingPlatform" BeforeTargets="Restore;Build" | ||
| Condition=" '$(TargetFramework)' != '' and '$(_IsWindowsTfm)' != 'true' "> | ||
| <Error Text="MSTest.Windows.AppTesting requires a Windows target framework (e.g. net8.0-windows). Current TargetFramework: '$(TargetFramework)'." /> | ||
|
Evangelink marked this conversation as resolved.
|
||
| </Target> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="MSTest.Windows.AppTesting" Sdk="MSTest"> | ||
| <Version Condition=" '$(ManagePackageVersionsCentrally)' != 'true' ">$(MSTestWindowsAppTestingVersion)</Version> | ||
| </PackageReference> | ||
| <PackageVersion Include="MSTest.Windows.AppTesting" Version="$(MSTestWindowsAppTestingVersion)" | ||
| Condition=" '$(ManagePackageVersionsCentrally)' == 'true' " /> | ||
| </ItemGroup> | ||
|
|
||
| <!-- | ||
| Implicit imports | ||
| Ensure feature is available and user hasn't opted-out from it. | ||
| See https://github.com/dotnet/sdk/blob/f9fdf2c7d94bc86dc443e5a9ffecbd1962b1d85d/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.CSharp.props#L26-L34 | ||
| --> | ||
| <ItemGroup Condition=" '$(ImplicitUsings)' == 'true' OR '$(ImplicitUsings)' == 'enable' "> | ||
| <Using Include="Microsoft.MSTest.Windows.AppTesting" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
134 changes: 134 additions & 0 deletions
134
src/TestFramework/TestFramework.Windows.AppTesting/ApplicationTest.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System.ComponentModel; | ||
|
|
||
| namespace Microsoft.MSTest.Windows.AppTesting; | ||
|
|
||
| /// <summary> | ||
| /// Base test class that manages the lifecycle of a desktop application process. | ||
| /// Analogous to <c>BrowserTest</c> in the Playwright MSTest integration. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// Override <see cref="ApplicationPath"/> and optionally <see cref="ApplicationArguments"/> | ||
| /// to configure which application to launch. | ||
| /// If <see cref="ApplicationPath"/> is not overridden, the test will attempt to read | ||
| /// the path from the <c>DESKTOP_TEST_APP_PATH</c> environment variable. | ||
| /// </remarks> | ||
| [STATestClass] | ||
| public class ApplicationTest : AutomationTest | ||
| { | ||
| private const string AppPathEnvVar = "DESKTOP_TEST_APP_PATH"; | ||
|
|
||
| /// <summary> | ||
| /// Gets the process of the application under test. | ||
| /// Available after <see cref="ApplicationSetup"/> has run. | ||
| /// </summary> | ||
| public Process AppProcess { get; private set; } = null!; | ||
|
|
||
| /// <summary> | ||
| /// Gets the path to the application executable to launch. | ||
| /// Override this property to specify the application under test. | ||
| /// Defaults to the <c>DESKTOP_TEST_APP_PATH</c> environment variable. | ||
| /// </summary> | ||
| public virtual string ApplicationPath | ||
| { | ||
| get | ||
| { | ||
| string? envPath = Environment.GetEnvironmentVariable(AppPathEnvVar); | ||
| return string.IsNullOrWhiteSpace(envPath) | ||
| ? throw new InvalidOperationException( | ||
| $"Override {nameof(ApplicationPath)} or set the '{AppPathEnvVar}' environment variable to the path of the application under test.") | ||
| : envPath; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the command-line arguments to pass when launching the application. | ||
| /// Override to customize. Defaults to <see langword="null"/> (no arguments). | ||
| /// </summary> | ||
| public virtual string? ApplicationArguments => null; | ||
|
|
||
| /// <summary> | ||
| /// Gets the timeout to wait for the application's main window to become available. | ||
| /// Override to customize. Defaults to 10 seconds. | ||
| /// </summary> | ||
| public virtual TimeSpan ApplicationStartTimeout => TimeSpan.FromSeconds(10); | ||
|
|
||
| /// <summary> | ||
| /// Launches the application under test before each test method. | ||
| /// </summary> | ||
| [TestInitialize] | ||
| public void ApplicationSetup() | ||
| { | ||
| ProcessStartInfo startInfo = new(ApplicationPath); | ||
| if (ApplicationArguments is not null) | ||
| { | ||
| startInfo.Arguments = ApplicationArguments; | ||
| } | ||
|
|
||
| AppProcess = Process.Start(startInfo) | ||
| ?? throw new InvalidOperationException($"Failed to start process: {ApplicationPath}"); | ||
|
|
||
| // Wait for the main window to be created | ||
| var sw = Stopwatch.StartNew(); | ||
| while (AppProcess.MainWindowHandle == IntPtr.Zero && sw.Elapsed < ApplicationStartTimeout) | ||
| { | ||
|
Evangelink marked this conversation as resolved.
|
||
| AppProcess.Refresh(); | ||
| if (AppProcess.HasExited) | ||
| { | ||
| throw new InvalidOperationException( | ||
| $"Application '{ApplicationPath}' exited with code {AppProcess.ExitCode} before a main window was created."); | ||
| } | ||
|
|
||
| TimeSpan remainingTime = ApplicationStartTimeout - sw.Elapsed; | ||
| if (remainingTime > TimeSpan.Zero) | ||
| { | ||
| Thread.Sleep(remainingTime < TimeSpan.FromMilliseconds(50) | ||
| ? remainingTime | ||
| : TimeSpan.FromMilliseconds(50)); | ||
| } | ||
| } | ||
|
|
||
| AppProcess.Refresh(); | ||
| if (AppProcess.MainWindowHandle == IntPtr.Zero) | ||
|
Evangelink marked this conversation as resolved.
|
||
| { | ||
| throw new TimeoutException( | ||
| $"Application '{ApplicationPath}' did not create a main window within {ApplicationStartTimeout}."); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Closes and disposes the application after each test method. | ||
| /// </summary> | ||
| [TestCleanup] | ||
| public void ApplicationTearDown() | ||
| { | ||
| Process? appProcess = AppProcess; | ||
|
|
||
| try | ||
| { | ||
| if (appProcess is not null && !appProcess.HasExited) | ||
| { | ||
| try | ||
| { | ||
| _ = appProcess.CloseMainWindow(); | ||
| if (!appProcess.WaitForExit(5000)) | ||
| { | ||
| appProcess.Kill(entireProcessTree: true); | ||
| _ = appProcess.WaitForExit(5000); | ||
| } | ||
|
Evangelink marked this conversation as resolved.
|
||
| } | ||
| catch (Exception ex) when (ex is InvalidOperationException or Win32Exception) | ||
| { | ||
| // The process exited or became inaccessible between state checks and shutdown operations. | ||
| } | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| appProcess?.Dispose(); | ||
| AppProcess = null!; | ||
| } | ||
| } | ||
| } | ||
18 changes: 18 additions & 0 deletions
18
src/TestFramework/TestFramework.Windows.AppTesting/AutomationTest.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| namespace Microsoft.MSTest.Windows.AppTesting; | ||
|
|
||
| /// <summary> | ||
| /// Base test class for desktop UI automation tests. | ||
| /// This is the root of the desktop testing class hierarchy, analogous to | ||
| /// <c>PlaywrightTest</c> in the Playwright MSTest integration. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// This layer exists to match the Playwright base class hierarchy pattern | ||
| /// and provides the extension point for future automation-level configuration. | ||
| /// </remarks> | ||
| [TestClass] | ||
| public class AutomationTest | ||
| { | ||
| } |
4 changes: 4 additions & 0 deletions
4
src/TestFramework/TestFramework.Windows.AppTesting/GlobalUsings.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| global using Microsoft.VisualStudio.TestTools.UnitTesting; |
2 changes: 2 additions & 0 deletions
2
src/TestFramework/TestFramework.Windows.AppTesting/PublicAPI/PublicAPI.Shipped.txt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| #nullable enable | ||
|
|
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.