Isolate plugins in an out-of-process COM host

WSL plugin DLLs are moved out of wslservice.exe into a separate
wslpluginhost.exe COM server so plugin code can no longer crash or
destabilize the service. Each plugin is activated in its own host
process (CLSCTX_LOCAL_SERVER, SYSTEM-only via AppID) and reached
through a versioned COM interface defined in WslPluginHost.idl. All
hosts are tied to a service-owned job object and terminate when
wslservice exits. The plugin API is unchanged; existing plugins run
unmodified.

A crashing or disconnected host is classified by IsHostCrash
(RPC_E_DISCONNECTED, RPC_E_SERVER_DIED[_DNE], CO_E_OBJNOTCONNECTED,
RPC_S_SERVER_UNAVAILABLE, RPC_S_CALL_FAILED[_DNE]); the service logs
it and continues instead of treating it as a fatal plugin error.
RPC_E_CALL_REJECTED is intentionally excluded as a transient busy
state rather than a dead host.

Plugin->service callbacks (MountFolder, ExecuteBinary, and the WSLC
session APIs) arrive on a different COM thread than the outbound hook,
so they cannot re-enter the lock held during the hook:
- VM path: LxssUserSessionImpl guards callbacks with a shared_mutex
  (shared for callbacks, exclusive in _VmTerminate after OnVmStopping
  drains in-flight callbacks before the utility VM is destroyed).
- WSLC path: PluginManager resolves sessions through its own
  reference map under a dedicated lock, and WSLCSessionManager
  releases its session lock before any plugin notification fires, so
  callbacks never re-enter the session lock. A session is registered
  in the reference map but not published until OnWslcSessionCreated
  succeeds, so a vetoed or race-lost session is never handed out.

Proxy/stub is consolidated into wslserviceproxystub.dll. One new exe,
no new DLLs.

Tests
- HostCrashIsolation: kills wslpluginhost.exe mid-OnVmStarted and
  verifies the service survives and m_initOnce stays sticky.
- ConcurrentCallbacks: four plugin threads hammer MountFolder and
  ExecuteBinary, exercising the shared callback lock.
- AsyncApiCallFromWorker: a plugin worker thread calls into the
  service post-hook (cross-apartment, non-COM-initialized).
- CallbacksDuringTerminationDoNotCrash: worker threads race
  _VmTerminate's exclusive lock and VM teardown, then wind down
  deterministically after OnVmStopping signals them and are joined on
  the next session start.
- Existing WSL1 plugin tests broadened alongside the refactor.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Ben Hillis

.pipelines/build-stage.yml

@@ -32,8 +32,8 @@ parameters:
   - name: targets
     type: object
     default:
-      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin;wslcsession;wslc;wsltests;wslcsdk"
-        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll,wslcsession.exe,wslc.exe,wslcsdk.dll"
+      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslpluginhost;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin;wslcsession;wslc;wsltests;wslcsdk"
+        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslpluginhost.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll,wslcsession.exe,wslc.exe,wslcsdk.dll"
       - target: "msixgluepackage"
         pattern: "gluepackage.msix"
       - target: "msipackage;wslcsdkcs"

CMakeLists.txt

@@ -535,6 +535,7 @@ add_subdirectory(src/windows/wsl)
 add_subdirectory(src/windows/wslg)
 add_subdirectory(src/windows/wslhost)
 add_subdirectory(src/windows/wslrelay)
+add_subdirectory(src/windows/wslpluginhost)
 add_subdirectory(src/windows/wslinstall)
 add_subdirectory(src/windows/wslc)
 add_subdirectory(src/windows/WslcSDK)

msipackage/CMakeLists.txt

@@ -17,7 +17,7 @@ set(OUTPUT_PACKAGE ${BIN}/wsl.msi)
 set(PACKAGE_WIX_IN ${CMAKE_CURRENT_LIST_DIR}/package.wix.in)
 set(PACKAGE_WIX ${BIN}/package.wix)
 set(CAB_CACHE ${BIN}/cab)
-set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll;wslc.exe;wslcsession.exe)
+set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslpluginhost.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll;wslc.exe;wslcsession.exe)
 if (WSL_BUILD_WSL_SETTINGS)
     list(APPEND WINDOWS_BINARIES "wslsettings/wslsettings.dll;wslsettings/wslsettings.exe;libwsl.dll")
 endif()
@@ -57,7 +57,7 @@ add_custom_command(
 
 add_custom_target(msipackage DEPENDS ${OUTPUT_PACKAGE})
 set_target_properties(msipackage PROPERTIES EXCLUDE_FROM_ALL FALSE SOURCES ${PACKAGE_WIX_IN})
-add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslserviceproxystub init initramfs wslinstall msixgluepackage wslc wslcsession)
+add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslpluginhost wslserviceproxystub init initramfs wslinstall msixgluepackage wslc wslcsession)
 
 if (WSL_BUILD_WSL_SETTINGS)
     add_dependencies(msipackage wslsettings libwsl)

msipackage/package.wix.in

@@ -31,6 +31,7 @@
                 <File Id="container.exe" Name="container.exe" Source="${PACKAGE_INPUT_DIR}/wslc.exe" />
                 <File Id="wslhost.exe" Name="wslhost.exe" Source="${PACKAGE_INPUT_DIR}/wslhost.exe" />
                 <File Id="wslrelay.exe" Name="wslrelay.exe" Source="${PACKAGE_INPUT_DIR}/wslrelay.exe" />
+                <File Id="wslpluginhost.exe" Name="wslpluginhost.exe" Source="${PACKAGE_INPUT_DIR}/wslpluginhost.exe" />
                 <File Id="wslserviceproxystub.dll" Name="wslserviceproxystub.dll" Source="${PACKAGE_INPUT_DIR}/wslserviceproxystub.dll" />
                 <File Id="wsldeps.dll" Name="wsldeps.dll" Source="${PACKAGE_INPUT_DIR}/wsldeps.dll" />
 
@@ -177,6 +178,35 @@
                     <RegistryValue Value='"[INSTALLDIR]wslhost.exe"' Type="string" />
                 </RegistryKey>
 
+                <!-- WslPluginHost - out-of-process plugin isolation (process spawned by service) -->
+                <!-- WslPluginHost AppID — SYSTEM-only activation -->
+                <!-- O:SYG:SYD:(A;;CCDCSW;;;SY) -->
+                <RegistryKey Root="HKCR" Key="AppID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AccessPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                    <RegistryValue Name="LaunchPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AppId" Value="{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}" Type="string" />
+                    <RegistryValue Value="WslPluginHost" Type="string" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}\LocalServer32">
+                    <RegistryValue Value='"[INSTALLDIR]wslpluginhost.exe"' Type="string" />
+                </RegistryKey>
+
+                <!-- WslPluginHost interfaces — use the shared wslserviceproxystub.dll -->
+                <RegistryKey Root="HKCR" Key="Interface\{A2B3C4D5-E6F7-4890-AB12-CD34EF56A789}">
+                    <RegistryValue Value="IWslPluginHostCallback" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="Interface\{B3C4D5E6-F7A8-4901-BC23-DE45FA67B890}">
+                    <RegistryValue Value="IWslPluginHost" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+
                 <!-- wsldevicehost.dll -->
                 <RegistryKey Root="HKCR" Key="AppID\{17696EAC-9568-4CF5-BB8C-82515AAD6C09}">
                     <RegistryValue Name="DllSurrogate" Value="" Type="string" />

src/windows/common/precomp.h

@@ -84,6 +84,7 @@ Module Name:
 #include <optional>
 #include <filesystem>
 #include <mutex>
+#include <shared_mutex>
 #include <chrono>
 #include <codecvt>
 #include <random>

src/windows/service/exe/CMakeLists.txt

@@ -58,7 +58,7 @@ set(HEADERS
     WSLCPluginNotifier.h)
 
 add_executable(wslservice ${SOURCES} ${HEADERS})
-add_dependencies(wslservice wslserviceidl wslservicemc)
+add_dependencies(wslservice wslserviceidl wslservicemc wslpluginhostidl)
 add_compile_definitions(__WRL_CLASSIC_COM__)
 add_compile_definitions(__WRL_DISABLE_STATIC_INITIALIZE__)
 add_compile_definitions(USE_COM_CONTEXT_DEF=1)

src/windows/service/exe/LxssUserSession.cpp

@@ -2631,13 +2631,18 @@ std::shared_ptr<LxssRunningInstance> LxssUserSessionImpl::_CreateInstance(_In_op
                     registration.Write(Property::OsVersion, distributionInfo->Version);
                 }
 
-                // This needs to be done before plugins are notifed because they might try to run a command inside the distribution.
-                m_runningInstances[registration.Id()] = instance;
+                // This needs to be done before plugins are notified because they might try to run a command inside the distribution.
+                {
+                    std::unique_lock callbackLock(m_callbackLock);
+                    m_runningInstances[registration.Id()] = instance;
+                }
 
                 if (version == LXSS_WSL_VERSION_2)
                 {
-                    auto cleanupOnFailure =
-                        wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() { m_runningInstances.erase(registration.Id()); });
+                    auto cleanupOnFailure = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() {
+                        std::unique_lock callbackLock(m_callbackLock);
+                        m_runningInstances.erase(registration.Id());
+                    });
                     m_pluginManager.OnDistributionStarted(&m_session, instance->DistributionInformation());
                     cleanupOnFailure.release();
                 }
@@ -2873,7 +2878,13 @@ void LxssUserSessionImpl::_CreateVm()
         m_vmId.store(vmId);
 
         // Create the utility VM and register for callbacks.
-        m_utilityVm = WslCoreVm::Create(m_userToken, std::move(config), vmId);
+        // Publish m_utilityVm under m_callbackLock exclusive to honor the dual-lock
+        // invariant for mutations of m_utilityVm; this is uncontended here because
+        // no plugin callbacks can race against initial creation.
+        {
+            std::unique_lock callbackLock(m_callbackLock);
+            m_utilityVm = WslCoreVm::Create(m_userToken, std::move(config), vmId);
+        }
 
         if (m_httpProxyStateTracker)
         {
@@ -3604,17 +3615,26 @@ bool LxssUserSessionImpl::_TerminateInstanceInternal(_In_ LPCGUID DistroGuid, _I
                     m_pluginManager.OnDistributionStopping(&m_session, wslcoreInstance->DistributionInformation());
                 }
 
-                instance->second->Stop();
+                m_lifetimeManager.RemoveCallback(clientKey);
 
-                const auto clientId = instance->second->GetClientId();
+                // Stop the instance and remove it from m_runningInstances atomically
+                // under m_callbackLock. This prevents plugin callbacks (which hold
+                // m_callbackLock shared) from finding a stopped-but-still-listed
+                // instance between Stop() and erase.
+                ULONG clientId;
                 {
-                    auto lock = m_terminatedInstanceLock.lock_exclusive();
-                    m_terminatedInstances.push_back(std::move(instance->second));
-                }
+                    std::unique_lock callbackLock(m_callbackLock);
 
-                m_lifetimeManager.RemoveCallback(clientKey);
+                    instance->second->Stop();
+                    clientId = instance->second->GetClientId();
+
+                    {
+                        auto lock = m_terminatedInstanceLock.lock_exclusive();
+                        m_terminatedInstances.push_back(std::move(instance->second));
+                    }
 
-                m_runningInstances.erase(instance);
+                    m_runningInstances.erase(instance);
+                }
 
                 // If the instance that was terminated was a WSL2 instance,
                 // check if the VM is now idle.
@@ -3642,7 +3662,10 @@ void LxssUserSessionImpl::_UpdateInit(_In_ const LXSS_DISTRO_CONFIGURATION& Conf
 
 HRESULT LxssUserSessionImpl::MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name)
 {
-    std::lock_guard lock(m_instanceLock);
+    // Shared lock prevents _VmTerminate from destroying the VM while we use it.
+    // Do NOT acquire m_instanceLock — callbacks arrive on a different COM thread
+    // from the notification thread that holds m_instanceLock.
+    std::shared_lock lock(m_callbackLock);
     RETURN_HR_IF(E_NOT_VALID_STATE, !m_utilityVm);
 
     m_utilityVm->MountRootNamespaceFolder(HostPath, GuestPath, ReadOnly, Name);
@@ -3651,7 +3674,9 @@ HRESULT LxssUserSessionImpl::MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In
 
 HRESULT LxssUserSessionImpl::CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* Socket)
 {
-    std::lock_guard lock(m_instanceLock);
+    // Shared lock prevents _VmTerminate from destroying the VM or instances
+    // while we use them. See MountRootNamespaceFolder for rationale.
+    std::shared_lock lock(m_callbackLock);
     RETURN_HR_IF(E_NOT_VALID_STATE, !m_utilityVm);
 
     if (Distro == nullptr)
@@ -3660,9 +3685,16 @@ HRESULT LxssUserSessionImpl::CreateLinuxProcess(_In_opt_ const GUID* Distro, _In
     }
     else
     {
-        const auto distro = _RunningInstance(Distro);
-        THROW_HR_IF(WSL_E_VM_MODE_INVALID_STATE, !distro);
-
+        // Look up the running instance directly instead of calling _RunningInstance,
+        // which accesses m_lockedDistributions (guarded only by m_instanceLock).
+        // m_runningInstances is safe to read under m_callbackLock (shared).
+        // The _EnsureNotLocked check is unnecessary here: _ConversionBegin removes
+        // a distribution from m_runningInstances before adding it to m_lockedDistributions,
+        // so a locked distribution will never be found in this lookup.
+        const auto instance = m_runningInstances.find(*Distro);
+        THROW_HR_IF(WSL_E_VM_MODE_INVALID_STATE, instance == m_runningInstances.end());
+
+        const auto distro = instance->second;
         const auto wsl2Distro = dynamic_cast<WslCoreInstance*>(distro.get());
         THROW_HR_IF(WSL_E_WSL2_NEEDED, !wsl2Distro);
 
@@ -3900,7 +3932,12 @@ void LxssUserSessionImpl::_VmTerminate()
         m_telemetryThread.join();
     }
 
-    m_utilityVm.reset();
+    // Acquire exclusive callback lock to wait for any in-flight plugin callbacks
+    // (MountRootNamespaceFolder, CreateLinuxProcess) to complete before destroying the VM.
+    {
+        std::unique_lock callbackLock(m_callbackLock);
+        m_utilityVm.reset();
+    }
     m_vmId.store(GUID_NULL);
 
     // Reset the user's token since its lifetime is tied to the VM.

src/windows/service/exe/LxssUserSession.h

@@ -310,6 +310,10 @@ class DECLSPEC_UUID("a9b7a1b9-0671-405c-95f1-e0612cb4ce7e") LxssUserSession
 /// </summary>
 class LxssUserSessionImpl
 {
+    // Plugin callbacks arrive on a different COM RPC thread and use m_callbackLock
+    // (shared) instead of m_instanceLock to access m_utilityVm and m_runningInstances.
+    friend class wsl::windows::service::PluginHostCallbackImpl;
+
 public:
     LxssUserSessionImpl(_In_ PSID userSid, _In_ DWORD sessionId, _Inout_ wsl::windows::service::PluginManager& pluginManager);
     virtual ~LxssUserSessionImpl();
@@ -363,11 +367,6 @@ class LxssUserSessionImpl
     /// </summary>
     void ClearDiskStateInRegistry(_In_opt_ LPCWSTR Disk);
 
-    /// <summary>
-    /// Start a process in the root namespace or in a user distribution.
-    /// </summary>
-    HRESULT CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* socket);
-
     /// <summary>
     /// Enumerates registered distributions, optionally including ones that are
     /// currently being registered, unregistered, or converted.
@@ -443,8 +442,6 @@ class LxssUserSessionImpl
 
     HRESULT MoveDistribution(_In_ LPCGUID DistroGuid, _In_ LPCWSTR Location);
 
-    HRESULT MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name);
-
     /// <summary>
     /// Registers a distribution.
     /// </summary>
@@ -533,6 +530,18 @@ class LxssUserSessionImpl
     static CreateLxProcessContext s_GetCreateProcessContext(_In_ const GUID& DistroGuid, _In_ bool SystemDistro);
 
 private:
+    /// <summary>
+    /// Plugin callback methods — called from PluginHostCallbackImpl on a COM RPC
+    /// thread during plugin notifications. These acquire m_callbackLock (shared)
+    /// instead of m_instanceLock, preventing _VmTerminate from destroying the VM
+    /// while a callback is in-flight. Access is restricted via friend declaration.
+    /// </summary>
+    _Requires_lock_not_held_(m_instanceLock)
+    HRESULT MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name);
+
+    _Requires_lock_not_held_(m_instanceLock)
+    HRESULT CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* socket);
+
     /// <summary>
     /// Adds a distro to the list of converting distros.
     /// </summary>
@@ -794,7 +803,9 @@ class LxssUserSessionImpl
     std::recursive_timed_mutex m_instanceLock;
 
     /// <summary>
-    /// Contains the currently running utility VM's.
+    /// Contains the currently running instances.
+    /// Reads guarded by m_instanceLock OR m_callbackLock (shared).
+    /// Mutations require BOTH m_instanceLock AND m_callbackLock (exclusive).
     /// </summary>
     _Guarded_by_(m_instanceLock) std::map<GUID, std::shared_ptr<LxssRunningInstance>, wsl::windows::common::helpers::GuidLess> m_runningInstances;
 
@@ -811,9 +822,24 @@ class LxssUserSessionImpl
 
     /// <summary>
     /// The running utility vm for WSL2 distributions.
-    ///
+    /// Reads guarded by m_instanceLock OR m_callbackLock (shared).
+    /// Mutations require BOTH m_instanceLock AND m_callbackLock (exclusive).
+    /// </summary>
     _Guarded_by_(m_instanceLock) std::unique_ptr<WslCoreVm> m_utilityVm;
 
+    /// <summary>
+    /// Reader-writer lock protecting m_utilityVm and m_runningInstances for
+    /// plugin callbacks. Callbacks take a shared (read) lock; _VmTerminate and
+    /// instance mutations take an exclusive (write) lock.
+    ///
+    /// Mutations of m_runningInstances/m_utilityVm require BOTH m_instanceLock
+    /// AND m_callbackLock (exclusive). Reads are safe under either lock alone.
+    ///
+    /// Lock ordering: m_instanceLock → m_callbackLock (never reverse).
+    /// Callbacks must NEVER acquire m_instanceLock (deadlock with notification thread).
+    /// </summary>
+    std::shared_mutex m_callbackLock;
+
     std::atomic<GUID> m_vmId{GUID_NULL};
 
     /// <summary>