Stabilize Windows MSI recovery packaging

Author: danjboyd

.github/workflows/windows-packaging.yml

@@ -33,15 +33,13 @@ jobs:
 
   build-msi:
     needs: resolve-version
-    uses: danjboyd/gnustep-packager/.github/workflows/package-gnustep-app.yml@bca864ff163e129100881145e017429fed155bf7
+    uses: danjboyd/gnustep-packager/.github/workflows/package-gnustep-app.yml@4fc362a68b3e55191942c01a92cf2f8da82031bb
     with:
       manifest-path: packaging/manifests/windows-msi.manifest.json
       backend: msi
       package-version: ${{ needs.resolve-version.outputs.package_version }}
       packager-repository: danjboyd/gnustep-packager
-      packager-ref: bca864ff163e129100881145e017429fed155bf7
-      msys2-packages: >-
-        mingw-w64-clang-x86_64-cmark
+      packager-ref: 4fc362a68b3e55191942c01a92cf2f8da82031bb
       run-validation: true
       run-smoke: true
       upload-artifacts: true

ObjcMarkdownViewer/OMDAppDelegate.m

@@ -3532,14 +3532,8 @@ - (void)drawRect:(NSRect)dirtyRect
 
 static void OMDAddPreferencesPopupOverlay(NSView *container, NSPopUpButton *popup)
 {
-    if (container == nil || popup == nil) {
-        return;
-    }
-    OMDPreferencesPopupOverlayView *overlay = [[[OMDPreferencesPopupOverlayView alloc] initWithFrame:[popup frame]
-                                                                                         popupButton:popup] autorelease];
-    [container addSubview:overlay
-               positioned:NSWindowAbove
-               relativeTo:popup];
+    (void)container;
+    (void)popup;
 }
 
 @interface OMDRoundedCardView : OMDFlippedFillView

ObjcMarkdownViewer/main.m

@@ -71,11 +71,16 @@ static void OMDStartupTrace(NSString *message)
 {
 #if defined(_WIN32)
     NSString *bundleRoot = OMDWindowsInstallRoot();
-    NSString *defaultsPath = [[bundleRoot stringByAppendingPathComponent:@"clang64"]
-        stringByAppendingPathComponent:@"bin\\defaults.exe"];
-
-    if ([[NSFileManager defaultManager] isExecutableFileAtPath:defaultsPath]) {
-        return defaultsPath;
+    NSArray *relativePaths = [NSArray arrayWithObjects:
+        @"runtime\\bin\\defaults.exe",
+        @"clang64\\bin\\defaults.exe",
+        nil];
+
+    for (NSString *relativePath in relativePaths) {
+        NSString *defaultsPath = [bundleRoot stringByAppendingPathComponent:relativePath];
+        if ([[NSFileManager defaultManager] isExecutableFileAtPath:defaultsPath]) {
+            return defaultsPath;
+        }
     }
 #endif
     return @"defaults.exe";
@@ -121,7 +126,7 @@ static BOOL OMDWindowsBundledExecutableExists(NSString *relativePath)
 {
 #if defined(_WIN32)
     NSString *bundleRoot = OMDWindowsInstallRoot();
-    NSString *bundledThemesRoot = nil;
+    NSMutableArray *bundledThemeRoots = [NSMutableArray array];
     NSString *userThemesRoot = [[[NSHomeDirectory() stringByAppendingPathComponent:@"GNUstep"]
         stringByAppendingPathComponent:@"Library"] stringByAppendingPathComponent:@"Themes"];
     NSString *systemThemesRoot = [[[@"C:\\clang64" stringByAppendingPathComponent:@"lib"]
@@ -130,12 +135,17 @@ static BOOL OMDWindowsBundledExecutableExists(NSString *relativePath)
     NSString *themeName = nil;
 
     if (bundleRoot != nil && [bundleRoot length] > 0) {
-        bundledThemesRoot = [[[[bundleRoot stringByAppendingPathComponent:@"clang64"]
+        [bundledThemeRoots addObject:[[[[bundleRoot stringByAppendingPathComponent:@"runtime"]
+            stringByAppendingPathComponent:@"lib"] stringByAppendingPathComponent:@"GNUstep"]
+            stringByAppendingPathComponent:@"Themes"]];
+        [bundledThemeRoots addObject:[[[[bundleRoot stringByAppendingPathComponent:@"clang64"]
             stringByAppendingPathComponent:@"lib"] stringByAppendingPathComponent:@"GNUstep"]
-            stringByAppendingPathComponent:@"Themes"];
+            stringByAppendingPathComponent:@"Themes"]];
         for (themeName in preferredThemes) {
-            if (OMDWindowsThemeBundleExistsInDirectory(bundledThemesRoot, themeName)) {
-                return themeName;
+            for (NSString *bundledThemesRoot in bundledThemeRoots) {
+                if (OMDWindowsThemeBundleExistsInDirectory(bundledThemesRoot, themeName)) {
+                    return themeName;
+                }
             }
         }
     }

OpenIssues.md

@@ -118,8 +118,27 @@
   - The remote Windows build reached the `ObjcMarkdown` compile and failed on missing `cmark.h`.
   - The immediate local fix was to update the ad hoc remote build script to install `mingw-w64-clang-x86_64-cmark` before invoking the packaging pipeline.
   - The long-term fix should be to declare `cmark` in the manifest under `hostDependencies.windows.msys2Packages` and then repin/use the updated `gnustep-packager` commit so host dependency realization happens systematically.
+- **Phase 10A-C Recovery Findings (2026-04-30)**:
+  - Connected to the known-good libvirt Windows VM at `172.17.2.148` as `Administrator`; hostname `OTVM-WIN-05SPHS`.
+  - Captured environment and payload evidence in `dist/phase10-msi/` and summarized it in [docs/windows-phase10-msi-recovery.md](/home/danboyd/git/ObjcMarkdown/docs/windows-phase10-msi-recovery.md).
+  - The known-good VM has `ObjcMarkdown` at `87ea70ed2df2bac8c634b6b82471339ce5dc8e6d` and `plugins-themes-WinUITheme` at `48d21f0a2ae97ca70a03197d93a305144635517f`.
+  - The known-good runtime is not an existing `dist/packaging/windows/stage`; it is a source-tree app build using `C:\\Users\\Administrator\\AppData\\Local\\gnustep-cli`.
+  - A temporary imported-payload stage produced an MSI through `gnustep-packager`; clean Windows install succeeded, then validation failed because the imported payload did not include TinyTeX.
+  - The normal source stage includes TinyTeX but differs materially from the imported known-good payload, especially project DLL placement and overall runtime size.
+  - Normal source staging on the gnustep-cli runtime still logs `/etc/profile` and `/c/...` path warnings, confirming a `gnustep-cli-new` mount-convention gap.
+- **Phase 10D-G Recovery Work (2026-04-30)**:
+  - [packaging/manifests/windows-msi.manifest.json](/home/danboyd/git/ObjcMarkdown/packaging/manifests/windows-msi.manifest.json) now declares `mingw-w64-clang-x86_64-cmark` under `hostDependencies.windows.msys2Packages`, declares required Windows theme inputs, sets packaged default theme `WinUITheme`, and ignores confirmed Windows system DLLs `DNSAPI.dll` and `IPHLPAPI.DLL` during MSI runtime closure checks.
+  - [.github/workflows/windows-packaging.yml](/home/danboyd/git/ObjcMarkdown/.github/workflows/windows-packaging.yml) is repinned to `gnustep-packager` commit `4fc362a68b3e55191942c01a92cf2f8da82031bb`, which includes manifest-driven host dependency provisioning.
+  - [scripts/windows/build-from-powershell.ps1](/home/danboyd/git/ObjcMarkdown/scripts/windows/build-from-powershell.ps1) and [packaging/scripts/ensure-windows-theme-inputs.ps1](/home/danboyd/git/ObjcMarkdown/packaging/scripts/ensure-windows-theme-inputs.ps1) now detect whether the managed MSYS runtime exposes Windows drives as `/c` or `/cygdrive/c` and only source `/etc/profile` when it exists.
+  - The known-good VM validates the fixed build script against `C:\\Users\\Administrator\\AppData\\Local\\gnustep-cli`: `-Task command -Command pwd` resolves to `/cygdrive/c/Users/Administrator/git/ObjcMarkdown`, and `-Task stage -StageDir dist/phase10-source-stage-fixed/windows/stage` completes cleanly.
+  - The temporary imported-payload MSI path is quarantined as evidence only under ignored `dist/phase10-msi/`; it is not a release path because it omits TinyTeX and imports the full mutable `gnustep-cli` runtime.
+- **Manual MSI Validation Findings (2026-04-30)**:
+  - Candidate `dist/packaging/windows/packages/phase10-manual/ObjcMarkdown-0.1.1-rc2-win64.msi` installed and launched on clean validation VM `lease-20260430203645-pbs16d` at `172.17.2.177`.
+  - The installed payload includes `WinUITheme.dll` under `C:\\Users\\Administrator\\AppData\\Local\\ObjcMarkdown\\runtime\\lib\\GNUstep\\Themes\\WinUITheme.theme`, but the app launched with the GNUstep theme active.
+  - Runtime inspection showed `GSTheme=WinUITheme` in the app defaults domain, while `NSGlobalDomain` had no `GSTheme`; app startup was still searching the old `clang64` packaged layout instead of the MSI's `runtime` layout for `defaults.exe` and bundled themes.
+  - Local fix: [ObjcMarkdownViewer/main.m](/home/danboyd/git/ObjcMarkdown/ObjcMarkdownViewer/main.m) now searches `runtime\\bin\\defaults.exe` before `clang64\\bin\\defaults.exe` and checks `runtime\\lib\\GNUstep\\Themes` before the legacy packaged theme path.
+  - Preferences validation found that clicking the `GNUstep Theme` popup opened the neighboring `Layout Mode` popup; Linux validation also showed broken hover tracking because Preferences used a custom popup overlay and manually drawn popup panel instead of native `NSPopUpButton` menu tracking.
+  - Local fix: [ObjcMarkdownViewer/OMDAppDelegate.m](/home/danboyd/git/ObjcMarkdown/ObjcMarkdownViewer/OMDAppDelegate.m) now disables the custom Preferences popup overlay path so the theme and layout controls use native `NSPopUpButton` popup behavior.
 - **Next Step**:
-  - Update `ObjcMarkdown`’s Windows packaging manifest to declare `mingw-w64-clang-x86_64-cmark` under `hostDependencies.windows.msys2Packages`.
-  - Repin the repo to the new `gnustep-packager` commit that includes host dependency provisioning.
-  - Rebuild the MSI through the normal packaging path instead of the increasingly patched ad hoc OCI remote bundle.
-  - Once a fresh MSI is produced, push the `.msi` and portable `.zip` to a Windows validation VM and re-run manual/UAT verification.
+  - Rebuild the MSI through the normal source-built packaging path with the app-side runtime-layout and native-popup fixes.
+  - Once a fresh MSI is produced, push the `.msi` and portable `.zip` to a clean Windows validation VM and re-run manual/UAT verification.

README.md

@@ -167,7 +167,7 @@ Release flow:
 - Ensure the target commit has already passed the separate Linux CI workflow if you want a GNUstep/Linux gate before release tagging.
 - Push an annotated tag like `v0.1.0`.
 - GitHub Actions runs `linux-appimage` as a thin caller to the reusable `gnustep-packager` workflow pinned to `bca864ff163e129100881145e017429fed155bf7`, using this repo's Linux manifest, stage script, and self-hosted GNUstep preflight.
-- GitHub Actions runs `windows-packaging` as a thin caller to the same pinned reusable workflow, using this repo's Windows MSI manifest and normalized Windows stage script. The staged Windows payload includes the GNUstep runtime, bundled Windows themes, and TinyTeX runtime for external LaTeX rendering. Windows releases are expected to bundle `WinUITheme` and use it as the default packaged theme.
+- GitHub Actions runs `windows-packaging` as a thin caller to the reusable `gnustep-packager` workflow pinned to `4fc362a68b3e55191942c01a92cf2f8da82031bb`, using this repo's Windows MSI manifest and normalized Windows stage script. The Windows manifest owns app-specific host dependencies such as `mingw-w64-clang-x86_64-cmark`. The staged Windows payload includes the GNUstep runtime, bundled Windows themes, and TinyTeX runtime for external LaTeX rendering. Windows releases are expected to bundle `WinUITheme` and use it as the default packaged theme.
 - Each tagged packaging workflow then downloads its `-packages` artifact and attaches the release files to the matching GitHub Release page. Linux publishes the `.AppImage` and `.zsync`; Windows publishes the `.msi` and portable ZIP, along with generated sidecars such as `.update-feed.json`.
 - Clean-machine Windows validation is documented in [docs/windows-otvm-msi-validation.md](docs/windows-otvm-msi-validation.md). Going forward, the supported Debian and Windows VM path is libvirt-backed `OracleTestVMs` leases. The older direct-OCI helper has been retired; [docs/windows-oci-msi-validation.md](docs/windows-oci-msi-validation.md) is kept only as a retirement note.
 

Roadmap.md

@@ -143,7 +143,7 @@ Scope:
 - configure the Windows caller to use:
   - `windows-latest`
   - the packager MSI baseline
-  - `msys2-packages` for app-specific dependencies such as `mingw-w64-clang-x86_64-cmark`
+  - manifest-declared host dependencies for app-specific packages such as `mingw-w64-clang-x86_64-cmark`
 - keep CI validation enabled for both backends and preserve any extra app-specific smoke we still need beyond the shared packager validation
 
 ### Phase 8E: External-Only Backend Packaging Cleanup
@@ -245,6 +245,108 @@ Acceptance criteria:
 - `Phase 9D`: separate Windows build and clean-test VMs can be launched from `OracleTestVMs`, the clean-test VM contains sample Markdown files, and the MSI can be validated there without relying on build-machine state
 - `Phase 9E`: the operator can follow repo-documented instructions to connect to the Linux VM and the Windows test VM and perform manual validation of AppImage and MSI behavior
 
+## Phase 10: Golden Windows MSI Reproduction Through gnustep-packager
+
+Goal:
+- use the known-good Windows build VM as a controlled reference point so `gnustep-packager` and `gnustep-cli-new` can reproduce a working MSI without relying on session memory, mutable machine state, or ad hoc repair scripts
+
+### Phase 10A: Snapshot The Known-Good Windows Build VM
+
+Scope:
+- connect to the known-good Windows VM through `OracleTestVMs`/libvirt SSH and record its exact build identity:
+  - `ObjcMarkdown` commit
+  - `plugins-themes-winuitheme` commit
+  - `plugins-themes-win11theme` commit
+  - `gnustep-packager` commit if present
+- export the Windows build environment facts needed for reproduction:
+  - MSYS2 package list
+  - `C:\\msys64\\clang64` GNUstep layout
+  - `GNUSTEP_*`, `PATH`, `MSYSTEM`, and `MSYS2_LOCATION`
+  - successful build, stage, and install/runtime tree locations
+- save the result as a committed diagnostic note or generated evidence bundle referenced from repo documentation
+
+### Phase 10B: Prove MSI Generation From The Known-Good Payload
+
+Scope:
+- add a temporary, explicitly marked payload-import workflow for recovery testing only
+- copy the VM's known-good app/runtime/theme payload into `dist/packaging/windows/stage`
+- run only the `gnustep-packager` MSI backend against that imported stage
+- validate the resulting MSI on a clean `OracleTestVMs` Windows VM
+- use the result to separate installer-generation defects from source-build/staging defects
+
+### Phase 10C: Diff Known-Good, Staged, And Installed Payloads
+
+Scope:
+- generate relative path, size, hash, and DLL dependency reports for:
+  - the known-good VM payload
+  - `packaging/scripts/stage-windows.ps1` output
+  - the tree installed by the MSI
+- identify meaningful differences rather than accepting broad directory drift
+- make validation fail for missing or incorrect release-critical content:
+  - `WinUITheme.theme/WinUITheme.dll`
+  - packaged default `GSTheme=WinUITheme`
+  - `cmark` runtime dependency coverage
+  - GNUstep backend bundle
+  - TinyTeX binaries
+  - unresolved non-system DLLs
+
+### Phase 10D: Move VM Facts Into Declarative Packager Inputs
+
+Scope:
+- declare app-specific Windows host dependencies directly in `packaging/manifests/windows-msi.manifest.json`
+- keep `mingw-w64-clang-x86_64-cmark` in the manifest rather than relying only on workflow inputs
+- add any additional MSYS2 packages proven necessary by the known-good VM snapshot
+- keep Windows theme requirements explicit through packaging inputs and validation rather than hand-prepared machine state
+
+### Phase 10E: Repin To The Required gnustep-packager Baseline
+
+Scope:
+- repin the Windows packaging workflow from the older reusable-workflow commit to the `gnustep-packager` commit that contains:
+  - host dependency provisioning
+  - first-class GNUstep theme input handling and validation
+  - packaged default-theme support
+  - Windows executable smoke validation
+- update packaging docs and release notes with the exact packager commit used for the next MSI recovery build
+- verify the repinned workflow still produces the expected MSI, portable ZIP, update feed, metadata, and diagnostics artifacts
+
+### Phase 10F: Harden gnustep-cli-new Against The Golden VM
+
+Scope:
+- treat the known-good VM's `C:\\msys64` state as the acceptance target for `gnustep-cli-new`
+- make `gnustep-cli-new` install or verify the same MSYS2/GNUstep prerequisites required by this app
+- add or update a `doctor`/probe output that reports the key differences between a fresh toolchain and the known-good build VM
+- remove recovery-only bootstrap steps once `gnustep-cli-new` can create an equivalent packaging-capable environment
+
+### Phase 10G: Retire The Temporary Payload Import Path
+
+Scope:
+- remove or quarantine the payload-import workflow after source-built staging matches the known-good payload and clean-machine MSI validation passes
+- keep the payload comparison and environment probe scripts as regression tools
+- document the final normal path as:
+  - `gnustep-cli-new` provisions or verifies the toolchain
+  - repo scripts build and stage the app
+  - `gnustep-packager` creates the MSI and portable ZIP
+  - `OracleTestVMs` validates the installer on a clean Windows VM
+
+Immediate next steps:
+- `Phase 10A`: completed on `2026-04-30` by exporting known-good VM evidence from `Administrator@172.17.2.148` on `OTVM-WIN-05SPHS`
+- `Phase 10B`: completed on `2026-04-30` enough to prove the MSI backend can package the imported known-good payload; clean install succeeded, but release validation failed because the imported payload lacks TinyTeX
+- `Phase 10C`: completed on `2026-04-30` with a payload comparison between the imported known-good stage, normal source stage, and MSI-installed tree
+- `Phase 10D`: completed on `2026-04-30` by moving `mingw-w64-clang-x86_64-cmark`, Windows theme inputs, packaged default theme, and confirmed system-DLL ignores into the Windows manifest
+- `Phase 10E`: completed on `2026-04-30` by repinning `.github/workflows/windows-packaging.yml` to `gnustep-packager` commit `4fc362a68b3e55191942c01a92cf2f8da82031bb`
+- `Phase 10F`: completed on `2026-04-30` for this repo's Windows build/theme scripts by resolving the active MSYS drive mount convention and making `/etc/profile` optional under `gnustep-cli-new`
+- `Phase 10G`: import-lane retirement started on `2026-04-30`; the imported MSI is retained only as ignored evidence under `dist/phase10-msi/`, and the release path is the normal source-built stage plus `gnustep-packager` MSI and clean `OracleTestVMs` validation
+- next verification gate: rebuild the MSI through the normal source-built path and validate that release artifact on a clean Windows VM
+
+Acceptance criteria:
+- `Phase 10A`: the known-good VM's build inputs, package inventory, environment, and payload locations are captured in a durable repo-referenced artifact
+- `Phase 10B`: `gnustep-packager` can create an MSI from the imported known-good payload and that MSI passes clean Windows validation
+- `Phase 10C`: validation reports clearly explain payload differences between the known-good VM, source-built stage, and MSI-installed tree
+- `Phase 10D`: app-specific Windows host dependencies and theme requirements are declared in manifests or packaging inputs, not hidden in VM state
+- `Phase 10E`: the Windows packaging workflow is pinned to a `gnustep-packager` baseline that includes the required MSI recovery features
+- `Phase 10F`: `gnustep-cli-new` can provision or verify a fresh Windows build environment equivalent to the known-good VM for this app's packaging needs
+- `Phase 10G`: the normal source-built `gnustep-packager` MSI path passes clean-machine validation without the temporary payload-import workflow
+
 ## Deferred Work
 
 These are interesting, but they are not the current release gate: