feat(nimbus): add iOS end-to-end enrollment integration test (#15403)

Because

* The iOS Nimbus integration test has been dormant since the CircleCI
scaffolding was removed; without live CI coverage, changes to the recipe
contract regress silently. This is the iOS counterpart to #15340 /
#15345.
* Unlike Fenix (signed APK on a TaskCluster public route), firefox-ios
publishes no prebuilt simulator artifact. We build from source on every
run.
* macos-15 GHA runners can't host Docker (Apple Virtualization rejects
nested-virt boot), so the Experimenter stack and the iOS build can't
share a job.

This commit

* Splits the workflow into two jobs coordinating via a recipe-JSON
artifact: `Create iOS recipe` (ubuntu) brings up the Experimenter stack
and writes a bucket-allocated recipe; `iOS Enrollment` (macos-15)
downloads it and enrolls firefox-ios against it.
* Builds the Fennec scheme on Xcode 26.2 / iPhone 17 / iOS 26.2, then
re-codesigns the bundle leaves-to-root with a minimal
`com.apple.security.application-groups` entitlement so the simulator
resolves the app group container (without it, `Experiments.dbPath` is
nil and Nimbus refuses to initialize).
* Verifies enrollment by polling
`<app-container>/Library/Caches/Logs/Firefox.log` for the `<slug> |
<features> | <branch>` row that `nimbus-cli enroll`'s embedded
`--log-state` produces, asserting the branch matches one of the recipe's
two ratio-1 branches.
* Pins the firefox-ios SHA in `firefox_ios_main_build.env`; adds an
`ios/main` entry to `update-firefox.yml` so the daily bumper updates the
pinned SHA and triggers the workflow against new firefox-ios HEAD.
* Gates the workflow on iOS-specific paths only — backend changes are
covered by the Fenix test, which exercises the same Experimenter
emission path.

Fixes #15402

Author: jaredlockhart

.github/workflows/ios-integration-test.yml

@@ -0,0 +1,200 @@
+name: iOS Enrollment Integration Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "3 12 * * *"
+  push:
+    branches:
+      - main
+      - update_firefox_ios_main
+  pull_request:
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  create-recipe:
+    name: "Create iOS recipe"
+    runs-on: ubuntu-24.04
+    outputs:
+      should-run: ${{ steps.check-paths.outputs.should-run }}
+    env:
+      INTEGRATION_TEST_NGINX_URL: https://localhost
+      IOS_CHANNEL: developer
+      IOS_RECIPE_PATH: ${{ github.workspace }}/ios_recipe.json
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/check-changed-paths
+        id: check-paths
+        with:
+          paths: "experimenter/tests/integration/nimbus/ios/ experimenter/tests/firefox_ios_main_build.env"
+
+      - uses: ./.github/actions/setup-cached-build
+        if: steps.check-paths.outputs.should-run == 'true'
+
+      - name: Set up Python
+        if: steps.check-paths.outputs.should-run == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install poetry
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: pipx install poetry
+
+      - name: Bring up Experimenter stack
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          cp .env.integration-tests .env
+          make refresh SKIP_DUMMY=1 up_prod_detached
+
+      - name: Wait for Experimenter backend to be ready
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: curl --retry 60 --retry-delay 5 --retry-all-errors -sfk -o /dev/null https://localhost/__lbheartbeat__
+
+      - name: Install test deps
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: poetry -C experimenter/tests install --no-root
+
+      - name: Create recipe
+        if: steps.check-paths.outputs.should-run == 'true'
+        working-directory: experimenter/tests
+        run: poetry run pytest -m ios_create_recipe -o addopts= -p no:rerunfailures integration/nimbus/ios
+
+      - name: Upload recipe artifact
+        if: steps.check-paths.outputs.should-run == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ios-recipe
+          path: ${{ github.workspace }}/ios_recipe.json
+          if-no-files-found: error
+          retention-days: 1
+
+  ios-test:
+    name: "iOS Enrollment"
+    needs: create-recipe
+    if: needs.create-recipe.outputs.should-run == 'true'
+    runs-on: macos-15
+    env:
+      IOS_CHANNEL: developer
+      XCODE_VERSION: "26.2"
+      IOS_VERSION: "26.2"
+      IOS_SIMULATOR: "iPhone 17"
+      IOS_RECIPE_PATH: ${{ github.workspace }}/ios_recipe.json
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Read firefox-ios SHA
+        run: |
+          . experimenter/tests/firefox_ios_main_build.env
+          if [ -z "$FIREFOX_IOS_SHA" ]; then
+            echo "::error::FIREFOX_IOS_SHA is empty in experimenter/tests/firefox_ios_main_build.env."
+            exit 1
+          fi
+          echo "FIREFOX_IOS_SHA=$FIREFOX_IOS_SHA" >> "$GITHUB_ENV"
+
+      - name: Checkout firefox-ios
+        uses: actions/checkout@v6
+        with:
+          repository: mozilla-mobile/firefox-ios
+          ref: ${{ env.FIREFOX_IOS_SHA }}
+          path: firefox-ios-checkout
+
+      - name: Select Xcode ${{ env.XCODE_VERSION }}
+        run: |
+          sudo rm -rf /Applications/Xcode.app
+          sudo xcode-select -s /Applications/Xcode_${{ env.XCODE_VERSION }}.app/Contents/Developer
+
+      - name: Bootstrap firefox-ios
+        working-directory: firefox-ios-checkout
+        run: ./bootstrap.sh firefox --force
+
+      - name: Build Fennec scheme
+        working-directory: firefox-ios-checkout/firefox-ios
+        run: |
+          xcodebuild -resolvePackageDependencies -onlyUsePackageVersionsFromResolvedFile
+          xcodebuild build \
+            -project Client.xcodeproj \
+            -scheme Fennec \
+            -destination "platform=iOS Simulator,name=${IOS_SIMULATOR},OS=${IOS_VERSION}" \
+            -derivedDataPath ./DerivedData \
+            COMPILER_INDEX_STORE_ENABLE=NO \
+            CODE_SIGN_IDENTITY= CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO \
+            ENABLE_USER_SCRIPT_SANDBOXING=NO \
+            ARCHS="arm64"
+
+      - name: Locate and ad-hoc sign Client.app
+        run: |
+          APP_PATH=$(find firefox-ios-checkout/firefox-ios/DerivedData/Build/Products -path "*iphonesimulator*/Client.app" -type d | head -1)
+          if [ -z "$APP_PATH" ]; then
+            echo "::error::Could not locate Client.app in DerivedData"
+            find firefox-ios-checkout/firefox-ios/DerivedData/Build/Products -maxdepth 3 -type d || true
+            exit 1
+          fi
+          # Sign leaves-to-root. --deep is deprecated and doesn't always
+          # reach every embedded binary (dylibs, extensions, nested
+          # frameworks) reliably on macOS 15.
+          find "$APP_PATH" -name '*.dylib' -print0 | xargs -0 -I{} codesign --force --sign - --preserve-metadata=entitlements {}
+          find "$APP_PATH" -type d -name '*.framework' -print0 | xargs -0 -I{} codesign --force --sign - --preserve-metadata=entitlements {}
+          find "$APP_PATH" -type d -name '*.appex' -print0 | xargs -0 -I{} codesign --force --sign - --preserve-metadata=entitlements {}
+          # Apply a minimal entitlements plist with just the app group
+          # (group.org.mozilla.ios.Fennec) declaration so containerURL
+          # (forSecurityApplicationGroupIdentifier:) resolves to a real
+          # path. Without it, Nimbus refuses to init ("Nimbus didn't get
+          # to create, because of a nil dbPath"). The full Fennec
+          # entitlements plist has com.apple.developer.* keys that
+          # require a provisioning profile — SpringBoard rejects launch
+          # of ad-hoc-signed apps that claim them.
+          ENTITLEMENTS=$(mktemp -t simulator-entitlements-XXXXXX).plist
+          cat > "$ENTITLEMENTS" <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+          <plist version="1.0">
+          <dict>
+            <key>com.apple.security.application-groups</key>
+            <array>
+              <string>group.org.mozilla.ios.Fennec</string>
+            </array>
+          </dict>
+          </plist>
+          EOF
+          codesign --force --sign - --entitlements "$ENTITLEMENTS" "$APP_PATH"
+          echo "IOS_APP_PATH=${GITHUB_WORKSPACE}/${APP_PATH}" >> "$GITHUB_ENV"
+
+      - name: Install nimbus-cli
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/mozilla/application-services/main/install-nimbus-cli.sh -o /tmp/install-nimbus-cli.sh
+          sudo bash /tmp/install-nimbus-cli.sh --directory /usr/local/bin
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Install test deps
+        run: poetry -C experimenter/tests install --no-root
+
+      - name: Download recipe artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ios-recipe
+          path: ${{ github.workspace }}
+
+      - name: Boot iOS simulator
+        run: |
+          xcrun simctl boot "${IOS_SIMULATOR}"
+          xcrun simctl bootstatus "${IOS_SIMULATOR}" -b
+          # Launch Simulator.app so there's a real SpringBoard to foreground
+          # the app — without it, simctl-launched apps can be stuck in a
+          # background-ish state where AppDelegate never finishes launching.
+          open -ga Simulator
+          sleep 15
+
+      - name: Run iOS enrollment test
+        run: make integration_test_nimbus_ios

.github/workflows/update-firefox.yml

@@ -15,6 +15,7 @@ on:
           - desktop-beta
           - fenix-beta
           - fenix-release
+          - ios-main
 
 jobs:
   update:
@@ -39,6 +40,9 @@ jobs:
           - application: fenix
             channel: release
             display_name: Firefox Fenix Release
+          - application: ios
+            channel: main
+            display_name: Firefox iOS
     steps:
       - name: Check if this variant should run
         id: should-run

Makefile

@@ -262,6 +262,9 @@ integration_test_nimbus_fenix:
 	mkdir -p experimenter/tests/integration/test-reports
 	cd experimenter/tests && poetry run pytest -m fenix_enrollment -o addopts= -p no:rerunfailures --junitxml=integration/test-reports/fenix_enrollment.xml integration/nimbus/android $(PYTEST_ARGS)
 
+integration_test_nimbus_ios:
+	cd experimenter/tests && poetry run pytest -m ios_enrollment -o addopts= -p no:rerunfailures integration/nimbus/ios $(PYTEST_ARGS)
+
 # cirrus
 CIRRUS_ENABLE = export CIRRUS=1 &&
 CIRRUS_RUFF_FORMAT_CHECK = ruff format --check --diff .

experimenter/tests/firefox_ios_main_build.env

@@ -0,0 +1 @@
+FIREFOX_IOS_SHA="87dcc1641f953be97c43d83936112eba85d8e126"

experimenter/tests/integration/nimbus/ios/README.md

@@ -0,0 +1,25 @@
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture(name="ios_channel")
+def fixture_ios_channel():
+    return os.environ.get("IOS_CHANNEL", "developer")
+
+
+@pytest.fixture(name="ios_app_path")
+def fixture_ios_app_path():
+    return os.environ["IOS_APP_PATH"]
+
+
+@pytest.fixture(name="ios_recipe_path")
+def fixture_ios_recipe_path():
+    return Path(os.environ["IOS_RECIPE_PATH"])
+
+
+@pytest.fixture(name="ios_recipe")
+def fixture_ios_recipe(ios_recipe_path):
+    return json.loads(ios_recipe_path.read_text())

experimenter/tests/integration/nimbus/ios/conftest.py

@@ -0,0 +1,44 @@
+import json
+import os
+import time
+
+import pytest
+import requests
+from nimbus.models.base_dataclass import BaseExperimentApplications
+from nimbus.utils import helpers
+
+IOS_APP = BaseExperimentApplications.FIREFOX_IOS.value
+RECIPE_POLL_TIMEOUT = 60
+
+
+def wait_for_recipe(slug):
+    url = f"{os.environ.get('INTEGRATION_TEST_NGINX_URL', 'https://nginx')}/api/v6/experiments/{slug}/"
+    deadline = time.time() + RECIPE_POLL_TIMEOUT
+    while time.time() < deadline:
+        try:
+            resp = requests.get(url, verify=False, timeout=5)
+            if resp.status_code == 200 and resp.json().get("bucketConfig"):
+                return resp.json()
+        except (requests.RequestException, ValueError):
+            pass
+        time.sleep(1)
+    pytest.fail(f"Timed out waiting for recipe at {url}")
+
+
+@pytest.mark.ios_create_recipe
+def test_create_ios_recipe(ios_channel, ios_recipe_path):
+    slug = f"ios-{ios_channel}-integration-test"
+    feature_id = helpers.get_feature_id_as_string("no-feature-ios", IOS_APP)
+    assert feature_id
+    helpers.create_experiment(
+        slug,
+        IOS_APP,
+        data={
+            "feature_config_ids": [int(feature_id)],
+            "channel": ios_channel,
+            "firefox_min_version": "",
+        },
+        targeting="no_targeting",
+    )
+    helpers.launch_to_preview(slug)
+    ios_recipe_path.write_text(json.dumps(wait_for_recipe(slug)))

experimenter/tests/integration/nimbus/ios/test_ios_create_recipe.py

@@ -0,0 +1,52 @@
+import re
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+
+NIMBUS_CLI_APP = "firefox_ios"
+BUNDLE_ID = "org.mozilla.ios.Fennec"
+LOG_POLL_DEADLINE_SECONDS = 180
+LOG_POLL_INTERVAL_SECONDS = 2
+
+
+@pytest.mark.ios_enrollment
+def test_ios_enrollment(ios_channel, ios_app_path, ios_recipe, ios_recipe_path):
+    experiment_slug = ios_recipe["slug"]
+    subprocess.check_call(["xcrun", "simctl", "install", "booted", ios_app_path])
+    subprocess.check_call(
+        [
+            "nimbus-cli",
+            "--app",
+            NIMBUS_CLI_APP,
+            "--channel",
+            ios_channel,
+            "enroll",
+            experiment_slug,
+            "--branch",
+            "control",
+            "--file",
+            str(ios_recipe_path),
+            "--preserve-targeting",
+            "--preserve-bucketing",
+            "--reset-app",
+            "--no-validate",
+        ]
+    )
+
+    container = subprocess.check_output(
+        ["xcrun", "simctl", "get_app_container", "booted", BUNDLE_ID, "data"],
+        text=True,
+    ).strip()
+    log_path = Path(container) / "Library" / "Caches" / "Logs" / "Firefox.log"
+    pattern = re.compile(rf"{re.escape(experiment_slug)}\s*\|\s*\S+\s*\|\s*(\S+)")
+    deadline = time.time() + LOG_POLL_DEADLINE_SECONDS
+    while time.time() < deadline:
+        if log_path.exists():
+            match = pattern.search(log_path.read_text())
+            if match:
+                assert match.group(1).strip() in {"control", "treatment-a"}
+                return
+        time.sleep(LOG_POLL_INTERVAL_SECONDS)
+    pytest.fail(f"No enrollment row for {experiment_slug}")