Merge pull request #8170 from woocommerce/feat/8166-native-jetpack-installation-ab-testing

Native Jetpack Installation: Apply A/B test

Author: selanthiraiyan

Experiments/Experiments/ABTest.swift

@@ -16,6 +16,11 @@ public enum ABTest: String, CaseIterable {
     ///
     case abTestLoginWithWPComOnly = "woocommerceios_login_wpcom_only"
 
+    /// A/B test to measure the sign-in success rate when native Jetpack installation experience is enabled
+    /// Experiment ref: pbxNRc-29W-p2
+    ///
+    case nativeJetpackSetupFlow = "woocommerceios_login_jetpack_setup_flow"
+
     /// A/B test for the Products Onboarding banner on the My Store dashboard.
     /// Experiment ref: pbxNRc-26F-p2
     case productsOnboardingBanner = "woocommerceios_products_onboarding_first_product_banner"
@@ -34,7 +39,7 @@ public enum ABTest: String, CaseIterable {
     /// When adding a new experiment, add it to the appropriate case depending on its context (logged-in or logged-out experience).
     public var context: ExperimentContext {
         switch self {
-        case .productsOnboardingBanner, .productsOnboardingTemplateProducts:
+        case .productsOnboardingBanner, .productsOnboardingTemplateProducts, .nativeJetpackSetupFlow:
             return .loggedIn
         case .aaTestLoggedOut, .abTestLoginWithWPComOnly:
             return .loggedOut
@@ -47,6 +52,7 @@ public enum ABTest: String, CaseIterable {
 public extension ABTest {
     /// Start the AB Testing platform if any experiment exists for the provided context
     ///
+    @MainActor
     static func start(for context: ExperimentContext) async {
         let experiments = ABTest.allCases.filter { $0.context == context }
 

Experiments/Experiments/DefaultFeatureFlagService.swift

@@ -51,8 +51,6 @@ public struct DefaultFeatureFlagService: FeatureFlagService {
                 .performanceMonitoringUserInteraction:
             // Disabled by default to avoid costs spikes, unless in internal testing builds.
             return buildConfig == .alpha
-        case .nativeJetpackSetupFlow:
-            return buildConfig == .localDeveloper || buildConfig == .alpha
         case .analyticsHub:
             return buildConfig == .localDeveloper || buildConfig == .alpha
         case .tapToPayOnIPhone:

Experiments/Experiments/FeatureFlag.swift

@@ -130,11 +130,6 @@ public enum FeatureFlag: Int {
     /// - Note: The app will ignore this if `performanceMonitoring` is `false`.
     case performanceMonitoringViewController
 
-    /// Temporary feature flag for the native Jetpack setup flow.
-    /// TODO-8075: replace this with A/B test.
-    ///
-    case nativeJetpackSetupFlow
-
     /// Temporary feature flag for the native Jetpack setup flow.
     ///
     case analyticsHub

WooCommerce/Classes/Analytics/WooAnalytics.swift

@@ -1,3 +1,4 @@
+import Experiments
 import Foundation
 import UIKit
 import WordPressShared
@@ -60,6 +61,15 @@ public extension WooAnalytics {
         }
 
         analyticsProvider.refreshUserData()
+
+        // Refreshes A/B experiments since `ExPlat.shared` is reset after each `TracksProvider.refreshUserData` call
+        // and any A/B test assignments that come back after the shared instance is reset won't be saved for later
+        // access.
+        let context: ExperimentContext = ServiceLocator.stores.isAuthenticated ?
+            .loggedIn: .loggedOut
+        Task { @MainActor in
+            await ABTest.start(for: context)
+        }
     }
 
     /// Track a spcific event without any associated properties

WooCommerce/Classes/AppDelegate.swift

@@ -73,15 +73,12 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         // ever new source code is injected into our application.
         Inject.animation = .interactiveSpring()
 
-        Task { @MainActor in
-            await startABTesting()
-
-            // Upgrade check...
-            // This has to be called after A/B testing setup in `startABTesting` if any of the Tracks events
-            // in `checkForUpgrades` is used as an exposure event for an experiment.
-            // For example, `application_installed` could be the exposure event for logged-out experiments.
-            checkForUpgrades()
-        }
+        // Upgrade check...
+        // This has to be called after A/B testing setup in `setupAnalytics` (which calls
+        // `WooAnalytics.refreshUserData`) if any of the Tracks events in `checkForUpgrades` is
+        // used as an exposure event for an experiment.
+        // For example, `application_installed` could be the exposure event for logged-out experiments.
+        checkForUpgrades()
 
         return true
     }
@@ -372,13 +369,6 @@ private extension AppDelegate {
         }
     }
 
-    /// Starts the AB testing platform and fetches test assignments for the current context
-    ///
-    func startABTesting() async {
-        let context: ExperimentContext = ServiceLocator.stores.isAuthenticated ? .loggedIn : .loggedOut
-        await ABTest.start(for: context)
-    }
-
     /// Tracks if the application was opened via a widget tap.
     ///
     func trackWidgetTappedIfNeeded(userActivity: NSUserActivity) {

WooCommerce/Classes/Authentication/AuthenticationManager.swift

@@ -761,8 +761,7 @@ private extension AuthenticationManager {
         }
 
         // Shows the native Jetpack flow during the site discovery flow.
-        // TODO-8075: replace feature flag with A/B testing
-        if featureFlagService.isFeatureFlagEnabled(.nativeJetpackSetupFlow) {
+        if ABTest.nativeJetpackSetupFlow.variation != .control {
             return jetpackSetupUI(for: site.url,
                                   connectionMissingOnly: site.hasJetpack && site.isJetpackActive,
                                   in: navigationController)

WooCommerce/Classes/Authentication/Epilogue/StorePickerViewController.swift

@@ -201,7 +201,6 @@ final class StorePickerViewController: UIViewController {
         switch configuration {
         case .login:
             startListeningToNotifications()
-            startABTesting()
         case .switchingStores:
             secondaryActionButton.isHidden = true
         default:
@@ -574,17 +573,6 @@ private extension StorePickerViewController {
         fancyAlert.transitioningDelegate = AppDelegate.shared.tabBarController
         present(fancyAlert, animated: true)
     }
-
-    /// Refreshes the AB testing assignments (refresh is needed after a user logs in)
-    ///
-    func startABTesting() {
-        guard ServiceLocator.stores.isAuthenticated else {
-            return
-        }
-        Task { @MainActor in
-            await ABTest.start(for: .loggedIn)
-        }
-    }
 }
 
 // MARK: Transition Controller Delegate

WooCommerce/Classes/Yosemite/DefaultStoresManager.swift

@@ -178,11 +178,6 @@ class DefaultStoresManager: StoresManager {
 
         updateAndReloadWidgetInformation(with: nil)
 
-        // Refresh the A/B test assignments for the logged-out context
-        Task { @MainActor in
-            await ABTest.start(for: .loggedOut)
-        }
-
         NotificationCenter.default.post(name: .logOutEventReceived, object: nil)
 
         return self