Merge pull request #7996 from woocommerce/issue/7904-simplified-login-ab-testing

itsmeichigo · web-flow · commit 47784c564572 · 2022-11-03T19:34:56.000+07:00
Simplified login: Apply A/B test
diff --git a/Experiments/Experiments/ABTest.swift b/Experiments/Experiments/ABTest.swift
@@ -16,6 +16,11 @@ public enum ABTest: String, CaseIterable {
     /// Experiment ref: pbxNRc-1S0-p2
     case aaTestLoggedOut = "woocommerceios_explat_aa_test_logged_out_202211"
 
+    /// A/B test to measure the sign-in success rate when only WPCom login is enabled.
+    /// Experiment ref: pbxNRc-27s-p2
+    ///
+    case abTestLoginWithWPComOnly = "woocommerceios_login_wpcom_only"
+
     /// A/B test for the Products Onboarding banner on the My Store dashboard.
     /// Experiment ref: pbxNRc-26F-p2
     case productsOnboardingBanner = "woocommerceios_products_onboarding_first_product_banner"
@@ -32,7 +37,7 @@ public enum ABTest: String, CaseIterable {
         switch self {
         case .aaTestLoggedIn202210, .productsOnboardingBanner:
             return .loggedIn
-        case .aaTestLoggedOut:
+        case .aaTestLoggedOut, .abTestLoginWithWPComOnly:
             return .loggedOut
         case .null:
             return .none
diff --git a/Experiments/Experiments/DefaultFeatureFlagService.swift b/Experiments/Experiments/DefaultFeatureFlagService.swift
@@ -34,17 +34,11 @@ public struct DefaultFeatureFlagService: FeatureFlagService {
         case .searchProductsBySKU:
             return true
         case .wpcomSignup:
-            guard isFeatureFlagEnabled(.simplifiedLoginFlowI1) else {
-                return buildConfig == .localDeveloper || buildConfig == .alpha
-            }
-            // To disable automatically sending signup link for unknown email IDs
             return false
         case .inAppPurchases:
             return buildConfig == .localDeveloper || buildConfig == .alpha
         case .storeCreationMVP:
             return buildConfig == .localDeveloper || buildConfig == .alpha
-        case .simplifiedLoginFlowI1:
-            return buildConfig == .localDeveloper || buildConfig == .alpha
         case .justInTimeMessagesOnDashboard:
             return buildConfig == .localDeveloper || buildConfig == .alpha
         case .productsOnboarding:
diff --git a/Experiments/Experiments/FeatureFlag.swift b/Experiments/Experiments/FeatureFlag.swift
@@ -82,11 +82,6 @@ public enum FeatureFlag: Int {
     ///
     case storeCreationMVP
 
-    /// Temporary feature flag for the simplified login flow.
-    /// TODO: replace with A/B testing.
-    ///
-    case simplifiedLoginFlowI1
-
     /// Just In Time Messages on Dashboard
     ///
     case justInTimeMessagesOnDashboard
diff --git a/WooCommerce/Classes/Authentication/AuthenticationManager.swift b/WooCommerce/Classes/Authentication/AuthenticationManager.swift
@@ -55,7 +55,7 @@ class AuthenticationManager: Authentication {
     func initialize(loggedOutAppSettings: LoggedOutAppSettingsProtocol) {
         let isWPComMagicLinkPreferredToPassword = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.loginMagicLinkEmphasis)
         let isWPComMagicLinkShownAsSecondaryActionOnPasswordScreen = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.loginMagicLinkEmphasisM2)
-        let isSimplifiedLoginI1Enabled = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.simplifiedLoginFlowI1)
+        let isSimplifiedLoginI1Enabled = ABTest.abTestLoginWithWPComOnly.variation != .control
         let isStoreCreationMVPEnabled = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.storeCreationMVP)
         let configuration = WordPressAuthenticatorConfiguration(wpcomClientId: ApiCredentials.dotcomAppId,
                                                                 wpcomSecret: ApiCredentials.dotcomSecret,
@@ -81,9 +81,9 @@ class AuthenticationManager: Authentication {
                                                                 enableWPComLoginOnlyInPrologue: isSimplifiedLoginI1Enabled,
                                                                 enableSiteCreation: isStoreCreationMVPEnabled,
                                                                 enableSocialLogin: !isSimplifiedLoginI1Enabled,
-                                                                emphasizeEmailForWPComPassword: isSimplifiedLoginI1Enabled,
-                                                                wpcomPasswordInstructions: isSimplifiedLoginI1Enabled ?
-                                                                AuthenticationConstants.wpcomPasswordInstructions : nil)
+                                                                emphasizeEmailForWPComPassword: true,
+                                                                wpcomPasswordInstructions:
+                                                                AuthenticationConstants.wpcomPasswordInstructions)
 
         let systemGray3LightModeColor = UIColor(red: 199/255.0, green: 199/255.0, blue: 204/255.0, alpha: 1)
         let systemLabelLightModeColor = UIColor(red: 0, green: 0, blue: 0, alpha: 1)
diff --git a/WooCommerce/Classes/Authentication/Navigation Exceptions/NotWPAccountViewModel.swift b/WooCommerce/Classes/Authentication/Navigation Exceptions/NotWPAccountViewModel.swift
@@ -21,9 +21,7 @@ final class NotWPAccountViewModel: ULErrorViewModel {
 
     let isPrimaryButtonHidden: Bool
 
-    var secondaryButtonTitle: String {
-        isSimplifiedLoginI1Enabled ? Localization.tryAnotherAddress : Localization.restartLogin
-    }
+    let secondaryButtonTitle = Localization.tryAnotherAddress
     let isSecondaryButtonHidden: Bool
 
     private weak var viewController: UIViewController?
@@ -42,15 +40,13 @@ final class NotWPAccountViewModel: ULErrorViewModel {
         return button
     }()
 
-    private let isSimplifiedLoginI1Enabled: Bool
     private let analytics: Analytics
     private var storePickerCoordinator: StorePickerCoordinator?
 
     init(error: Error,
          analytics: Analytics = ServiceLocator.analytics,
          featureFlagService: FeatureFlagService = ServiceLocator.featureFlagService) {
         self.analytics = analytics
-        self.isSimplifiedLoginI1Enabled = featureFlagService.isFeatureFlagEnabled(.simplifiedLoginFlowI1)
         if let error = error as? SignInError,
            case let .invalidWPComEmail(source) = error,
            source == .wpComSiteAddress {
@@ -60,23 +56,14 @@ final class NotWPAccountViewModel: ULErrorViewModel {
         } else {
             isSecondaryButtonHidden = false
 
-            if isSimplifiedLoginI1Enabled {
-                primaryButtonTitle = Localization.createAnAccount
-                isPrimaryButtonHidden = !featureFlagService.isFeatureFlagEnabled(.storeCreationMVP)
-            } else {
-                primaryButtonTitle = Localization.loginWithSiteAddress
-                isPrimaryButtonHidden = false
-            }
+            primaryButtonTitle = Localization.createAnAccount
+            isPrimaryButtonHidden = !featureFlagService.isFeatureFlagEnabled(.storeCreationMVP)
         }
     }
 
     // MARK: - Actions
     func didTapPrimaryButton(in viewController: UIViewController?) {
-        if isSimplifiedLoginI1Enabled {
-            createAnAccountButtonTapped(in: viewController)
-        } else {
-            loginWithSiteAddressButtonTapped()
-        }
+        createAnAccountButtonTapped(in: viewController)
     }
 
     func didTapSecondaryButton(in viewController: UIViewController?) {
@@ -110,11 +97,6 @@ private extension NotWPAccountViewModel {
         viewController?.present(fancyAlert, animated: true)
     }
 
-    func loginWithSiteAddressButtonTapped() {
-        let popCommand = NavigateToEnterSite()
-        popCommand.execute(from: viewController)
-    }
-
     func createAnAccountButtonTapped(in viewController: UIViewController?) {
         analytics.track(.createAccountOnInvalidEmailScreenTapped)
         guard let viewController,
@@ -141,10 +123,6 @@ private extension NotWPAccountViewModel {
                                                             comment: "Button linking to webview that explains what Jetpack is"
                                                             + "Presented when logging in with a site address that does not have a valid Jetpack installation")
 
-        static let loginWithSiteAddress = NSLocalizedString("Log in with your store address",
-                                                            comment: "Action button linking to instructions for enter another store."
-                                                            + "Presented when logging in with an email address that is not a WordPress.com account")
-
         static let createAnAccount = NSLocalizedString("Create An Account",
                                                        comment: "Action button linking to create WooCommerce store flow."
                                                        + "Presented when logging in with an email address that is not a WordPress.com account")
diff --git a/WooCommerce/Classes/Authentication/Prologue/LoginPrologueViewController.swift b/WooCommerce/Classes/Authentication/Prologue/LoginPrologueViewController.swift
@@ -12,6 +12,7 @@ final class LoginPrologueViewController: UIViewController {
     private let isFeatureCarouselShown: Bool
     private let analytics: Analytics
     private let featureFlagService: FeatureFlagService
+    private let isSimplifiedLogin: Bool
 
     /// Background View, to be placed surrounding the bottom area.
     ///
@@ -44,6 +45,7 @@ final class LoginPrologueViewController: UIViewController {
         self.isFeatureCarouselShown = isFeatureCarouselShown
         self.analytics = analytics
         self.featureFlagService = featureFlagService
+        self.isSimplifiedLogin = ABTest.abTestLoginWithWPComOnly.variation != .control
         super.init(nibName: nil, bundle: nil)
     }
 
@@ -97,7 +99,7 @@ private extension LoginPrologueViewController {
         let pageTypes: [LoginProloguePageType] = {
             if isFeatureCarouselShown {
                 return [.stats, .orderManagement, .products, .reviews]
-            } else if featureFlagService.isFeatureFlagEnabled(.simplifiedLoginFlowI1) {
+            } else if isSimplifiedLogin {
                 return [.simplifiedLoginI1Intro]
             } else {
                 return [.getStarted]
@@ -125,7 +127,7 @@ private extension LoginPrologueViewController {
         guard isNewToWooCommerceButtonShown else {
             return newToWooCommerceButton.isHidden = true
         }
-        let title = featureFlagService.isFeatureFlagEnabled(.simplifiedLoginFlowI1) ? Localization.learnMoreAboutWoo : Localization.newToWooCommerce
+        let title = isSimplifiedLogin ? Localization.learnMoreAboutWoo : Localization.newToWooCommerce
         newToWooCommerceButton.setTitle(title, for: .normal)
         newToWooCommerceButton.applyLinkButtonStyle()
         newToWooCommerceButton.titleLabel?.numberOfLines = 0
diff --git a/WooCommerce/WooCommerceTests/Authentication/NotWPAccountViewModelTests.swift b/WooCommerce/WooCommerceTests/Authentication/NotWPAccountViewModelTests.swift
@@ -67,24 +67,9 @@ final class NotWPAccountViewModelTests: XCTestCase {
         XCTAssertEqual(auxiliaryButtonTitle, AuthenticationConstants.whatIsWPComLinkTitle)
     }
 
-    func test_viewmodel_provides_expected_title_for_primary_button_when_simplified_login_feature_flag_is_off() {
+    func test_viewmodel_provides_expected_title_for_primary_button() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: false)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                               featureFlagService: featureFlagService)
-
-        // When
-        let primaryButtonTitle = viewModel.primaryButtonTitle
-
-        // Then
-        XCTAssertEqual(primaryButtonTitle, Expectations.loginWithSiteAddressTitle)
-    }
-
-    func test_viewmodel_provides_expected_title_for_primary_button_when_simplified_login_feature_flag_is_on() {
-        // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                               featureFlagService: featureFlagService)
+        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom))
         // When
         let primaryButtonTitle = viewModel.primaryButtonTitle
 
@@ -105,23 +90,9 @@ final class NotWPAccountViewModelTests: XCTestCase {
         XCTAssertEqual(primaryButtonTitle, Expectations.restartLoginTitle)
     }
 
-    func test_primary_button_title_is_login_with_site_address_for_invalidWPComEmail_from_wpCom_error_when_simplified_login_feature_flag_is_off() {
-        // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: false)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              featureFlagService: featureFlagService)
-        // When
-        let primaryButtonTitle = viewModel.primaryButtonTitle
-
-        // Then
-        XCTAssertEqual(primaryButtonTitle, Expectations.loginWithSiteAddressTitle)
-    }
-
-    func test_primary_button_title_is_create_an_account_for_invalidWPComEmail_from_wpCom_error_when_simplified_login_feature_flag_is_on() {
+    func test_primary_button_title_is_create_an_account_for_invalidWPComEmail_from_wpCom_error() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              featureFlagService: featureFlagService)
+        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom))
         // When
         let primaryButtonTitle = viewModel.primaryButtonTitle
 
@@ -139,29 +110,18 @@ final class NotWPAccountViewModelTests: XCTestCase {
         XCTAssertFalse(viewModel.isPrimaryButtonHidden)
     }
 
-    func test_primary_button_is_not_hidden_for_invalidWPComEmail_from_wpCom_error_when_simplified_login_feature_flag_is_off() {
+    func test_primary_button_is_not_hidden_for_invalidWPComEmail_from_wpCom_error_when_store_creation_is_on() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: false)
+        let featureFlagService = MockFeatureFlagService(isStoreCreationMVPEnabled: true)
         let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
                                               featureFlagService: featureFlagService)
         // Then
         XCTAssertFalse(viewModel.isPrimaryButtonHidden)
     }
 
-    func test_primary_button_is_not_hidden_for_invalidWPComEmail_from_wpCom_error_when_simplified_login_is_on_and_store_creation_is_on() {
+    func test_primary_button_is_hidden_for_invalidWPComEmail_from_wpCom_error_when_store_creation_is_off() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true,
-                                                        isStoreCreationMVPEnabled: true)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              featureFlagService: featureFlagService)
-        // Then
-        XCTAssertFalse(viewModel.isPrimaryButtonHidden)
-    }
-
-    func test_primary_button_is_hidden_for_invalidWPComEmail_from_wpCom_error_when_simplified_login_is_on_and_store_creation_is_off() {
-        // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true,
-                                                        isStoreCreationMVPEnabled: false)
+        let featureFlagService = MockFeatureFlagService(isStoreCreationMVPEnabled: false)
         let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
                                               featureFlagService: featureFlagService)
         // Then
@@ -188,24 +148,9 @@ final class NotWPAccountViewModelTests: XCTestCase {
 
     // MARK: - `secondaryButtonTitle`
 
-    func test_viewmodel_provides_expected_title_for_secondary_button_when_simplified_login_feature_flag_is_off() {
-        // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: false)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                               featureFlagService: featureFlagService)
-
-        // When
-        let secondaryButtonTitle = viewModel.secondaryButtonTitle
-
-        // Then
-        XCTAssertEqual(secondaryButtonTitle, Expectations.restartLoginTitle)
-    }
-
-    func test_viewmodel_provides_expected_title_for_secondary_button_when_simplified_login_feature_flag_is_on() {
+    func test_viewmodel_provides_expected_title_for_secondary_button() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                               featureFlagService: featureFlagService)
+        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom))
         // When
         let secondaryButtonTitle = viewModel.secondaryButtonTitle
 
@@ -232,10 +177,8 @@ final class NotWPAccountViewModelTests: XCTestCase {
 
     func test_tapping_auxiliary_button_tracks_what__is_wordpress_com_event() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true)
         let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              analytics: analytics,
-                                              featureFlagService: featureFlagService)
+                                              analytics: analytics)
 
         // When
         viewModel.didTapAuxiliaryButton(in: nil)
@@ -244,26 +187,10 @@ final class NotWPAccountViewModelTests: XCTestCase {
         XCTAssertNotNil(analyticsProvider.receivedEvents.first(where: { $0 == "what_is_wordpress_com_on_invalid_email_screen" }))
     }
 
-    func test_tapping_primary_button_does_not_track_create_account_event_when_simplified_login_feature_flag_is_off() {
+    func test_tapping_primary_button_tracks_create_account_event() {
         // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: false)
         let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              analytics: analytics,
-                                              featureFlagService: featureFlagService)
-
-        // When
-        viewModel.didTapPrimaryButton(in: nil)
-
-        // Then
-        XCTAssertNil(analyticsProvider.receivedEvents.first(where: { $0 == "create_account_on_invalid_email_screen" }))
-    }
-
-    func test_tapping_primary_button_tracks_create_account_event_when_simplified_login_feature_flag_is_on() {
-        // Given
-        let featureFlagService = MockFeatureFlagService(isSimplifiedLoginFlowI1Enabled: true)
-        let viewModel = NotWPAccountViewModel(error: SignInError.invalidWPComEmail(source: .wpCom),
-                                              analytics: analytics,
-                                              featureFlagService: featureFlagService)
+                                              analytics: analytics)
 
         // When
         viewModel.didTapPrimaryButton(in: nil)
diff --git a/WooCommerce/WooCommerceTests/Mocks/MockFeatureFlagService.swift b/WooCommerce/WooCommerceTests/Mocks/MockFeatureFlagService.swift
