chore(tests): end to end testing again (#11428)

* update sign up flow for updated design

* accesibility labels for easier testing

* assert on welcome screen to make wait reliably

* update login flow

* get rid of broken license check for now

* attempt to install maestro in ci

* attempt to run a test in ci

* attempt with s3 download for now

* need to check out project

* try using m2 pro

* ignore maestro folder for now

Co-authored-by: George <[email protected]>

* convenience script for installing app in booted sims

Co-authored-by: George <[email protected]>

* move flows to top level for now

Co-authored-by: George <[email protected]>

* attempt to run tests in parallel in ci

Co-authored-by: George <[email protected]>

* increase timeout for maestro tests

Co-authored-by: George <[email protected]>

* basic deeplink testing

* try using xcode 16.2

* add missing idb install

* don't clear state for deeplink tests to make more reliable

* try just one flow again

* try just booting one simulator

* try parallelizing using circle parallelism

Co-authored-by: George <[email protected]>

* duplicate assertion in deeplink test

* fix yml matching

* add launch args dep and convenience methods for sign in and sign out

Co-authored-by: George <[email protected]>

* use signOut method instead of buggy clearState

Co-authored-by: George <[email protected]>

* move launch args to real deps for now

* remove dev check for now

* move maestro launch vals to hook to make more reliable, comment out problematic native module for now

* script to workaround xarg exit code behavior

* only sign out if currently signed in to prevent ui breakage

* add required appID in config

* run tests in sequence to avoid exit code not being propogated

* comment auth model lint error for now

* maestro init not dependent on login state, to avoid double sign out

* remove conditional check, assert we are in signup flow

Co-authored-by: George <[email protected]>

* revert me: recaptcha debugging

* Revert "revert me: recaptcha debugging"

This reverts commit b54fd42.

* fix deeplink assertion

* make sure brokenflows are not run

* quick login for deeplink test

* run tests after beta

* upload build to s3 and use in maestro tests

* gate maestro init behind check

* remove logs

* test beta + e2e workflow

* check to avoid crash when native module called early

* bring back native module code, wait on hydration

* tie e2e to beta ios builds for now

* fix zip file creation

* update version number

* add active arch flag to speed up sim build

* upgrade to xcode 16.2 and 18.2 sim

* upload sim builds to s3

* make e2e dependent on test app rather than betas

* clean up

* typo in script

* pr review: remove hardcoded appId

* pr review: move script to utils

* pr review: move env vars to file

---------

Co-authored-by: George <[email protected]>

Author: brainbicycle

.circleci/config.yml

@@ -175,7 +175,7 @@ jobs:
       BUNDLE_PATH: .vendor
 
     macos:
-      xcode: 16.1
+      xcode: 16.2
       resource_class: macos.m1.medium.gen1
 
     steps:
@@ -191,12 +191,58 @@ jobs:
           name: Update rollout if needed
           command: ./scripts/deploys/update-android-rollout-if-needed
 
+  run-e2e-tests:
+    parallelism: 4
+
+    environment:
+      BUNDLE_PATH: .vendor # path to install gems and use for caching
+
+    macos:
+      xcode: 16.2
+      resource_class: m2pro.medium
+
+    steps:
+      - attach_workspace:
+          at: ../workspace
+      - macos/preboot-simulator:
+          version: "18.2"
+          platform: "iOS"
+          device: "iPhone 16 Pro"
+      - run:
+          name: Install maestro
+          command: curl -Ls "https://get.maestro.mobile.dev" | bash
+      - run:
+          name: Install idb
+          command: brew tap facebook/fb && brew install idb-companion
+      - run:
+          name: Clear project dir
+          command: |
+            rm -rf  /Users/distiller/project/*
+      - checkout
+      - setup-awscli
+      - run:
+          name: Set up maestro env
+          command: ./scripts/setup/setup-maestro-env.sh
+      - run:
+          name: Download app zip
+          command: aws s3 cp s3://artsy-citadel/eigen/builds/ios/Artsy-latest.zip ./Artsy.zip
+      - run:
+          name: Unzip the app
+          command: unzip Artsy.zip
+      - run:
+          name: Install app in booted sims
+          command: ./scripts/utils/install_app_in_booted_sims.sh
+      - run:
+          name: Run Maestro Tests
+          no_output_timeout: 25m
+          command: ./scripts/utils/run_maestro_shard.sh
+
   deploy-nightly-beta:
     environment:
       BUNDLE_PATH: .vendor
 
     macos:
-      xcode: 16.1
+      xcode: 16.2
       resource_class: macos.m1.medium.gen1
 
     steps:
@@ -217,7 +263,7 @@ jobs:
       BUNDLE_PATH: .vendor
 
     macos:
-      xcode: 16.1
+      xcode: 16.2
       resource_class: macos.m1.medium.gen1
 
     steps:
@@ -309,7 +355,7 @@ jobs:
       BUNDLE_PATH: .vendor # path to install gems and use for caching
 
     macos:
-      xcode: 16.1
+      xcode: 16.2
       resource_class: macos.m1.medium.gen1
 
     steps:
@@ -383,7 +429,7 @@ jobs:
       BUNDLE_PATH: .vendor # path to install gems and use for caching
 
     macos:
-      xcode: 16.1
+      xcode: 16.2
       resource_class: macos.m1.medium.gen1
 
     steps:
@@ -423,7 +469,7 @@ jobs:
       - install-gems
       - install-cocoapods
       - macos/preboot-simulator:
-          version: "18.1"
+          version: "18.2"
           platform: "iOS"
           device: "iPhone 16 Pro"
       - build-app-ios
@@ -518,8 +564,12 @@ workflows:
               only:
                 - main
     jobs:
+      - build-test-app-ios
       - deploy-nightly-beta
       - run-nightly-tasks
+      - run-e2e-tests:
+          requires:
+            - build-test-app-ios
 
   flag-check:
     triggers:
@@ -600,3 +650,11 @@ workflows:
           requires:
             - build-test-app-ios
             - build-test-app-android
+
+      - run-e2e-tests:
+          filters:
+            branches:
+              only:
+                - beta-ios
+          requires:
+            - build-test-app-ios

.gitignore

@@ -151,6 +151,9 @@ tsconfig.tsbuildinfo
 # Temporary files created by Metro to check the health of the file watcher
 .metro-health-check*
 
+# maestro test artifacts
+.maestro/*
+
 #########################################################
 # Keep this rule last.
 # Keep these, so we can keep the folders these appear in.

e2e/flows/onboarding/onboardingQuiz.js e2e/broken_flows/onboardingQuiz.jse2e/flows/onboarding/onboardingQuiz.js renamed to e2e/broken_flows/onboardingQuiz.js

@@ -1,9 +1,4 @@
 appId: ${MAESTRO_APP_ID}
 ---
-- runFlow:
-    file: flows/onboarding/login.yml
-- runFlow:
-    file: flows/onboarding/signupWithoutOnboarding.yml
-# This flow is broken because of performance issues and hangs of the Art Taste Quiz Screen
-# - runFlow:
-# file: flows/onboarding/onboardingQuiz.yml
+flows:
+  - "flows/*"

e2e/flows/onboarding/onboardingQuiz.yml e2e/broken_flows/onboardingQuiz.ymle2e/flows/onboarding/onboardingQuiz.yml renamed to e2e/broken_flows/onboardingQuiz.yml

@@ -0,0 +1,23 @@
+appId: ${MAESTRO_APP_ID}
+---
+- launchApp:
+    clearState: false
+    arguments:
+      email: ${MAESTRO_TEST_EMAIL}
+      password: ${MAESTRO_TEST_PASSWORD}
+- assertVisible: "New Works for You"
+- killApp
+- openLink:
+    link: https://www.artsy.net/artist/kaws
+    autoVerify: false
+- assertVisible: "Kaws"
+- killApp
+- openLink:
+    link: https://click.artsy.net/f/a/epPWiX0dJaHHMiXAnSrzSg~~/AAQRxQA~/RgRj95QSP0SQaHR0cHM6Ly93d3cuYXJ0c3kubmV0L2NvbGxlY3Rpb24vYXVjdGlvbi1oaWdobGlnaHRzP3V0bV9zb3VyY2U9YnJhemUmdXRtX21lZGl1bT1lbWFpbCZ1dG1fY2FtcGFpZ249bnRmJnV0bV90ZXJtPTYyMTRmYjlkZWI0ZWZkN2RiNjBjZTcwNDk1NWU4NDQ1VwNzcGNCCmIFEg8VYlEJKGBSEmJyaWJlY2sxQGdtYWlsLmNvbVgEAAAAfw~~
+    autoVerify: false
+- assertVisible: "Highlights at Auction This Week"
+- killApp
+- openLink:
+    link: "https://email-link.artsy.net/ls/click?upn=u001.XPbk8dXjc6dwd18Yq2uPM3eTWB4YbDUmeJTAXXTF513sTSKEFRIPxrFgGj8gd2LN9AcRIza5p2-2Fi0huqjv6GWgdnpwA-2FRyTEIYA-2BT1-2F0josxHnYTzJpKeJEmkfzDQlMEY9y6FknqDddY-2BKmm5qtTrTIF2hxySbdUboaYf5ofK-2Bj-2FgSolzBiLvvzai-2FmvjWbSpNpJHWATexQixwLakesmmZUBoRoLkCnxs59pBxJPbB3JZyP13cv77lbfn1WzJ86fiW-2B5MQcd2qss7wLGBI-2BZuy-2FxDkbFuDzpfYPDlaUs1-2BuyO9rqleUCwXPEuyWXZ3aGJrh-_dWIRBTTKlL2IhQqV5a1Cy38Sd4xylePmkTfQkXpdWOLZbG0vcWuzG-2BSSjZ7xaSUgPR0lxFBD0Zz4mpyMqiSRd7GblcbPN8eH0NYfUK6pmADWc2UoFXyJdeq-2F-2BOyOeWlS7O-2FOrHgB9-2BOGtV1nkIFPq97vJTNmWtBMDOt77SsursARR1LHG7CnzBEedV5yll40aEh9LTjA47-2BfpvonTwdxvYfd6ZWsZN2uRczvCs34ipSKUClLSzq0xCn8IjAixssMm2IreBk6LmNzHPC1YLqmZ77pv0qTxX095mxh8mk7f9rJJ8Tg1fyfRFriXGsQqgHlY-2BqTf2aAU6LS9NVXirqdp20-2BHXhzUZbaCnkQNASABts91iKoZaDeV4xnYuuo3L-2F8"
+    autoVerify: false
+- assertVisible: "The artwork you were looking for isn't available."

e2e/config.yml

@@ -0,0 +1,13 @@
+appId: ${MAESTRO_APP_ID}
+---
+- launchApp:
+    clearState: false
+    arguments:
+      shouldSignOut: "true"
+- assertVisible: "Sign up or log in"
+- tapOn: "Email Input"
+- inputText: ${MAESTRO_TEST_EMAIL}
+- tapOn: "Continue.*"
+- inputText: ${MAESTRO_TEST_PASSWORD}
+- tapOn: "Continue.*"
+- assertVisible: "New Works for You"