Full run e2e on every branch (#36)

* Full run e2e on every branch

* Fix incorrect caveat config

* refactor(e2e): remove build step from CI and improve dependency polling logic, remove explicit turbo cache dir

* Add timeout, to fail after dependencies are not available before 5 minutes. Fix issue where dependency is marked as unavailable even when it's responding, if the method is not supported.

* Conditionally log service availability to prevent misleading logging (#40)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: jeff.smale <jeff.smale@consensys.net>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: jeff.smale <jeff.smale@consensys.net>

Author: jeffsmale90

.github/workflows/build-lint-test.yml

@@ -26,7 +26,7 @@ jobs:
         uses: MetaMask/action-checkout-and-setup@v1
         with:
           is-high-risk-environment: false
-          cache-node-modules: ${{ matrix.node-version == '22.x' }}
+          cache-node-modules: true
           submodules: 'recursive'
 
       - name: Install Foundry

.github/workflows/end-to-end-tests.yml

@@ -13,28 +13,21 @@ on:
       NPM_CLIENT:
         type: string
         default: 'yarn'
-      RUN_FULL_TESTS:
-        type: boolean
-        default: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') }}
   workflow_dispatch:
-    inputs:
-      RUN_FULL_TESTS:
-        type: boolean
-        default: true
 
 jobs:
   pipeline:
     name: Turbo Pipeline
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x, 20.x]
+        node-version: [18.x, 20.x, 22.x]
     steps:
       - name: Checkout and setup environment
         uses: MetaMask/action-checkout-and-setup@v1
         with:
           is-high-risk-environment: false
-          cache-node-modules: ${{ matrix.node-version == '22.x' }}
+          cache-node-modules: true
           submodules: 'recursive'
 
       - name: Install Foundry
@@ -54,15 +47,5 @@ jobs:
       - name: Install Yarn dependencies
         run: yarn install
 
-      - name: Run build
-        run: yarn build
-        env:
-          NODE_ENV: production
-
       - name: Run end-to-end tests
-        run: |
-          if [[ "${{ inputs.RUN_FULL_TESTS }}" == "true" ]]; then
-            yarn e2etest:full
-          else
-            yarn e2etest:smoketest
-          fi
+        run: yarn test:e2e

.github/workflows/main.yml

@@ -83,7 +83,6 @@ jobs:
         with:
           commit-starts-with: 'Release [version],Release v[version],Release/[version],Release/v[version],Release `[version]`'
 
-
   publish-release:
     needs: is-release
     if: needs.is-release.outputs.IS_RELEASE == 'true'

CONTRIBUTING.md

@@ -42,7 +42,7 @@ yarn test
 Run the `delegator-e2e` integration test suite:
 
 ```sh
-yarn e2etest:full
+yarn e2etest
 ```
 
 # Contributing

package.json

@@ -13,8 +13,8 @@
   ],
   "scripts": {
     "allow-scripts": "echo 'n/a'",
-    "build": "turbo build --cache-dir=.turbo",
-    "build:packages": "turbo run build --cache-dir=.turbo --filter='!./packages/delegator-e2e'",
+    "build": "turbo build",
+    "build:packages": "turbo run build --filter='!./packages/delegator-e2e'",
     "changelog:update": "turbo run changelog:update --filter='!./packages/delegator-e2e'",
     "changelog:validate": "turbo run changelog:validate --filter='!./packages/delegator-e2e'",
     "clean": "turbo clean",
@@ -23,8 +23,7 @@
     "generate:abi": "cd ./shared/abi/scripts && ./generate-abi.sh",
     "lint": "turbo lint",
     "test": "turbo run test --filter='!./packages/delegator-e2e'",
-    "e2etest:full": "turbo run e2etest:full --filter='./packages/delegator-e2e'",
-    "e2etest:smoketest": "turbo run e2etest:smoketest --filter='./packages/delegator-e2e'",
+    "test:e2e": "turbo run test:e2e --filter='./packages/delegator-e2e'",
     "test:watch": "turbo run test:watch"
   },
   "devDependencies": {

packages/delegator-e2e/README.md

@@ -20,10 +20,7 @@ When developing end-to-end tests, you can run step 2 repeatedly without having t
 
 ### Test Types
 
-There are two types of test suites available:
-
-- **Full Tests** (`yarn e2etest:full`): Runs the complete test suite including all test cases. This is used in the main branch and release branches to ensure comprehensive testing.
-- **Smoke Tests** (`yarn e2etest:smoketest`): Runs only the main test cases, skipping edge cases and alternative scenarios. This is used in feature branches for faster feedback during development.
+The end-to-end test suite runs the complete test suite including all test cases using `yarn test:e2e`.
 
 ## Adding tests
 
@@ -42,7 +39,7 @@ Some guidelines for creating tests:
 
 ### Test Structure
 
-When creating a test file, follow this structure to support both full and smoke test runs:
+When creating a test file, follow this structure:
 
 ```typescript
 import { beforeEach, expect, test } from 'vitest';
@@ -63,12 +60,10 @@ beforeEach(async () => {
   - Any important context
 */
 
-// Main test case that will run in both full and smoke tests
-test('maincase: Description of the main functionality', async () => {
+test('Description of the main functionality', async () => {
   ...
 });
 
-// Additional test cases that will only run in full test suite
 test('Description of edge case or alternative scenario', async () => {
   ...
 });
@@ -80,29 +75,15 @@ test('Description of failure case', async () => {
 
 Key points about test structure:
 
-1. **Main Test Case**:
-   - Prefix the main test case with `maincase:` in the test description
-   - This test will run in both full and smoke test suites
-   - Should cover the primary happy path functionality
-
-2. **Additional Test Cases**:
-   - These will only run in the full test suite
-   - Can include:
-     - Edge cases
-     - Alternative scenarios
-     - Failure cases
-
-3. **Test Organization**:
+1. **Test Organization**:
    - Use `beforeEach` for common setup
    - Group related assertions together
    - Use descriptive test names that explain the scenario
    - Include a block comment describing the user story
 
-4. **Common Patterns**:
+2. **Common Patterns**:
    - Use helper functions from `utils/helpers.ts` for common operations
    - Use assertion functions from `utils/assertions.ts` for consistent validation
    - Generate unique addresses and keys for each test
    - Clean up state in `beforeEach` to ensure test isolation
-   - `runTest_expectSuccess` and `runTest_expectFailure` to abstract common functionality out of each test within the file
-
-The smoke test runner will only execute tests with `maincase:` in their description, while the full test runner will execute all test cases. This allows for quick feedback during development while maintaining comprehensive testing in the main and release branches.
+   - `runTest_expectSuccess` and `runTest_expectFailure` to abstract common functionality out of each test within the file

packages/delegator-e2e/package.json

@@ -14,10 +14,8 @@
   "scripts": {
     "build": "cd contracts && forge build && cd ..",
     "setup": "yarn build && docker compose up -d && npx tsx src/await-dependencies.ts",
-    "run-test:full": "npx vitest run",
-    "run-test:smoketest": "npx vitest run --testNamePattern maincase",
-    "e2etest:full": "yarn setup && yarn run-test:full; EXIT_CODE=$?; yarn teardown; exit $EXIT_CODE",
-    "e2etest:smoketest": "yarn setup && yarn run-test:smoketest; EXIT_CODE=$?; yarn teardown; exit $EXIT_CODE",
+    "test": "npx vitest run",
+    "test:e2e": "yarn setup && yarn test; EXIT_CODE=$?; yarn teardown; exit $EXIT_CODE",
     "teardown": "docker compose down"
   },
   "peerDependencies": {

packages/delegator-e2e/src/await-dependencies.ts

@@ -11,7 +11,10 @@ import { deployDeleGatorEnvironment } from '@metamask/delegation-toolkit/utils';
 import { ENTRYPOINT_ADDRESS_V07 } from 'permissionless';
 import { writeFile } from 'fs/promises';
 
-const POLL_INTERVAL_MS = 1000;
+const POLL_INTERVAL_MS = 1_000;
+// timeout is 5 minutes, which is sufficiently long to never trigger a false positive
+const TIMEOUT_MS = 5 * 60_000;
+let hasTimedOut = false;
 
 const waitFor = async (name: string, url: string) => {
   let isAvailable: boolean | undefined = undefined;
@@ -24,20 +27,21 @@ const waitFor = async (name: string, url: string) => {
   });
 
   do {
-    if (!isAvailable) {
+    if (isAvailable !== undefined) {
       // Only add a delay if it's not the first time
       await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
     }
 
     await client
       .request({ method: 'web3_clientVersion' })
       .then(() => (isAvailable = true))
-      .catch((e: any) => {
-        isAvailable = (e as Error).name !== 'HttpRequestError';
-      });
-  } while (!isAvailable);
+      // as soon as the node is responding (even if it's MethodNotFoundRpcError)
+      .catch((e) => (isAvailable = (e as Error).name !== 'HttpRequestError'));
+  } while (!isAvailable && !hasTimedOut);
 
-  console.log(`${name} is available`);
+  if (isAvailable) {
+    console.log(`${name} is available`);
+  }
 };
 
 const deployEnvironment = async () => {
@@ -61,13 +65,29 @@ const deployEnvironment = async () => {
 };
 
 (async () => {
-  await waitFor('Blockchain node', nodeUrl);
+  const startTime = Date.now();
 
-  const environment = await deployEnvironment();
-  await writeFile('./.gator-env.json', JSON.stringify(environment, null, 2));
+  const timeoutRef = setTimeout(() => (hasTimedOut = true), TIMEOUT_MS);
 
-  await Promise.all([
+  await waitFor('Blockchain node', nodeUrl);
+
+  const waitingForDependencies = Promise.all([
     waitFor('Bundler', bundlerUrl),
     waitFor('Mock paymaster', paymasterUrl),
   ]);
+
+  const environment = await deployEnvironment();
+  await writeFile('./.gator-env.json', JSON.stringify(environment, null, 2));
+
+  await waitingForDependencies;
+
+  clearTimeout(timeoutRef);
+
+  if (hasTimedOut) {
+    console.error('Timed out waiting for dependencies');
+    process.exitCode = 1;
+  } else {
+    const duration = Date.now() - startTime;
+    console.log(`Dependencies ready in ${duration}ms`);
+  }
 })();

packages/delegator-e2e/test/experimental/erc7710sendTransactionWithDelegation.test.ts

@@ -238,15 +238,15 @@ test('Bob attempts to call the increment function directly', async () => {
 test('Bob sends a native value transaction with delegation', async () => {
   await deploySmartAccount(aliceSmartAccount);
 
-  const allowance = 100n;
+  const maxAmount = 100n;
   const recipient = randomAddress();
 
   const { DelegationManager: delegationManager } =
     aliceSmartAccount.environment;
 
   const caveats = createCaveatBuilder(aliceSmartAccount.environment).addCaveat(
     'nativeTokenTransferAmount',
-    allowance,
+    { maxAmount },
   );
 
   const delegation = createDelegation({
@@ -268,13 +268,13 @@ test('Bob sends a native value transaction with delegation', async () => {
     chain,
   }).extend(erc7710WalletActions());
 
-  await fundAddress(aliceSmartAccount.address, allowance);
+  await fundAddress(aliceSmartAccount.address, maxAmount);
 
   const transactionHash = await bobWalletClient.sendTransactionWithDelegation({
     account: bob,
     chain,
     to: recipient,
-    value: allowance,
+    value: maxAmount,
     permissionsContext,
     delegationManager,
   });
@@ -283,5 +283,5 @@ test('Bob sends a native value transaction with delegation', async () => {
 
   const balance = await publicClient.getBalance({ address: recipient });
 
-  expect(balance).toEqual(allowance);
+  expect(balance).toEqual(maxAmount);
 });

turbo.json

@@ -13,10 +13,7 @@
     "test": {
       "dependsOn": ["^build", "^test"]
     },
-    "e2etest:full": {
-      "dependsOn": ["^build"]
-    },
-    "e2etest:smoketest": {
+    "test:e2e": {
       "dependsOn": ["^build"]
     },
     "test:watch": {