test: e2e supports easy tagging of tests and running of tags

.github/workflows/e2e.yml

@@ -174,7 +174,7 @@ jobs:
           SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
           SANITY_E2E_DATASET: pr-${{ matrix.project }}-${{ github.event.number }}
         # As e2e:build ran in the `install` job, turbopack restores it from cache here
-        run: pnpm e2e:build && pnpm test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: pnpm e2e:build && pnpm test:e2e --excludeTag "@drag-drop" --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
       - uses: actions/upload-artifact@v4
         if: always()

package.json

@@ -90,7 +90,7 @@
     "test": "run-s test:vitest",
     "test:vitest": "vitest --run",
     "test:vitest:watch": "vitest --watch",
-    "test:e2e": "playwright test",
+    "test:e2e": "node -r esbuild-register ./scripts/test-e2e",
     "test:exports": "turbo run test --filter=@repo/test-exports",
     "tsdoc:dev": "sanity-tsdoc dev",
     "updated": "lerna updated",

scripts/test-e2e.mts

@@ -0,0 +1,102 @@
+import path from 'node:path'
+
+import cac from 'cac'
+import execa from 'execa'
+
+export const E2E_ANNOTATION_TAGS = {
+  flake: '@flake',
+  dragDrop: '@drag-drop',
+  pte: '@pte',
+  nightly: '@nightly',
+}
+export type E2E_ANNOTATION_TAG = (typeof E2E_ANNOTATION_TAGS)[keyof typeof E2E_ANNOTATION_TAGS]
+const VALID_E2E_TAGS = Object.values(E2E_ANNOTATION_TAGS)
+
+function validateE2ETags(tags: string): string[] {
+  const tagList = tags.split(',')
+  const invalidTags = tagList.filter((tag) => !VALID_E2E_TAGS.includes(tag as E2E_ANNOTATION_TAG))
+
+  if (invalidTags.length > 0) {
+    throw new Error(
+      `Invalid tag(s): ${invalidTags.join(', ')}. Valid tags are: ${VALID_E2E_TAGS.join(', ')}`,
+    )
+  }
+
+  return tagList
+}
+
+/**
+ * @example
+ * Run E2E tests with specific tags:
+ *
+ * ```bash
+ * // Run tests with the "@flake" tag
+ * pnpm test:e2e --tag "@flake"
+ *
+ * // Run tests without the "@pte" tag
+ * pnpm test:e2e --excludeTag "@pte"
+ *
+ * // Run tests with the @flake and @drag-drop tags
+ * pnpm test:e2e --tag @flake,@drag-drop
+ *
+ * // Run tests without the @flake, @drag-drop, and @pte tags
+ * pnpm test:e2e --excludeTag @flake,@drag-drop,@pte
+ * ```
+ */
+
+// Only run the CLI when this is the main entry point
+// This prevents double execution when imported by Playwright
+if (
+  process.argv[1] === path.resolve(process.cwd(), 'scripts/test-e2e') ||
+  process.argv[1].endsWith('/scripts/test-e2e')
+) {
+  const cli = cac('test-e2e')
+
+  cli
+    .option('--tag <tags>', 'Run tests with specific tags (comma-separated)')
+    .option('--excludeTag <tags>', 'Run tests without specific tags (comma-separated)')
+    .help()
+
+  // Allow passing any other arguments directly to Playwright
+  cli
+    .command('[...args]', 'Additional arguments passed to Playwright')
+    .allowUnknownOptions() // This is the key fix - allow unknown options like --project
+    .action(async (args, options) => {
+      const playwrightArgs: string[] = [...args]
+
+      // Add all unknown options to the playwright args
+      for (const [key, value] of Object.entries(options)) {
+        if (key !== 'tag' && key !== 'excludeTag' && key !== '--') {
+          // Handle boolean flags vs flags with values
+          if (value === true) {
+            playwrightArgs.push(`--${key}`)
+          } else if (value !== false) {
+            playwrightArgs.push(`--${key}`, String(value))
+          }
+        }
+      }
+
+      // Process include tags
+      if (options.tag) {
+        const tags = validateE2ETags(options.tag)
+        playwrightArgs.push('--grep', tags.join('|'))
+      }
+
+      // Process exclude tags
+      if (options.excludeTag) {
+        const tags = validateE2ETags(options.excludeTag)
+        playwrightArgs.push('--grep-invert', tags.join('|'))
+      }
+
+      console.log(`[running] playwright test ${playwrightArgs.join(' ')}`)
+
+      try {
+        await execa('npx', ['playwright', 'test', ...playwrightArgs], {stdio: 'inherit'})
+      } catch (error) {
+        console.error(error)
+        process.exit(1)
+      }
+    })
+
+  cli.parse()
+}

test/e2e/.eslintrc

@@ -0,0 +1,6 @@
+{
+    "rules": {
+        "react-hooks/rules-of-hooks": "off",
+        "max-nested-callbacks": "off"
+    }
+}

test/e2e/tests/comments/inline.spec.ts

@@ -1,6 +1,8 @@
 import {expect, type Page} from '@playwright/test'
 import {test} from '@sanity/test'
 
+import {E2E_ANNOTATION_TAGS} from '../../../../scripts/test-e2e.mjs'
+
 const WAIT_OPTIONS = {
   timeout: 20 * 1000, // 20 seconds
 }
@@ -169,20 +171,24 @@ async function inlineCommentCreationTest(props: InlineCommentCreationTestProps)
   )
 }
 
-test.describe('Inline comments:', () => {
-  test('should create inline comment', async ({page, createDraftDocument}) => {
-    await inlineCommentCreationTest({page, createDraftDocument})
-  })
-
-  test('should resolve inline comment', async ({page, createDraftDocument}) => {
-    // 1. Create a new inline comment
-    await inlineCommentCreationTest({page, createDraftDocument})
-
-    // 2. Resolve the comment by clicking the status button in the comments list item.
-    await page.getByTestId('comments-list-item-status-button').waitFor(WAIT_OPTIONS)
-    await page.locator('[data-testid="comments-list-item-status-button"]').click()
-
-    // 3. Verify that the text is no longer highlighted in the editor.
-    await expect(page.locator('[data-inline-comment-state="added"]')).not.toBeVisible()
-  })
-})
+test.describe(
+  'Inline comments:',
+  {tag: [E2E_ANNOTATION_TAGS.pte, E2E_ANNOTATION_TAGS.nightly]},
+  () => {
+    test('should create inline comment', async ({page, createDraftDocument}) => {
+      await inlineCommentCreationTest({page, createDraftDocument})
+    })
+
+    test('should resolve inline comment', async ({page, createDraftDocument}) => {
+      // 1. Create a new inline comment
+      await inlineCommentCreationTest({page, createDraftDocument})
+
+      // 2. Resolve the comment by clicking the status button in the comments list item.
+      await page.getByTestId('comments-list-item-status-button').waitFor(WAIT_OPTIONS)
+      await page.locator('[data-testid="comments-list-item-status-button"]').click()
+
+      // 3. Verify that the text is no longer highlighted in the editor.
+      await expect(page.locator('[data-inline-comment-state="added"]')).not.toBeVisible()
+    })
+  },
+)

test/e2e/tests/inputs/array.spec.ts

@@ -4,59 +4,61 @@ import path from 'node:path'
 import {expect, type Page} from '@playwright/test'
 import {test} from '@sanity/test'
 
+import {E2E_ANNOTATION_TAGS} from '../../../../scripts/test-e2e.mjs'
 import {createFileDataTransferHandle} from '../../helpers'
 
 const fileName = 'capybara.jpg'
 const image = readFileSync(path.join(__dirname, '..', '..', 'resources', fileName))
 
-test(`file drop event should not propagate to dialog parent`, async ({
-  page,
-  createDraftDocument,
-}) => {
-  await createDraftDocument('/test/content/input-standard;arraysTest')
-
-  await expect(page.getByTestId('document-panel-scroller')).toBeAttached({
-    timeout: 40000,
-  })
-  const list = page.getByTestId('field-arrayOfMultipleTypes').locator('#arrayOfMultipleTypes')
-  const item = list.locator('[data-ui="Grid"] > div')
-
-  const dataTransfer = await createFileDataTransferHandle(
-    {page},
-    {
-      buffer: image,
-      fileName,
-      fileOptions: {
-        type: 'image/jpeg',
+test(
+  `file drop event should not propagate to dialog parent`,
+  {tag: [E2E_ANNOTATION_TAGS.dragDrop, E2E_ANNOTATION_TAGS.nightly]},
+  async ({page, createDraftDocument}) => {
+    await createDraftDocument('/test/content/input-standard;arraysTest')
+
+    await expect(page.getByTestId('document-panel-scroller')).toBeAttached({
+      timeout: 40000,
+    })
+    const list = page.getByTestId('field-arrayOfMultipleTypes').locator('#arrayOfMultipleTypes')
+    const item = list.locator('[data-ui="Grid"] > div')
+
+    const dataTransfer = await createFileDataTransferHandle(
+      {page},
+      {
+        buffer: image,
+        fileName,
+        fileOptions: {
+          type: 'image/jpeg',
+        },
       },
-    },
-  )
+    )
 
-  await expect(list).toBeVisible()
+    await expect(list).toBeVisible()
 
-  // Drop the file.
-  await list.dispatchEvent('drop', {dataTransfer})
+    // Drop the file.
+    await list.dispatchEvent('drop', {dataTransfer})
 
-  // Ensure the list contains one item.
-  expect(item).toHaveCount(1)
+    // Ensure the list contains one item.
+    expect(item).toHaveCount(1)
 
-  // Open the dialog.
-  await page.getByRole('button', {name: fileName}).click()
-  await expect(page.getByRole('dialog')).toBeVisible()
+    // Open the dialog.
+    await page.getByRole('button', {name: fileName}).click()
+    await expect(page.getByRole('dialog')).toBeVisible()
 
-  // Drop the file again; this time, while the dialog is open.
-  //
-  // - The drop event should not propagate to the parent.
-  // - Therefore, the drop event should not cause the image to be added to the list again.
-  await page.getByRole('dialog').dispatchEvent('drop', {dataTransfer})
+    // Drop the file again; this time, while the dialog is open.
+    //
+    // - The drop event should not propagate to the parent.
+    // - Therefore, the drop event should not cause the image to be added to the list again.
+    await page.getByRole('dialog').dispatchEvent('drop', {dataTransfer})
 
-  // Close the dialog.
-  await page.keyboard.press('Escape')
-  await expect(page.getByRole('dialog')).not.toBeVisible()
+    // Close the dialog.
+    await page.keyboard.press('Escape')
+    await expect(page.getByRole('dialog')).not.toBeVisible()
 
-  // Ensure the list still contains one item.
-  expect(item).toHaveCount(1)
-})
+    // Ensure the list still contains one item.
+    expect(item).toHaveCount(1)
+  },
+)
 
 test(`Scenario: Adding a new type from multiple options`, async ({page, createDraftDocument}) => {
   await createDraftDocument('/test/content/input-standard;arraysTest')