feat: add alpha playwright suite support to the cli

Author: umutuzgur

.github/workflows/release-canary.yml

@@ -19,15 +19,16 @@ jobs:
           node-version: '18.x'
           registry-url: 'https://registry.npmjs.org'
       # Extract the dynamic value from the canary label if present
-      - name: Extract dynamic value from canary label
+      - name: Extract CANARY_TAG
         id: extract-canary
         run: |
-          if [[ "${GITHUB_EVENT_LABEL_NAME}" == canary:* ]]; then
-            CANARY_TAG="${GITHUB_EVENT_LABEL_NAME#canary:}"
-            echo "CANARY_TAG=${CANARY_TAG}" >> $GITHUB_ENV
-          fi
-        env:
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
+          export LABELS_JSON='${{ toJson(github.event.pull_request.labels) }}'
+          CANARY_TAG=$(node -e "
+            const labels = JSON.parse(process.env.LABELS_JSON || '[]');
+            const canaryLabel = labels.find(label => label.name.startsWith('canary:'));
+            if (canaryLabel) console.log(canaryLabel.name.split(':')[1]);
+          ")
+          echo "CANARY_TAG=$CANARY_TAG" >> $GITHUB_ENV
       # Ensure that the README is published with the package
       - run: rm -f packages/cli/README.md && cp README.md packages/cli
       - run: echo "PR_VERSION=0.0.0-pr.${{github.event.pull_request.number}}.$(git rev-parse --short HEAD)" >> $GITHUB_ENV
@@ -36,10 +37,13 @@ jobs:
       # Publish to npm with the additional tag if CANARY_TAG is set
       - run: |
           npm publish --workspace packages/cli --tag experimental
-           if [[ -n "${{ env.CANARY_TAG }}" ]]; then
-             npm dist-tag add checkly@${{ env.PR_VERSION }} ${{ env.CANARY_TAG }}
-           fi
+          if [[ -n "$CANARY_TAG" ]]; then
+            echo "Publishing with additional tag: $CANARY_TAG"
+            npm dist-tag add checkly@$PR_VERSION $CANARY_TAG
+          fi
         env:
+          CANARY_TAG: ${{ env.CANARY_TAG }}
+          PR_VERSION: ${{ env.PR_VERSION }}
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
       - uses: marocchino/sticky-pull-request-comment@v2
         with:

package-lock.json

@@ -73,9 +73,11 @@
     "@oclif/plugin-not-found": "^3.2.44",
     "@oclif/plugin-plugins": "^5.4.36",
     "@oclif/plugin-warn-if-update-available": "^3.1.35",
+    "@types/archiver": "6.0.3",
     "@typescript-eslint/typescript-estree": "^8.30.0",
     "acorn": "^8.14.1",
     "acorn-walk": "^8.3.4",
+    "archiver": "7.0.1",
     "axios": "^1.8.4",
     "chalk": "^4.1.2",
     "ci-info": "^4.2.0",
@@ -89,6 +91,7 @@
     "jwt-decode": "^3.1.2",
     "log-symbols": "^4.1.0",
     "luxon": "^3.6.1",
+    "minimatch": "9.0.5",
     "mqtt": "^5.11.0",
     "open": "^8.4.2",
     "p-queue": "^6.6.2",

packages/cli/package.json

@@ -11,7 +11,7 @@ import type { Runtime } from '../rest/runtimes'
 import {
   Check, AlertChannelSubscription, AlertChannel, CheckGroup, Dashboard,
   MaintenanceWindow, PrivateLocation, PrivateLocationCheckAssignment, PrivateLocationGroupAssignment,
-  Project, ProjectData, BrowserCheck,
+  Project, ProjectData, BrowserCheck, PlaywrightCheck,
 } from '../constructs'
 import chalk from 'chalk'
 import { splitConfigFilePath, getGitInformation } from '../services/util'
@@ -114,6 +114,9 @@ export default class Deploy extends AuthCommand {
       defaultRuntimeId: account.runtimeId,
       verifyRuntimeDependencies,
       checklyConfigConstructs,
+      playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath,
+      include: checklyConfig.checks?.include,
+      playwrightChecks: checklyConfig.checks?.playwrightChecks,
     })
     const repoInfo = getGitInformation(project.repoUrl)
 
@@ -128,6 +131,19 @@ export default class Deploy extends AuthCommand {
         }
         check.snapshots = await uploadSnapshots(check.rawSnapshots)
       }
+
+      for (const check of Object.values(project.data.check)) {
+        // TODO: Improve bundling and uploading
+        if (!(check instanceof PlaywrightCheck) || check.codeBundlePath) {
+          continue
+        }
+        const {
+          relativePlaywrightConfigPath, browsers, key,
+        } = await PlaywrightCheck.bundleProject(check.playwrightConfigPath, check.include)
+        check.codeBundlePath = key
+        check.browsers = browsers
+        check.playwrightConfigPath = relativePlaywrightConfigPath
+      }
     }
 
     const projectPayload = project.synthesize(false)

packages/cli/src/commands/deploy.ts

@@ -16,7 +16,7 @@ import { loadChecklyConfig } from '../services/checkly-config-loader'
 import { filterByFileNamePattern, filterByCheckNamePattern, filterByTags } from '../services/test-filters'
 import type { Runtime } from '../rest/runtimes'
 import { AuthCommand } from './authCommand'
-import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
+import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, PlaywrightCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
 import type { Region } from '..'
 import { splitConfigFilePath, getGitInformation, getCiInformation, getEnvs } from '../services/util'
 import { createReporters, ReporterType } from '../reporters/reporter'
@@ -146,6 +146,7 @@ export default class Test extends AuthCommand {
       'update-snapshots': updateSnapshots,
       retries,
       'verify-runtime-dependencies': verifyRuntimeDependencies,
+      playwrightConfig,
     } = flags
     const filePatterns = argv as string[]
 
@@ -182,6 +183,9 @@ export default class Test extends AuthCommand {
       defaultRuntimeId: account.runtimeId,
       verifyRuntimeDependencies,
       checklyConfigConstructs,
+      playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath,
+      include: checklyConfig.checks?.include,
+      playwrightChecks: checklyConfig.checks?.playwrightChecks,
     })
     const checks = Object.entries(project.data.check)
       .filter(([, check]) => {
@@ -230,6 +234,19 @@ export default class Test extends AuthCommand {
       check.snapshots = await uploadSnapshots(check.rawSnapshots)
     }
 
+    for (const check of checks) {
+      // TODO: Improve bundling and uploading
+      if (!(check instanceof PlaywrightCheck) || check.codeBundlePath) {
+        continue
+      }
+      const {
+        relativePlaywrightConfigPath, browsers, key,
+      } = await PlaywrightCheck.bundleProject(check.playwrightConfigPath, check.include)
+      check.codeBundlePath = key
+      check.browsers = browsers
+      check.playwrightConfigPath = relativePlaywrightConfigPath
+    }
+
     if (this.fancy) {
       ux.action.stop()
     }

packages/cli/src/commands/test.ts

@@ -32,3 +32,4 @@ export * from './telegram-alert-channel'
 export * from './status-page'
 export * from './status-page-service'
 export * from './incident'
+export * from './playwright-check'

packages/cli/src/constructs/index.ts

@@ -0,0 +1,124 @@
+import fs from 'fs'
+import type { AxiosResponse } from 'axios'
+import { Check, CheckProps } from './check'
+import { Session } from './project'
+import {
+  bundlePlayWrightProject, cleanup,
+} from '../services/util'
+import { checklyStorage } from '../rest/api'
+import { ValidationError } from './validator-error'
+
+export interface PlaywrightCheckProps extends CheckProps {
+  playwrightConfigPath: string
+  codeBundlePath?: string
+  installCommand?: string
+  testCommand?: string
+  pwProjects?: string|string[]
+  pwTags?: string|string[]
+  browsers?: string[]
+  include?: string|string[]
+  groupName?: string
+  logicalId: string
+}
+
+export class PlaywrightCheck extends Check {
+  installCommand?: string
+  testCommand: string
+  playwrightConfigPath: string
+  pwProjects: string[]
+  pwTags: string[]
+  codeBundlePath?: string
+  browsers?: string[]
+  include: string[]
+  groupName?: string
+  name: string
+  constructor (logicalId: string, props: PlaywrightCheckProps) {
+    super(logicalId, props)
+    this.logicalId = logicalId
+    this.name = props.name
+    this.codeBundlePath = props.codeBundlePath
+    this.installCommand = props.installCommand
+    this.browsers = props.browsers
+    this.pwProjects = props.pwProjects
+      ? (Array.isArray(props.pwProjects) ? props.pwProjects : [props.pwProjects])
+      : []
+    this.pwTags = props.pwTags
+      ? (Array.isArray(props.pwTags) ? props.pwTags : [props.pwTags])
+      : []
+    this.include = props.include
+      ? (Array.isArray(props.include) ? props.include : [props.include])
+      : []
+    this.testCommand = props.testCommand ?? 'npx playwright test'
+    if (!fs.existsSync(props.playwrightConfigPath)) {
+      throw new ValidationError(`Playwright config doesnt exist ${props.playwrightConfigPath}`)
+    }
+    this.groupName = props.groupName
+    this.playwrightConfigPath = props.playwrightConfigPath
+    this.applyGroup(this.groupName)
+    Session.registerConstruct(this)
+  }
+
+  applyGroup (groupName?: string) {
+    if (!groupName) {
+      return
+    }
+    const checkGroups = Session.project?.data?.['check-group']
+    if (!checkGroups) {
+      return
+    }
+    const group = Object.values(checkGroups).find(group => group.name === groupName)
+    if (group) {
+      this.groupId = group.ref()
+    } else {
+      throw new ValidationError(`Error: No group named "${groupName}". Please verify the group exists in your code or create it.`)
+    }
+  }
+
+  getSourceFile () {
+    return this.__checkFilePath ?? this.logicalId
+  }
+
+  static buildTestCommand (
+    testCommand: string, playwrightConfigPath: string, playwrightProject?: string[], playwrightTag?: string[],
+  ) {
+    return `${testCommand} --config ${playwrightConfigPath}${playwrightProject?.length ? ' --project ' + playwrightProject.map(project => `"${project}"`).join(' ') : ''}${playwrightTag?.length ? ' --grep="' + playwrightTag.join('|') + '"' : ''}`
+  }
+
+  static async bundleProject (playwrightConfigPath: string, include: string[]) {
+    let dir = ''
+    try {
+      const {
+        outputFile, browsers, relativePlaywrightConfigPath,
+      } = await bundlePlayWrightProject(playwrightConfigPath, include)
+      dir = outputFile
+      const { data: { key } } = await PlaywrightCheck.uploadPlaywrightProject(dir)
+      return { key, browsers, relativePlaywrightConfigPath }
+    } finally {
+      await cleanup(dir)
+    }
+  }
+
+  static async uploadPlaywrightProject (dir: string): Promise<AxiosResponse> {
+    const { size } = await fs.promises.stat(dir)
+    const stream = fs.createReadStream(dir)
+    return checklyStorage.uploadCodeBundle(stream, size)
+  }
+
+
+  synthesize () {
+    const testCommand = PlaywrightCheck.buildTestCommand(
+      this.testCommand,
+      this.playwrightConfigPath,
+      this.pwProjects,
+      this.pwTags,
+    )
+    return {
+      ...super.synthesize(),
+      checkType: 'PLAYWRIGHT',
+      codeBundlePath: this.codeBundlePath,
+      testCommand,
+      installCommand: this.installCommand,
+      browsers: this.browsers,
+    }
+  }
+}

packages/cli/src/constructs/playwright-check.ts

@@ -42,14 +42,14 @@ export default class ListReporter extends AbstractListReporter {
     printLn(formatCheckTitle(CheckStatus.RETRIED, checkResult, { printRetryDuration: true }))
     printLn(indentString(formatCheckResult(checkResult), 4), 2, 1)
     if (links) {
-      printLn(indentString('View result: ' + chalk.underline.cyan(`${links.testResultLink}`), 4))
+      printLn(indentString('View result: ' + chalk.underline.cyan(links.testResultLink), 4))
       if (links.testTraceLinks?.length) {
         // TODO: print all video files URLs
-        printLn(indentString('View trace : ' + chalk.underline.cyan(links.testTraceLinks.join(', ')), 4))
+        printLn(indentString('View trace : ' + links.testTraceLinks.map(link => chalk.underline.cyan(link)).join(', '), 4))
       }
       if (links.videoLinks?.length) {
         // TODO: print all trace files URLs
-        printLn(indentString('View video : ' + chalk.underline.cyan(`${links.videoLinks.join(', ')}`), 4))
+        printLn(indentString('View video : ' + links.videoLinks.map(link => chalk.underline.cyan(link)).join(', '), 4))
       }
       printLn('')
     }
@@ -76,14 +76,14 @@ export default class ListReporter extends AbstractListReporter {
     }
 
     if (links) {
-      printLn(indentString('View result: ' + chalk.underline.cyan(`${links.testResultLink}`), 4))
+      printLn(indentString('View result: ' + chalk.underline.cyan(links.testResultLink), 4))
       if (links.testTraceLinks?.length) {
         // TODO: print all video files URLs
-        printLn(indentString('View trace : ' + chalk.underline.cyan(links.testTraceLinks.join(', ')), 4))
+        printLn(indentString('View trace : ' + links.testTraceLinks.map(link => chalk.underline.cyan(link)).join(', '), 4))
       }
       if (links.videoLinks?.length) {
         // TODO: print all trace files URLs
-        printLn(indentString('View video : ' + chalk.underline.cyan(`${links.videoLinks.join(', ')}`), 4))
+        printLn(indentString('View video : ' + links.videoLinks.map(link => chalk.underline.cyan(link)).join(', '), 4))
       }
       printLn('')
     }

packages/cli/src/reporters/list.ts

@@ -15,6 +15,14 @@ class ChecklyStorage {
     )
   }
 
+  uploadCodeBundle (stream: Readable, size: number) {
+    return this.api.post<{ key: string }>(
+      '/next/checkly-storage/upload-code-bundle',
+      stream,
+      { headers: { 'Content-Type': 'application/octet-stream', 'content-length': size } },
+    )
+  }
+
   download (key: string) {
     return this.api.post('/next/checkly-storage/download', { key }, { responseType: 'stream' })
   }

packages/cli/src/rest/checkly-storage.ts

@@ -0,0 +1,43 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  testMatch: 'tests.*.ts',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'Chromium',
+    }
+  ],
+
+});