Let's include E2E tests for the website

.github/workflows/playwright.yml

@@ -0,0 +1,41 @@
+name: E2E Website tests
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test-docs:
+    name: E2E Testing
+    runs-on: ubuntu-latest
+    env:
+      CI: true
+    defaults:
+      run:
+        working-directory: scorecards-site
+
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v5
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        run: |
+          yarn install --frozen-lockfile
+          yarn add -D @playwright/test chromatic @chromatic-com/playwright
+          yarn playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: yarn playwright test
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+
+      - name: Run test and Upload the generated Chromatic archives
+        run: yarn chromatic --playwright -t="$CHROMATIC_PROJECT_TOKEN"
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          ENABLE_SNAPSHOTS: true

scorecards-site/.gitignore

@@ -88,3 +88,11 @@ sw.*
 
 # Vim swap files
 *.swp
+
+# Playwright
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+/playwright/.auth/
+.last-run.json

scorecards-site/README.md

@@ -29,6 +29,77 @@ $ yarn generate
 For detailed explanation on how things work, check out the
 [documentation](https://nuxtjs.org).
 
+## E2E testing
+
+In order to run the e2e tests we use Playwright, but we **do not include it as a project dependency**.  
+To run tests locally, you must install Playwright manually and avoid committing these changes.
+
+### Temporary local installation
+
+```sh
+yarn add -D @playwright/test chromatic @chromatic-com/playwright
+```
+
+These changes are temporary and should not be committed.
+
+For reference you can follow the steps in the[e2e testing pipeline](../.github/workflows/playwright.yml)
+
+### Run the tests
+
+Basic test run:
+
+```sh
+yarn playwright test
+```
+
+Run tests with visual (HTML) reporting:
+
+```sh
+yarn playwright test --reporter=html
+```
+
+Open the HTML report:
+
+```sh
+yarn playwright show-report
+```
+
+### Testing approach
+
+Test files are located in the `tests-e2e` directory.
+They are used to perform basic interaction tests such as:
+
+- clicking buttons
+- navigation and URL redirections
+- rendering checks
+- simple behavior flows
+
+To prevent visual regressions, we generate screenshots at key test steps and compare them against stored reference files.
+
+Snapshots are located in `tests-e2e/*-snapshots`
+
+If differences are detected, the HTML report will show a diff view and the tests will fail.
+
+### Important
+
+The E2E tests will **build the website and serve it locally during execution**.  
+Before running the tests:
+
+- stop any local development server you may have running
+- ensure that `http://localhost:3000` is **not in use**
+
+If this port is busy, the tests will fail.
+
+### Clean up
+
+Once you are done with the local tests:
+
+- Remove all temporarily installed Playwright dependencies: `rm -rf node_modules`
+- Discard the `devDependencies` modifications in `package.json` and `yarn.lock`.
+- Reinstall your project dependencies:`yarn`
+
+This restores your environment to a clean state.
+
 ## Special Directories
 
 You can create the following extra directories, some of which have special

scorecards-site/playwright.config.js

@@ -0,0 +1,46 @@
+import { defineConfig, devices } from '@playwright/test'
+/**
+ * @see https://playwright.dev/docs/test-configuration
+ */
+export default defineConfig({
+  testDir: './tests-e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    /* Test against mobile viewports. */
+    {
+      name: 'Mobile Chrome',
+      use: { ...devices['Pixel 5'] },
+    },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: 'yarn build && yarn start',
+    url: 'http://localhost:3000',
+    reuseExistingServer: !process.env.CI,
+  },
+})

scorecards-site/tests-e2e/home.spec.js

@@ -0,0 +1,66 @@
+// @ts-check
+import path from 'path'
+import fs from 'fs'
+import { test, expect } from '@chromatic-com/playwright'
+
+const mockRepoDataPath = path.resolve(
+  __dirname,
+  'mock-data',
+  'github-repo.json'
+)
+const mockRepoData = fs.readFileSync(mockRepoDataPath, 'utf-8')
+
+const mockCommitsDataPath = path.resolve(
+  __dirname,
+  'mock-data',
+  'github-commits.json'
+)
+const mockCommitsData = fs.readFileSync(mockCommitsDataPath, 'utf-8')
+
+test('Home Page', async ({ page }) => {
+  // Mock GitHub repo metadata used by Header.getTotalCommits()
+  await page.route('https://api.github.com/repos/ossf/scorecard', (route) => {
+    route.fulfill({
+      status: 200,
+      contentType: 'application/json',
+      body: mockRepoData,
+    })
+  })
+
+  // Mock commits endpoint used by Header.fetchData()
+  await page.route(
+    'https://api.github.com/repos/ossf/scorecard/commits*',
+    (route) => {
+      route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: mockCommitsData,
+      })
+    }
+  )
+
+  await page.goto('http://localhost:3000/')
+
+  // Expect a title "to contain" a substring.
+  await expect(page).toHaveTitle(/OpenSSF Scorecard/)
+
+  // Check the video player is present and in autoplay mode
+  const videoElement = page.locator('video:visible')
+  await expect(videoElement).toBeVisible()
+  const autoplay = await videoElement.getAttribute('autoplay')
+  expect(autoplay).not.toBeNull()
+
+  // Hide all video players for consistent screenshots
+  await page.evaluate(() => {
+    document
+      .querySelectorAll('video')
+      .forEach((v) => (v.style.display = 'none'))
+  })
+
+  // Prevent Visual Regressions
+  if (process.env.ENABLE_SNAPSHOTS) {
+    expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(
+      'home.png'
+    )
+  }
+})