Adds reflow scan plugin (#173)

Adds a built-in scan that will detect if a page horizontally scrolls on
a mobile screen size (320x256).

Example reflow issue:

<img width="1247" height="1163" alt="Screen shot of a GitHub issue with
the title 'page requires horizontal scrolling', and the issue body
stating the issue is found at 320x256px"
src="https://github.com/user-attachments/assets/ed9e4111-2aeb-4ab0-8df3-a5984a846628"
/>

Updates made:
* Passes in `url` from the plugin manager so it can be accessed in the
plugins (which I can see being needed for most plugins at this point)
* Removes test plugin in favor of pointing to a real plugin, updates
docs accordingly
* Updates the pluginManager to account for an edge case with duplicate
named plugins running multiple times
* Makes `ruleId` and `html` optional in the issue body - handles
conditionals if these are not passed in and adds tests
* Adds new error to the sites/site-with-errors 404 page to check in the
site-with-errors test.

[Testing - staff only]: 
* [Workflow test w/no scans input passed
in](https://github.com/github/accessibility-scorecard/actions/runs/23449717365)
* [Workflow test w/`include_screenshots`
on](https://github.com/github/accessibility-scorecard/actions/runs/23448598781)
* [Workflow test w/`include_screenshots`
off](https://github.com/github/accessibility-scorecard/actions/runs/23449359031)

Author: lindseywild

.github/actions/file/src/generateIssueBody.ts

@@ -19,21 +19,20 @@ export function generateIssueBody(finding: Finding, screenshotRepo: string): str
   }
 
   const acceptanceCriteria = `## Acceptance Criteria
-  - [ ] The specific axe violation reported in this issue is no longer reproducible.
-  - [ ] The fix MUST meet WCAG 2.1 guidelines OR the accessibility standards specified by the repository or organization.
-  - [ ] A test SHOULD be added to ensure this specific axe violation does not regress.
-  - [ ] This PR MUST NOT introduce any new accessibility issues or regressions.
-  `
+- [ ] The specific violation reported in this issue is no longer reproducible.
+- [ ] The fix MUST meet WCAG 2.1 guidelines OR the accessibility standards specified by the repository or organization.
+- [ ] A test SHOULD be added to ensure this specific violation does not regress.
+- [ ] This PR MUST NOT introduce any new accessibility issues or regressions.`
 
   const body = `## What
-  An accessibility scan flagged the element \`${finding.html}\` on ${finding.url} because ${finding.problemShort}. Learn more about why this was flagged by visiting ${finding.problemUrl}.
+An accessibility scan ${finding.html ? `flagged the element \`${finding.html}\`` : `found an issue on ${finding.url}`} because ${finding.problemShort}. Learn more about why this was flagged by visiting ${finding.problemUrl}.
 
-  ${screenshotSection ?? ''}
-  To fix this, ${finding.solutionShort}.
-  ${solutionLong ? `\nSpecifically:\n\n${solutionLong}` : ''}
+${screenshotSection ?? ''}
+To fix this, ${finding.solutionShort}.
+${solutionLong ? `\nSpecifically:\n\n${solutionLong}` : ''}
 
-  ${acceptanceCriteria}
-  `
+${acceptanceCriteria}
+`
 
   return body
 }

.github/actions/file/src/openIssue.ts

@@ -21,7 +21,12 @@ export async function openIssue(octokit: Octokit, repoWithOwner: string, finding
   const owner = repoWithOwner.split('/')[0]
   const repo = repoWithOwner.split('/')[1]
 
-  const labels = [`${finding.scannerType} rule: ${finding.ruleId}`, `${finding.scannerType}-scanning-issue`]
+  const labels = [`${finding.scannerType}-scanning-issue`]
+  // Only include a ruleId label when it's defined
+  if (finding.ruleId) {
+    labels.push(`${finding.scannerType} rule: ${finding.ruleId}`)
+  }
+
   const title = truncateWithEllipsis(
     `Accessibility issue: ${finding.problemShort[0].toUpperCase() + finding.problemShort.slice(1)} on ${new URL(finding.url).pathname}`,
     GITHUB_ISSUE_TITLE_MAX_LENGTH,

.github/actions/file/src/types.d.ts

@@ -1,8 +1,8 @@
 export type Finding = {
   scannerType: string
-  ruleId: string
+  ruleId?: string
   url: string
-  html: string
+  html?: string
   problemShort: string
   problemUrl: string
   solutionShort: string

.github/actions/file/src/updateFilingsWithNewFindings.ts

@@ -5,7 +5,10 @@ function getFilingKey(filing: ResolvedFiling | RepeatedFiling): string {
 }
 
 function getFindingKey(finding: Finding): string {
-  return `${finding.url};${finding.ruleId};${finding.html}`
+  if (finding.ruleId && finding.html) {
+    return `${finding.url};${finding.ruleId};${finding.html}`
+  }
+  return `${finding.url};${finding.scannerType};${finding.problemUrl}`
 }
 
 export function updateFilingsWithNewFindings(

.github/actions/file/tests/generateIssueBody.test.ts

@@ -11,13 +11,21 @@ const baseFinding = {
   solutionShort: 'ensure the contrast between foreground and background colors meets WCAG thresholds',
 }
 
+const findingWithEmptyOptionalFields = {
+  scannerType: 'reflow',
+  url: 'https://example.com/404',
+  problemShort: 'page requires horizontal scrolling at 320x256 viewport',
+  problemUrl: 'https://www.w3.org/WAI/WCAG21/Understanding/reflow.html',
+  solutionShort: 'ensure content is responsive and does not require horizontal scrolling at small viewport sizes',
+}
+
 describe('generateIssueBody', () => {
   it('includes acceptance criteria and omits the Specifically section when solutionLong is missing', () => {
     const body = generateIssueBody(baseFinding, 'github/accessibility-scanner')
 
     expect(body).toContain('## What')
     expect(body).toContain('## Acceptance Criteria')
-    expect(body).toContain('The specific axe violation reported in this issue is no longer reproducible.')
+    expect(body).toContain('The specific violation reported in this issue is no longer reproducible.')
     expect(body).not.toContain('Specifically:')
   })
 
@@ -61,4 +69,11 @@ describe('generateIssueBody', () => {
     expect(body).not.toContain('View screenshot')
     expect(body).not.toContain('.screenshots')
   })
+
+  it('uses url fallback when html is not present', () => {
+    const body = generateIssueBody(findingWithEmptyOptionalFields, 'github/accessibility-scanner')
+
+    expect(body).toContain(`found an issue on ${findingWithEmptyOptionalFields.url}`)
+    expect(body).not.toContain('flagged the element')
+  })
 })

.github/actions/file/tests/openIssue.test.ts

@@ -60,7 +60,7 @@ describe('openIssue', () => {
     expect(octokit.request).toHaveBeenCalledWith(
       expect.any(String),
       expect.objectContaining({
-        labels: ['axe rule: color-contrast', 'axe-scanning-issue'],
+        labels: ['axe-scanning-issue', 'axe rule: color-contrast'],
       }),
     )
   })

.github/actions/find/src/pluginManager.ts

@@ -12,14 +12,18 @@ const __dirname = path.dirname(__filename)
 
 type PluginDefaultParams = {
   page: playwright.Page
-  addFinding: (findingData: Finding) => void
+  addFinding: (findingData: Finding) => Promise<void>
 }
 
 type Plugin = {
   name: string
   default: (options: PluginDefaultParams) => Promise<void>
 }
 
+// Built-in plugin names shipped with the scanner.
+// Used to skip duplicates when loading custom plugins.
+const BUILT_IN_PLUGINS = ['reflow-scan']
+
 const plugins: Plugin[] = []
 let pluginsLoaded = false
 
@@ -65,7 +69,7 @@ export async function loadCustomPlugins() {
 
   // - currently, the plugin manager will abort loading
   //   all plugins if there's an error
-  // - the problem with this is that if a scanner user doesnt
+  // - the problem with this is that if a scanner user doesn't
   //   have custom plugins, they won't have a 'scanner-plugins' folder
   //   which will cause an error and abort loading all plugins, including built-in ones
   // - so for custom plugins, if the path doesn't exist, we can return early
@@ -75,19 +79,32 @@ export async function loadCustomPlugins() {
     return
   }
 
-  await loadPluginsFromPath({pluginsPath})
+  await loadPluginsFromPath({pluginsPath, skipBuiltInPlugins: BUILT_IN_PLUGINS})
 }
 
 // exported for mocking/testing. not for actual use
-export async function loadPluginsFromPath({pluginsPath}: {pluginsPath: string}) {
+export async function loadPluginsFromPath({
+  pluginsPath,
+  skipBuiltInPlugins,
+}: {
+  pluginsPath: string
+  skipBuiltInPlugins?: string[]
+}) {
   try {
     const res = fs.readdirSync(pluginsPath)
     for (const pluginFolder of res) {
       const pluginFolderPath = path.join(pluginsPath, pluginFolder)
 
       if (fs.existsSync(pluginFolderPath) && fs.lstatSync(pluginFolderPath).isDirectory()) {
-        core.info(`Found plugin: ${pluginFolder}`)
-        plugins.push(await dynamicImport(path.join(pluginsPath, pluginFolder, 'index.js')))
+        const plugin = await dynamicImport(path.join(pluginsPath, pluginFolder, 'index.js'))
+
+        if (skipBuiltInPlugins?.includes(plugin.name)) {
+          core.info(`Skipping built-in plugin: ${plugin.name}`)
+          continue
+        }
+
+        core.info(`Found plugin: ${plugin.name}`)
+        plugins.push(plugin)
       }
     }
   } catch (e) {

.github/actions/find/src/types.d.ts

@@ -1,7 +1,7 @@
 export type Finding = {
   scannerType: string
   url: string
-  html: string
+  html?: string
   problemShort: string
   problemUrl: string
   solutionShort: string

.github/actions/find/tests/pluginManager.test.ts

@@ -12,12 +12,17 @@ vi.mock('../src/pluginManager.js', {spy: true})
 vi.mock('@actions/core', {spy: true})
 
 describe('loadPlugins', () => {
-  vi.spyOn(dynamicImportModule, 'dynamicImport').mockImplementation(path => Promise.resolve(path))
+  let dynamicImportCallCount = 0
+  vi.spyOn(dynamicImportModule, 'dynamicImport').mockImplementation(() => {
+    dynamicImportCallCount++
+    return Promise.resolve({name: `plugin-${dynamicImportCallCount}`, default: vi.fn()})
+  })
   beforeEach(() => {
+    dynamicImportCallCount = 0
     // @ts-expect-error - we don't need the full fs readdirsync
     // method signature here
-    vi.spyOn(fs, 'readdirSync').mockImplementation(readPath => {
-      return [readPath + '/plugin-1', readPath + '/plugin-2']
+    vi.spyOn(fs, 'readdirSync').mockImplementation(() => {
+      return ['folder-a', 'folder-b']
     })
     vi.spyOn(fs, 'lstatSync').mockImplementation(() => {
       return {
@@ -61,4 +66,27 @@ describe('loadPlugins', () => {
       expect(logSpy).toHaveBeenCalledWith(pluginManager.abortError)
     })
   })
+
+  describe('when a custom plugin folder matches a built-in plugin name', () => {
+    beforeEach(() => {
+      // @ts-expect-error - we don't need the full fs readdirsync
+      // method signature here
+      vi.spyOn(fs, 'readdirSync').mockImplementation(() => {
+        return ['reflow-scan']
+      })
+      vi.spyOn(dynamicImportModule, 'dynamicImport').mockImplementation(() => {
+        return Promise.resolve({name: 'reflow-scan', default: vi.fn()})
+      })
+    })
+
+    it('skips the built-in name in custom plugins and only loads it once', async () => {
+      pluginManager.clearCache()
+      const infoSpy = vi.spyOn(core, 'info').mockImplementation(() => {})
+      const plugins = await pluginManager.loadPlugins()
+      // Built-in loads it, custom skips the folder by name
+      expect(plugins.length).toBe(1)
+      expect(plugins[0].name).toBe('reflow-scan')
+      expect(infoSpy).toHaveBeenCalledWith('Skipping built-in plugin: reflow-scan')
+    })
+  })
 })

.github/scanner-plugins/reflow-scan/index.js

@@ -0,0 +1,31 @@
+export default async function reflowScan({page, addFinding} = {}) {
+  const originalViewport = page.viewportSize()
+  const url = page.url()
+  // Check for horizontal scrolling at 320x256 viewport
+  try {
+    await page.setViewportSize({width: 320, height: 256})
+    const scrollWidth = await page.evaluate(() => document.documentElement.scrollWidth)
+    const clientWidth = await page.evaluate(() => document.documentElement.clientWidth)
+
+    // If horizontal scroll is required (with 1px tolerance for rounding)
+    if (scrollWidth > clientWidth + 1) {
+      await addFinding({
+        scannerType: 'reflow-scan',
+        url,
+        problemShort: 'page requires horizontal scrolling at 320x256 viewport',
+        problemUrl: 'https://www.w3.org/WAI/WCAG21/Understanding/reflow.html',
+        solutionShort: 'ensure content is responsive and does not require horizontal scrolling at small viewport sizes',
+        solutionLong: `The page has a scroll width of ${scrollWidth}px but a client width of only ${clientWidth}px at a 320x256 viewport, requiring horizontal scrolling. This violates WCAG 2.1 Level AA Success Criterion 1.4.10 (Reflow).`,
+      })
+    }
+  } catch (e) {
+    console.error('Error checking horizontal scroll:', e)
+  } finally {
+    // Restore original viewport so subsequent scans (e.g. Axe) aren't affected
+    if (originalViewport) {
+      await page.setViewportSize(originalViewport)
+    }
+  }
+}
+
+export const name = 'reflow-scan'