[Scout][a11y] adding axe-core validation

.buildkite/scripts/steps/security/third_party_packages.txt

@@ -19,4 +19,5 @@ xml-formatter
 @storybook/addon-docs
 dompurify
 magic-bytes.js
+@axe-core/playwright
 @opentelemetry/exporter-trace-otlp-grpc

package.json

@@ -1467,6 +1467,7 @@
   "devDependencies": {
     "@apidevtools/swagger-parser": "^12.1.0",
     "@arizeai/phoenix-client": "^4.2.0",
+    "@axe-core/playwright": "^4.11.0",
     "@babel/cli": "^7.24.7",
     "@babel/core": "^7.24.7",
     "@babel/eslint-parser": "^7.24.7",
@@ -1886,7 +1887,7 @@
     "ajv-draft-04": "^1.0.0",
     "argsplit": "^1.0.5",
     "autoprefixer": "^10.4.7",
-    "axe-core": "^4.10.0",
+    "axe-core": "^4.11.0",
     "babel-jest": "^29.7.0",
     "babel-loader": "^9.1.3",
     "babel-plugin-add-module-exports": "^1.0.4",

renovate.json

@@ -244,7 +244,8 @@
     {
       "groupName": "axe-core",
       "matchDepNames": [
-        "axe-core"
+        "axe-core",
+        "@axe-core/playwright"
       ],
       "reviewers": [
         "team:appex-qa"

src/platform/packages/shared/kbn-axe-config/index.ts

@@ -50,3 +50,8 @@ export const AXE_OPTIONS = {
     },
   },
 };
+
+export const AXE_IMPACT_LEVELS: Array<'minor' | 'moderate' | 'serious' | 'critical'> = [
+  'critical',
+  'serious',
+];

src/platform/packages/shared/kbn-scout/moon.yml

@@ -39,6 +39,7 @@ dependsOn:
   - '@kbn/core-http-common'
   - '@kbn/streams-schema'
   - '@kbn/streamlang'
+  - '@kbn/axe-config'
 tags:
   - test-helper
   - package

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/test/scout_page/index.ts

@@ -8,6 +8,7 @@
  */
 
 import type { Page } from '@playwright/test';
+import type { RunA11yScanOptions } from '../../../../utils';
 import type { PathOptions } from '../../../../../common/services/kibana_url';
 
 /**
@@ -40,6 +41,19 @@ export type ScoutPage = Page & {
    * @returns A Promise that resolves once the the element with the css selector is focused, or an error occurs.
    */
   keyTo: (selector: string, key: string, maxElementsToTraverse?: number) => Promise<void>;
+
+  /**
+   * Performs an accessibility (a11y) scan of the current page using axe-core.
+   * Use this in tests to collect formatted violation summaries (one string per violation).
+   *
+   * @param options - Optional accessibility scan configuration (e.g. selectors to include, exclude, timeout). 
+   * @returns A Promise resolving to an object with a 'violations' array containing
+   *          human-readable formatted strings for each detected violation (empty if none).
+   */
+  checkA11y: (options?: RunA11yScanOptions) => Promise<{
+    violations: string[];
+  }>;
+
   /**
    * Types text into an input field character by character with a specified delay between each character.
    * @param selector - The css selector for the input element.

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/test/scout_page/single_thread.ts

@@ -12,7 +12,7 @@ import type { Page } from '@playwright/test';
 import { test as base } from '@playwright/test';
 import type { ScoutPage } from '.';
 import type { PathOptions } from '../../../../../common/services/kibana_url';
-import { keyTo } from '../../../../utils';
+import { keyTo, checkA11y } from '../../../../utils';
 import type { KibanaUrl, ScoutLogger } from '../../worker';
 
 /**
@@ -115,6 +115,9 @@ export function extendPlaywrightPage({
   ) => {
     return await keyTo(page, selector, key, maxElementsToTraverse);
   };
+
+  extendedPage.checkA11y = (options) => checkA11y(page, kbnUrl, options);
+
   // Method to type text with delay character by character
   extendedPage.typeWithDelay = (selector: string, text: string, options?: { delay: number }) =>
     typeWithDelay(page, selector, text, options);

src/platform/packages/shared/kbn-scout/src/playwright/utils/axe.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Page } from '@playwright/test';
+import type { Result } from 'axe-core';
+import AxeBuilder from '@axe-core/playwright';
+import { AXE_OPTIONS, AXE_IMPACT_LEVELS } from '@kbn/axe-config';
+
+import type { KibanaUrl } from '../../..';
+
+export interface RunA11yScanOptions {
+  /** Optional CSS selectors to include in analysis */
+  include?: string[];
+  /** Optional CSS selectors to exclude from analysis */
+  exclude?: string[];
+  /** Timeout in ms for the scan (defaults 10000) */
+  timeoutMs?: number;
+}
+
+const runA11yScan = async (
+  page: Page,
+  { include = [], exclude = [], timeoutMs = 10000 }: RunA11yScanOptions = {}
+) => {
+  const builder = new AxeBuilder({ page });
+  builder.options(AXE_OPTIONS);
+
+  if (include) {
+    for (const selector of include) {
+      builder.include(selector);
+    }
+  }
+
+  if (exclude) {
+    for (const selector of exclude) {
+      builder.exclude(selector);
+    }
+  }
+
+  const analysisPromise = builder.analyze();
+  let timeoutId: ReturnType<typeof setTimeout> | undefined;
+
+  const result = await Promise.race([
+    analysisPromise,
+    new Promise<never>((_, reject) => {
+      timeoutId = setTimeout(
+        () => reject(new Error(`Axe accessibility scan timed out after ${timeoutMs}ms`)),
+        timeoutMs
+      );
+    }),
+  ]);
+
+  if (timeoutId) {
+    clearTimeout(timeoutId);
+  }
+
+  let violations: Result[] = result.violations;
+
+  if (AXE_IMPACT_LEVELS?.length) {
+    violations = violations.filter((v) => v.impact && AXE_IMPACT_LEVELS.includes(v.impact));
+  }
+
+  return { violations };
+};
+
+export const checkA11y = async (page: Page, kbnUrl?: KibanaUrl, options?: RunA11yScanOptions) => {
+  const { violations } = await runA11yScan(page, options);
+
+  const formatA11yViolation = (v: Result): string => {
+    const nodesSection = v.nodes
+      .map((n, idx) => {
+        const selectors = n.target.join(', ');
+        const failure = n.failureSummary?.trim() || 'No failure summary provided';
+        return `  ${idx + 1}. Selectors: ${selectors}\n     Failure: ${failure}`;
+      })
+      .join('\n');
+
+    return [
+      `\nAccessibility violation detected!\n`,
+      `  Rule: ${v.id}. Impact: (${v.impact ?? 'impact unknown'})`,
+      `  Description: ${v.description}`,
+      `  Help: ${v.help}. See more: ${v.helpUrl}`,
+      `  Page: ${kbnUrl}`,
+      `  Nodes:\n${nodesSection}`,
+    ]
+      .join('\n')
+      .trim();
+  };
+
+  return {
+    violations: violations.map((v) => formatA11yViolation(v)),
+  };
+};

src/platform/packages/shared/kbn-scout/src/playwright/utils/index.ts

@@ -10,3 +10,4 @@
 export { isValidUTCDate, formatTime, getPlaywrightGrepTag, execPromise } from './runner_utils';
 export { resolveSelector, type SelectorInput } from './locator_helper';
 export { keyTo } from './a11y_utils';
+export { checkA11y, type RunA11yScanOptions } from './axe';

src/platform/packages/shared/kbn-scout/test/scout/ui/tests/axe.spec.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { test, expect } from '../../../../src/playwright';
+
+test.describe('runA11yScan', { tag: ['@svlSecurity', '@ess'] }, () => {
+  test('returns violations array (empty for basic accessible markup)', async ({ page }) => {
+    await page.setContent(`
+      <html lang="en">
+      <head>
+        <title>AXE Core check for basic accessible markup</title>
+      </head>
+      <main>
+        <h1>AXE Core check for basic accessible markup</h1>
+        <button aria-label="Close dialog">X</button>
+        <form>
+          <label for="email">Email</label>
+          <input id="email" type="email" />
+        </form>
+      </main>
+      </html>
+    `);
+
+    const { violations } = await page.checkA11y();
+
+    expect(Array.isArray(violations)).toBe(true);
+
+    // Basic page should have no serious/critical violations
+    expect(violations).toHaveLength(0);
+  });
+
+  test('returns violations related to the CSS "include" selector only', async ({ page }) => {
+    await page.setContent(`
+      <main>
+        <h1>AXE Core check for basic accessible markup</h1>
+        <form>
+          <label>Email</label>
+          <input id="email" type="email" />
+        </form>
+      </main>
+    `);
+
+    const { violations } = await page.checkA11y({ include: ['form'] });
+
+    expect(violations).toHaveLength(1);
+  });
+});

src/platform/packages/shared/kbn-scout/tsconfig.json

@@ -35,5 +35,6 @@
     "@kbn/core-http-common",
     "@kbn/streams-schema",
     "@kbn/streamlang",
+    "@kbn/axe-config"
   ]
 }

yarn.lock

@@ -1148,6 +1148,13 @@
   resolved "https://registry.yarnpkg.com/@aws/lambda-invoke-store/-/lambda-invoke-store-0.0.1.tgz#92d792a7dda250dfcb902e13228f37a81be57c8f"
   integrity sha512-ORHRQ2tmvnBXc8t/X9Z8IcSbBA4xTLKuN873FopzklHMeqBst7YG0d+AX97inkvDX+NChYtSr+qGfcqGFaI8Zw==
 
+"@axe-core/playwright@^4.11.0":
+  version "4.11.0"
+  resolved "https://registry.yarnpkg.com/@axe-core/playwright/-/playwright-4.11.0.tgz#64beab80764c1f3f0ec4ac21f9b2c2d7df508958"
+  integrity sha512-70vBT/Ylqpm65RQz2iCG2o0JJCEG/WCNyefTr2xcOcr1CoSee60gNQYUMZZ7YukoKkFLv26I/jjlsvwwp532oQ==
+  dependencies:
+    axe-core "~4.11.0"
+
 "@babel/cli@^7.24.7":
   version "7.24.7"
   resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.24.7.tgz#eb2868c1fa384b17ea88d60107577d3e6fd05c4e"
@@ -16113,16 +16120,16 @@ aws4@^1.13.2, aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.13.2.tgz#0aa167216965ac9474ccfa83892cfb6b3e1e52ef"
   integrity sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==
 
-axe-core@^4.10.0:
-  version "4.10.0"
-  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.10.0.tgz#d9e56ab0147278272739a000880196cdfe113b59"
-  integrity sha512-Mr2ZakwQ7XUAjp7pAwQWRhhK8mQQ6JAaNWSjmjxil0R8BPioMtQsTLOolGYkji1rcL++3dCqZA3zWqpT+9Ew6g==
-
 axe-core@^4.10.3:
   version "4.10.3"
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.10.3.tgz#04145965ac7894faddbac30861e5d8f11bfd14fc"
   integrity sha512-Xm7bpRXnDSX2YE2YFfBk2FnF0ep6tmG7xPh8iHee8MIcrgq762Nkce856dYtJYLkuIoYZvGfTs/PbZhideTcEg==
 
+axe-core@^4.11.0, axe-core@~4.11.0:
+  version "4.11.0"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.11.0.tgz#16f74d6482e343ff263d4f4503829e9ee91a86b6"
+  integrity sha512-ilYanEU8vxxBexpJd8cWM4ElSQq4QctCLKih0TSfjIfCQTeyH/6zVrmIJfLPrKTKJRbiG+cfnZbQIjAlJmF1jQ==
+
 axe-core@^4.2.0, axe-core@^4.6.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.2.tgz#040a7342b20765cb18bb50b628394c21bccc17a0"