[Scout][a11y] adding axe-core validation

.buildkite/scripts/steps/security/third_party_packages.txt

@@ -19,4 +19,5 @@ xml-formatter
 @storybook/addon-docs
 dompurify
 magic-bytes.js
+@axe-core/playwright
 @opentelemetry/exporter-trace-otlp-grpc

.eslintrc.js

@@ -2692,6 +2692,7 @@ module.exports = {
       ],
       rules: {
         '@kbn/eslint/scout_no_describe_configure': 'error',
+        '@kbn/eslint/require_include_in_check_a11y': 'warn',
       },
     },
     {

package.json

@@ -1470,6 +1470,7 @@
   "devDependencies": {
     "@apidevtools/swagger-parser": "^12.1.0",
     "@arizeai/phoenix-client": "^4.2.0",
+    "@axe-core/playwright": "^4.11.0",
     "@babel/cli": "^7.24.7",
     "@babel/core": "^7.24.7",
     "@babel/eslint-parser": "^7.24.7",
@@ -1888,7 +1889,7 @@
     "ajv-draft-04": "^1.0.0",
     "argsplit": "^1.0.5",
     "autoprefixer": "^10.4.7",
-    "axe-core": "^4.10.0",
+    "axe-core": "^4.11.0",
     "babel-jest": "^29.7.0",
     "babel-loader": "^9.1.3",
     "babel-plugin-add-module-exports": "^1.0.4",

packages/kbn-eslint-plugin-eslint/index.js

@@ -25,5 +25,6 @@ module.exports = {
     deployment_agnostic_test_context: require('./rules/deployment_agnostic_test_context'),
     scout_no_describe_configure: require('./rules/scout_no_describe_configure'),
     require_kbn_fs: require('./rules/require_kbn_fs'),
+    require_include_in_check_a11y: require('./rules/require_include_in_check_a11y'),
   },
 };

packages/kbn-eslint-plugin-eslint/rules/require_include_in_check_a11y.js

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+const isName = (node, expected) =>
+  (node?.type === 'Identifier' && node.name === expected) ||
+  (node?.type === 'Literal' && node.value === expected);
+
+module.exports = {
+  meta: {
+    type: 'suggestion',
+    docs: {
+      description:
+        'Warn when page.checkA11y is called without the include option to encourage scoped a11y scans.',
+      recommended: false,
+    },
+    messages: {
+      requireIncludeInCheckA11y:
+        'We recommend running checkA11y with the include parameter set to the root element you are testing. This makes the tests more isolated and reduces the time required to analyze the DOM structure.',
+    },
+    schema: [],
+  },
+
+  create(context) {
+    const isCheckA11yCall = (node) => {
+      const callee = node && node.callee;
+      if (!callee || callee.type !== 'MemberExpression') return false;
+      return isName(callee.property, 'checkA11y');
+    };
+
+    const hasIncludeProperty = (objExpr) => {
+      for (const prop of objExpr.properties) {
+        if (prop.type === 'Property' && isName(prop.key, 'include')) {
+          return true;
+        }
+      }
+      return false;
+    };
+
+    return {
+      CallExpression(node) {
+        if (!isCheckA11yCall(node)) return;
+
+        const args = node.arguments || [];
+        if (args.length === 0) {
+          context.report({ node, messageId: 'requireIncludeInCheckA11y' });
+          return;
+        }
+
+        const firstArg = args[0];
+        if (!firstArg || firstArg.type !== 'ObjectExpression') {
+          context.report({ node, messageId: 'requireIncludeInCheckA11y' });
+          return;
+        }
+
+        if (!hasIncludeProperty(firstArg)) {
+          context.report({ node, messageId: 'requireIncludeInCheckA11y' });
+        }
+      },
+    };
+  },
+};

packages/kbn-eslint-plugin-eslint/rules/require_include_in_check_a11y.test.js

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+const { RuleTester } = require('eslint');
+const rule = require('./require_include_in_check_a11y');
+const dedent = require('dedent');
+
+const ruleTester = new RuleTester({
+  parser: require.resolve('@typescript-eslint/parser'),
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 2018,
+  },
+});
+
+const MESSAGE =
+  'We recommend running checkA11y with the include parameter set to the root element you are testing. This makes the tests more isolated and reduces the time required to analyze the DOM structure.';
+
+ruleTester.run('@kbn/eslint/require_include_in_check_a11y', rule, {
+  valid: [
+    {
+      code: dedent`
+        page.checkA11y({ include: '#root' });
+      `,
+    },
+    {
+      code: dedent`
+        page.checkA11y({ include: rootEl });
+      `,
+    },
+    {
+      code: dedent`
+        something.checkA11y({ include: '#app', exclude: ['#legacy'] });
+      `,
+    },
+    {
+      code: dedent`
+        page['checkA11y']({ include: '#root' });
+      `,
+    },
+    {
+      code: dedent`
+        page.checkA11y({ include: '#root', other: true });
+      `,
+    },
+    {
+      code: dedent`
+        page.checkA11y({ ['include']: '#root' });
+      `,
+    },
+  ],
+
+  invalid: [
+    {
+      code: dedent`
+        page.checkA11y();
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      code: dedent`
+        page.checkA11y({});
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      code: dedent`
+        page.checkA11y({ foo: 1 });
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      code: dedent`
+        page.checkA11y({ 'include ': '#root' });
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      code: dedent`
+        page.checkA11y(config);
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      code: dedent`
+        page['checkA11y']();
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      // include only in second arg (rule checks first arg)
+      code: dedent`
+        page.checkA11y({}, { include: '#root' });
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+    {
+      // Non-object first argument
+      code: dedent`
+        page.checkA11y('not an object');
+      `,
+      errors: [{ line: 1, message: MESSAGE }],
+    },
+  ],
+});

renovate.json

@@ -244,7 +244,8 @@
     {
       "groupName": "axe-core",
       "matchDepNames": [
-        "axe-core"
+        "axe-core",
+        "@axe-core/playwright"
       ],
       "reviewers": [
         "team:appex-qa"

src/platform/packages/shared/kbn-axe-config/index.ts

@@ -50,3 +50,8 @@ export const AXE_OPTIONS = {
     },
   },
 };
+
+export const AXE_IMPACT_LEVELS: Array<'minor' | 'moderate' | 'serious' | 'critical'> = [
+  'critical',
+  'serious',
+];

src/platform/packages/shared/kbn-scout/moon.yml

@@ -39,6 +39,7 @@ dependsOn:
   - '@kbn/core-http-common'
   - '@kbn/streams-schema'
   - '@kbn/streamlang'
+  - '@kbn/axe-config'
 tags:
   - test-helper
   - package

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/test/scout_page/index.ts

@@ -8,6 +8,7 @@
  */
 
 import type { Page } from '@playwright/test';
+import type { RunA11yScanOptions } from '../../../../utils';
 import type { PathOptions } from '../../../../../common/services/kibana_url';
 
 /**
@@ -40,6 +41,19 @@ export type ScoutPage = Page & {
    * @returns A Promise that resolves once the the element with the css selector is focused, or an error occurs.
    */
   keyTo: (selector: string, key: string, maxElementsToTraverse?: number) => Promise<void>;
+
+  /**
+   * Performs an accessibility (a11y) scan of the current page using axe-core.
+   * Use this in tests to collect formatted violation summaries (one string per violation).
+   *
+   * @param options - Optional accessibility scan configuration (e.g. selectors to include, exclude, timeout).
+   * @returns A Promise resolving to an object with a 'violations' array containing
+   *          human-readable formatted strings for each detected violation (empty if none).
+   */
+  checkA11y: (options?: RunA11yScanOptions) => Promise<{
+    violations: string[];
+  }>;
+
   /**
    * Types text into an input field character by character with a specified delay between each character.
    * @param selector - The css selector for the input element.

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/test/scout_page/single_thread.ts

@@ -12,7 +12,7 @@ import type { Page } from '@playwright/test';
 import { test as base } from '@playwright/test';
 import type { ScoutPage } from '.';
 import type { PathOptions } from '../../../../../common/services/kibana_url';
-import { keyTo } from '../../../../utils';
+import { keyTo, checkA11y } from '../../../../utils';
 import type { KibanaUrl, ScoutLogger } from '../../worker';
 
 /**
@@ -115,6 +115,9 @@ export function extendPlaywrightPage({
   ) => {
     return await keyTo(page, selector, key, maxElementsToTraverse);
   };
+
+  extendedPage.checkA11y = (options) => checkA11y(page, options);
+
   // Method to type text with delay character by character
   extendedPage.typeWithDelay = (selector: string, text: string, options?: { delay: number }) =>
     typeWithDelay(page, selector, text, options);

src/platform/packages/shared/kbn-scout/src/playwright/utils/axe.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Page } from '@playwright/test';
+import type { Result } from 'axe-core';
+import AxeBuilder from '@axe-core/playwright';
+import { AXE_OPTIONS, AXE_IMPACT_LEVELS } from '@kbn/axe-config';
+
+export interface RunA11yScanOptions {
+  /** Optional CSS selectors to include in analysis */
+  include?: string[];
+  /** Optional CSS selectors to exclude from analysis */
+  exclude?: string[];
+  /** Timeout in ms for the scan (defaults 10000) */
+  timeoutMs?: number;
+}
+
+/**
+ * Runs an Axe accessibility scan
+ *
+ * Failure modes:
+ * - Timeout: Can occur on large or complex DOMs. The scan rejects with a timeout Error; tests should
+ *   be made more isolated by narrowing scope via the `include` option.
+ * - Axe/Playwright errors: Underlying errors propagate and indicate a failed scan.
+ */
+const runA11yScan = async (
+  page: Page,
+  { include = [], exclude = [], timeoutMs = 10000 }: RunA11yScanOptions = {}
+) => {
+  const builder = new AxeBuilder({ page });
+  builder.options(AXE_OPTIONS);
+
+  for (const selector of include) {
+    builder.include(selector);
+  }
+
+  for (const selector of exclude) {
+    builder.exclude(selector);
+  }
+
+  const analysisPromise = builder.analyze();
+  let timeoutId: ReturnType<typeof setTimeout> | undefined;
+
+  const result = await Promise.race([
+    analysisPromise,
+    new Promise<never>((_, reject) => {
+      timeoutId = setTimeout(
+        () => reject(new Error(`Axe accessibility scan timed out after ${timeoutMs}ms`)),
+        timeoutMs
+      );
+    }),
+  ]);
+
+  if (timeoutId) {
+    clearTimeout(timeoutId);
+  }
+
+  let violations: Result[] = result.violations;
+
+  if (AXE_IMPACT_LEVELS?.length) {
+    violations = violations.filter((v) => v.impact && AXE_IMPACT_LEVELS.includes(v.impact));
+  }
+
+  return { violations };
+};
+
+export const checkA11y = async (page: Page, options?: RunA11yScanOptions) => {
+  const { violations } = await runA11yScan(page, options);
+
+  const formatA11yViolation = (v: Result): string => {
+    const nodesSection = v.nodes
+      .map((n, idx) => {
+        const selectors = `${n.target.join(', ')}(xpath: ${n.xpath})`;
+        const failure = n.failureSummary?.trim() || 'No failure summary provided';
+        return `  ${idx + 1}. Selectors: ${selectors}\n     Failure: ${failure}`;
+      })
+      .join('\n');
+
+    return [
+      `\nAccessibility violation detected!\n`,
+      `  Rule: ${v.id}. Impact: (${v.impact ?? 'impact unknown'})`,
+      `  Description: ${v.description}`,
+      `  Help: ${v.help}. See more: ${v.helpUrl}`,
+      `  Page: ${page.url()}`,
+      `  Nodes:\n${nodesSection}`,
+    ]
+      .join('\n')
+      .trim();
+  };
+
+  return {
+    violations: violations.map((v) => formatA11yViolation(v)),
+  };
+};

src/platform/packages/shared/kbn-scout/src/playwright/utils/index.ts

@@ -10,3 +10,4 @@
 export { isValidUTCDate, formatTime, getPlaywrightGrepTag, execPromise } from './runner_utils';
 export { resolveSelector, type SelectorInput } from './locator_helper';
 export { keyTo } from './a11y_utils';
+export { checkA11y, type RunA11yScanOptions } from './axe';