✨ Add CLI to validate elastic/workflows examples in CI

Author: dej611

.buildkite/pipelines/on_merge.yml

@@ -172,6 +172,25 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/workflows/validate_examples.sh
+    label: 'Validate elastic/workflows examples'
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
+      spotZones: us-central1-b,us-central1-c,us-central1-f
+      diskSizeGb: 105
+    key: validate_workflow_examples
+    timeout_in_minutes: 20
+    artifact_paths:
+      - target/workflow-examples-junit.xml
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   #  - command: .buildkite/scripts/steps/checks/api_contracts.sh
   #    label: 'Check API Contracts'
   #    agents:

.buildkite/pipelines/workflows_examples_dryrun.yml

@@ -0,0 +1,12 @@
+steps:
+  - command: .buildkite/scripts/steps/workflows/validate_examples.sh
+    label: Validate elastic/workflows examples against the current Kibana schema
+    timeout_in_minutes: 20
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
+    artifact_paths:
+      - target/workflow-examples-junit.xml

.buildkite/scripts/steps/workflows/validate_examples.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+WORKFLOWS_REPO_URL="${WORKFLOWS_REPO_URL:-https://github.com/elastic/workflows}"
+WORKFLOWS_BRANCH="${WORKFLOWS_BRANCH:-main}"
+WORKFLOWS_CHECKOUT="$PARENT_DIR/workflows"
+EXAMPLES_SUBPATH="${WORKFLOWS_EXAMPLES_SUBPATH:-examples}"
+JUNIT_OUT="$KIBANA_DIR/target/workflow-examples-junit.xml"
+
+WORKFLOW_SCHEMA_PATH_PATTERN='^(src/platform/packages/shared/kbn-workflows/|src/platform/packages/shared/kbn-workflows-examples-cli/|src/platform/packages/shared/kbn-workflows-yaml/|src/platform/plugins/shared/workflows_management/|\.buildkite/scripts/steps/workflows/)'
+
+report_main_step () {
+  echo "--- $1"
+}
+
+skip_unless_workflow_schema_changed () {
+  if [[ "${WORKFLOWS_VALIDATE_FORCE:-}" == "true" ]]; then
+    echo "WORKFLOWS_VALIDATE_FORCE=true — running validation regardless of changed paths"
+    return 0
+  fi
+
+  report_main_step "Checking whether this merge touched workflow schema paths"
+  local changed
+  if ! changed="$(git -C "$KIBANA_DIR" diff --name-only HEAD~1 HEAD 2>/dev/null)"; then
+    echo "Warning: could not diff HEAD~1..HEAD; running validation anyway." >&2
+    return 0
+  fi
+
+  if echo "$changed" | grep -qE "$WORKFLOW_SCHEMA_PATH_PATTERN"; then
+    echo "Workflow schema paths changed — running validation"
+    return 0
+  fi
+
+  echo "No workflow schema changes in HEAD~1..HEAD — skipping validation"
+  exit 0
+}
+
+clone_workflows_examples () {
+  report_main_step "Cloning elastic/workflows @ $WORKFLOWS_BRANCH"
+  rm -rf "$WORKFLOWS_CHECKOUT"
+  git clone --filter=blob:none --branch "$WORKFLOWS_BRANCH" --single-branch \
+    "$WORKFLOWS_REPO_URL" "$WORKFLOWS_CHECKOUT"
+}
+
+main () {
+  skip_unless_workflow_schema_changed
+  clone_workflows_examples
+
+  report_main_step "Bootstrapping Kibana"
+  cd "$KIBANA_DIR"
+  .buildkite/scripts/bootstrap.sh
+
+  local examples_dir="$WORKFLOWS_CHECKOUT/$EXAMPLES_SUBPATH"
+  if [[ ! -d "$examples_dir" ]; then
+    echo "Error: examples directory '$examples_dir' does not exist." >&2
+    echo "Override the subpath with WORKFLOWS_EXAMPLES_SUBPATH if upstream layout changed." >&2
+    exit 1
+  fi
+
+  mkdir -p "$(dirname "$JUNIT_OUT")"
+
+  report_main_step "Validating workflow examples"
+  node scripts/validate_workflow_examples.js \
+    --dir "$examples_dir" \
+    --junit-out "$JUNIT_OUT"
+}
+
+main

package.json

@@ -1892,6 +1892,7 @@
     "@kbn/validate-oas": "link:src/platform/packages/private/kbn-validate-oas",
     "@kbn/web-worker-stub": "link:packages/kbn-web-worker-stub",
     "@kbn/whereis-pkg-cli": "link:packages/kbn-whereis-pkg-cli",
+    "@kbn/workflows-examples-cli": "link:src/platform/packages/shared/kbn-workflows-examples-cli",
     "@kbn/workspaces": "link:src/platform/packages/shared/kbn-workspaces",
     "@kbn/yarn-install-scripts": "link:packages/kbn-yarn-install-scripts",
     "@kbn/yarn-lock-validator": "link:packages/kbn-yarn-lock-validator",

scripts/validate_workflow_examples.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/setup-node-env');
+require('@kbn/workflows-examples-cli').runValidateExamplesCli();

src/cli/tsconfig.json

@@ -27,6 +27,7 @@
     "@kbn/config",
     "@kbn/dev-utils",
     "@kbn/projects-solutions-groups",
+    "@kbn/workflows-examples-cli",
   ],
   "exclude": [
     "target/**/*",

src/platform/packages/shared/kbn-workflows-examples-cli/README.md

@@ -0,0 +1,69 @@
+# @kbn/workflows-examples-cli
+
+Static validation for workflow YAML examples.
+
+Used in CI to detect when YAML parsing or basic schema constraints break the
+example workflows published in
+[`elastic/workflows`](https://github.com/elastic/workflows). Runs without
+booting Kibana.
+
+## Usage
+
+```
+node scripts/validate_workflow_examples --dir <path-to-examples> [--junit-out <path>]
+```
+
+- `--dir` (required): directory of `.yml`/`.yaml` files. Walked recursively;
+  dotfiles and hidden directories are skipped.
+- `--junit-out` (optional): writes a JUnit XML report for Buildkite to pick up.
+  When set, the CLI also fails if no YAML examples are found (CI misconfiguration guard).
+
+Exits non-zero on any failure.
+
+## What this CLI validates
+
+- YAML syntax errors (parser-level failures).
+- Examples exceeding `MAX_WORKFLOW_YAML_LENGTH`.
+- Structural schema regressions on the workflow definition — top-level `name`,
+  `enabled`, `triggers`, `inputs`, `settings`, `steps` — using a schema built
+  from:
+  - **Static connectors** via `getAllStaticConnectors()` in `@kbn/workflows`
+    (Elasticsearch/Kibana built-ins, stack connectors such as `slack`, `http`,
+    `inference`, `jira`, etc., and connector-specs sub-actions such as
+    `virustotal.*`).
+  - **Extension steps** registered by platform plugins (`data.*`, `ai.*`,
+    `cases.*`, `search.rerank`, `ai.agent`) via
+    `getExtensionStepContracts()`.
+  - Validation runs with `loose: true` (same mode as the YAML editor).
+
+## What this CLI does **not** validate strictly
+
+- **`security.*` steps** (`security.buildAlertEntityGraph`,
+  `security.renderAlertNarrative`): included as permissive `z.any()` placeholders
+  because the security solution plugin is not importable from this platform
+  package. Param drift for those step types is not caught here.
+- **Dynamic connectors** resolved at runtime from the Actions client (only the
+  static catalog is available offline).
+- **End-to-end execution** against a running Kibana stack.
+
+This CLI is a **merge gate** on workflow-schema changes: it catches the cheapest
+classes of regression without booting Kibana. It does not replace functional or
+API integration tests.
+
+## CI behavior
+
+On-merge validation (`.buildkite/scripts/steps/workflows/validate_examples.sh`)
+runs only when the merge commit touches workflow schema paths, unless
+`WORKFLOWS_VALIDATE_FORCE=true`. Upstream examples are cloned from the
+`main` branch of [`elastic/workflows`](https://github.com/elastic/workflows)
+at run time (no pinned ref).
+
+## Programmatic use
+
+```ts
+import {
+  runValidation,
+  validateExampleYaml,
+  buildWorkflowSchema,
+} from '@kbn/workflows-examples-cli';
+```

src/platform/packages/shared/kbn-workflows-examples-cli/index.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import Path from 'path';
+import { mkdir, writeFile } from 'fs/promises';
+import { run } from '@kbn/dev-cli-runner';
+import { createFailError } from '@kbn/dev-cli-errors';
+
+import { runValidation } from './src/run_validation';
+import { renderJUnitXml } from './src/junit_report';
+
+export { runValidation } from './src/run_validation';
+export { validateExampleYaml } from './src/validate_example';
+export type { ValidationOutcome, SchemaIssue } from './src/validate_example';
+export { buildWorkflowSchema } from './src/build_schema';
+export { discoverExampleFiles } from './src/discover_examples';
+export { renderJUnitXml } from './src/junit_report';
+export type { ExampleResult } from './src/junit_report';
+
+export function runValidateExamplesCli(): void {
+  run(
+    async ({ flagsReader, log }) => {
+      const dir = flagsReader.requiredString('dir');
+      const rootDir = Path.resolve(dir);
+      const junitOutFlag = flagsReader.string('junit-out');
+
+      const summary = await runValidation({ rootDir, log });
+
+      if (junitOutFlag) {
+        const junitPath = Path.resolve(junitOutFlag);
+        await mkdir(Path.dirname(junitPath), { recursive: true });
+        await writeFile(junitPath, renderJUnitXml(summary.results), 'utf8');
+        log.info(`Wrote JUnit report to ${junitPath}`);
+      }
+
+      if (summary.empty && junitOutFlag) {
+        throw createFailError(`No workflow YAML files found under ${rootDir}`);
+      }
+
+      if (summary.failed > 0) {
+        throw createFailError(
+          `${summary.failed} of ${summary.results.length} workflow example(s) failed validation`
+        );
+      }
+    },
+    {
+      description:
+        'Validate workflow YAML examples (from elastic/workflows or any directory) against the Kibana workflow schema.',
+      usage: 'node scripts/validate_workflow_examples --dir <path> [--junit-out <path>]',
+      flags: {
+        string: ['dir', 'junit-out'],
+        help: `
+          --dir            (required) Directory containing workflow YAML examples (.yml/.yaml).
+                            The directory is walked recursively; dotfiles and hidden directories
+                            are skipped.
+          --junit-out      Optional path to write a JUnit XML report (consumed by Buildkite).
+        `,
+      },
+    }
+  );
+}

src/platform/packages/shared/kbn-workflows-examples-cli/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-workflows-examples-cli'],
+};

src/platform/packages/shared/kbn-workflows-examples-cli/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/workflows-examples-cli",
+  "owner": "@elastic/workflows-eng",
+  "group": "platform",
+  "visibility": "shared",
+  "devOnly": true
+}