[8.19] [CI / FIPS] Add support for FIPS 140-3 agents (#243112) (#245757)

# Backport

This will backport the following commits from `main` to `8.19`:
- [[CI / FIPS] Add support for FIPS 140-3 agents
(#243112)](#243112)

<!--- Backport version: 10.2.0 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Brad
White","email":"Ikuni17@users.noreply.github.com"},"sourceCommit":{"committedDate":"2025-12-10T02:52:04Z","message":"[CI
/ FIPS] Add support for FIPS 140-3 agents (#243112)\n\n##
Summary\nCloses #242952\n\n- Adds support for FIPS 140-3 agents in CI\n-
Adds two GH labels for choosing the agent's FIPS
version\n`ci:enable-fips-140-2-agent`, `ci:enable-fips-140-3-agent` and
removes\nthe old label `ci:enable-fips-agent`\n - Updated existing PRs
to new `140-2` label\n- Updates the FIPS pipeline to use the store cache
step and missing disk\nsize updates.\n- Removes bootstrapping from
`Verify FIPS Enabled` step because it isn't\nneeded.\n- Adds
`fipsIsEnabled` to `@kbn/test` to reduce redundant checks in
some\ntests\n- Combines the following env vars into one,
`TEST_ENABLE_FIPS_VERSION`,\nwhich contains `\"140-2\"` or `\"140-3\"`
when FIPS should be enabled for\nCI.\n - `KBN_ENABLE_FIPS`\n- This
initially was to enable FIPS in Kibana itself, but is no
longer\nnecessary since it is through configuration `yml`.\n -
`FTR_ENABLE_FIPS_AGENT`\n- Was used as a boolean, but didn't describe
the new usage with version.\n - `KBN_FIPS_VERSION`\n- Original idea from
#243434 for usage with `FTR_ENABLE_FIPS_AGENT`, but\nseemed unnecessary
to have two vars.\n\n### Testing\n- [Daily
140-3](https://buildkite.com/elastic/kibana-fips/builds/942)\n- [Daily
140-2](https://buildkite.com/elastic/kibana-fips/builds/944)\n-
[PR\n140-3](https://buildkite.com/elastic/kibana-pull-request/builds/369297)\n-
[PR\n140-2](https://buildkite.com/elastic/kibana-pull-request/builds/369323)\n-
Non FIPS in this
PR","sha":"0c2707e6c998bafc6f3cb41a3a8c135d1760e503","branchLabelMapping":{"^v9.3.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:all-open","v9.3.0"],"title":"[CI
/ FIPS] Add support for FIPS 140-3
agents","number":243112,"url":"https://github.com/elastic/kibana/pull/243112","mergeCommit":{"message":"[CI
/ FIPS] Add support for FIPS 140-3 agents (#243112)\n\n##
Summary\nCloses #242952\n\n- Adds support for FIPS 140-3 agents in CI\n-
Adds two GH labels for choosing the agent's FIPS
version\n`ci:enable-fips-140-2-agent`, `ci:enable-fips-140-3-agent` and
removes\nthe old label `ci:enable-fips-agent`\n - Updated existing PRs
to new `140-2` label\n- Updates the FIPS pipeline to use the store cache
step and missing disk\nsize updates.\n- Removes bootstrapping from
`Verify FIPS Enabled` step because it isn't\nneeded.\n- Adds
`fipsIsEnabled` to `@kbn/test` to reduce redundant checks in
some\ntests\n- Combines the following env vars into one,
`TEST_ENABLE_FIPS_VERSION`,\nwhich contains `\"140-2\"` or `\"140-3\"`
when FIPS should be enabled for\nCI.\n - `KBN_ENABLE_FIPS`\n- This
initially was to enable FIPS in Kibana itself, but is no
longer\nnecessary since it is through configuration `yml`.\n -
`FTR_ENABLE_FIPS_AGENT`\n- Was used as a boolean, but didn't describe
the new usage with version.\n - `KBN_FIPS_VERSION`\n- Original idea from
#243434 for usage with `FTR_ENABLE_FIPS_AGENT`, but\nseemed unnecessary
to have two vars.\n\n### Testing\n- [Daily
140-3](https://buildkite.com/elastic/kibana-fips/builds/942)\n- [Daily
140-2](https://buildkite.com/elastic/kibana-fips/builds/944)\n-
[PR\n140-3](https://buildkite.com/elastic/kibana-pull-request/builds/369297)\n-
[PR\n140-2](https://buildkite.com/elastic/kibana-pull-request/builds/369323)\n-
Non FIPS in this
PR","sha":"0c2707e6c998bafc6f3cb41a3a8c135d1760e503"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.3.0","branchLabelMappingKey":"^v9.3.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/243112","number":243112,"mergeCommit":{"message":"[CI
/ FIPS] Add support for FIPS 140-3 agents (#243112)\n\n##
Summary\nCloses #242952\n\n- Adds support for FIPS 140-3 agents in CI\n-
Adds two GH labels for choosing the agent's FIPS
version\n`ci:enable-fips-140-2-agent`, `ci:enable-fips-140-3-agent` and
removes\nthe old label `ci:enable-fips-agent`\n - Updated existing PRs
to new `140-2` label\n- Updates the FIPS pipeline to use the store cache
step and missing disk\nsize updates.\n- Removes bootstrapping from
`Verify FIPS Enabled` step because it isn't\nneeded.\n- Adds
`fipsIsEnabled` to `@kbn/test` to reduce redundant checks in
some\ntests\n- Combines the following env vars into one,
`TEST_ENABLE_FIPS_VERSION`,\nwhich contains `\"140-2\"` or `\"140-3\"`
when FIPS should be enabled for\nCI.\n - `KBN_ENABLE_FIPS`\n- This
initially was to enable FIPS in Kibana itself, but is no
longer\nnecessary since it is through configuration `yml`.\n -
`FTR_ENABLE_FIPS_AGENT`\n- Was used as a boolean, but didn't describe
the new usage with version.\n - `KBN_FIPS_VERSION`\n- Original idea from
#243434 for usage with `FTR_ENABLE_FIPS_AGENT`, but\nseemed unnecessary
to have two vars.\n\n### Testing\n- [Daily
140-3](https://buildkite.com/elastic/kibana-fips/builds/942)\n- [Daily
140-2](https://buildkite.com/elastic/kibana-fips/builds/944)\n-
[PR\n140-3](https://buildkite.com/elastic/kibana-pull-request/builds/369297)\n-
[PR\n140-2](https://buildkite.com/elastic/kibana-pull-request/builds/369323)\n-
Non FIPS in this
PR","sha":"0c2707e6c998bafc6f3cb41a3a8c135d1760e503"}}]}] BACKPORT-->

---------

Co-authored-by: Alex Szabo <alex.szabo@elastic.co>

Author: Ikuni17

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -33,19 +33,37 @@ spec:
           branch: main
           cronline: 0 5 * * * America/New_York
           env:
-            KBN_FIPS_VERSION: '140-2'
+            TEST_ENABLE_FIPS_VERSION: '140-2'
         '140-2 Daily build (9.1)':
           message: 140-2 Daily build
           branch: '9.1'
           cronline: 0 5 * * * America/New_York
           env:
-            KBN_FIPS_VERSION: '140-2'
+            TEST_ENABLE_FIPS_VERSION: '140-2'
         '140-2 Daily build (8.19)':
           message: 140-2 Daily build
           branch: '8.19'
           cronline: 0 5 * * * America/New_York
           env:
-            KBN_FIPS_VERSION: '140-2'
+            TEST_ENABLE_FIPS_VERSION: '140-2'
+        '140-3 Daily build (main)':
+          message: 140-3 Daily build
+          branch: main
+          cronline: 0 5 * * * America/New_York
+          env:
+            TEST_ENABLE_FIPS_VERSION: '140-3'
+        '140-3 Daily build (9.1)':
+          message: 140-3 Daily build
+          branch: '9.1'
+          cronline: 0 5 * * * America/New_York
+          env:
+            TEST_ENABLE_FIPS_VERSION: '140-3'
+        '140-3 Daily build (8.19)':
+          message: 140-3 Daily build
+          branch: '8.19'
+          cronline: 0 5 * * * America/New_York
+          env:
+            TEST_ENABLE_FIPS_VERSION: '140-3'
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-utils/agent_images.ts

@@ -8,7 +8,9 @@
  */
 
 import { dump } from 'js-yaml';
-import { BuildkiteClient, BuildkiteAgentTargetingRule } from './buildkite';
+import type { BuildkiteAgentTargetingRule } from './buildkite';
+import { BuildkiteClient } from './buildkite';
+import { FIPS_VERSION, prHasFIPSLabel } from './pr_labels';
 
 const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
 const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
@@ -21,31 +23,46 @@ const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
   diskSizeGb: 105,
 };
 
-const FIPS_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
-  provider: 'gcp',
-  image: 'family/kibana-fips-ubuntu-2404',
-  imageProject: ELASTIC_IMAGES_PROD_PROJECT,
-  diskSizeGb: 105,
-};
-
 const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
-const FTR_ENABLE_FIPS_AGENT = process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() === 'true';
+const USE_FIPS_IMAGE_FOR_PR = process.env.TEST_ENABLE_FIPS_VERSION?.match(
+  new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
+);
 const USE_QA_IMAGE_FOR_PR = process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i);
 
+const getFIPSImage = () => {
+  let image: string;
+
+  if (
+    process.env.TEST_ENABLE_FIPS_VERSION === FIPS_VERSION.THREE ||
+    prHasFIPSLabel(FIPS_VERSION.THREE)
+  ) {
+    image = 'family/kibana-fips-140-3-ubuntu-2404';
+  } else {
+    image = 'family/kibana-fips-140-2-ubuntu-2404';
+  }
+
+  return {
+    provider: 'gcp',
+    image,
+    imageProject: ELASTIC_IMAGES_PROD_PROJECT,
+    diskSizeGb: 105,
+  };
+};
+
 // Narrow the return type with overloads
 function getAgentImageConfig(): BuildkiteAgentTargetingRule;
 function getAgentImageConfig(options: { returnYaml: true }): string;
 function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAgentTargetingRule {
   const bk = new BuildkiteClient();
   let config: BuildkiteAgentTargetingRule;
 
-  if (FTR_ENABLE_FIPS_AGENT || GITHUB_PR_LABELS.includes('ci:enable-fips-agent')) {
-    config = FIPS_AGENT_IMAGE_CONFIG;
+  if (USE_FIPS_IMAGE_FOR_PR || prHasFIPSLabel()) {
+    config = getFIPSImage();
 
     bk.setAnnotation(
       'agent image config',
       'info',
-      '#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```KBN_ENABLE_FIPS``` environment variable and/or the ```ci:enable-fips-agent``` Github label.'
+      '#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```TEST_ENABLE_FIPS_VERSION``` environment variable and/or the ```ci:enable-fips-<version>-agent``` Github label.'
     );
   } else {
     config = DEFAULT_AGENT_IMAGE_CONFIG;

.buildkite/pipeline-utils/buildkite/emitPipeline.ts

@@ -19,7 +19,7 @@ import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
 import { serverless, stateful } from '../../ftr_configs_manifests.json';
-import { expandAgentQueue } from '#pipeline-utils';
+import { collectEnvFromLabels, expandAgentQueue } from '#pipeline-utils';
 
 const ALL_FTR_MANIFEST_REL_PATHS = serverless.concat(stateful);
 
@@ -208,32 +208,6 @@ function getEnabledFtrConfigs(patterns?: string[], solutions?: string[]) {
   }
 }
 
-/**
- * Collects environment variables from labels on the PR
- * TODO: extract this (and other functions from this big file) to a separate module
- */
-function collectEnvFromLabels() {
-  const LABEL_MAPPING: Record<string, Record<string, string>> = {
-    'ci:use-chrome-beta': {
-      USE_CHROME_BETA: 'true',
-    },
-  };
-
-  const envFromlabels: Record<string, string> = {};
-  if (!process.env.GITHUB_PR_LABELS) {
-    return envFromlabels;
-  } else {
-    const labels = process.env.GITHUB_PR_LABELS.split(',');
-    labels.forEach((label) => {
-      const env = LABEL_MAPPING[label];
-      if (env) {
-        Object.assign(envFromlabels, env);
-      }
-    });
-    return envFromlabels;
-  }
-}
-
 export async function pickTestGroupRunOrder() {
   const bk = new BuildkiteClient();
   const ciStats = new CiStatsClient();

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -13,3 +13,4 @@ export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
 export * from './utils';
+export * from './pr_labels';

.buildkite/pipeline-utils/index.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export enum FIPS_VERSION {
+  TWO = '140-2',
+  THREE = '140-3',
+}
+
+export const FIPS_GH_LABELS = {
+  [FIPS_VERSION.TWO]: 'ci:enable-fips-140-2-agent',
+  [FIPS_VERSION.THREE]: 'ci:enable-fips-140-3-agent',
+};
+
+/**
+ * Checks if the PR has a specific FIPS label or ANY FIPS label when no version is passed.
+ */
+export function prHasFIPSLabel(version?: FIPS_VERSION): boolean {
+  const labels = process.env.GITHUB_PR_LABELS ?? '';
+
+  if (!labels) {
+    return false;
+  }
+
+  if (version) {
+    return labels.includes(FIPS_GH_LABELS[version]);
+  }
+
+  return Object.values(FIPS_GH_LABELS).some((label) => labels.includes(label));
+}
+
+/**
+ * Available auto-mapped label options, respected by 'collectEnvFromLabels' function.
+ */
+export const LABEL_MAPPING: Record<string, Record<string, string>> = {
+  'ci:use-chrome-beta': {
+    USE_CHROME_BETA: 'true', // Use if you want to run tests with Chrome Beta
+  },
+  [FIPS_GH_LABELS[FIPS_VERSION.TWO]]: {
+    TEST_ENABLE_FIPS_VERSION: FIPS_VERSION.TWO,
+  },
+  [FIPS_GH_LABELS[FIPS_VERSION.THREE]]: {
+    TEST_ENABLE_FIPS_VERSION: FIPS_VERSION.THREE,
+  },
+};
+
+/**
+ * This function reads available GITHUB_LABELS and maps them to environment variables.
+ */
+export function collectEnvFromLabels(
+  labels = process.env.GITHUB_PR_LABELS
+): Record<string, string> {
+  const envFromlabels: Record<string, string> = {};
+
+  if (labels) {
+    const labelArray = labels.split(',');
+    labelArray.forEach((label) => {
+      const env = LABEL_MAPPING[label];
+      if (env) {
+        Object.assign(envFromlabels, env);
+      }
+    });
+  }
+
+  return envFromlabels;
+}

.buildkite/pipeline-utils/pr_labels.ts

@@ -1,6 +1,5 @@
 env:
   DISABLE_CI_STATS_SHIPPING: 'true'
-  KBN_ENABLE_FIPS: 'true'
   TEST_BROWSER_HEADLESS: 1
 agents:
   provider: 'gcp'
@@ -13,14 +12,27 @@ steps:
     timeout_in_minutes: 10
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 115
 
   - wait
 
+  - command: .buildkite/scripts/steps/store_cache.sh
+    label: Store Cache for build
+    timeout_in_minutes: 10
+    id: store_cache
+    soft_fail: true
+    depends_on:
+      - terrazzo-initial-pipeline-upload
+    agents:
+      machineType: n2-standard-2
+      diskSizeGb: 95
+
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution
     agents:
       machineType: n2-standard-8
       preemptible: true
+      diskSizeGb: 150
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     depends_on: pre-build
@@ -44,6 +56,9 @@ steps:
     label: 'Pick Test Group Run Order'
     depends_on: build
     timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+      diskSizeGb: 115
     env:
       FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
       FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'

.buildkite/pipelines/fips.yml

@@ -7,12 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { emitPipeline, getPipeline } from '#pipeline-utils';
+import { emitPipeline, getAgentImageConfig, getPipeline } from '#pipeline-utils';
 
 (async () => {
   const pipeline: string[] = [];
 
   try {
+    pipeline.push(getAgentImageConfig({ returnYaml: true }));
     pipeline.push(getPipeline('.buildkite/pipelines/fips.yml', false));
 
     emitPipeline(pipeline);

.buildkite/pipelines/fips/fips_pipeline.ts

@@ -3,8 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
-export DISABLE_BOOTSTRAP_VALIDATION=false
-.buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
 
 echo --- Verify FIPS enabled

.buildkite/pipelines/fips/verify_fips_enabled.sh

@@ -10,7 +10,7 @@
 import { groups } from './groups.json';
 import { TestSuiteType } from './constants';
 import type { BuildkiteStep } from '#pipeline-utils';
-import { expandAgentQueue } from '#pipeline-utils';
+import { expandAgentQueue, collectEnvFromLabels } from '#pipeline-utils';
 
 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@@ -143,9 +143,11 @@ if (totalJobs > MAX_JOBS) {
 }
 
 const steps: BuildkiteStep[] = [];
+const envFromLabels = collectEnvFromLabels(process.env.GITHUB_PR_LABELS);
 const pipeline = {
   env: {
     IGNORE_SHIP_CI_STATS_ERROR: 'true',
+    ...envFromLabels,
   },
   steps,
 };