Skip to content

Commit 666d1e8

Browse files
Ikuni17Ikuni17
authored
Merge branch '9.2' into renovate/9.2-ftr
2 parents a0303aa + 56f666d commit 666d1e8

224 files changed

Lines changed: 7283 additions & 2421 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,37 @@ spec:
3333
branch: main
3434
cronline: 0 5 * * * America/New_York
3535
env:
36-
KBN_FIPS_VERSION: '140-2'
36+
TEST_ENABLE_FIPS_VERSION: '140-2'
3737
'140-2 Daily build (9.1)':
3838
message: 140-2 Daily build
3939
branch: '9.1'
4040
cronline: 0 5 * * * America/New_York
4141
env:
42-
KBN_FIPS_VERSION: '140-2'
42+
TEST_ENABLE_FIPS_VERSION: '140-2'
4343
'140-2 Daily build (8.19)':
4444
message: 140-2 Daily build
4545
branch: '8.19'
4646
cronline: 0 5 * * * America/New_York
4747
env:
48-
KBN_FIPS_VERSION: '140-2'
48+
TEST_ENABLE_FIPS_VERSION: '140-2'
49+
'140-3 Daily build (main)':
50+
message: 140-3 Daily build
51+
branch: main
52+
cronline: 0 5 * * * America/New_York
53+
env:
54+
TEST_ENABLE_FIPS_VERSION: '140-3'
55+
'140-3 Daily build (9.1)':
56+
message: 140-3 Daily build
57+
branch: '9.1'
58+
cronline: 0 5 * * * America/New_York
59+
env:
60+
TEST_ENABLE_FIPS_VERSION: '140-3'
61+
'140-3 Daily build (8.19)':
62+
message: 140-3 Daily build
63+
branch: '8.19'
64+
cronline: 0 5 * * * America/New_York
65+
env:
66+
TEST_ENABLE_FIPS_VERSION: '140-3'
4967
teams:
5068
kibana-operations:
5169
access_level: MANAGE_BUILD_AND_READ

‎.buildkite/pipeline-utils/agent_images.ts‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import { dump } from 'js-yaml';
1111
import type { BuildkiteAgentTargetingRule } from './buildkite';
1212
import { BuildkiteClient } from './buildkite';
13+
import { FIPS_VERSION, prHasFIPSLabel } from './pr_labels';
1314

1415
const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
1516
const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
@@ -19,32 +20,49 @@ const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
1920
provider: 'gcp',
2021
image: 'family/kibana-ubuntu-2404',
2122
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
22-
};
23-
24-
const FIPS_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
25-
provider: 'gcp',
26-
image: 'family/kibana-fips-ubuntu-2404',
27-
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
23+
diskSizeGb: 105,
2824
};
2925

3026
const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
31-
const FTR_ENABLE_FIPS_AGENT = process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() === 'true';
27+
const USE_FIPS_IMAGE_FOR_PR = process.env.TEST_ENABLE_FIPS_VERSION?.match(
28+
new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
29+
);
3230
const USE_QA_IMAGE_FOR_PR = process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i);
3331

32+
const getFIPSImage = () => {
33+
let image: string;
34+
35+
if (
36+
process.env.TEST_ENABLE_FIPS_VERSION === FIPS_VERSION.THREE ||
37+
prHasFIPSLabel(FIPS_VERSION.THREE)
38+
) {
39+
image = 'family/kibana-fips-140-3-ubuntu-2404';
40+
} else {
41+
image = 'family/kibana-fips-140-2-ubuntu-2404';
42+
}
43+
44+
return {
45+
provider: 'gcp',
46+
image,
47+
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
48+
diskSizeGb: 105,
49+
};
50+
};
51+
3452
// Narrow the return type with overloads
3553
function getAgentImageConfig(): BuildkiteAgentTargetingRule;
3654
function getAgentImageConfig(options: { returnYaml: true }): string;
3755
function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAgentTargetingRule {
3856
const bk = new BuildkiteClient();
3957
let config: BuildkiteAgentTargetingRule;
4058

41-
if (FTR_ENABLE_FIPS_AGENT || GITHUB_PR_LABELS.includes('ci:enable-fips-agent')) {
42-
config = FIPS_AGENT_IMAGE_CONFIG;
59+
if (USE_FIPS_IMAGE_FOR_PR || prHasFIPSLabel()) {
60+
config = getFIPSImage();
4361

4462
bk.setAnnotation(
4563
'agent image config',
4664
'info',
47-
'#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```KBN_ENABLE_FIPS``` environment variable and/or the ```ci:enable-fips-agent``` Github label.'
65+
'#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```TEST_ENABLE_FIPS_VERSION``` environment variable and/or the ```ci:enable-fips-<version>-agent``` Github label.'
4866
);
4967
} else {
5068
config = DEFAULT_AGENT_IMAGE_CONFIG;
@@ -61,7 +79,7 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAge
6179
return config;
6280
}
6381

64-
const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
82+
const expandAgentQueue = (queueName: string = 'n2-4-spot', diskSizeGb?: number) => {
6583
const [kind, cores, addition] = queueName.split('-');
6684
const additionalProps =
6785
{
@@ -72,6 +90,7 @@ const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
7290
return {
7391
...getAgentImageConfig(),
7492
machineType: `${kind}-standard-${cores}`,
93+
...(diskSizeGb ? { diskSizeGb } : {}),
7594
...additionalProps,
7695
};
7796
};

‎.buildkite/pipeline-utils/buildkite/client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ export interface BuildkiteAgentTargetingRule {
5151
machineType?: string;
5252
minCpuPlatform?: string;
5353
preemptible?: boolean;
54+
diskSizeGb?: number;
5455
}
5556

5657
export interface BuildkiteCommandStep {

‎.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -512,10 +512,7 @@ export async function pickTestGroupRunOrder() {
512512
parallelism: unit.count,
513513
timeout_in_minutes: 120,
514514
key: 'jest',
515-
agents: {
516-
...expandAgentQueue('n2-4-spot'),
517-
diskSizeGb: 115,
518-
},
515+
agents: expandAgentQueue('n2-4-spot', 110),
519516
env: {
520517
SCOUT_TARGET_TYPE: 'local',
521518
},
@@ -537,7 +534,7 @@ export async function pickTestGroupRunOrder() {
537534
parallelism: integration.count,
538535
timeout_in_minutes: 120,
539536
key: 'jest-integration',
540-
agents: expandAgentQueue('n2-4-spot'),
537+
agents: expandAgentQueue('n2-4-spot', 105),
541538
env: {
542539
SCOUT_TARGET_TYPE: 'local',
543540
},
@@ -575,7 +572,7 @@ export async function pickTestGroupRunOrder() {
575572
label: title,
576573
command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
577574
timeout_in_minutes: 120,
578-
agents: expandAgentQueue(queue),
575+
agents: expandAgentQueue(queue, 105),
579576
env: {
580577
SCOUT_TARGET_TYPE: 'local',
581578
FTR_CONFIG_GROUP_KEY: key,

‎.buildkite/pipeline-utils/pr_labels.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,46 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10+
export enum FIPS_VERSION {
11+
TWO = '140-2',
12+
THREE = '140-3',
13+
}
14+
15+
export const FIPS_GH_LABELS = {
16+
[FIPS_VERSION.TWO]: 'ci:enable-fips-140-2-agent',
17+
[FIPS_VERSION.THREE]: 'ci:enable-fips-140-3-agent',
18+
};
19+
20+
/**
21+
* Checks if the PR has a specific FIPS label or ANY FIPS label when no version is passed.
22+
*/
23+
export function prHasFIPSLabel(version?: FIPS_VERSION): boolean {
24+
const labels = process.env.GITHUB_PR_LABELS ?? '';
25+
26+
if (!labels) {
27+
return false;
28+
}
29+
30+
if (version) {
31+
return labels.includes(FIPS_GH_LABELS[version]);
32+
}
33+
34+
return Object.values(FIPS_GH_LABELS).some((label) => labels.includes(label));
35+
}
36+
1037
/**
1138
* Available auto-mapped label options, respected by 'collectEnvFromLabels' function.
1239
*/
1340
export const LABEL_MAPPING: Record<string, Record<string, string>> = {
1441
'ci:use-chrome-beta': {
1542
USE_CHROME_BETA: 'true', // Use if you want to run tests with Chrome Beta
1643
},
44+
[FIPS_GH_LABELS[FIPS_VERSION.TWO]]: {
45+
TEST_ENABLE_FIPS_VERSION: FIPS_VERSION.TWO,
46+
},
47+
[FIPS_GH_LABELS[FIPS_VERSION.THREE]]: {
48+
TEST_ENABLE_FIPS_VERSION: FIPS_VERSION.THREE,
49+
},
1750
};
1851

1952
/**

‎.buildkite/pipelines/fips.yml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
env:
22
DISABLE_CI_STATS_SHIPPING: 'true'
3-
KBN_ENABLE_FIPS: 'true'
43
TEST_BROWSER_HEADLESS: 1
54
agents:
65
provider: 'gcp'
@@ -13,14 +12,27 @@ steps:
1312
timeout_in_minutes: 10
1413
agents:
1514
machineType: n2-standard-2
15+
diskSizeGb: 115
1616

1717
- wait
1818

19+
- command: .buildkite/scripts/steps/store_cache.sh
20+
label: Store Cache for build
21+
timeout_in_minutes: 10
22+
id: store_cache
23+
soft_fail: true
24+
depends_on:
25+
- terrazzo-initial-pipeline-upload
26+
agents:
27+
machineType: n2-standard-2
28+
diskSizeGb: 95
29+
1930
- command: .buildkite/scripts/steps/build_kibana.sh
2031
label: Build Kibana Distribution
2132
agents:
2233
machineType: n2-standard-8
2334
preemptible: true
35+
diskSizeGb: 150
2436
key: build
2537
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
2638
depends_on: pre-build
@@ -44,6 +56,9 @@ steps:
4456
label: 'Pick Test Group Run Order'
4557
depends_on: build
4658
timeout_in_minutes: 10
59+
agents:
60+
machineType: n2-standard-2
61+
diskSizeGb: 115
4762
env:
4863
FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
4964
FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'

‎.buildkite/pipelines/fips/fips_pipeline.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
import { emitPipeline, getPipeline } from '#pipeline-utils';
10+
import { emitPipeline, getAgentImageConfig, getPipeline } from '#pipeline-utils';
1111

1212
(async () => {
1313
const pipeline: string[] = [];
1414

1515
try {
16+
pipeline.push(getAgentImageConfig({ returnYaml: true }));
1617
pipeline.push(getPipeline('.buildkite/pipelines/fips.yml', false));
1718

1819
emitPipeline(pipeline);

‎.buildkite/pipelines/fips/verify_fips_enabled.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
set -euo pipefail
44

55
source .buildkite/scripts/common/util.sh
6-
export DISABLE_BOOTSTRAP_VALIDATION=false
7-
.buildkite/scripts/bootstrap.sh
86
.buildkite/scripts/download_build_artifacts.sh
97

108
echo --- Verify FIPS enabled

‎.buildkite/pipelines/on_merge.yml‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ steps:
1010
imageProject: elastic-images-prod
1111
provider: gcp
1212
machineType: n2-standard-2
13-
diskSizeGb: 115
1413
retry:
1514
automatic:
1615
- exit_status: '*'
@@ -54,7 +53,7 @@ steps:
5453
provider: gcp
5554
machineType: n2-highcpu-8
5655
preemptible: true
57-
diskSizeGb: 115
56+
diskSizeGb: 105
5857
timeout_in_minutes: 60
5958
retry:
6059
automatic:
@@ -69,7 +68,7 @@ steps:
6968
provider: gcp
7069
machineType: n2-standard-2
7170
preemptible: true
72-
diskSizeGb: 115
71+
diskSizeGb: 105
7372
timeout_in_minutes: 60
7473
retry:
7574
automatic:
@@ -84,7 +83,7 @@ steps:
8483
provider: gcp
8584
machineType: n2-standard-16
8685
preemptible: true
87-
diskSizeGb: 115
86+
diskSizeGb: 105
8887
timeout_in_minutes: 60
8988
retry:
9089
automatic:
@@ -99,7 +98,7 @@ steps:
9998
provider: gcp
10099
machineType: n2-standard-32
101100
preemptible: true
102-
diskSizeGb: 115
101+
diskSizeGb: 105
103102
timeout_in_minutes: 60
104103
retry:
105104
automatic:
@@ -116,7 +115,7 @@ steps:
116115
diskType: 'hyperdisk-balanced'
117116
preemptible: true
118117
spotZones: us-central1-a,us-central1-b,us-central1-c
119-
diskSizeGb: 115
118+
diskSizeGb: 105
120119
timeout_in_minutes: 60
121120
retry:
122121
automatic:
@@ -148,7 +147,7 @@ steps:
148147
provider: gcp
149148
machineType: n2-highmem-4
150149
preemptible: true
151-
diskSizeGb: 115
150+
diskSizeGb: 105
152151
timeout_in_minutes: 80
153152
retry:
154153
automatic:
@@ -164,7 +163,7 @@ steps:
164163
imageProject: elastic-images-prod
165164
provider: gcp
166165
machineType: n2-standard-2
167-
diskSizeGb: 115
166+
diskSizeGb: 105
168167
timeout_in_minutes: 10
169168
depends_on:
170169
- build
@@ -181,7 +180,7 @@ steps:
181180
imageProject: elastic-images-prod
182181
provider: gcp
183182
machineType: n2-standard-2
184-
diskSizeGb: 115
183+
diskSizeGb: 105
185184
timeout_in_minutes: 10
186185
env:
187186
JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@@ -199,7 +198,7 @@ steps:
199198
imageProject: elastic-images-prod
200199
provider: gcp
201200
machineType: n2-standard-4
202-
diskSizeGb: 115
201+
diskSizeGb: 105
203202
key: build_scout_tests
204203
timeout_in_minutes: 10
205204
depends_on:

0 commit comments

Comments
 (0)