Skip to content

Commit 2086838

Browse files
committed
ci: update buildkite logic to run backports
1 parent 1928582 commit 2086838

35 files changed

Lines changed: 10554 additions & 104 deletions

.buildkite/hooks/pre-command

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,19 @@ source .buildkite/scripts/utils.sh
66

77
echo '--- Setting job environment variables'
88

9+
910
# kibanamachine token to access org
10-
GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
11+
GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/ci/elastic-elastic-charts/kibanamachine)
1112
export GITHUB_TOKEN
1213

1314
# charts github app to send checks
14-
GITHUB_AUTH=$(retry 5 5 vault read -field=auth secret/datavis/github)
15+
GITHUB_AUTH=$(retry 5 5 vault read -field=auth secret/ci/elastic-elastic-charts/github | base64 -d)
1516
export GITHUB_AUTH
1617

17-
FIREBASE_AUTH=$(retry 5 5 vault read -field=auth secret/datavis/firebase)
18+
FIREBASE_AUTH=$(retry 5 5 vault read -field=auth secret/ci/elastic-elastic-charts/firebase | base64 -d)
1819
export FIREBASE_AUTH
1920

20-
BUILDKITE_TOKEN=$(retry 5 5 vault read -field=token secret/datavis/buildkite)
21+
BUILDKITE_TOKEN=$(retry 5 5 vault read -field=token secret/ci/elastic-elastic-charts/buildkite)
2122
export BUILDKITE_TOKEN
2223

2324
source .buildkite/scripts/node_setup.sh

.buildkite/hooks/pre-exit

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ set -euo pipefail
44

55
echo '--- Update commit status for jobs'
66

7-
npx ts-node .buildkite/scripts/pre_exit.ts
7+
# npx ts-node .buildkite/scripts/pre_exit.ts

.buildkite/pipelines/pull_request/pipeline.sh

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,8 @@ set -euo pipefail
44

55
cd '.buildkite'
66

7-
if [[ "${ELASTIC_BUILDKITE_INFRA:-}" =~ ^(1|true)$ ]]; then
8-
# The new infra requires the pipeline to emit the steps
9-
yarn -s print:pipeline
10-
else
11-
# The old infra required pipeline upload
12-
echo '--- Build pipeline'
13-
14-
yarn build:pipeline
15-
fi
7+
# Yarn install completely silently:
8+
yarn install 1>&2
9+
10+
# The new infra requires the pipeline to emit the steps
11+
yarn -s print:pipeline

.buildkite/pipelines/pull_request/pipeline.ts

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ import {
2525
storybookStep,
2626
typeCheckStep,
2727
firebasePreDeployStep,
28+
chartsPackageStep,
2829
} from '../../steps';
2930
import type { Step, CustomCommandStep } from '../../utils';
3031
import { bkEnv, ChangeContext, uploadPipeline } from '../../utils';
31-
import { getBuildConfig } from '../../utils/build';
3232
import { MetaDataKeys } from '../../utils/constants';
3333
import { createDeployment, createDeploymentStatus, createOrUpdateDeploymentComment } from '../../utils/deployment';
3434

@@ -41,15 +41,6 @@ void (async () => {
4141
const changeCtx = new ChangeContext();
4242
await changeCtx.init();
4343

44-
// Update main job status
45-
await updateCheckStatus(
46-
{
47-
status: 'in_progress',
48-
details_url: bkEnv.buildUrl,
49-
},
50-
getBuildConfig().main.id,
51-
);
52-
5344
const steps: Step[] = [
5445
jestStep(),
5546
eslintStep(),
@@ -59,6 +50,7 @@ void (async () => {
5950
docsStep(),
6051
storybookStep(),
6152
e2eServerStep(),
53+
chartsPackageStep(),
6254
firebasePreDeployStep(),
6355
ghpDeployStep(),
6456
playwrightVrtStep(),

.buildkite/pull-requests.json

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"jobs": [
3+
{
4+
"repoOwner": "elastic",
5+
"repoName": "elastic-charts",
6+
"pipelineSlug": "elastic-charts-build",
7+
8+
"enabled": true,
9+
"allow_org_users": true,
10+
"allowed_repo_permissions": ["admin", "write"],
11+
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
12+
"set_commit_status": true,
13+
"build_on_commit": true,
14+
"build_on_comment": true,
15+
"build_drafts": true,
16+
"build_on_ready": true,
17+
"cancel_intermediate_builds": true,
18+
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
19+
"always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
20+
"skip_ci_labels": [],
21+
"skip_target_branches": [],
22+
"enable_skippable_commits": true,
23+
"skip_ci_on_only_changed": [],
24+
"always_require_ci_on_changed": [],
25+
"enable_trigger_checkbox": false,
26+
"kibana_versions_check": false,
27+
"kibana_build_reuse": false,
28+
"kibana_build_reuse_pipeline_slugs": [],
29+
"kibana_build_reuse_regexes": []
30+
},
31+
{
32+
"repoOwner": "elastic",
33+
"repoName": "elastic-charts",
34+
"pipelineSlug": "elastic-charts-build",
35+
36+
"enabled": true,
37+
"allow_org_users": true,
38+
"cancel_intermediate_builds": true,
39+
"allowed_repo_permissions": ["admin", "write"],
40+
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
41+
"set_commit_status": true,
42+
"build_on_commit": true,
43+
"build_on_comment": false,
44+
"build_drafts": false,
45+
"build_on_ready": false,
46+
"enable_skippable_commits": false,
47+
"skip_ci_on_only_changed": [],
48+
"always_require_ci_on_changed": [],
49+
"enable_trigger_checkbox": false,
50+
"target_branch": "main"
51+
}
52+
]
53+
}

.buildkite/scripts/node_setup.sh

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ source .buildkite/scripts/utils.sh
66

77
echo "--- Setup Node"
88

9-
KIBANA_DIR=$(pwd)
10-
CACHE_DIR="$HOME/.kibana"
9+
ELASTIC_CHARTS_DIR=$(pwd)
10+
CACHE_DIR="$HOME/.elastic-charts"
1111

12-
NODE_VERSION="$(cat "$KIBANA_DIR/.node-version")"
12+
NODE_VERSION="$(cat "$ELASTIC_CHARTS_DIR/.node-version")"
1313
NODE_DIR="$CACHE_DIR/node/$NODE_VERSION"
1414
NODE_BIN_DIR="$NODE_DIR/bin"
1515
YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache"
@@ -70,7 +70,14 @@ if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
7070
npm_install_global yarn "^$YARN_VERSION"
7171
fi
7272

73-
yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"
73+
# Move the prewarmed offline cache into the repo so it's visible inside the
74+
# docker plugin container (mounted at /app). The container reads the location
75+
# from YARN_YARN_OFFLINE_MIRROR set by the docker plugin environment.
76+
if [[ -d "$YARN_OFFLINE_CACHE" ]]; then
77+
echo " -- moving yarn offline cache to $ELASTIC_CHARTS_DIR/.yarn-offline-cache"
78+
mv -v "$YARN_OFFLINE_CACHE" "$ELASTIC_CHARTS_DIR/.yarn-offline-cache"
79+
yarn config set yarn-offline-mirror "$ELASTIC_CHARTS_DIR/.yarn-offline-cache"
80+
fi
7481

7582
YARN_GLOBAL_BIN=$(yarn global bin)
7683
export YARN_GLOBAL_BIN

.buildkite/scripts/pre_exit.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ void (async function () {
1515
const { checkId, jobId, jobUrl } = bkEnv;
1616

1717
if (checkId && jobId && !skipChecks.has(checkId)) {
18-
await yarnInstall();
18+
await yarnInstall('.buildkite');
1919
const jobStatus = await getJobStatus(jobId);
2020

2121
if (jobStatus) {
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import fs from 'fs';
10+
import path from 'path';
11+
12+
import {
13+
bkEnv,
14+
CHARTS_PACKAGE_MANIFEST_FILENAME,
15+
createChartsPackageManifest,
16+
exec,
17+
setChartsPackageMetadata,
18+
startGroup,
19+
uploadArtifacts,
20+
yarnInstall,
21+
} from '../../utils';
22+
23+
interface NpmPackResult {
24+
filename: string;
25+
version: string;
26+
}
27+
28+
void (async () => {
29+
const prNumber = bkEnv.pullRequestNumber;
30+
const commitSha = bkEnv.commit;
31+
32+
if (!prNumber || !commitSha) {
33+
throw new Error('Charts package tarballs are only published for pull request builds with a commit SHA');
34+
}
35+
36+
await yarnInstall();
37+
38+
startGroup('Preparing @elastic/charts package files');
39+
await exec('node ./packages/charts/scripts/move_txt_files.js');
40+
41+
startGroup('Building @elastic/charts package');
42+
await exec('yarn build', {
43+
cwd: 'packages/charts',
44+
});
45+
46+
startGroup('Packing @elastic/charts package');
47+
const packOutput = await exec('npm pack --json', {
48+
cwd: 'packages/charts',
49+
stdio: 'pipe',
50+
});
51+
const [packResult] = JSON.parse(packOutput) as NpmPackResult[];
52+
53+
if (!packResult?.filename || !packResult.version) {
54+
throw new Error('Failed to parse npm pack output for @elastic/charts');
55+
}
56+
57+
const commitShortSha = commitSha.slice(0, 7);
58+
const liveTarballFilename = `elastic-charts-v${packResult.version}-pr-${prNumber}.tgz`;
59+
const commitTarballFilename = `elastic-charts-v${packResult.version}-pr-${prNumber}-${commitShortSha}.tgz`;
60+
const artifactDir = '.buildkite/artifacts/packages';
61+
const packagedTarballPath = path.join('packages/charts', packResult.filename);
62+
const liveArtifactTarballPath = path.join(artifactDir, liveTarballFilename);
63+
const commitArtifactTarballPath = path.join(artifactDir, commitTarballFilename);
64+
const manifestPath = path.join(artifactDir, CHARTS_PACKAGE_MANIFEST_FILENAME);
65+
const chartsPackageMetadata = {
66+
liveTarballFilename,
67+
commitTarballFilename,
68+
version: packResult.version,
69+
commitSha,
70+
commitShortSha,
71+
};
72+
73+
fs.mkdirSync(artifactDir, { recursive: true });
74+
fs.rmSync(liveArtifactTarballPath, { force: true });
75+
fs.rmSync(commitArtifactTarballPath, { force: true });
76+
fs.rmSync(manifestPath, { force: true });
77+
fs.renameSync(packagedTarballPath, liveArtifactTarballPath);
78+
fs.copyFileSync(liveArtifactTarballPath, commitArtifactTarballPath);
79+
80+
fs.writeFileSync(manifestPath, JSON.stringify(createChartsPackageManifest(chartsPackageMetadata, prNumber), null, 2));
81+
82+
await setChartsPackageMetadata(chartsPackageMetadata);
83+
84+
await uploadArtifacts(liveArtifactTarballPath);
85+
await uploadArtifacts(commitArtifactTarballPath);
86+
await uploadArtifacts(manifestPath);
87+
})();

.buildkite/scripts/steps/e2e_reports.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ void (async () => {
224224
body: `Your latest commit indicated you would like me to update the vrt screenshots but this PR disallows edits. Please update your PR to allow edits and tell me to \`test this\` again.
225225
<img width="297" alt="image" src="https://user-images.githubusercontent.com/19007109/175552884-7f8e4bba-3440-444b-b19c-de15d618ac23.png">`,
226226
});
227+
throw new Error('CI run in update mode but PR does not allow edits');
227228
}
228229
}
229230
}

.buildkite/scripts/steps/eslint.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ void (async () => {
2828
// TODO: fix this to where we can install only the necessary packages in one script
2929
await yarnInstall('e2e');
3030
await yarnInstall('.buildkite');
31-
await yarnInstall('github_bot');
3231

3332
// TODO: fix this to avoid requiring build to run linting
3433
// currently the storybook and others are loosely coupled to @elastic/charts/src

0 commit comments

Comments
 (0)