Skip to content

Commit 8b49b84

Browse files
donoghucdonoghuc
committed
Merge remote-tracking branch 'upstream/feature/fedramp-high-8.x' into merge-feature-fedramp-high-8.x
2 parents d0d3697 + 6f89de5 commit 8b49b84

File tree

56 files changed

+1721
-58
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1721
-58
lines changed

‎.buildkite/pull_request_pipeline.yml

+174-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,27 @@ steps:
1919
export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
2020
ci/license_check.sh -m 4G
2121
22+
steps:
23+
- label: ":passport_control: License check - Fedramp High Mode"
24+
key: "license-check-fedramp-high"
25+
agents:
26+
provider: gcp
27+
imageProject: elastic-images-prod
28+
image: family/platform-ingest-logstash-ubuntu-2204
29+
machineType: "n2-standard-4"
30+
diskSizeGb: 64
31+
retry:
32+
automatic:
33+
- limit: 3
34+
command: |
35+
set -euo pipefail
36+
37+
source .buildkite/scripts/common/container-agent.sh
38+
export JRUBY_OPTS="-J-Xmx1g"
39+
export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
40+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
41+
docker run -e ORG_GRADLE_PROJECT_fedrampHighMode=true test-runner-image ci/license_check.sh -m 4G
42+
2243
- label: ":rspec: Ruby unit tests"
2344
key: "ruby-unit-tests"
2445
agents:
@@ -37,6 +58,25 @@ steps:
3758
artifact_paths:
3859
- "coverage/coverage.json"
3960

61+
- label: ":rspec: Ruby unit tests - FIPS mode"
62+
key: "ruby-unit-tests-fips"
63+
agents:
64+
provider: gcp
65+
imageProject: elastic-images-prod
66+
image: family/platform-ingest-logstash-ubuntu-2204
67+
machineType: "n2-standard-4"
68+
diskSizeGb: 64
69+
retry:
70+
automatic:
71+
- limit: 3
72+
command: |
73+
set -euo pipefail
74+
75+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
76+
docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true rubyTests
77+
artifact_paths:
78+
- "coverage/coverage.json"
79+
4080
- label: ":java: Java unit tests"
4181
key: "java-unit-tests"
4282
agents:
@@ -58,6 +98,29 @@ steps:
5898
- "**/jacocoTestReport.xml"
5999
- "**/build/classes/**/*.*"
60100

101+
- label: ":java: Java unit tests - FIPS mode"
102+
key: "java-unit-tests-fips"
103+
agents:
104+
provider: gcp
105+
imageProject: elastic-images-prod
106+
image: family/platform-ingest-logstash-ubuntu-2204
107+
machineType: "n2-standard-4"
108+
diskSizeGb: 64
109+
retry:
110+
automatic:
111+
- limit: 3
112+
env:
113+
ENABLE_SONARQUBE: true
114+
command: |
115+
set -euo pipefail
116+
117+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
118+
docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true javaTests
119+
artifact_paths:
120+
- "**/build/test-results/javaTests/TEST-*.xml"
121+
- "**/jacocoTestReport.xml"
122+
- "**/build/classes/**/*.*"
123+
61124
- label: ":sonarqube: Continuous Code Inspection"
62125
if: |
63126
build.pull_request.id != null ||
@@ -79,8 +142,84 @@ steps:
79142
manual:
80143
allowed: true
81144

145+
- label: "Observability SRE container smoke test"
146+
key: "observability-sre-container-smoke-test"
147+
agents:
148+
provider: gcp
149+
imageProject: elastic-images-prod
150+
image: family/platform-ingest-logstash-ubuntu-2204
151+
machineType: "n2-standard-4"
152+
diskSizeGb: 64
153+
retry:
154+
automatic:
155+
- limit: 3
156+
command: |
157+
set -euo pipefail
158+
source .buildkite/scripts/common/vm-agent.sh
159+
QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
160+
# Build the image locally with the gradle task
161+
./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
162+
# Ensure it can at least start logstash
163+
docker run docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
164+
logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }'
165+
# Run the smoke tests on the PR code
166+
docker tag docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
167+
pr-built-observability-sre-image
168+
./gradlew observabilitySREsmokeTests --stacktrace
169+
170+
- label: ":lab_coat: Integration Tests - FIPS mode / part 1-of-3"
171+
key: "integration-tests-fips-part-1-of-3"
172+
agents:
173+
provider: gcp
174+
imageProject: elastic-images-prod
175+
image: family/platform-ingest-logstash-ubuntu-2204
176+
machineType: "n2-standard-4"
177+
diskSizeGb: 64
178+
retry:
179+
automatic:
180+
- limit: 3
181+
command: |
182+
set -euo pipefail
183+
184+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
185+
docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split 0 3
186+
187+
- label: ":lab_coat: Integration Tests - FIPS mode / part 2-of-3"
188+
key: "integration-tests-fips-part-2-of-3"
189+
agents:
190+
provider: gcp
191+
imageProject: elastic-images-prod
192+
image: family/platform-ingest-logstash-ubuntu-2204
193+
machineType: "n2-standard-4"
194+
diskSizeGb: 64
195+
retry:
196+
automatic:
197+
- limit: 3
198+
command: |
199+
set -euo pipefail
200+
201+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
202+
docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split 1 3
203+
204+
- label: ":lab_coat: Integration Tests - FIPS mode / part 3-of-3"
205+
key: "integration-tests-fips-part-3-of-3"
206+
agents:
207+
provider: gcp
208+
imageProject: elastic-images-prod
209+
image: family/platform-ingest-logstash-ubuntu-2204
210+
machineType: "n2-standard-4"
211+
diskSizeGb: 64
212+
retry:
213+
automatic:
214+
- limit: 3
215+
command: |
216+
set -euo pipefail
217+
218+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
219+
docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split 2 3
220+
82221
- label: ":lab_coat: Integration Tests / part 1-of-3"
83-
key: "integration-tests-part-1-of-3"
222+
key: "integration-tests-part-1"
84223
agents:
85224
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
86225
cpu: "8"
@@ -226,6 +365,40 @@ steps:
226365
source .buildkite/scripts/common/container-agent.sh
227366
x-pack/ci/integration_tests.sh
228367
368+
- label: ":lab_coat: x-pack unit tests - FIPS mode"
369+
key: "x-pack-unit-tests-fips"
370+
agents:
371+
provider: gcp
372+
imageProject: elastic-images-prod
373+
image: family/platform-ingest-logstash-ubuntu-2204
374+
machineType: "n2-standard-4"
375+
diskSizeGb: 64
376+
retry:
377+
automatic:
378+
- limit: 3
379+
command: |
380+
set -euo pipefail
381+
382+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
383+
docker run -e FEDRAMP_HIGH_MODE=true test-runner-image x-pack/ci/unit_tests.sh
384+
385+
- label: ":lab_coat: x-pack integration - FIPS mode"
386+
key: "integration-tests-x-pack-fips"
387+
agents:
388+
provider: gcp
389+
imageProject: elastic-images-prod
390+
image: family/platform-ingest-logstash-ubuntu-2204
391+
machineType: "n2-standard-4"
392+
diskSizeGb: 64
393+
retry:
394+
automatic:
395+
- limit: 3
396+
command: |
397+
set -euo pipefail
398+
399+
docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
400+
docker run -e FEDRAMP_HIGH_MODE=true test-runner-image x-pack/ci/integration_tests.sh
401+
229402
- wait: ~
230403
continue_on_failure: true
231404

‎.buildkite/scripts/common/qualified-version.sh

+5
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ export QUALIFIED_VERSION="$(
1717
# e.g: 8.0.0-alpha1
1818
printf '%s' "${VERSION_QUALIFIER:+-${VERSION_QUALIFIER}}"
1919
20+
# Include git SHA if requested
21+
if [[ -n "${INCLUDE_COMMIT_ID:+x}" ]]; then
22+
printf '%s' "-$(git rev-parse --short HEAD)"
23+
fi
24+
2025
# add the SNAPSHOT tag unless WORKFLOW_TYPE=="staging" or RELEASE=="1"
2126
if [[ ! ( "${WORKFLOW_TYPE:-}" == "staging" || "${RELEASE:+$RELEASE}" == "1" ) ]]; then
2227
printf '%s' "-SNAPSHOT"

‎.buildkite/scripts/dra/build-and-push-observability-sre.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
# Script to build and publish ObservabilitySRE container
3+
# Currently this is built on a host with the target architecture.
4+
# This allows us to utilize the make file for building the container and
5+
# to ensure the best compatability with the host architecture.
6+
# A later step in CI will take care of pushing a tag that references the right
7+
# image using `docker manifest` commands.
8+
9+
echo "Setting up environment"
10+
source .buildkite/scripts/common/vm-agent.sh
11+
source .buildkite/scripts/dra/docker-env-setup.sh
12+
13+
echo "Building ObservabilitySRE container"
14+
./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
15+
16+
echo "Pushing ObservabilitySRE container to Docker repository"
17+
docker_login
18+
19+
# Get qualified version without SHA (this is what the gradle task will produce)
20+
QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
21+
22+
# Set environment variable to include SHA and get version with SHA
23+
QUALIFIED_VERSION_WITH_SHA="$(INCLUDE_COMMIT_ID=1 .buildkite/scripts/common/qualified-version.sh)"
24+
25+
REGISTRY_PATH=docker.elastic.co/logstash/logstash-observability-sre
26+
27+
# Current architecture
28+
ARCH="${ARCH:-x86_64}" # Default to x86_64 if ARCH is not set
29+
echo "Architecture: ${ARCH}"
30+
31+
# Create the full tag with SHA and architecture
32+
FULL_TAG="${QUALIFIED_VERSION_WITH_SHA}-${ARCH}"
33+
echo "Tagging and pushing: ${REGISTRY_PATH}:${QUALIFIED_VERSION} as ${REGISTRY_PATH}:${FULL_TAG}"
34+
docker tag ${REGISTRY_PATH}:${QUALIFIED_VERSION} ${REGISTRY_PATH}:${FULL_TAG}
35+
docker push ${REGISTRY_PATH}:${FULL_TAG}
36+
37+
# Teardown Docker environment
38+
source .buildkite/scripts/dra/docker-env-teardown.sh

‎.buildkite/scripts/dra/generatesteps.py

+61-1
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,61 @@ def package_aarch64_docker_step(branch, workflow_type):
7979

8080
return step
8181

82+
def ship_observability_sre_image_steps(branch, workflow_type):
83+
step = f'''
84+
- label: ":package: Build & Ship aarch64 ObservabilitySRE container / {branch}-{workflow_type.upper()}"
85+
key: "logstash_build_and_ship_observability_sre_aarch64"
86+
soft_fail: true
87+
depends_on: logstash_publish_dra
88+
agents:
89+
provider: aws
90+
imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
91+
instanceType: "m6g.4xlarge"
92+
diskSizeGb: 200
93+
artifact_paths:
94+
- "**/*.hprof"
95+
command: |
96+
export WORKFLOW_TYPE="{workflow_type}"
97+
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
98+
export ARCH="aarch64"
99+
eval "$(rbenv init -)"
100+
.buildkite/scripts/dra/build-and-push-observability-sre.sh
101+
- label: ":package: Build & Ship x86_64 ObservabilitySRE container / {branch}-{workflow_type.upper()}"
102+
key: "logstash_build_and_ship_observability_sre_x86_64"
103+
soft_fail: true
104+
depends_on: logstash_publish_dra
105+
agents:
106+
provider: gcp
107+
imageProject: elastic-images-prod
108+
image: family/platform-ingest-logstash-ubuntu-2204
109+
machineType: "n2-standard-16"
110+
diskSizeGb: 200
111+
artifact_paths:
112+
- "**/*.hprof"
113+
command: |
114+
export WORKFLOW_TYPE="{workflow_type}"
115+
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
116+
export ARCH="x86_64"
117+
eval "$(rbenv init -)"
118+
.buildkite/scripts/dra/build-and-push-observability-sre.sh
119+
- label: ":docker: Create & Push ObservabilitySRE Multi-Arch Manifest / {branch}-{workflow_type.upper()}"
120+
key: "logstash_create_observability_sre_manifest"
121+
depends_on:
122+
- "logstash_build_and_ship_observability_sre_aarch64"
123+
- "logstash_build_and_ship_observability_sre_x86_64"
124+
agents:
125+
provider: gcp
126+
imageProject: elastic-images-prod
127+
image: family/platform-ingest-logstash-ubuntu-2204
128+
machineType: "n2-standard-8"
129+
command: |
130+
export WORKFLOW_TYPE="{workflow_type}"
131+
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
132+
eval "$(rbenv init -)"
133+
.buildkite/scripts/dra/multi-architecture-observability-sre.sh
134+
'''
135+
return step
136+
82137
def publish_dra_step(branch, workflow_type, depends_on):
83138
step = f'''
84139
- label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts"
@@ -139,9 +194,14 @@ def build_steps_to_yaml(branch, workflow_type):
139194
"steps": build_steps_to_yaml(branch, workflow_type),
140195
})
141196

142-
# Final step: pull artifacts built above and publish them via the release-manager
197+
# Pull artifacts built above and publish them via the release-manager
143198
structure["steps"].extend(
144199
yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
145200
)
146201

202+
# Once published, do the same for observabilitySRE image
203+
structure["steps"].extend(
204+
yaml.safe_load(ship_observability_sre_image_steps(branch, workflow_type)),
205+
)
206+
147207
print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))

‎.buildkite/scripts/dra/multi-architecture-observability-sre.sh

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
# Script to create and push Docker manifest for multi-architecture support
3+
# This MUST be run after build-and-push-observabilty-sre.sh!
4+
5+
source .buildkite/scripts/common/vm-agent.sh
6+
source .buildkite/scripts/dra/docker-env-setup.sh
7+
8+
docker_login
9+
10+
# Set INCLUDE_COMMIT_ID to include git SHA in version
11+
QUALIFIED_VERSION="$(INCLUDE_COMMIT_ID=1 .buildkite/scripts/common/qualified-version.sh)"
12+
REGISTRY_PATH=docker.elastic.co/logstash/logstash-observability-sre
13+
14+
# Architecture-specific tags (created by the build steps)
15+
X86_64_TAG="${QUALIFIED_VERSION}-x86_64"
16+
AARCH64_TAG="${QUALIFIED_VERSION}-aarch64"
17+
18+
# Target manifest tags - already has SHA from QUALIFIED_VERSION
19+
VERSION_MANIFEST_TAG="${QUALIFIED_VERSION}"
20+
21+
# Create and push manifest with version (which already includes SHA)
22+
echo "Creating manifest list for: ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}"
23+
docker manifest create ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
24+
${REGISTRY_PATH}:${X86_64_TAG} \
25+
${REGISTRY_PATH}:${AARCH64_TAG}
26+
27+
docker manifest annotate ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
28+
${REGISTRY_PATH}:${X86_64_TAG} --os linux --arch amd64
29+
30+
docker manifest annotate ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
31+
${REGISTRY_PATH}:${AARCH64_TAG} --os linux --arch arm64
32+
33+
echo "Pushing manifest: ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}"
34+
docker manifest push ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}
35+
36+
# Also create version without SHA for effective "latest" tag
37+
BASE_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
38+
echo "Creating manifest list for: ${REGISTRY_PATH}:${BASE_VERSION}"
39+
docker manifest create ${REGISTRY_PATH}:${BASE_VERSION} \
40+
${REGISTRY_PATH}:${X86_64_TAG} \
41+
${REGISTRY_PATH}:${AARCH64_TAG}
42+
43+
docker manifest annotate ${REGISTRY_PATH}:${BASE_VERSION} \
44+
${REGISTRY_PATH}:${X86_64_TAG} --os linux --arch amd64
45+
46+
docker manifest annotate ${REGISTRY_PATH}:${BASE_VERSION} \
47+
${REGISTRY_PATH}:${AARCH64_TAG} --os linux --arch arm64
48+
49+
echo "Pushing manifest: ${REGISTRY_PATH}:${BASE_VERSION}"
50+
docker manifest push ${REGISTRY_PATH}:${BASE_VERSION}
51+
52+
# Teardown Docker environment
53+
source .buildkite/scripts/dra/docker-env-teardown.sh

0 commit comments

Comments
 (0)