diff --git a/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml b/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml index 53a59fc35..e51ca9694 100644 --- a/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml +++ b/prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml @@ -18,9 +18,12 @@ data: * See the details [here](https://github.com/prometheus/test-infra/tree/master/prombench/manifests/prombench#benchmarking-from-the-different-directory), defaults to `manifests/prombench`. * `--bench.version=` * See the details [here](https://github.com/prometheus/test-infra/tree/master/prombench/manifests/prombench#benchmarking-from-the-custom-test-infra-commitbranch), defaults to `master`. + * `--bench.use-registry=` + * If `true`, benchmark will use the Prometheus from the quay.io/prometheus/prometheus for the "release" version. Any other value will cause benchmark to rebuild the prometheus from the provided (start or restart argument) reference, same as the PR Prometheus. By default it builds from source. **Examples:** * `/prombench v3.0.0` + * `/prombench v3.0.0 --bench.use-registry=true` * `/prombench v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc --bench.directory=manifests/prombench` verify_user: true @@ -32,11 +35,12 @@ data: - name: restart event_type: prombench_restart - arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) + arg_regex: (\S+) arg_name: RELEASE flag_args: bench.directory: BENCHMARK_DIRECTORY bench.version: BENCHMARK_VERSION + bench.use-registry: BENCHMARK_USE_REGISTRY comment_template: | ⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️ @@ -65,11 +69,12 @@ data: - name: "" # start is a default (empty command). event_type: prombench_start - arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*) + arg_regex: (\S+) arg_name: RELEASE flag_args: bench.directory: BENCHMARK_DIRECTORY bench.version: BENCHMARK_VERSION + bench.use-registry: BENCHMARK_USE_REGISTRY label: prombench comment_template: | ⏱️ Welcome to Prometheus Benchmarking Tool. ⏱️ diff --git a/prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml index 4e3e56e2d..4a369af63 100644 --- a/prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml @@ -766,7 +766,7 @@ spec: image: docker.io/prominfra/prometheus-builder:master imagePullPolicy: Always env: - - name: PR_NUMBER + - name: REFERENCE value: "{{ .PR_NUMBER }}" - name: VOLUME_DIR value: "/prometheus-builder" # same as mountPath diff --git a/prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml b/prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml index 652ff3373..75155d08c 100644 --- a/prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml @@ -761,11 +761,39 @@ spec: - prometheus securityContext: runAsUser: 0 + initContainers: + - name: prometheus-builder + image: docker.io/prominfra/prometheus-builder:master + imagePullPolicy: Always + env: + - name: REFERENCE + value: "{{ .RELEASE }}" + - name: USE_REGISTRY + value: "{{ .BENCHMARK_USE_REGISTRY }}" + - name: VOLUME_DIR + value: "/prometheus-builder" # same as mountPath + - name: GITHUB_ORG + value: "{{ .GITHUB_ORG }}" + - name: GITHUB_REPO + value: "{{ .GITHUB_REPO }}" + volumeMounts: + - name: prometheus-executable + mountPath: /prometheus-builder containers: - name: prometheus - image: quay.io/prometheus/prometheus:{{ .RELEASE }} + image: quay.io/prometheus/busybox:latest imagePullPolicy: Always - command: [ "/bin/prometheus" ] + # The prometheus-builder takes a while to build + # so make sure to start it before the release deployment. + # Mark it ready only when prometheus is started. + # This way we have the least time difference in the scraped metrics. + readinessProbe: + tcpSocket: + port: 9090 + initialDelaySeconds: 30 + periodSeconds: 2 + failureThreshold: 30 + command: ["/usr/bin/prometheus"] args: [ "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-release", "--storage.tsdb.path=/prometheus", @@ -781,6 +809,8 @@ spec: mountPath: /etc/prometheus - name: instance-ssd mountPath: /prometheus + - name: prometheus-executable + mountPath: /usr/bin ports: - name: prom-web containerPort: 9090 @@ -792,6 +822,8 @@ spec: hostPath: # /mnt is where GKE keeps it's SSD. path: /mnt/disks/ssd0 + - name: prometheus-executable + emptyDir: {} terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }} diff --git a/tools/comment-monitor/internal/command_test.go b/tools/comment-monitor/internal/command_test.go index a0014b934..dcf57e931 100644 --- a/tools/comment-monitor/internal/command_test.go +++ b/tools/comment-monitor/internal/command_test.go @@ -265,6 +265,19 @@ func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) { // Not matching cases. {comment: ""}, {comment: "How to start prombench? I think it was something like:\n\n /prombench main\n\nYolo"}, + // Extended reference building from source cases + { + comment: "/prombench d40aff95265ae8af512d58dd7b76c77abfb9611f\nSome text after", + expect: testCommand(eventTypeStart, map[string]string{"RELEASE": "d40aff95265ae8af512d58dd7b76c77abfb9611f"}), + }, + { + comment: "/prombench some-branch/yolo --bench.use-registry=true\nSome text after", + expect: testCommand(eventTypeStart, map[string]string{"RELEASE": "some-branch/yolo", "BENCHMARK_USE_REGISTRY": "true"}), + }, + { + comment: "/prombench restart some-branch/yolo --bench.directory=manifests/prombench --bench.use-registry=true\nSome text after", + expect: testCommand(eventTypeRestart, map[string]string{"RELEASE": "some-branch/yolo", "BENCHMARK_DIRECTORY": "manifests/prombench", "BENCHMARK_USE_REGISTRY": "true"}), + }, }) } diff --git a/tools/prometheus-builder/build.sh b/tools/prometheus-builder/build.sh index a07328e62..9b11fc01e 100755 --- a/tools/prometheus-builder/build.sh +++ b/tools/prometheus-builder/build.sh @@ -1,34 +1,82 @@ #!/bin/bash +# This script created a Prometheus binary to use in benchmarks and place it in $VOLUME/prometheus. +# It uses $REFERENCE variable to build or fetch the binary. +# If $USE_REGISTRY is set to "true" it uses the binary from the quay.io/prometheus/prometheus:${REFERENCE}, +# otherwise it builds it from source, from the given reference (PR number, branch or git SHA). + +# Default values DIR="/go/src/github.com/prometheus/prometheus" -if [[ -z $PR_NUMBER || -z $VOLUME_DIR || -z $GITHUB_ORG || -z $GITHUB_REPO ]]; then - echo "ERROR:: environment variables not set correctly" +REFERENCE=${REFERENCE} + +# We want builder to work with the old scenarios, so support PR_NUMBER var for compatibility. +if [[ -z ${REFERENCE} ]]; then + REFERENCE=${PR_NUMBER} +fi + +if [[ -z ${REFERENCE} || -z ${VOLUME_DIR} || -z ${GITHUB_ORG} || -z ${GITHUB_REPO} ]]; then + echo "ERROR:: environment variables not set correctly, requires REFERENCE (or PR_NUMBER), VOLUME_DIR, GITHUB_ORG, GITHUB_REPO" exit 1; fi +# Fetch from quay if requested. +if [[ "${USE_REGISTRY}" == "true" ]]; then + echo ">> USE_REGISTRY is enabled." + echo ">> Attempting to download binary from quay.io/prometheus/prometheus:${REFERENCE}" + + IMAGE="quay.io/prometheus/prometheus:${REFERENCE}" + + if ! CONTAINER_ID=$(docker create "${IMAGE}"); then + echo "ERROR:: Could not pull or create container from ${IMAGE}" + exit 1 + fi + + echo ">> Extracting prometheus binary from ${CONTAINER_ID} container..." + if ! docker cp "${CONTAINER_ID}:/bin/prometheus" "${VOLUME_DIR}/prometheus"; then + echo "ERROR:: Failed to copy binary from container" + docker rm -v "${CONTAINER_ID}" >/dev/null + exit 1 + fi + docker rm -v "${CONTAINER_ID}" >/dev/null + echo ">> Binary successfully downloaded and copied." + exit 0 +fi + # Clone the repository with a shallow clone echo ">> Cloning repository $GITHUB_ORG/$GITHUB_REPO (shallow clone)" -if ! git clone --depth 1 https://github.com/$GITHUB_ORG/$GITHUB_REPO.git $DIR; then +if ! git clone --depth 1 "https://github.com/$GITHUB_ORG/$GITHUB_REPO.git" "$DIR"; then echo "ERROR:: Cloning of repo $GITHUB_ORG/$GITHUB_REPO failed" exit 1; fi -cd $DIR || exit 1 +cd "$DIR" || exit 1 -echo ">> Fetching Pull Request $GITHUB_ORG/$GITHUB_REPO/pull/$PR_NUMBER" -if ! git fetch origin pull/$PR_NUMBER/head:pr-branch; then - echo "ERROR:: Fetching of PR $PR_NUMBER failed" - exit 1; -fi +echo ">> Resolving git state for building from source from ${REFERENCE}..." + +# Attempt 1: Try pulling PR first which will only work if REFERENCE is a PR number. +if git fetch origin "pull/${REFERENCE}/head:pr-branch" 2>/dev/null; then + echo ">> Successfully fetched PR reference: pull/${REFERENCE}/head" + git checkout pr-branch +else + # Attempt 2: If PR fetch fails, try assuming it's a branch or Git SHA + echo ">> Reference 'pull/${REFERENCE}/head' not found; assuming reference is not a PR number. Trying to fetch '${REFERENCE}' as a remote branch or SHA..." -git checkout pr-branch + # We fetch specifically the ref to FETCH_HEAD to avoid naming conflicts + if git fetch origin "${REFERENCE}"; then + echo ">> Successfully fetched reference: ${REFERENCE}" + git checkout FETCH_HEAD + else + echo "ERROR:: Could not resolve '${REFERENCE}' as a Pull Request, remote branch, or Git SHA." + exit 1 + fi +fi -echo ">> Creating prometheus binaries" +echo ">> Creating prometheus binary using promu" if ! make build PROMU_BINARIES="prometheus"; then echo "ERROR:: Building of binaries failed" exit 1; fi echo ">> Copy files to volume" -cp prometheus $VOLUME_DIR/prometheus +cp prometheus "$VOLUME_DIR/prometheus" \ No newline at end of file