feat: add support for building the "release" Prometheus from source

prombench/manifests/cluster-infra/7a_commentmonitor_configmap_noparse.yaml

@@ -18,9 +18,12 @@ data:
           * See the details [here](https://github.com/prometheus/test-infra/tree/master/prombench/manifests/prombench#benchmarking-from-the-different-directory), defaults to `manifests/prombench`.
         * `--bench.version=<branch | @commit>`
           * See the details [here](https://github.com/prometheus/test-infra/tree/master/prombench/manifests/prombench#benchmarking-from-the-custom-test-infra-commitbranch), defaults to `master`.
+        * `--bench.use-registry=<true | anything else>`
+          * If `true`, benchmark will use the Prometheus from the quay.io/prometheus/prometheus for the "release" version. Any other value will cause benchmark to rebuild the prometheus from the provided (start or restart argument) reference, same as the PR Prometheus. By default it builds from source.
 
         **Examples:**
         * `/prombench v3.0.0`
+        * `/prombench v3.0.0 --bench.use-registry=true`
         * `/prombench v3.0.0 --bench.version=@aca1803ccf5d795eee4b0848707eab26d05965cc --bench.directory=manifests/prombench`
 
       verify_user: true
@@ -32,11 +35,12 @@ data:
 
       - name: restart
         event_type: prombench_restart
-        arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*)
+        arg_regex: (\S+)
         arg_name: RELEASE
         flag_args:
           bench.directory: BENCHMARK_DIRECTORY
           bench.version: BENCHMARK_VERSION
+          bench.use-registry: BENCHMARK_USE_REGISTRY
         comment_template: |
           ⏱️ Welcome (again) to Prometheus Benchmarking Tool. ⏱️
 
@@ -65,11 +69,12 @@ data:
 
       - name: "" # start is a default (empty command).
         event_type: prombench_start
-        arg_regex: (master|main|v[0-9]+\.[0-9]+\.[0-9]+\S*)
+        arg_regex: (\S+)
         arg_name: RELEASE
         flag_args:
           bench.directory: BENCHMARK_DIRECTORY
           bench.version: BENCHMARK_VERSION
+          bench.use-registry: BENCHMARK_USE_REGISTRY
         label: prombench
         comment_template: |
           ⏱️ Welcome to Prometheus Benchmarking Tool. ⏱️

prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml

@@ -766,7 +766,7 @@ spec:
         image: docker.io/prominfra/prometheus-builder:master
         imagePullPolicy: Always
         env:
-        - name: PR_NUMBER
+        - name: REFERENCE
           value: "{{ .PR_NUMBER }}"
         - name: VOLUME_DIR
           value: "/prometheus-builder" # same as mountPath

prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml

@@ -761,11 +761,39 @@ spec:
                 - prometheus
       securityContext:
         runAsUser: 0
+      initContainers:
+      - name: prometheus-builder
+        image: docker.io/prominfra/prometheus-builder:master
+        imagePullPolicy: Always
+        env:
+        - name: REFERENCE
+          value: "{{ .RELEASE }}"
+        - name: USE_REGISTRY
+          value: "{{ .BENCHMARK_USE_REGISTRY }}"
+        - name: VOLUME_DIR
+          value: "/prometheus-builder" # same as mountPath
+        - name: GITHUB_ORG
+          value: "{{ .GITHUB_ORG }}"
+        - name: GITHUB_REPO
+          value: "{{ .GITHUB_REPO }}"
+        volumeMounts:
+        - name: prometheus-executable
+          mountPath: /prometheus-builder
       containers:
       - name: prometheus
-        image: quay.io/prometheus/prometheus:{{ .RELEASE }}
+        image: quay.io/prometheus/busybox:latest
         imagePullPolicy: Always
-        command: [ "/bin/prometheus" ]
+        # The prometheus-builder takes a while to build
+        # so make sure to start it before the release deployment.
+        # Mark it ready only when prometheus is started.
+        # This way we have the least time difference in the scraped metrics.
+        readinessProbe:
+          tcpSocket:
+            port: 9090
+          initialDelaySeconds: 30
+          periodSeconds: 2
+          failureThreshold: 30
+        command: ["/usr/bin/prometheus"]
         args: [
           "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-release",
           "--storage.tsdb.path=/prometheus",
@@ -781,6 +809,8 @@ spec:
           mountPath: /etc/prometheus
         - name: instance-ssd
           mountPath: /prometheus
+        - name: prometheus-executable
+          mountPath: /usr/bin
         ports:
         - name: prom-web
           containerPort: 9090
@@ -792,6 +822,8 @@ spec:
         hostPath:
           # /mnt is where GKE keeps it's SSD.
           path: /mnt/disks/ssd0
+      - name: prometheus-executable
+        emptyDir: {}
       terminationGracePeriodSeconds: 300
       nodeSelector:
         node-name: prometheus-{{ .PR_NUMBER }}

tools/comment-monitor/internal/command_test.go

@@ -265,6 +265,19 @@ func TestParseCommand_ProdCommentMonitorConfig(t *testing.T) {
 		// Not matching cases.
 		{comment: ""},
 		{comment: "How to start prombench? I think it was something like:\n\n /prombench main\n\nYolo"},
+		// Extended reference building from source cases
+		{
+			comment: "/prombench d40aff95265ae8af512d58dd7b76c77abfb9611f\nSome text after",
+			expect:  testCommand(eventTypeStart, map[string]string{"RELEASE": "d40aff95265ae8af512d58dd7b76c77abfb9611f"}),
+		},
+		{
+			comment: "/prombench some-branch/yolo --bench.use-registry=true\nSome text after",
+			expect:  testCommand(eventTypeStart, map[string]string{"RELEASE": "some-branch/yolo", "BENCHMARK_USE_REGISTRY": "true"}),
+		},
+		{
+			comment: "/prombench restart some-branch/yolo --bench.directory=manifests/prombench --bench.use-registry=true\nSome text after",
+			expect:  testCommand(eventTypeRestart, map[string]string{"RELEASE": "some-branch/yolo", "BENCHMARK_DIRECTORY": "manifests/prombench", "BENCHMARK_USE_REGISTRY": "true"}),
+		},
 	})
 }
 

tools/prometheus-builder/build.sh

@@ -1,34 +1,82 @@
 #!/bin/bash
 
+# This script created a Prometheus binary to use in benchmarks and place it in $VOLUME/prometheus.
+# It uses $REFERENCE variable to build or fetch the binary.
+# If $USE_REGISTRY is set to "true" it uses the binary from the quay.io/prometheus/prometheus:${REFERENCE},
+# otherwise it builds it from source, from the given reference (PR number, branch or git SHA).
+
+# Default values
 DIR="/go/src/github.com/prometheus/prometheus"
 
-if [[ -z $PR_NUMBER || -z $VOLUME_DIR || -z $GITHUB_ORG || -z $GITHUB_REPO ]]; then
-    echo "ERROR:: environment variables not set correctly"
+REFERENCE=${REFERENCE}
+
+# We want builder to work with the old scenarios, so support PR_NUMBER var for compatibility.
+if [[ -z ${REFERENCE} ]]; then
+  REFERENCE=${PR_NUMBER}
+fi
+
+if [[ -z ${REFERENCE} || -z ${VOLUME_DIR} || -z ${GITHUB_ORG} || -z ${GITHUB_REPO} ]]; then
+    echo "ERROR:: environment variables not set correctly, requires REFERENCE (or PR_NUMBER), VOLUME_DIR, GITHUB_ORG, GITHUB_REPO"
     exit 1;
 fi
 
+# Fetch from quay if requested.
+if [[ "${USE_REGISTRY}" == "true" ]]; then
+    echo ">> USE_REGISTRY is enabled."
+    echo ">> Attempting to download binary from quay.io/prometheus/prometheus:${REFERENCE}"
+
+    IMAGE="quay.io/prometheus/prometheus:${REFERENCE}"
+
+    if ! CONTAINER_ID=$(docker create "${IMAGE}"); then
+        echo "ERROR:: Could not pull or create container from ${IMAGE}"
+        exit 1
+    fi
+
+    echo ">> Extracting prometheus binary from ${CONTAINER_ID} container..."
+    if ! docker cp "${CONTAINER_ID}:/bin/prometheus" "${VOLUME_DIR}/prometheus"; then
+        echo "ERROR:: Failed to copy binary from container"
+        docker rm -v "${CONTAINER_ID}" >/dev/null
+        exit 1
+    fi
+    docker rm -v "${CONTAINER_ID}" >/dev/null
+    echo ">> Binary successfully downloaded and copied."
+    exit 0
+fi
+
 # Clone the repository with a shallow clone
 echo ">> Cloning repository $GITHUB_ORG/$GITHUB_REPO (shallow clone)"
-if ! git clone --depth 1 https://github.com/$GITHUB_ORG/$GITHUB_REPO.git $DIR; then
+if ! git clone --depth 1 "https://github.com/$GITHUB_ORG/$GITHUB_REPO.git" "$DIR"; then
     echo "ERROR:: Cloning of repo $GITHUB_ORG/$GITHUB_REPO failed"
     exit 1;
 fi
 
-cd $DIR || exit 1
+cd "$DIR" || exit 1
 
-echo ">> Fetching Pull Request $GITHUB_ORG/$GITHUB_REPO/pull/$PR_NUMBER"
-if ! git fetch origin pull/$PR_NUMBER/head:pr-branch; then
-    echo "ERROR:: Fetching of PR $PR_NUMBER failed"
-    exit 1;
-fi
+echo ">> Resolving git state for building from source from ${REFERENCE}..."
+
+# Attempt 1: Try pulling PR first which will only work if REFERENCE is a PR number.
+if git fetch origin "pull/${REFERENCE}/head:pr-branch" 2>/dev/null; then
+    echo ">> Successfully fetched PR reference: pull/${REFERENCE}/head"
+    git checkout pr-branch
+else
+    # Attempt 2: If PR fetch fails, try assuming it's a branch or Git SHA
+    echo ">> Reference 'pull/${REFERENCE}/head' not found; assuming reference is not a PR number. Trying to fetch '${REFERENCE}' as a remote branch or SHA..."
 
-git checkout pr-branch
+    # We fetch specifically the ref to FETCH_HEAD to avoid naming conflicts
+    if git fetch origin "${REFERENCE}"; then
+        echo ">> Successfully fetched reference: ${REFERENCE}"
+        git checkout FETCH_HEAD
+    else
+        echo "ERROR:: Could not resolve '${REFERENCE}' as a Pull Request, remote branch, or Git SHA."
+        exit 1
+    fi
+fi
 
-echo ">> Creating prometheus binaries"
+echo ">> Creating prometheus binary using promu"
 if ! make build PROMU_BINARIES="prometheus"; then
     echo "ERROR:: Building of binaries failed"
     exit 1;
 fi
 
 echo ">> Copy files to volume"
-cp prometheus               $VOLUME_DIR/prometheus
+cp prometheus "$VOLUME_DIR/prometheus"