add OLMv1 benchmark test steps

Signed-off-by: Jian Zhang <[email protected]>

Author: jianzhangbjz

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.19-nightly-x86.yaml

@@ -15,6 +15,15 @@ resources:
       cpu: 100m
       memory: 200Mi
 tests:
+- as: olmv1-benchmark-test
+  cron: 0 0,4 * * *
+  steps:
+    allow_skip_on_success: true
+    cluster_profile: gcp-qe
+    test:
+    - ref: olmv1-performance
+    workflow: cucushift-installer-rehearse-gcp-ipi
+  timeout: 3h0m0s
 - as: netpol-24nodes
   cron: 0 12 6,13,20,27 * *
   steps:

ci-operator/jobs/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main-periodics.yaml

@@ -4424,6 +4424,82 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build02
+  cron: 0 0,4 * * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+    timeout: 3h0m0s
+  extra_refs:
+  - base_ref: main
+    org: openshift-eng
+    repo: ocp-qe-perfscale-ci
+  labels:
+    ci-operator.openshift.io/cloud: gcp
+    ci-operator.openshift.io/cloud-cluster-profile: gcp-qe
+    ci-operator.openshift.io/variant: aws-4.19-nightly-x86
+    ci.openshift.io/generator: prowgen
+    job-release: "4.19"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-eng-ocp-qe-perfscale-ci-main-aws-4.19-nightly-x86-olmv1-benchmark-test
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=olmv1-benchmark-test
+      - --variant=aws-4.19-nightly-x86
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: '@yearly'

ci-operator/step-registry/olmv1/performance/OWNERS

@@ -0,0 +1,11 @@
+approvers:
+  - Xia-Zhao-rh
+  - kuiwang02
+  - bandrade
+  - jianzhangbjz
+  - anpingli
+reviewers:
+  - Xia-Zhao-rh
+  - kuiwang02
+  - bandrade
+  - jianzhangbjz

ci-operator/step-registry/olmv1/performance/olmv1-performance-commands.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+set -x
+# cat /etc/os-release
+# oc config view
+# oc projects
+oc version
+oc get co
+oc get nodes
+python --version
+pushd /tmp
+python -m virtualenv ./venv_qe
+source ./venv_qe/bin/activate
+
+# connected to ES server
+ES_SECRETS_PATH=${ES_SECRETS_PATH:-/secret}
+ES_HOST=${ES_HOST:-"search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com"}
+ES_PASSWORD=$(cat "${ES_SECRETS_PATH}/password")
+ES_USERNAME=$(cat "${ES_SECRETS_PATH}/username")
+if [ -e "${ES_SECRETS_PATH}/host" ]; then
+    ES_HOST=$(cat "${ES_SECRETS_PATH}/host")
+fi
+
+# # download kube-burner-ocp
+# KUBE_BURNER_VERSION=1.6.8
+
+git clone https://github.com/kube-burner/kube-burner-ocp.git --branch main --depth 1
+pushd kube-burner-ocp
+make build
+./bin/amd64/kube-burner-ocp olm -h
+
+# REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking";
+# LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/e2e-benchmarking/releases/latest" | jq -r '.tag_name');
+# TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)";
+# git clone $REPO_URL $TAG_OPTION --depth 1
+# pushd e2e-benchmarking/workloads/kube-burner-ocp-wrapper
+# export WORKLOAD=olm
+
+export ITERATIONS=30
+export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@$ES_HOST"
+
+if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then
+    EXTRA_FLAGS+=" --local-indexing"
+fi
+EXTRA_FLAGS+=" --gc-metrics=true --profile-type=${PROFILE_TYPE}"
+
+export EXTRA_FLAGS
+export ADDITIONAL_PARAMS
+
+export WORKLOAD=olm LOG_LEVEL=debug
+
+cmd="./bin/amd64/kube-burner-ocp ${WORKLOAD} --log-level=${LOG_LEVEL} --qps=${QPS} --burst=${BURST} --gc=${GC} --uuid ${UUID} --iterations=${ITERATIONS}"
+
+# If ES_SERVER is specified
+if [[ -n ${ES_SERVER} ]]; then
+  curl -k -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/ripsaw-kube-burner/_doc -d "${METADATA}" -o /dev/null
+  cmd+=" --es-server=${ES_SERVER} --es-index=ripsaw-kube-burner"
+fi
+# If PERFORMANCE_PROFILE is specified
+if [[ -n ${PERFORMANCE_PROFILE} && ${WORKLOAD} =~ "rds-core" ]]; then
+  cmd+=" --perf-profile=${PERFORMANCE_PROFILE}"
+fi
+
+# Enable pprof collection
+if $PPROF; then
+  cmd+=" --pprof"
+fi
+
+# Capture the exit code of the run, but don't exit the script if it fails.
+set +e
+
+echo $cmd
+JOB_START=${JOB_START:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")};
+$cmd
+exit_code=$?
+JOB_END=${JOB_END:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")};
+if [ $exit_code -eq 0 ]; then
+  JOB_STATUS="success"
+else
+  JOB_STATUS="failure"
+fi
+
+wget http://github.com/cloud-bulldozer/e2e-benchmarking/blob/master/utils/index.sh
+export JOB_START="$JOB_START" JOB_END="$JOB_END" JOB_STATUS="$JOB_STATUS" UUID="$UUID" WORKLOAD="$WORKLOAD" ES_SERVER="$ES_SERVER" 
+source index.sh || true
+
+
+rm -f ${SHARED_DIR}/index.json
+
+folder_name=$(ls -t -d /tmp/*/ | head -1)
+jq ".iterations = $ITERATIONS" $folder_name/index_data.json >> ${SHARED_DIR}/index_data.json
+
+cp "${SHARED_DIR}"/index_data.json "${SHARED_DIR}"/${WORKLOAD}-index_data.json 
+cp "${SHARED_DIR}"/${WORKLOAD}-index_data.json  "${ARTIFACT_DIR}"/${WORKLOAD}-index_data.json
+
+
+if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then
+    metrics_folder_name=$(find . -maxdepth 1 -type d -name 'collected-metric*' | head -n 1)
+    cp -r "${metrics_folder_name}" "${ARTIFACT_DIR}/"
+fi
+
+echo "OLMv1 benchmark test finised"
+

ci-operator/step-registry/olmv1/performance/olmv1-performance-ref.metadata.json

@@ -0,0 +1,18 @@
+{
+	"path": "olmv1/performance/olmv1-performance-ref.yaml",
+	"owners": {
+		"approvers": [
+			"Xia-Zhao-rh",
+			"kuiwang02",
+			"bandrade",
+			"jianzhangbjz",
+			"anpingli"
+		],
+		"reviewers": [
+			"Xia-Zhao-rh",
+			"kuiwang02",
+			"bandrade",
+			"jianzhangbjz"
+		]
+	}
+}

ci-operator/step-registry/olmv1/performance/olmv1-performance-ref.yaml

@@ -0,0 +1,68 @@
+ref:
+  as: olmv1-performance
+  cli: latest
+  from_image:
+    namespace: ci
+    name: tests-private-burner
+    tag: "4.19"
+  resources:
+    requests:
+      cpu: 200m
+      memory: 500Mi
+  timeout: 30m
+  grace_period: 15m0s
+  env:
+  - name: E2E_VERSION
+    default: "v2.3.5"
+    documentation: |-
+      Override the e2e version
+  - name: CHURN
+    default: "false"
+    documentation: |-
+      Default is false, which means the workload with delete and recreate a set number of namespaces
+  # - name: EXTRA_FLAGS
+  #   default: "--churn-duration=20m --timeout=5h"
+  #   documentation: |-
+  #     Default is Churn Duration for 20 minutes
+  - name: PROFILE_TYPE
+    default: "both"
+    documentation: |-
+      Kube-burner indexing profile type
+  - name: GC
+    default: "true"
+    documentation: |-
+      Default is true, which means clean up the pod/resource that kube-burner ocp created, you can set it to false to keep the resource
+  - name: ITERATION_MULTIPLIER_ENV
+    default: "9"
+    documentation: |-
+      The number of iterations per worker nodes to create
+  - name: ENABLE_LOCAL_INDEX
+    default: "false"
+    documentation: |-
+      Trigger to enable local indexing
+  - name: ES_SECRETS_PATH
+    default: ""
+    documentation: |-
+      Override elasticsearch secrets path.
+  - name: ADDITIONAL_PARAMS
+    default: '{}'
+    documentation: |-
+      JSON value to upload additonal metadata to the fingerprint
+  - name: PPROF
+    default: "true"
+  commands: olmv1-performance-commands.sh
+  credentials:
+  - namespace: test-credentials
+    name: ocp-perfscale-prod-es-creds
+    mount_path: /prod-secret
+  - namespace: test-credentials # this entry injects the custom credential
+    name: ocp-qe-perfscale-es
+    mount_path: /secret
+  - namespace: test-credentials
+    name: stackrox-perfscale-elasticsearch
+    mount_path: /secret_stackrox
+  - namespace: test-credentials
+    name: ocp-perfscale-horreum-poc
+    mount_path: /horreum-secret
+  documentation: |-
+    Run olmv1 benchmark test.