ci: add nightly integration scenario for Opensearch with basic auth (#3725)

Author: bkenez

.github/workflows/test-integration-runner.yaml

@@ -203,6 +203,12 @@ env:
   # Camunda registry auth to access WebModeler Docker image since it's not public.
   TEST_DOCKER_USERNAME_CAMUNDA_CLOUD: ${{ secrets.DISTRO_CI_DOCKER_USERNAME_CAMUNDA }}
   TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD: ${{ secrets.DISTRO_CI_DOCKER_PASSWORD_CAMUNDA }}
+  # OpenSearch credentials
+  OPENSEARCH_USERNAME: ${{ secrets.OPENSEARCH_USERNAME }}
+  OPENSEARCH_PASSWORD: ${{ secrets.OPENSEARCH_PASSWORD }}
+  OPENSEARCH_PROTOCOL: "https"
+  OPENSEARCH_HOST: "search-qa-e2e-5q5uium4w7pgfz7i5tviimmmgm.eu-north-1.es.amazonaws.com"
+  OPENSEARCH_PORT: "443"
   # OIDC.
   ENTRA_PARENT_APP_CLIENT_ID: ${{ secrets.ENTRA_PARENT_APP_CLIENT_ID }}
   ENTRA_PARENT_APP_CLIENT_SECRET: ${{ secrets.ENTRA_PARENT_APP_CLIENT_SECRET }}
@@ -425,6 +431,13 @@ jobs:
             ${{ env.TEST_HELM_EXTRA_ARGS }}
             --set global.ingress.host=${{ steps.vars.outputs.ingress-host }}
         run: |
+          # Configure OpenSearch environment if using opensearch scenario
+          if [[ "${{ inputs.scenario }}" == "opensearch" ]] && [[ -n "${OPENSEARCH_USERNAME}" ]] && [[ -n "${OPENSEARCH_PASSWORD}" ]]; then
+            # Apply environment variable substitution to values file
+            values_file="charts/${{ inputs.camunda-helm-dir }}/test/integration/scenarios/chart-full-setup/values-integration-test-ingress-${TEST_VALUES_SCENARIO}.yaml"
+            envsubst < "$values_file" > /tmp/values-substituted.yaml && mv /tmp/values-substituted.yaml "$values_file"
+          fi
+          
           echo "Extra values from workflow:"
           echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
           task -d ${CI_TASKS_BASE_DIR}/chart-full-setup setup.pre
@@ -649,6 +662,13 @@ jobs:
           EXTRA_VALUES: ${{ inputs.extra-values }}
         continue-on-error: false
         run: |
+          # Configure OpenSearch environment if using opensearch scenario
+          if [[ "${{ inputs.scenario }}" == "opensearch" ]] && [[ -n "${OPENSEARCH_USERNAME}" ]] && [[ -n "${OPENSEARCH_PASSWORD}" ]]; then
+            # Apply environment variable substitution to values file
+            values_file="charts/${{ inputs.camunda-helm-dir }}/test/integration/scenarios/chart-full-setup/values-integration-test-ingress-${TEST_VALUES_SCENARIO}.yaml"
+            envsubst < "$values_file" > /tmp/values-substituted.yaml && mv /tmp/values-substituted.yaml "$values_file"
+          fi
+          
           echo "Extra values from workflow:"
           echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
           task -d ${CI_TASKS_BASE_DIR}/chart-full-setup setup.venom.env

charts/camunda-platform-8.8/templates/common/constraints.tpl

@@ -581,7 +581,7 @@ The keys moved in the 8.8 Alpha 8 version.
   - renamed: global.security.authentication => orchestration.security.authentication
   */}}
   {{ include "camundaPlatform.keyRenamed" (dict
-    "condition" (.Values.global.security.authentication)
+    "condition" (and (hasKey .Values.global "security") (hasKey .Values.global.security "authentication"))
     "oldName" "global.security.authentication"
     "newName" "orchestration.security.authentication"
   ) }}
@@ -590,7 +590,7 @@ The keys moved in the 8.8 Alpha 8 version.
   - renamed: global.security.authorizations => orchestration.security.authorizations
   */}}
   {{ include "camundaPlatform.keyRenamed" (dict
-    "condition" (.Values.global.security.authorizations)
+    "condition" (and (hasKey .Values.global "security") (hasKey .Values.global.security "authorizations"))
     "oldName" "global.security.authorizations"
     "newName" "orchestration.security.authorizations"
   ) }}
@@ -599,7 +599,7 @@ The keys moved in the 8.8 Alpha 8 version.
   - renamed: global.security.initialization => orchestration.security.initialization
   */}}
   {{ include "camundaPlatform.keyRenamed" (dict
-    "condition" (.Values.global.security.initialization)
+    "condition" (and (hasKey .Values.global "security") (hasKey .Values.global.security "initialization"))
     "oldName" "global.security.initialization"
     "newName" "orchestration.security.initialization"
   ) }}

charts/camunda-platform-8.8/test/ci-test-config.yaml

@@ -62,6 +62,11 @@ integration:
           exclude: []
           shortname: oske
           auth: keycloak
+        - name: opensearch
+          enabled: true
+          shortname: osba
+          exclude: [identity,console,orchestration-grpc]
+          auth: basic
         - name: identity-migration
           enabled: true
           exclude: []

charts/camunda-platform-8.8/test/integration/scenarios/chart-full-setup/values-integration-test-ingress-basic.yaml

@@ -1,6 +1,8 @@
 global:
   secrets:
     autoGenerated: false
+  exporter:
+    enabled: false
   ingress:
     enabled: true
     className: nginx
@@ -13,6 +15,9 @@ global:
     annotations:
       external-dns.alpha.kubernetes.io/hostname: "{{ .Values.global.ingress.host }}"
       external-dns.alpha.kubernetes.io/ttl: "60"
+
+orchestration:
+  contextPath: "/orchestration"
   security:
     authentication:
       method: basic

charts/camunda-platform-8.8/test/integration/scenarios/chart-full-setup/values-integration-test-ingress-opensearch.yaml

@@ -1,22 +1,27 @@
 global:
   elasticsearch:
     enabled: false
+  exporter:
+    enabled: true
   opensearch:
     enabled: true
+    aws:
+      enabled: false
     auth:
-      username: camunda
-      password: camunda
+      username: "$OPENSEARCH_USERNAME"
+      password: "$OPENSEARCH_PASSWORD"
     url:
-      protocol: https
-      host: camunda.com
-      port: 443
+      protocol: $OPENSEARCH_PROTOCOL
+      host: "$OPENSEARCH_HOST"
+      port: $OPENSEARCH_PORT
+    prefix: "custom-zeebe"
 
 elasticsearch:
   enabled: false
 
 orchestration:
-  # index:
-  #   prefix: app
+  index:
+    prefix: "custom-zeebe"
   env:
     # group1
     - name: CAMUNDA_DATA_SECONDARYSTORAGE_OPENSEARCH_INDEXPREFIX

charts/camunda-platform-8.8/test/integration/testsuites/tests/connectors.spec.ts

@@ -37,7 +37,11 @@ test.describe("connectors", () => {
 
   test.beforeAll(async ({ playwright }) => {
     api = await playwright.request.newContext();
-    venomJWT = await fetchToken(config.venomID, config.venomSec, api, config);
+    if (config.authType !== "basic") {
+      venomJWT = await fetchToken(config.venomID, config.venomSec, api, config);
+    } else {
+      venomJWT = "";
+    }
   });
 
   test("Connectors inbound page", async () => {

charts/camunda-platform-8.8/test/integration/testsuites/tests/core-rest.spec.ts

@@ -60,7 +60,11 @@ test.describe("orchestration-rest", () => {
 
   test.beforeAll(async ({ playwright }) => {
     api = await playwright.request.newContext();
-    venomJWT = await fetchToken(config.venomID, config.venomSec, api, config);
+    if (config.authType !== "basic") {
+      venomJWT = await fetchToken(config.venomID, config.venomSec, api, config);
+    } else {
+      venomJWT = "";
+    }
   });
 
   for (const [label, url, method, body] of [

charts/camunda-platform-8.8/test/integration/testsuites/tests/helper.ts

@@ -25,7 +25,7 @@ export async function fetchToken(
   }
 }
 
-export const authHeader = async (api: APIRequestContext, config: any) => {
+export const authHeader = async (api: APIRequestContext, config: any): Promise<string> => {
   if (config.authType === "basic") {
     return `Basic ${Buffer.from(
       `${config.demoUser}:${config.demoPass}`,

scripts/run-integration-tests.sh

@@ -43,6 +43,12 @@ setup_env_file() {
   local is_ci="$7"
 
   export TEST_INGRESS_HOST="$hostname"
+  
+  # Only export TEST_AUTH_TYPE for 8.8+ where the template uses it
+  if [[ "$test_suite_path" == *"8.8"* ]]; then
+    export TEST_AUTH_TYPE="$test_auth_type"
+  fi
+  
   envsubst <"$test_suite_path"/vars/playwright/files/playwright-job-vars.env.template >"$env_file"
 
   # during helm install, we create a secret with the credentials for the services
@@ -140,8 +146,8 @@ ABSOLUTE_CHART_PATH=""
 NAMESPACE=""
 SHOW_HTML_REPORT=false
 VERBOSE=false
-TEST_AUTH_TYPE="keycloak"
-TEST_EXCLUDE=""
+TEST_AUTH_TYPE="${TEST_AUTH_TYPE:-keycloak}"
+TEST_EXCLUDE="${TEST_EXCLUDE:-}"
 IS_CI=true
 
 check_required_cmds

test/integration/scenarios/chart-full-setup/Taskfile.yaml

@@ -152,7 +152,7 @@ tasks:
           }"
       {{ end }}
     - |
-      {{ if eq .TEST_VALUES_SCENARIO "opensearch" }}
+      {{ if and (eq .TEST_VALUES_SCENARIO "opensearch") .TEST_OPENSEARCH_PREFIX }}
         sed -i 's/custom/{{ .TEST_OPENSEARCH_PREFIX }}/g' {{ .TEST_VALUES_BASE_DIR }}/chart-full-setup/values-integration-test-ingress-{{ .TEST_VALUES_SCENARIO }}.yaml
       {{ end }}
     - |