ci(deploy-camunda): native --extra-values on matrix run with per-scenario overrides

.github/workflows/test-integration-runner.yaml

@@ -528,8 +528,7 @@ jobs:
           ENTRA_APP_OBJECT_ID: ${{ steps.setup.outputs.ENTRA_APP_OBJECT_ID }}
           ENTRA_APP_DIRECTORY_ID: ${{ steps.setup.outputs.ENTRA_APP_DIRECTORY_ID }}
         run: |
-          echo "Extra values from workflow:"
-          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
+          echo "$EXTRA_VALUES" > /tmp/extra-values-file.yaml
           deploy-camunda entra update-redirect-uris \
             --ingress-host "$TEST_INGRESS_HOST" \
             --log-level info
@@ -548,11 +547,10 @@ jobs:
         env:
           EXTRA_VALUES: ${{ inputs.extra-values }}
         run: |
-          # Persist user-supplied extra values for `deploy-camunda matrix run` to consume.
-          # The runner uses a fixed location (/tmp/extra-values-file.yaml) which is added to
-          # the helm values chain by deploy.Execute via flags.Deployment.ExtraValues.
-          echo "Extra values from workflow:"
-          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
+          # Persist user-supplied extra values for `deploy-camunda matrix run` to consume
+          # via --extra-values (flags.Deployment.ExtraValues → digest-overlay strip).
+          # Written without echoing to stdout to avoid leaking sensitive values into logs.
+          echo "$EXTRA_VALUES" > /tmp/extra-values-file.yaml
 
       - name: Helm - OCI Registry Login
         if: inputs.helmChartVersion != ''
@@ -678,7 +676,8 @@ jobs:
           # Forward workflow extra-values via --extra-values (NOT
           # --extra-helm-arg=--values=…): only this path lands in
           # flags.Deployment.ExtraValues, which engages the digest-overlay strip.
-          # Upgrade flow omits this; runner_upgrade.go also nils Step 1's copy.
+          # The upgrade flow forwards the same way; runner_upgrade.go nils Step 1's
+          # copy so the override applies to Step 2 (the upgraded chart) only.
           EXTRA_VALUES_ARGS=()
           if [[ -s /tmp/extra-values-file.yaml ]] && grep -q '[^[:space:]]' /tmp/extra-values-file.yaml; then
             EXTRA_VALUES_ARGS=(--extra-values /tmp/extra-values-file.yaml)
@@ -903,8 +902,7 @@ jobs:
           ENTRA_APP_OBJECT_ID: ${{ steps.setup.outputs.ENTRA_APP_OBJECT_ID }}
           ENTRA_APP_DIRECTORY_ID: ${{ steps.setup.outputs.ENTRA_APP_DIRECTORY_ID }}
         run: |
-          echo "Extra values from workflow:"
-          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
+          echo "$EXTRA_VALUES" > /tmp/extra-values-file.yaml
           deploy-camunda entra update-redirect-uris \
             --ingress-host "$TEST_INGRESS_HOST" \
             --log-level info
@@ -954,6 +952,7 @@ jobs:
           E2E_TESTS_LICENSE_KEY: ${{ secrets.E2E_TESTS_LICENSE_KEY || env.E2E_TESTS_LICENSE_KEY }}
           CAMUNDA_HELM_DIR: ${{ inputs.camunda-helm-dir }}
           INFRA_TYPE: ${{ inputs.infra-type }}
+          EXTRA_VALUES: ${{ inputs.extra-values }}
           # Selection + composition model environment variables.
           TEST_IDENTITY: ${{ inputs.test-identity }}
           TEST_PERSISTENCE: ${{ inputs.test-persistence }}
@@ -1037,14 +1036,17 @@ jobs:
             fi
           fi
 
-          # NOTE: extra-values (e.g. the run-built image tag) are intentionally NOT
-          # forwarded for upgrade flows. `matrix run` runs upgrade as a two-step
-          # orchestration — Step 1 installs the previous released version, Step 2
-          # upgrades to the current chart — and --extra-helm-arg fans out to BOTH
-          # steps, which would inject the new image tag into the previous-version
-          # install and break the upgrade-from-stable baseline. Step-2-only / per-
-          # scenario injection is tracked in #6312. (The install flow, being single-
-          # step, forwards the file safely; see that step.)
+          # Persist user-supplied extra values, then forward via --extra-values.
+          # `matrix run` runs upgrade as two steps and applies --extra-values to
+          # Step 2 only (runner_upgrade.go nils Step 1's copy), so the run-built
+          # image tag lands on the upgraded chart without contaminating the
+          # previous-version install. Written without echoing to stdout to avoid
+          # leaking potentially sensitive values into workflow logs.
+          echo "$EXTRA_VALUES" > /tmp/extra-values-file.yaml
+          EXTRA_VALUES_ARGS=()
+          if [[ -s /tmp/extra-values-file.yaml ]] && grep -q '[^[:space:]]' /tmp/extra-values-file.yaml; then
+            EXTRA_VALUES_ARGS=(--extra-values /tmp/extra-values-file.yaml)
+          fi
 
           deploy-camunda matrix run \
             --repo-root "${GITHUB_WORKSPACE}" \
@@ -1063,6 +1065,7 @@ jobs:
             ${ENV_FILE_ARGS[@]+"${ENV_FILE_ARGS[@]}"} \
             ${CHART_REF_ARGS[@]+"${CHART_REF_ARGS[@]}"} \
             ${EXTRA_HELM_ARGS[@]+"${EXTRA_HELM_ARGS[@]}"} \
+            ${EXTRA_VALUES_ARGS[@]+"${EXTRA_VALUES_ARGS[@]}"} \
             --log-level info
 
       - name: Display Docker Image Git Commits (Upgrade)

docs/adr/0093-adopt-composable-ci-scenario-registry-for-per-version-test-matrix.md

@@ -89,7 +89,7 @@ green for one nightly cycle on `main`. Normative constraints:
    ```text
    charts/camunda-platform-<X.Y>/test/ci/registry/
      manifest.yaml             # ordered list of scenario IDs with numeric tier and enabled
-     scenarios/<id>.yaml       # one scenario per file; inlines all `CIScenario` fields (identity, persistence, platforms, infra-type, features, shortname, prefix-key, helmVersion, skip-e2e, skip-it, qa, image-tags, upgrade, enterprise) except `flow`, `pre-install`, `post-deploy`, and `dependencies`
+     scenarios/<id>.yaml       # one scenario per file; inlines all `CIScenario` fields (identity, persistence, platforms, infra-type, features, extra-values, shortname, prefix-key, helmVersion, skip-e2e, skip-it, qa, image-tags, upgrade, enterprise) except `flow`, `pre-install`, `post-deploy`, and `dependencies`
      hooks/<id>.yaml           # named pre-install / post-deploy LifecycleHook blocks
      dependencies/<id>.yaml    # named ChartDependency entries (chart, version, repo, values-file)
    ```
@@ -122,6 +122,10 @@ green for one nightly cycle on `main`. Normative constraints:
    - Referenced basenames (hook manifests, hook scripts, dependency
      values-files, scenario values-layer files, feature values-files)
      resolve under `test/integration/scenarios/`.
+   - `extra-values` relative paths resolve under the scenario's
+     `chart-full-setup/` dir and MUST NOT escape it via `..` traversal;
+     absolute paths are runtime-supplied (e.g. per-PR image-tag overlay)
+     and are intentionally not validated at load time.
    - Platform × flow combinations are not denied by
      `.github/config/permitted-flows.yaml`.
 

scripts/deploy-camunda/cmd/matrix.go

@@ -606,13 +606,13 @@ This command calls deploy.Execute() for each matrix entry.`,
 	f.BoolVar(&ensureDockerHub, "ensure-docker-hub", false, "Ensure Docker Hub registry pull secret is created in each entry's namespace")
 	f.BoolVar(&useLatest, "use-latest", false, "Use values-latest.yaml from each chart root instead of values-digest.yaml")
 	f.BoolVar(&useQA, "use-qa", false, "Force the base-qa layer to be included for all entries, regardless of per-scenario qa config")
-	f.BoolVar(&forceImageOverrides, "force-image-overrides", false, "Bypass OCI immutability guard: allow Go-layer image overlays (base-image-tags, chart-root overlays) even when --chart-ref is set. Note: env-file IMAGE_TAG keys stripped at the workflow layer are not restored by this flag.")
+	f.BoolVar(&forceImageOverrides, "force-image-overrides", false, "Bypass OCI immutability guard: allow chart-root image overlays when --chart-ref is set (env-file IMAGE_TAG keys stripped at the workflow layer are not restored).")
 	f.BoolVarP(&yes, "yes", "y", false, "Skip confirmation prompts (e.g., e2e threshold warning)")
 	f.StringVar(&logDir, "log-dir", "", "Write logs to this directory and show a live status table (auto-generated when running in a TTY)")
 	f.StringArrayVar(&extraHelmArgs, "extra-helm-arg", nil, "Extra argument appended to every helm command (repeatable, e.g. --extra-helm-arg=--set-file=global.license.secret.inlineSecret=/tmp/license.txt)")
 	f.StringSliceVar(&extraHelmSets, "extra-helm-set", nil, "Extra helm --set key=value pair applied to every entry (comma-separated or repeatable, e.g. orchestration.upgrade.allowPreReleaseImages=true)")
-	f.StringArrayVar(&extraValues, "extra-values", nil, "Additional Helm values files appended last for every entry (repeatable; NOT comma-split, unlike `deploy --extra-values` — use the flag multiple times for multiple files). Use this, not --extra-helm-arg=--values=…, for image overrides; only this path triggers the digest-overlay strip. In two-step upgrade flows the file is applied to Step 2 only — Step 1 installs the previously released chart and intentionally ignores --extra-values.")
-	f.StringVar(&namespaceOverride, "namespace-override", "", "Override the computed namespace for every entry. Use only with filters that narrow the run to a single entry (typically called from per-scenario CI workflows that pre-create the namespace).")
+	f.StringArrayVar(&extraValues, "extra-values", nil, "Additional Helm values files appended last for every entry (repeatable; not comma-split — use the flag multiple times for multiple files). Engages digest-overlay strip; prefer over --extra-helm-arg=--values=. In two-step upgrade flows, applied to Step 2 only.")
+	f.StringVar(&namespaceOverride, "namespace-override", "", "Override the computed namespace for every entry (use with filters that narrow to a single entry — per-scenario CI workflows that pre-create the namespace).")
 	f.StringVar(&chartRef, "chart-ref", "", "Override chart source with an OCI reference or .tgz path (e.g., oci://registry.camunda.cloud/team-distribution/camunda-platform). Values are still resolved from the local repo via --repo-root.")
 	f.StringVar(&chartRefVersion, "chart-version", "", "Chart version to install from --chart-ref (e.g., 13-rc-latest). Only meaningful when --chart-ref is set.")
 	f.IntVar(&tier, "tier", 0, "Filter entries by tier (1=PR CI, 2=merge-queue only; 0=all)")

scripts/deploy-camunda/matrix/config.go

@@ -172,6 +172,12 @@ type CIScenario struct {
 	Persistence string   `yaml:"persistence,omitempty"`
 	Features    []string `yaml:"features,omitempty"`
 
+	// ExtraValues lists scenario-specific values files (paths relative to the
+	// scenario's chart-full-setup dir) appended to the helm values chain after
+	// any global --extra-values. Lets a scenario specialize without losing the
+	// global override.
+	ExtraValues []string `yaml:"extra-values,omitempty"`
+
 	// Base modifier flags.
 	QA         bool `yaml:"qa,omitempty"`
 	ImageTags  bool `yaml:"image-tags,omitempty"`

scripts/deploy-camunda/matrix/matrix.go

@@ -30,6 +30,12 @@ type Entry struct {
 	Persistence string   `json:"persistence,omitempty"`
 	Features    []string `json:"features,omitempty"`
 
+	// ExtraValues are scenario-declared values files (paths relative to the
+	// scenario's chart-full-setup dir) appended to the helm values chain AFTER
+	// any global --extra-values, so a scenario specializes without losing the
+	// global override (precedence: defaults < global --extra-values < per-scenario).
+	ExtraValues []string `json:"extraValues,omitempty"`
+
 	// Base modifier flags.
 	QA         bool `json:"qa,omitempty"`
 	ImageTags  bool `json:"imageTags,omitempty"`
@@ -211,6 +217,7 @@ func Generate(repoRoot string, opts GenerateOptions) ([]Entry, error) {
 						Identity:     scenario.Identity,
 						Persistence:  scenario.Persistence,
 						Features:     scenario.Features,
+						ExtraValues:  scenario.ExtraValues,
 						QA:           scenario.QA,
 						ImageTags:    scenario.ImageTags,
 						Upgrade:      scenario.Upgrade,

scripts/deploy-camunda/matrix/matrix_test.go

@@ -2077,11 +2077,15 @@ func TestExtraValues_PropagatesToDeploymentFlags(t *testing.T) {
 //
 // Mirrors TestChartRefOverride_UpgradeStep1Unaffected above.
 func TestExtraValues_UpgradeStep1Cleared(t *testing.T) {
-	// Base flags as executeEntry would build for a single-step install or
-	// Step 2 of an upgrade — ExtraValues populated from the caller.
+	// Base flags as executeEntry would build them after appendScenarioExtraValues
+	// runs — a merged slice of global + per-scenario paths. This mirrors the
+	// production state that arrives at executeTwoStepUpgrade.
 	baseFlags := &config.RuntimeFlags{
 		Deployment: config.DeploymentFlags{
-			ExtraValues: []string{"/tmp/engine-image.yaml"},
+			ExtraValues: []string{
+				"/tmp/engine-image.yaml", // global --extra-values (e.g. per-PR image tag)
+				"/repo/charts/camunda-platform-8.10/test/integration/scenarios/chart-full-setup/values/extra/tuning.yaml", // per-scenario, pre-resolved by appendScenarioExtraValues
+			},
 		},
 	}
 
@@ -2091,18 +2095,56 @@ func TestExtraValues_UpgradeStep1Cleared(t *testing.T) {
 	step1Flags := *baseFlags
 	step1Flags.Deployment.ExtraValues = nil
 
+	// Both global and per-scenario paths must be absent from Step 1.
 	if step1Flags.Deployment.ExtraValues != nil {
 		t.Errorf("Step 1 ExtraValues = %v, want nil (must not inherit caller overrides)", step1Flags.Deployment.ExtraValues)
 	}
-	// Step 2 inherits from baseFlags and must keep the override.
+	// Step 2 inherits from baseFlags and must keep both paths.
 	step2Flags := *baseFlags
-	if len(step2Flags.Deployment.ExtraValues) != 1 || step2Flags.Deployment.ExtraValues[0] != "/tmp/engine-image.yaml" {
-		t.Errorf("Step 2 ExtraValues = %v, want [/tmp/engine-image.yaml]", step2Flags.Deployment.ExtraValues)
+	if len(step2Flags.Deployment.ExtraValues) != 2 {
+		t.Errorf("Step 2 ExtraValues = %v, want both global and per-scenario paths", step2Flags.Deployment.ExtraValues)
 	}
 	// Nil-out on the shallow copy must not affect the parent (regression
 	// guard for the slice-header sharing trap that runner_upgrade.go's
 	// hook-slice detach comment already calls out).
-	if len(baseFlags.Deployment.ExtraValues) != 1 {
+	if len(baseFlags.Deployment.ExtraValues) != 2 {
 		t.Errorf("baseFlags.ExtraValues mutated by Step 1 nil-out (got %v)", baseFlags.Deployment.ExtraValues)
 	}
 }
+
+// TestAppendScenarioExtraValues pins the #6312 per-scenario precedence: a
+// scenario's declared extra-values are appended AFTER any global --extra-values
+// (so the per-scenario file wins within the chain's `extra` slot), and relative
+// paths resolve against the scenario dir while absolute paths pass through.
+func TestAppendScenarioExtraValues(t *testing.T) {
+	scenarioDir := "/repo/charts/camunda-platform-8.10/test/integration/scenarios/chart-full-setup"
+	global := []string{"/tmp/global-image.yaml"}
+	entry := Entry{ExtraValues: []string{"values/extra/image.yaml", "/abs/override.yaml"}}
+
+	got := appendScenarioExtraValues(append([]string(nil), global...), entry, scenarioDir)
+
+	want := []string{
+		"/tmp/global-image.yaml",
+		filepath.Join(scenarioDir, "values/extra/image.yaml"),
+		"/abs/override.yaml",
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d files %v, want %d %v", len(got), got, len(want), want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("file[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+
+	// Global override must come before per-scenario so the per-scenario file
+	// is last-wins in helm's -f merge within the extra slot.
+	if got[0] != global[0] {
+		t.Errorf("global --extra-values must come first, got %q", got[0])
+	}
+
+	// Nil per-scenario list leaves the global slice untouched.
+	if out := appendScenarioExtraValues([]string{"/tmp/g.yaml"}, Entry{}, scenarioDir); len(out) != 1 || out[0] != "/tmp/g.yaml" {
+		t.Errorf("empty ExtraValues should pass global through unchanged, got %v", out)
+	}
+}

scripts/deploy-camunda/matrix/registry.go

@@ -56,6 +56,7 @@ type registryScenario struct {
 	Identity    string            `yaml:"identity,omitempty"`
 	Persistence string            `yaml:"persistence,omitempty"`
 	Features    []string          `yaml:"features,omitempty"`
+	ExtraValues []string          `yaml:"extra-values,omitempty"`
 	QA          bool              `yaml:"qa,omitempty"`
 	ImageTags   bool              `yaml:"image-tags,omitempty"`
 	Upgrade     bool              `yaml:"upgrade,omitempty"`
@@ -210,6 +211,7 @@ func LoadRegistry(chartDir string) (*CITestConfig, error) {
 				Identity:     rscn.Identity,
 				Persistence:  rscn.Persistence,
 				Features:     rscn.Features,
+				ExtraValues:  rscn.ExtraValues,
 				QA:           rscn.QA,
 				ImageTags:    rscn.ImageTags,
 				Upgrade:      rscn.Upgrade,