ci(deploy-camunda): native --extra-values on matrix run with per-scenario overrides

Closes #6312.

Two gaps remained after #6375 shipped the CLI flag:

Gap B — upgrade flow never forwarded the file. The CLI already applied
--extra-values to Step 2 only (runner_upgrade.go nils Step 1's copy), but
the workflow job omitted EXTRA_VALUES entirely. Wire it: add the env var,
tee to the fixed path, guard with EXTRA_VALUES_ARGS, inject into the
matrix run invocation — mirroring the install-flow pattern.

Gap A — per-scenario extra-values. Scenarios can now declare their own
values files under extra-values in ci-test-config.yaml / the registry
format. appendScenarioExtraValues resolves relative paths against
chart-full-setup and appends them after the global --extra-values, so
per-scenario files win within the extra slot. Precedence:
  chart defaults < global --extra-values < per-scenario extra-values < scenario layers

All three propagation hops are tested:
  - TestLoadRegistryCarriesExtraValues: YAML → CIScenario
  - TestGenerate_PropagatesExtraValues: CIScenario → Entry
  - TestAppendScenarioExtraValues: precedence + path resolution
  - TestRegistryValidatorRejectsMissingExtraValues: validator parity

Also debloated three flag descriptions in cmd/matrix.go.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: Ian-wang-liyang

.github/workflows/test-integration-runner.yaml

@@ -669,7 +669,8 @@ jobs:
           # Forward workflow extra-values via --extra-values (NOT
           # --extra-helm-arg=--values=…): only this path lands in
           # flags.Deployment.ExtraValues, which engages the digest-overlay strip.
-          # Upgrade flow omits this; runner_upgrade.go also nils Step 1's copy.
+          # The upgrade flow forwards the same way; runner_upgrade.go nils Step 1's
+          # copy so the override applies to Step 2 (the upgraded chart) only.
           EXTRA_VALUES_ARGS=()
           if [[ -s /tmp/extra-values-file.yaml ]] && grep -q '[^[:space:]]' /tmp/extra-values-file.yaml; then
             EXTRA_VALUES_ARGS=(--extra-values /tmp/extra-values-file.yaml)
@@ -937,6 +938,7 @@ jobs:
           E2E_TESTS_LICENSE_KEY: ${{ secrets.E2E_TESTS_LICENSE_KEY || env.E2E_TESTS_LICENSE_KEY }}
           CAMUNDA_HELM_DIR: ${{ inputs.camunda-helm-dir }}
           INFRA_TYPE: ${{ inputs.infra-type }}
+          EXTRA_VALUES: ${{ inputs.extra-values }}
           # Selection + composition model environment variables.
           TEST_IDENTITY: ${{ inputs.test-identity }}
           TEST_PERSISTENCE: ${{ inputs.test-persistence }}
@@ -1020,14 +1022,18 @@ jobs:
             fi
           fi
 
-          # NOTE: extra-values (e.g. the run-built image tag) are intentionally NOT
-          # forwarded for upgrade flows. `matrix run` runs upgrade as a two-step
-          # orchestration — Step 1 installs the previous released version, Step 2
-          # upgrades to the current chart — and --extra-helm-arg fans out to BOTH
-          # steps, which would inject the new image tag into the previous-version
-          # install and break the upgrade-from-stable baseline. Step-2-only / per-
-          # scenario injection is tracked in #6312. (The install flow, being single-
-          # step, forwards the file safely; see that step.)
+          # Persist user-supplied extra values, then forward via --extra-values.
+          # `matrix run` runs upgrade as two steps and applies --extra-values to
+          # Step 2 only (runner_upgrade.go nils Step 1's copy), so the run-built
+          # image tag lands on the upgraded chart without contaminating the
+          # previous-version install. The tee runs here unconditionally so the file
+          # exists regardless of auth type (the OIDC configure step also tees it).
+          echo "Extra values from workflow:"
+          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
+          EXTRA_VALUES_ARGS=()
+          if [[ -s /tmp/extra-values-file.yaml ]] && grep -q '[^[:space:]]' /tmp/extra-values-file.yaml; then
+            EXTRA_VALUES_ARGS=(--extra-values /tmp/extra-values-file.yaml)
+          fi
 
           cd ./scripts/deploy-camunda
           go run . matrix run \
@@ -1047,6 +1053,7 @@ jobs:
             ${ENV_FILE_ARGS[@]+"${ENV_FILE_ARGS[@]}"} \
             ${CHART_REF_ARGS[@]+"${CHART_REF_ARGS[@]}"} \
             ${EXTRA_HELM_ARGS[@]+"${EXTRA_HELM_ARGS[@]}"} \
+            ${EXTRA_VALUES_ARGS[@]+"${EXTRA_VALUES_ARGS[@]}"} \
             --log-level info
 
       - name: Display Docker Image Git Commits (Upgrade)

scripts/deploy-camunda/cmd/matrix.go

@@ -610,13 +610,13 @@ This command calls deploy.Execute() for each matrix entry.`,
 	f.BoolVar(&ensureDockerHub, "ensure-docker-hub", false, "Ensure Docker Hub registry pull secret is created in each entry's namespace")
 	f.BoolVar(&useLatest, "use-latest", false, "Use values-latest.yaml from each chart root instead of values-digest.yaml")
 	f.BoolVar(&useQA, "use-qa", false, "Force the base-qa layer to be included for all entries, regardless of per-scenario qa config")
-	f.BoolVar(&forceImageOverrides, "force-image-overrides", false, "Bypass OCI immutability guard: allow Go-layer image overlays (base-image-tags, chart-root overlays) even when --chart-ref is set. Note: env-file IMAGE_TAG keys stripped at the workflow layer are not restored by this flag.")
+	f.BoolVar(&forceImageOverrides, "force-image-overrides", false, "Bypass OCI immutability guard: allow chart-root image overlays when --chart-ref is set (env-file IMAGE_TAG keys stripped at the workflow layer are not restored).")
 	f.BoolVarP(&yes, "yes", "y", false, "Skip confirmation prompts (e.g., e2e threshold warning)")
 	f.StringVar(&logDir, "log-dir", "", "Write logs to this directory and show a live status table (auto-generated when running in a TTY)")
 	f.StringArrayVar(&extraHelmArgs, "extra-helm-arg", nil, "Extra argument appended to every helm command (repeatable, e.g. --extra-helm-arg=--set-file=global.license.secret.inlineSecret=/tmp/license.txt)")
 	f.StringSliceVar(&extraHelmSets, "extra-helm-set", nil, "Extra helm --set key=value pair applied to every entry (comma-separated or repeatable, e.g. orchestration.upgrade.allowPreReleaseImages=true)")
-	f.StringArrayVar(&extraValues, "extra-values", nil, "Additional Helm values files appended last for every entry (repeatable; NOT comma-split, unlike `deploy --extra-values` — use the flag multiple times for multiple files). Use this, not --extra-helm-arg=--values=…, for image overrides; only this path triggers the digest-overlay strip. In two-step upgrade flows the file is applied to Step 2 only — Step 1 installs the previously released chart and intentionally ignores --extra-values.")
-	f.StringVar(&namespaceOverride, "namespace-override", "", "Override the computed namespace for every entry. Use only with filters that narrow the run to a single entry (typically called from per-scenario CI workflows that pre-create the namespace).")
+	f.StringArrayVar(&extraValues, "extra-values", nil, "Additional Helm values files appended last for every entry (repeatable; not comma-split — use the flag multiple times for multiple files). Engages digest-overlay strip; prefer over --extra-helm-arg=--values=. In two-step upgrade flows, applied to Step 2 only.")
+	f.StringVar(&namespaceOverride, "namespace-override", "", "Override the computed namespace for every entry (use with filters that narrow to a single entry — per-scenario CI workflows that pre-create the namespace).")
 	f.StringVar(&chartRef, "chart-ref", "", "Override chart source with an OCI reference or .tgz path (e.g., oci://registry.camunda.cloud/team-distribution/camunda-platform). Values are still resolved from the local repo via --repo-root.")
 	f.StringVar(&chartRefVersion, "chart-version", "", "Chart version to install from --chart-ref (e.g., 13-rc-latest). Only meaningful when --chart-ref is set.")
 	f.IntVar(&tier, "tier", 0, "Filter entries by tier (1=PR CI, 2=merge-queue only; 0=all)")

scripts/deploy-camunda/matrix/config.go

@@ -172,6 +172,12 @@ type CIScenario struct {
 	Persistence string   `yaml:"persistence,omitempty"`
 	Features    []string `yaml:"features,omitempty"`
 
+	// ExtraValues lists scenario-specific values files (paths relative to the
+	// scenario's chart-full-setup dir) appended to the helm values chain after
+	// any global --extra-values. Lets a scenario specialize without losing the
+	// global override.
+	ExtraValues []string `yaml:"extra-values,omitempty"`
+
 	// Base modifier flags.
 	QA         bool `yaml:"qa,omitempty"`
 	ImageTags  bool `yaml:"image-tags,omitempty"`

scripts/deploy-camunda/matrix/matrix.go

@@ -30,6 +30,12 @@ type Entry struct {
 	Persistence string   `json:"persistence,omitempty"`
 	Features    []string `json:"features,omitempty"`
 
+	// ExtraValues are scenario-declared values files (paths relative to the
+	// scenario's chart-full-setup dir) appended to the helm values chain AFTER
+	// any global --extra-values, so a scenario specializes without losing the
+	// global override (precedence: defaults < global --extra-values < per-scenario).
+	ExtraValues []string `json:"extraValues,omitempty"`
+
 	// Base modifier flags.
 	QA         bool `json:"qa,omitempty"`
 	ImageTags  bool `json:"imageTags,omitempty"`
@@ -212,6 +218,7 @@ func Generate(repoRoot string, opts GenerateOptions) ([]Entry, error) {
 						Identity:     scenario.Identity,
 						Persistence:  scenario.Persistence,
 						Features:     scenario.Features,
+						ExtraValues:  scenario.ExtraValues,
 						QA:           scenario.QA,
 						ImageTags:    scenario.ImageTags,
 						Upgrade:      scenario.Upgrade,

scripts/deploy-camunda/matrix/matrix_test.go

@@ -2106,3 +2106,40 @@ func TestExtraValues_UpgradeStep1Cleared(t *testing.T) {
 		t.Errorf("baseFlags.ExtraValues mutated by Step 1 nil-out (got %v)", baseFlags.Deployment.ExtraValues)
 	}
 }
+
+// TestAppendScenarioExtraValues pins the #6312 per-scenario precedence: a
+// scenario's declared extra-values are appended AFTER any global --extra-values
+// (so the per-scenario file wins within the chain's `extra` slot), and relative
+// paths resolve against the scenario dir while absolute paths pass through.
+func TestAppendScenarioExtraValues(t *testing.T) {
+	scenarioDir := "/repo/charts/camunda-platform-8.10/test/integration/scenarios/chart-full-setup"
+	global := []string{"/tmp/global-image.yaml"}
+	entry := Entry{ExtraValues: []string{"values/extra/image.yaml", "/abs/override.yaml"}}
+
+	got := appendScenarioExtraValues(append([]string(nil), global...), entry, scenarioDir)
+
+	want := []string{
+		"/tmp/global-image.yaml",
+		filepath.Join(scenarioDir, "values/extra/image.yaml"),
+		"/abs/override.yaml",
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d files %v, want %d %v", len(got), got, len(want), want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("file[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+
+	// Global override must come before per-scenario so the per-scenario file
+	// is last-wins in helm's -f merge within the extra slot.
+	if got[0] != global[0] {
+		t.Errorf("global --extra-values must come first, got %q", got[0])
+	}
+
+	// Nil per-scenario list leaves the global slice untouched.
+	if out := appendScenarioExtraValues([]string{"/tmp/g.yaml"}, Entry{}, scenarioDir); len(out) != 1 || out[0] != "/tmp/g.yaml" {
+		t.Errorf("empty ExtraValues should pass global through unchanged, got %v", out)
+	}
+}

scripts/deploy-camunda/matrix/registry.go

@@ -56,6 +56,7 @@ type registryScenario struct {
 	Identity    string            `yaml:"identity,omitempty"`
 	Persistence string            `yaml:"persistence,omitempty"`
 	Features    []string          `yaml:"features,omitempty"`
+	ExtraValues []string          `yaml:"extra-values,omitempty"`
 	QA          bool              `yaml:"qa,omitempty"`
 	ImageTags   bool              `yaml:"image-tags,omitempty"`
 	Upgrade     bool              `yaml:"upgrade,omitempty"`
@@ -211,6 +212,7 @@ func LoadRegistry(chartDir string) (*CITestConfig, error) {
 				Identity:     rscn.Identity,
 				Persistence:  rscn.Persistence,
 				Features:     rscn.Features,
+				ExtraValues:  rscn.ExtraValues,
 				QA:           rscn.QA,
 				ImageTags:    rscn.ImageTags,
 				Upgrade:      rscn.Upgrade,

scripts/deploy-camunda/matrix/registry_test.go

@@ -288,6 +288,71 @@ func TestLoadRegistryRejectsPathTraversalManifestID(t *testing.T) {
 	}
 }
 
+// TestLoadRegistryCarriesExtraValues pins the #6312 loader plumbing: a
+// scenario's `extra-values` list must flow from scenarios/<id>.yaml into the
+// assembled CIScenario so it can be threaded into the deploy values chain.
+func TestLoadRegistryCarriesExtraValues(t *testing.T) {
+	_, chartDir, regDir := syntheticChart(t)
+	writeManifest(t, regDir, "    - id: alpha\n      shortname: alph\n      enabled: true\n")
+	writeFile(t, filepath.Join(regDir, "scenarios", "alpha.yaml"),
+		"name: alpha\nauth: keycloak\nflows: [install]\nidentity: keycloak\npersistence: elasticsearch\nplatforms: [gke]\nextra-values:\n  - values/extra/image.yaml\n  - values/extra/tuning.yaml\n")
+	// The validator (run inside LoadRegistry) resolves relative extra-values
+	// against chart-full-setup, so the referenced files must exist.
+	extraDir := filepath.Join(chartDir, "test", "integration", "scenarios", "chart-full-setup", "values", "extra")
+	if err := os.MkdirAll(extraDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeFile(t, filepath.Join(extraDir, "image.yaml"), "{}\n")
+	writeFile(t, filepath.Join(extraDir, "tuning.yaml"), "{}\n")
+
+	cfg, err := LoadRegistry(chartDir)
+	if err != nil {
+		t.Fatalf("LoadRegistry: %v", err)
+	}
+	scns := cfg.Integration.Case.PR.Scenarios
+	if len(scns) != 1 {
+		t.Fatalf("want 1 scenario, got %d", len(scns))
+	}
+	got := scns[0].ExtraValues
+	want := []string{"values/extra/image.yaml", "values/extra/tuning.yaml"}
+	if len(got) != len(want) {
+		t.Fatalf("ExtraValues = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("ExtraValues[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+// TestGenerate_PropagatesExtraValues pins hop 2 of the #6312 chain: the
+// CIScenario→Entry copy in Generate. A regression deleting that copy would
+// leave the loader test green while silently dropping the field before deploy.
+func TestGenerate_PropagatesExtraValues(t *testing.T) {
+	dir, chartDir, regDir := syntheticChart(t)
+	writeFile(t, filepath.Join(dir, "charts", "chart-versions.yaml"),
+		"camundaVersions:\n  supportStandard:\n    - \"99.99\"\n")
+	writeManifest(t, regDir, "    - id: alpha\n      shortname: alph\n      enabled: true\n")
+	writeFile(t, filepath.Join(regDir, "scenarios", "alpha.yaml"),
+		"name: alpha\nauth: keycloak\nflows: [install]\nidentity: keycloak\npersistence: elasticsearch\nplatforms: [gke]\nextra-values:\n  - values/extra/image.yaml\n")
+	extraDir := filepath.Join(chartDir, "test", "integration", "scenarios", "chart-full-setup", "values", "extra")
+	if err := os.MkdirAll(extraDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeFile(t, filepath.Join(extraDir, "image.yaml"), "{}\n")
+
+	entries, err := Generate(dir, GenerateOptions{Versions: []string{"99.99"}})
+	if err != nil {
+		t.Fatalf("Generate: %v", err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("want 1 entry, got %d: %+v", len(entries), entries)
+	}
+	if !reflect.DeepEqual(entries[0].ExtraValues, []string{"values/extra/image.yaml"}) {
+		t.Errorf("Entry.ExtraValues = %v, want [values/extra/image.yaml]", entries[0].ExtraValues)
+	}
+}
+
 // TestLoadRegistryRejectsMalformedManifest surfaces YAML parse failures at
 // load time with the manifest path in the error so authors can locate the
 // broken file.
@@ -350,6 +415,29 @@ func TestRegistryValidatorRejectsMissingFeatureValues(t *testing.T) {
 	}
 }
 
+// TestRegistryValidatorRejectsMissingExtraValues exercises checkExtraValues:
+// a scenario's relative extra-values path must resolve under chart-full-setup;
+// a dangling reference is caught at validation, not at deploy time. Absolute
+// paths are runtime-supplied and intentionally skipped.
+func TestRegistryValidatorRejectsMissingExtraValues(t *testing.T) {
+	abs := absChartDir(t)
+	cfg, err := LoadRegistry(abs)
+	if err != nil {
+		t.Fatalf("LoadRegistry: %v", err)
+	}
+	cfg.Integration.Case.PR.Scenarios[0].ExtraValues = []string{"values/extra/nope.yaml"}
+	err = (&RegistryValidator{ChartDir: abs}).Validate(cfg)
+	if err == nil || !strings.Contains(err.Error(), "nope.yaml") {
+		t.Fatalf("want missing-extra-values error, got: %v", err)
+	}
+
+	// An absolute path is not validated (runtime-supplied).
+	cfg.Integration.Case.PR.Scenarios[0].ExtraValues = []string{"/tmp/runtime-supplied.yaml"}
+	if err := (&RegistryValidator{ChartDir: abs}).Validate(cfg); err != nil {
+		t.Fatalf("absolute extra-values must skip validation, got: %v", err)
+	}
+}
+
 // TestRegistryValidatorRejectsMissingDepValuesFile exercises checkDep.
 // values-file paths are repo-root-relative (matching runner.go:1742); a
 // dangling reference must be caught at validation, not at deploy time.

scripts/deploy-camunda/matrix/registry_validator.go

@@ -53,7 +53,8 @@ func (v *RegistryValidator) Validate(cfg *CITestConfig) error {
 	scenariosDir := filepath.Join(v.ChartDir, "test", "integration", "scenarios")
 	resourcesDir := filepath.Join(scenariosDir, "common", "resources")
 	scriptsDir := filepath.Join(scenariosDir, "pre-setup-scripts")
-	featuresDir := filepath.Join(scenariosDir, "chart-full-setup", "values", "features")
+	chartFullSetupDir := filepath.Join(scenariosDir, "chart-full-setup")
+	featuresDir := filepath.Join(chartFullSetupDir, "values", "features")
 
 	// Referenced-set tracking — feeds the orphan walks below. Populated as a
 	// side effect of checkHook so every hook iteration path (PR/Nightly/
@@ -93,6 +94,18 @@ func (v *RegistryValidator) Validate(cfg *CITestConfig) error {
 			problems = append(problems, fmt.Sprintf("%s: feature %q: missing values file at %s", ctx, feature, fPath))
 		}
 	}
+	// Per-scenario extra-values resolution: relative paths resolve against the
+	// scenario's chart-full-setup dir (matching appendScenarioExtraValues in
+	// runner_execute.go); absolute paths are runtime-supplied and not validated.
+	checkExtraValues := func(ctx, ev string) {
+		if filepath.IsAbs(ev) {
+			return
+		}
+		path := filepath.Join(chartFullSetupDir, ev)
+		if info, err := os.Stat(path); err != nil || info.IsDir() {
+			problems = append(problems, fmt.Sprintf("%s: extra-values %q: missing values file at %s", ctx, ev, path))
+		}
+	}
 	checkDep := func(ctx string, dep ChartDependency) {
 		if dep.ValuesFile == "" {
 			return
@@ -141,6 +154,9 @@ func (v *RegistryValidator) Validate(cfg *CITestConfig) error {
 		for _, feat := range scn.Features {
 			checkFeature(label, feat)
 		}
+		for _, ev := range scn.ExtraValues {
+			checkExtraValues(label, ev)
+		}
 		for _, dep := range scn.Dependencies {
 			checkDep(label, dep)
 		}
@@ -179,6 +195,9 @@ func (v *RegistryValidator) Validate(cfg *CITestConfig) error {
 		checkHook(label+" pre-install", scn.PreInstall)
 		checkHook(label+" post-infra", scn.PostInfra)
 		checkHook(label+" post-deploy", scn.PostDeploy)
+		for _, ev := range scn.ExtraValues {
+			checkExtraValues(label, ev)
+		}
 	}
 
 	// Dependency-profile pre-install hooks: validate even profiles that no

scripts/deploy-camunda/matrix/runner_execute.go

@@ -53,6 +53,20 @@ func companionChartsForEntry(entry Entry, repoRoot string) []config.CompanionCha
 	return charts
 }
 
+// appendScenarioExtraValues resolves a scenario's declared extra-values files
+// against scenarioDir (absolute paths pass through) and appends them to base.
+// Resolving to absolute paths here sidesteps helm's CWD-relative -f resolution.
+func appendScenarioExtraValues(base []string, entry Entry, scenarioDir string) []string {
+	for _, p := range entry.ExtraValues {
+		if filepath.IsAbs(p) {
+			base = append(base, p)
+			continue
+		}
+		base = append(base, filepath.Join(scenarioDir, p))
+	}
+	return base
+}
+
 // executeEntry deploys a single matrix entry by constructing RuntimeFlags and calling deploy.Execute().
 // The flow determines the execution strategy:
 //   - Two-step upgrade (upgrade-patch, upgrade-minor): Step 1 installs old version, Step 2 upgrades.
@@ -157,7 +171,10 @@ func executeEntry(ctx context.Context, entry Entry, opts RunOptions) RunResult {
 			Timeout:              opts.HelmTimeout,
 			DeleteNamespaceFirst: opts.DeleteNamespaceFirst,
 			ExtraHelmArgs:        append([]string(nil), opts.ExtraHelmArgs...),
-			ExtraValues:          append([]string(nil), opts.ExtraValues...),
+			// Global --extra-values first, then scenario-declared extra-values
+			// (resolved against the scenario dir) so the per-scenario files win
+			// within the chain's `extra` slot.
+			ExtraValues: appendScenarioExtraValues(append([]string(nil), opts.ExtraValues...), entry, scenarioDir),
 			// Always include allowPreReleaseImages=true for CI deployments —
 			// matches the legacy Taskfile install/upgrade behaviour. User-supplied
 			// --extra-helm-set values are merged on top and take precedence.

scripts/deploy-camunda/matrix/runner_upgrade.go

@@ -180,8 +180,9 @@ func executeTwoStepUpgrade(ctx context.Context, entry Entry, flags *config.Runti
 	step1Flags.Selection.UpgradeFlow = false     // Step 1 is a fresh install, no base-upgrade.yaml.
 	step1Flags.Chart.ChartRootOverlays = nil     // Step 1 installs old version from repo — no chart-root overlays.
 	step1Flags.Chart.SkipDependencyUpdate = true // Repo charts don't need local dep update.
-	// Step 1 installs the previously released chart; caller --extra-values
-	// (e.g. per-PR image tag) belongs to Step 2 only.
+	// Step 1 installs the previously released chart; both global --extra-values
+	// (e.g. per-PR image tag) and scenario-declared extra-values belong to Step 2
+	// only — they are combined in flags.Deployment.ExtraValues, which this nils.
 	step1Flags.Deployment.ExtraValues = nil
 	step1Flags.Test.RunIntegrationTests = false // Don't run tests after Step 1.
 	step1Flags.Test.RunE2ETests = false