feat(raycicmd): add matrix expansion and filtering support

Integrate matrix_expand into conversion steps. This allows pipeline steps to selectively depend on a subset of matrix-defined
steps, reducing unnecessary steps when tag filtering can filter those out.

- Introduced a new YAML configuration for matrix selection tests in Buildkite.
- Implemented matrix expansion logic in the converter, allowing for dynamic step generation based on matrix configurations.

Topic: matrix-converter
Relative: matrix-expand
Labels: draft

Signed-off-by: andrew <andrew@anyscale.com>

Author: andrew-anyscale

.buildkite/matrix-test.rayci.yaml

@@ -0,0 +1,38 @@
+group: matrix selection test
+sort_key: "99"
+tags:
+  - matrix-test
+steps:
+  # Matrix step with 5 values - only "selected-1" and "selected-2" should be run
+  # when filtered (others are skipped)
+  - label: "Matrix {{matrix.variant}}"
+    key: matrix-step
+    command: |
+      echo "Running variant: {{matrix.variant}}"
+      if [ "{{matrix.variant}}" != "selected-1" ] && [ "{{matrix.variant}}" != "selected-2" ]; then
+        echo "ERROR: This variant should not have been spawned!"
+        echo "Only 'selected-1' and 'selected-2' variants should run when matrix-test tag is used."
+        exit 1
+      fi
+      echo "Correct variant spawned."
+    depends_on: forge
+    matrix:
+      setup:
+        variant:
+          - "selected-1"
+          - "selected-2"
+          - "skipped-1"
+          - "skipped-2"
+          - "skipped-3"
+
+  # This step depends on only the "selected" variants
+  - label: "Depends on selected only"
+    key: depends-on-selected
+    command: echo "This step depends on only the selected variants"
+    tags:
+      - always
+    depends_on:
+      - key: matrix-step
+        matrix:
+          variant: ["selected-1", "selected-2"]
+      - forge

raycicmd/converter.go

@@ -115,13 +115,164 @@ func stepTags(step map[string]any) []string {
 	return nil
 }
 
+// matrixExpansionContext holds state for matrix expansion.
+type matrixExpansionContext struct {
+	// registry maps base step keys to their matrix configs
+	registry map[string]*matrixConfig
+	// expandedKeys maps base step keys to their expanded keys
+	expandedKeys map[string][]string
+}
+
+func newMatrixExpansionContext() *matrixExpansionContext {
+	return &matrixExpansionContext{
+		registry:     make(map[string]*matrixConfig),
+		expandedKeys: make(map[string][]string),
+	}
+}
+
+// expandMatrixInGroups preprocesses pipeline groups to expand matrix steps.
+// Returns the modified groups and a context for selector expansion.
+func expandMatrixInGroups(gs []*pipelineGroup) ([]*pipelineGroup, *matrixExpansionContext, error) {
+	ctx := newMatrixExpansionContext()
+	var result []*pipelineGroup
+
+	for _, g := range gs {
+		newGroup := &pipelineGroup{
+			filename:      g.filename,
+			sortKey:       g.sortKey,
+			Group:         g.Group,
+			Key:           g.Key,
+			Tags:          g.Tags,
+			SortKey:       g.SortKey,
+			DependsOn:     g.DependsOn,
+			DefaultJobEnv: g.DefaultJobEnv,
+		}
+
+		for _, step := range g.Steps {
+			matrixDef, hasMatrix := step["matrix"]
+			if !hasMatrix {
+				// No matrix, keep step as-is
+				newGroup.Steps = append(newGroup.Steps, step)
+				continue
+			}
+
+			baseKey := stepKey(step)
+			if baseKey == "" {
+				// No key - pass through to Buildkite for native matrix handling
+				newGroup.Steps = append(newGroup.Steps, step)
+				continue
+			}
+
+			// Parse matrix configuration
+			cfg, err := parseMatrixConfig(matrixDef)
+			if err != nil {
+				return nil, nil, fmt.Errorf("parse matrix in step %q: %w", stepKey(step), err)
+			}
+
+			// Validate label has placeholder
+			if label, ok := step["label"].(string); ok {
+				if !hasMatrixPlaceholder(label) {
+					return nil, nil, fmt.Errorf("matrix step %q: label must contain {{matrix...}} placeholder", baseKey)
+				}
+			}
+
+			// Register for selector expansion
+			ctx.registry[baseKey] = cfg
+
+			// Expand matrix into individual steps
+			instances := expandMatrix(cfg)
+			if len(instances) == 0 {
+				return nil, nil, fmt.Errorf("matrix step %q: no instances after expansion", baseKey)
+			}
+
+			var expandedKeysList []string
+			for _, inst := range instances {
+				// Deep copy and substitute
+				expandedStep := deepCopyStepMap(step)
+				expandedStep = substituteMatrixValues(expandedStep, inst.Values).(map[string]any)
+
+				// Generate expanded key and update name/key fields
+				expandedKey := generateMatrixInstanceKey(baseKey, cfg, inst.Values)
+				if _, hasName := expandedStep["name"]; hasName {
+					expandedStep["name"] = expandedKey
+				} else {
+					expandedStep["key"] = expandedKey
+				}
+				delete(expandedStep, "matrix")
+
+				// Add auto-generated tags (inherit original + add matrix tags)
+				originalTags := stepTags(step)
+				matrixTags := generateMatrixTags(inst.Values)
+				allTags := append([]string{}, originalTags...)
+				allTags = append(allTags, matrixTags...)
+				if len(allTags) > 0 {
+					expandedStep["tags"] = allTags
+				}
+
+				expandedKeysList = append(expandedKeysList, expandedKey)
+				newGroup.Steps = append(newGroup.Steps, expandedStep)
+			}
+
+			ctx.expandedKeys[baseKey] = expandedKeysList
+
+			// Create meta wait-step with original key.
+			// Note: The meta-step intentionally has no tags. It is included in
+			// filtered output only via dependency propagation - when a downstream
+			// step depends on the base key, the meta-step (and all its expanded
+			// step dependencies) will be pulled in. If no downstream step depends
+			// on the base key, the meta-step is excluded, allowing tag filters to
+			// select only specific matrix variants.
+			metaStep := map[string]any{
+				"wait":       nil,
+				"key":        baseKey,
+				"depends_on": expandedKeysList,
+			}
+			newGroup.Steps = append(newGroup.Steps, metaStep)
+		}
+
+		result = append(result, newGroup)
+	}
+
+	return result, ctx, nil
+}
+
+// expandDependsOnSelectors processes depends_on to expand matrix selectors.
+func expandDependsOnSelectors(dependsOn any, ctx *matrixExpansionContext) ([]string, error) {
+	selectors, err := parseMatrixDependsOn(dependsOn)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []string
+	for _, sel := range selectors {
+		if sel.Matrix == nil {
+			// Simple key reference
+			result = append(result, sel.Key)
+		} else {
+			// Selector with matrix filter - expand to matching keys
+			matches, err := expandMatrixSelector(sel, ctx.registry, ctx.expandedKeys)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, matches...)
+		}
+	}
+
+	return result, nil
+}
+
 func (c *converter) convertGroups(gs []*pipelineGroup, filter *stepFilter) (
 	[]*bkPipelineGroup, error,
 ) {
+	expandedGroups, matrixCtx, err := expandMatrixInGroups(gs)
+	if err != nil {
+		return nil, fmt.Errorf("expand matrix: %w", err)
+	}
+
 	set := newStepNodeSet()
 	var groupNodes []*stepNode
 
-	for i, g := range gs {
+	for i, g := range expandedGroups {
 		groupNode := &stepNode{
 			id:       fmt.Sprintf("g%d", i),
 			key:      g.Key,
@@ -169,8 +320,20 @@ func (c *converter) convertGroups(gs []*pipelineGroup, filter *stepFilter) (
 		for _, step := range groupNode.subSteps {
 			// Track step dependencies.
 			if dependsOn, ok := step.src["depends_on"]; ok {
-				deps := toStringList(dependsOn)
-				for _, dep := range deps {
+				// Expand matrix selectors in depends_on
+				expandedDeps, err := expandDependsOnSelectors(dependsOn, matrixCtx)
+				if err != nil {
+					return nil, fmt.Errorf("expand depends_on for step %q: %w", step.key, err)
+				}
+
+				// Update the step source with expanded deps for Buildkite output
+				if len(expandedDeps) == 1 {
+					step.src["depends_on"] = expandedDeps[0]
+				} else {
+					step.src["depends_on"] = expandedDeps
+				}
+
+				for _, dep := range expandedDeps {
 					if depNode, ok := set.byKey(dep); ok {
 						set.addDep(step.id, depNode.id)
 					}