feat: support needs[].parallel.matrix and matrix expressions (#1848)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: inistor

src/data-expander.ts

@@ -116,6 +116,7 @@ export function needsComplex (data: any) {
         ...(data.project ? {project: data.project} : {}),
         ...(data.ref ? {ref: data.ref} : {}),
         ...(data.optional ? {optional: data.optional} : {}),
+        ...(data.parallel ? {parallel: data.parallel} : {}),
     };
 
     // In needs:project/needs:pipeline, `optional` is not an allowed property

src/executor.ts

@@ -3,6 +3,7 @@ import {Job} from "./job.js";
 import assert, {AssertionError} from "node:assert";
 import {Argv} from "./argv.js";
 import pMap from "p-map";
+import {matrixSelectorMatches} from "./parallel.js";
 
 export class Executor {
 
@@ -87,7 +88,16 @@ export class Executor {
         const toWaitFor = [];
         assert(job.needs != null, chalk`${job.name}.needs cannot be null in getNeededToWaitFor`);
         for (const need of job.needs) {
-            const baseJobs = jobs.filter(j => j.baseName === need.job);
+            let baseJobs = jobs.filter(j => j.baseName === need.job);
+            if (need.parallel?.matrix && baseJobs.length > 0) {
+                if (baseJobs.every(j => j.matrixVariables == null)) {
+                    throw new AssertionError({message: chalk`{blueBright ${job.name}} uses needs.parallel.matrix targeting {blueBright ${need.job}}, but {blueBright ${need.job}} has no parallel:matrix configuration`});
+                }
+                baseJobs = baseJobs.filter(j => matrixSelectorMatches(j.matrixVariables, need.parallel!.matrix));
+                if (baseJobs.length === 0 && !need.optional) {
+                    throw new AssertionError({message: chalk`{blueBright ${job.name}} needs.parallel.matrix selector for {blueBright ${need.job}} matched zero permutations`});
+                }
+            }
             for (const j of baseJobs) {
                 if (j.when === "never" && !need.optional) {
                     throw new AssertionError({message: chalk`{blueBright ${j.name}} is when:never, but its needed by {blueBright ${job.name}}`});

src/job.ts

@@ -61,6 +61,7 @@ export interface Need {
     ref?: string;
     pipeline?: string;
     project?: string;
+    parallel?: {matrix: {[key: string]: string | number | (string | number)[]}[]};
 }
 
 const isGlob = (str: string) => /[*?{}(|)[\]]/.test(str);
@@ -98,6 +99,7 @@ export class Job {
     readonly argv: Argv;
     readonly name: string;
     readonly baseName: string;
+    readonly matrixVariables: {[key: string]: string} | null;
     readonly dependencies: string[] | null;
     readonly environment?: {name: string; url: string | null; deployment_tier: string | null; action: string | null};
     readonly jobId: number;
@@ -150,6 +152,7 @@ export class Job {
         this.gitData = opt.gitData;
         this.name = opt.name;
         this.baseName = opt.baseName;
+        this.matrixVariables = opt.matrixVariables;
         this.jobId = this.generateJobId();
         this.jobData = opt.data;
         this.pipelineIid = opt.pipelineIid;

src/parallel.ts

@@ -1,10 +1,106 @@
-import assert from "node:assert";
+import chalk from "chalk-template";
+import assert, {AssertionError} from "node:assert";
 import deepExtend from "deep-extend";
+import {Need} from "./job.js";
+
+type MatrixSelector = NonNullable<Need["parallel"]>["matrix"];
+type MatrixVariables = {[key: string]: string};
 
 export function isPlainParallel (jobData: any) {
     return Number.isInteger(jobData.parallel);
 }
 
+// Expand a parallel.matrix-shaped selector into concrete {key: value} bindings.
+// Mirrors GitLab's expansion: scalars are wrapped to single-element arrays, and
+// arrays produce a cartesian product across keys within an entry.
+export function expandMatrixSelector (matrix: MatrixSelector): MatrixVariables[] {
+    const expanded: MatrixVariables[] = [];
+    for (const entry of matrix) {
+        let inner: MatrixVariables[] = [{}];
+        for (const [key, raw] of Object.entries(entry)) {
+            const values = Array.isArray(raw) ? raw : [raw];
+            const next: MatrixVariables[] = [];
+            for (const clone of inner) {
+                for (const v of values) {
+                    next.push({...clone, [key]: String(v)});
+                }
+            }
+            inner = next;
+        }
+        expanded.push(...inner);
+    }
+    return expanded;
+}
+
+// Returns true if `producerVars` matches at least one expanded selector entry.
+// Match semantics: every key in the selector entry must equal the producer's value;
+// the producer may have extra keys not mentioned in the selector.
+export function matrixSelectorMatches (producerVars: MatrixVariables | null, matrix: MatrixSelector): boolean {
+    if (!producerVars) return false;
+    return expandMatrixSelector(matrix).some(sel =>
+        Object.entries(sel).every(([k, v]) => String(producerVars[k]) === String(v)),
+    );
+}
+
+// Find the consumer's `need` (if any) that this `producer` job satisfies.
+// Used by both the executor (waitFor) and the artifacts producer pipeline so
+// the two paths apply the same matching rules.
+export function findMatchingNeed (consumerNeeds: ReadonlyArray<Need> | null, producerName: string, producerBaseName: string): Need | undefined {
+    return consumerNeeds?.find(n => n.job === producerName || n.job === producerBaseName);
+}
+
+// `$[[ matrix.IDENTIFIER ]]` — see https://docs.gitlab.com/ci/yaml/matrix_expressions/.
+// Identifier charset matches upstream gitlab-org/gitlab's MatrixInterpolator regex
+// `[a-zA-Z0-9_-]+` (letters, digits, underscore, hyphen).
+const MATRIX_EXPR_RE = /\$\[\[\s*matrix\.([a-zA-Z0-9_-]+)\s*\]\]/g;
+const MATRIX_EXPR_TEST_RE = /\$\[\[\s*matrix\./;
+
+function valueContainsMatrixExpr (v: unknown): boolean {
+    if (Array.isArray(v)) return v.some(valueContainsMatrixExpr);
+    if (typeof v !== "string") return false;
+    return MATRIX_EXPR_TEST_RE.test(v);
+}
+
+// Detect whether any need's parallel.matrix selector contains a `$[[ matrix.X ]]` expression.
+// Used to surface a clear error if a non-parallel-matrix consumer references such expressions.
+export function needsContainMatrixExpressions (needs: Need[]): boolean {
+    return needs.some(n =>
+        n.parallel?.matrix?.some(entry => Object.values(entry).some(valueContainsMatrixExpr)) ?? false,
+    );
+}
+
+function substituteMatrixExpr (value: unknown, consumerVars: MatrixVariables, jobName: string): unknown {
+    if (Array.isArray(value)) return value.map(v => substituteMatrixExpr(v, consumerVars, jobName));
+    if (typeof value !== "string") return value;
+    return value.replace(MATRIX_EXPR_RE, (_, key) => {
+        if (!(key in consumerVars)) {
+            // Wording mirrors upstream gitlab-org/gitlab's MatrixInterpolator error
+            // ("'X' does not exist in matrix configuration") so log greps line up.
+            throw new AssertionError({message: chalk`{blueBright ${jobName}}: '{yellow ${key}}' does not exist in matrix configuration`});
+        }
+        return consumerVars[key];
+    });
+}
+
+// Substitute `$[[ matrix.X ]]` references inside each need's parallel.matrix
+// selector, using the consumer permutation's own matrix bindings. Returns a
+// shallow-cloned needs array (only entries with parallel.matrix are rebuilt);
+// non-substituted fields on each need are shared with the input by reference.
+export function resolveNeedsMatrixExpressions (needs: Need[], consumerVars: MatrixVariables, jobName: string): Need[] {
+    return needs.map(n => {
+        if (!n?.parallel?.matrix) return n;
+        return {
+            ...n,
+            parallel: {
+                ...n.parallel,
+                matrix: n.parallel.matrix.map(entry =>
+                    Object.fromEntries(Object.entries(entry).map(([k, v]) => [k, substituteMatrixExpr(v, consumerVars, jobName)])),
+                ) as MatrixSelector,
+            },
+        };
+    });
+}
+
 export function matrixVariablesList (jobData: any, jobName: string): {[key: string]: string}[] | null[] {
     if (isPlainParallel(jobData)) {
         return new Array(jobData.parallel).fill(null);

src/parser.ts

@@ -7,7 +7,7 @@ import prettyHrtime from "pretty-hrtime";
 import {Job} from "./job.js";
 import * as DataExpander from "./data-expander.js";
 import {Utils} from "./utils.js";
-import assert from "node:assert";
+import assert, {AssertionError} from "node:assert";
 import {Validator} from "./validator.js";
 import * as parallel from "./parallel.js";
 import {GitData} from "./git-data.js";
@@ -193,10 +193,22 @@ export class Parser {
                     matrixJobName = `${jobName}: [${nodeIndex}/${parallelMatrixVariablesList.length}]`;
                 }
 
+                // Resolve `$[[ matrix.X ]]` expressions in needs.parallel.matrix per-permutation
+                // (https://docs.gitlab.com/ci/yaml/matrix_expressions/). Each consumer permutation
+                // gets its own substituted needs so producers can be matched 1:1.
+                let permutationJobData = jobData;
+                if (jobData.needs && parallel.needsContainMatrixExpressions(jobData.needs) && !parallelMatrixVariables) {
+                    throw new AssertionError({message: chalk`{blueBright ${matrixJobName}} uses $[[ matrix.X ]] expressions in needs.parallel.matrix but is not parallelized with parallel:matrix`});
+                }
+                if (parallelMatrixVariables && jobData.needs?.some((n: any) => n.parallel?.matrix)) {
+                    const resolvedNeeds = parallel.resolveNeedsMatrixExpressions(jobData.needs, parallelMatrixVariables, matrixJobName);
+                    permutationJobData = {...jobData, needs: resolvedNeeds};
+                }
+
                 const job = new Job({
                     argv,
                     writeStreams,
-                    data: jobData,
+                    data: permutationJobData,
                     name: matrixJobName,
                     baseName: jobName,
                     globalVariables: gitlabData.variables,

src/producers.ts

@@ -1,5 +1,6 @@
 import {Utils} from "./utils.js";
 import {Job} from "./job.js";
+import {findMatchingNeed, matrixSelectorMatches} from "./parallel.js";
 
 export class Producers {
 
@@ -30,6 +31,10 @@ export class Producers {
             if (potential.artifacts == null) continue;
             if (potential.when == "never") continue;
             if (!producerSet.has(potential.name) && !producerSet.has(potential.baseName)) continue;
+
+            const matchedNeed = findMatchingNeed(job.needs, potential.name, potential.baseName);
+            if (matchedNeed?.parallel?.matrix && !matrixSelectorMatches(potential.matrixVariables, matchedNeed.parallel.matrix)) continue;
+
             producers.push(potential);
         }
         return producers.map(producer => {

tests/test-cases/needs-parallel-matrix-artifacts/.gitlab-ci.yml

@@ -0,0 +1,29 @@
+---
+# Each producer permutation creates a unique artifact file. Each consumer
+# permutation, via $[[ matrix.X ]], should pull ONLY its matching producer's
+# artifact — proving the artifact cascade respects the matrix selector.
+stages: [build, test]
+
+build:
+  stage: build
+  parallel:
+    matrix:
+      - PROVIDER: [aws, gcp]
+  script:
+    - echo "$PROVIDER-data" > "tag-$PROVIDER.txt"
+  artifacts:
+    paths:
+      - "tag-*.txt"
+
+test:
+  stage: test
+  parallel:
+    matrix:
+      - PROVIDER: [aws, gcp]
+  needs:
+    - job: build
+      parallel:
+        matrix:
+          - PROVIDER: $[[ matrix.PROVIDER ]]
+  script:
+    - cat tag-*.txt

tests/test-cases/needs-parallel-matrix-artifacts/integration.test.ts

@@ -0,0 +1,27 @@
+import {WriteStreamsMock} from "../../../src/write-streams.js";
+import {handler} from "../../../src/handler.js";
+import chalk from "chalk-template";
+
+test.concurrent("needs-parallel-matrix-artifacts cascades only the matching producer's artifacts to each consumer permutation", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/needs-parallel-matrix-artifacts",
+        shellIsolation: true,
+        stateDir: ".gitlab-ci-local-needs-parallel-matrix-artifacts",
+    }, writeStreams);
+
+    // Positive: each test permutation reads its own producer's artifact via `cat tag-*.txt`.
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([
+        chalk`{blueBright test: [aws] } {greenBright >} aws-data`,
+        chalk`{blueBright test: [gcp] } {greenBright >} gcp-data`,
+    ]));
+
+    // Negative: the other matrix permutation's artifact must NOT have leaked into
+    // the consumer's working dir. If it did, `cat tag-*.txt` would emit both lines.
+    expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([
+        chalk`{blueBright test: [aws] } {greenBright >} gcp-data`,
+    ]));
+    expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([
+        chalk`{blueBright test: [gcp] } {greenBright >} aws-data`,
+    ]));
+});

tests/test-cases/needs-parallel-matrix-error/.gitlab-ci.yml

@@ -0,0 +1,54 @@
+---
+stages: [build, test]
+
+# Producer is NOT parallelized — using needs.parallel.matrix on it must error.
+build-job:
+  stage: build
+  script:
+    - echo "build"
+
+test-job:
+  stage: test
+  needs:
+    - job: build-job
+      parallel:
+        matrix:
+          - NAME: foo
+  script:
+    - echo "test"
+
+# Producer uses plain `parallel: <integer>` (no matrix) — selector still must error.
+build-int:
+  stage: build
+  parallel: 3
+  script:
+    - echo "build-int $CI_NODE_INDEX/$CI_NODE_TOTAL"
+
+test-plain-parallel:
+  stage: test
+  needs:
+    - job: build-int
+      parallel:
+        matrix:
+          - NAME: foo
+  script:
+    - echo "test-plain-parallel"
+
+# Producer is parallel:matrix but the selector matches zero permutations.
+build-foo:
+  stage: build
+  parallel:
+    matrix:
+      - NAME: [foo, bar]
+  script:
+    - echo "build-foo $NAME"
+
+test-zero-match:
+  stage: test
+  needs:
+    - job: build-foo
+      parallel:
+        matrix:
+          - NAME: nope
+  script:
+    - echo "test-zero-match"

tests/test-cases/needs-parallel-matrix-error/integration.test.ts

@@ -0,0 +1,36 @@
+import {WriteStreamsMock} from "../../../src/write-streams.js";
+import {handler} from "../../../src/handler.js";
+import chalk from "chalk-template";
+
+test.concurrent("needs-parallel-matrix-error <test-job> rejects when producer has no parallel:matrix", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await expect(handler({
+        cwd: "tests/test-cases/needs-parallel-matrix-error",
+        job: ["test-job"],
+        needs: true,
+    }, writeStreams)).rejects.toThrow(
+        chalk`{blueBright test-job} uses needs.parallel.matrix targeting {blueBright build-job}, but {blueBright build-job} has no parallel:matrix configuration`,
+    );
+});
+
+test.concurrent("needs-parallel-matrix-error <test-plain-parallel> rejects when producer is plain parallel:<int>", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await expect(handler({
+        cwd: "tests/test-cases/needs-parallel-matrix-error",
+        job: ["test-plain-parallel"],
+        needs: true,
+    }, writeStreams)).rejects.toThrow(
+        chalk`{blueBright test-plain-parallel} uses needs.parallel.matrix targeting {blueBright build-int}, but {blueBright build-int} has no parallel:matrix configuration`,
+    );
+});
+
+test.concurrent("needs-parallel-matrix-error <test-zero-match> rejects when selector matches zero permutations", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await expect(handler({
+        cwd: "tests/test-cases/needs-parallel-matrix-error",
+        job: ["test-zero-match"],
+        needs: true,
+    }, writeStreams)).rejects.toThrow(
+        chalk`{blueBright test-zero-match} needs.parallel.matrix selector for {blueBright build-foo} matched zero permutations`,
+    );
+});