diff --git a/src/data-expander.ts b/src/data-expander.ts index a4768996b..4a3241ec3 100644 --- a/src/data-expander.ts +++ b/src/data-expander.ts @@ -116,6 +116,7 @@ export function needsComplex (data: any) { ...(data.project ? {project: data.project} : {}), ...(data.ref ? {ref: data.ref} : {}), ...(data.optional ? {optional: data.optional} : {}), + ...(data.parallel ? {parallel: data.parallel} : {}), }; // In needs:project/needs:pipeline, `optional` is not an allowed property diff --git a/src/executor.ts b/src/executor.ts index 4290edd8e..d06c8bea9 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -3,6 +3,7 @@ import {Job} from "./job.js"; import assert, {AssertionError} from "node:assert"; import {Argv} from "./argv.js"; import pMap from "p-map"; +import {matrixSelectorMatches} from "./parallel.js"; export class Executor { @@ -87,7 +88,16 @@ export class Executor { const toWaitFor = []; assert(job.needs != null, chalk`${job.name}.needs cannot be null in getNeededToWaitFor`); for (const need of job.needs) { - const baseJobs = jobs.filter(j => j.baseName === need.job); + let baseJobs = jobs.filter(j => j.baseName === need.job); + if (need.parallel?.matrix && baseJobs.length > 0) { + if (baseJobs.every(j => j.matrixVariables == null)) { + throw new AssertionError({message: chalk`{blueBright ${job.name}} uses needs.parallel.matrix targeting {blueBright ${need.job}}, but {blueBright ${need.job}} has no parallel:matrix configuration`}); + } + baseJobs = baseJobs.filter(j => matrixSelectorMatches(j.matrixVariables, need.parallel!.matrix)); + if (baseJobs.length === 0 && !need.optional) { + throw new AssertionError({message: chalk`{blueBright ${job.name}} needs.parallel.matrix selector for {blueBright ${need.job}} matched zero permutations`}); + } + } for (const j of baseJobs) { if (j.when === "never" && !need.optional) { throw new AssertionError({message: chalk`{blueBright ${j.name}} is when:never, but its needed by {blueBright ${job.name}}`}); diff --git a/src/job.ts b/src/job.ts index 23823b978..d30abaf3c 100644 --- a/src/job.ts +++ b/src/job.ts @@ -61,6 +61,7 @@ export interface Need { ref?: string; pipeline?: string; project?: string; + parallel?: {matrix: {[key: string]: string | number | (string | number)[]}[]}; } const isGlob = (str: string) => /[*?{}(|)[\]]/.test(str); @@ -98,6 +99,7 @@ export class Job { readonly argv: Argv; readonly name: string; readonly baseName: string; + readonly matrixVariables: {[key: string]: string} | null; readonly dependencies: string[] | null; readonly environment?: {name: string; url: string | null; deployment_tier: string | null; action: string | null}; readonly jobId: number; @@ -150,6 +152,7 @@ export class Job { this.gitData = opt.gitData; this.name = opt.name; this.baseName = opt.baseName; + this.matrixVariables = opt.matrixVariables; this.jobId = this.generateJobId(); this.jobData = opt.data; this.pipelineIid = opt.pipelineIid; diff --git a/src/parallel.ts b/src/parallel.ts index d913d1666..642eb7666 100644 --- a/src/parallel.ts +++ b/src/parallel.ts @@ -1,10 +1,106 @@ -import assert from "node:assert"; +import chalk from "chalk-template"; +import assert, {AssertionError} from "node:assert"; import deepExtend from "deep-extend"; +import {Need} from "./job.js"; + +type MatrixSelector = NonNullable["matrix"]; +type MatrixVariables = {[key: string]: string}; export function isPlainParallel (jobData: any) { return Number.isInteger(jobData.parallel); } +// Expand a parallel.matrix-shaped selector into concrete {key: value} bindings. +// Mirrors GitLab's expansion: scalars are wrapped to single-element arrays, and +// arrays produce a cartesian product across keys within an entry. +export function expandMatrixSelector (matrix: MatrixSelector): MatrixVariables[] { + const expanded: MatrixVariables[] = []; + for (const entry of matrix) { + let inner: MatrixVariables[] = [{}]; + for (const [key, raw] of Object.entries(entry)) { + const values = Array.isArray(raw) ? raw : [raw]; + const next: MatrixVariables[] = []; + for (const clone of inner) { + for (const v of values) { + next.push({...clone, [key]: String(v)}); + } + } + inner = next; + } + expanded.push(...inner); + } + return expanded; +} + +// Returns true if `producerVars` matches at least one expanded selector entry. +// Match semantics: every key in the selector entry must equal the producer's value; +// the producer may have extra keys not mentioned in the selector. +export function matrixSelectorMatches (producerVars: MatrixVariables | null, matrix: MatrixSelector): boolean { + if (!producerVars) return false; + return expandMatrixSelector(matrix).some(sel => + Object.entries(sel).every(([k, v]) => String(producerVars[k]) === String(v)), + ); +} + +// Find the consumer's `need` (if any) that this `producer` job satisfies. +// Used by both the executor (waitFor) and the artifacts producer pipeline so +// the two paths apply the same matching rules. +export function findMatchingNeed (consumerNeeds: ReadonlyArray | null, producerName: string, producerBaseName: string): Need | undefined { + return consumerNeeds?.find(n => n.job === producerName || n.job === producerBaseName); +} + +// `$[[ matrix.IDENTIFIER ]]` — see https://docs.gitlab.com/ci/yaml/matrix_expressions/. +// Identifier charset matches upstream gitlab-org/gitlab's MatrixInterpolator regex +// `[a-zA-Z0-9_-]+` (letters, digits, underscore, hyphen). +const MATRIX_EXPR_RE = /\$\[\[\s*matrix\.([a-zA-Z0-9_-]+)\s*\]\]/g; +const MATRIX_EXPR_TEST_RE = /\$\[\[\s*matrix\./; + +function valueContainsMatrixExpr (v: unknown): boolean { + if (Array.isArray(v)) return v.some(valueContainsMatrixExpr); + if (typeof v !== "string") return false; + return MATRIX_EXPR_TEST_RE.test(v); +} + +// Detect whether any need's parallel.matrix selector contains a `$[[ matrix.X ]]` expression. +// Used to surface a clear error if a non-parallel-matrix consumer references such expressions. +export function needsContainMatrixExpressions (needs: Need[]): boolean { + return needs.some(n => + n.parallel?.matrix?.some(entry => Object.values(entry).some(valueContainsMatrixExpr)) ?? false, + ); +} + +function substituteMatrixExpr (value: unknown, consumerVars: MatrixVariables, jobName: string): unknown { + if (Array.isArray(value)) return value.map(v => substituteMatrixExpr(v, consumerVars, jobName)); + if (typeof value !== "string") return value; + return value.replace(MATRIX_EXPR_RE, (_, key) => { + if (!(key in consumerVars)) { + // Wording mirrors upstream gitlab-org/gitlab's MatrixInterpolator error + // ("'X' does not exist in matrix configuration") so log greps line up. + throw new AssertionError({message: chalk`{blueBright ${jobName}}: '{yellow ${key}}' does not exist in matrix configuration`}); + } + return consumerVars[key]; + }); +} + +// Substitute `$[[ matrix.X ]]` references inside each need's parallel.matrix +// selector, using the consumer permutation's own matrix bindings. Returns a +// shallow-cloned needs array (only entries with parallel.matrix are rebuilt); +// non-substituted fields on each need are shared with the input by reference. +export function resolveNeedsMatrixExpressions (needs: Need[], consumerVars: MatrixVariables, jobName: string): Need[] { + return needs.map(n => { + if (!n?.parallel?.matrix) return n; + return { + ...n, + parallel: { + ...n.parallel, + matrix: n.parallel.matrix.map(entry => + Object.fromEntries(Object.entries(entry).map(([k, v]) => [k, substituteMatrixExpr(v, consumerVars, jobName)])), + ) as MatrixSelector, + }, + }; + }); +} + export function matrixVariablesList (jobData: any, jobName: string): {[key: string]: string}[] | null[] { if (isPlainParallel(jobData)) { return new Array(jobData.parallel).fill(null); diff --git a/src/parser.ts b/src/parser.ts index 7566f1377..8abfab4af 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -7,7 +7,7 @@ import prettyHrtime from "pretty-hrtime"; import {Job} from "./job.js"; import * as DataExpander from "./data-expander.js"; import {Utils} from "./utils.js"; -import assert from "node:assert"; +import assert, {AssertionError} from "node:assert"; import {Validator} from "./validator.js"; import * as parallel from "./parallel.js"; import {GitData} from "./git-data.js"; @@ -193,10 +193,22 @@ export class Parser { matrixJobName = `${jobName}: [${nodeIndex}/${parallelMatrixVariablesList.length}]`; } + // Resolve `$[[ matrix.X ]]` expressions in needs.parallel.matrix per-permutation + // (https://docs.gitlab.com/ci/yaml/matrix_expressions/). Each consumer permutation + // gets its own substituted needs so producers can be matched 1:1. + let permutationJobData = jobData; + if (jobData.needs && parallel.needsContainMatrixExpressions(jobData.needs) && !parallelMatrixVariables) { + throw new AssertionError({message: chalk`{blueBright ${matrixJobName}} uses $[[ matrix.X ]] expressions in needs.parallel.matrix but is not parallelized with parallel:matrix`}); + } + if (parallelMatrixVariables && jobData.needs?.some((n: any) => n.parallel?.matrix)) { + const resolvedNeeds = parallel.resolveNeedsMatrixExpressions(jobData.needs, parallelMatrixVariables, matrixJobName); + permutationJobData = {...jobData, needs: resolvedNeeds}; + } + const job = new Job({ argv, writeStreams, - data: jobData, + data: permutationJobData, name: matrixJobName, baseName: jobName, globalVariables: gitlabData.variables, diff --git a/src/producers.ts b/src/producers.ts index f75433cbc..39eb7bb61 100644 --- a/src/producers.ts +++ b/src/producers.ts @@ -1,5 +1,6 @@ import {Utils} from "./utils.js"; import {Job} from "./job.js"; +import {findMatchingNeed, matrixSelectorMatches} from "./parallel.js"; export class Producers { @@ -30,6 +31,10 @@ export class Producers { if (potential.artifacts == null) continue; if (potential.when == "never") continue; if (!producerSet.has(potential.name) && !producerSet.has(potential.baseName)) continue; + + const matchedNeed = findMatchingNeed(job.needs, potential.name, potential.baseName); + if (matchedNeed?.parallel?.matrix && !matrixSelectorMatches(potential.matrixVariables, matchedNeed.parallel.matrix)) continue; + producers.push(potential); } return producers.map(producer => { diff --git a/tests/test-cases/needs-parallel-matrix-artifacts/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-artifacts/.gitlab-ci.yml new file mode 100644 index 000000000..ec68549a4 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-artifacts/.gitlab-ci.yml @@ -0,0 +1,29 @@ +--- +# Each producer permutation creates a unique artifact file. Each consumer +# permutation, via $[[ matrix.X ]], should pull ONLY its matching producer's +# artifact — proving the artifact cascade respects the matrix selector. +stages: [build, test] + +build: + stage: build + parallel: + matrix: + - PROVIDER: [aws, gcp] + script: + - echo "$PROVIDER-data" > "tag-$PROVIDER.txt" + artifacts: + paths: + - "tag-*.txt" + +test: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + needs: + - job: build + parallel: + matrix: + - PROVIDER: $[[ matrix.PROVIDER ]] + script: + - cat tag-*.txt diff --git a/tests/test-cases/needs-parallel-matrix-artifacts/integration.test.ts b/tests/test-cases/needs-parallel-matrix-artifacts/integration.test.ts new file mode 100644 index 000000000..c2c8f1be5 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-artifacts/integration.test.ts @@ -0,0 +1,27 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-artifacts cascades only the matching producer's artifacts to each consumer permutation", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-artifacts", + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-artifacts", + }, writeStreams); + + // Positive: each test permutation reads its own producer's artifact via `cat tag-*.txt`. + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{blueBright test: [aws] } {greenBright >} aws-data`, + chalk`{blueBright test: [gcp] } {greenBright >} gcp-data`, + ])); + + // Negative: the other matrix permutation's artifact must NOT have leaked into + // the consumer's working dir. If it did, `cat tag-*.txt` would emit both lines. + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{blueBright test: [aws] } {greenBright >} gcp-data`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{blueBright test: [gcp] } {greenBright >} aws-data`, + ])); +}); diff --git a/tests/test-cases/needs-parallel-matrix-error/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-error/.gitlab-ci.yml new file mode 100644 index 000000000..f72e08050 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-error/.gitlab-ci.yml @@ -0,0 +1,54 @@ +--- +stages: [build, test] + +# Producer is NOT parallelized — using needs.parallel.matrix on it must error. +build-job: + stage: build + script: + - echo "build" + +test-job: + stage: test + needs: + - job: build-job + parallel: + matrix: + - NAME: foo + script: + - echo "test" + +# Producer uses plain `parallel: ` (no matrix) — selector still must error. +build-int: + stage: build + parallel: 3 + script: + - echo "build-int $CI_NODE_INDEX/$CI_NODE_TOTAL" + +test-plain-parallel: + stage: test + needs: + - job: build-int + parallel: + matrix: + - NAME: foo + script: + - echo "test-plain-parallel" + +# Producer is parallel:matrix but the selector matches zero permutations. +build-foo: + stage: build + parallel: + matrix: + - NAME: [foo, bar] + script: + - echo "build-foo $NAME" + +test-zero-match: + stage: test + needs: + - job: build-foo + parallel: + matrix: + - NAME: nope + script: + - echo "test-zero-match" diff --git a/tests/test-cases/needs-parallel-matrix-error/integration.test.ts b/tests/test-cases/needs-parallel-matrix-error/integration.test.ts new file mode 100644 index 000000000..8cfaf9d56 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-error/integration.test.ts @@ -0,0 +1,36 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-error rejects when producer has no parallel:matrix", async () => { + const writeStreams = new WriteStreamsMock(); + await expect(handler({ + cwd: "tests/test-cases/needs-parallel-matrix-error", + job: ["test-job"], + needs: true, + }, writeStreams)).rejects.toThrow( + chalk`{blueBright test-job} uses needs.parallel.matrix targeting {blueBright build-job}, but {blueBright build-job} has no parallel:matrix configuration`, + ); +}); + +test.concurrent("needs-parallel-matrix-error rejects when producer is plain parallel:", async () => { + const writeStreams = new WriteStreamsMock(); + await expect(handler({ + cwd: "tests/test-cases/needs-parallel-matrix-error", + job: ["test-plain-parallel"], + needs: true, + }, writeStreams)).rejects.toThrow( + chalk`{blueBright test-plain-parallel} uses needs.parallel.matrix targeting {blueBright build-int}, but {blueBright build-int} has no parallel:matrix configuration`, + ); +}); + +test.concurrent("needs-parallel-matrix-error rejects when selector matches zero permutations", async () => { + const writeStreams = new WriteStreamsMock(); + await expect(handler({ + cwd: "tests/test-cases/needs-parallel-matrix-error", + job: ["test-zero-match"], + needs: true, + }, writeStreams)).rejects.toThrow( + chalk`{blueBright test-zero-match} needs.parallel.matrix selector for {blueBright build-foo} matched zero permutations`, + ); +}); diff --git a/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/.gitlab-ci.yml new file mode 100644 index 000000000..bb5eb31d9 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/.gitlab-ci.yml @@ -0,0 +1,23 @@ +--- +# Consumer is NOT parallel:matrix but uses $[[ matrix.X ]] in needs.parallel.matrix. +# GitLab matrix expressions can only reference identifiers from the current job's +# parallel:matrix configuration; using them in a non-parallelized consumer is invalid. +stages: [build, test] + +linux:build: + stage: build + parallel: + matrix: + - PROVIDER: [aws, gcp] + script: + - echo "build $PROVIDER" + +linux:test: + stage: test + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: $[[ matrix.PROVIDER ]] + script: + - echo "test" diff --git a/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/integration.test.ts b/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/integration.test.ts new file mode 100644 index 000000000..79d92b033 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer/integration.test.ts @@ -0,0 +1,12 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-expressions-non-parallel-consumer rejects $[[ matrix.X ]] in a non-parallel consumer", async () => { + const writeStreams = new WriteStreamsMock(); + await expect(handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions-non-parallel-consumer", + }, writeStreams)).rejects.toThrow( + chalk`{blueBright linux:test} uses $[[ matrix.X ]] expressions in needs.parallel.matrix but is not parallelized with parallel:matrix`, + ); +}); diff --git a/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/.gitlab-ci.yml new file mode 100644 index 000000000..92bb041af --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/.gitlab-ci.yml @@ -0,0 +1,26 @@ +--- +# Consumer references $[[ matrix.NOPE ]] but NOPE is not a key in its own +# parallel:matrix; this must error per the docs (matrix expressions can only +# reference identifiers from the current job's matrix configuration). +stages: [build, test] + +linux:build: + stage: build + parallel: + matrix: + - PROVIDER: [aws, gcp] + script: + - echo "build $PROVIDER" + +linux:test: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: $[[ matrix.NOPE ]] + script: + - echo "test $PROVIDER" diff --git a/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/integration.test.ts b/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/integration.test.ts new file mode 100644 index 000000000..26349ba8a --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier/integration.test.ts @@ -0,0 +1,12 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-expressions-unknown-identifier rejects $[[ matrix.X ]] referencing unknown matrix key", async () => { + const writeStreams = new WriteStreamsMock(); + await expect(handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions-unknown-identifier", + }, writeStreams)).rejects.toThrow( + chalk`'{yellow NOPE}' does not exist in matrix configuration`, + ); +}); diff --git a/tests/test-cases/needs-parallel-matrix-expressions/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-expressions/.gitlab-ci.yml new file mode 100644 index 000000000..cf344c9a5 --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions/.gitlab-ci.yml @@ -0,0 +1,122 @@ +--- +# Each consumer permutation depends on exactly its matching producer permutation +# via $[[ matrix.IDENTIFIER ]] expressions resolved per-permutation. +stages: [build, test] + +linux:build: + stage: build + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + script: + - echo "build $PROVIDER/$STACK" + +# Bare canonical form — matches the example in gitlab-org/gitlab#423553 verbatim. +linux:test: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: $[[ matrix.PROVIDER ]] + STACK: $[[ matrix.STACK ]] + script: + - echo "test $PROVIDER/$STACK" + +# Array-wrapped expression form — covers the Array.isArray branch in substituteMatrixExpr. +linux:test-array: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: ["$[[ matrix.PROVIDER ]]"] + STACK: ["$[[ matrix.STACK ]]"] + script: + - echo "test-array $PROVIDER/$STACK" + +# Mixed: one literal key + one expression key — selects all STACKs for one PROVIDER. +linux:test-mixed: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: $[[ matrix.PROVIDER ]] + script: + - echo "test-mixed $PROVIDER/$STACK" + +# Hyphenated identifier — upstream allows `[a-zA-Z0-9_-]` in matrix names. +build-hyphen: + stage: build + parallel: + matrix: + - MY-VAR: [foo, bar] + script: + - echo "build-hyphen" + +test-hyphen: + stage: test + parallel: + matrix: + - MY-VAR: [foo, bar] + needs: + - job: build-hyphen + parallel: + matrix: + - MY-VAR: $[[ matrix.MY-VAR ]] + script: + - echo "test-hyphen" + +# Concatenation: two expressions in the same string, plus a non-matrix `$VAR` left +# untouched. Producer's TAG matches the resolved value `build--`. +build-concat: + stage: build + parallel: + matrix: + - TAG: [build-aws-monitoring, build-aws-app1, build-gcp-monitoring, build-gcp-app1] + script: + - echo "build-concat" + +test-concat: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + needs: + - job: build-concat + parallel: + matrix: + - TAG: build-$[[ matrix.PROVIDER ]]-$[[ matrix.STACK ]] + script: + - echo "test-concat" + +# Whitespace variation: `$[[matrix.X]]` (no spaces) must match the same as `$[[ matrix.X ]]`. +test-whitespace: + stage: test + parallel: + matrix: + - PROVIDER: [aws, gcp] + STACK: [monitoring, app1] + needs: + - job: linux:build + parallel: + matrix: + - PROVIDER: $[[matrix.PROVIDER]] + STACK: $[[ matrix.STACK ]] + script: + - echo "test-whitespace" diff --git a/tests/test-cases/needs-parallel-matrix-expressions/integration.test.ts b/tests/test-cases/needs-parallel-matrix-expressions/integration.test.ts new file mode 100644 index 000000000..11632b7bf --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-expressions/integration.test.ts @@ -0,0 +1,159 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-expressions array-form expression resolves 1:1", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["linux:test: [aws,monitoring]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-aws-monitoring", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,monitoring]}`, + chalk`{black.bgGreenBright PASS } {blueBright linux:test: [aws,monitoring]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,app1]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,monitoring]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,app1]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions a different permutation resolves 1:1", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["linux:test: [gcp,app1]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-gcp-app1", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,app1]}`, + chalk`{black.bgGreenBright PASS } {blueBright linux:test: [gcp,app1]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,monitoring]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,app1]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,monitoring]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions array-form expression resolves 1:1", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["linux:test-array: [aws,monitoring]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-array-aws-monitoring", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,monitoring] }`, + chalk`{black.bgGreenBright PASS } {blueBright linux:test-array: [aws,monitoring]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,app1] }`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions hyphenated matrix identifier resolves correctly", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["test-hyphen: [foo]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-hyphen-foo", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-hyphen: [foo]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-hyphen: [foo]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-hyphen: [bar]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions multiple expressions concatenated in one string resolve correctly", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["test-concat: [aws,monitoring]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-concat-aws-monitoring", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-concat: [build-aws-monitoring]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-concat: [aws,monitoring]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-concat: [build-gcp-app1]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions whitespace variations parse equivalently", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["test-whitespace: [aws,monitoring]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-whitespace-aws-monitoring", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,monitoring] }`, + chalk`{black.bgGreenBright PASS } {blueBright test-whitespace: [aws,monitoring]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,app1] }`, + ])); +}); + +test.concurrent("needs-parallel-matrix-expressions mixed literal+expression keeps PROVIDER bound, broadens STACK", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-expressions", + job: ["linux:test-mixed: [aws,monitoring]"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-expressions-mixed-aws-monitoring", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,monitoring] }`, + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [aws,app1] }`, + chalk`{black.bgGreenBright PASS } {blueBright linux:test-mixed: [aws,monitoring]}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,monitoring] }`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright linux:build: [gcp,app1] }`, + ])); +}); diff --git a/tests/test-cases/needs-parallel-matrix-static/.gitlab-ci.yml b/tests/test-cases/needs-parallel-matrix-static/.gitlab-ci.yml new file mode 100644 index 000000000..30e082e5d --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-static/.gitlab-ci.yml @@ -0,0 +1,77 @@ +--- +stages: [build, test] + +build-job: + stage: build + parallel: + matrix: + - NAME: [foo, bar, beb] + script: + - echo "build NAME=$NAME" + +# Multi-key producer — exercises cartesian expansion and partial selectors below. +build-multi: + stage: build + parallel: + matrix: + - ARCH: [x64, arm] + OS: [linux, mac] + script: + - echo "build-multi $ARCH/$OS" + +# Selects exactly one permutation of build-job. +test-single: + stage: test + needs: + - job: build-job + parallel: + matrix: + - NAME: foo + script: + - echo "test-single ran" + +# Selects two permutations via array selector. +test-array: + stage: test + needs: + - job: build-job + parallel: + matrix: + - NAME: [foo, bar] + script: + - echo "test-array ran" + +# Partial selector — mentions only ARCH; should match both OS permutations of x64. +test-partial: + stage: test + needs: + - job: build-multi + parallel: + matrix: + - ARCH: x64 + script: + - echo "test-partial ran" + +# Full selector — picks exactly one permutation of build-multi. +test-full: + stage: test + needs: + - job: build-multi + parallel: + matrix: + - ARCH: x64 + OS: linux + script: + - echo "test-full ran" + +# Optional + zero-match selector — should run alone, no error. +test-optional-zero: + stage: test + needs: + - job: build-job + optional: true + parallel: + matrix: + - NAME: nope + script: + - echo "test-optional-zero ran" diff --git a/tests/test-cases/needs-parallel-matrix-static/integration.test.ts b/tests/test-cases/needs-parallel-matrix-static/integration.test.ts new file mode 100644 index 000000000..2ee2a8e1b --- /dev/null +++ b/tests/test-cases/needs-parallel-matrix-static/integration.test.ts @@ -0,0 +1,116 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import chalk from "chalk-template"; + +test.concurrent("needs-parallel-matrix-static only triggers build-job: [foo]", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-static", + job: ["test-single"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-static-single", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [foo]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-single}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [bar]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [beb]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-static triggers build-job: [foo] and [bar] only", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-static", + job: ["test-array"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-static-array", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [foo]}`, + chalk`{black.bgGreenBright PASS } {blueBright build-job: [bar]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-array}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [beb]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-static partial selector matches both OS permutations of ARCH=x64", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-static", + job: ["test-partial"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-static-partial", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [x64,linux]}`, + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [x64,mac]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-partial}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [arm,linux]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [arm,mac]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-static full multi-key selector picks one permutation", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-static", + job: ["test-full"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-static-full", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [x64,linux]}`, + chalk`{black.bgGreenBright PASS } {blueBright test-full }`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [x64,mac]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [arm,linux]}`, + ])); + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-multi: [arm,mac]}`, + ])); +}); + +test.concurrent("needs-parallel-matrix-static optional selector matching nothing runs consumer alone", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/needs-parallel-matrix-static", + job: ["test-optional-zero"], + needs: true, + shellIsolation: true, + stateDir: ".gitlab-ci-local-needs-parallel-matrix-static-optional-zero", + }, writeStreams); + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright test-optional-zero}`, + ])); + + expect(writeStreams.stdoutLines).not.toEqual(expect.arrayContaining([ + chalk`{black.bgGreenBright PASS } {blueBright build-job: [foo]}`, + ])); +});