diff --git a/src/parser-includes.ts b/src/parser-includes.ts index e6d73cd46..9919267c6 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -26,14 +26,31 @@ type ParserIncludesInitOptions = { }; type ParsedComponent = { + _cache: { + version: string | null | undefined; + effectiveRef: string | undefined; + sha: string | undefined; + }; + gitData: GitData; domain: string; port: string; projectPath: string; + componentPath: string; name: string; - ref: string; + reference: string; + version: string | null; + effectiveRef: string; + sha: string; isLocal: boolean; }; +type GitRemoteInfoContext = { + gitData: GitData; + domain: string; + port: string; + projectPath: string; +}; + export class ParserIncludes { private static count: number = 0; @@ -100,7 +117,7 @@ export class ParserIncludes { componentParseCache.set(index, component); if (!component.isLocal) { - promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.ref, component.name)); + promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.effectiveRef, component.componentPath)); } } @@ -142,13 +159,13 @@ export class ParserIncludes { const component = componentParseCache.get(index); assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`); // Gitlab allows two different file paths to include a component - const files = [`${component.name}.yml`, `${component.name}/template.yml`]; + const files = [`${component.componentPath}.yml`, `${component.componentPath}/template.yml`]; let file = null; for (const f of files) { let searchPath = `${cwd}/${f}`; if (!component.isLocal) { - searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`; + searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.effectiveRef}/${f}`; } if (fs.existsSync(searchPath)) { file = searchPath; @@ -158,13 +175,15 @@ export class ParserIncludes { (component.port ? `:${component.port}` : "") + `/${component.projectPath}\``); // Extract component name for component-specific inputs - const componentName = component.name.replace(/^templates\//, ""); + const componentName = component.componentPath.replace(/^templates\//, ""); const fileComponentInputs = isStructured ? (fileInputs[componentName] ?? {}) : {}; const cliComponentSpecificInputs = cliComponentInputs[componentName] ?? {}; const mergedInputs = {...(value.inputs ?? {}), ...globalInputs, ...fileComponentInputs, ...cliComponentSpecificInputs}; - const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs}, expandVariables, writeStreams); - // Expand local includes inside to a "project"-like include - fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts); + const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs, component}, expandVariables, writeStreams); + if (!component.isLocal) { + // Expand local includes inside to a "project"-like include + fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.effectiveRef, opts); + } includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); @@ -236,42 +255,70 @@ export class ParserIncludes { assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`); const pattern = /(?[^/:\s]+)(:(?\d+))?\/(?.+)\/(?[^@]+)@(?.+)/; // https://regexr.com/7v7hm const gitRemoteMatch = pattern.exec(component); - if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`); - - const {domain, projectPath, port} = gitRemoteMatch.groups; - let ref = gitRemoteMatch.groups["ref"]; + const {domain, projectPath, port, componentName, ref} = gitRemoteMatch.groups; const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && ref === gitData.commit.SHA; - - if (!isLocalComponent) { - const semanticVersionRangesPattern = /^\d+(\.\d+)?$/; - if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) { - // https://docs.gitlab.com/ci/components/#semantic-version-ranges - let stdout; - if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") { - stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout; - } else { - stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; - } - assert(stdout); - const tags = stdout - .split("\n") - .map((line) => { - return line - .split("\t")[1] - .split("/")[2]; - }); - const _ref = resolveSemanticVersionRange(ref, tags); - assert(_ref, `This GitLab CI configuration is invalid: component: \`${component}\` - The ref (${ref}) is invalid`); - ref = _ref; - } - } return { - domain: domain, - port: port, - projectPath: projectPath, - name: `templates/${gitRemoteMatch.groups["componentName"]}`, - ref: ref, + _cache: { + version: undefined, + effectiveRef: undefined, + sha: undefined, + }, + gitData, + domain, + port, + projectPath, + componentPath: `templates/${componentName}`, + name: componentName, + reference: ref, + get version () { + if (this._cache.version === undefined) { + if (this.isLocal) { + this._cache.version = this.gitData.commit.SHA; + } else { + const semanticVersionRangesPattern = /^\d+(\.\d+)?$/; + if (this.reference == "~latest" || semanticVersionRangesPattern.test(this.reference)) { + // https://docs.gitlab.com/ci/components/#semantic-version-ranges + const stdout = getGitRemoteInfo(this, "--tags"); + const tags = stdout.split("\n").map(line => line.split("\t")[1].split("/")[2]); + const version = resolveSemanticVersionRange(this.reference, tags); + assert(version, `This GitLab CI configuration is invalid: component: \`${this.name}\` - The reference (${this.reference}) is invalid`); + this._cache.version = version; + } else { + this._cache.version = null; + } + } + } + return this._cache.version; + }, + get effectiveRef () { + if (this._cache.effectiveRef === undefined) { + this._cache.effectiveRef = this.version ?? this.reference; + } + return this._cache.effectiveRef; + }, + get sha () { + if (this._cache.sha === undefined) { + if (this.isLocal) { + this._cache.sha = this.gitData.commit.SHA; + } else if (/^[0-9a-f]{40}$/.test(this.effectiveRef)) { + // effectiveRef may already be a sha, if so return it directly + this._cache.sha = this.effectiveRef; + } else { + const stdout = getGitRemoteInfo(this); + const lines = stdout.split("\n"); + // annotated tags: prefer the deref'd commit sha (refs/tags/x^{}) + const match = lines.find(line => line.endsWith(`refs/tags/${this.effectiveRef}^{}`)) ?? + lines.find(line => + line.endsWith(`refs/tags/${this.effectiveRef}`) || + line.endsWith(`refs/heads/${this.effectiveRef}`), + ); + assert(match, `Could not resolve commit SHA for ${this.effectiveRef} in ${this.projectPath}`); + this._cache.sha = match.split("\t")[0]; + } + } + return this._cache.sha; + }, isLocal: isLocalComponent, }; } @@ -458,3 +505,13 @@ export async function resolveIncludeLocal (pattern: string, cwd: string) { const re2js = RE2JS.compile(`^${pattern}`); return repoFiles.filter((f: any) => re2js.matches(f)); } + +export function getGitRemoteInfo (ctx: GitRemoteInfoContext, ...args: string[]) { + const cmdArgs = ["git", "ls-remote", ...args]; + if (ctx.gitData.remote.schema == "git" || ctx.gitData.remote.schema == "ssh") { + cmdArgs.push(`git@${ctx.domain}:${ctx.projectPath}`); + } else { + cmdArgs.push(`${ctx.gitData.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`); + } + return Utils.syncSpawn(cmdArgs).stdout; +} diff --git a/src/parser.ts b/src/parser.ts index 8abfab4af..5e4abe968 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -354,7 +354,6 @@ export class Parser { if (isGitlabSpecFile(fileData[0])) { const inputsSpecification: any = fileData[0]; const uninterpolatedConfigurations: any = fileData[1]; - const interpolatedConfigurations = JSON.stringify(uninterpolatedConfigurations) .replaceAll( /(?.)?(?.)?\$\[\[\s*inputs.(?[\w-]+)\s*\|?\s*(?.*?)\s*\]\](?[^$])?/g // https://regexr.com/81c16 @@ -398,6 +397,11 @@ export class Parser { default: Utils.switchStatementExhaustiveCheck(inputType); } + }) + .replaceAll( // https://docs.gitlab.com/ci/components/#use-component-context-in-components + /\$\[\[\s*component\.(?name|reference|version|sha)\s*\]\]/g // regexr.com/8lotc + , (_: string, interpolationKey: string) => { + return getComponentValue(filePath, ctx, interpolationKey) || _; }); return JSON.parse(interpolatedConfigurations); } @@ -409,6 +413,12 @@ function isGitlabSpecFile (fileData: any) { return "spec" in fileData; } +function getComponentValue (filePath: string, ctx: any, interpolationKey: string) { + const {component} = ctx; + assert(component !== undefined, chalk`This GitLab CI configuration is invalid: \`{blueBright ${filePath}}\`: \`{blueBright component.${interpolationKey}}\` cannot be used outside a component.`); + return component[interpolationKey]; +} + function validateInterpolationKey (ctx: any) { const {configFilePath, interpolationKey, inputsSpecification} = ctx; const invalidInterpolationKeyErr = chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: unknown interpolation key: \`${interpolationKey}\`.`; @@ -427,12 +437,6 @@ function validateInput (ctx: any) { const {configFilePath, interpolationKey, inputsSpecification} = ctx; const inputValue = getInputValue(ctx); - const options = inputsSpecification.spec.inputs[interpolationKey]?.options; - if (options) { - assert(options.includes(inputValue), - chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${inputValue}}\` cannot be used because it is not in the list of allowed options.`); - } - const expectedInputType = getExpectedInputType(ctx); assert(INCLUDE_INPUTS_SUPPORTED_TYPES.includes(expectedInputType), chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: header:spec:inputs:{blueBright ${interpolationKey}} input type unknown value: {blueBright ${expectedInputType}}.`); @@ -441,6 +445,19 @@ function validateInput (ctx: any) { assert(inputType === expectedInputType, chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: provided value is not a {blueBright ${expectedInputType}}.`); + const options = inputsSpecification.spec.inputs[interpolationKey]?.options; + if (options) { + if (inputType == "array") { + for (const itemValue of inputValue) { + assert(options.includes(itemValue), + chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${itemValue}}\` cannot be used because it is not in the list of allowed options.`); + } + } else { + assert(options.includes(inputValue), + chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${inputValue}}\` cannot be used because it is not in the list of allowed options.`); + } + } + const regex = inputsSpecification.spec.inputs[interpolationKey]?.regex; if (regex) { ctx.writeStreams?.stderr(chalk`{black.bgYellowBright WARN } spec:inputs:regex is currently not supported via gitlab-ci-local. This will just be a no-op.\n`); diff --git a/tests/test-cases/include-component/component-local/templates/my-component.yml b/tests/test-cases/include-component/component-local/templates/my-component.yml index 89a12337f..dc3b103d3 100644 --- a/tests/test-cases/include-component/component-local/templates/my-component.yml +++ b/tests/test-cases/include-component/component-local/templates/my-component.yml @@ -5,5 +5,7 @@ spec: default: test --- component-job: - script: echo job 1 + script: + - echo job 1 from $[[ component.name ]] ($[[ component.version ]]/$[[ component.reference ]]). + - Sha is $[[ component.sha ]] stage: $[[ inputs.stage ]] diff --git a/tests/test-cases/include-component/integration.test.ts b/tests/test-cases/include-component/integration.test.ts index 1b4739c0b..b907b47a8 100644 --- a/tests/test-cases/include-component/integration.test.ts +++ b/tests/test-cases/include-component/integration.test.ts @@ -3,6 +3,7 @@ import {handler} from "../../../src/handler.js"; import assert, {AssertionError} from "assert"; import {initSpawnSpy} from "../../mocks/utils.mock.js"; import {WhenStatics} from "../../mocks/when-statics.js"; +import {Utils} from "../../../src/utils.js"; beforeAll(() => { initSpawnSpy(WhenStatics.all); @@ -175,6 +176,7 @@ test.concurrent("include-component component (protocol: https) (~latest semver)" }); test.concurrent("include-component local component", async () => { + const sha = Utils.syncSpawn(["git", "rev-parse", "HEAD"]).stdout.trimEnd();; const writeStreams = new WriteStreamsMock(); await handler({ @@ -182,7 +184,6 @@ test.concurrent("include-component local component", async () => { preview: true, stateDir: ".gitlab-ci-local-include-component-local-component", }, writeStreams); - const expected = `--- stages: - .pre @@ -190,7 +191,8 @@ stages: - .post component-job: script: - - echo job 1 + - echo job 1 from my-component (${sha}/${sha}). + - Sha is ${sha} stage: my-stage`; expect(writeStreams.stdoutLines[0]).toEqual(expected); diff --git a/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci-input-template.yml b/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci-input-template.yml new file mode 100644 index 000000000..edfabbd81 --- /dev/null +++ b/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci-input-template.yml @@ -0,0 +1,14 @@ +--- +spec: + inputs: + options_input: + type: array + default: [] + options: + - foo + - bar + - baz +--- +scan-website: + script: + - echo $[[ inputs.options_input ]] diff --git a/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci.yml b/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci.yml new file mode 100644 index 000000000..54ece4e30 --- /dev/null +++ b/tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci.yml @@ -0,0 +1,7 @@ +--- +include: + - local: '/.gitlab-ci-input-template.yml' + inputs: + options_input: ["foo", "baz"] +stages: + - test diff --git a/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci-input-template.yml b/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci-input-template.yml new file mode 100644 index 000000000..edfabbd81 --- /dev/null +++ b/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci-input-template.yml @@ -0,0 +1,14 @@ +--- +spec: + inputs: + options_input: + type: array + default: [] + options: + - foo + - bar + - baz +--- +scan-website: + script: + - echo $[[ inputs.options_input ]] diff --git a/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci.yml b/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci.yml new file mode 100644 index 000000000..44afa75c8 --- /dev/null +++ b/tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci.yml @@ -0,0 +1,7 @@ +--- +include: + - local: '/.gitlab-ci-input-template.yml' + inputs: + options_input: ["bar", "fizz"] +stages: + - test diff --git a/tests/test-cases/include-inputs/integration.test.ts b/tests/test-cases/include-inputs/integration.test.ts index eabd4d6e2..360152096 100644 --- a/tests/test-cases/include-inputs/integration.test.ts +++ b/tests/test-cases/include-inputs/integration.test.ts @@ -321,6 +321,44 @@ test.concurrent("include-inputs options validation", async () => { throw new Error("Error is expected but not thrown/caught"); }); +test.concurrent("include-inputs options array1 validation", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/include-inputs/input-templates/options-array1-validation", + preview: true, + }, writeStreams); + + const expected = `--- +stages: + - .pre + - test + - .post +scan-website: + script: + - echo ["foo","baz"]`; + + expect(writeStreams.stdoutLines[0]).toEqual(expected); +}); + +test.concurrent("include-inputs options array2 validation", async () => { + try { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/include-inputs/input-templates/options-array2-validation", + preview: true, + }, writeStreams); + } catch (e: any) { + assert(e instanceof AssertionError, "e is not instanceof AssertionError"); + expect(e.message).toContain("This GitLab CI configuration is invalid:"); + expect(e.message).toContain( + chalk`\`{blueBright options_input}\` input: \`{blueBright fizz}\` cannot be used because it is not in the list of allowed options.`, + ); + return; + } + + throw new Error("Error is expected but not thrown/caught"); +}); + test.concurrent("include-inputs too many functions in interpolation block", async () => { const writeStreams = new WriteStreamsMock(); try {