Allow to use component.reference, component.sha, component.version and component.name (#1836)

Co-authored-by: Mads Jon Nielsen <madsjon@gmail.com>

Author: jrd

src/parser-includes.ts

@@ -26,14 +26,31 @@ type ParserIncludesInitOptions = {
 };
 
 type ParsedComponent = {
+    _cache: {
+        version: string | null | undefined;
+        effectiveRef: string | undefined;
+        sha: string | undefined;
+    };
+    gitData: GitData;
     domain: string;
     port: string;
     projectPath: string;
+    componentPath: string;
     name: string;
-    ref: string;
+    reference: string;
+    version: string | null;
+    effectiveRef: string;
+    sha: string;
     isLocal: boolean;
 };
 
+type GitRemoteInfoContext = {
+    gitData: GitData;
+    domain: string;
+    port: string;
+    projectPath: string;
+};
+
 export class ParserIncludes {
     private static count: number = 0;
 
@@ -100,7 +117,7 @@ export class ParserIncludes {
                 componentParseCache.set(index, component);
                 if (!component.isLocal)
                 {
-                    promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.ref, component.name));
+                    promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.effectiveRef, component.componentPath));
                 }
             }
 
@@ -142,13 +159,13 @@ export class ParserIncludes {
                 const component = componentParseCache.get(index);
                 assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`);
                 // Gitlab allows two different file paths to include a component
-                const files = [`${component.name}.yml`, `${component.name}/template.yml`];
+                const files = [`${component.componentPath}.yml`, `${component.componentPath}/template.yml`];
 
                 let file = null;
                 for (const f of files) {
                     let searchPath = `${cwd}/${f}`;
                     if (!component.isLocal) {
-                        searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`;
+                        searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.effectiveRef}/${f}`;
                     }
                     if (fs.existsSync(searchPath)) {
                         file = searchPath;
@@ -158,13 +175,15 @@ export class ParserIncludes {
                                     (component.port ? `:${component.port}` : "") + `/${component.projectPath}\``);
 
                 // Extract component name for component-specific inputs
-                const componentName = component.name.replace(/^templates\//, "");
+                const componentName = component.componentPath.replace(/^templates\//, "");
                 const fileComponentInputs = isStructured ? (fileInputs[componentName] ?? {}) : {};
                 const cliComponentSpecificInputs = cliComponentInputs[componentName] ?? {};
                 const mergedInputs = {...(value.inputs ?? {}), ...globalInputs, ...fileComponentInputs, ...cliComponentSpecificInputs};
-                const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs}, expandVariables, writeStreams);
-                // Expand local includes inside to a "project"-like include
-                fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts);
+                const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs, component}, expandVariables, writeStreams);
+                if (!component.isLocal) {
+                    // Expand local includes inside to a "project"-like include
+                    fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.effectiveRef, opts);
+                }
                 includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
             } else if (value["template"]) {
                 const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]);
@@ -236,42 +255,70 @@ export class ParserIncludes {
         assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
         const pattern = /(?<domain>[^/:\s]+)(:(?<port>\d+))?\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // https://regexr.com/7v7hm
         const gitRemoteMatch = pattern.exec(component);
-
         if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`);
-
-        const {domain, projectPath, port} = gitRemoteMatch.groups;
-        let ref = gitRemoteMatch.groups["ref"];
+        const {domain, projectPath, port, componentName, ref} = gitRemoteMatch.groups;
         const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && ref === gitData.commit.SHA;
-
-        if (!isLocalComponent) {
-            const semanticVersionRangesPattern = /^\d+(\.\d+)?$/;
-            if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) {
-                // https://docs.gitlab.com/ci/components/#semantic-version-ranges
-                let stdout;
-                if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") {
-                    stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout;
-                } else {
-                    stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout;
-                }
-                assert(stdout);
-                const tags = stdout
-                    .split("\n")
-                    .map((line) => {
-                        return line
-                            .split("\t")[1]
-                            .split("/")[2];
-                    });
-                const _ref = resolveSemanticVersionRange(ref, tags);
-                assert(_ref, `This GitLab CI configuration is invalid: component: \`${component}\` - The ref (${ref}) is invalid`);
-                ref = _ref;
-            }
-        }
         return {
-            domain: domain,
-            port: port,
-            projectPath: projectPath,
-            name: `templates/${gitRemoteMatch.groups["componentName"]}`,
-            ref: ref,
+            _cache: {
+                version: undefined,
+                effectiveRef: undefined,
+                sha: undefined,
+            },
+            gitData,
+            domain,
+            port,
+            projectPath,
+            componentPath: `templates/${componentName}`,
+            name: componentName,
+            reference: ref,
+            get version () {
+                if (this._cache.version === undefined) {
+                    if (this.isLocal) {
+                        this._cache.version = this.gitData.commit.SHA;
+                    } else {
+                        const semanticVersionRangesPattern = /^\d+(\.\d+)?$/;
+                        if (this.reference == "~latest" || semanticVersionRangesPattern.test(this.reference)) {
+                            // https://docs.gitlab.com/ci/components/#semantic-version-ranges
+                            const stdout = getGitRemoteInfo(this, "--tags");
+                            const tags = stdout.split("\n").map(line => line.split("\t")[1].split("/")[2]);
+                            const version = resolveSemanticVersionRange(this.reference, tags);
+                            assert(version, `This GitLab CI configuration is invalid: component: \`${this.name}\` - The reference (${this.reference}) is invalid`);
+                            this._cache.version = version;
+                        } else {
+                            this._cache.version = null;
+                        }
+                    }
+                }
+                return this._cache.version;
+            },
+            get effectiveRef () {
+                if (this._cache.effectiveRef === undefined) {
+                    this._cache.effectiveRef = this.version ?? this.reference;
+                }
+                return this._cache.effectiveRef;
+            },
+            get sha () {
+                if (this._cache.sha === undefined) {
+                    if (this.isLocal) {
+                        this._cache.sha = this.gitData.commit.SHA;
+                    } else if (/^[0-9a-f]{40}$/.test(this.effectiveRef)) {
+                        // effectiveRef may already be a sha, if so return it directly
+                        this._cache.sha = this.effectiveRef;
+                    } else {
+                        const stdout = getGitRemoteInfo(this);
+                        const lines = stdout.split("\n");
+                        // annotated tags: prefer the deref'd commit sha (refs/tags/x^{})
+                        const match = lines.find(line => line.endsWith(`refs/tags/${this.effectiveRef}^{}`)) ??
+                            lines.find(line =>
+                                line.endsWith(`refs/tags/${this.effectiveRef}`) ||
+                                line.endsWith(`refs/heads/${this.effectiveRef}`),
+                            );
+                        assert(match, `Could not resolve commit SHA for ${this.effectiveRef} in ${this.projectPath}`);
+                        this._cache.sha = match.split("\t")[0];
+                    }
+                }
+                return this._cache.sha;
+            },
             isLocal: isLocalComponent,
         };
     }
@@ -458,3 +505,13 @@ export async function resolveIncludeLocal (pattern: string, cwd: string) {
     const re2js = RE2JS.compile(`^${pattern}`);
     return repoFiles.filter((f: any) => re2js.matches(f));
 }
+
+export function getGitRemoteInfo (ctx: GitRemoteInfoContext, ...args: string[]) {
+    const cmdArgs = ["git", "ls-remote", ...args];
+    if (ctx.gitData.remote.schema == "git" || ctx.gitData.remote.schema == "ssh") {
+        cmdArgs.push(`git@${ctx.domain}:${ctx.projectPath}`);
+    } else {
+        cmdArgs.push(`${ctx.gitData.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`);
+    }
+    return Utils.syncSpawn(cmdArgs).stdout;
+}

src/parser.ts

@@ -354,7 +354,6 @@ export class Parser {
         if (isGitlabSpecFile(fileData[0])) {
             const inputsSpecification: any = fileData[0];
             const uninterpolatedConfigurations: any = fileData[1];
-
             const interpolatedConfigurations = JSON.stringify(uninterpolatedConfigurations)
                 .replaceAll(
                     /(?<firstChar>.)?(?<secondChar>.)?\$\[\[\s*inputs.(?<interpolationKey>[\w-]+)\s*\|?\s*(?<interpolationFunctions>.*?)\s*\]\](?<lastChar>[^$])?/g // https://regexr.com/81c16
@@ -398,6 +397,11 @@ export class Parser {
                             default:
                                 Utils.switchStatementExhaustiveCheck(inputType);
                         }
+                    })
+                .replaceAll( // https://docs.gitlab.com/ci/components/#use-component-context-in-components
+                    /\$\[\[\s*component\.(?<interpolationKey>name|reference|version|sha)\s*\]\]/g // regexr.com/8lotc
+                    , (_: string, interpolationKey: string) => {
+                        return getComponentValue(filePath, ctx, interpolationKey) || _;
                     });
             return JSON.parse(interpolatedConfigurations);
         }
@@ -409,6 +413,12 @@ function isGitlabSpecFile (fileData: any) {
     return "spec" in fileData;
 }
 
+function getComponentValue (filePath: string, ctx: any, interpolationKey: string) {
+    const {component} = ctx;
+    assert(component !== undefined, chalk`This GitLab CI configuration is invalid: \`{blueBright ${filePath}}\`: \`{blueBright component.${interpolationKey}}\` cannot be used outside a component.`);
+    return component[interpolationKey];
+}
+
 function validateInterpolationKey (ctx: any) {
     const {configFilePath, interpolationKey, inputsSpecification} = ctx;
     const invalidInterpolationKeyErr = chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: unknown interpolation key: \`${interpolationKey}\`.`;
@@ -427,12 +437,6 @@ function validateInput (ctx: any) {
     const {configFilePath, interpolationKey, inputsSpecification} = ctx;
     const inputValue = getInputValue(ctx);
 
-    const options = inputsSpecification.spec.inputs[interpolationKey]?.options;
-    if (options) {
-        assert(options.includes(inputValue),
-            chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${inputValue}}\` cannot be used because it is not in the list of allowed options.`);
-    }
-
     const expectedInputType = getExpectedInputType(ctx);
     assert(INCLUDE_INPUTS_SUPPORTED_TYPES.includes(expectedInputType),
         chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: header:spec:inputs:{blueBright ${interpolationKey}} input type unknown value: {blueBright ${expectedInputType}}.`);
@@ -441,6 +445,19 @@ function validateInput (ctx: any) {
     assert(inputType === expectedInputType,
         chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: provided value is not a {blueBright ${expectedInputType}}.`);
 
+    const options = inputsSpecification.spec.inputs[interpolationKey]?.options;
+    if (options) {
+        if (inputType == "array") {
+            for (const itemValue of inputValue) {
+                assert(options.includes(itemValue),
+                    chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${itemValue}}\` cannot be used because it is not in the list of allowed options.`);
+            }
+        } else {
+            assert(options.includes(inputValue),
+                chalk`This GitLab CI configuration is invalid: \`{blueBright ${configFilePath}}\`: \`{blueBright ${interpolationKey}}\` input: \`{blueBright ${inputValue}}\` cannot be used because it is not in the list of allowed options.`);
+        }
+    }
+
     const regex = inputsSpecification.spec.inputs[interpolationKey]?.regex;
     if (regex) {
         let re: RegExp;

tests/test-cases/include-component/component-local/templates/my-component.yml

@@ -5,5 +5,7 @@ spec:
       default: test
 ---
 component-job:
-  script: echo job 1
+  script:
+    - echo job 1 from $[[ component.name ]] ($[[ component.version ]]/$[[ component.reference ]]).
+    - Sha is $[[ component.sha ]]
   stage: $[[ inputs.stage ]]

tests/test-cases/include-component/integration.test.ts

@@ -3,6 +3,7 @@ import {handler} from "../../../src/handler.js";
 import assert, {AssertionError} from "assert";
 import {initSpawnSpy} from "../../mocks/utils.mock.js";
 import {WhenStatics} from "../../mocks/when-statics.js";
+import {Utils} from "../../../src/utils.js";
 
 beforeAll(() => {
     initSpawnSpy(WhenStatics.all);
@@ -175,22 +176,23 @@ test.concurrent("include-component component (protocol: https) (~latest semver)"
 });
 
 test.concurrent("include-component local component", async () => {
+    const sha = Utils.syncSpawn(["git", "rev-parse", "HEAD"]).stdout.trimEnd();;
     const writeStreams = new WriteStreamsMock();
 
     await handler({
         cwd: "tests/test-cases/include-component/component-local",
         preview: true,
         stateDir: ".gitlab-ci-local-include-component-local-component",
     }, writeStreams);
-
     const expected = `---
 stages:
   - .pre
   - my-stage
   - .post
 component-job:
   script:
-    - echo job 1
+    - echo job 1 from my-component (${sha}/${sha}).
+    - Sha is ${sha}
   stage: my-stage`;
 
     expect(writeStreams.stdoutLines[0]).toEqual(expected);

tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci-input-template.yml

@@ -0,0 +1,14 @@
+---
+spec:
+  inputs:
+    options_input:
+      type: array
+      default: []
+      options:
+        - foo
+        - bar
+        - baz
+---
+scan-website:
+  script:
+    - echo $[[ inputs.options_input ]]

tests/test-cases/include-inputs/input-templates/options-array1-validation/.gitlab-ci.yml

@@ -0,0 +1,7 @@
+---
+include:
+  - local: '/.gitlab-ci-input-template.yml'
+    inputs:
+      options_input: ["foo", "baz"]
+stages:
+  - test

tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci-input-template.yml

@@ -0,0 +1,14 @@
+---
+spec:
+  inputs:
+    options_input:
+      type: array
+      default: []
+      options:
+        - foo
+        - bar
+        - baz
+---
+scan-website:
+  script:
+    - echo $[[ inputs.options_input ]]

tests/test-cases/include-inputs/input-templates/options-array2-validation/.gitlab-ci.yml

@@ -0,0 +1,7 @@
+---
+include:
+  - local: '/.gitlab-ci-input-template.yml'
+    inputs:
+      options_input: ["bar", "fizz"]
+stages:
+  - test

tests/test-cases/include-inputs/integration.test.ts

@@ -321,6 +321,44 @@ test.concurrent("include-inputs options validation", async () => {
     throw new Error("Error is expected but not thrown/caught");
 });
 
+test.concurrent("include-inputs options array1 validation", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/include-inputs/input-templates/options-array1-validation",
+        preview: true,
+    }, writeStreams);
+
+    const expected = `---
+stages:
+  - .pre
+  - test
+  - .post
+scan-website:
+  script:
+    - echo ["foo","baz"]`;
+
+    expect(writeStreams.stdoutLines[0]).toEqual(expected);
+});
+
+test.concurrent("include-inputs options array2 validation", async () => {
+    try {
+        const writeStreams = new WriteStreamsMock();
+        await handler({
+            cwd: "tests/test-cases/include-inputs/input-templates/options-array2-validation",
+            preview: true,
+        }, writeStreams);
+    } catch (e: any) {
+        assert(e instanceof AssertionError, "e is not instanceof AssertionError");
+        expect(e.message).toContain("This GitLab CI configuration is invalid:");
+        expect(e.message).toContain(
+            chalk`\`{blueBright options_input}\` input: \`{blueBright fizz}\` cannot be used because it is not in the list of allowed options.`,
+        );
+        return;
+    }
+
+    throw new Error("Error is expected but not thrown/caught");
+});
+
 test.concurrent("include-inputs too many functions in interpolation block", async () => {
     const writeStreams = new WriteStreamsMock();
     try {