Skip to content

feat: added parsing and interpolation for component.* variables #1839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
N1lzh wants to merge 7 commits into
base: master
Choose a base branch
from
+124 −38
Open

feat: added parsing and interpolation for component.* variables #1839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ae20b31
added parsing and interpolation for component.* variables
nhoff-inform Apr 22, 2026
f919e7f
reverted personal changes
nhoff-inform Apr 22, 2026
04a1e1f
reverted to original bun.lock
nhoff-inform Apr 22, 2026
36e5670
fix: do not reject valid component.version when null by asserting not…
nhoff-inform Apr 24, 2026
022bea9
fix: handle commit sha component refs and return correct sha for anno…
nhoff-inform Apr 24, 2026
f9c3b3a
fix: wrap interpolated values with JSON.stringify before inserting
nhoff-inform Apr 24, 2026
d9852a3
fix: reverted changes again
nhoff-inform Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 92 additions & 36 deletions src/parser-includes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,12 @@ type ParsedComponent = {
domain: string;
port: string;
projectPath: string;
componentPath: string;
effectiveRef: string;
name: string;
ref: string;
reference: string;
version: string | undefined;
sha: string;
isLocal: boolean;
};

Expand Down Expand Up @@ -96,11 +100,11 @@ export class ParserIncludes {
} else if (value["remote"]) {
promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes, writeStreams));
} else if (value["component"]) {
const component = this.parseIncludeComponent(value["component"], gitData);
const component = this.parseIncludeComponent(value["component"], gitData, opts);
componentParseCache.set(index, component);
if (!component.isLocal)
{
promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.ref, component.name));
promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.effectiveRef, component.componentPath));
}
}

Expand Down Expand Up @@ -140,15 +144,16 @@ export class ParserIncludes {
}
} else if (value["component"]) {
const component = componentParseCache.get(index);

assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`);
// Gitlab allows two different file paths to include a component
const files = [`${component.name}.yml`, `${component.name}/template.yml`];
const files = [`${component.componentPath}.yml`, `${component.componentPath}/template.yml`];

let file = null;
for (const f of files) {
let searchPath = `${cwd}/${f}`;
if (!component.isLocal) {
searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`;
searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.effectiveRef}/${f}`;
}
if (fs.existsSync(searchPath)) {
file = searchPath;
Expand All @@ -158,13 +163,13 @@ export class ParserIncludes {
(component.port ? `:${component.port}` : "") + `/${component.projectPath}\``);

// Extract component name for component-specific inputs
const componentName = component.name.replace(/^templates\//, "");
const componentName = component.componentPath.replace(/^templates\//, "");
const fileComponentInputs = isStructured ? (fileInputs[componentName] ?? {}) : {};
const cliComponentSpecificInputs = cliComponentInputs[componentName] ?? {};
const mergedInputs = {...(value.inputs ?? {}), ...globalInputs, ...fileComponentInputs, ...cliComponentSpecificInputs};
const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs}, expandVariables, writeStreams);
const fileDoc = await Parser.loadYaml(file, {inputs: mergedInputs, component}, expandVariables, writeStreams);
// Expand local includes inside to a "project"-like include
fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts);
fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.effectiveRef, opts);
includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
} else if (value["template"]) {
const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]);
Expand Down Expand Up @@ -232,46 +237,42 @@ export class ParserIncludes {
};
}

static parseIncludeComponent (component: string, gitData: GitData): ParsedComponent {
static parseIncludeComponent (component: string, gitData: GitData, opts: ParserIncludesInitOptions): ParsedComponent {
assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
const pattern = /(?<domain>[^/:\s]+)(:(?<port>\d+))?\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // https://regexr.com/7v7hm
const gitRemoteMatch = pattern.exec(component);

if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`);

const {domain, projectPath, port} = gitRemoteMatch.groups;
let ref = gitRemoteMatch.groups["ref"];
const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && ref === gitData.commit.SHA;

if (!isLocalComponent) {
const semanticVersionRangesPattern = /^\d+(\.\d+)?$/;
if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) {
// https://docs.gitlab.com/ci/components/#semantic-version-ranges
let stdout;
if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") {
stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout;
} else {
stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout;
}
assert(stdout);
const tags = stdout
.split("\n")
.map((line) => {
return line
.split("\t")[1]
.split("/")[2];
});
const _ref = resolveSemanticVersionRange(ref, tags);
assert(_ref, `This GitLab CI configuration is invalid: component: \`${component}\` - The ref (${ref}) is invalid`);
ref = _ref;
}

let reference = gitRemoteMatch.groups["ref"];
const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && reference === gitData.commit.SHA;

const context = {
isLocal: isLocalComponent,
projectPath: projectPath,
remote: gitData.remote,
domain: domain,
port: port,
component: component,
cwd: opts.cwd
}
let version = getComponentVersion(context, reference);
let effectiveRef = version ?? reference;
let sha = getComponentSha(context, effectiveRef);
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
let name = gitRemoteMatch.groups["componentName"];

return {
domain: domain,
port: port,
projectPath: projectPath,
name: `templates/${gitRemoteMatch.groups["componentName"]}`,
ref: ref,
componentPath: `templates/${name}`,
effectiveRef: version ?? reference,
name: name,
reference: reference,
sha: sha,
version: version,
isLocal: isLocalComponent,
};
}
Expand Down Expand Up @@ -458,3 +459,58 @@ export async function resolveIncludeLocal (pattern: string, cwd: string) {
const re2js = RE2JS.compile(`^${pattern}`);
return repoFiles.filter((f: any) => re2js.matches(f));
}

export function getComponentVersion(ctx: any, reference: string) {
if (!ctx.isLocal) {
const semanticVersionRangesPattern = /^\d+(\.\d+)?$/;
if (reference == "~latest" || semanticVersionRangesPattern.test(reference)) {
// https://docs.gitlab.com/ci/components/#semantic-version-ranges
let stdout;
if (ctx.remote.schema == "git" || ctx.remote.schema == "ssh") {
stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${ctx.domain}:${ctx.projectPath}`]).stdout;
} else {
stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${ctx.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`]).stdout;
}
assert(stdout);
const tags = stdout
.split("\n")
.map((line) => {
return line
.split("\t")[1]
.split("/")[2];
});
const version = resolveSemanticVersionRange(reference, tags);
assert(version, `This GitLab CI configuration is invalid: component: \`${ctx.component}\` - The ref (${reference}) is invalid`);
return version;
}
}

return undefined;
}

export function getComponentSha(ctx: any, effectiveRef: string) {
if (ctx.isLocal) {
return Utils.syncSpawn(["git", "rev-parse", effectiveRef], ctx.cwd).stdout.trimEnd();
}

// effectiveRef may already be a sha, if so return it directly
if (/^[0-9a-f]{40}$/.test(effectiveRef)) {
return effectiveRef;
}

let stdout;
if (ctx.remote.schema == "git" || ctx.remote.schema == "ssh") {
stdout = Utils.syncSpawn(["git", "ls-remote", `git@${ctx.domain}:${ctx.projectPath}`]).stdout;
} else {
stdout = Utils.syncSpawn(["git", "ls-remote", `${ctx.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`]).stdout;
}

const match = stdout.split("\n").find(line =>
Comment thread
N1lzh marked this conversation as resolved.
line.endsWith(`refs/tags/${effectiveRef}^{} `) ||
line.endsWith(`refs/tags/${effectiveRef}`) ||
line.endsWith(`refs/heads/${effectiveRef}`)
);

assert(match, `Could not resolve commit SHA for ${effectiveRef} in ${ctx.projectPath}`);
return match.split("\t")[0];
}
34 changes: 32 additions & 2 deletions src/parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ export class Parser {
const inputsSpecification: any = fileData[0];

@jrd jrd Apr 29, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be also be parsed for component.XXX interpolation.

const uninterpolatedConfigurations: any = fileData[1];

const interpolatedConfigurations = JSON.stringify(uninterpolatedConfigurations)
const configurationWithInterpolatedInputs = JSON.stringify(uninterpolatedConfigurations)
.replaceAll(
/(?<firstChar>.)?(?<secondChar>.)?\$\[\[\s*inputs.(?<interpolationKey>[\w-]+)\s*\|?\s*(?<interpolationFunctions>.*?)\s*\]\](?<lastChar>[^$])?/g // https://regexr.com/81c16
, (_: string, firstChar: string, secondChar: string, interpolationKey: string, interpolationFunctions: string, lastChar: string) => {
Expand Down Expand Up @@ -372,7 +372,37 @@ export class Parser {
Utils.switchStatementExhaustiveCheck(inputType);
}
});
return JSON.parse(interpolatedConfigurations);

// NOTE: Replacement of the component interpolation keys behaves slightly different. It currently does not check whether the interpolation key
// is specified in the specs via component: [{interpolationKey}, ...]
const configurationWithInterpolatedInputsAndComponent = configurationWithInterpolatedInputs
.replaceAll(
/(?<firstChar>.)?(?<secondChar>.)?\$\[\[\s*component.(?<interpolationKey>[\w-]+)\s*\]\](?<lastChar>[^$])?/g,

@jrd jrd Apr 29, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

firstChar, secondChar and lastChar are useless for component interpolation as this is just a raw replace and there is no array or object type. This makes it more complicated that it should.

(_, firstChar, secondChar, interpolationKey, lastChar) => {
firstChar ??= "";
secondChar ??= "";
lastChar ??= "";

const { component } = ctx;
if (!component) return _;

let componentValue
const properties = [ "name", "reference", "version", "sha" ];
let foundKey = false;
properties.forEach(property => {
if (property === interpolationKey) {
foundKey = true;
componentValue = component[interpolationKey];
}
})

assert(foundKey, chalk`This GitLab CI configuration is invalid: \`{blueBright ${ctx.configFilePath}}\`: unknown interpolation key: \`${interpolationKey}\`.`);
return firstChar + secondChar + componentValue + lastChar;
}
);


return JSON.parse(configurationWithInterpolatedInputsAndComponent);
}
return fileData[0];
}
Expand Down
Loading