Use some in-memory cache for git ls-remote identical calls

Author: jrd

src/parser-includes.ts

@@ -53,6 +53,7 @@ type GitRemoteInfoContext = {
 
 export class ParserIncludes {
     private static count: number = 0;
+    private static gitRemoteInfoCache: Record<string, string> = {};
 
     static resetCount (): void {
         this.count = 0;
@@ -114,13 +115,13 @@ export class ParserIncludes {
                 promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes, writeStreams));
             } else if (value["component"]) {
                 const component = this.parseIncludeComponent(value["component"], gitData);
-                componentParseCache.set(index, component);
-                if (!component.isLocal)
-                {
-                    promises.push(this.downloadIncludeComponent(opts, component.projectPath, component.effectiveRef, component.componentPath));
-                }
+                promises.push((async (componentParseCache: Map<number, ParsedComponent>, component: ParsedComponent, opts: ParserIncludesInitOptions) => {
+                    if (!component.isLocal) {
+                        await this.downloadIncludeComponent(opts, component.projectPath, component.effectiveRef, component.componentPath);
+                    }
+                    componentParseCache.set(index, component);
+                })(componentParseCache, component, opts));
             }
-
         }
 
         await Promise.all(promises);
@@ -251,13 +252,30 @@ export class ParserIncludes {
         };
     }
 
+    static getGitRemoteInfo (ctx: GitRemoteInfoContext, ...args: string[]): string {
+        const cmdArgs = ["git", "ls-remote", ...args];
+        if (ctx.gitData.remote.schema == "git" || ctx.gitData.remote.schema == "ssh") {
+            cmdArgs.push(`git@${ctx.domain}:${ctx.projectPath}`);
+        } else {
+            cmdArgs.push(`${ctx.gitData.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`);
+        }
+        const cmdStr = cmdArgs.join(" ");
+        let info = this.gitRemoteInfoCache[cmdStr];
+        if (!(cmdStr in this.gitRemoteInfoCache)) {
+            info = Utils.syncSpawn(cmdArgs).stdout;
+            this.gitRemoteInfoCache[cmdStr] = info;
+        }
+        return info;
+    };
+
     static parseIncludeComponent (component: string, gitData: GitData): ParsedComponent {
         assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
         const pattern = /(?<domain>[^/:\s]+)(:(?<port>\d+))?\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // https://regexr.com/7v7hm
         const gitRemoteMatch = pattern.exec(component);
         if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`);
         const {domain, projectPath, port, componentName, ref} = gitRemoteMatch.groups;
         const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && ref === gitData.commit.SHA;
+        const parserIncludes = this; // eslint-disable-line @typescript-eslint/no-this-alias
         return {
             _cache: {
                 version: undefined,
@@ -279,7 +297,7 @@ export class ParserIncludes {
                         const semanticVersionRangesPattern = /^\d+(\.\d+)?$/;
                         if (this.reference == "~latest" || semanticVersionRangesPattern.test(this.reference)) {
                             // https://docs.gitlab.com/ci/components/#semantic-version-ranges
-                            const stdout = getGitRemoteInfo(this, "--tags");
+                            const stdout = parserIncludes.getGitRemoteInfo(this, "--tags");
                             const tags = stdout.split("\n").map(line => line.split("\t")[1].split("/")[2]);
                             const version = resolveSemanticVersionRange(this.reference, tags);
                             assert(version, `This GitLab CI configuration is invalid: component: \`${this.name}\` - The reference (${this.reference}) is invalid`);
@@ -305,7 +323,7 @@ export class ParserIncludes {
                         // effectiveRef may already be a sha, if so return it directly
                         this._cache.sha = this.effectiveRef;
                     } else {
-                        const stdout = getGitRemoteInfo(this);
+                        const stdout = parserIncludes.getGitRemoteInfo(this);
                         const lines = stdout.split("\n");
                         // annotated tags: prefer the deref'd commit sha (refs/tags/x^{})
                         const match = lines.find(line => line.endsWith(`refs/tags/${this.effectiveRef}^{}`)) ??
@@ -505,13 +523,3 @@ export async function resolveIncludeLocal (pattern: string, cwd: string) {
     const re2js = RE2JS.compile(`^${pattern}`);
     return repoFiles.filter((f: any) => re2js.matches(f));
 }
-
-export function getGitRemoteInfo (ctx: GitRemoteInfoContext, ...args: string[]) {
-    const cmdArgs = ["git", "ls-remote", ...args];
-    if (ctx.gitData.remote.schema == "git" || ctx.gitData.remote.schema == "ssh") {
-        cmdArgs.push(`git@${ctx.domain}:${ctx.projectPath}`);
-    } else {
-        cmdArgs.push(`${ctx.gitData.remote.schema}://${ctx.domain}:${ctx.port ?? 443}/${ctx.projectPath}.git`);
-    }
-    return Utils.syncSpawn(cmdArgs).stdout;
-}