From 1da981ec1167eaff9518480c5034b0c037120770 Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Tue, 1 Jul 2025 11:38:30 +0200 Subject: [PATCH 1/2] fix(include-parser): use git ls-files instead of globby to memoize repository files --- src/job.ts | 2 +- src/parser-includes.ts | 13 ++++++------- src/utils.ts | 8 ++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/job.ts b/src/job.ts index e428fa4f5..560380d02 100644 --- a/src/job.ts +++ b/src/job.ts @@ -1546,7 +1546,7 @@ export class Job { if (include["local"]) { const expandedInclude = Utils.expandText(include["local"], this._variables); validateIncludeLocal(expandedInclude); - const files = resolveIncludeLocal(expandedInclude, cwd); + const files = await resolveIncludeLocal(expandedInclude, cwd); if (files.length == 0) { throw new AssertionError({message: `Local include file \`${include["local"]}\` specified in \`.${this.name}\` cannot be found!`}); } diff --git a/src/parser-includes.ts b/src/parser-includes.ts index f11fcf0a9..5f7bd9bdd 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -7,7 +7,6 @@ import assert, {AssertionError} from "assert"; import chalk from "chalk"; import {Parser} from "./parser.js"; import axios from "axios"; -import globby from "globby"; import path from "path"; import semver from "semver"; import {RE2JS} from "re2js"; @@ -93,9 +92,9 @@ export class ParserIncludes { } if (value["local"]) { validateIncludeLocal(value["local"]); - const files = resolveIncludeLocal(value["local"], cwd); + const files = await resolveIncludeLocal(value["local"], cwd); if (files.length == 0) { - throw new AssertionError({message: `Local include file cannot be found ${value["local"]}`}); + throw new AssertionError({message: `Local include file cannot be found ${cwd}${value["local"]}`}); } for (const localFile of files) { const content = await Parser.loadYaml(localFile, {inputs: value.inputs ?? {}}, expandVariables); @@ -327,11 +326,11 @@ export class ParserIncludes { static readonly memoLocalRepoFiles = (() => { const cache = new Map(); - return (path: string) => { + return async (path: string) => { let result = cache.get(path); if (typeof result !== "undefined") return result; - result = globby.sync(path, {dot: true, gitignore: true}); + result = (await Utils.getTrackedFiles(path)).map(p => `${path}/${p}`); cache.set(path, result); return result; }; @@ -361,8 +360,8 @@ export function resolveSemanticVersionRange (range: string, gitTags: string[]) { return found; } -export function resolveIncludeLocal (pattern: string, cwd: string) { - const repoFiles = ParserIncludes.memoLocalRepoFiles(cwd); +export async function resolveIncludeLocal (pattern: string, cwd: string) { + const repoFiles = await ParserIncludes.memoLocalRepoFiles(cwd); if (!pattern.startsWith("/")) pattern = `/${pattern}`; // Ensure pattern starts with `/` pattern = `${cwd}${pattern}`; diff --git a/src/utils.ts b/src/utils.ts index e61524bee..222d59822 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -434,4 +434,12 @@ export class Utils { // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f throw new Error(`Unhandled case ${param}`); } + + static async getTrackedFiles (cwd: string): Promise { + const lsFilesRes = await Utils.bash("git ls-files --deduplicate", cwd); + if (lsFilesRes.exitCode != 0) { + throw new Error(`Failed to list tracked files in ${cwd}: ${lsFilesRes.stderr}`); + } + return lsFilesRes.stdout.split("\n"); + } } From 86d6540de52a23442e2c04ea9445b07ca245283b Mon Sep 17 00:00:00 2001 From: SuperEwald Date: Tue, 1 Jul 2025 11:52:30 +0200 Subject: [PATCH 2/2] change back exception message to adhere with tests --- src/parser-includes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 5f7bd9bdd..24a897d61 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -94,7 +94,7 @@ export class ParserIncludes { validateIncludeLocal(value["local"]); const files = await resolveIncludeLocal(value["local"], cwd); if (files.length == 0) { - throw new AssertionError({message: `Local include file cannot be found ${cwd}${value["local"]}`}); + throw new AssertionError({message: `Local include file cannot be found ${value["local"]}`}); } for (const localFile of files) { const content = await Parser.loadYaml(localFile, {inputs: value.inputs ?? {}}, expandVariables);