fix(include-parser): infinite recursion or out-of-memory with symlinks/large projects (#1598)

* fix(include-parser): use git ls-files instead of globby to memoize
repository files

* change back exception message to adhere with tests

Author: superewald

src/job.ts

@@ -1546,7 +1546,7 @@ export class Job {
             if (include["local"]) {
                 const expandedInclude = Utils.expandText(include["local"], this._variables);
                 validateIncludeLocal(expandedInclude);
-                const files = resolveIncludeLocal(expandedInclude, cwd);
+                const files = await resolveIncludeLocal(expandedInclude, cwd);
                 if (files.length == 0) {
                     throw new AssertionError({message: `Local include file \`${include["local"]}\` specified in \`.${this.name}\` cannot be found!`});
                 }

src/parser-includes.ts

@@ -7,7 +7,6 @@ import assert, {AssertionError} from "assert";
 import chalk from "chalk";
 import {Parser} from "./parser.js";
 import axios from "axios";
-import globby from "globby";
 import path from "path";
 import semver from "semver";
 import {RE2JS} from "re2js";
@@ -93,7 +92,7 @@ export class ParserIncludes {
             }
             if (value["local"]) {
                 validateIncludeLocal(value["local"]);
-                const files = resolveIncludeLocal(value["local"], cwd);
+                const files = await resolveIncludeLocal(value["local"], cwd);
                 if (files.length == 0) {
                     throw new AssertionError({message: `Local include file cannot be found ${value["local"]}`});
                 }
@@ -327,11 +326,11 @@ export class ParserIncludes {
 
     static readonly memoLocalRepoFiles = (() => {
         const cache = new Map<string, string[]>();
-        return (path: string) => {
+        return async (path: string) => {
             let result = cache.get(path);
             if (typeof result !== "undefined") return result;
 
-            result = globby.sync(path, {dot: true, gitignore: true});
+            result = (await Utils.getTrackedFiles(path)).map(p => `${path}/${p}`);
             cache.set(path, result);
             return result;
         };
@@ -361,8 +360,8 @@ export function resolveSemanticVersionRange (range: string, gitTags: string[]) {
     return found;
 }
 
-export function resolveIncludeLocal (pattern: string, cwd: string) {
-    const repoFiles = ParserIncludes.memoLocalRepoFiles(cwd);
+export async function resolveIncludeLocal (pattern: string, cwd: string) {
+    const repoFiles = await ParserIncludes.memoLocalRepoFiles(cwd);
 
     if (!pattern.startsWith("/")) pattern = `/${pattern}`; // Ensure pattern starts with `/`
     pattern = `${cwd}${pattern}`;

src/utils.ts

@@ -434,4 +434,12 @@ export class Utils {
         // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f
         throw new Error(`Unhandled case ${param}`);
     }
+
+    static async getTrackedFiles (cwd: string): Promise<string[]> {
+        const lsFilesRes = await Utils.bash("git ls-files --deduplicate", cwd);
+        if (lsFilesRes.exitCode != 0) {
+            throw new Error(`Failed to list tracked files in ${cwd}: ${lsFilesRes.stderr}`);
+        }
+        return lsFilesRes.stdout.split("\n");
+    }
 }