New optional .gitlab-ci-local-ignores file to list file to ignore to sync with jobs.

The file could also be specified on command line.

Format should follow the `rsync` ignore format.

Also never sync the `.git/lfs` directory which, if it exists, is usually
huge.

Author: jrd

src/argv.ts

@@ -60,6 +60,7 @@ export function injectGclVariableEnvVars (argv: {variable?: string[]; [key: stri
 export class Argv {
     static readonly default = {
         "variablesFile": ".gitlab-ci-local-variables.yml",
+        "ignoresFile": ".gitlab-ci-local-ignores",
         "inputsFile": ".gitlab-ci-local-inputs.yml",
         "evaluateRuleChanges": true,
         "ignoreSchemaPaths": [] as string[],
@@ -170,6 +171,10 @@ export class Argv {
         return this.map.get("inputsFile") ?? Argv.default.inputsFile;
     }
 
+    get ignoresFile (): string {
+        return this.map.get("ignoresFile") ?? Argv.default.ignoresFile;
+    }
+
     get evaluateRuleChanges (): boolean {
         return this.map.get("evaluateRuleChanges") ?? Argv.default.evaluateRuleChanges;
     }

src/handler.ts

@@ -81,7 +81,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
             pipelineIid = await state.getPipelineIid(cwd, stateDir);
         }
         parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
-        await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker");
+        await Utils.rsyncTrackedFiles(cwd, stateDir, `${cwd}/${argv.ignoresFile}`, ".docker");
         await Commander.runJobs(argv, parser, writeStreams);
         if (argv.needs || argv.onlyNeeds) {
             writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
@@ -94,7 +94,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
         const time = process.hrtime();
         const pipelineIid = await state.getPipelineIid(cwd, stateDir);
         parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
-        await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker");
+        await Utils.rsyncTrackedFiles(cwd, stateDir, `${cwd}/${argv.ignoresFile}`, ".docker");
         await Commander.runJobsInStage(argv, parser, writeStreams);
         writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
     } else {
@@ -105,7 +105,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
         const time = process.hrtime();
         const pipelineIid = await state.incrementPipelineIid(cwd, stateDir);
         parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
-        await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker");
+        await Utils.rsyncTrackedFiles(cwd, stateDir, `${cwd}/${argv.ignoresFile}`, ".docker");
         await Commander.runPipeline(argv, parser, writeStreams);
         if (childPipelineDepth == 0) writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
     }

src/index.ts

@@ -138,6 +138,12 @@ process.on("SIGUSR2", async () => {
             requiresArg: true,
             default: Argv.default.inputsFile,
         })
+        .option("ignores-file", {
+            type: "string",
+            description: "Path to a ignores file",
+            requiresArg: true,
+            default: Argv.default.ignoresFile,
+        })
         .option("completion", {
             type: "boolean",
             description: "Generate tab completion script",

src/job.ts

@@ -920,7 +920,7 @@ If you know what you're doing and would like to suppress this warning, use one o
 
         // Copy git tracked files to build folder if shell isolation enabled.
         if (!imageName && this.argv.shellIsolation) {
-            await Utils.rsyncTrackedFiles(cwd, stateDir, `${safeJobName}`);
+            await Utils.rsyncTrackedFiles(cwd, stateDir, `${cwd}/${this.argv.ignoresFile}`, `${safeJobName}`);
         }
 
         if (this.interactive) {

src/utils.ts

@@ -377,10 +377,19 @@ export class Utils {
         return !relative.startsWith("..");
     }
 
-    static async rsyncTrackedFiles (cwd: string, stateDir: string, target: string): Promise<{hrdeltatime: [number, number]}> {
+    static async rsyncTrackedFiles (cwd: string, stateDir: string, ignoresFile: string, target: string): Promise<{hrdeltatime: [number, number]}> {
         const time = process.hrtime();
         await fs.mkdirp(`${cwd}/${stateDir}/builds/${target}`);
-        await Utils.bash(`rsync -a --delete-excluded --delete --exclude-from=<(git ls-files -o --directory | awk '{print "/"$0}') --exclude ${stateDir}/ ./ ${stateDir}/builds/${target}/`, cwd);
+        const cmd = [
+            "rsync -a --delete-excluded --delete",
+            `--exclude-from=<(git ls-files -o --directory | awk '{print "/"$0}')`, // eslint-disable-line @stylistic/quotes
+            `--exclude-from=<(cat ${Utils.safeBashString(ignoresFile)} 2>/dev/null || true)`,
+            "--exclude .git/lfs",
+            `--exclude ${stateDir}/`,
+            "./",
+            `${stateDir}/builds/${target}/`,
+        ].join(" ");
+        await Utils.bash(cmd, cwd);
         return {hrdeltatime: process.hrtime(time)};
     }
 

tests/test-cases/project-ignores-file/.gitlab-ci.yml

@@ -0,0 +1,5 @@
+---
+test-job:
+  script:
+    - tree -a
+    - find $STATE_DIR -type f -name '*.txt' -print | wc -l

tests/test-cases/project-ignores-file/gitlab-ci-local-ignores-test-job

@@ -0,0 +1 @@
+*-big-*

tests/test-cases/project-ignores-file/integration.test.ts

@@ -0,0 +1,34 @@
+import {WriteStreamsMock} from "../../../src/write-streams.js";
+import {handler} from "../../../src/handler.js";
+
+const cwd = "tests/test-cases/project-ignores-file";
+const ignoresFile = "gitlab-ci-local-ignores-test-job";
+
+test.concurrent("project-no-ignores-file", async () => {
+    const writeStreams = new WriteStreamsMock();
+    const stateDir = ".gitlab-ci-local-no-ignores-file";
+    await handler({
+        cwd,
+        job: ["test-job"],
+        noColor: true,
+        stateDir,
+        variable: [`STATE_DIR=${stateDir}`],
+    }, writeStreams);
+    const expected = ["test-job > 2"];
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected));
+});
+
+test.concurrent("project-ignores-file", async () => {
+    const writeStreams = new WriteStreamsMock();
+    const stateDir = ".gitlab-ci-local-ignores-file";
+    await handler({
+        cwd,
+        job: ["test-job"],
+        ignoresFile,
+        noColor: true,
+        stateDir,
+        variable: [`STATE_DIR=${stateDir}`],
+    }, writeStreams);
+    const expected = ["test-job > 1"];
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected));
+});

tests/test-cases/project-ignores-file/test-big-file.txt

@@ -0,0 +1 @@
+I should be ignored

tests/test-cases/project-ignores-file/test-small-file.txt

@@ -0,0 +1 @@
+I should be rsynced