From dc2b57bdf4d8d46720871c64db5ad0e883535a5c Mon Sep 17 00:00:00 2001 From: Cyrille Pontvieux Date: Tue, 16 Jun 2026 09:46:50 +0200 Subject: [PATCH] New optional `.gitlab-ci-local-ignores` file to list file to ignore to sync with jobs. The file could also be specified on command line. Format should follow the `rsync` ignore format. Also never sync the `.git/lfs` directory which, if it exists, is usually huge. --- src/argv.ts | 5 +++ src/handler.ts | 6 ++-- src/index.ts | 6 ++++ src/job.ts | 2 +- src/utils.ts | 13 +++++-- .../project-ignores-file/.gitlab-ci.yml | 5 +++ .../gitlab-ci-local-ignores-test-job | 1 + .../project-ignores-file/integration.test.ts | 34 +++++++++++++++++++ .../project-ignores-file/test-big-file.txt | 1 + .../project-ignores-file/test-small-file.txt | 1 + 10 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 tests/test-cases/project-ignores-file/.gitlab-ci.yml create mode 100644 tests/test-cases/project-ignores-file/gitlab-ci-local-ignores-test-job create mode 100644 tests/test-cases/project-ignores-file/integration.test.ts create mode 100644 tests/test-cases/project-ignores-file/test-big-file.txt create mode 100644 tests/test-cases/project-ignores-file/test-small-file.txt diff --git a/src/argv.ts b/src/argv.ts index 4c30e7cb4..7bfe803f6 100644 --- a/src/argv.ts +++ b/src/argv.ts @@ -60,6 +60,7 @@ export function injectGclVariableEnvVars (argv: {variable?: string[]; [key: stri export class Argv { static readonly default = { "variablesFile": ".gitlab-ci-local-variables.yml", + "ignoresFile": ".gitlab-ci-local-ignores", "inputsFile": ".gitlab-ci-local-inputs.yml", "evaluateRuleChanges": true, "ignoreSchemaPaths": [] as string[], @@ -170,6 +171,10 @@ export class Argv { return this.map.get("inputsFile") ?? Argv.default.inputsFile; } + get ignoresFile (): string { + return this.map.get("ignoresFile") ?? Argv.default.ignoresFile; + } + get evaluateRuleChanges (): boolean { return this.map.get("evaluateRuleChanges") ?? Argv.default.evaluateRuleChanges; } diff --git a/src/handler.ts b/src/handler.ts index 206a29e24..336baad76 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -81,7 +81,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ pipelineIid = await state.getPipelineIid(cwd, stateDir); } parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); - await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); + await Utils.rsyncTrackedFiles(cwd, stateDir, path.resolve(cwd, argv.ignoresFile), ".docker"); await Commander.runJobs(argv, parser, writeStreams); if (argv.needs || argv.onlyNeeds) { writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); @@ -94,7 +94,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ const time = process.hrtime(); const pipelineIid = await state.getPipelineIid(cwd, stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); - await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); + await Utils.rsyncTrackedFiles(cwd, stateDir, path.resolve(cwd, argv.ignoresFile), ".docker"); await Commander.runJobsInStage(argv, parser, writeStreams); writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } else { @@ -105,7 +105,7 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ const time = process.hrtime(); const pipelineIid = await state.incrementPipelineIid(cwd, stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); - await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); + await Utils.rsyncTrackedFiles(cwd, stateDir, path.resolve(cwd, argv.ignoresFile), ".docker"); await Commander.runPipeline(argv, parser, writeStreams); if (childPipelineDepth == 0) writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } diff --git a/src/index.ts b/src/index.ts index b57f9ab24..874f14366 100644 --- a/src/index.ts +++ b/src/index.ts @@ -138,6 +138,12 @@ process.on("SIGUSR2", async () => { requiresArg: true, default: Argv.default.inputsFile, }) + .option("ignores-file", { + type: "string", + description: "Path to an ignores file", + requiresArg: true, + default: Argv.default.ignoresFile, + }) .option("completion", { type: "boolean", description: "Generate tab completion script", diff --git a/src/job.ts b/src/job.ts index 76bafba33..a42b0d323 100644 --- a/src/job.ts +++ b/src/job.ts @@ -920,7 +920,7 @@ If you know what you're doing and would like to suppress this warning, use one o // Copy git tracked files to build folder if shell isolation enabled. if (!imageName && this.argv.shellIsolation) { - await Utils.rsyncTrackedFiles(cwd, stateDir, `${safeJobName}`); + await Utils.rsyncTrackedFiles(cwd, stateDir, path.resolve(cwd, this.argv.ignoresFile), `${safeJobName}`); } if (this.interactive) { diff --git a/src/utils.ts b/src/utils.ts index 6a0c4e2ff..055cc84e4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -378,10 +378,19 @@ export class Utils { return !relative.startsWith(".."); } - static async rsyncTrackedFiles (cwd: string, stateDir: string, target: string): Promise<{hrdeltatime: [number, number]}> { + static async rsyncTrackedFiles (cwd: string, stateDir: string, ignoresFile: string, target: string): Promise<{hrdeltatime: [number, number]}> { const time = process.hrtime(); await fs.mkdirp(`${cwd}/${stateDir}/builds/${target}`); - await Utils.bash(`rsync -a --delete-excluded --delete --exclude-from=<(git ls-files -o --directory | awk '{print "/"$0}') --exclude ${stateDir}/ ./ ${stateDir}/builds/${target}/`, cwd); + const cmd = [ + "rsync -a --delete-excluded --delete", + "--exclude-from=<(git ls-files -o --directory | awk '{print \"/\"$0}')", + ...await fs.pathExists(ignoresFile) ? [`--exclude-from=${Utils.safeBashString(ignoresFile)}`] : [], + "--exclude .git/lfs", + `--exclude ${stateDir}/`, + "./", + `${stateDir}/builds/${target}/`, + ].join(" "); + await Utils.bash(cmd, cwd); return {hrdeltatime: process.hrtime(time)}; } diff --git a/tests/test-cases/project-ignores-file/.gitlab-ci.yml b/tests/test-cases/project-ignores-file/.gitlab-ci.yml new file mode 100644 index 000000000..125a9156c --- /dev/null +++ b/tests/test-cases/project-ignores-file/.gitlab-ci.yml @@ -0,0 +1,5 @@ +--- +test-job: + script: + - find . -type f + - find $STATE_DIR -type f -name '*.txt' -print | wc -l diff --git a/tests/test-cases/project-ignores-file/gitlab-ci-local-ignores-test-job b/tests/test-cases/project-ignores-file/gitlab-ci-local-ignores-test-job new file mode 100644 index 000000000..76853fa42 --- /dev/null +++ b/tests/test-cases/project-ignores-file/gitlab-ci-local-ignores-test-job @@ -0,0 +1 @@ +*-big-* diff --git a/tests/test-cases/project-ignores-file/integration.test.ts b/tests/test-cases/project-ignores-file/integration.test.ts new file mode 100644 index 000000000..9b5a73c45 --- /dev/null +++ b/tests/test-cases/project-ignores-file/integration.test.ts @@ -0,0 +1,34 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; + +const cwd = "tests/test-cases/project-ignores-file"; +const ignoresFile = "gitlab-ci-local-ignores-test-job"; + +test.concurrent("project-no-ignores-file", async () => { + const writeStreams = new WriteStreamsMock(); + const stateDir = ".gitlab-ci-local-no-ignores-file"; + await handler({ + cwd, + job: ["test-job"], + noColor: true, + stateDir, + variable: [`STATE_DIR=${stateDir}`], + }, writeStreams); + const expected = ["test-job > 2"]; + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); + +test.concurrent("project-ignores-file", async () => { + const writeStreams = new WriteStreamsMock(); + const stateDir = ".gitlab-ci-local-ignores-file"; + await handler({ + cwd, + job: ["test-job"], + ignoresFile, + noColor: true, + stateDir, + variable: [`STATE_DIR=${stateDir}`], + }, writeStreams); + const expected = ["test-job > 1"]; + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); diff --git a/tests/test-cases/project-ignores-file/test-big-file.txt b/tests/test-cases/project-ignores-file/test-big-file.txt new file mode 100644 index 000000000..707f160f4 --- /dev/null +++ b/tests/test-cases/project-ignores-file/test-big-file.txt @@ -0,0 +1 @@ +I should be ignored diff --git a/tests/test-cases/project-ignores-file/test-small-file.txt b/tests/test-cases/project-ignores-file/test-small-file.txt new file mode 100644 index 000000000..f539d72df --- /dev/null +++ b/tests/test-cases/project-ignores-file/test-small-file.txt @@ -0,0 +1 @@ +I should be rsynced