Remote variable array to support multiple variable files. (#1697)

* Refactor remoteVariables handling to support multiple values and update documentation

* Update src/argv.ts

Co-authored-by: Mads Jon Nielsen <madsjon@gmail.com>

* refactor: replace deepmerge with deep-extend for merging remote file data

---------

Co-authored-by: Mads Jon Nielsen <madsjon@gmail.com>

Author: bbrala

README.md

@@ -277,6 +277,8 @@ Variables will now appear in your jobs, if project or group matches git remote,
 
 ### Remote file variables
 
+You can also fetch variable files from remote git repositories. If you need multiple variable files, repeat the `--remote-variables` flag.
+
 ```shell
 gitlab-ci-local --remote-variables git@gitlab.com:firecow/example.git=gitlab-variables.yml=master
 ```

src/argv.ts

@@ -165,7 +165,7 @@ export class Argv {
         return this.map.get("pullPolicy") ?? "if-not-present";
     }
 
-    get remoteVariables (): string {
+    get remoteVariables (): string[] {
         return this.map.get("remoteVariables");
     }
 

src/index.ts

@@ -153,7 +153,7 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
             requiresArg: false,
         })
         .option("remote-variables", {
-            type: "string",
+            type: "array",
             description: "Fetch variables file from remote location",
             requiresArg: false,
         })

src/variables-from-files.ts

@@ -7,6 +7,7 @@ import {Argv} from "./argv.js";
 import assert from "assert";
 import {Utils} from "./utils.js";
 import dotenv from "dotenv";
+import deepExtend from "deep-extend";
 
 export interface CICDVariable {
     type: "file" | "variable";
@@ -33,13 +34,19 @@ export class VariablesFromFiles {
         let homeFileData: any = {};
 
         if (remoteVariables && !autoCompleting) {
-            const match = /(?<url>git@.*?)=(?<file>.*?)=(?<ref>.*)/.exec(remoteVariables);
-            assert(match != null, "--remote-variables is malformed use 'git@gitlab.com:firecow/example.git=gitlab-variables.yml=master' syntax");
-            const url = match.groups?.url;
-            const file = match.groups?.file;
-            const ref = match.groups?.ref;
-            const res = await Utils.bash(`set -eou pipefail; git archive --remote=${url} ${ref} ${file} | tar -xO ${file}`, cwd);
-            remoteFileData = yaml.load(`${res.stdout}`);
+            for (let i = 0; i < remoteVariables.length; i++) {
+                const match = /(?<url>git@.*?)=(?<file>.*?)=(?<ref>.*)/.exec(remoteVariables[i]);
+                assert(match != null, "--remote-variables is malformed use 'git@gitlab.com:firecow/example.git=gitlab-variables.yml=master' syntax");
+                const url = match.groups?.url;
+                const file = match.groups?.file;
+                const ref = match.groups?.ref;
+                const res = await Utils.bash(`set -eou pipefail; git archive --remote=${url} ${ref} ${file} | tar -xO ${file}`, cwd);
+                const loadedYaml = yaml.load(`${res.stdout}`);
+                // Check if loadedYaml is an object
+                if (typeof loadedYaml === "object" && loadedYaml !== null) {
+                    remoteFileData = deepExtend(remoteFileData, loadedYaml);
+                }
+            }
         }
 
         if (await fs.pathExists(homeVariablesFile)) {

tests/test-cases/remote-variables-file/integration.test.ts

@@ -18,7 +18,7 @@ test.concurrent("remote-variables-file <test-job>", async () => {
     await handler({
         cwd: "tests/test-cases/remote-variables-file",
         job: ["test-job"],
-        remoteVariables: "git@gitlab.com:example/firecow.git=variables.yml=main",
+        remoteVariables: ["git@gitlab.com:example/firecow.git=variables.yml=main"],
     }, writeStreams);
 
     const expected = [