feat: option to start a local docker registry

src/argv.ts

@@ -350,4 +350,8 @@ export class Argv {
     get childPipelineDepth (): number {
         return this.map.get("childPipelineDepth");
     }
+
+    get registry (): boolean {
+        return this.map.get("registry") ?? false;
+    }
 }

src/handler.ts

@@ -69,6 +69,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
         Commander.runCsv(parser, writeStreams, argv.listCsvAll);
     } else if (argv.job.length > 0) {
         assert(argv.stage === null, "You cannot use --stage when starting individual jobs");
+        if (argv.registry) {
+            await Utils.startDockerRegistry(argv);
+        }
         generateGitIgnore(cwd, stateDir);
         const time = process.hrtime();
         if (argv.needs || argv.onlyNeeds) {
@@ -82,6 +85,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
             writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
         }
     } else if (argv.stage) {
+        if (argv.registry) {
+            await Utils.startDockerRegistry(argv);
+        }
         generateGitIgnore(cwd, stateDir);
         const time = process.hrtime();
         const pipelineIid = await state.getPipelineIid(cwd, stateDir);
@@ -90,6 +96,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
         await Commander.runJobsInStage(argv, parser, writeStreams);
         writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`);
     } else {
+        if (argv.registry) {
+            await Utils.startDockerRegistry(argv);
+        }
         generateGitIgnore(cwd, stateDir);
         const time = process.hrtime();
         await state.incrementPipelineIid(cwd, stateDir);
@@ -101,5 +110,8 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
     }
     writeStreams.flush();
 
+    if (argv.registry) {
+        await Utils.stopDockerRegistry(argv.containerExecutable);
+    }
     return cleanupJobResources(jobs);
 }

src/index.ts

@@ -323,6 +323,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
             default: true,
             description: "Enables color",
         })
+        .option("registry", {
+            type: "boolean",
+            requiresArg: false,
+            description: "Start a local docker registry and configure gitlab-ci-local containers to use that by default",
+        })
         .completion("completion", false, (current: string, yargsArgv: any, completionFilter: any, done: (completions: string[]) => any) => {
             try {
                 if (current.startsWith("-")) {

src/job.ts

@@ -329,7 +329,7 @@ export class Job {
             predefinedVariables["CI_NODE_INDEX"] = `${opt.nodeIndex}`;
         }
         predefinedVariables["CI_NODE_TOTAL"] = `${opt.nodesTotal}`;
-        predefinedVariables["CI_REGISTRY"] = `local-registry.${this.gitData.remote.host}`;
+        predefinedVariables["CI_REGISTRY"] = predefinedVariables["CI_REGISTRY"] = this.argv.registry ? Utils.gclRegistryPrefix : `local-registry.${this.gitData.remote.host}`;
         predefinedVariables["CI_REGISTRY_IMAGE"] = `$CI_REGISTRY/${predefinedVariables["CI_PROJECT_PATH"].toLowerCase()}`;
         return predefinedVariables;
     }
@@ -886,6 +886,11 @@ export class Job {
             });
         }
 
+        if (this.argv.registry) {
+            expanded["CI_REGISTRY_USER"] = expanded["CI_REGISTRY_USER"] ?? `${Utils.gclRegistryPrefix}.user`;
+            expanded["CI_REGISTRY_PASSWORD"] = expanded["CI_REGISTRY_PASSWORD"] ?? `${Utils.gclRegistryPrefix}.password`;
+        }
+
         this.refreshLongRunningSilentTimeout(writeStreams);
 
         if (imageName && !this._containerId) {
@@ -954,6 +959,12 @@ export class Job {
                 dockerCmd += `--network ${this._serviceNetworkId} --network-alias build `;
             }
 
+            if (this.argv.registry) {
+                dockerCmd += `--network ${Utils.gclRegistryPrefix}.net `;
+                dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/containers/certs.d:ro `;
+                dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/docker/certs.d:ro `;
+            }
+
             dockerCmd += `--volume ${buildVolumeName}:${this.ciProjectDir} `;
             dockerCmd += `--volume ${tmpVolumeName}:${this.fileVariablesDir} `;
             dockerCmd += `--workdir ${this.ciProjectDir} `;

src/utils.ts

@@ -5,7 +5,7 @@ import {Job, JobRule} from "./job.js";
 import fs from "fs-extra";
 import checksum from "checksum";
 import base64url from "base64url";
-import execa from "execa";
+import execa, {ExecaError} from "execa";
 import assert from "assert";
 import {CICDVariable} from "./variables-from-files.js";
 import {GitData, GitSchema} from "./git-data.js";
@@ -438,6 +438,101 @@ export class Utils {
         throw new Error(`Unhandled case ${param}`);
     }
 
+    static async dockerVolumeFileExists (containerExecutable: string, path: string, volume: string): Promise<boolean> {
+        try {
+            await Utils.spawn([containerExecutable, "run", "--rm", "-v", `${volume}:/mnt/vol`, "alpine", "ls", `/mnt/vol/${path}`]);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    static gclRegistryPrefix: string = "registry.gcl.local";
+    static async startDockerRegistry (argv: Argv): Promise<void> {
+        const gclRegistryCertVol = `${this.gclRegistryPrefix}.certs`;
+        const gclRegistryDataVol = `${this.gclRegistryPrefix}.data`;
+        const gclRegistryNet = `${this.gclRegistryPrefix}.net`;
+
+        // create cert volume
+        try {
+            await Utils.spawn(`${argv.containerExecutable} volume create ${gclRegistryCertVol}`.split(" "));
+        } catch (err) {
+            if (err instanceof Error && !err.message.endsWith("already exists"))
+                throw err;
+        }
+
+        // create self-signed cert/key files for https support
+        if (!await this.dockerVolumeFileExists(argv.containerExecutable, `${this.gclRegistryPrefix}.crt`, gclRegistryCertVol)) {
+            const opensslArgs = [
+                "req", "-newkey", "rsa:4096", "-nodes", "-sha256",
+                "-keyout", `/certs/${this.gclRegistryPrefix}.key`,
+                "-x509", "-days", "365",
+                "-out", `/certs/${this.gclRegistryPrefix}.crt`,
+                "-subj", `/CN=${this.gclRegistryPrefix}`,
+                "-addext", `subjectAltName=DNS:${this.gclRegistryPrefix}`,
+            ];
+            const generateCertsInPlace = [
+                argv.containerExecutable, "run", "--rm", "-v", `${gclRegistryCertVol}:/certs`, "--entrypoint", "sh", "alpine/openssl", "-c",
+                [
+                    "openssl", ...opensslArgs,
+                    "&&", "mkdir", "-p", `/certs/${this.gclRegistryPrefix}`,
+                    "&&", "cp", `/certs/${this.gclRegistryPrefix}.crt`, `/certs/${this.gclRegistryPrefix}/ca.crt`,
+                ].join(" "),
+            ];
+            await Utils.spawn(generateCertsInPlace);
+        }
+
+        // create data volume
+        try {
+            await Utils.spawn([argv.containerExecutable, "volume", "create", gclRegistryDataVol]);
+        } catch (err) {
+            if (err instanceof Error && !err.message.endsWith("already exists"))
+                throw err;
+        }
+
+        // create network
+        try {
+            await Utils.spawn([argv.containerExecutable, "network", "create", gclRegistryNet]);
+        } catch (err) {
+            if (err instanceof Error && !err.message.includes("already exists"))
+                throw err;
+        }
+
+        await Utils.spawn([argv.containerExecutable, "rm", "-f", this.gclRegistryPrefix]);
+        await Utils.spawn([
+            argv.containerExecutable, "run", "-d", "--name", this.gclRegistryPrefix,
+            "--network", gclRegistryNet,
+            "--volume", `${gclRegistryDataVol}:/var/lib/registry`,
+            "--volume", `${gclRegistryCertVol}:/certs:ro`,
+            "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:443",
+            "-e", `REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${this.gclRegistryPrefix}.crt`,
+            "-e", `REGISTRY_HTTP_TLS_KEY=/certs/${this.gclRegistryPrefix}.key`,
+            "registry",
+        ]);
+
+        try {
+            await execa(argv.containerExecutable, [
+                "run", "--rm",
+                "--network", gclRegistryNet,
+                "--entrypoint", "sh",
+                "curlimages/curl",
+                "-c", `until [ "$(curl -s -o /dev/null -k -w "%{http_code}" https://${this.gclRegistryPrefix}:443)" = "200" ]; do sleep 1; done;`,
+            ], {
+                timeout: 4000,
+            });
+        } catch (err) {
+            await this.stopDockerRegistry(argv.containerExecutable);
+            if ((err as ExecaError).timedOut) {
+                throw "local docker registry port check timed out";
+            }
+            throw err;
+        }
+    }
+
+    static async stopDockerRegistry (containerExecutable: string): Promise<void> {
+        await Utils.spawn([containerExecutable, "rm", "-f", this.gclRegistryPrefix]);
+    }
+
     static async getTrackedFiles (cwd: string): Promise<string[]> {
         const lsFilesRes = await Utils.bash("git ls-files --deduplicate", cwd);
         if (lsFilesRes.exitCode != 0) {

tests/test-cases/local-registry/.gitlab-ci.yml

@@ -0,0 +1,17 @@
+---
+registry-variables:
+  image: alpine:latest
+  script:
+    - echo "CI_REGISTRY=$CI_REGISTRY"
+    - echo "CI_REGISTRY_USER=$CI_REGISTRY_USER"
+    - echo "CI_REGISTRY_PASSWORD=$CI_REGISTRY_PASSWORD"
+
+registry-login-docker:
+  image: docker:dind
+  script:
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+
+registry-login-oci:
+  image: quay.io/podman/stable
+  script:
+    - echo "$CI_REGISTRY_PASSWORD" | podman login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

tests/test-cases/local-registry/integration.test.ts

@@ -0,0 +1,46 @@
+import {WriteStreamsMock} from "../../../src/write-streams.js";
+import {handler} from "../../../src/handler.js";
+import {Utils} from "../../../src/utils.js";
+
+test("local-registry ci variables", async () => {
+    const writeStreams = new WriteStreamsMock;
+    await handler({
+        cwd: "tests/test-cases/local-registry",
+        job: ["registry-variables"],
+        registry: true,
+        noColor: true,
+    }, writeStreams);
+
+    const expected = [
+        `registry-variables > CI_REGISTRY=${Utils.gclRegistryPrefix}`,
+        `registry-variables > CI_REGISTRY_USER=${Utils.gclRegistryPrefix}.user`,
+        `registry-variables > CI_REGISTRY_PASSWORD=${Utils.gclRegistryPrefix}.password`,
+    ];
+
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected));
+});
+
+test("local-registry login <docker>", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/local-registry",
+        job: ["registry-login-docker"],
+        registry: true,
+        noColor: true,
+    }, writeStreams);
+
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(["registry-login-docker > Login Succeeded"]));
+});
+
+test("local-registry login <oci>", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/local-registry",
+        job: ["registry-login-oci"],
+        registry: true,
+        privileged: true,
+        noColor: true,
+    }, writeStreams);
+
+    expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(["registry-login-oci > Login Succeeded!"]));
+});