bootstrap image

.github/workflows/images.yaml

@@ -36,6 +36,7 @@ jobs:
           - executor-debug
           - executor-slim
           - warmer
+          - bootstrap
 
         include:
           - image: executor
@@ -66,6 +67,13 @@ jobs:
             tag: ${{ github.sha }}-warmer
             release-tag: warmer
 
+          - image: bootstrap
+            target: kaniko-debug-2
+            platforms: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le
+            image-name: martizih/kaniko
+            tag: ${{ github.sha }}-bootstrap
+            release-tag: bootstrap
+
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1

README.md

@@ -98,6 +98,7 @@ _If you are interested in contributing to kaniko, see
       - [Running kaniko in gVisor](#running-kaniko-in-gvisor)
       - [Running kaniko in Google Cloud Build](#running-kaniko-in-google-cloud-build)
       - [Running kaniko in Docker](#running-kaniko-in-docker)
+    - [Bootstrapping Kaniko](#bootstrapping-kaniko)
     - [Caching](#caching)
       - [Caching Layers](#caching-layers)
       - [Caching Base Images](#caching-base-images)
@@ -529,6 +530,53 @@ destination:
 ./run_in_docker.sh /workspace/Dockerfile /home/user/kaniko-project gcr.io/$PROJECT_ID/$TAG
 ```
 
+### Bootstrapping Kaniko
+Kaniko's approach to buliding docker images is unique. It doesn't build up and snapshot an image in a separate container, instead it will build up and snapshot the image in the container where kaniko is installed. The benefit is that kaniko can run without any privileges, because it's not actually using any containerization technologies, the downside is that sometimes builds can yield surprising results - bootstrapping kaniko image with kaniko builder is one such case. 
+
+The good news is, it's all technically possible, but it's a bit more involved than building other images.
+
+The first problem we face is that the kaniko binaries are installed in `/kaniko` directory. Which means that this directory must also be ignored during snapshots, less we would leak our build tool into any image we produce. But this also means that kaniko by default can't build a kaniko image with the binaries installed in `/kaniko` directory. Luckily there is an override for that, that allows us to move all the binaries to a different location before the build ie. `--kaniko-dir=/kaniko2`.
+
+The second problem only affects build using the `debug` image, ie. gitlab-runner. The shell that is spawned in the debug image is in `/busybox`, similarly we can't snapshot files in that directory. Unfortunately there is no override to move those binaries. But with a bit creativity we can create a bootstrap image that has the shell installed into a different location ie. `/busybox2` and then use that bootstrap image to build the actual new debug image.
+
+```yaml
+bootstrap:
+  extends:
+    - .build
+  image:
+    name: gcr.io/kaniko-project/executor:v1.24.0-debug
+    entrypoint: [""]
+  needs: []
+  variables:
+    IMAGE: ${CI_REGISTRY_IMAGE}/bootstrap:latest
+    EXTRA_ARGS: >-
+      --build-arg=TARGETARCH=amd64
+      --build-arg=TARGETOS=linux
+      --kaniko-dir=/kaniko2
+      --target=kaniko-debug-2
+    KANIKO_DIR: /kaniko2
+
+build:
+  extends:
+    - .build
+  image:
+    name: ${CI_REGISTRY_IMAGE}/bootstrap:latest
+    entrypoint: [""]
+  needs: [bootstrap]
+  variables:
+    IMAGE: ${CI_REGISTRY_IMAGE}/kaniko:latest
+    EXTRA_ARGS: >-
+      --build-arg=TARGETARCH=amd64
+      --build-arg=TARGETOS=linux
+      --kaniko-dir=/kaniko2
+      --target=kaniko-debug
+    KANIKO_DIR: /kaniko2
+```
+This is just an illustrative extract, please find the full [.gitlab-ci.yml](./docs/bootstrap.gitlab-ci.yml) here.
+
+With this two step approach we can now indeed bootstrap kaniko in kaniko. However, it is not really necessary to rebuild that intermediate bootstrap image every time, we can reuse it from a different build. Hence I provide a dedicated image `martizih/kaniko:bootstrap` that can be used for that purpose.
+
+
 ### Caching
 
 #### Caching Layers

deploy/Dockerfile

@@ -49,22 +49,20 @@
 # Get ACR docker env credential helper
 RUN go install github.com/chrismellard/docker-credential-acr-env
 
-RUN \
-  --mount=type=cache,target=/root/.cache/go-build \
-  --mount=type=cache,target=/go/pkg \
-  make out/executor out/warmer
+RUN make out/executor out/warmer
 
 # Generate latest ca-certificates
 FROM debian:bookworm-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
 # use musl busybox since it's staticly compiled on all platforms
 FROM busybox:musl AS busybox
+RUN mkdir -p /kaniko && chmod 777 /kaniko
 
 FROM scratch AS kaniko-base-slim
 
 # Create kaniko directory with world write permission to allow non root run
-RUN --mount=from=busybox,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"]
+COPY --from=busybox /kaniko /kaniko
 
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/
 COPY files/nsswitch.conf /etc/nsswitch.conf
@@ -82,7 +80,7 @@
 COPY --from=builder /kaniko/.docker /kaniko/.docker
 
 ENV DOCKER_CONFIG=/kaniko/.docker/
 ENV DOCKER_CREDENTIAL_GCR_CONFIG=/kaniko/.config/gcloud/docker_credential_gcr_config.json
 WORKDIR /workspace
 
 ### FINAL STAGES ###
@@ -112,6 +110,19 @@
 RUN ["/busybox/mkdir", "-p", "/bin"]
 RUN ["/busybox/ln", "-s", "/busybox/sh", "/bin/sh"]
 
+FROM kaniko-executor AS kaniko-debug-2
+
+ENV PATH=/usr/local/bin:/kaniko:/busybox2
+
+COPY --from=builder /src/out/warmer /kaniko/warmer
+
+COPY --from=busybox /bin /busybox2
+# Declare /busybox as a volume to get it automatically in the path to ignore
+VOLUME /busybox2
+
+RUN ["/busybox2/mkdir", "-p", "/bin"]
+RUN ["/busybox2/ln", "-s", "/busybox2/sh", "/bin/sh"]
+
 FROM kaniko-base-slim AS kaniko-slim
 
 COPY --from=builder /src/out/executor /kaniko/executor

docs/bootstrap.gitlab-ci.yml

@@ -0,0 +1,64 @@
+workflow:
+  auto_cancel:
+    on_new_commit: interruptible
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+default:
+  interruptible: true
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+
+stages:
+  - build
+
+.build:
+  stage: build
+  script:
+    - /kaniko/executor
+      --context $CI_PROJECT_DIR
+      --dockerfile deploy/Dockerfile
+      --destination=${IMAGE}
+      --cache=true
+      --cache-copy-layers
+      --cache-dir /cache/kaniko
+      --cache-repo ${CI_REGISTRY_IMAGE}/cache
+      --skip-unused-stages
+      $EXTRA_ARGS
+
+bootstrap:
+  extends:
+    - .build
+  image:
+    name: gcr.io/kaniko-project/executor:v1.24.0-debug
+    entrypoint: [""]
+  needs: []
+  variables:
+    IMAGE: ${CI_REGISTRY_IMAGE}/bootstrap:latest
+    EXTRA_ARGS: >-
+      --build-arg=TARGETARCH=amd64
+      --build-arg=TARGETOS=linux  
+      --kaniko-dir=/kaniko2
+      --target=kaniko-debug-2
+    KANIKO_DIR: /kaniko2
+
+build:
+  extends:
+    - .build
+  image:
+    name: ${CI_REGISTRY_IMAGE}/bootstrap:latest
+    entrypoint: [""]
+  needs: [bootstrap]
+  variables:
+    IMAGE: ${CI_REGISTRY_IMAGE}/kaniko:latest
+    EXTRA_ARGS: >-
+      --build-arg=TARGETARCH=amd64
+      --build-arg=TARGETOS=linux  
+      --kaniko-dir=/kaniko2
+      --target=kaniko-debug
+    KANIKO_DIR: /kaniko2