Merge branch 'main' into wip/mend0za/action-raw-test

Author: obbardc

.github/workflows/ci.yaml

@@ -2,25 +2,57 @@ name: Build and Test
 
 env:
   GITHUB_TAG: ghcr.io/${{ github.repository }}
-  DOCKERHUB_TAG: godebos/debos
 
 on:
   push:
     branches-ignore:
       - '*.tmp'
+    tags:
+      - '*'
+  # Build at 04:00am every Monday
+  schedule:
+    - cron: "0 4 * * 1"
   pull_request:
   workflow_dispatch:
 
 jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        variant:
+          - debos-arch
+          - debos-bookworm
+          - debos-bullseye
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    container:
+      image: ghcr.io/go-debos/test-containers/${{matrix.variant}}:main
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Test build
+      run: go build -o debos cmd/debos/debos.go
+
+    - name: Run unit tests
+      run: go test -v ./... | tee test.out
+
+    - name: Ensure no tests were skipped
+      run: "! grep -q SKIP test.out"
+
   build:
     name: Build Docker container
     runs-on: ubuntu-latest
+    needs: test
     steps:
       - name: Repository checkout
         uses: actions/checkout@v3
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Use cache
         uses: actions/cache@v3
@@ -29,7 +61,7 @@ jobs:
           key: ${{ runner.os }}-docker
 
       - name: Build Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: false
@@ -56,7 +88,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Use cache
         uses: actions/cache@v3
@@ -65,7 +97,7 @@ jobs:
           key: ${{ runner.os }}-docker
 
       - name: Build Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: false
@@ -92,12 +124,16 @@ jobs:
           - { name: "uml", backend: "--fakemachine-backend=uml" }
         test:
           - { name: "recipes", case: "recipes" }
-          - { name: "debian (amd64}", case: "debian", variables: "-t architecture:amd64" }
+          - { name: "debian (amd64)", case: "debian", variables: "-t architecture:amd64" }
           - { name: "debian (arm64)", case: "debian", variables: "-t architecture:arm64" }
           - { name: "debian (armhf)", case: "debian", variables: "-t architecture:armhf" }
         include:
+          - backend: { name: "arch", backend: "--fakemachine-backend=qemu" }
+            test: { name: "arch", case: "arch" }
           - backend: { name: "qemu", backend: "--fakemachine-backend=qemu" }
             test: { name: "partitioning", case: "partitioning" }
+          - backend: { name: "uml", backend: "--fakemachine-backend=uml" }
+            test: { name: "apertis", case: "apertis" }
           - backend: { name: "uml", backend: "--fakemachine-backend=uml" }
             test: { name: "partitioning", case: "partitioning" }
           - backend: { name: "qemu", backend: "--fakemachine-backend=qemu" }
@@ -133,22 +169,25 @@ jobs:
           --tmpfs /run
           --privileged
           -e TMP=/scratch
+          -e SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1
           debos -v
           ${{matrix.backend.backend}}
           ${{matrix.test.variables}}
           ${{matrix.test.case}}/test.yaml
 
-  # Job to key the bors success status against
-  bors:
-    name: bors
-    if: success()
+  # Job to key success status against
+  allgreen:
+    name: allgreen
+    if: always()
     needs:
       - unit-tests
       - recipe-tests
     runs-on: ubuntu-latest
     steps:
-      - name: Mark the job as a success
-        run: exit 0
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
 
   publish-github:
     name: Publish to GHCR
@@ -167,7 +206,7 @@ jobs:
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.GITHUB_TAG }}
           tags: |
@@ -178,14 +217,14 @@ jobs:
             "type=ref,event=pr"
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Use cache
         uses: actions/cache@v3
@@ -194,7 +233,7 @@ jobs:
           key: ${{ runner.os }}-docker
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true
@@ -203,12 +242,37 @@ jobs:
           file: docker/Dockerfile
           cache-from: type=local,src=/tmp/.build-cache
 
+  check-dockerhub-secrets:
+    name: Check DockerHub secrets exist
+    runs-on: ubuntu-latest
+    outputs:
+      has-secrets: ${{ steps.check-secrets.outputs.has-secrets }}
+    steps:
+      - id: check-secrets
+        name: Check secrets exist
+        run: |
+          if [[ "${{ secrets.DOCKERHUB_IMAGE }}" != "" && \
+                "${{ secrets.DOCKERHUB_USERNAME }}" != "" && \
+                "${{ secrets.DOCKERHUB_PASSWORD }}" != "" ]]; \
+          then
+            echo "Dockerhub secrets exist"
+            echo "has-secrets=true" >> $GITHUB_OUTPUT
+          else
+            echo "Dockerhub secrets do not exist; not pushing to Dockerhub"
+            echo "Please set the following secrets on GitHub (settings > secrets > actions > new):"
+            echo "DOCKERHUB_IMAGE, DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD"
+            echo "has-secrets=false" >> $GITHUB_OUTPUT
+          fi
+
   publish-dockerhub:
     name: Publish to DockerHub
     needs:
+      - check-dockerhub-secrets
       - unit-tests
       - recipe-tests
-    if: github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+    if: |
+      needs.check-dockerhub-secrets.outputs.has-secrets == 'true' &&
+      github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -220,9 +284,9 @@ jobs:
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
-          images: ${{ env.DOCKERHUB_TAG }}
+          images: ${{ secrets.DOCKERHUB_IMAGE }}
           tags: |
             "type=ref,event=branch"
             "type=ref,suffix=-{{sha}},event=branch"
@@ -231,14 +295,14 @@ jobs:
             "type=ref,event=pr"
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         continue-on-error: true
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Use cache
         uses: actions/cache@v3
@@ -247,7 +311,7 @@ jobs:
           key: ${{ runner.os }}-docker
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         continue-on-error: true
         with:
           context: .

README.md

@@ -45,6 +45,8 @@ Some of the actions provided by debos to customize and produce images are:
 * ostree-deploy: deploy an OSTree branch to the image
 * overlay: do a recursive copy of directories or files to the target filesystem
 * pack: create a tarball with the target filesystem
+* pacman: install packages and their dependencies with pacman
+* pacstrap: construct the target rootfs with pacstrap
 * raw: directly write a file to the output image at a given offset
 * recipe: includes the recipe actions at the given path
 * run: allows to run a command or script in the filesystem or in the host
@@ -66,8 +68,8 @@ See [docker/README.md](https://github.com/go-debos/debos/blob/master/docker/READ
 
     sudo apt install golang git libglib2.0-dev libostree-dev qemu-system-x86 \
          qemu-user-static debootstrap systemd-container
-    export GOPATH=/opt/src/gocode # or whatever suites your needs
-    go get -u github.com/go-debos/debos/cmd/debos
+    export GOPATH=/opt/src/gocode # or whatever suits your needs
+    go install -v github.com/go-debos/debos/cmd/debos@latest
     /opt/src/gocode/bin/debos --help
 
 ## Simple example
@@ -82,10 +84,10 @@ make a tarball.
 
     actions:
       - action: debootstrap
-        suite: "buster"
+        suite: bookworm
         components:
           - main
-          - non-free
+          - non-free-firmware
         mirror: https://deb.debian.org/debian
         variant: minbase
 
@@ -112,7 +114,8 @@ this:
 
 ## Other examples
 
-This example builds a customized image for a Raspberry Pi 3.
+Example recipes are collected in a separate repository:
+
 https://github.com/go-debos/debos-recipes
 
 ## Environment variables
@@ -136,7 +139,7 @@ no_proxy defined, both will be propagated to fakemachine respecting the case.
 The command line options --environ-var and -e can be used to specify,
 overwrite, and unset environment variables for fakemachine with the syntax:
 
-$ debos -e ENVIRONVAR:VALUE ...
+    $ debos -e ENVIRONVAR:VALUE ...
 
 To unset an enviroment variable, or in other words, to prevent an environment
 variable to be propagated to fakemachine, use the same syntax without a value.
@@ -162,6 +165,18 @@ Fakemachine can use different virtualisation backends to spawn the virtualmachin
 for more information see the documentation under the [fakemachine repository](https://github.com/go-debos/fakemachine).
 
 By default the backend will automatically be selected based on what is supported
-on the host machine, but this can be overridden using the `--fakemachine-backend`
+on the host machine, but this can be overridden using the `--fakemachine-backend` / `-b`
 option. If no backends are supported, debos reverts to running the recipe on the
 host without creating a fakemachine.
+
+Performance of the backends is roughly as follows: `kvm` is faster than `uml` is faster than `qemu`.
+Using `--disable-fakemachine` is slightly faster than `kvm`, but requires root permissions.
+
+Numbers for running [pine-a64-plus/debian.yaml](https://github.com/go-debos/debos-recipes/blob/9a25b4be6c9136f4a27e542f39ab7e419fc852c9/pine-a64-plus/debian.yaml) on an Intel Pentium G4560T with SSD:
+
+| Backend | Wall Time | Prerequisites |
+| --- | --- | --- |
+| `--disable-fakemachine` | 8 min | root permissions |
+| `-b kvm` | 9 min | access to `/dev/kvm` |
+| `-b uml` | 18 min | package `user-mode-linux` installed  |
+| `-b qemu` | 166 min | none |

actions/actions_doc.go

@@ -2,5 +2,16 @@
 
 /*
 Package 'actions' implements 'debos' modules used for OS creation.
+
+The origin property
+
+Several actions have the 'origin' property. Possible values for the
+'origin' property are:
+
+  1) 'recipe' ....... directory the recipe is in
+  2) 'filesystem' ... target filesystem root directory from previous filesystem-deploy action or
+                      a previous ostree action.
+  3) name property of a previous download action
+
 */
 package actions

actions/apt_action.go

@@ -3,7 +3,7 @@ Apt Action
 
 Install packages and their dependencies to the target rootfs with 'apt'.
 
-Yaml syntax:
+ # Yaml syntax:
  - action: apt
    recommends: bool
    unauthenticated: bool