Configuration files and Dockerfiles for every parts of the project

.github/workflows/build-container-prep-image.yml

@@ -0,0 +1,42 @@
+name: Build and publish HPCS container preparation image
+on: [push]
+
+env:
+  IMAGE_NAME: hpcs/container-prep
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Build image
+        run: git lfs pull ; docker build . -f ./client/container_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.github/workflows/build-data-prep-image.yml

@@ -0,0 +1,42 @@
+name: Build and publish HPCS data preparation image
+on: [push]
+
+env:
+  IMAGE_NAME: hpcs/data-prep
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Build image
+        run: git lfs pull ; docker build . -f ./client/data_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.github/workflows/build-job-prep-image.yml

@@ -0,0 +1,42 @@
+name: Build and publish HPCS job preparation image
+on: [push]
+
+env:
+  IMAGE_NAME: hpcs/job-prep
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Build image
+        run: git lfs pull ; docker build . -f ./client/job_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.github/workflows/build-server-image.yml

@@ -0,0 +1,42 @@
+name: Build and publish HPCS server image
+on: [push]
+
+env:
+  IMAGE_NAME: hpcs/server
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Build image
+        run: git lfs pull ; docker build . -f ./server/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

client/container_preparation/Dockerfile

@@ -1,23 +1,24 @@
 # Using Python original Docker image
-FROM python:3.9-bullseye
-
-# Update & Upgrade global packages base
-RUN apt update -y \
-	&& apt upgrade -y
+FROM --platform=linux/amd64 python:3.9-alpine
 
 # Install necessary packages
-RUN apt install -y \
+RUN apk add \
     git \
     curl \
-    jq
+    jq \
+    build-base \
+    libffi-dev
+
+RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
+ENV PATH="$PATH:/root/.cargo/bin"
 
 # Install spire-agent
-RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz
-RUN tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire
+RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
+RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
 RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
 
 # Install pyspiffe package
-RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git
+RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
 
 # Create code directory, output directory
 RUN mkdir /container_preparation /output ; chmod -R 777 /output

client/container_preparation/entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 ## This entrypoint wraps up the container preparation with the agent spawning and the key shipping.
 #
@@ -10,6 +10,7 @@ docker_path="/var/run/docker.sock"
 parse_args() {
     while [[ "$#" -gt 0 ]]; do
         case "$1" in
+            --config) config="$2"; shift 2 ;;
             -b|--base-oci-image) base_oci_image="$2"; shift 2 ;;
             -s|--sif-path) sif_path="$2"; shift 2 ;;
             -e|--encrypted) encrypted=true; shift ;;
@@ -26,7 +27,7 @@ parse_args() {
     done
 
     # Check for required arguments
-    if [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
+    if [ -z "$config" ] || [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
         echo echo "Please provides options for both of these programs : "
         python3 ./prepare_container.py --help
         python3 ./utils/ship_a_key.py --help
@@ -66,11 +67,14 @@ echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Entering ent
 if [ -n "$encrypted" ]; then
     echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Encryption mode is on. Registering and running SPIRE Agent"
 
-    python3 ./utils/spawn_agent.py > /dev/null 2> /dev/null || exit 1 &
+    python3 ./utils/spawn_agent.py --config $config > /dev/null 2> /dev/null &
     spire_agent_pid=$!
 
 fi
 
+
+ps $spire_agent_pid > /dev/null || ( echo "spire agent died, aborting" ; end_entrypoint "$spire_agent_pid" 1)
+
 #
 ## [END] Perform node attestation
 #
@@ -110,13 +114,13 @@ else
 
     if [ -z "$users" ]; then
         # If the user provided only groups
-        python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+        python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
     elif [ -z "$groups" ] ; then
         # If the user provided only users
-        python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+        python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
     else
         # If the user provided both
-        python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
+        python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
     fi
 
     echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Key written to the vault"

client/container_preparation/input_logic/run.sh

@@ -7,10 +7,10 @@ PATH="$PATH:/sd-container/tools/input_logic/"
 echo "[SD-Container][Input-Logic] : Getting data decryption key from vault"
 
 # Get token via vault login. The data_login environment variable need to be exported from calling script
-data_token=$(curl -s --request POST --data "$data_login" http://${vault}/v1/auth/jwt/login | jq '.auth.client_token' -r)  || exit 1
+data_token=$(curl -s --request POST --data "$data_login" $vault/v1/auth/jwt/login | jq '.auth.client_token' -r)  || exit 1
 
 # Use the token to access the key. The data_path environment variable needs to be exported from calling script
-data_key=$(curl -s -H "X-Vault-Token: $data_token" http://${vault}/v1/kv/data/${data_path} | jq '.data.data.key' -r)    || exit 1
+data_key=$(curl -s -H "X-Vault-Token: $data_token" $vault/v1/kv/data/${data_path} | jq '.data.data.key' -r)    || exit 1
 
 # Write the key in an encrypted volume
 echo "$data_key" > /sd-container/encrypted/decryption_key

client/container_preparation/lib/image_build.py

@@ -112,11 +112,11 @@ def create_sif_image(
         encrypted (bool, optional): Wether or not to encrypt the container, keys are generated relatively to the current path, and are called "keys, keys.pub". Defaults to False.
     """
     # Check that the build environment exists
-    build_env_exists = check_build_env_exists()
+    build_env_exists = check_build_env_exists(docker_client=docker_client)
     if not build_env_exists:
         print("Build environment container image doesn't exist, building it")
-        build_build_env()
-        build_env_exists = check_build_env_exists()
+        build_build_env(docker_client=docker_client)
+        build_env_exists = check_build_env_exists(docker_client=docker_client)
 
     # Fixing the build environment image tag
     build_env_image_tag = build_env_exists.attrs["RepoTags"][0]

client/container_preparation/tools/docker/build_env/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 
 # Update & Upgrade global packages base
 RUN apt update -y \
@@ -17,8 +17,8 @@ RUN apt install \
 RUN DEBIAN_FRONTEND=noninteractive apt install -y cryptsetup
 
 # Get apptainer installer and install it
-RUN wget -q https://github.com/apptainer/apptainer/releases/download/v1.2.5/apptainer_1.2.5_amd64.deb
-RUN apt install -y ./apptainer_1.2.5_amd64.deb
+RUN wget -q https://github.com/apptainer/apptainer/releases/download/v1.3.0/apptainer_1.3.0_amd64.deb
+RUN apt install -y ./apptainer_1.3.0_amd64.deb
 
 # Create a directory to eventually map for output image
 RUN mkdir /output

client/container_preparation/tools/docker/docker_utils.py

@@ -1,23 +1,21 @@
 import docker, os
 
 
-def check_build_env_exists():
+def check_build_env_exists(docker_client : docker.DockerClient):
     """Verify that the build environment (docker image sd-container/build_env) exists.
 
     Returns:
         bool: Wether or not the sd-container/build_env image exists.
     """
     # Check the build env exists
-    docker_client = docker.from_env()
     try:
         return docker_client.images.get("sd-container/build_env")
     except docker.errors.ImageNotFound:
         return False
 
 
-def build_build_env():
+def build_build_env(docker_client : docker.DockerClient):
     """Builds the build environment"""
-    docker_client = docker.from_env()
     docker_client.images.build(
         path=f"{os.path.realpath(os.path.dirname(__file__))}/build_env",
         dockerfile="./Dockerfile",

client/data_preparation/Dockerfile

@@ -1,23 +1,26 @@
 # Using Python original Docker image
-FROM python:3.9-bullseye
-
-# Update & Upgrade global packages base
-RUN apt update -y \
-	&& apt upgrade -y
+FROM --platform=linux/amd64 python:3.9-alpine
 
 # Install necessary packages
-RUN apt install -y \
+RUN apk add \
     git \
     curl \
-    jq
+    jq \
+    build-base \
+    libffi-dev
+
+# Install Rust
+RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
+ENV PATH="$PATH:/root/.cargo/bin"
 
 # Install spire-agent
-RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz
-RUN tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire
+
+RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
+RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
 RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
 
 # Install pyspiffe package
-RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git
+RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
 
 # Create code directory, output directory
 RUN mkdir /data_preparation /output