Skip to content

Commit 3fadcc5

Browse files
mmatthiesencscmmatthiesencsc
authored
Merge pull request #1 from CSCfi/feature/dockerfile_everywhere
Configuration files and Dockerfiles for every parts of the project
2 parents bbda921 + e859010 commit 3fadcc5

32 files changed

+576
-162
lines changed

‎.github/workflows/build-container-prep-image.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build and publish HPCS container preparation image
2+
on: [push]
3+
4+
env:
5+
IMAGE_NAME: hpcs/container-prep
6+
7+
jobs:
8+
push:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
packages: write
12+
contents: read
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
lfs: 'true'
17+
18+
- name: Build image
19+
run: git lfs pull ; docker build . -f ./client/container_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
20+
21+
- name: Log in to registry
22+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
23+
24+
- name: Push image
25+
run: |
26+
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
27+
28+
# This changes all uppercase characters to lowercase.
29+
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
30+
31+
# This strips the git ref prefix from the version.
32+
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
33+
34+
# This strips the "v" prefix from the tag name.
35+
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
36+
37+
# This uses the Docker `latest` tag convention.
38+
[ "$VERSION" == "main" ] && VERSION=latest
39+
echo IMAGE_ID=$IMAGE_ID
40+
echo VERSION=$VERSION
41+
docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
42+
docker push $IMAGE_ID:$VERSION

‎.github/workflows/build-data-prep-image.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build and publish HPCS data preparation image
2+
on: [push]
3+
4+
env:
5+
IMAGE_NAME: hpcs/data-prep
6+
7+
jobs:
8+
push:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
packages: write
12+
contents: read
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
lfs: 'true'
17+
18+
- name: Build image
19+
run: git lfs pull ; docker build . -f ./client/data_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
20+
21+
- name: Log in to registry
22+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
23+
24+
- name: Push image
25+
run: |
26+
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
27+
28+
# This changes all uppercase characters to lowercase.
29+
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
30+
31+
# This strips the git ref prefix from the version.
32+
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
33+
34+
# This strips the "v" prefix from the tag name.
35+
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
36+
37+
# This uses the Docker `latest` tag convention.
38+
[ "$VERSION" == "main" ] && VERSION=latest
39+
echo IMAGE_ID=$IMAGE_ID
40+
echo VERSION=$VERSION
41+
docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
42+
docker push $IMAGE_ID:$VERSION

‎.github/workflows/build-job-prep-image.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build and publish HPCS job preparation image
2+
on: [push]
3+
4+
env:
5+
IMAGE_NAME: hpcs/job-prep
6+
7+
jobs:
8+
push:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
packages: write
12+
contents: read
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
lfs: 'true'
17+
18+
- name: Build image
19+
run: git lfs pull ; docker build . -f ./client/job_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
20+
21+
- name: Log in to registry
22+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
23+
24+
- name: Push image
25+
run: |
26+
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
27+
28+
# This changes all uppercase characters to lowercase.
29+
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
30+
31+
# This strips the git ref prefix from the version.
32+
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
33+
34+
# This strips the "v" prefix from the tag name.
35+
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
36+
37+
# This uses the Docker `latest` tag convention.
38+
[ "$VERSION" == "main" ] && VERSION=latest
39+
echo IMAGE_ID=$IMAGE_ID
40+
echo VERSION=$VERSION
41+
docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
42+
docker push $IMAGE_ID:$VERSION

‎.github/workflows/build-server-image.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build and publish HPCS server image
2+
on: [push]
3+
4+
env:
5+
IMAGE_NAME: hpcs/server
6+
7+
jobs:
8+
push:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
packages: write
12+
contents: read
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
lfs: 'true'
17+
18+
- name: Build image
19+
run: git lfs pull ; docker build . -f ./server/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
20+
21+
- name: Log in to registry
22+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
23+
24+
- name: Push image
25+
run: |
26+
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
27+
28+
# This changes all uppercase characters to lowercase.
29+
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
30+
31+
# This strips the git ref prefix from the version.
32+
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
33+
34+
# This strips the "v" prefix from the tag name.
35+
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
36+
37+
# This uses the Docker `latest` tag convention.
38+
[ "$VERSION" == "main" ] && VERSION=latest
39+
echo IMAGE_ID=$IMAGE_ID
40+
echo VERSION=$VERSION
41+
docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
42+
docker push $IMAGE_ID:$VERSION

‎client/container_preparation/Dockerfile

+11-10
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,24 @@
11
# Using Python original Docker image
2-
FROM python:3.9-bullseye
3-
4-
# Update & Upgrade global packages base
5-
RUN apt update -y \
6-
&& apt upgrade -y
2+
FROM --platform=linux/amd64 python:3.9-alpine
73

84
# Install necessary packages
9-
RUN apt install -y \
5+
RUN apk add \
106
git \
117
curl \
12-
jq
8+
jq \
9+
build-base \
10+
libffi-dev
11+
12+
RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
13+
ENV PATH="$PATH:/root/.cargo/bin"
1314

1415
# Install spire-agent
15-
RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz
16-
RUN tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire
16+
RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
17+
RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
1718
RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
1819

1920
# Install pyspiffe package
20-
RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git
21+
RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
2122

2223
# Create code directory, output directory
2324
RUN mkdir /container_preparation /output ; chmod -R 777 /output

‎client/container_preparation/entrypoint.sh

+10-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/sh
22
#
33
## This entrypoint wraps up the container preparation with the agent spawning and the key shipping.
44
#
@@ -10,6 +10,7 @@ docker_path="/var/run/docker.sock"
1010
parse_args() {
1111
while [[ "$#" -gt 0 ]]; do
1212
case "$1" in
13+
--config) config="$2"; shift 2 ;;
1314
-b|--base-oci-image) base_oci_image="$2"; shift 2 ;;
1415
-s|--sif-path) sif_path="$2"; shift 2 ;;
1516
-e|--encrypted) encrypted=true; shift ;;
@@ -26,7 +27,7 @@ parse_args() {
2627
done
2728

2829
# Check for required arguments
29-
if [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
30+
if [ -z "$config" ] || [ -z "$base_oci_image" ] || [ -z "$sif_path" ] || [ -z "$data_path" ] || [ -z "$data_path_at_rest" ] || ( [ -z "$users" ] && [ -z "$groups" ] ) || [ -z "$compute_nodes" ]; then
3031
echo echo "Please provides options for both of these programs : "
3132
python3 ./prepare_container.py --help
3233
python3 ./utils/ship_a_key.py --help
@@ -66,11 +67,14 @@ echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Entering ent
6667
if [ -n "$encrypted" ]; then
6768
echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Encryption mode is on. Registering and running SPIRE Agent"
6869

69-
python3 ./utils/spawn_agent.py > /dev/null 2> /dev/null || exit 1 &
70+
python3 ./utils/spawn_agent.py --config $config > /dev/null 2> /dev/null &
7071
spire_agent_pid=$!
7172

7273
fi
7374

75+
76+
ps $spire_agent_pid > /dev/null || ( echo "spire agent died, aborting" ; end_entrypoint "$spire_agent_pid" 1)
77+
7478
#
7579
## [END] Perform node attestation
7680
#
@@ -110,13 +114,13 @@ else
110114

111115
if [ -z "$users" ]; then
112116
# If the user provided only groups
113-
python3 ./utils/ship_a_key.py --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
117+
python3 ./utils/ship_a_key.py --config $config --username "$username" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
114118
elif [ -z "$groups" ] ; then
115119
# If the user provided only users
116-
python3 ./utils/ship_a_key.py --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
120+
python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
117121
else
118122
# If the user provided both
119-
python3 ./utils/ship_a_key.py --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
123+
python3 ./utils/ship_a_key.py --config $config --username "$username" -u "$users" -g "$groups" -c "$compute_nodes" --data-path "$data_path" --data-path-at-rest "$data_path_at_rest" -i "$spiffeID" || end_entrypoint "$spire_agent_pid" 1
120124
fi
121125

122126
echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Key written to the vault"

‎client/container_preparation/input_logic/run.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ PATH="$PATH:/sd-container/tools/input_logic/"
77
echo "[SD-Container][Input-Logic] : Getting data decryption key from vault"
88

99
# Get token via vault login. The data_login environment variable need to be exported from calling script
10-
data_token=$(curl -s --request POST --data "$data_login" http://${vault}/v1/auth/jwt/login | jq '.auth.client_token' -r) || exit 1
10+
data_token=$(curl -s --request POST --data "$data_login" $vault/v1/auth/jwt/login | jq '.auth.client_token' -r) || exit 1
1111

1212
# Use the token to access the key. The data_path environment variable needs to be exported from calling script
13-
data_key=$(curl -s -H "X-Vault-Token: $data_token" http://${vault}/v1/kv/data/${data_path} | jq '.data.data.key' -r) || exit 1
13+
data_key=$(curl -s -H "X-Vault-Token: $data_token" $vault/v1/kv/data/${data_path} | jq '.data.data.key' -r) || exit 1
1414

1515
# Write the key in an encrypted volume
1616
echo "$data_key" > /sd-container/encrypted/decryption_key

‎client/container_preparation/lib/image_build.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,11 @@ def create_sif_image(
112112
encrypted (bool, optional): Wether or not to encrypt the container, keys are generated relatively to the current path, and are called "keys, keys.pub". Defaults to False.
113113
"""
114114
# Check that the build environment exists
115-
build_env_exists = check_build_env_exists()
115+
build_env_exists = check_build_env_exists(docker_client=docker_client)
116116
if not build_env_exists:
117117
print("Build environment container image doesn't exist, building it")
118-
build_build_env()
119-
build_env_exists = check_build_env_exists()
118+
build_build_env(docker_client=docker_client)
119+
build_env_exists = check_build_env_exists(docker_client=docker_client)
120120

121121
# Fixing the build environment image tag
122122
build_env_image_tag = build_env_exists.attrs["RepoTags"][0]

‎client/container_preparation/tools/docker/build_env/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ubuntu:22.04
1+
FROM --platform=linux/amd64 ubuntu:22.04
22

33
# Update & Upgrade global packages base
44
RUN apt update -y \
@@ -17,8 +17,8 @@ RUN apt install \
1717
RUN DEBIAN_FRONTEND=noninteractive apt install -y cryptsetup
1818

1919
# Get apptainer installer and install it
20-
RUN wget -q https://github.com/apptainer/apptainer/releases/download/v1.2.5/apptainer_1.2.5_amd64.deb
21-
RUN apt install -y ./apptainer_1.2.5_amd64.deb
20+
RUN wget -q https://github.com/apptainer/apptainer/releases/download/v1.3.0/apptainer_1.3.0_amd64.deb
21+
RUN apt install -y ./apptainer_1.3.0_amd64.deb
2222

2323
# Create a directory to eventually map for output image
2424
RUN mkdir /output

‎client/container_preparation/tools/docker/docker_utils.py

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,21 @@
11
import docker, os
22

33

4-
def check_build_env_exists():
4+
def check_build_env_exists(docker_client : docker.DockerClient):
55
"""Verify that the build environment (docker image sd-container/build_env) exists.
66
77
Returns:
88
bool: Wether or not the sd-container/build_env image exists.
99
"""
1010
# Check the build env exists
11-
docker_client = docker.from_env()
1211
try:
1312
return docker_client.images.get("sd-container/build_env")
1413
except docker.errors.ImageNotFound:
1514
return False
1615

1716

18-
def build_build_env():
17+
def build_build_env(docker_client : docker.DockerClient):
1918
"""Builds the build environment"""
20-
docker_client = docker.from_env()
2119
docker_client.images.build(
2220
path=f"{os.path.realpath(os.path.dirname(__file__))}/build_env",
2321
dockerfile="./Dockerfile",

‎client/data_preparation/Dockerfile

+13-10
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
# Using Python original Docker image
2-
FROM python:3.9-bullseye
3-
4-
# Update & Upgrade global packages base
5-
RUN apt update -y \
6-
&& apt upgrade -y
2+
FROM --platform=linux/amd64 python:3.9-alpine
73

84
# Install necessary packages
9-
RUN apt install -y \
5+
RUN apk add \
106
git \
117
curl \
12-
jq
8+
jq \
9+
build-base \
10+
libffi-dev
11+
12+
# Install Rust
13+
RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
14+
ENV PATH="$PATH:/root/.cargo/bin"
1315

1416
# Install spire-agent
15-
RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz
16-
RUN tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire
17+
18+
RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
19+
RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
1720
RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
1821

1922
# Install pyspiffe package
20-
RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git
23+
RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
2124

2225
# Create code directory, output directory
2326
RUN mkdir /data_preparation /output

0 commit comments

Comments
 (0)