Skip to content

Commit 91f3ee3

Browse files
mnoergaardmnoergaard
authored
Merge pull request #60 from openneuropet/single-docker-file
Single docker file + Apptainer Support
2 parents d3411ce + ad0cf5d commit 91f3ee3

File tree

5 files changed

+619
-6
lines changed

5 files changed

+619
-6
lines changed

‎.github/scripts/util_free_space.sh‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
set -eux
21+
22+
df -h
23+
echo "::group::/usr/local/*"
24+
du -hsc /usr/local/*
25+
echo "::endgroup::"
26+
# ~1GB
27+
sudo rm -rf \
28+
/usr/local/aws-sam-cil \
29+
/usr/local/julia* || :
30+
echo "::group::/usr/local/bin/*"
31+
du -hsc /usr/local/bin/*
32+
echo "::endgroup::"
33+
# ~1GB (From 1.2GB to 214MB)
34+
sudo rm -rf \
35+
/usr/local/bin/aliyun \
36+
/usr/local/bin/azcopy \
37+
/usr/local/bin/bicep \
38+
/usr/local/bin/cmake-gui \
39+
/usr/local/bin/cpack \
40+
/usr/local/bin/helm \
41+
/usr/local/bin/hub \
42+
/usr/local/bin/kubectl \
43+
/usr/local/bin/minikube \
44+
/usr/local/bin/node \
45+
/usr/local/bin/packer \
46+
/usr/local/bin/pulumi* \
47+
/usr/local/bin/sam \
48+
/usr/local/bin/stack \
49+
/usr/local/bin/terraform || :
50+
# 142M
51+
sudo rm -rf /usr/local/bin/oc || : \
52+
echo "::group::/usr/local/share/*"
53+
du -hsc /usr/local/share/*
54+
echo "::endgroup::"
55+
# 506MB
56+
sudo rm -rf /usr/local/share/chromium || :
57+
# 1.3GB
58+
sudo rm -rf /usr/local/share/powershell || :
59+
echo "::group::/usr/local/lib/*"
60+
du -hsc /usr/local/lib/*
61+
echo "::endgroup::"
62+
# 15GB
63+
sudo rm -rf /usr/local/lib/android || :
64+
# 341MB
65+
sudo rm -rf /usr/local/lib/heroku || :
66+
# 1.2GB
67+
sudo rm -rf /usr/local/lib/node_modules || :
68+
echo "::group::/opt/*"
69+
du -hsc /opt/*
70+
echo "::endgroup::"
71+
# 679MB
72+
sudo rm -rf /opt/az || :
73+
echo "::group::/opt/microsoft/*"
74+
du -hsc /opt/microsoft/*
75+
echo "::endgroup::"
76+
# 197MB
77+
sudo rm -rf /opt/microsoft/powershell || :
78+
echo "::group::/opt/hostedtoolcache/*"
79+
du -hsc /opt/hostedtoolcache/*
80+
echo "::endgroup::"
81+
# 5.3GB
82+
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
83+
# 1.4GB
84+
sudo rm -rf /opt/hostedtoolcache/go || :
85+
# 489MB
86+
sudo rm -rf /opt/hostedtoolcache/PyPy || :
87+
# 376MB
88+
sudo rm -rf /opt/hostedtoolcache/node || :
89+
# Remove Web browser packages
90+
#sudo apt purge -y \
91+
# firefox \
92+
# google-chrome-stable \
93+
# microsoft-edge-stable
94+
df -h

‎.github/workflows/apptainer-build-local.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: Test Apptainer Build Local
2+
3+
on:
4+
pull_request:
5+
branches: [ master ]
6+
workflow_run:
7+
workflows: ["Docker Build (Local)"]
8+
types:
9+
- completed
10+
workflow_dispatch: # Allow manual triggering
11+
12+
jobs:
13+
Test-Apptainer-Local-Uploads-and-Build:
14+
runs-on: ubuntu-latest
15+
if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
16+
17+
env:
18+
COMPOSE_FILE: ./docker-compose.yml
19+
REGISTRY_PREFIX: docker.io/library
20+
SERVER_NAME: localhost
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
with:
25+
submodules: true # Ensure we get the submodule info
26+
27+
- name: Set repository name
28+
run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV
29+
30+
- name: Cache test data
31+
uses: actions/cache@v4
32+
id: test-data-cache
33+
with:
34+
path: test/test_data
35+
key: test-data-${{ hashFiles('.gitmodules') }}
36+
restore-keys: |
37+
test-data-
38+
39+
- name: Get test data if not cached
40+
if: steps.test-data-cache.outputs.cache-hit != 'true'
41+
run: |
42+
make get-test-data
43+
44+
- name: Cache Docker images
45+
uses: actions/cache@v4
46+
id: docker-cache
47+
with:
48+
path: |
49+
mongo.tar
50+
api.tar
51+
handler.tar
52+
ui.tar
53+
key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }}
54+
restore-keys: |
55+
docker-local-images-
56+
57+
- name: Setup environment
58+
run: |
59+
# copy the example.env to .env
60+
cp example.env .env
61+
62+
# Update .env to use the hostname
63+
sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env
64+
65+
- name: Build or load images
66+
run: |
67+
if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then
68+
echo "Loading cached images..."
69+
if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then
70+
echo "Failed to load cached images, building instead..."
71+
docker compose -f ${{ env.COMPOSE_FILE }} build
72+
else
73+
rm mongo.tar api.tar handler.tar ui.tar
74+
fi
75+
else
76+
echo "Building images..."
77+
docker compose -f ${{ env.COMPOSE_FILE }} build
78+
fi
79+
80+
# Tag the images to match docker-compose
81+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest
82+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest
83+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest
84+
85+
- name: Clean Up some More Space
86+
run: bash .github/scripts/util_free_space.sh
87+
88+
- name: Setup Apptainer
89+
run: |
90+
sudo apt update
91+
sudo apt install -y software-properties-common
92+
sudo add-apt-repository -y ppa:apptainer/ppa
93+
sudo apt update
94+
sudo apt install -y apptainer
95+
96+
apptainer --version
97+
98+
- name: Build Single Docker Image
99+
run: |
100+
docker build -f EverythingDockerfile -t ezbids-everything .
101+
102+
- name: Build Apptainer Image
103+
run: |
104+
apptainer build ezbids-everything.sif docker-daemon://ezbids-everything:latest
105+
106+
- name: Run Apptainer Image
107+
run: |
108+
apptainer instance start --fakeroot --writable-tmpfs --cleanenv --no-home ezbids-everything.sif ezbids-everything
109+
110+
- name: Install uv
111+
run: |
112+
curl -LsSf https://astral.sh/uv/install.sh | sh
113+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
114+
115+
- name: Verify uv installation
116+
run: uv --version
117+
118+
- name: Verify Apptainer image is running
119+
run: |
120+
apptainer instance list | grep ezbids-everything
121+
if [ $? -ne 0 ]; then
122+
echo "Apptainer image is not running"
123+
exit 1
124+
fi
125+
126+
- name: Run upload test
127+
run: |
128+
make test-upload
129+
130+
- name: Verify Apptainer image is running after upload
131+
run: |
132+
apptainer instance list | grep ezbids-everything
133+
if [ $? -ne 0 ]; then
134+
echo "Apptainer image is not running"
135+
exit 1
136+
fi
137+
138+

‎.github/workflows/docker-build-everything-image.yaml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: Test Docker Build Everything Image
2+
3+
on:
4+
pull_request:
5+
branches: [ master ]
6+
workflow_run:
7+
workflows: ["Docker Build (Local)"]
8+
types:
9+
- completed
10+
workflow_dispatch: # Allow manual triggering
11+
12+
jobs:
13+
Test-Docker-Build-Everything-Image:
14+
runs-on: ubuntu-latest
15+
if: ${{ github.event_name == 'pull_request' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
16+
17+
env:
18+
COMPOSE_FILE: ./docker-compose.yml
19+
REGISTRY_PREFIX: docker.io/library
20+
SERVER_NAME: localhost
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
with:
25+
submodules: true # Ensure we get the submodule info
26+
27+
- name: Set repository name
28+
run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV
29+
30+
- name: Cache test data
31+
uses: actions/cache@v4
32+
id: test-data-cache
33+
with:
34+
path: test/test_data
35+
key: test-data-${{ hashFiles('.gitmodules') }}
36+
restore-keys: |
37+
test-data-
38+
39+
- name: Get test data if not cached
40+
if: steps.test-data-cache.outputs.cache-hit != 'true'
41+
run: |
42+
make get-test-data
43+
44+
- name: Cache Docker images
45+
uses: actions/cache@v4
46+
id: docker-cache
47+
with:
48+
path: |
49+
mongo.tar
50+
api.tar
51+
handler.tar
52+
ui.tar
53+
key: docker-local-images-${{ github.event.workflow_run.head_sha || github.sha }}
54+
restore-keys: |
55+
docker-local-images-
56+
57+
- name: Setup environment
58+
run: |
59+
# copy the example.env to .env
60+
cp example.env .env
61+
62+
# Update .env to use the hostname
63+
sed -i "s/SERVER_NAME=.*/SERVER_NAME=localhost/" .env
64+
65+
- name: Build or load images
66+
run: |
67+
if [ -f mongo.tar ] && [ -f api.tar ] && [ -f handler.tar ] && [ -f ui.tar ]; then
68+
echo "Loading cached images..."
69+
if ! docker load < mongo.tar || ! docker load < api.tar || ! docker load < handler.tar || ! docker load < ui.tar; then
70+
echo "Failed to load cached images, building instead..."
71+
docker compose -f ${{ env.COMPOSE_FILE }} build
72+
else
73+
rm mongo.tar api.tar handler.tar ui.tar
74+
fi
75+
else
76+
echo "Building images..."
77+
docker compose -f ${{ env.COMPOSE_FILE }} build
78+
fi
79+
80+
# Tag the images to match docker-compose
81+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest api:latest
82+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest handler:latest
83+
docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ui:latest
84+
85+
- name: Build Single Docker Image
86+
run: |
87+
docker build -f EverythingDockerfile -t ezbids-everything .
88+
89+
- name: Start Services
90+
run: |
91+
docker run -d -p 27017:27017 -p 8082:8082 -p 3000:3000 -p 8000:8000 -v /tmp/ezbids-workdir:/tmp ezbids-everything
92+
- name: Install uv
93+
run: |
94+
curl -LsSf https://astral.sh/uv/install.sh | sh
95+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
96+
97+
- name: Verify uv installation
98+
run: uv --version
99+
100+
- name: Run upload test
101+
run: |
102+
make test-upload
103+
104+
105+

0 commit comments

Comments
 (0)