Skip to content

Commit 19411ed

Browse files
SalimKayalleafty
SalimKayal
and
leafty
authored
feat: Add multi-arch support, update run-image, and streamline build process (#97)
Co-authored-by: Flora Thiebaut <flora.thiebaut@sdsc.ethz.ch>
1 parent 66b0754 commit 19411ed

19 files changed

Lines changed: 469 additions & 96 deletions

File tree

‎.github/workflows/test.yaml‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,47 @@ jobs:
7272
needs:
7373
- build-devcontainer
7474
steps:
75+
- name: Free Disk Space (Ubuntu)
76+
uses: jlumbroso/free-disk-space@main
77+
with:
78+
tool-cache: true
79+
7580
- uses: actions/checkout@v4
81+
82+
- name: Set up Docker Buildx
83+
uses: docker/setup-buildx-action@v3
84+
with:
85+
driver: docker-container
86+
87+
- name: Login to GHCR
88+
uses: docker/login-action@v3
89+
with:
90+
registry: ghcr.io
91+
username: ${{ github.actor }}
92+
password: ${{ secrets.GITHUB_TOKEN }}
93+
94+
- name: Docker metadata
95+
id: meta
96+
uses: docker/metadata-action@v5
97+
with:
98+
images: ghcr.io/${{ github.repository }}/run-image
99+
tags: |
100+
type=ref,event=tag
101+
type=sha,format=short
102+
103+
- name: Build and push run-image
104+
uses: docker/build-push-action@v5
105+
with:
106+
context: ./run-image
107+
file: ./run-image/Dockerfile
108+
platforms: linux/amd64
109+
target: run
110+
push: true
111+
tags: ${{ steps.meta.outputs.tags }}
112+
labels: io.buildpacks.base.distro.name=ubuntu,io.buildpacks.base.distro.version=24.04
113+
cache-from: type=gha
114+
cache-to: type=gha,mode=max
115+
76116
- name: Run linting
77117
id: lint
78118
uses: devcontainers/ci@v0.3
@@ -82,11 +122,17 @@ jobs:
82122
push: never
83123
skipContainerUserIdUpdate: false
84124
cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
85-
- name: Run tests
125+
126+
- name: Run tests with custom run image
86127
id: tests
87128
uses: devcontainers/ci@v0.3
88129
with:
89130
runCmd: |
131+
go tool tomljson builders/selector/builder.toml | \
132+
go tool yq -p json -o json '.run.images[0].image="'"${{ steps.meta.outputs.tags }}"'"' | \
133+
go tool jsontoml > builders/selector/builder.toml.tmp && \
134+
mv builders/selector/builder.toml.tmp builders/selector/builder.toml
135+
cat builders/selector/builder.toml # Print for debugging
90136
make tests
91137
push: never
92138
skipContainerUserIdUpdate: false

‎.github/workflows/update-versions.yml‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ jobs:
3737
run: |
3838
echo "REPO=${GITHUB_REPOSITORY@L}" >> "${GITHUB_OUTPUT}"
3939
40-
build-and-push-run-image:
40+
build-and-push-build-run-images:
4141
if: github.event_name == 'workflow_dispatch'
4242
runs-on: ubuntu-latest
4343
needs: validate-tag
@@ -66,15 +66,46 @@ jobs:
6666
with:
6767
context: ./run-image
6868
file: ./run-image/Dockerfile
69+
platforms: linux/amd64,linux/arm64
70+
target: run
6971
push: true
70-
tags: ghcr.io/${{ needs.validate-tag.outputs.repo_lower }}/base-image:${{ github.event.inputs.version }}
72+
tags: ghcr.io/${{ needs.validate-tag.outputs.repo_lower }}/run-image:${{ github.event.inputs.version }}
73+
labels: io.buildpacks.base.distro.name=ubuntu,io.buildpacks.base.distro.version=24.04
74+
cache-from: type=gha
75+
cache-to: type=gha,mode=max
76+
77+
- name: Build and push cuda run-image
78+
uses: docker/build-push-action@v5
79+
with:
80+
context: ./run-image
81+
file: ./run-image/Dockerfile
82+
platforms: linux/amd64,linux/arm64
83+
build-args: base_image=nvcr.io/nvidia/cuda-dl-base:25.10-cuda13.0-devel-ubuntu24.04
84+
target: run
85+
push: true
86+
tags: ghcr.io/${{ needs.validate-tag.outputs.repo_lower }}/cuda-run-image:${{ github.event.inputs.version }}
87+
labels: io.buildpacks.base.distro.name=ubuntu,io.buildpacks.base.distro.version=24.04
88+
cache-from: type=gha
89+
cache-to: type=gha,mode=max
90+
91+
- name: Build and push cuda build-image
92+
uses: docker/build-push-action@v5
93+
with:
94+
context: ./run-image
95+
file: ./run-image/Dockerfile
96+
platforms: linux/amd64,linux/arm64
97+
build-args: base_image=nvcr.io/nvidia/cuda-dl-base:25.10-cuda13.0-devel-ubuntu24.04
98+
target: build
99+
push: true
100+
tags: ghcr.io/${{ needs.validate-tag.outputs.repo_lower }}/cuda-build-image:${{ github.event.inputs.version }}
101+
labels: io.buildpacks.base.distro.name=ubuntu,io.buildpacks.base.distro.version=24.04
71102
cache-from: type=gha
72103
cache-to: type=gha,mode=max
73104

74105
create-version-update-pr:
75106
if: github.event_name == 'workflow_dispatch'
76107
runs-on: ubuntu-latest
77-
needs: [validate-tag, build-and-push-run-image]
108+
needs: [validate-tag, build-and-push-build-run-images]
78109
permissions:
79110
contents: write
80111
pull-requests: write
@@ -147,7 +178,7 @@ jobs:
147178
username: ${{ github.actor }}
148179
password: ${{ secrets.GITHUB_TOKEN }}
149180

150-
- uses: buildpacks/github-actions/setup-pack@v5.9.3
181+
- uses: buildpacks/github-actions/setup-pack@v5.9.5
151182

152183
- name: publish buildpacks
153184
run: make publish_buildpacks

‎Makefile‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ endef
4444
.PHONY: all buildpacks builders samples run_image
4545

4646
## Tools
47+
TOMLJSON = $(LOCALBIN)/tomljson
48+
JSONTOML = $(LOCALBIN)/jsontoml
49+
GO_TOML_VERSION ?= v2.2.4
50+
4751
YQ = $(LOCALBIN)/yq
4852
YQ_VERSION ?= v4.45.1
4953

@@ -56,6 +60,16 @@ SHELLCHECK_VERSION ?= "v0.10.0"
5660
GOLANG_CI_LINT = $(LOCALBIN)/golangci-lint
5761
GOLANG_CI_LINT_VERSION ?= "v2.1.5"
5862

63+
.PHONY: tomljson
64+
tomljson: $(TOMLJSON)
65+
$(TOMLJSON): $(LOCALBIN)
66+
$(call go-install-tool, $(TOMLJSON),github.com/pelletier/go-toml/v2/cmd/tomljson,$(GO_TOML_VERSION))
67+
68+
.PHONY: jsontoml
69+
jsontoml: $(JSONTOML)
70+
$(JSONTOML): $(LOCALBIN)
71+
$(call go-install-tool, $(JSONTOML),github.com/pelletier/go-toml/v2/cmd/jsontoml,$(GO_TOML_VERSION))
72+
5973
.PHONY: yq
6074
yq: $(YQ)
6175
$(YQ): $(LOCALBIN)
@@ -107,11 +121,11 @@ REGISTRY_HOST=ghcr.io
107121
REGISTRY_REPO=swissdatasciencecenter/renku-frontend-buildpacks
108122

109123
run_image:
110-
bash ./scripts/publish_run_image.sh $(REGISTRY_HOST)/$(REGISTRY_REPO)/base-image
124+
bash ./scripts/publish_run_image.sh $(REGISTRY_HOST)/$(REGISTRY_REPO)/run-image
111125

112126
.PHONY: publish_run_image
113127
publish_run_image: run_image
114-
bash ./scripts/publish_run_image.sh $(REGISTRY_HOST)/$(REGISTRY_REPO)/base-image --publish
128+
bash ./scripts/publish_run_image.sh $(REGISTRY_HOST)/$(REGISTRY_REPO)/run-image --publish
115129

116130
.PHONY: publish_buildpacks
117131
publish_buildpacks: yq
@@ -161,7 +175,11 @@ update-builder-versions:
161175
@echo "Updating builder versions to $(RELEASE_VERSION)..."
162176
@for builder in $(BUILDERS); do \
163177
FILE="builders/$$builder/builder.toml"; \
164-
./scripts/update_builder_versions.sh "$(REGISTRY_HOST)/$(REGISTRY_REPO)/base-image" "$(RELEASE_VERSION)" "$$FILE"; \
178+
if [ $$(grep $(REGISTRY_REPO) $$FILE|wc -l) -eq 2 ];then \
179+
./scripts/update_builder_versions.sh "$(RELEASE_VERSION)" "$$FILE" true; \
180+
else \
181+
./scripts/update_builder_versions.sh "$(RELEASE_VERSION)" "$$FILE"; \
182+
fi \
165183
done
166184

167185
.PHONY: update-action-versions

‎builders/cuda-selector/builder.toml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
description = "CUDA enabled builder for Renku frontends and environments."
2+
3+
[[build.env]]
4+
name = "BP_RENKU_FRONTENDS"
5+
value = "vscodium"
6+
7+
[[buildpacks]]
8+
uri = "docker://ghcr.io/swissdatasciencecenter/vscodium-buildpack/vscodium:0.5.1"
9+
id = "vscodium"
10+
version = "0.5.1"
11+
12+
[[buildpacks]]
13+
uri = "../../buildpacks/conda-nodefaults"
14+
version = "0.1.0"
15+
16+
[[buildpacks]]
17+
uri = "../../buildpacks/kernel-installer"
18+
version = "0.1.0"
19+
20+
[[buildpacks]]
21+
uri = "../../buildpacks/jupyterlab"
22+
version = "0.1.0"
23+
24+
[[buildpacks]]
25+
uri = "../../buildpacks/ttyd"
26+
version = "0.1.0"
27+
28+
[[buildpacks]]
29+
uri = "../../buildpacks/python-dependency-manager"
30+
version = "0.1.0"
31+
32+
[[buildpacks]]
33+
uri = "../../buildpacks/rstudio"
34+
version = "0.1.0"
35+
36+
[[buildpacks]]
37+
uri = "docker://docker.io/paketobuildpacks/python:2.38.0"
38+
version = "2.38.0"
39+
40+
[[buildpacks]]
41+
id = "paketo-buildpacks/miniconda"
42+
uri = "docker://docker.io/paketobuildpacks/miniconda:0.11.5"
43+
version = "0.11.5"
44+
45+
[lifecycle]
46+
version = "0.20.16"
47+
48+
[[order]]
49+
50+
[[order.group]]
51+
id = "renku/conda-nodefaults"
52+
version = "0.1.0"
53+
optional = true
54+
[[order.group]]
55+
id = "paketo-buildpacks/miniconda"
56+
version = "0.11.5"
57+
[[order.group]]
58+
id = "paketo-buildpacks/python"
59+
version = "2.38.0"
60+
[[order.group]]
61+
id = "renku/python-dependency-manager"
62+
version = "0.1.0"
63+
[[order.group]]
64+
id = "renku/jupyterlab"
65+
version = "0.1.0"
66+
[[order.group]]
67+
id = "renku/kernel-installer"
68+
version = "0.1.0"
69+
70+
[[order]]
71+
[[order.group]]
72+
id = "renku/conda-nodefaults"
73+
version = "0.1.0"
74+
optional = true
75+
[[order.group]]
76+
id = "paketo-buildpacks/python"
77+
version = "2.38.0"
78+
[[order.group]]
79+
id = "renku/python-dependency-manager"
80+
version = "0.1.0"
81+
[[order.group]]
82+
id = "renku/ttyd"
83+
version = "0.1.0"
84+
85+
[[order]]
86+
87+
[[order.group]]
88+
id = "renku/conda-nodefaults"
89+
version = "0.1.0"
90+
optional = true
91+
[[order.group]]
92+
id = "paketo-buildpacks/python"
93+
version = "2.38.0"
94+
[[order.group]]
95+
id = "renku/python-dependency-manager"
96+
version = "0.1.0"
97+
[[order.group]]
98+
id = "vscodium"
99+
version = "0.5.1"
100+
101+
[[order]]
102+
103+
[[order.group]]
104+
id = "renku/conda-nodefaults"
105+
version = "0.1.0"
106+
optional = true
107+
[[order.group]]
108+
id = "paketo-buildpacks/miniconda"
109+
version = "0.11.5"
110+
[[order.group]]
111+
id = "renku/rstudio"
112+
version = "0.1.0"
113+
114+
[build]
115+
image = "ghcr.io/swissdatasciencecenter/renku-frontend-buildpacks/cuda-build-image:0.1.0"
116+
117+
[run]
118+
[[run.images]]
119+
image = "ghcr.io/swissdatasciencecenter/renku-frontend-buildpacks/cuda-run-image:0.1.0"
120+
mirrors = []
121+
122+
[[targets]]
123+
os = "linux"
124+
arch = "amd64"
125+
126+
[[targets]]
127+
os = "linux"
128+
arch = "arm64"

0 commit comments

Comments
 (0)