Skip to content

Commit 372ef1b

Browse files
riccardopinosioriccardopinosio
committed
cicd improvements
1 parent 819435e commit 372ef1b

8 files changed

Lines changed: 204 additions & 53 deletions

File tree

‎.github/workflows/build-push.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Create image
2+
3+
on: workflow_dispatch
4+
5+
permissions:
6+
contents: write
7+
packages: write
8+
checks: write
9+
attestations: write
10+
id-token: write
11+
12+
env:
13+
REGISTRY: ghcr.io
14+
IMAGE_NAME: ${{ github.repository }}
15+
16+
jobs:
17+
build-and-push:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v4
22+
23+
- name: Log in to the Container registry
24+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
25+
with:
26+
registry: ${{ env.REGISTRY }}
27+
username: ${{ github.actor }}
28+
password: ${{ secrets.GITHUB_TOKEN }}
29+
30+
- name: Extract metadata (tags, labels) for Docker
31+
id: meta
32+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
33+
with:
34+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
35+
36+
- name: Build and push Docker image
37+
id: push
38+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
39+
with:
40+
context: .
41+
push: true
42+
tags: ${{ steps.meta.outputs.tags }}
43+
labels: ${{ steps.meta.outputs.labels }}
44+
45+
- name: Generate artifact attestation
46+
uses: actions/attest-build-provenance@v1
47+
with:
48+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
49+
subject-digest: ${{ steps.push.outputs.digest }}
50+
push-to-registry: true

‎.github/workflows/release.yaml‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,14 @@ on:
77

88
permissions:
99
contents: write
10+
packages: write
1011
checks: write
12+
attestations: write
13+
id-token: write
14+
15+
env:
16+
REGISTRY: ghcr.io
17+
IMAGE_NAME: ${{ github.repository }}
1118

1219
jobs:
1320
test:
@@ -56,6 +63,43 @@ jobs:
5663
report_paths: './testTarget/unit/*.xml'
5764
fail_on_failure: true
5865
require_tests: true
66+
67+
build-and-push:
68+
runs-on: ubuntu-latest
69+
needs: test
70+
steps:
71+
- name: Checkout repository
72+
uses: actions/checkout@v4
73+
74+
- name: Log in to the Container registry
75+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
76+
with:
77+
registry: ${{ env.REGISTRY }}
78+
username: ${{ github.actor }}
79+
password: ${{ secrets.GITHUB_TOKEN }}
80+
81+
- name: Extract metadata (tags, labels) for Docker
82+
id: meta
83+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
84+
with:
85+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
86+
87+
- name: Build and push Docker image
88+
id: push
89+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
90+
with:
91+
context: .
92+
push: true
93+
tags: ${{ steps.meta.outputs.tags }}
94+
labels: ${{ steps.meta.outputs.labels }}
95+
96+
- name: Generate artifact attestation
97+
uses: actions/attest-build-provenance@v1
98+
with:
99+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
100+
subject-digest: ${{ steps.push.outputs.digest }}
101+
push-to-registry: true
102+
59103
release:
60104
name: Release
61105
runs-on: ubuntu-latest
@@ -78,11 +122,11 @@ jobs:
78122
- uses: actions/download-artifact@v4
79123
with:
80124
name: onnxruntime-linux-x64-gpu
81-
path: .
125+
path: ./onnxruntime-linux-x64-gpu
82126
- name: Display structure of downloaded files
83127
run: ls -R
84128
- uses: ncipollo/release-action@v1
85129
with:
86-
artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu.zip, hugot-cli-linux-x64"
130+
artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu, hugot-cli-linux-x64"
87131
generateReleaseNotes: true
88132
skipIfReleaseExists: true

‎Dockerfile‎

Lines changed: 11 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
#--- dockerfile with hugot dependencies and cli (cpu only) ---
2+
13
ARG GO_VERSION=1.22.5
24
ARG RUST_VERSION=1.79
35
ARG ONNXRUNTIME_VERSION=1.18.0
46
ARG BUILD_PLATFORM=linux/amd64
5-
ARG CGO_LDFLAGS="-L./usr/lib/libtokenizers.a"
7+
68
#--- rust build of tokenizer ---
79

810
FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer
@@ -11,23 +13,16 @@ RUN git clone https://github.com/knights-analytics/tokenizers -b namespace && \
1113
cd tokenizers && \
1214
cargo build --release
1315

14-
#--- build and test layer ---
16+
#--- build layer ---
1517

1618
FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-build
1719
ARG GO_VERSION
1820
ARG ONNXRUNTIME_VERSION
19-
ARG CGO_LDFLAGS
2021

2122
RUN dnf -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git && \
2223
ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \
2324
dnf install -y 'dnf-command(config-manager)' && \
2425
dnf config-manager --add-repo https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/x86_64/os/ && \
25-
# from fedora
26-
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora39/x86_64/cuda-fedora39.repo && \
27-
dnf install -y cuda-cudart-12-4 libcublas-12-4 libcurand-12-4 libcufft-12-4 && \
28-
# from rhel
29-
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo && \
30-
dnf install -y libcudnn8 && \
3126
dnf clean all
3227

3328
# go
@@ -42,38 +37,16 @@ COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtok
4237
# onnxruntime cpu and gpu
4338
RUN curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
4439
tar -xzf onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
45-
mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so && \
46-
curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
47-
tar -xzf onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
48-
mv ./onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/lib /usr/lib64/onnxruntime-gpu
49-
50-
# build gotestsum and test2json
51-
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w" cmd/test2json && mv test2json /usr/local/bin/test2json && \
52-
curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.12.0/gotestsum_1.12.0_linux_amd64.tar.gz && \
53-
tar -xzf gotestsum_1.12.0_linux_amd64.tar.gz --directory /usr/local/bin
40+
mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so
5441

5542
# build cli binary
5643
COPY . /build
5744
WORKDIR /build
58-
RUN cd ./cmd && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
45+
RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
5946

60-
# NON-PRIVILEDGED USER
61-
# create non-priviledged testuser with id: 1000
62-
RUN dnf install --disablerepo=* --enablerepo=amazonlinux --allowerasing -y dirmngr sudo which && dnf clean all
63-
RUN useradd -u 1000 -m testuser && chown -R testuser:testuser /build && usermod -a -G wheel testuser
64-
RUN echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser
47+
#--- final layer ---
48+
FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS final
6549

66-
# ENTRYPOINT
67-
COPY ./scripts/entrypoint.sh /entrypoint.sh
68-
# convert windows line endings if present
69-
RUN sed -i 's/\r//g' /entrypoint.sh
70-
RUN chmod +x /entrypoint.sh
71-
ENTRYPOINT ["/entrypoint.sh"]
72-
73-
# artifacts layer
74-
FROM --platform=$BUILD_PLATFORM scratch AS artifacts
75-
76-
COPY --from=hugot-build /usr/lib64/onnxruntime.so onnxruntime-linux-x64.so
77-
COPY --from=hugot-build /usr/lib64/onnxruntime-gpu onnxruntime-linux-x64-gpu
78-
COPY --from=hugot-build /usr/lib/libtokenizers.a libtokenizers.a
79-
COPY --from=hugot-build /build/cmd/target /hugot-cli-linux-x64
50+
COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtokenizers.a
51+
COPY --from=hugot-build /build/cmd/target /hugot-cli
52+
COPY --from=hugot-build /usr/lib64/onnxruntime.so /usr/lib64/onnxruntime.so

‎Dockerfile.test‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#--- dockerfile to test hugot ---
2+
3+
ARG GO_VERSION=1.22.5
4+
ARG RUST_VERSION=1.79
5+
ARG ONNXRUNTIME_VERSION=1.18.0
6+
ARG BUILD_PLATFORM=linux/amd64
7+
8+
#--- rust build of tokenizer ---
9+
10+
FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer
11+
12+
RUN git clone https://github.com/knights-analytics/tokenizers -b namespace && \
13+
cd tokenizers && \
14+
cargo build --release
15+
16+
#--- build and test layer ---
17+
18+
FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-build
19+
ARG GO_VERSION
20+
ARG ONNXRUNTIME_VERSION
21+
22+
RUN dnf -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git && \
23+
ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \
24+
dnf install -y 'dnf-command(config-manager)' && \
25+
dnf config-manager --add-repo https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/x86_64/os/ && \
26+
# from fedora
27+
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora39/x86_64/cuda-fedora39.repo && \
28+
dnf install -y cuda-cudart-12-4 libcublas-12-4 libcurand-12-4 libcufft-12-4 && \
29+
# from rhel
30+
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo && \
31+
dnf install -y libcudnn8 && \
32+
dnf clean all
33+
34+
# go
35+
RUN curl -LO https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz && \
36+
tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \
37+
rm go${GO_VERSION}.linux-amd64.tar.gz
38+
ENV PATH="$PATH:/usr/local/go/bin"
39+
40+
# tokenizer
41+
COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtokenizers.a
42+
43+
# onnxruntime cpu and gpu
44+
RUN curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
45+
tar -xzf onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
46+
mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so && \
47+
curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
48+
tar -xzf onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
49+
mv ./onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/lib /usr/lib64/onnxruntime-gpu
50+
51+
# build gotestsum and test2json
52+
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w" cmd/test2json && mv test2json /usr/local/bin/test2json && \
53+
curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.12.0/gotestsum_1.12.0_linux_amd64.tar.gz && \
54+
tar -xzf gotestsum_1.12.0_linux_amd64.tar.gz --directory /usr/local/bin
55+
56+
# build cli binary
57+
COPY . /build
58+
WORKDIR /build
59+
RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
60+
61+
# NON-PRIVILEDGED USER
62+
# create non-priviledged testuser with id: 1000
63+
RUN dnf install --disablerepo=* --enablerepo=amazonlinux --allowerasing -y dirmngr sudo which && dnf clean all
64+
RUN useradd -u 1000 -m testuser && chown -R testuser:testuser /build && usermod -a -G wheel testuser
65+
RUN echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser
66+
67+
# ENTRYPOINT
68+
COPY ./scripts/entrypoint.sh /entrypoint.sh
69+
# convert windows line endings if present
70+
RUN sed -i 's/\r//g' /entrypoint.sh
71+
RUN chmod +x /entrypoint.sh
72+
ENTRYPOINT ["/entrypoint.sh"]
73+
74+
# artifacts layer
75+
FROM --platform=$BUILD_PLATFORM scratch AS artifacts
76+
77+
COPY --from=hugot-build /usr/lib64/onnxruntime.so onnxruntime-linux-x64.so
78+
COPY --from=hugot-build /usr/lib64/onnxruntime-gpu onnxruntime-linux-x64-gpu
79+
COPY --from=hugot-build /usr/lib/libtokenizers.a libtokenizers.a
80+
COPY --from=hugot-build /build/cmd/target /hugot-cli-linux-x64

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ Hugot can be used in two ways: as a library in your go application, or as a comm
4545

4646
To use Hugot as a library in your application, you will need the following two dependencies on your system:
4747

48-
- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it.
48+
- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it. Alternatively, you can explicitly specify the path to the folder with the `libtokenizers.a` file using the `CGO_LDFLAGS` env variable, see the [dockerfile](./Dockerfile).
4949
- the onnxruntime.go file obtained from the releases section of this page (if you want to use alternative architectures from `linux/amd64` you will have to download it from [the onnxruntime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example). Hugot looks for this file at /usr/lib/onnxruntime.so or /usr/lib64/onnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewSession()`, e.g:
5050

5151
```

‎compose-dev.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ services:
44
platform: linux/amd64
55
build:
66
context: .
7-
dockerfile: ./Dockerfile
7+
dockerfile: ./Dockerfile.test
88
target: hugot-build
99
volumes:
1010
- $src_dir:/home/testuser/repositories/hugot

‎compose-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ services:
55
container_name: hugot
66
build:
77
context: .
8-
dockerfile: ./Dockerfile
8+
dockerfile: ./Dockerfile.test
99
target: hugot-build
1010
volumes:
1111
- $test_folder:/test

‎scripts/run-unit-tests.sh‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,24 @@ set -e
44

55
# Directory of *this* script
66
this_dir="$( cd "$( dirname "$0" )" && pwd )"
7-
export src_dir="$(realpath "${this_dir}/..")"
7+
src_dir="$(realpath "${this_dir}/..")"
8+
export src_dir
89

9-
export commit_hash=$(git rev-parse --short HEAD)
10-
export test_folder="$src_dir/testTarget"
11-
mkdir -p $test_folder
12-
export host_uid=$(id -u "$USER")
10+
commit_hash=$(git rev-parse --short HEAD)
11+
export commit_hash
12+
test_folder="$src_dir/testTarget"
13+
export test_folder
14+
mkdir -p "$test_folder"
15+
host_uid=$(id -u "$USER")
16+
export host_uid
1317

1418
# build with compose
15-
docker compose -f $src_dir/compose-test.yaml build
19+
docker compose -f "$src_dir/compose-test.yaml" build
1620
echo "Running tests for commit hash: $commit_hash"
17-
docker compose -f $src_dir/compose-test.yaml up && \
18-
docker compose -f $src_dir/compose-test.yaml logs --no-color >& $test_folder/logs.txt
19-
docker compose -f $src_dir/compose-test.yaml rm -fsv
21+
docker compose -f "$src_dir/compose-test.yaml" up && \
22+
docker compose -f "$src_dir/compose-test.yaml" logs --no-color >& "$test_folder/logs.txt"
23+
docker compose -f "$src_dir/compose-test.yaml" rm -fsv
2024

2125
echo "Extracting lib artifacts"
22-
docker build . --output "$src_dir/artifacts" --target artifacts
26+
docker build -f ./Dockerfile.test . --output "$src_dir/artifacts" --target artifacts
2327
echo "lib artifacts extracted"

0 commit comments

Comments
 (0)