Skip to content

Commit 3ce4a63

Browse files
dereknoladereknola
authored
Build k3s overhaul (#12200)
* Add full ci support without Dapper * Seperate git and other version tags, improves caching on binary builds * Use new local targets for build-k3s.yaml workflow * Allow optional ghcr build caching * Build binary using GHA native commands * Use internal setup-go action for e2e.yaml * Add emulation builds to k3s-build.yaml (for arm32 and future riscv64) * Be consistent in k3s artifact names * Fix package/dockerfile warnings * Fix install script for PR installs Signed-off-by: Derek Nola <[email protected]>
1 parent 1d104e3 commit 3ce4a63

13 files changed

+208
-65
lines changed

‎.drone.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,7 @@ steps:
576576
commands:
577577
- DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test .
578578
- apk add make git bash
579-
- GOCOVER=1 make local
579+
- GOCOVER=1 make local-binary
580580
- cp dist/artifacts/* /tmp/artifacts/
581581
volumes:
582582
- name: cache

‎.github/workflows/build-k3s.yaml

+99-15
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
name: Build K3s
2-
31
on:
42
workflow_call:
53
inputs:
64
arch:
75
type: string
8-
description: 'Architecture to build (ubuntu-latest or ubuntu-24.04-arm)'
9-
default: 'ubuntu-latest'
6+
description: 'Architecture to build (amd64, arm64, or arm)'
7+
default: 'amd64'
108
os:
119
type: string
1210
description: 'Target OS (linux or windows)'
@@ -15,31 +13,117 @@ on:
1513
type: boolean
1614
required: false
1715
default: false
16+
cache:
17+
type: string
18+
description: 'Cache mode: "read", "write", or empty for no cache'
19+
required: false
20+
default: ''
21+
22+
# Note that is workflow requires the following permissions:
23+
# contents: read
24+
# If using the cache: write option, you will need:
25+
# packages: write
26+
# If using the cache: read option, you will need:
27+
# packages: read
28+
1829

19-
permissions:
20-
contents: read
2130

2231
jobs:
2332
build:
24-
name: Build K3s (${{ inputs.os }} on ${{ inputs.arch }})
25-
runs-on: ${{ inputs.arch }}
33+
name: Build # DO NOT CHANGE THIS NAME, we rely on it for INSTALL_K3S_PR functionality
34+
runs-on: ${{ contains(inputs.arch, 'arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
2635
timeout-minutes: 20
2736
env:
2837
BIN_EXT: ${{ inputs.os == 'windows' && '.exe' || '' }}
29-
ARTIFACT_EXT: ${{ inputs.os == 'windows' && '-windows' || (contains(inputs.arch, 'arm') && '-arm64' || '') }}
38+
ARCH_EXT: ${{ inputs.os == 'windows' && '-windows' || format('-{0}', inputs.arch) }}
3039
GOOS: ${{ inputs.os }}
3140
steps:
3241
- name: Checkout K3s
3342
uses: actions/checkout@v4
34-
35-
- name: Build K3s binary
43+
44+
- name: Set up QEMU
45+
if: inputs.arch == 'arm'
46+
uses: docker/setup-qemu-action@v3
47+
with:
48+
platforms: linux/arm/v7
49+
50+
- name: Set up Docker Buildx
51+
uses: docker/setup-buildx-action@v3
52+
53+
- name: Determine Git Version Info
54+
id: git_vars
55+
run: |
56+
source ./scripts/git_version.sh
57+
{
58+
echo "git_tag=${GIT_TAG}"
59+
echo "tree_state=${TREE_STATE}"
60+
echo "commit=${COMMIT}"
61+
echo "dirty=${DIRTY}"
62+
} >> "$GITHUB_OUTPUT"
63+
64+
- name: Login to GitHub Container Registry
65+
if: inputs.cache == 'write'
66+
uses: docker/login-action@v3
67+
with:
68+
registry: ghcr.io
69+
username: ${{ github.repository_owner }}
70+
password: ${{ secrets.GITHUB_TOKEN }}
71+
72+
- name: Build K3s Binary Native
73+
if: inputs.arch != 'arm'
74+
env:
75+
DOCKER_BUILD_SUMMARY: false
76+
uses: docker/build-push-action@v6
77+
with:
78+
context: .
79+
file: ./Dockerfile.local
80+
target: result
81+
# Defined actions like this don't ingest GITHUB_ENV, so use outputs
82+
# and manual set the build arguments
83+
build-args: |
84+
GIT_TAG=${{ steps.git_vars.outputs.git_tag }}
85+
TREE_STATE=${{ steps.git_vars.outputs.tree_state }}
86+
COMMIT=${{ steps.git_vars.outputs.commit }}
87+
DIRTY=${{ steps.git_vars.outputs.dirty }}
88+
cache-from: ${{ inputs.cache != '' && format('type=registry,ref=ghcr.io/{0}:cache-{1}', github.repository, inputs.arch) || '' }}
89+
cache-to: ${{ inputs.cache == 'write' && format('type=registry,ref=ghcr.io/{0}:cache-{1},mode=max', github.repository, inputs.arch) || '' }}
90+
push: false
91+
provenance: mode=min
92+
outputs: type=local,dest=.
93+
94+
- name: Build K3s Binary Emulated
95+
if: inputs.arch != 'arm64' && inputs.arch != 'amd64'
96+
env:
97+
PLATFORM: ${{ inputs.arch == 'arm' && 'linux/arm/v7' || format('linux/{0}', inputs.arch) }}
98+
DOCKER_BUILD_SUMMARY: false
99+
uses: docker/build-push-action@v6
100+
with:
101+
context: .
102+
file: ./Dockerfile.local
103+
target: result
104+
build-args: |
105+
GIT_TAG=${{ steps.git_vars.outputs.git_tag }}
106+
TREE_STATE=${{ steps.git_vars.outputs.tree_state }}
107+
COMMIT=${{ steps.git_vars.outputs.commit }}
108+
DIRTY=${{ steps.git_vars.outputs.dirty }}
109+
push: false
110+
provenance: mode=min
111+
platforms: ${{ env.PLATFORM }}
112+
outputs: type=local,dest=.
113+
114+
- name: Caculate binary checksum
36115
run: |
37-
DOCKER_BUILDKIT=1 SKIP_IMAGE=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 GOCOVER=1 GOOS=${{ env.GOOS }} make
38-
sha256sum dist/artifacts/k3s${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.BIN_EXT }}.sha256sum
116+
if [ ${{ inputs.arch }} == 'amd64' ]; then
117+
sha256sum dist/artifacts/k3s${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.BIN_EXT }}.sha256sum
118+
elif [ ${{ inputs.arch }} == "arm" ]; then
119+
sha256sum dist/artifacts/k3s-armhf | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.ARCH_EXT }}.sha256sum
120+
else
121+
sha256sum dist/artifacts/k3s${{ env.ARCH_EXT }}${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.ARCH_EXT }}${{ env.BIN_EXT }}.sha256sum
122+
fi
39123
40124
- name: Build K3s image
41125
if: inputs.upload-image == true && inputs.os == 'linux'
42-
run: make package-image
126+
run: ./scripts/package-image
43127

44128
- name: "Save K3s image"
45129
if: inputs.upload-image == true && inputs.os == 'linux'
@@ -48,5 +132,5 @@ jobs:
48132
- name: "Upload K3s Artifacts"
49133
uses: actions/upload-artifact@v4
50134
with:
51-
name: k3s${{ env.ARTIFACT_EXT }}
135+
name: k3s${{ env.ARCH_EXT }}
52136
path: dist/artifacts/k3s*

‎.github/workflows/e2e.yaml

+13-15
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,22 @@ permissions:
2929

3030
jobs:
3131
build:
32+
permissions:
33+
contents: read
34+
packages: write # permissions cannot be conditional, so we need to set this for all jobs
3235
uses: ./.github/workflows/build-k3s.yaml
3336
with:
3437
upload-image: true
38+
cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }}
3539
build-arm64:
3640
uses: ./.github/workflows/build-k3s.yaml
41+
permissions:
42+
contents: read
43+
packages: write
3744
with:
38-
arch: ubuntu-24.04-arm
45+
arch: arm64
3946
upload-image: true
47+
cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }}
4048
e2e:
4149
name: "E2E Tests"
4250
needs: build
@@ -71,18 +79,15 @@ jobs:
7179
- name: "Vagrant Plugin(s)"
7280
run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
7381
- name: Install Go
74-
uses: actions/setup-go@v5
75-
with:
76-
go-version-file: 'go.mod'
77-
cache: false
82+
uses: ./.github/actions/setup-go
7883
- name: Install Kubectl
7984
run: |
8085
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
8186
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
8287
- name: "Download k3s binary"
8388
uses: actions/download-artifact@v4
8489
with:
85-
name: k3s
90+
name: k3s-amd64
8691
path: ./dist/artifacts
8792

8893
- name: Run ${{ matrix.etest }} Test
@@ -177,17 +182,10 @@ jobs:
177182
steps:
178183
- name: Checkout
179184
uses: actions/checkout@v4
180-
- name: "Download K3s image (amd64)"
181-
if: ${{ matrix.arch == 'amd64' }}
185+
- name: "Download K3s image"
182186
uses: actions/download-artifact@v4
183187
with:
184-
name: k3s
185-
path: ./dist/artifacts
186-
- name: "Download K3s image (arm64)"
187-
if: ${{ matrix.arch == 'arm64' }}
188-
uses: actions/download-artifact@v4
189-
with:
190-
name: k3s-arm64
188+
name: k3s-${{ matrix.arch }}
191189
path: ./dist/artifacts
192190
- name: Load and set K3s image
193191
run: |

‎.github/workflows/install.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
- name: "Download k3s binary"
5555
uses: actions/download-artifact@v4
5656
with:
57-
name: k3s
57+
name: k3s-amd64
5858
path: tests/install/${{ matrix.vm }}
5959
- name: "Vagrant Up"
6060
run: vagrant up --no-tty --no-provision

‎.github/workflows/integration.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,13 @@ env:
2929

3030
jobs:
3131
build:
32+
permissions:
33+
contents: read
34+
packages: read
3235
uses: ./.github/workflows/build-k3s.yaml
3336
with:
3437
os: linux
38+
cache: read
3539
build-windows:
3640
uses: ./.github/workflows/build-k3s.yaml
3741
with:
@@ -56,7 +60,7 @@ jobs:
5660
- name: "Download k3s binary"
5761
uses: actions/download-artifact@v4
5862
with:
59-
name: k3s
63+
name: k3s-amd64
6064
path: ./dist/artifacts
6165
- name: Run Integration Tests
6266
run: |

‎.github/workflows/trivy.yaml

+1-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ jobs:
3636

3737
- name: Build K3s Image
3838
run: |
39-
make local
40-
make package-image
39+
make local-image
4140
make tag-image-latest
4241
4342
- name: Download Rancher's VEX Hub report

‎Dockerfile.local

+22-5
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,36 @@ ENV SRC_DIR=/go/src/github.com/k3s-io/k3s
3232
WORKDIR ${SRC_DIR}/
3333

3434

35-
FROM infra AS build
36-
37-
ARG SKIP_VALIDATE
38-
35+
FROM infra AS manifests
36+
ARG GIT_TAG
37+
ARG TREE_STATE
38+
ARG COMMIT
39+
ARG DIRTY
40+
ARG GOOS
41+
ENV NO_DAPPER=true
42+
# Used by both build and validate stages, better caching if we do this in a separate stage
3943
COPY ./scripts/ ./scripts
4044
COPY ./go.mod ./go.sum ./main.go ./
4145
COPY ./manifests ./manifests
4246
RUN mkdir -p bin dist
4347
RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
4448
./scripts/download
4549

50+
51+
FROM manifests AS validate
52+
ARG SKIP_VALIDATE
53+
COPY . .
54+
RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
55+
--mount=type=cache,id=gobuild,target=/root/.cache/go-build \
56+
--mount=type=cache,id=lint,target=/root/.cache/golangci-lint \
57+
./scripts/validate
58+
59+
60+
FROM manifests AS build
61+
ARG GOCOVER
62+
ARG DEBUG
4663
COPY ./cmd ./cmd
4764
COPY ./tests ./tests
48-
COPY ./.git ./.git
4965
COPY ./pkg ./pkg
5066
RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
5167
--mount=type=cache,id=gobuild,target=/root/.cache/go-build \
@@ -58,6 +74,7 @@ RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
5874

5975
RUN ./scripts/binary_size_check.sh
6076

77+
6178
FROM scratch AS result
6279
ENV SRC_DIR=/go/src/github.com/k3s-io/k3s
6380
COPY --from=build ${SRC_DIR}/dist /dist

‎Makefile

+34-4
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,38 @@ format:
4343
gofmt -s -l -w $(GO_FILES)
4444
goimports -w $(GO_FILES)
4545

46-
.PHONY: local
47-
local:
46+
47+
.PHONY: local-validate
48+
local-validate:
49+
DOCKER_BUILDKIT=1 docker build \
50+
--build-arg="SKIP_VALIDATE=$(SKIP_VALIDATE)" \
51+
--build-arg="DEBUG=$(DEBUG)" \
52+
--progress=plain \
53+
-f Dockerfile.local --target=validate .
54+
55+
.PHONY: local-binary
56+
local-binary:
57+
@echo "INFO: Building K3s binaries and assets..."
58+
. ./scripts/git_version.sh && \
4859
DOCKER_BUILDKIT=1 docker build \
49-
--build-arg="REPO TAG GITHUB_TOKEN GOLANG GOCOVER DEBUG" \
50-
-t k3s-local -f Dockerfile.local --output=. .
60+
--build-arg "GIT_TAG=$$GIT_TAG" \
61+
--build-arg "TREE_STATE=$$TREE_STATE" \
62+
--build-arg "COMMIT=$$COMMIT" \
63+
--build-arg "DIRTY=$$DIRTY" \
64+
--build-arg="GOCOVER=$(GOCOVER)" \
65+
--build-arg="GOOS=$(GOOS)" \
66+
--build-arg="DEBUG=$(DEBUG)" \
67+
-f Dockerfile.local --target=result --output=. .
68+
69+
.PHONY: local-image
70+
local-image: local-binary
71+
@echo "INFO: Building K3s image..."
72+
./scripts/package-image
73+
74+
.PHONY: local-airgap
75+
local-airgap:
76+
@echo "INFO: Building K3s airgap tarball..."
77+
./scripts/package-airgap
78+
79+
.PHONY: local-ci
80+
local-ci: local-binary local-image local-airgap

‎install.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -514,8 +514,8 @@ get_pr_artifact_url() {
514514
# GET request to the GitHub API to retrieve the Build workflow associated with the commit
515515
run_id=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/commits/${commit_id}/check-runs?check_name=build%20%2F%20Build" | jq -r '[.check_runs | sort_by(.id) | .[].details_url | split("/")[7]] | last')
516516

517-
# Extract the artifact ID for the "k3s" artifact
518-
GITHUB_PR_URL=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/actions/runs/${run_id}/artifacts" | jq -r '.artifacts[] | select(.name == "k3s") | .archive_download_url')
517+
# Extract the artifact ID for the "k3s" (old) or "k3s-amd64" (new) artifact
518+
GITHUB_PR_URL=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/actions/runs/${run_id}/artifacts" | jq -r '.artifacts[] | select(.name == "k3s" or .name == "k3s-amd64") | .archive_download_url')
519519
}
520520

521521
# --- download binary from github url ---

‎install.sh.sha256sum

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
9d5fc42bf825d3e8dcc8682c8bac071b1de18019af81f85519ccbe5c919e0896 install.sh
1+
9ca7930c31179d83bc13de20078fd8ad3e1ee00875b31f39a7e524ca4ef7d9de install.sh

‎package/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM alpine:3.21 as base
1+
FROM alpine:3.21 AS base
22
RUN apk add -U ca-certificates zstd tzdata
33
COPY build/out/data-linux.tar.zst /
44
RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware /image/var/lib/rancher/k3s/data/cni && \
@@ -8,7 +8,7 @@ RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/li
88
echo "root:x:0:" > /image/etc/group && \
99
cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
1010

11-
FROM scratch as collect
11+
FROM scratch AS collect
1212
ARG DRONE_TAG="dev"
1313
COPY --from=base /image /
1414
COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo

0 commit comments

Comments
 (0)