Skip to content

Commit 74a1b18

Browse files
kvinwangkvinwang
committed
fix: track shared scripts in git and add CI sync check
Instead of gitignoring the per-service copies and copying them at build time, track them in git so `docker build` works standalone without build-image.sh. - Remove .gitignore files, track pin-packages.sh and config-qemu.sh in each service's shared/ directory - build-image.sh now verifies copies match build/shared/ (not copies) - CI: add check-shared-scripts job that fails if copies are out of sync - CI: remove manual copy steps (no longer needed) - CI: remove obsolete .GIT_REV file creation from kms workflows - CI: add DSTACK_SRC_URL build-arg to gateway workflows
1 parent 53d7e94 commit 74a1b18

16 files changed

Lines changed: 221 additions & 42 deletions

File tree

‎.github/workflows/docker-build-check.yml‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,47 @@ on:
1111
branches: [ master, next, dev-* ]
1212

1313
jobs:
14+
check-shared-scripts:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v4
18+
19+
- name: Verify shared build scripts are in sync
20+
run: |
21+
failed=false
22+
for dir in gateway/dstack-app/builder/shared kms/dstack-app/builder/shared verifier/builder/shared; do
23+
if ! diff -q build/shared/pin-packages.sh "$dir/pin-packages.sh" >/dev/null 2>&1; then
24+
echo "ERROR: $dir/pin-packages.sh is out of sync with build/shared/pin-packages.sh"
25+
failed=true
26+
fi
27+
done
28+
for dir in kms/dstack-app/builder/shared verifier/builder/shared; do
29+
if ! diff -q build/shared/config-qemu.sh "$dir/config-qemu.sh" >/dev/null 2>&1; then
30+
echo "ERROR: $dir/config-qemu.sh is out of sync with build/shared/config-qemu.sh"
31+
failed=true
32+
fi
33+
done
34+
if [ "$failed" = "true" ]; then
35+
echo ""
36+
echo "Run the following to fix:"
37+
echo " cp build/shared/pin-packages.sh gateway/dstack-app/builder/shared/"
38+
echo " cp build/shared/pin-packages.sh kms/dstack-app/builder/shared/"
39+
echo " cp build/shared/pin-packages.sh verifier/builder/shared/"
40+
echo " cp build/shared/config-qemu.sh kms/dstack-app/builder/shared/"
41+
echo " cp build/shared/config-qemu.sh verifier/builder/shared/"
42+
exit 1
43+
fi
44+
echo "All shared build scripts are in sync."
45+
1446
kms:
1547
runs-on: ubuntu-latest
48+
needs: check-shared-scripts
1649
steps:
1750
- uses: actions/checkout@v4
1851

1952
- name: Set up Docker Buildx
2053
uses: docker/setup-buildx-action@v3
2154

22-
- name: Copy shared build scripts
23-
run: |
24-
cp build/shared/pin-packages.sh kms/dstack-app/builder/shared/
25-
cp build/shared/config-qemu.sh kms/dstack-app/builder/shared/
26-
2755
- name: Build KMS Docker image
2856
uses: docker/build-push-action@v5
2957
with:
@@ -43,15 +71,13 @@ jobs:
4371
4472
gateway:
4573
runs-on: ubuntu-latest
74+
needs: check-shared-scripts
4675
steps:
4776
- uses: actions/checkout@v4
4877

4978
- name: Set up Docker Buildx
5079
uses: docker/setup-buildx-action@v3
5180

52-
- name: Copy shared build scripts
53-
run: cp build/shared/pin-packages.sh gateway/dstack-app/builder/shared/
54-
5581
- name: Build Gateway Docker image
5682
uses: docker/build-push-action@v5
5783
with:
@@ -61,20 +87,17 @@ jobs:
6187
provenance: false
6288
build-args: |
6389
DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
90+
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
6491
6592
verifier:
6693
runs-on: ubuntu-latest
94+
needs: check-shared-scripts
6795
steps:
6896
- uses: actions/checkout@v4
6997

7098
- name: Set up Docker Buildx
7199
uses: docker/setup-buildx-action@v3
72100

73-
- name: Copy shared build scripts
74-
run: |
75-
cp build/shared/pin-packages.sh verifier/builder/shared/
76-
cp build/shared/config-qemu.sh verifier/builder/shared/
77-
78101
- name: Build Verifier Docker image
79102
uses: docker/build-push-action@v5
80103
with:
@@ -85,3 +108,4 @@ jobs:
85108
provenance: false
86109
build-args: |
87110
DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
111+
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}

‎.github/workflows/gateway-release.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ jobs:
3838
- name: Set up Docker Buildx
3939
uses: docker/setup-buildx-action@v3
4040

41-
- name: Copy shared build scripts
42-
run: cp build/shared/pin-packages.sh gateway/dstack-app/builder/shared/
43-
4441
- name: Get Git commit timestamps
4542
run: |
4643
echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV

‎.github/workflows/kms-release.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,6 @@ jobs:
3838
- name: Set up Docker Buildx
3939
uses: docker/setup-buildx-action@v3
4040

41-
- name: Copy shared build scripts
42-
run: |
43-
cp build/shared/pin-packages.sh kms/dstack-app/builder/shared/
44-
cp build/shared/config-qemu.sh kms/dstack-app/builder/shared/
45-
4641
- name: Get Git commit timestamps
4742
run: |
4843
echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV

‎.github/workflows/verifier-release.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,6 @@ jobs:
3737
- name: Set up Docker Buildx
3838
uses: docker/setup-buildx-action@v3
3939

40-
- name: Copy shared build scripts
41-
run: |
42-
cp build/shared/pin-packages.sh verifier/builder/shared/
43-
cp build/shared/config-qemu.sh verifier/builder/shared/
44-
4540
- name: Get Git commit timestamps
4641
run: |
4742
echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV

‎build/shared/build-lib.sh‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,27 @@ docker_build() {
6767
extract_packages "$image_name" "$pkg_list_file"
6868
}
6969

70-
# Copy shared build scripts into the local shared directory used by Dockerfile COPY.
71-
sync_shared_scripts() {
70+
# Verify that local copies of shared scripts match the canonical versions in build/shared/.
71+
check_shared_scripts() {
7272
local dest_dir=$1
7373
local need_qemu=${2:-false}
74+
local canonical="$REPO_ROOT/build/shared"
75+
local failed=false
7476

75-
cp "$REPO_ROOT/build/shared/pin-packages.sh" "$dest_dir/pin-packages.sh"
77+
if ! diff -q "$canonical/pin-packages.sh" "$dest_dir/pin-packages.sh" &>/dev/null; then
78+
echo "ERROR: $dest_dir/pin-packages.sh is out of sync with build/shared/pin-packages.sh" >&2
79+
echo " Run: cp build/shared/pin-packages.sh $dest_dir/" >&2
80+
failed=true
81+
fi
7682
if [ "$need_qemu" = "true" ]; then
77-
cp "$REPO_ROOT/build/shared/config-qemu.sh" "$dest_dir/config-qemu.sh"
83+
if ! diff -q "$canonical/config-qemu.sh" "$dest_dir/config-qemu.sh" &>/dev/null; then
84+
echo "ERROR: $dest_dir/config-qemu.sh is out of sync with build/shared/config-qemu.sh" >&2
85+
echo " Run: cp build/shared/config-qemu.sh $dest_dir/" >&2
86+
failed=true
87+
fi
88+
fi
89+
if [ "$failed" = "true" ]; then
90+
exit 1
7891
fi
7992
}
8093

‎gateway/dstack-app/builder/build-image.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV")
2626
DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git}
2727

2828
ensure_buildkit
29-
sync_shared_scripts "$SHARED_DIR"
29+
check_shared_scripts "$SHARED_DIR"
3030

3131
touch "$SHARED_DIR/builder-pinned-packages.txt"
3232
touch "$SHARED_DIR/pinned-packages.txt"

‎gateway/dstack-app/builder/shared/.gitignore‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎gateway/dstack-app/builder/shared/pin-packages.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network>
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
7+
# Pin APT packages to exact versions from a frozen Debian snapshot.
8+
# Usage: pin-packages.sh <pkg-list-file>
9+
#
10+
# This script:
11+
# 1. Points APT at a frozen snapshot.debian.org mirror (reproducible package sources)
12+
# 2. Reads package=version pairs from the given file and creates APT pin preferences
13+
# with priority 1001 to force exact versions
14+
15+
set -e
16+
17+
PKG_LIST=$1
18+
SNAPSHOT_DATE=${SNAPSHOT_DATE:-20260317T000000Z}
19+
20+
if [ -z "$PKG_LIST" ]; then
21+
echo "Usage: $0 <pkg-list-file>" >&2
22+
exit 1
23+
fi
24+
25+
echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${SNAPSHOT_DATE} bookworm main" > /etc/apt/sources.list
26+
echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_DATE} bookworm-security main" >> /etc/apt/sources.list
27+
echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/10no-check-valid-until
28+
29+
mkdir -p /etc/apt/preferences.d
30+
while IFS= read -r line; do
31+
pkg=$(echo "$line" | cut -d= -f1)
32+
ver=$(echo "$line" | cut -d= -f2)
33+
if [ -n "$pkg" ] && [ -n "$ver" ]; then
34+
printf 'Package: %s\nPin: version %s\nPin-Priority: 1001\n\n' "$pkg" "$ver" >> /etc/apt/preferences.d/pinned-packages
35+
fi
36+
done < "$PKG_LIST"

‎kms/dstack-app/builder/build-image.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV")
2626
DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git}
2727

2828
ensure_buildkit
29-
sync_shared_scripts "$SHARED_DIR" true
29+
check_shared_scripts "$SHARED_DIR" true
3030

3131
touch "$SHARED_DIR/builder-pinned-packages.txt"
3232
touch "$SHARED_DIR/qemu-pinned-packages.txt"

‎kms/dstack-app/builder/shared/.gitignore‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

0 commit comments

Comments
 (0)