Skip to content

Commit 7df0fec

Browse files
YuweiXiaoclaude
YuweiXiao
and
claude
committed
ci: add CI/CD workflows, Docker Bake, and extract extension fetch script
Add GitHub Actions workflows for CI (format check + regression tests) and multi-arch Docker builds with manifest merging. Extract DuckDB extension download into a reusable script. Harden docker.yaml against shell injection by passing refs via env vars, scope permissions to contents:read, and add concurrency guards. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 806ee87 commit 7df0fec

5 files changed

Lines changed: 288 additions & 8 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: ["main"]
6+
tags: ["v*"]
7+
pull_request:
8+
9+
# Cancel in-progress runs for the same branch (except main)
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: ${{ !contains(github.ref, 'main') }}
13+
14+
jobs:
15+
format:
16+
name: "Format check"
17+
runs-on: ubuntu-24.04
18+
steps:
19+
- name: Checkout code
20+
uses: actions/checkout@v4
21+
22+
- uses: dtolnay/rust-toolchain@stable
23+
with:
24+
components: rustfmt
25+
26+
- name: Check Rust formatting
27+
run: cargo fmt --all -- --check
28+
29+
test:
30+
name: "Regression tests (PG${{ matrix.pg_version }})"
31+
runs-on: ubuntu-24.04
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
pg_version: [18]
36+
37+
steps:
38+
- name: Checkout code
39+
uses: actions/checkout@v4
40+
41+
- name: Set up Docker Buildx
42+
uses: docker/setup-buildx-action@v3
43+
44+
- name: Build pg_duckpipe builder image
45+
uses: docker/build-push-action@v6
46+
with:
47+
context: .
48+
target: builder
49+
build-args: PG_VERSION=${{ matrix.pg_version }}
50+
load: true
51+
tags: pg_duckpipe:builder-${{ matrix.pg_version }}
52+
cache-from: type=gha,scope=ci-builder-pg${{ matrix.pg_version }}
53+
cache-to: type=gha,mode=max,scope=ci-builder-pg${{ matrix.pg_version }}
54+
55+
- name: Run regression tests
56+
id: run-tests
57+
run: |
58+
docker run --rm \
59+
-v /tmp/regression-output-${{ matrix.pg_version }}:/results \
60+
pg_duckpipe:builder-${{ matrix.pg_version }} bash -c '
61+
set -euo pipefail
62+
63+
# Pre-fetch the ducklake DuckDB extension needed by tests
64+
/build/docker/fetch-ducklake-ext.sh
65+
chown -R postgres:postgres /var/lib/postgresql/.duckdb /build
66+
67+
# pg_regress refuses to run as root — switch to the postgres user
68+
su postgres -s /bin/bash -c "
69+
export LD_LIBRARY_PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/lib
70+
make -C /build/test/regression check-regression \
71+
PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config
72+
" || {
73+
mkdir -p /results
74+
cp /build/test/regression/regression.diffs /results/ 2>/dev/null || true
75+
cp /build/test/regression/log/postmaster.log /results/ 2>/dev/null || true
76+
exit 1
77+
}
78+
'
79+
80+
- name: Upload regression artifacts
81+
if: failure() && steps.run-tests.outcome == 'failure'
82+
uses: actions/upload-artifact@v4
83+
with:
84+
name: regression-output-pg${{ matrix.pg_version }}
85+
path: /tmp/regression-output-${{ matrix.pg_version }}/
86+
87+
- name: Print regression.diffs on failure
88+
if: failure() && steps.run-tests.outcome == 'failure'
89+
run: |
90+
echo "=== regression.diffs ==="
91+
cat /tmp/regression-output-${{ matrix.pg_version }}/regression.diffs || echo "(not found)"
92+
echo ""
93+
echo "=== postmaster.log (last 100 lines) ==="
94+
tail -100 /tmp/regression-output-${{ matrix.pg_version }}/postmaster.log || echo "(not found)"
95+
exit 1

‎.github/workflows/docker.yaml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: Build Docker
2+
3+
permissions:
4+
contents: read
5+
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.ref }}
8+
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
9+
10+
on:
11+
push:
12+
tags: ["v*"]
13+
pull_request:
14+
paths:
15+
- Dockerfile
16+
- docker-bake.hcl
17+
- ".github/workflows/docker.yaml"
18+
schedule:
19+
# Rebuild nightly to pick up base-image updates from pg_ducklake
20+
- cron: "44 4 * * *"
21+
workflow_dispatch:
22+
23+
jobs:
24+
docker_build:
25+
name: Build Docker image for Postgres ${{ matrix.postgres }} on ${{ matrix.runner }}
26+
strategy:
27+
matrix:
28+
postgres: ["18"]
29+
runner: ["ubuntu-24.04", "ubuntu-24.04-arm"]
30+
31+
runs-on: ${{ matrix.runner }}
32+
33+
env:
34+
BUILDKIT_PROGRESS: plain
35+
PG_VERSION: ${{ matrix.postgres }}
36+
RAW_REF: ${{ github.head_ref || github.ref_name }}
37+
outputs:
38+
branch_tag: ${{ steps.params.outputs.branch_tag }}
39+
target_repo: ${{ steps.params.outputs.target_repo }}
40+
41+
steps:
42+
- name: Login to Docker Hub
43+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
44+
uses: docker/login-action@v3
45+
with:
46+
username: pgducklake
47+
password: ${{ secrets.DOCKERHUB_TOKEN }}
48+
49+
- name: Checkout pg_duckpipe code
50+
uses: actions/checkout@v4
51+
52+
- name: Compute job parameters
53+
id: params
54+
run: |
55+
# Sanitise branch name to a valid Docker tag component (max 112 chars)
56+
BRANCH=$(printf '%s' "$RAW_REF" \
57+
| sed 's/[^a-zA-Z0-9\-\.]/-/g' \
58+
| cut -c 1-112 | tr '[:upper:]' '[:lower:]' \
59+
| sed -e 's/-*$//')
60+
61+
# Set platform depending on which runner we're using
62+
if [ "${{ matrix.runner }}" = "ubuntu-24.04" ]; then
63+
PLATFORM=amd64
64+
else
65+
PLATFORM=arm64
66+
fi
67+
68+
# main branch and release tags publish to pgducklake/pgduckpipe;
69+
# all other branches go to the staging pgducklake/pipe-ci-builds repo.
70+
git fetch --tags --force
71+
if [ "$BRANCH" = "main" ] || git rev-parse --verify "$BRANCH"^{tag} > /dev/null 2>&1; then
72+
TARGET_REPO='pgducklake/pgduckpipe'
73+
else
74+
TARGET_REPO='pgducklake/pipe-ci-builds'
75+
fi
76+
77+
LATEST_IMAGE="pgducklake/pipe-ci-builds:${{ matrix.postgres }}-${PLATFORM}-${BRANCH}-latest"
78+
79+
echo "platform=$PLATFORM" >> "$GITHUB_OUTPUT"
80+
echo "branch_tag=$BRANCH" >> "$GITHUB_OUTPUT"
81+
echo "target_repo=$TARGET_REPO" >> "$GITHUB_OUTPUT"
82+
echo "latest_image=$LATEST_IMAGE" >> "$GITHUB_OUTPUT"
83+
84+
- name: Attempt to pull previous image for layer cache
85+
run: |
86+
docker pull "${{ steps.params.outputs.latest_image }}" || true
87+
docker pull moby/buildkit:buildx-stable-1
88+
89+
- name: Set up Docker Buildx
90+
uses: docker/setup-buildx-action@v3
91+
with:
92+
platforms: linux/${{ steps.params.outputs.platform }}
93+
94+
- name: docker bake
95+
uses: docker/bake-action@v5
96+
with:
97+
targets: pg_duckpipe_${{ matrix.postgres }}
98+
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
99+
set: |
100+
*.platform=linux/${{ steps.params.outputs.platform }}
101+
*.cache-from=type=registry,ref=${{ steps.params.outputs.latest_image }}
102+
*.cache-from=type=gha,scope=${{ github.workflow }}
103+
*.cache-to=type=gha,mode=max,scope=${{ github.workflow }}
104+
pg_duckpipe.tags=pgducklake/pipe-ci-builds:${{ matrix.postgres }}-${{ steps.params.outputs.platform }}-${{ github.sha }}
105+
pg_duckpipe.tags=${{ steps.params.outputs.latest_image }}
106+
107+
docker_merge:
108+
name: Merge Docker image for Postgres ${{ matrix.postgres }}
109+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
110+
strategy:
111+
matrix:
112+
postgres: ["18"]
113+
114+
runs-on: ubuntu-24.04
115+
needs: docker_build
116+
117+
steps:
118+
- name: Login to Docker Hub
119+
uses: docker/login-action@v3
120+
with:
121+
username: pgducklake
122+
password: ${{ secrets.DOCKERHUB_TOKEN }}
123+
124+
- name: Merge amd64 + arm64 into a multi-arch manifest
125+
run: |
126+
docker pull --platform linux/amd64 \
127+
pgducklake/pipe-ci-builds:${{ matrix.postgres }}-amd64-${{ github.sha }}
128+
docker pull --platform linux/arm64 \
129+
pgducklake/pipe-ci-builds:${{ matrix.postgres }}-arm64-${{ github.sha }}
130+
131+
BRANCH="${{ needs.docker_build.outputs.branch_tag }}"
132+
TARGET_REPO="${{ needs.docker_build.outputs.target_repo }}"
133+
134+
echo "Pushing merged manifest to '${TARGET_REPO}'"
135+
docker buildx imagetools create \
136+
--tag "${TARGET_REPO}:${{ matrix.postgres }}-${BRANCH}" \
137+
--tag "pgducklake/pipe-ci-builds:${{ matrix.postgres }}-${{ github.sha }}" \
138+
"pgducklake/pipe-ci-builds:${{ matrix.postgres }}-amd64-${{ github.sha }}" \
139+
"pgducklake/pipe-ci-builds:${{ matrix.postgres }}-arm64-${{ github.sha }}"
140+
141+
docker buildx imagetools inspect \
142+
"pgducklake/pipe-ci-builds:${{ matrix.postgres }}-${{ github.sha }}"

‎Dockerfile‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
5959
###############################################################################
6060
### OUTPUT — pg_ducklake + pg_duckpipe, ready to use
6161
###############################################################################
62-
FROM pgducklake/pgducklake:${PG_VERSION}-main
62+
FROM pgducklake/pgducklake:${PG_VERSION}-main AS output
6363
ARG PG_VERSION
6464

6565
USER root
@@ -98,14 +98,9 @@ RUN echo "/usr/lib/postgresql/${PG_VERSION}/lib" > /etc/ld.so.conf.d/duckdb.conf
9898
# survives the Docker volume mount at /var/lib/postgresql/data.
9999
#
100100
# DuckDB version must match libduckdb.so shipped by pg_ducklake in this image.
101+
COPY docker/fetch-ducklake-ext.sh /usr/local/bin/fetch-ducklake-ext.sh
101102
RUN apt-get update -qq && apt-get install -y --no-install-recommends curl && \
102-
DUCKDB_VER="v1.4.3" && \
103-
ARCH="$(uname -m | sed 's/x86_64/linux_amd64/;s/aarch64/linux_arm64/')" && \
104-
EXT_DIR="/var/lib/postgresql/.duckdb/extensions/${DUCKDB_VER}/${ARCH}" && \
105-
mkdir -p "${EXT_DIR}" && \
106-
curl -fsSL \
107-
"https://extensions.duckdb.org/${DUCKDB_VER}/${ARCH}/ducklake.duckdb_extension.gz" \
108-
| gzip -d > "${EXT_DIR}/ducklake.duckdb_extension" && \
103+
/usr/local/bin/fetch-ducklake-ext.sh && \
109104
chown -R postgres:postgres /var/lib/postgresql/.duckdb && \
110105
apt-get remove -y curl && apt-get autoremove -y && \
111106
rm -rf /var/lib/apt/lists/*

‎docker-bake.hcl‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
variable "REPO" {
2+
default = "pgducklake/pgduckpipe"
3+
}
4+
5+
variable "PG_VERSION" {
6+
default = "18"
7+
}
8+
9+
# Base target: defines build args, target stage, and default tag.
10+
# The docker.yaml workflow overrides tags via `set: pg_duckpipe.tags=...`.
11+
target "pg_duckpipe" {
12+
args = {
13+
PG_VERSION = "${PG_VERSION}"
14+
}
15+
target = "output"
16+
tags = ["${REPO}:${PG_VERSION}-dev"]
17+
}
18+
19+
target "pg_duckpipe_18" {
20+
inherits = ["pg_duckpipe"]
21+
args = {
22+
PG_VERSION = "18"
23+
}
24+
}
25+
26+
target "default" {
27+
inherits = ["pg_duckpipe_18"]
28+
}

‎docker/fetch-ducklake-ext.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
# Download the ducklake DuckDB extension to the standard offline cache directory.
3+
#
4+
# Usage: fetch-ducklake-ext.sh [duckdb-version]
5+
# Default version: v1.4.3 (must match libduckdb.so shipped by pg_ducklake)
6+
#
7+
# Installs to: /var/lib/postgresql/.duckdb/extensions/<ver>/<arch>/ducklake.duckdb_extension
8+
9+
set -euo pipefail
10+
11+
DUCKDB_VER="${1:-v1.4.3}"
12+
ARCH=$(uname -m | sed 's/x86_64/linux_amd64/;s/aarch64/linux_arm64/')
13+
EXT_DIR="/var/lib/postgresql/.duckdb/extensions/${DUCKDB_VER}/${ARCH}"
14+
15+
mkdir -p "${EXT_DIR}"
16+
curl -fsSL \
17+
"https://extensions.duckdb.org/${DUCKDB_VER}/${ARCH}/ducklake.duckdb_extension.gz" \
18+
| gzip -d > "${EXT_DIR}/ducklake.duckdb_extension"
19+
20+
echo "ducklake ${DUCKDB_VER} (${ARCH}) installed to ${EXT_DIR}"

0 commit comments

Comments
 (0)