Skip to content
CI

Add DuckLake round-trip + iceberg integration tests #1490

Sign in to view logs

Add DuckLake round-trip + iceberg integration tests

Add DuckLake round-trip + iceberg integration tests #1490

Workflow file for this run

.github/workflows/ci.yml at 908b7fb
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
permissions:
contents: read
jobs:
lint:
runs-on: ubuntu-24.04-arm
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
with:
version: v2.11.4
controlplane-no-libduckdb:
name: Verify cmd/duckgres-controlplane does not link libduckdb
runs-on: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Build cmd/duckgres-controlplane (default tags)
run: go build ./cmd/duckgres-controlplane/...
- name: Build cmd/duckgres-controlplane (-tags kubernetes)
run: go build -tags kubernetes ./cmd/duckgres-controlplane/...
# Lock in the binary-split goal: cmd/duckgres-controlplane must remain
# free of github.com/duckdb/duckdb-go in its transitive import graph,
# both with and without the kubernetes build tag. If this guard breaks,
# someone added an import that pulls libduckdb into the CP build —
# likely via a server/* leaf that grew a duckdb-go dependency. Find
# the offending package with `go list -deps -json ./cmd/duckgres-controlplane`
# and route around it (subpackage extraction, registration hook, or
# interface boundary) rather than relaxing this check.
- name: go list -deps controlplane is duckdb-free (default tags)
run: |
set -euo pipefail
if go list -deps ./cmd/duckgres-controlplane | grep -F "github.com/duckdb/duckdb-go"; then
echo "::error::cmd/duckgres-controlplane transitively imports duckdb-go (default tags)" >&2
exit 1
fi
if go list -deps ./controlplane | grep -F "github.com/duckdb/duckdb-go"; then
echo "::error::controlplane package transitively imports duckdb-go (default tags)" >&2
exit 1
fi
- name: go list -deps controlplane is duckdb-free (-tags kubernetes)
run: |
set -euo pipefail
if go list -tags kubernetes -deps ./cmd/duckgres-controlplane | grep -F "github.com/duckdb/duckdb-go"; then
echo "::error::cmd/duckgres-controlplane transitively imports duckdb-go (-tags kubernetes)" >&2
exit 1
fi
if go list -tags kubernetes -deps ./controlplane | grep -F "github.com/duckdb/duckdb-go"; then
echo "::error::controlplane package transitively imports duckdb-go (-tags kubernetes)" >&2
exit 1
fi
unit-tests:
runs-on: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Run unit tests
run: just test-unit
integration-tests:
needs: unit-tests
runs-on: ubuntu-24.04-arm
services:
# PostgreSQL for comparison tests
postgres:
image: public.ecr.aws/docker/library/postgres:16-alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
ports:
- 127.0.0.1:35432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
# DuckLake metadata store
ducklake-metadata:
image: public.ecr.aws/docker/library/postgres:16-alpine
env:
POSTGRES_USER: ducklake
POSTGRES_PASSWORD: ducklake
POSTGRES_DB: ducklake
ports:
- 127.0.0.1:35433:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Build
run: go build -o duckgres .
- name: Start MinIO
run: |
docker run -d --name minio \
-p 39000:9000 \
-e MINIO_ROOT_USER=minioadmin \
-e MINIO_ROOT_PASSWORD=minioadmin \
quay.io/minio/minio:latest server /data
# Wait for MinIO to be ready
for i in {1..30}; do
curl -sf http://localhost:39000/minio/health/live && break
sleep 1
done
- name: Create MinIO bucket
run: |
# Install mc client
curl -sL https://dl.min.io/client/mc/release/linux-arm64/mc -o /tmp/mc
chmod +x /tmp/mc
# Configure and create bucket
/tmp/mc alias set minio http://localhost:39000 minioadmin minioadmin
/tmp/mc mb minio/ducklake --ignore-existing
- name: Run integration tests
run: just test-integration
controlplane-tests:
needs: unit-tests
runs-on: ubuntu-24.04-arm
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Run control plane tests
run: just test-controlplane
configstore-integration-tests:
needs: unit-tests
runs-on: ubuntu-24.04-arm
services:
postgres:
image: public.ecr.aws/docker/library/postgres:16-alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
ports:
- 127.0.0.1:35432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Run config store integration tests
run: just test-configstore-integration
controlplane-k8s-tests:
needs: unit-tests
runs-on: ubuntu-24.04-arm
services:
postgres:
image: public.ecr.aws/docker/library/postgres:16-alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
ports:
- 127.0.0.1:35432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Run Kubernetes-tagged control plane package tests
run: just test-controlplane-k8s
k8s-integration-tests:
needs: unit-tests
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
# id-token: write lets this job request a GitHub OIDC token, which
# aws-actions/configure-aws-credentials trades for STS-vended AWS
# credentials by assuming github-duckgres-iceberg-ci-testing-role in mw-dev.
# The role's trust policy is scoped to repo:PostHog/duckgres:*; its
# IAM policy is scoped to the iceberg test buckets only. Provisioned
# by PostHog/posthog-cloud-infra#8124.
permissions:
id-token: write
contents: read
env:
DUCKGRES_KIND_CLUSTER_NAME: duckgres
DUCKGRES_KIND_NODE_IMAGE: kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865
# Iceberg integration test (tests/k8s/iceberg_test.go) fails openly
# when any of these is unset — see its godoc for the rationale. The
# AWS_* credentials are populated by configure-aws-credentials below
# via OIDC; the three iceberg-specific values are bucket coordinates
# provisioned in mw-dev.
DUCKGRES_K8S_ICEBERG_TABLE_BUCKET_ARN: arn:aws:s3tables:us-east-1:373313242555:bucket/posthog-duckgres-iceberg-test-mw-dev
DUCKGRES_K8S_ICEBERG_REGION: us-east-1
DUCKGRES_K8S_ICEBERG_DATA_BUCKET: posthog-duckgres-iceberg-test-data-mw-dev
services:
postgres:
image: public.ecr.aws/docker/library/postgres:16-alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
ports:
- 127.0.0.1:35432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up just
uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
- name: Set up Go
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
with:
go-version-file: go.mod
- name: Install kind
env:
KIND_VERSION: v0.24.0
run: |
curl -sSL -o /tmp/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-arm64"
chmod +x /tmp/kind
sudo mv /tmp/kind /usr/local/bin/kind
kind --version
- name: Clear Docker Hub credentials for kind pulls
run: |
docker logout registry-1.docker.io || true
docker logout docker.io || true
docker logout https://index.docker.io/v1/ || true
- name: Pre-pull kind node image
run: |
for attempt in 1 2 3; do
if docker pull "${DUCKGRES_KIND_NODE_IMAGE}"; then
exit 0
fi
sleep $((attempt * 5))
done
exit 1
- name: Configure AWS credentials via OIDC
# Trades the GitHub-issued OIDC token for STS credentials by
# assuming github-duckgres-iceberg-ci-testing-role in mw-dev. Exposes
# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN
# in the job env, which iceberg_test.go reads via os.Getenv.
# Pinned to the same commit cloud-infra workflows use.
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v4.0.2
with:
aws-region: us-east-1
role-to-assume: arn:aws:iam::373313242555:role/github-duckgres-iceberg-ci-testing-role
role-duration-seconds: 3600
- name: DEBUG probe iceberg REST endpoint as OIDC role
# Temporary diagnostic: same signed call to the S3 Tables iceberg
# REST endpoint we know returns 200 for SSO admin, run here from the
# OIDC role's STS session. Dumps full response so we can see the
# exact AWS error message and the calling principal ARN.
run: |
set -x
aws sts get-caller-identity
curl -sS --aws-sigv4 "aws:amz:us-east-1:s3tables" \
--user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \
-H "x-amz-security-token: $AWS_SESSION_TOKEN" \
-i \
"https://s3tables.us-east-1.amazonaws.com/iceberg/v1/config?warehouse=arn%3Aaws%3As3tables%3Aus-east-1%3A373313242555%3Abucket%2Fposthog-duckgres-iceberg-test-mw-dev" \
-w "\nHTTP_STATUS=%{http_code}\n" || true
- name: Run Kubernetes integration tests
run: just test-k8s-integration