Skip to content

Commit 53b2ca2

Browse files
mumoshumumoshu
committed
Add workflow for building and publishing container images
1 parent b7901f3 commit 53b2ca2

2 files changed

Lines changed: 255 additions & 0 deletions

File tree

‎.github/workflows/build-and-publish.yml‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
name: Build and Publish Connector
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
tags:
8+
- 'v*'
9+
paths:
10+
- 'destination-surrealdb/**'
11+
- '.github/workflows/build-and-publish.yml'
12+
- 'Makefile'
13+
workflow_dispatch:
14+
15+
env:
16+
REGISTRY: ghcr.io
17+
IMAGE_NAME: surrealdb/airbyte-destination-surrealdb
18+
19+
jobs:
20+
build-and-publish:
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 60
23+
permissions:
24+
contents: read
25+
packages: write
26+
strategy:
27+
matrix:
28+
architecture: ["linux/amd64", "linux/arm64"]
29+
steps:
30+
- name: Checkout repository
31+
uses: actions/checkout@v4
32+
33+
- name: Free up disk space
34+
run: |
35+
echo "=== Initial Disk Usage ==="
36+
df -h
37+
38+
echo "=== Cleaning up disk space ==="
39+
# Remove unnecessary packages and caches
40+
sudo rm -rf /usr/share/dotnet
41+
sudo rm -rf /opt/ghc
42+
sudo rm -rf /usr/local/share/boost
43+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
44+
sudo rm -rf /usr/local/lib/android
45+
sudo rm -rf /usr/share/swift
46+
sudo rm -rf /opt/hostedtoolcache
47+
48+
# Clean package manager caches
49+
sudo apt-get clean
50+
sudo apt-get autoremove -y
51+
52+
# Clean Docker system
53+
docker system prune -af --volumes || true
54+
55+
# Clear various caches
56+
sudo rm -rf /var/cache/* || true
57+
sudo rm -rf /tmp/* || true
58+
59+
echo "=== Disk Usage After Cleanup ==="
60+
df -h
61+
62+
- name: Configure system limits for dagger
63+
run: |
64+
echo "Current system limits:"
65+
echo " inotify max_user_instances: $(cat /proc/sys/fs/inotify/max_user_instances)"
66+
echo " inotify max_user_watches: $(cat /proc/sys/fs/inotify/max_user_watches)"
67+
echo " ulimit -n: $(ulimit -n)"
68+
69+
echo "Configuring system limits for better dagger performance..."
70+
echo fs.inotify.max_user_instances=8192 | sudo tee -a /etc/sysctl.conf
71+
echo fs.inotify.max_user_watches=1048576 | sudo tee -a /etc/sysctl.conf
72+
sudo sysctl -p
73+
74+
- name: Set up Docker Buildx
75+
uses: docker/setup-buildx-action@v3
76+
77+
- name: Log in to Container Registry
78+
uses: docker/login-action@v3
79+
with:
80+
registry: ${{ env.REGISTRY }}
81+
username: ${{ github.actor }}
82+
password: ${{ secrets.GITHUB_TOKEN }}
83+
84+
- name: Extract architecture for build
85+
id: arch
86+
run: |
87+
arch="${{ matrix.architecture }}"
88+
echo "airbyte_arch=$arch" >> $GITHUB_OUTPUT
89+
echo "platform_arch=$(echo $arch | cut -d'/' -f2)" >> $GITHUB_OUTPUT
90+
91+
- name: Generate image tags
92+
id: tags
93+
run: |
94+
# Base image name
95+
IMAGE_BASE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
96+
97+
# Get commit SHA (short)
98+
COMMIT_SHA=$(echo ${{ github.sha }} | cut -c1-7)
99+
100+
# Initialize tags array
101+
TAGS=()
102+
103+
# Always add commit SHA tag
104+
TAGS+=("${IMAGE_BASE}:${COMMIT_SHA}-${{ steps.arch.outputs.platform_arch }}")
105+
106+
# Add branch-specific tags
107+
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
108+
TAGS+=("${IMAGE_BASE}:canary-${{ steps.arch.outputs.platform_arch }}")
109+
fi
110+
111+
# Add tag-specific tags if this is a tag push
112+
if [[ "${{ github.ref }}" == refs/tags/* ]]; then
113+
VERSION=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
114+
TAGS+=("${IMAGE_BASE}:${VERSION}-${{ steps.arch.outputs.platform_arch }}")
115+
fi
116+
117+
# Join tags with newlines for output
118+
printf -v TAGS_STRING '%s\n' "${TAGS[@]}"
119+
echo "tags<<EOF" >> $GITHUB_OUTPUT
120+
echo "$TAGS_STRING" >> $GITHUB_OUTPUT
121+
echo "EOF" >> $GITHUB_OUTPUT
122+
123+
# Also set a single tag for the build process
124+
echo "primary_tag=${TAGS[0]}" >> $GITHUB_OUTPUT
125+
126+
- name: Build connector with airbyte-ci
127+
timeout-minutes: 45
128+
uses: devcontainers/ci@v0.3
129+
env:
130+
BUILDKIT_PROGRESS: plain
131+
DOCKER_BUILDKIT: 1
132+
with:
133+
subFolder: .
134+
env: |
135+
BUILDKIT_PROGRESS=plain
136+
DOCKER_BUILDKIT=1
137+
runCmd: |
138+
echo "=== Building connector for architecture: ${{ steps.arch.outputs.airbyte_arch }} ==="
139+
echo "Target image tag: ${{ steps.tags.outputs.primary_tag }}"
140+
141+
echo "=== Environment Information ==="
142+
echo "Available memory: $(free -h)"
143+
echo "Available disk space: $(df -h /)"
144+
echo "Docker version: $(docker --version)"
145+
146+
echo "=== Running airbyte-ci build ==="
147+
# Setup development environment (clone Airbyte repo and copy connector)
148+
make setup-dev
149+
150+
# Build with airbyte-ci for the specific architecture
151+
cd airbyte
152+
airbyte-ci --show-dagger-logs connectors \
153+
--name destination-surrealdb \
154+
build \
155+
--architecture ${{ steps.arch.outputs.airbyte_arch }}
156+
157+
echo "=== Build completed ==="
158+
# Verify the image was built
159+
docker images | grep destination-surrealdb || echo "No destination-surrealdb images found"
160+
161+
echo "=== Tagging and pushing images ==="
162+
# Tag and push each target tag
163+
while IFS= read -r tag; do
164+
if [ -n "$tag" ]; then
165+
echo "Tagging: $SOURCE_IMAGE -> $tag"
166+
docker tag "$SOURCE_IMAGE" "$tag"
167+
echo "Pushing: $tag"
168+
docker push "$tag"
169+
fi
170+
done <<< "${{ steps.tags.outputs.tags }}"
171+
172+
echo "=== All images pushed successfully ==="
173+
push: never
174+
175+
create-manifest:
176+
needs: build-and-publish
177+
runs-on: ubuntu-latest
178+
permissions:
179+
contents: read
180+
packages: write
181+
steps:
182+
- name: Log in to Container Registry
183+
uses: docker/login-action@v3
184+
with:
185+
registry: ${{ env.REGISTRY }}
186+
username: ${{ github.actor }}
187+
password: ${{ secrets.GITHUB_TOKEN }}
188+
189+
- name: Generate manifest tags
190+
id: manifest_tags
191+
run: |
192+
# Base image name
193+
IMAGE_BASE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
194+
195+
# Get commit SHA (short)
196+
COMMIT_SHA=$(echo ${{ github.sha }} | cut -c1-7)
197+
198+
# Initialize tags array (without architecture suffix)
199+
TAGS=()
200+
201+
# Always add commit SHA tag
202+
TAGS+=("${IMAGE_BASE}:${COMMIT_SHA}")
203+
204+
# Add branch-specific tags
205+
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
206+
TAGS+=("${IMAGE_BASE}:canary")
207+
fi
208+
209+
# Add tag-specific tags if this is a tag push
210+
if [[ "${{ github.ref }}" == refs/tags/* ]]; then
211+
VERSION=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
212+
TAGS+=("${IMAGE_BASE}:${VERSION}")
213+
fi
214+
215+
# Join tags with newlines for output
216+
printf -v TAGS_STRING '%s\n' "${TAGS[@]}"
217+
echo "tags<<EOF" >> $GITHUB_OUTPUT
218+
echo "$TAGS_STRING" >> $GITHUB_OUTPUT
219+
echo "EOF" >> $GITHUB_OUTPUT
220+
221+
- name: Create and push multi-architecture manifests
222+
run: |
223+
IMAGE_BASE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
224+
225+
while IFS= read -r tag; do
226+
if [ -n "$tag" ]; then
227+
echo "Creating manifest for: $tag"
228+
229+
# Create and push manifest
230+
docker manifest create "$tag" \
231+
"${tag}-amd64" \
232+
"${tag}-arm64"
233+
234+
docker manifest push "$tag"
235+
236+
echo "Manifest created and pushed: $tag"
237+
fi
238+
done <<< "${{ steps.manifest_tags.outputs.tags }}"
239+
240+
echo "=== All manifests created successfully ==="

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,21 @@ This is the repository for developing the Airbyte SurrealDB destination connecto
44

55
The actual connector code is located in the [destination-surrealdb](./destination-surrealdb) directory, which will eventually be contributed to [connectors](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors) in the Airbyte repository.
66

7+
### Using Published Images
8+
9+
You can pull and use the published images:
10+
11+
```bash
12+
# Pull latest multi-architecture image
13+
docker pull ghcr.io/surrealdb/airbyte-destination-surrealdb:latest
14+
15+
# Pull specific version
16+
docker pull ghcr.io/surrealdb/airbyte-destination-surrealdb:v0.1.0
17+
18+
# Pull specific commit
19+
docker pull ghcr.io/surrealdb/airbyte-destination-surrealdb:a1b2c3d
20+
```
21+
722
## Development
823

924
We have two options for developing the connector:

0 commit comments

Comments
 (0)