Skip to content

Commit b0fbd8c

Browse files
Amr-ShamsAmr-Shams
committed
Add S3 storage test workflow with Garage
Signed-off-by: Amr-Shams <amr.shams2015.as@gmail.com>
1 parent 437445e commit b0fbd8c

1 file changed

Lines changed: 373 additions & 0 deletions

File tree

‎.github/workflows/storage.yml‎

Lines changed: 373 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,373 @@
1+
name: Storage
2+
3+
on:
4+
push:
5+
branches:
6+
- ci/storage-tests
7+
pull_request:
8+
branches:
9+
- main
10+
workflow_dispatch:
11+
12+
jobs:
13+
s3-garage:
14+
runs-on: ubuntu-latest
15+
16+
env:
17+
PG_VERSION: "17"
18+
PG_DATABASE: mydb
19+
PG_USER_NAME: myuser
20+
PG_USER_PASSWORD: mypass
21+
PG_REPL_USER_NAME: repl
22+
PG_REPL_PASSWORD: replpass
23+
GARAGE_VERSION: "v2.2.0"
24+
GARAGE_BUCKET: pgmoneta-bucket
25+
GARAGE_REGION: garage
26+
GARAGE_RPC_SECRET: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
27+
BASE_DIR: /tmp/pgmoneta-storage-test
28+
LOG_DIR: /tmp/pgmoneta-storage-test/log
29+
30+
steps:
31+
- uses: actions/checkout@v6
32+
33+
- name: Install dependencies
34+
run: |
35+
sudo apt-get update -y
36+
sudo apt-get install -y \
37+
gcc cmake make \
38+
libev-dev libssl-dev \
39+
libsystemd-dev zlib1g-dev \
40+
libzstd-dev liblz4-dev \
41+
libssh-dev libbz2-dev \
42+
libarchive-dev libyaml-dev \
43+
libncurses-dev \
44+
check python3-docutils
45+
46+
- name: Install PostgreSQL ${{ env.PG_VERSION }}
47+
run: |
48+
sudo apt-get install -y curl ca-certificates
49+
sudo install -d /usr/share/postgresql-common/pgdg
50+
sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
51+
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
52+
sudo apt-get update -y
53+
sudo apt-get install -y postgresql-${{ env.PG_VERSION }} postgresql-server-dev-${{ env.PG_VERSION }}
54+
echo "/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> $GITHUB_PATH
55+
56+
- name: Install pgmoneta_ext
57+
run: |
58+
cd /tmp
59+
git clone --branch main --single-branch --depth 1 https://github.com/pgmoneta/pgmoneta_ext.git
60+
cd pgmoneta_ext
61+
mkdir build && cd build
62+
cmake -DDOCS=false ..
63+
make
64+
sudo make install
65+
66+
- name: Build pgmoneta
67+
run: |
68+
mkdir build && cd build
69+
cmake -DCMAKE_BUILD_TYPE=Debug -DDOCS=FALSE ..
70+
make -j$(nproc)
71+
72+
- name: Set up PostgreSQL
73+
run: |
74+
CONF_DIR=${{ github.workspace }}/test/postgresql/src/postgresql${{ env.PG_VERSION }}/conf
75+
SCRIPT_DIR=${{ github.workspace }}/test/postgresql/src/postgresql${{ env.PG_VERSION }}/root/usr/bin
76+
77+
sudo mkdir -p /conf /pgconf /pgdata /pgwal /pglog
78+
sudo cp "$CONF_DIR"/* /conf/
79+
sudo chown -R postgres:postgres /conf /pgconf /pgdata /pgwal /pglog
80+
sudo chmod -R 777 /conf /pgconf /pgdata /pgwal /pglog
81+
82+
sudo -u postgres bash -c "
83+
export PG_MAX_CONNECTIONS=100
84+
export PG_SHARED_BUFFERS=256MB
85+
export PG_WORK_MEM=4MB
86+
export PG_MAX_PARALLEL_WORKERS=8
87+
export PG_EFFECTIVE_CACHE_SIZE=4GB
88+
export PG_MAX_WAL_SIZE=1GB
89+
export PG_LOG_LEVEL=debug5
90+
export PG_DATABASE=${{ env.PG_DATABASE }}
91+
export PG_USER_NAME=${{ env.PG_USER_NAME }}
92+
export PG_USER_PASSWORD=${{ env.PG_USER_PASSWORD }}
93+
export PG_REPL_USER_NAME=${{ env.PG_REPL_USER_NAME }}
94+
export PG_REPL_PASSWORD=${{ env.PG_REPL_PASSWORD }}
95+
96+
/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/initdb -k -X /pgwal/ /pgdata/
97+
98+
sed -i 's/PG_MAX_CONNECTIONS/100/g' /conf/postgresql.conf
99+
sed -i 's/PG_SHARED_BUFFERS/256MB/g' /conf/postgresql.conf
100+
sed -i 's/PG_WORK_MEM/4MB/g' /conf/postgresql.conf
101+
sed -i 's/PG_MAX_PARALLEL_WORKERS/8/g' /conf/postgresql.conf
102+
sed -i 's/PG_EFFECTIVE_CACHE_SIZE/4GB/g' /conf/postgresql.conf
103+
sed -i 's/PG_MAX_WAL_SIZE/1GB/g' /conf/postgresql.conf
104+
sed -i 's/PG_LOG_LEVEL/debug5/g' /conf/postgresql.conf
105+
106+
sed -i 's/PG_DATABASE/${{ env.PG_DATABASE }}/g' /conf/pg_hba.conf
107+
sed -i 's/PG_USER_NAME/${{ env.PG_USER_NAME }}/g' /conf/pg_hba.conf
108+
sed -i 's/PG_REPL_USER_NAME/${{ env.PG_REPL_USER_NAME }}/g' /conf/pg_hba.conf
109+
110+
cp /conf/postgresql.conf /pgdata/
111+
cp /conf/pg_hba.conf /pgdata/
112+
113+
sed -i 's/PG_DATABASE/${{ env.PG_DATABASE }}/g' /conf/setup.sql
114+
sed -i 's/PG_USER_NAME/${{ env.PG_USER_NAME }}/g' /conf/setup.sql
115+
sed -i 's/PG_USER_PASSWORD/${{ env.PG_USER_PASSWORD }}/g' /conf/setup.sql
116+
sed -i 's/PG_REPL_USER_NAME/${{ env.PG_REPL_USER_NAME }}/g' /conf/setup.sql
117+
sed -i 's/PG_REPL_PASSWORD/${{ env.PG_REPL_PASSWORD }}/g' /conf/setup.sql
118+
119+
/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_ctl -D /pgdata/ start
120+
sleep 3
121+
/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/psql -q -h /tmp -f /conf/setup.sql postgres
122+
"
123+
124+
/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_isready -h localhost -p 5432
125+
126+
- name: Set up Garage
127+
run: |
128+
curl -fsSL -o /tmp/garage \
129+
"https://garagehq.deuxfleurs.fr/_releases/${{ env.GARAGE_VERSION }}/x86_64-unknown-linux-musl/garage"
130+
chmod +x /tmp/garage
131+
132+
mkdir -p /tmp/garage-data/meta /tmp/garage-data/data
133+
134+
cat > /tmp/garage.toml <<EOF
135+
metadata_dir = "/tmp/garage-data/meta"
136+
data_dir = "/tmp/garage-data/data"
137+
db_engine = "sqlite"
138+
replication_factor = 1
139+
compression_level = 1
140+
rpc_bind_addr = "[::]:3901"
141+
rpc_public_addr = "127.0.0.1:3901"
142+
rpc_secret = "${{ env.GARAGE_RPC_SECRET }}"
143+
144+
[s3_api]
145+
s3_region = "${{ env.GARAGE_REGION }}"
146+
api_bind_addr = "[::]:3900"
147+
root_domain = ".s3.garage.localhost"
148+
149+
[s3_web]
150+
bind_addr = "[::]:3902"
151+
root_domain = ".web.garage.localhost"
152+
153+
[admin]
154+
api_bind_addr = "[::]:3903"
155+
EOF
156+
157+
/tmp/garage -c /tmp/garage.toml server &
158+
159+
echo "Waiting for Garage to be ready..."
160+
for i in $(seq 1 30); do
161+
if /tmp/garage -c /tmp/garage.toml status >/dev/null 2>&1; then
162+
echo "Garage is ready"
163+
break
164+
fi
165+
if [ "$i" -eq 30 ]; then
166+
echo "Garage failed to start"
167+
exit 1
168+
fi
169+
sleep 1
170+
done
171+
172+
NODE_ID=$(/tmp/garage -c /tmp/garage.toml status 2>&1 | awk '/^[0-9a-f]+[[:space:]]/ {print $1; exit}')
173+
/tmp/garage -c /tmp/garage.toml layout assign -z dc1 -c 1G "$NODE_ID"
174+
/tmp/garage -c /tmp/garage.toml layout apply --version 1
175+
176+
/tmp/garage -c /tmp/garage.toml bucket create ${{ env.GARAGE_BUCKET }}
177+
/tmp/garage -c /tmp/garage.toml key create pgmoneta-app-key
178+
179+
/tmp/garage -c /tmp/garage.toml bucket allow \
180+
--read --write --owner ${{ env.GARAGE_BUCKET }} --key pgmoneta-app-key
181+
182+
KEY_INFO=$(/tmp/garage -c /tmp/garage.toml key info pgmoneta-app-key --show-secret)
183+
echo "S3_ACCESS_KEY_ID=$(echo "$KEY_INFO" | awk '/Key ID/ {print $3; exit}')" >> $GITHUB_ENV
184+
echo "S3_SECRET_ACCESS_KEY=$(echo "$KEY_INFO" | awk '/Secret key/ {print $3; exit}')" >> $GITHUB_ENV
185+
186+
- name: Configure pgmoneta
187+
run: |
188+
mkdir -p ${{ env.BASE_DIR }}/{backup,log,conf}
189+
190+
cat > ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf <<EOF
191+
unix_socket_dir = /tmp/
192+
log_type = file
193+
log_level = info
194+
log_path = ${{ env.LOG_DIR }}/pgmoneta-cli.log
195+
EOF
196+
197+
cat > ${{ env.BASE_DIR }}/conf/pgmoneta.conf <<EOF
198+
[pgmoneta]
199+
host = localhost
200+
base_dir = ${{ env.BASE_DIR }}/backup
201+
compression = zstd
202+
retention = 7
203+
log_type = file
204+
log_level = debug5
205+
log_path = ${{ env.LOG_DIR }}/pgmoneta.log
206+
unix_socket_dir = /tmp/
207+
storage_engine = s3
208+
209+
[primary]
210+
host = localhost
211+
port = 5432
212+
user = ${{ env.PG_REPL_USER_NAME }}
213+
wal_slot = repl
214+
create_slot = yes
215+
s3_endpoint = localhost
216+
s3_port = 3900
217+
s3_region = ${{ env.GARAGE_REGION }}
218+
s3_use_tls = off
219+
s3_bucket = ${{ env.GARAGE_BUCKET }}
220+
s3_access_key_id = ${S3_ACCESS_KEY_ID}
221+
s3_secret_access_key = ${S3_SECRET_ACCESS_KEY}
222+
EOF
223+
224+
./build/src/pgmoneta-admin master-key -P ${{ env.PG_REPL_PASSWORD }}
225+
./build/src/pgmoneta-admin \
226+
-f ${{ env.BASE_DIR }}/conf/pgmoneta_users.conf \
227+
-U ${{ env.PG_REPL_USER_NAME }} \
228+
-P ${{ env.PG_REPL_PASSWORD }} user add
229+
230+
- name: Start pgmoneta
231+
run: |
232+
./build/src/pgmoneta \
233+
-c ${{ env.BASE_DIR }}/conf/pgmoneta.conf \
234+
-u ${{ env.BASE_DIR }}/conf/pgmoneta_users.conf -d
235+
sleep 5
236+
237+
for i in $(seq 1 10); do
238+
if ./build/src/pgmoneta-cli \
239+
-c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf status > /dev/null 2>&1; then
240+
echo "pgmoneta is ready"
241+
break
242+
fi
243+
if [ "$i" -eq 10 ]; then
244+
echo "pgmoneta failed to start"
245+
cat ${{ env.LOG_DIR }}/pgmoneta.log
246+
exit 1
247+
fi
248+
sleep 2
249+
done
250+
251+
- name: Test S3 backup
252+
run: |
253+
CLI="./build/src/pgmoneta-cli -c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf"
254+
255+
echo "=== Running backup ==="
256+
$CLI backup primary
257+
sleep 10
258+
259+
LABEL=$($CLI -F json status details | python3 -c "
260+
import sys, json
261+
data = json.load(sys.stdin)
262+
backups = data.get('Response', {}).get('Backups', [])
263+
if backups:
264+
print(backups[0].get('Label', ''))
265+
")
266+
267+
if [ -z "$LABEL" ]; then
268+
echo "FAIL: no backup label found"
269+
cat ${{ env.LOG_DIR }}/pgmoneta.log
270+
exit 1
271+
fi
272+
echo "Backup label: $LABEL"
273+
echo "BACKUP_LABEL=$LABEL" >> $GITHUB_ENV
274+
275+
- name: Test S3 list
276+
run: |
277+
CLI="./build/src/pgmoneta-cli -c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf"
278+
279+
echo "=== Listing S3 objects ==="
280+
UPLOAD_COUNT=$($CLI -F json s3 ls primary ${{ env.BACKUP_LABEL }} | python3 -c "
281+
import sys, json
282+
data = json.load(sys.stdin)
283+
objects = data.get('Response', {}).get('S3Objects', [])
284+
print(len(objects))
285+
")
286+
287+
if [ "$UPLOAD_COUNT" -eq 0 ] 2>/dev/null; then
288+
echo "FAIL: s3 ls returned no objects"
289+
cat ${{ env.LOG_DIR }}/pgmoneta.log
290+
exit 1
291+
fi
292+
293+
HAS_MANIFEST=$($CLI -F json s3 ls primary ${{ env.BACKUP_LABEL }} | python3 -c "
294+
import sys, json
295+
data = json.load(sys.stdin)
296+
objects = data.get('Response', {}).get('S3Objects', [])
297+
keys = [o.get('S3Key', '') for o in objects]
298+
print('yes' if any('backup.manifest' in k for k in keys) else 'no')
299+
")
300+
301+
if [ "$HAS_MANIFEST" != "yes" ]; then
302+
echo "FAIL: backup.manifest not found in S3 objects"
303+
exit 1
304+
fi
305+
306+
echo "S3 objects after backup: $UPLOAD_COUNT (manifest present)"
307+
echo "UPLOAD_COUNT=$UPLOAD_COUNT" >> $GITHUB_ENV
308+
309+
- name: Test S3 restore
310+
run: |
311+
CLI="./build/src/pgmoneta-cli -c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf"
312+
313+
echo "=== Restoring from S3 ==="
314+
$CLI s3 restore primary ${{ env.BACKUP_LABEL }}
315+
sleep 10
316+
317+
RESTORE_DIR="${{ env.BASE_DIR }}/backup/primary/backup/${{ env.BACKUP_LABEL }}"
318+
319+
if [ ! -d "$RESTORE_DIR/data" ]; then
320+
echo "FAIL: restored data directory not found at $RESTORE_DIR/data"
321+
cat ${{ env.LOG_DIR }}/pgmoneta.log
322+
exit 1
323+
fi
324+
325+
RESTORED_COUNT=$(find "$RESTORE_DIR" -type f | wc -l)
326+
327+
if [ "$RESTORED_COUNT" -ne "${{ env.UPLOAD_COUNT }}" ]; then
328+
echo "FAIL: file count mismatch (uploaded: ${{ env.UPLOAD_COUNT }}, restored: $RESTORED_COUNT)"
329+
echo "Restored files:"
330+
find "$RESTORE_DIR" -type f
331+
exit 1
332+
fi
333+
334+
echo "Restored files: $RESTORED_COUNT (matches upload count: ${{ env.UPLOAD_COUNT }})"
335+
336+
- name: Test S3 delete
337+
run: |
338+
CLI="./build/src/pgmoneta-cli -c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf"
339+
340+
echo "=== Deleting S3 objects ==="
341+
$CLI s3 delete primary ${{ env.BACKUP_LABEL }}
342+
sleep 5
343+
344+
REMAINING=$($CLI -F json s3 ls primary ${{ env.BACKUP_LABEL }} | python3 -c "
345+
import sys, json
346+
data = json.load(sys.stdin)
347+
objects = data.get('Response', {}).get('S3Objects', [])
348+
print(len(objects))
349+
")
350+
351+
if [ "$REMAINING" -ne 0 ] 2>/dev/null; then
352+
echo "FAIL: s3 delete did not remove all objects (remaining: $REMAINING)"
353+
cat ${{ env.LOG_DIR }}/pgmoneta.log
354+
exit 1
355+
fi
356+
echo "S3 objects after delete: 0"
357+
358+
- name: Shutdown pgmoneta
359+
if: always()
360+
run: |
361+
if [ -f /tmp/pgmoneta.localhost.pid ]; then
362+
./build/src/pgmoneta-cli \
363+
-c ${{ env.BASE_DIR }}/conf/pgmoneta_cli.conf shutdown || true
364+
sleep 3
365+
fi
366+
367+
- name: Upload logs
368+
if: always()
369+
uses: actions/upload-artifact@v5
370+
with:
371+
name: storage-test-logs
372+
path: ${{ env.LOG_DIR }}
373+
retention-days: 30

0 commit comments

Comments
 (0)