Skip to content

Commit 73a6838

Browse files
adonmadonm
committed
Simplify CSV report job
1 parent 314a61a commit 73a6838

3 files changed

Lines changed: 60 additions & 111 deletions

File tree

‎.github/workflows/ci-nightly.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ jobs:
4242
path: dist/consultations.sql.gz
4343
retention-days: 7
4444

45+
- name: Upload CSV report artifact
46+
if: always() && hashFiles('dist/consultations.csv') != ''
47+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
48+
with:
49+
name: consultations-csv-report
50+
path: dist/consultations.csv
51+
retention-days: 7
52+
4553
- name: Upload helm chart artifact
4654
if: always() && hashFiles('chart/**') != ''
4755
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

‎justfile‎

Lines changed: 51 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,141 +1,81 @@
11
ns := "harvest-consultations"
2-
dbHost := "mariadb"
32
table := "consultations"
4-
owner := "wagov-dtt"
5-
image := "harvest-duckdb"
3+
image := "ghcr.io/wagov-dtt/harvest-duckdb"
64

7-
# Versions come from chart metadata. Docker image tags are derived, not stored.
8-
chart_version := `awk '/^version:/ {print $2; exit}' chart/Chart.yaml`
9-
duckdb_version := `awk '/^appVersion:/ {gsub(/"/, "", $2); print $2; exit}' chart/Chart.yaml`
10-
duckdb_short := `awk '/^appVersion:/ {gsub(/[".]/, "", $2); print $2; exit}' chart/Chart.yaml`
5+
chart_version := `dasel -i yaml --compact '$this.version' < chart/Chart.yaml`
6+
duckdb_version := `dasel -i yaml --compact appVersion < chart/Chart.yaml | tr -d '"'`
7+
duckdb_short := `dasel -i yaml --compact appVersion < chart/Chart.yaml | tr -d '".'`
118
image_tag := chart_version + "-duckdb" + duckdb_short
9+
csv_sql := "LOAD mysql; ATTACH '' AS mysqldb (TYPE mysql); COPY (SELECT * FROM mysqldb." + table + ") TO '/dev/stdout' (HEADER);"
1210

1311
default:
1412
just --choose
1513

16-
# Bump chart/app versions. The Helm image tag is computed from Chart.yaml.
17-
# Usage: just bump-version 0.5.5 (same DuckDB)
18-
# just bump-version 0.5.5 1.6.0 (new DuckDB)
19-
bump-version chart duckdb=duckdb_version:
20-
@echo "=== chart: {{chart_version}} → {{chart}}"
21-
@echo "=== duckdb: {{duckdb_version}} → {{duckdb}}"
22-
sed -i 's/^version: .*/version: {{chart}}/' chart/Chart.yaml
23-
sed -i 's/^appVersion: .*/appVersion: "{{duckdb}}"/' chart/Chart.yaml
24-
sed -i 's/^ARG DUCKDB_VERSION=.*/ARG DUCKDB_VERSION={{duckdb}}/' Dockerfile
25-
@echo "=== image tag: {{chart}}-duckdb{{replace(duckdb, ".", "")}}"
26-
@echo "=== done — verify with: git diff"
27-
28-
clean:
29-
kind delete cluster --name harvest || true
30-
rm -rf dist
31-
32-
# Start local k8s cluster with kind
33-
kind-up:
34-
kind get clusters | grep -q harvest || kind create cluster --name harvest
14+
helm-install:
3515
helm upgrade --install harvest chart \
3616
--namespace {{ns}} --create-namespace \
37-
--set db.host={{dbHost}} \
3817
--set db.table={{table}}
3918

40-
# Forward mariadb from k8s cluster
41-
mariadb-svc: kind-up
42-
ss -ltpn | grep 3306 || kubectl port-forward service/mariadb 3306:3306 -n {{ns}} & sleep 1
19+
kind-up:
20+
kind get clusters | grep -q harvest || kind create cluster --name harvest
21+
just helm-install
4322

44-
# Create a one-off test job in the cluster
4523
test: kind-up
4624
kubectl delete job test -n {{ns}} --ignore-not-found
4725
kubectl create job test --from cronjob/harvest-cronjob -n {{ns}}
4826

49-
# Install/upgrade helm chart
50-
helm-install:
51-
helm upgrade --install harvest chart \
52-
--namespace {{ns}} --create-namespace \
53-
--set db.host={{dbHost}} \
54-
--set db.table={{table}}
55-
56-
# Build and push docker image.
57-
# just docker-build → uses {chart-version}-duckdb{short}
58-
# just docker-build edge → tags as :edge
59-
# just docker-build 0.5.6-duckdb170 → explicit tag
60-
docker-build tag=image_tag:
61-
docker buildx build --platform linux/amd64,linux/arm64 \
62-
--build-arg DUCKDB_VERSION="{{duckdb_version}}" \
63-
-t "ghcr.io/{{owner}}/{{image}}:{{tag}}" --push .
64-
65-
# Build release image with chart version from a git tag, without leading 'v'.
66-
docker-build-release chart: (docker-build chart + "-duckdb" + duckdb_short)
67-
68-
# Package helm chart
69-
helm-package version=chart_version:
70-
mkdir -p dist
71-
helm package chart --version "{{version}}" -d dist/
72-
73-
# === CI / nightly validation ===
74-
75-
# Full end-to-end test: kind cluster → helm deploy → run job → dump → validate
76-
ci-test:
27+
ci-test: kind-up
7728
#!/usr/bin/env bash
7829
set -euo pipefail
7930
80-
echo "=== CI: starting kind cluster ==="
81-
kind get clusters | grep -q harvest || kind create cluster --name harvest
82-
83-
echo "=== CI: installing helm chart ==="
84-
helm upgrade --install harvest chart \
85-
--namespace {{ns}} --create-namespace \
86-
--set db.host={{dbHost}} \
87-
--set db.table={{table}}
31+
wait_job() {
32+
kubectl wait --for=condition=complete "job/$1" -n {{ns}} --timeout="$2" || {
33+
kubectl logs "job/$1" -n {{ns}} --tail=50
34+
exit 1
35+
}
36+
}
8837
89-
echo "=== CI: waiting for MariaDB (up to 5 min) ==="
9038
kubectl rollout status statefulset/mariadb -n {{ns}} --timeout=300s
9139
92-
echo "=== CI: triggering harvest job ==="
9340
kubectl delete job ci-harvest -n {{ns}} --ignore-not-found
9441
kubectl create job ci-harvest --from cronjob/harvest-cronjob -n {{ns}}
42+
wait_job ci-harvest 300s
9543
96-
echo "=== CI: waiting for job to complete ==="
97-
kubectl wait --for=condition=complete job/ci-harvest -n {{ns}} --timeout=300s || {
98-
echo "Job failed — pod logs:"
99-
kubectl logs -l app=harvest-cronjob -n {{ns}} --tail=50
100-
exit 1
101-
}
102-
103-
echo "=== CI: dumping {{table}} table ==="
104-
POD=$(kubectl get pod -l app=mariadb -n {{ns}} -o jsonpath='{.items[0].metadata.name}')
10544
mkdir -p dist
106-
kubectl exec -n {{ns}} "$POD" -- \
45+
pod=$(kubectl get pod -l app=mariadb -n {{ns}} -o jsonpath='{.items[0].metadata.name}')
46+
kubectl exec -n {{ns}} "$pod" -- \
10747
sh -c 'MYSQL_PWD="$MARIADB_PASSWORD" exec mariadb-dump -u"$MARIADB_USER" -h 127.0.0.1 harvest {{table}}' \
10848
| gzip > dist/{{table}}.sql.gz
10949
110-
echo "=== CI: validating dump ==="
111-
gunzip -c dist/{{table}}.sql.gz | head -20 || true
112-
ROWS=$(gunzip -c dist/{{table}}.sql.gz | grep -c 'INSERT INTO' || echo 0)
113-
echo "Rows found: $ROWS"
114-
if [ "$ROWS" -eq 0 ]; then
115-
echo "ERROR: dump contains no INSERT statements"
116-
exit 1
117-
fi
118-
echo "=== CI: PASSED ==="
119-
120-
# Check for newer versions of pinned GitHub Actions
121-
check-actions:
122-
#!/usr/bin/env bash
123-
set -euo pipefail
124-
echo "Checking pinned action versions..."
125-
for f in .github/workflows/*.yaml; do
126-
echo " → $(basename "$f")"
127-
grep -oP 'uses:\s+\K\S+@\S+' "$f" | sort -u | while read -r ref; do
128-
repo="${ref%%@*}"
129-
sha="${ref##*@}"
130-
sha_short="${sha:0:7}"
131-
latest_tag=$(gh api "repos/${repo}/releases/latest" --jq '.tag_name' 2>/dev/null) || latest_tag="unknown"
132-
latest_sha=$(gh api "repos/${repo}/commits/${latest_tag}" --jq '.sha' 2>/dev/null) || latest_sha="unknown"
133-
if [ "$latest_sha" = "unknown" ]; then
134-
echo " ??? ${repo} (could not fetch latest)"
135-
elif [[ "$sha" != "$latest_sha"* ]]; then
136-
echo " OUTDATED: ${repo} → ${latest_tag} (pinned: ${sha_short}, latest: ${latest_sha:0:7})"
137-
else
138-
echo " OK: ${repo} @ ${latest_tag}"
139-
fi
140-
done
141-
done
50+
kubectl delete job ci-csv-report -n {{ns}} --ignore-not-found
51+
kubectl create job ci-csv-report --from cronjob/harvest-cronjob -n {{ns}} --dry-run=client -o yaml \
52+
| sed "s#\.read /etc/config/harvest.sql#{{csv_sql}}#" \
53+
| kubectl apply -f -
54+
wait_job ci-csv-report 120s
55+
kubectl logs job/ci-csv-report -n {{ns}} > dist/{{table}}.csv
56+
kubectl delete job ci-csv-report -n {{ns}} --ignore-not-found
57+
58+
gunzip -c dist/{{table}}.sql.gz | grep -q 'INSERT INTO'
59+
test -s dist/{{table}}.csv
60+
61+
clean:
62+
kind delete cluster --name harvest || true
63+
rm -rf dist
64+
65+
helm-package version=chart_version:
66+
mkdir -p dist
67+
helm package chart --version "{{version}}" -d dist/
68+
69+
docker-build tag=image_tag:
70+
docker buildx build --platform linux/amd64,linux/arm64 \
71+
--build-arg DUCKDB_VERSION="{{duckdb_version}}" \
72+
-t "{{image}}:{{tag}}" --push .
73+
74+
docker-build-release chart: (docker-build chart + "-duckdb" + duckdb_short)
75+
76+
bump-version chart duckdb=duckdb_version:
77+
dasel -i yaml -o yaml --root '$this.version = "{{chart}}"' < chart/Chart.yaml > chart/Chart.yaml.tmp
78+
dasel -i yaml -o yaml --root 'appVersion = "{{duckdb}}"' < chart/Chart.yaml.tmp > chart/Chart.yaml
79+
rm chart/Chart.yaml.tmp
80+
sed -i 's/^ARG DUCKDB_VERSION=.*/ARG DUCKDB_VERSION={{duckdb}}/' Dockerfile
81+
@echo "image tag: {{chart}}-duckdb{{replace(duckdb, ".", "")}}"

‎mise.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
[tools]
2+
dasel = "latest"
23
duckdb = "latest"
34
helm = "latest"
45
just = "latest"

0 commit comments

Comments
 (0)