Skip to content

Commit 6f12ed9

Browse files
committed
refactor: separate publication and team crawlers into dedicated workflow
1 parent b69dc00 commit 6f12ed9

3 files changed

Lines changed: 124 additions & 36 deletions

File tree

‎.github/workflows/scheduled-maintenance.yml‎

Lines changed: 14 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
1-
# Unified workflow to crawl publications, team photos, and apply LTS security patches
1+
# Weekly maintenance: apply LTS security patches and prune stale Keystatic branches.
2+
# Crawling hadylauw.com (publications, team photos) lives in update-publications-and-team.yml.
23
name: Scheduled Maintenance
34

45
on:
5-
# Run every Sunday at 00:00 UTC
6+
# Run every Sunday at 06:00 UTC (14:00 Singapore time), 12h after the daily
7+
# publications & team photos sync (18:00 UTC) so the two never overlap
68
schedule:
7-
- cron: "0 0 * * 0"
9+
- cron: "0 6 * * 0"
810

911
push:
1012
branches: ["main"]
1113
paths:
1214
- ".github/workflows/scheduled-maintenance.yml"
13-
- "scripts/crawl-publications.ts"
14-
- "scripts/crawl-team.ts"
1515

16-
# Allow manual trigger with selective options
16+
# Allow manual trigger
1717
workflow_dispatch:
18-
inputs:
19-
skip_crawlers:
20-
description: "Skip crawling external sites (security update only)"
21-
type: boolean
22-
default: false
2318

2419
permissions:
2520
contents: write
2621
pull-requests: read
2722

23+
# Shared with update-publications-and-team.yml so bot pushes to main never race
24+
concurrency:
25+
group: main-bot-writes
26+
cancel-in-progress: false
27+
2828
jobs:
2929
maintenance:
3030
runs-on: ubuntu-latest
@@ -47,24 +47,11 @@ jobs:
4747
- name: Install dependencies
4848
run: pnpm install --frozen-lockfile
4949

50-
- name: Run external crawlers concurrently
51-
if: ${{ !inputs.skip_crawlers }}
52-
run: |
53-
pnpm crawl:publications &
54-
PID_PUB=$!
55-
pnpm crawl:team &
56-
PID_TEAM=$!
57-
wait $PID_PUB $PID_TEAM
58-
5950
- name: Auto-patch security vulnerabilities
6051
run: |
6152
pnpm audit --fix update || true
6253
pnpm install
6354
64-
- name: Format generated publication and team data
65-
run: |
66-
pnpm exec biome check --write src/data/publications.ts src/data/teamPhotos.ts || true
67-
6855
- name: Validate application integrity
6956
run: |
7057
pnpm lint
@@ -76,7 +63,7 @@ jobs:
7663
- name: Check for changes
7764
id: check-changes
7865
run: |
79-
git add src/data/publications.ts src/data/teamPhotos.ts public/team pnpm-lock.yaml pnpm-workspace.yaml package.json apps/admin/package.json
66+
git add pnpm-lock.yaml pnpm-workspace.yaml package.json apps/admin/package.json
8067
git diff --cached --quiet || echo "changed=true" >> $GITHUB_OUTPUT
8168
8269
- name: Commit and push changes
@@ -85,13 +72,8 @@ jobs:
8572
git config --local user.email "github-actions[bot]@users.noreply.github.com"
8673
git config --local user.name "github-actions[bot]"
8774
88-
changes=()
89-
! git diff --cached --quiet src/data/publications.ts && changes+=("publications")
90-
! git diff --cached --quiet src/data/teamPhotos.ts public/team && changes+=("team photos")
91-
! git diff --cached --quiet pnpm-lock.yaml pnpm-workspace.yaml package.json apps/admin/package.json && changes+=("security patches")
92-
93-
msg="chore: update $(IFS=', '; echo "${changes[*]}") [automated]"
94-
git commit -m "$msg"
75+
git commit -m "chore: update security patches [automated]"
76+
git pull --rebase origin main
9577
git push
9678
9779
- name: Trigger deploy notice
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Daily sync of publications and team photos from hadylauw.com.
2+
# hadylauw.com is a Google Sites site with no RSS/Atom feed, sitemap, or webhook,
3+
# so we poll it. The crawlers' parsed output is deterministic, so a git diff on the
4+
# generated files is used as the change detector. Use "Run workflow" for an
5+
# immediate sync after editing hadylauw.com.
6+
name: Update Publications & Team Photos
7+
8+
on:
9+
# Run daily at 18:00 UTC (02:00 Singapore time)
10+
schedule:
11+
- cron: "0 18 * * *"
12+
13+
push:
14+
branches: ["main"]
15+
paths:
16+
- ".github/workflows/update-publications-and-team.yml"
17+
- "scripts/crawl-publications.ts"
18+
- "scripts/crawl-team.ts"
19+
20+
workflow_dispatch:
21+
22+
permissions:
23+
contents: write
24+
25+
# Shared with scheduled-maintenance.yml so bot pushes to main never race
26+
concurrency:
27+
group: main-bot-writes
28+
cancel-in-progress: false
29+
30+
jobs:
31+
sync:
32+
runs-on: ubuntu-latest
33+
timeout-minutes: 15
34+
steps:
35+
- name: Checkout
36+
uses: actions/checkout@v7
37+
with:
38+
# Push via deploy key so the push triggers the Cloudflare Pages deploy workflow
39+
ssh-key: ${{ secrets.DEPLOY_KEY }}
40+
41+
- name: Setup pnpm
42+
uses: pnpm/action-setup@v6
43+
44+
- name: Setup Node
45+
uses: actions/setup-node@v7
46+
with:
47+
node-version: "22"
48+
cache: pnpm
49+
50+
- name: Install dependencies
51+
run: pnpm install --frozen-lockfile
52+
53+
- name: Run external crawlers concurrently
54+
run: |
55+
pnpm crawl:publications &
56+
PID_PUB=$!
57+
pnpm crawl:team &
58+
PID_TEAM=$!
59+
wait $PID_PUB $PID_TEAM
60+
61+
- name: Format generated publication and team data
62+
run: |
63+
pnpm exec biome check --write src/data/publications.ts src/data/teamPhotos.ts || true
64+
65+
- name: Check for changes
66+
id: check-changes
67+
run: |
68+
git add src/data/publications.ts src/data/teamPhotos.ts public/team
69+
git diff --cached --quiet || echo "changed=true" >> "$GITHUB_OUTPUT"
70+
71+
- name: Validate application integrity
72+
if: steps.check-changes.outputs.changed == 'true'
73+
run: |
74+
pnpm lint
75+
pnpm typecheck:prepared
76+
77+
- name: Commit and push changes
78+
if: steps.check-changes.outputs.changed == 'true'
79+
run: |
80+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
81+
git config --local user.name "github-actions[bot]"
82+
83+
changes=()
84+
! git diff --cached --quiet src/data/publications.ts && changes+=("publications")
85+
! git diff --cached --quiet src/data/teamPhotos.ts public/team && changes+=("team photos")
86+
87+
msg="chore: update $(IFS=', '; echo "${changes[*]}") [automated]"
88+
git commit -m "$msg"
89+
git pull --rebase origin main
90+
git push
91+
92+
- name: Report result
93+
run: |
94+
if [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then
95+
echo "Changes pushed to main. Cloudflare Pages deployment will be triggered automatically."
96+
else
97+
echo "No changes on hadylauw.com. Nothing to commit."
98+
fi

‎docs/publishing-operations.md‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,10 +87,18 @@ branches, and jump directly to the build or open pull request.
8787
5. Advancing `main` via `DEPLOY_KEY` natively triggers
8888
`.github/workflows/nextjs.yml` to deploy the static website to Cloudflare
8989
Pages without redeploying the admin Worker.
90-
6. `.github/workflows/scheduled-maintenance.yml` runs dependency maintenance
91-
exclusively on `main` and automatically prunes abandoned `keystatic/*`
92-
branches that have `0` commits ahead of `main`, no open pull request, and no
93-
activity for over 7 days.
90+
6. `.github/workflows/scheduled-maintenance.yml` runs weekly (Sundays 06:00
91+
UTC) dependency maintenance exclusively on `main` and automatically prunes
92+
abandoned `keystatic/*` branches that have `0` commits ahead of `main`, no
93+
open pull request, and no activity for over 7 days.
94+
7. `.github/workflows/update-publications-and-team.yml` runs daily (18:00 UTC /
95+
02:00 SGT) to crawl publications and team photos from hadylauw.com. It commits
96+
and pushes to `main` (triggering a deploy) only when the generated data
97+
changed. hadylauw.com is a Google Sites site with no RSS/Atom feed or
98+
webhook, so it must be polled. For an immediate sync after editing
99+
hadylauw.com, use **Actions → Update Publications & Team Photos → Run workflow**.
100+
Both workflows share the `main-bot-writes` concurrency group so their pushes
101+
never race.
94102

95103
Protect `main` with the repository ruleset (`id: 20780485`), requiring pull
96104
requests, linear history, and the `audit` + `build` status checks.

0 commit comments

Comments
 (0)