Cluster — daily recompute #89
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cluster — daily recompute | |
| # Runs the K-Means neighborhood-clustering job once a day. | |
| # We can't rely on the API's in-process PeriodicTimer because the App Service | |
| # is on the student tier (no Always On), so the runtime can sleep on idle. | |
| # This workflow guarantees a recompute on a fixed schedule, independent of | |
| # whether anyone is hitting the API. | |
| on: | |
| schedule: | |
| # Every day at 03:00 UTC (06:00 Israel summer / 05:00 winter). | |
| # Off-peak so the GIS calls run when TA's ArcGIS server is least loaded. | |
| - cron: "0 3 * * *" | |
| # Allow manual triggering from the Actions tab when you want to force a run. | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| recompute: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Wake & cluster | |
| env: | |
| API_BASE: https://app-groundshare-api.azurewebsites.net | |
| ADMIN_TOKEN: ${{ secrets.CLUSTERING_ADMIN_TOKEN }} | |
| # How long to wait for the background recompute to finish, in seconds. | |
| # The job is fire-and-forget server-side, so we poll diagnostics instead | |
| # of holding one request open (which tripped Azure's 230s 504 timeout). | |
| POLL_TIMEOUT: "480" | |
| POLL_INTERVAL: "15" | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${ADMIN_TOKEN}" ]; then | |
| echo "::error::CLUSTERING_ADMIN_TOKEN secret is not set." | |
| exit 1 | |
| fi | |
| # Extracts a top-level JSON field as a string ("" if null/missing). | |
| json_field() { python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('$1') or '')"; } | |
| # 1. Wake the API (student tier sleeps on idle). The diagnostics | |
| # endpoint is anonymous and cheap, so it's a clean cold-start ping. | |
| echo "Pinging diagnostics to warm the App Service..." | |
| for attempt in 1 2 3; do | |
| if curl -fsS --max-time 60 "${API_BASE}/api/clustering/diagnostics" >/tmp/diag.json; then | |
| cat /tmp/diag.json; echo | |
| break | |
| fi | |
| echo "Attempt ${attempt} failed, retrying..." | |
| sleep 15 | |
| done | |
| # Record the timestamp BEFORE we trigger, so we can detect when the | |
| # background job has written a newer one. | |
| before=$(json_field clustersLastUpdated </tmp/diag.json) | |
| echo "clustersLastUpdated before run: '${before}'" | |
| # 2. Kick off the recompute. The endpoint is fire-and-forget and returns | |
| # 202 immediately (or while a run is already in progress). | |
| echo "Triggering /run-now..." | |
| http_code=$(curl -sS -o /tmp/run.json -w "%{http_code}" \ | |
| --max-time 60 \ | |
| -X POST \ | |
| -H "X-Admin-Token: ${ADMIN_TOKEN}" \ | |
| "${API_BASE}/api/clustering/run-now") | |
| echo "HTTP ${http_code}" | |
| cat /tmp/run.json; echo | |
| # 202 Accepted is the success path; tolerate 200 for backward-compat | |
| # during the deploy window when old/new code may briefly coexist. | |
| if [ "${http_code}" != "202" ] && [ "${http_code}" != "200" ]; then | |
| echo "::error::Cluster recompute trigger failed with HTTP ${http_code}" | |
| exit 1 | |
| fi | |
| # 3. Poll diagnostics until clustersLastUpdated advances past 'before'. | |
| echo "Polling for completion (timeout ${POLL_TIMEOUT}s)..." | |
| deadline=$(( $(date +%s) + POLL_TIMEOUT )) | |
| while :; do | |
| if curl -fsS --max-time 30 "${API_BASE}/api/clustering/diagnostics" >/tmp/diag.json; then | |
| after=$(json_field clustersLastUpdated </tmp/diag.json) | |
| if [ -n "${after}" ] && [ "${after}" != "${before}" ]; then | |
| echo "Recompute complete. clustersLastUpdated now: '${after}'" | |
| cat /tmp/diag.json; echo | |
| exit 0 | |
| fi | |
| echo " still running (clustersLastUpdated='${after}')..." | |
| else | |
| echo " diagnostics ping failed, will retry..." | |
| fi | |
| if [ "$(date +%s)" -ge "${deadline}" ]; then | |
| echo "::error::Cluster recompute did not complete within ${POLL_TIMEOUT}s" | |
| cat /tmp/diag.json 2>/dev/null || true | |
| exit 1 | |
| fi | |
| sleep "${POLL_INTERVAL}" | |
| done |