Sync Heatmaps #319
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Heatmaps | |
| on: | |
| schedule: | |
| - cron: "0 15 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| source: | |
| description: "Data source" | |
| required: false | |
| type: choice | |
| options: | |
| - strava | |
| - garmin | |
| default: strava | |
| update_readme_link: | |
| description: "Update README dashboard URL in this fork" | |
| required: false | |
| type: boolean | |
| default: true | |
| full_backfill: | |
| description: "Reset backfill cursor and re-fetch full history for the selected source" | |
| required: false | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| pages: read | |
| env: | |
| DASHBOARD_DATA_BRANCH: dashboard-data | |
| concurrency: | |
| group: sync-${{ github.repository }}-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Resolve source | |
| env: | |
| WORKFLOW_SOURCE: ${{ github.event_name == 'workflow_dispatch' && inputs.source || '' }} | |
| DASHBOARD_SOURCE: ${{ vars.DASHBOARD_SOURCE }} | |
| run: | | |
| source="${WORKFLOW_SOURCE:-${DASHBOARD_SOURCE:-strava}}" | |
| source="$(echo "${source}" | tr '[:upper:]' '[:lower:]' | xargs)" | |
| if [[ "${source}" != "strava" && "${source}" != "garmin" ]]; then | |
| echo "Unsupported source '${source}'. Expected strava or garmin." | |
| exit 1 | |
| fi | |
| echo "SYNC_SOURCE=${source}" >> "$GITHUB_ENV" | |
| echo "Using source: ${source}" | |
| - name: Validate required source secrets | |
| env: | |
| SYNC_SOURCE: ${{ env.SYNC_SOURCE }} | |
| STRAVA_CLIENT_ID: ${{ secrets.STRAVA_CLIENT_ID }} | |
| STRAVA_CLIENT_SECRET: ${{ secrets.STRAVA_CLIENT_SECRET }} | |
| STRAVA_REFRESH_TOKEN: ${{ secrets.STRAVA_REFRESH_TOKEN }} | |
| GARMIN_TOKENS_B64: ${{ secrets.GARMIN_TOKENS_B64 }} | |
| GARMIN_EMAIL: ${{ secrets.GARMIN_EMAIL }} | |
| GARMIN_PASSWORD: ${{ secrets.GARMIN_PASSWORD }} | |
| run: | | |
| missing=() | |
| if [[ "${SYNC_SOURCE}" == "strava" ]]; then | |
| [[ -n "$STRAVA_CLIENT_ID" ]] || missing+=("STRAVA_CLIENT_ID") | |
| [[ -n "$STRAVA_CLIENT_SECRET" ]] || missing+=("STRAVA_CLIENT_SECRET") | |
| [[ -n "$STRAVA_REFRESH_TOKEN" ]] || missing+=("STRAVA_REFRESH_TOKEN") | |
| elif [[ "${SYNC_SOURCE}" == "garmin" ]]; then | |
| if [[ -z "$GARMIN_TOKENS_B64" && ( -z "$GARMIN_EMAIL" || -z "$GARMIN_PASSWORD" ) ]]; then | |
| missing+=("GARMIN_TOKENS_B64 or GARMIN_EMAIL+GARMIN_PASSWORD") | |
| fi | |
| else | |
| missing+=("SYNC_SOURCE must be strava or garmin") | |
| fi | |
| if [ ${#missing[@]} -gt 0 ]; then | |
| echo "❌ Missing required repository secrets: ${missing[*]}" | |
| echo "Add them in Settings -> Secrets and variables -> Actions, then re-run this workflow." | |
| exit 1 | |
| fi | |
| - name: Write config.local.yaml | |
| env: | |
| SYNC_SOURCE: ${{ env.SYNC_SOURCE }} | |
| STRAVA_CLIENT_ID: ${{ secrets.STRAVA_CLIENT_ID }} | |
| STRAVA_CLIENT_SECRET: ${{ secrets.STRAVA_CLIENT_SECRET }} | |
| STRAVA_REFRESH_TOKEN: ${{ secrets.STRAVA_REFRESH_TOKEN }} | |
| DASHBOARD_STRAVA_PROFILE_URL: ${{ vars.DASHBOARD_STRAVA_PROFILE_URL }} | |
| DASHBOARD_STRAVA_ACTIVITY_LINKS: ${{ vars.DASHBOARD_STRAVA_ACTIVITY_LINKS }} | |
| DASHBOARD_GARMIN_PROFILE_URL: ${{ vars.DASHBOARD_GARMIN_PROFILE_URL }} | |
| DASHBOARD_GARMIN_ACTIVITY_LINKS: ${{ vars.DASHBOARD_GARMIN_ACTIVITY_LINKS }} | |
| GARMIN_TOKENS_B64: ${{ secrets.GARMIN_TOKENS_B64 }} | |
| GARMIN_EMAIL: ${{ secrets.GARMIN_EMAIL }} | |
| GARMIN_PASSWORD: ${{ secrets.GARMIN_PASSWORD }} | |
| DASHBOARD_DISTANCE_UNIT: ${{ vars.DASHBOARD_DISTANCE_UNIT }} | |
| DASHBOARD_ELEVATION_UNIT: ${{ vars.DASHBOARD_ELEVATION_UNIT }} | |
| DASHBOARD_WEEK_START: ${{ vars.DASHBOARD_WEEK_START }} | |
| run: | | |
| cat <<CONFIG > config.local.yaml | |
| source: "${SYNC_SOURCE}" | |
| CONFIG | |
| if [[ "${SYNC_SOURCE}" == "strava" ]]; then | |
| cat <<CONFIG >> config.local.yaml | |
| strava: | |
| client_id: "${STRAVA_CLIENT_ID}" | |
| client_secret: "${STRAVA_CLIENT_SECRET}" | |
| refresh_token: "${STRAVA_REFRESH_TOKEN}" | |
| CONFIG | |
| if [[ -n "${DASHBOARD_STRAVA_PROFILE_URL}" ]]; then | |
| echo " profile_url: \"${DASHBOARD_STRAVA_PROFILE_URL}\"" >> config.local.yaml | |
| fi | |
| normalized_links_value="$(echo "${DASHBOARD_STRAVA_ACTIVITY_LINKS}" | tr '[:upper:]' '[:lower:]' | xargs)" | |
| if [[ "${normalized_links_value}" == "1" || "${normalized_links_value}" == "true" || "${normalized_links_value}" == "yes" || "${normalized_links_value}" == "y" || "${normalized_links_value}" == "on" ]]; then | |
| echo " include_activity_urls: true" >> config.local.yaml | |
| fi | |
| else | |
| cat <<CONFIG >> config.local.yaml | |
| garmin: | |
| token_store_b64: "${GARMIN_TOKENS_B64}" | |
| email: "${GARMIN_EMAIL}" | |
| password: "${GARMIN_PASSWORD}" | |
| CONFIG | |
| if [[ -n "${DASHBOARD_GARMIN_PROFILE_URL}" ]]; then | |
| echo " profile_url: \"${DASHBOARD_GARMIN_PROFILE_URL}\"" >> config.local.yaml | |
| fi | |
| normalized_links_value="$(echo "${DASHBOARD_GARMIN_ACTIVITY_LINKS}" | tr '[:upper:]' '[:lower:]' | xargs)" | |
| if [[ "${normalized_links_value}" == "1" || "${normalized_links_value}" == "true" || "${normalized_links_value}" == "yes" || "${normalized_links_value}" == "y" || "${normalized_links_value}" == "on" ]]; then | |
| echo " include_activity_urls: true" >> config.local.yaml | |
| fi | |
| fi | |
| if [[ -n "${DASHBOARD_DISTANCE_UNIT}" || -n "${DASHBOARD_ELEVATION_UNIT}" ]]; then | |
| cat <<'CONFIG' >> config.local.yaml | |
| units: | |
| CONFIG | |
| if [[ -n "${DASHBOARD_DISTANCE_UNIT}" ]]; then | |
| echo " distance: \"${DASHBOARD_DISTANCE_UNIT}\"" >> config.local.yaml | |
| fi | |
| if [[ -n "${DASHBOARD_ELEVATION_UNIT}" ]]; then | |
| echo " elevation: \"${DASHBOARD_ELEVATION_UNIT}\"" >> config.local.yaml | |
| fi | |
| fi | |
| if [[ -n "${DASHBOARD_WEEK_START}" ]]; then | |
| echo "heatmaps:" >> config.local.yaml | |
| echo " week_start: \"${DASHBOARD_WEEK_START}\"" >> config.local.yaml | |
| fi | |
| echo "Generated config.local.yaml for source=${SYNC_SOURCE}" | |
| - name: Apply source-repo overrides (optional) | |
| if: ${{ github.repository == 'aspain/git-sweaty' }} | |
| env: | |
| DASHBOARD_EXCLUDE_TYPES: ${{ vars.DASHBOARD_EXCLUDE_TYPES }} | |
| run: | | |
| if [[ -z "${DASHBOARD_EXCLUDE_TYPES}" ]]; then | |
| echo "No source-repo overrides configured." | |
| exit 0 | |
| fi | |
| echo "activities:" >> config.local.yaml | |
| echo " exclude_types:" >> config.local.yaml | |
| IFS=',' read -ra TYPES <<< "${DASHBOARD_EXCLUDE_TYPES}" | |
| for raw_type in "${TYPES[@]}"; do | |
| sport_type="$(echo "${raw_type}" | xargs)" | |
| if [[ -n "${sport_type}" ]]; then | |
| echo " - ${sport_type}" >> config.local.yaml | |
| fi | |
| done | |
| - name: Restore persisted state | |
| env: | |
| FULL_BACKFILL: ${{ github.event_name == 'workflow_dispatch' && inputs.full_backfill || false }} | |
| run: | | |
| set -euo pipefail | |
| branch="${DASHBOARD_DATA_BRANCH}" | |
| if git ls-remote --exit-code --heads origin "${branch}" >/dev/null; then | |
| git fetch origin "${branch}" | |
| rm -rf data site/data.json | |
| git restore --source "origin/${branch}" --worktree -- data || true | |
| git restore --source "origin/${branch}" --worktree -- site/data.json || true | |
| echo "Restored persisted state from ${branch}." | |
| else | |
| echo "No ${branch} branch yet; starting with an empty state." | |
| fi | |
| if [ "${FULL_BACKFILL}" = "true" ]; then | |
| rm -f data/activities_normalized.json | |
| rm -f data/daily_aggregates.json | |
| rm -f data/backfill_state.json | |
| rm -f data/backfill_state_strava.json | |
| rm -f data/backfill_state_garmin.json | |
| rm -f data/last_sync_summary.json | |
| rm -f data/last_sync_summary.txt | |
| rm -f data/source_state.json | |
| rm -f site/data.json | |
| echo "Full backfill requested: reset persisted pipeline outputs and backfill cursor." | |
| fi | |
| - name: Run pipeline | |
| env: | |
| DASHBOARD_REPO: ${{ vars.DASHBOARD_REPO }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| UPDATE_README_LINK: ${{ github.event_name != 'workflow_dispatch' || inputs.update_readme_link }} | |
| run: | | |
| set -euo pipefail | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| args=() | |
| if [ "${UPDATE_README_LINK}" = "true" ]; then | |
| args+=(--update-readme-link) | |
| fi | |
| python scripts/run_pipeline.py "${args[@]}" | |
| - name: Stamp repository in site data | |
| env: | |
| DASHBOARD_REPO: ${{ vars.DASHBOARD_REPO }} | |
| run: | | |
| set -euo pipefail | |
| python - <<'PY' | |
| import json | |
| import os | |
| import re | |
| repo_re = re.compile(r"^[^/\s]+/[^/\s]+$") | |
| configured_repo = os.environ.get("DASHBOARD_REPO", "").strip() | |
| default_repo = os.environ.get("GITHUB_REPOSITORY", "").strip() | |
| configured_valid = bool(repo_re.match(configured_repo or "")) | |
| default_valid = bool(repo_re.match(default_repo or "")) | |
| if ( | |
| configured_valid | |
| and default_valid | |
| and configured_repo != default_repo | |
| and str(os.environ.get("GITHUB_ACTIONS", "")).strip().lower() == "true" | |
| ): | |
| repo = default_repo | |
| elif configured_valid: | |
| repo = configured_repo | |
| else: | |
| repo = default_repo | |
| path = "site/data.json" | |
| try: | |
| with open(path, "r", encoding="utf-8") as handle: | |
| payload = json.load(handle) | |
| except Exception: | |
| payload = {} | |
| if not isinstance(payload, dict): | |
| payload = {} | |
| if repo_re.match(repo or ""): | |
| payload["repo"] = repo | |
| with open(path, "w", encoding="utf-8") as handle: | |
| json.dump(payload, handle, ensure_ascii=True, indent=2, sort_keys=True) | |
| handle.write("\n") | |
| PY | |
| - name: Rotate Strava refresh secret (optional) | |
| env: | |
| SYNC_SOURCE: ${{ env.SYNC_SOURCE }} | |
| STRAVA_REFRESH_TOKEN: ${{ secrets.STRAVA_REFRESH_TOKEN }} | |
| STRAVA_SECRET_UPDATE_TOKEN: ${{ secrets.STRAVA_SECRET_UPDATE_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "${SYNC_SOURCE}" != "strava" ]]; then | |
| echo "Skipping refresh-token rotation: source=${SYNC_SOURCE}." | |
| exit 0 | |
| fi | |
| if [[ ! -f ".strava_token.json" ]]; then | |
| echo "No local Strava token cache found; skipping refresh-token rotation." | |
| exit 0 | |
| fi | |
| new_refresh_token="$(python -c 'import json; payload=json.load(open(".strava_token.json","r",encoding="utf-8")); value=payload.get("refresh_token"); print(value if isinstance(value,str) else "")' 2>/dev/null || true)" | |
| if [[ -z "${new_refresh_token}" ]]; then | |
| echo "No refreshed Strava token captured; skipping." | |
| exit 0 | |
| fi | |
| if [[ "${new_refresh_token}" == "${STRAVA_REFRESH_TOKEN}" ]]; then | |
| echo "Strava refresh token unchanged." | |
| exit 0 | |
| fi | |
| if [[ -z "${STRAVA_SECRET_UPDATE_TOKEN}" ]]; then | |
| echo "::warning::Strava refresh token rotated, but STRAVA_SECRET_UPDATE_TOKEN is not configured. Update STRAVA_REFRESH_TOKEN manually." | |
| exit 0 | |
| fi | |
| if ! command -v gh >/dev/null 2>&1; then | |
| echo "::warning::gh CLI is unavailable on runner; cannot auto-rotate STRAVA_REFRESH_TOKEN." | |
| exit 0 | |
| fi | |
| if printf '%s' "${new_refresh_token}" | GH_TOKEN="${STRAVA_SECRET_UPDATE_TOKEN}" gh secret set STRAVA_REFRESH_TOKEN --repo "${GITHUB_REPOSITORY}" --body -; then | |
| echo "Updated STRAVA_REFRESH_TOKEN from rotated Strava token." | |
| else | |
| echo "::warning::Failed to update STRAVA_REFRESH_TOKEN automatically. Update it manually." | |
| fi | |
| - name: Rotate Garmin token secret (optional) | |
| env: | |
| SYNC_SOURCE: ${{ env.SYNC_SOURCE }} | |
| GARMIN_TOKENS_B64: ${{ secrets.GARMIN_TOKENS_B64 }} | |
| GARMIN_SECRET_UPDATE_TOKEN: ${{ secrets.GARMIN_SECRET_UPDATE_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "${SYNC_SOURCE}" != "garmin" ]]; then | |
| echo "Skipping Garmin token rotation: source=${SYNC_SOURCE}." | |
| exit 0 | |
| fi | |
| if [[ ! -d ".garmin_token_store" ]]; then | |
| echo "No local Garmin token store found; skipping token rotation." | |
| exit 0 | |
| fi | |
| new_token_store="$(python - <<'PY' | |
| import base64 | |
| import io | |
| import os | |
| import zipfile | |
| token_dir = ".garmin_token_store" | |
| if not os.path.isdir(token_dir): | |
| raise SystemExit(0) | |
| buffer = io.BytesIO() | |
| with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as archive: | |
| for root, _dirs, files in os.walk(token_dir): | |
| for filename in sorted(files): | |
| full_path = os.path.join(root, filename) | |
| rel_path = os.path.relpath(full_path, token_dir) | |
| info = zipfile.ZipInfo(rel_path) | |
| info.date_time = (1980, 1, 1, 0, 0, 0) | |
| info.compress_type = zipfile.ZIP_DEFLATED | |
| with open(full_path, "rb") as handle: | |
| archive.writestr(info, handle.read()) | |
| data = buffer.getvalue() | |
| print(base64.b64encode(data).decode("ascii") if data else "") | |
| PY | |
| )" | |
| if [[ -z "${new_token_store}" ]]; then | |
| echo "No refreshed Garmin token store captured; skipping." | |
| exit 0 | |
| fi | |
| if [[ "${new_token_store}" == "${GARMIN_TOKENS_B64}" ]]; then | |
| echo "Garmin token store unchanged." | |
| exit 0 | |
| fi | |
| if [[ -z "${GARMIN_SECRET_UPDATE_TOKEN}" ]]; then | |
| echo "::warning::Garmin token store changed, but GARMIN_SECRET_UPDATE_TOKEN is not configured. Update GARMIN_TOKENS_B64 manually." | |
| exit 0 | |
| fi | |
| if ! command -v gh >/dev/null 2>&1; then | |
| echo "::warning::gh CLI is unavailable on runner; cannot auto-rotate GARMIN_TOKENS_B64." | |
| exit 0 | |
| fi | |
| if printf '%s' "${new_token_store}" | GH_TOKEN="${GARMIN_SECRET_UPDATE_TOKEN}" gh secret set GARMIN_TOKENS_B64 --repo "${GITHUB_REPOSITORY}" --body -; then | |
| echo "Updated GARMIN_TOKENS_B64 from refreshed Garmin token store." | |
| else | |
| echo "::warning::Failed to update GARMIN_TOKENS_B64 automatically. Update it manually." | |
| fi | |
| - name: Commit README link update | |
| run: | | |
| set -euo pipefail | |
| if git diff --quiet -- README.md; then | |
| echo "README unchanged." | |
| exit 0 | |
| fi | |
| git commit --only -m "docs: set dashboard URL for this fork" -- README.md | |
| if ! git push origin HEAD:${GITHUB_REF_NAME}; then | |
| echo "Warning: unable to push README update; continuing without it." | |
| fi | |
| - name: Publish generated data branch | |
| run: | | |
| set -euo pipefail | |
| branch="${DASHBOARD_DATA_BRANCH}" | |
| data_worktree="${RUNNER_TEMP}/dashboard-data-worktree" | |
| rm -rf "${data_worktree}" | |
| if git ls-remote --exit-code --heads origin "${branch}" >/dev/null; then | |
| git fetch origin "${branch}" | |
| git worktree add "${data_worktree}" "origin/${branch}" | |
| else | |
| git worktree add --detach "${data_worktree}" | |
| pushd "${data_worktree}" >/dev/null | |
| git checkout --orphan "${branch}" | |
| git rm -rf . >/dev/null 2>&1 || true | |
| popd >/dev/null | |
| fi | |
| find "${data_worktree}" -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + | |
| mkdir -p "${data_worktree}/site" | |
| cp -R data "${data_worktree}/data" | |
| cp site/data.json "${data_worktree}/site/data.json" | |
| pushd "${data_worktree}" >/dev/null | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "No dashboard data changes." | |
| popd >/dev/null | |
| git worktree remove "${data_worktree}" --force | |
| exit 0 | |
| fi | |
| summary_file="${GITHUB_WORKSPACE}/data/last_sync_summary.txt" | |
| message="Sync data: update dashboard data" | |
| if [ -s "${summary_file}" ]; then | |
| message="$(head -n 1 "${summary_file}")" | |
| fi | |
| git commit -m "${message}" | |
| git push origin HEAD:${branch} | |
| popd >/dev/null | |
| git worktree remove "${data_worktree}" --force |