Upload R2 snapshot #510
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload R2 snapshot | |
| # **What it does**: Uploads our site output directly to an R2 bucket consumed by the AI search instance that runs on Dev Docs. | |
| # **Why we have it**: At the moment, AI Search doesn't support native indexing by anchor headings via a [webcrawler](https://developers.cloudflare.com/ai-search/configuration/data-source/website/) (which was something Algolia did support). This is a workaround that will hopefully be removed in the future. | |
| # **What to do if it fails**: In most cases, this is fine -- individual failures are usually due to a newer version of the upload starting. So there's a slight delay in updating the bucket, but that delay ensures we're getting the most up to date content there. | |
| on: | |
| workflow_run: | |
| workflows: ["Publish"] | |
| types: [completed] | |
| permissions: | |
| actions: read | |
| contents: read | |
| jobs: | |
| upload: | |
| name: Upload production snapshot to R2 | |
| if: >- | |
| github.repository == 'cloudflare/cloudflare-docs' && | |
| github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.event == 'push' && | |
| github.event.workflow_run.head_branch == 'production' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - name: Download published site | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: site-html | |
| path: dist | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| - name: Upload snapshot | |
| env: | |
| R2_BUCKET: ${{ secrets.R2_SNAPSHOT_BUCKET }} | |
| R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AI_SEARCH_R2_SNAPSHOT_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AI_SEARCH_R2_SNAPSHOT_SECRET_ACCESS_KEY_ID }} | |
| AWS_DEFAULT_REGION: auto | |
| PUBLISHED_SHA: ${{ github.event.workflow_run.head_sha }} | |
| PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }} | |
| run: | | |
| set -euo pipefail | |
| : "${R2_BUCKET:?R2_SNAPSHOT_BUCKET secret is required}" | |
| : "${R2_ACCOUNT_ID:?CLOUDFLARE_ACCOUNT_ID secret is required}" | |
| : "${AWS_ACCESS_KEY_ID:?AI_SEARCH_R2_SNAPSHOT_ACCESS_KEY_ID secret is required}" | |
| : "${AWS_SECRET_ACCESS_KEY:?AI_SEARCH_R2_SNAPSHOT_SECRET_ACCESS_KEY_ID secret is required}" | |
| endpoint="https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com" | |
| timestamp=$(date -u '+%Y-%m-%dT%H-%M-%SZ') | |
| destination="s3://$R2_BUCKET/$timestamp-$PUBLISH_RUN_ID" | |
| manifest="$RUNNER_TEMP/manifest.json" | |
| checksums="$RUNNER_TEMP/MD5SUMS" | |
| success="$RUNNER_TEMP/_SUCCESS" | |
| if [ -z "$(find dist -type f -name '*.html' -print -quit)" ]; then | |
| echo "No HTML files found in the site artifact" >&2 | |
| exit 1 | |
| fi | |
| find . \( -path './.git' -o -path './dist' \) -prune -o -type f -print0 \ | |
| | sort -z \ | |
| | xargs -0 md5sum \ | |
| | sed 's| \./| source/|' > "$checksums" | |
| find dist -type f -name '*.html' -print0 \ | |
| | sort -z \ | |
| | xargs -0 md5sum >> "$checksums" | |
| jq -n \ | |
| --arg timestamp "$timestamp" \ | |
| --arg sha "$PUBLISHED_SHA" \ | |
| --arg publish_run_id "$PUBLISH_RUN_ID" \ | |
| '{timestamp: $timestamp, commitSha: $sha, publishRunId: $publish_run_id, checksumAlgorithm: "md5", checksumFile: "MD5SUMS"}' \ | |
| > "$manifest" | |
| touch "$success" | |
| aws configure set default.s3.max_concurrent_requests 32 | |
| aws s3 cp . "$destination/source/" \ | |
| --recursive --exclude '.git/*' --exclude 'dist/*' \ | |
| --no-follow-symlinks --no-progress --endpoint-url "$endpoint" | |
| aws s3 cp dist "$destination/dist/" \ | |
| --recursive --exclude '*' --include '*.html' --no-progress \ | |
| --endpoint-url "$endpoint" | |
| aws s3 cp "$checksums" "$destination/MD5SUMS" \ | |
| --no-progress --endpoint-url "$endpoint" | |
| aws s3 cp "$manifest" "$destination/manifest.json" \ | |
| --no-progress --endpoint-url "$endpoint" | |
| aws s3 cp "$success" "$destination/_SUCCESS" \ | |
| --no-progress --endpoint-url "$endpoint" | |
| - name: Notify Google Chat on failure | |
| if: failure() | |
| env: | |
| WEBHOOK_URL: ${{ secrets.CED_TEAM_ALERTS_CHANNEL_WEBHOOK }} | |
| ACTOR: ${{ github.event.workflow_run.actor.login }} | |
| REPO: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| PUBLISHED_SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| MESSAGE="*R2 production snapshot* failed (site already deployed).\nActor: $ACTOR\nCommit: $PUBLISHED_SHA\n<https://github.com/$REPO/actions/runs/$RUN_ID|View run>" | |
| JSON_PAYLOAD=$(jq -n --arg text "$MESSAGE" '{text: $text}') | |
| curl -X POST "$WEBHOOK_URL" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$JSON_PAYLOAD" |