Add upload inflection workflow for the inflection branch only #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload Inflection Data to Cloudflare R2 | |
| # Publishes the inflection data to Cloudflare R2. On push to the | |
| # `inflection` branch it reads the data version from | |
| # priv/localize/localize_inflection_sha (plus the pipeline revision) | |
| # and uploads that version when it is not already on R2 — i.e. the | |
| # version changed or was never uploaded. A push that leaves the | |
| # version unchanged finds it already on R2 and short-circuits to a | |
| # cheap no-op (no filter, so the never-uploaded case fires on any | |
| # push). Also reusable from ci.yml (post-merge on main, ahead of the | |
| # test matrix) and manually dispatchable for recovery. | |
| on: | |
| push: | |
| branches: [inflection] | |
| workflow_call: | |
| workflow_dispatch: | |
| env: | |
| MIX_ENV: test | |
| # Toolchain pins kept in step with the developer toolchain | |
| # (`mise current`) and with upload-locales.yml. ETF files are | |
| # serialized with the term_to_binary `:deterministic` option, so | |
| # the generated bytes are reproducible across OTP versions; the | |
| # pre-upload manifest check is the backstop if that ever changes. | |
| ELIXIR_VERSION: "1.20.2" | |
| OTP_VERSION: "29.0.3" | |
| R2_BUCKET: content | |
| R2_PREFIX: inflection | |
| # The conformance suite loads every supported locale's lexicon into | |
| # :persistent_term (BEAM literal area), so bump the literal super | |
| # carrier above its default. See plans/MF2_NAMESPACE_INFLECTION.md. | |
| ELIXIR_ERL_OPTIONS: "+MIscs 3072" | |
| permissions: | |
| contents: read | |
| jobs: | |
| build-and-upload: | |
| name: Build and upload inflection data | |
| runs-on: ubuntu-latest | |
| # Serialize publish runs so overlapping pushes cannot upload | |
| # concurrently; the queued run sees the version already on R2 | |
| # and skips. | |
| concurrency: | |
| group: inflection-publish | |
| cancel-in-progress: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # The pin lives in priv/localize/localize_inflection_sha and the | |
| # pipeline revision in Localize.Inflection.Provider — both read | |
| # without an Elixir compile for the cheap short-circuit check. | |
| - name: Read data version | |
| id: version | |
| run: | | |
| sha=$(tr -d '[:space:]' < priv/localize/localize_inflection_sha) | |
| revision=$(grep -o '@data_revision [0-9]*' lib/localize/inflection/provider.ex | grep -o '[0-9]*') | |
| data_version="${sha:0:12}-r${revision}" | |
| echo "data_version=${data_version}" >> "$GITHUB_OUTPUT" | |
| echo "upstream_sha=${sha}" >> "$GITHUB_OUTPUT" | |
| echo "Data version: ${data_version}" | |
| - name: Install rclone | |
| run: | | |
| curl -fsSL https://rclone.org/install.sh | sudo bash | |
| - name: Configure rclone for Cloudflare R2 | |
| env: | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| run: | | |
| mkdir -p ~/.config/rclone | |
| cat > ~/.config/rclone/rclone.conf << RCLONE | |
| [r2] | |
| type = s3 | |
| provider = Cloudflare | |
| access_key_id = ${R2_ACCESS_KEY_ID} | |
| secret_access_key = ${R2_SECRET_ACCESS_KEY} | |
| endpoint = https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com | |
| acl = private | |
| no_check_bucket = true | |
| RCLONE | |
| - name: Check if data version already exists on R2 | |
| id: check_existing | |
| env: | |
| DATA_VERSION: ${{ steps.version.outputs.data_version }} | |
| run: | | |
| existing=$(rclone lsf "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" --include "*.etf" 2>/dev/null | wc -l | tr -d '[:space:]') | |
| if [ "${existing}" -gt 0 ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Inflection data ${DATA_VERSION} already exists on R2 with ${existing} file(s); skipping." | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Set up Elixir | |
| if: steps.check_existing.outputs.skip != 'true' | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| elixir-version: ${{ env.ELIXIR_VERSION }} | |
| otp-version: ${{ env.OTP_VERSION }} | |
| - name: Cache deps and build | |
| if: steps.check_existing.outputs.skip != 'true' | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| deps | |
| _build | |
| key: ${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-mix-inflection-upload-${{ hashFiles('mix.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-mix-inflection-upload- | |
| - name: Cache upstream inflection sources | |
| if: steps.check_existing.outputs.skip != 'true' | |
| uses: actions/cache@v5 | |
| with: | |
| path: data/inflection | |
| key: inflection-source-${{ steps.version.outputs.upstream_sha }}-${{ hashFiles('data/mix/tasks/localize.inflection.download.ex') }} | |
| - name: Cache generated artifacts | |
| if: steps.check_existing.outputs.skip != 'true' | |
| uses: actions/cache@v5 | |
| id: artifact-cache | |
| with: | |
| path: priv/localize/inflection | |
| key: inflection-artifacts-${{ steps.version.outputs.upstream_sha }}-${{ hashFiles('data/inflection_gen/**', 'data/mix/tasks/localize.inflection.*.ex') }} | |
| - name: Install dependencies | |
| if: steps.check_existing.outputs.skip != 'true' | |
| run: mix deps.get | |
| - name: Compile | |
| if: steps.check_existing.outputs.skip != 'true' | |
| run: mix compile --warnings-as-errors | |
| - name: Download and generate inflection data | |
| if: steps.check_existing.outputs.skip != 'true' && steps.artifact-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mix localize.inflection.download | |
| mix localize.inflection.generate | |
| - name: Run inflection conformance suites | |
| if: steps.check_existing.outputs.skip != 'true' | |
| run: mix test test/localize/inflection/ | |
| # The committed manifest pins the exact bytes downloads are | |
| # verified against. Generation is deterministic, so the bytes | |
| # built here must match it; if they do not, uploading would make | |
| # every download fail integrity verification — fail before | |
| # touching R2 instead. Read the committed manifest from git | |
| # because generation rewrites it in the working tree. | |
| - name: Verify generated data matches the committed hash manifest | |
| if: steps.check_existing.outputs.skip != 'true' | |
| run: | | |
| git show HEAD:priv/localize/inflection_hashes.etf > /tmp/committed_inflection_hashes.etf | |
| mix run --no-start -e ' | |
| hashes = File.read!("/tmp/committed_inflection_hashes.etf") |> :erlang.binary_to_term() | |
| mismatches = | |
| Enum.filter(hashes, fn {file, expected} -> | |
| path = "priv/localize/inflection/#{file}" | |
| not File.exists?(path) or :crypto.hash(:sha256, File.read!(path)) != expected | |
| end) | |
| if mismatches == [] do | |
| IO.puts("All #{map_size(hashes)} generated files match the committed manifest.") | |
| else | |
| IO.puts("::error::Generated inflection data does not match the committed inflection_hashes.etf: #{inspect(Enum.map(mismatches, &elem(&1, 0)))}") | |
| IO.puts("Regenerate on the mise-current toolchain and commit the refreshed manifest.") | |
| System.halt(1) | |
| end | |
| ' | |
| - name: Upload to R2 | |
| if: steps.check_existing.outputs.skip != 'true' | |
| env: | |
| DATA_VERSION: ${{ steps.version.outputs.data_version }} | |
| run: | | |
| echo "Uploading to r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" | |
| rclone sync \ | |
| priv/localize/inflection/ \ | |
| "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" \ | |
| --include "*.etf" \ | |
| --transfers 16 \ | |
| --s3-upload-concurrency 2 \ | |
| --stats 10s \ | |
| --stats-one-line \ | |
| --verbose | |
| - name: Skipped summary | |
| if: steps.check_existing.outputs.skip == 'true' | |
| env: | |
| DATA_VERSION: ${{ steps.version.outputs.data_version }} | |
| run: | | |
| echo "Skipped: inflection data ${DATA_VERSION} already present." | |
| echo "Bump the upstream pin or data revision in Localize.Inflection.Provider and push a new tag to trigger a real upload." |