upload-inflection.yml

name: Upload Inflection Data to Cloudflare R2

# Publishes the inflection data to Cloudflare R2. On push to the
# `inflection` branch it reads the data version from
# priv/localize/localize_inflection_sha (plus the pipeline revision)
# and uploads that version when it is not already on R2 — i.e. the
# version changed or was never uploaded. A push that leaves the
# version unchanged finds it already on R2 and short-circuits to a
# cheap no-op (no filter, so the never-uploaded case fires on any
# push). Also reusable from ci.yml (post-merge on main, ahead of the
# test matrix) and manually dispatchable for recovery.
on:
  push:
    branches: [inflection]
  workflow_call:
  workflow_dispatch:

env:
  MIX_ENV: test
  # Toolchain pins kept in step with the developer toolchain
  # (`mise current`) and with upload-locales.yml. ETF files are
  # serialized with the term_to_binary `:deterministic` option, so
  # the generated bytes are reproducible across OTP versions; the
  # pre-upload manifest check is the backstop if that ever changes.
  ELIXIR_VERSION: "1.20.2"
  OTP_VERSION: "29.0.3"
  R2_BUCKET: content
  R2_PREFIX: inflection
  # The conformance suite loads every supported locale's lexicon into
  # :persistent_term (BEAM literal area), so bump the literal super
  # carrier above its default. See plans/MF2_NAMESPACE_INFLECTION.md.
  ELIXIR_ERL_OPTIONS: "+MIscs 3072"

permissions:
  contents: read

jobs:
  build-and-upload:
    name: Build and upload inflection data
    runs-on: ubuntu-latest
    # Serialize publish runs so overlapping pushes cannot upload
    # concurrently; the queued run sees the version already on R2
    # and skips.
    concurrency:
      group: inflection-publish
      cancel-in-progress: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      # The pin lives in priv/localize/localize_inflection_sha and the
      # pipeline revision in Localize.Inflection.Provider — both read
      # without an Elixir compile for the cheap short-circuit check.
      - name: Read data version
        id: version
        run: |
          sha=$(tr -d '[:space:]' < priv/localize/localize_inflection_sha)
          revision=$(grep -o '@data_revision [0-9]*' lib/localize/inflection/provider.ex | grep -o '[0-9]*')
          data_version="${sha:0:12}-r${revision}"
          echo "data_version=${data_version}" >> "$GITHUB_OUTPUT"
          echo "upstream_sha=${sha}" >> "$GITHUB_OUTPUT"
          echo "Data version: ${data_version}"

      - name: Install rclone
        run: |
          curl -fsSL https://rclone.org/install.sh | sudo bash

      - name: Configure rclone for Cloudflare R2
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
        run: |
          mkdir -p ~/.config/rclone
          cat > ~/.config/rclone/rclone.conf << RCLONE
          [r2]
          type = s3
          provider = Cloudflare
          access_key_id = ${R2_ACCESS_KEY_ID}
          secret_access_key = ${R2_SECRET_ACCESS_KEY}
          endpoint = https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com
          acl = private
          no_check_bucket = true
          RCLONE

      - name: Check if data version already exists on R2
        id: check_existing
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          existing=$(rclone lsf "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" --include "*.etf" 2>/dev/null | wc -l | tr -d '[:space:]')
          if [ "${existing}" -gt 0 ]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "::notice::Inflection data ${DATA_VERSION} already exists on R2 with ${existing} file(s); skipping."
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Set up Elixir
        if: steps.check_existing.outputs.skip != 'true'
        uses: erlef/setup-beam@v1
        with:
          elixir-version: ${{ env.ELIXIR_VERSION }}
          otp-version: ${{ env.OTP_VERSION }}

      - name: Cache deps and build
        if: steps.check_existing.outputs.skip != 'true'
        uses: actions/cache@v5
        with:
          path: |
            deps
            _build
          key: ${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-mix-inflection-upload-${{ hashFiles('mix.lock') }}
          restore-keys: |
            ${{ runner.os }}-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-mix-inflection-upload-

      - name: Cache upstream inflection sources
        if: steps.check_existing.outputs.skip != 'true'
        uses: actions/cache@v5
        with:
          path: data/inflection
          key: inflection-source-${{ steps.version.outputs.upstream_sha }}-${{ hashFiles('data/mix/tasks/localize.inflection.download.ex') }}

      - name: Cache generated artifacts
        if: steps.check_existing.outputs.skip != 'true'
        uses: actions/cache@v5
        id: artifact-cache
        with:
          path: priv/localize/inflection
          key: inflection-artifacts-${{ steps.version.outputs.upstream_sha }}-${{ hashFiles('data/inflection_gen/**', 'data/mix/tasks/localize.inflection.*.ex') }}

      - name: Install dependencies
        if: steps.check_existing.outputs.skip != 'true'
        run: mix deps.get

      - name: Compile
        if: steps.check_existing.outputs.skip != 'true'
        run: mix compile --warnings-as-errors

      - name: Download and generate inflection data
        if: steps.check_existing.outputs.skip != 'true' && steps.artifact-cache.outputs.cache-hit != 'true'
        run: |
          mix localize.inflection.download
          mix localize.inflection.generate

      - name: Run inflection conformance suites
        if: steps.check_existing.outputs.skip != 'true'
        run: mix test test/localize/inflection/

      # The committed manifest pins the exact bytes downloads are
      # verified against. Generation is deterministic, so the bytes
      # built here must match it; if they do not, uploading would make
      # every download fail integrity verification — fail before
      # touching R2 instead. Read the committed manifest from git
      # because generation rewrites it in the working tree.
      - name: Verify generated data matches the committed hash manifest
        if: steps.check_existing.outputs.skip != 'true'
        run: |
          git show HEAD:priv/localize/inflection_hashes.etf > /tmp/committed_inflection_hashes.etf
          mix run --no-start -e '
          hashes = File.read!("/tmp/committed_inflection_hashes.etf") |> :erlang.binary_to_term()

          mismatches =
            Enum.filter(hashes, fn {file, expected} ->
              path = "priv/localize/inflection/#{file}"
              not File.exists?(path) or :crypto.hash(:sha256, File.read!(path)) != expected
            end)

          if mismatches == [] do
            IO.puts("All #{map_size(hashes)} generated files match the committed manifest.")
          else
            IO.puts("::error::Generated inflection data does not match the committed inflection_hashes.etf: #{inspect(Enum.map(mismatches, &elem(&1, 0)))}")
            IO.puts("Regenerate on the mise-current toolchain and commit the refreshed manifest.")
            System.halt(1)
          end
          '

      - name: Upload to R2
        if: steps.check_existing.outputs.skip != 'true'
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          echo "Uploading to r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/"
          rclone sync \
            priv/localize/inflection/ \
            "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" \
            --include "*.etf" \
            --transfers 16 \
            --s3-upload-concurrency 2 \
            --stats 10s \
            --stats-one-line \
            --verbose

      - name: Skipped summary
        if: steps.check_existing.outputs.skip == 'true'
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          echo "Skipped: inflection data ${DATA_VERSION} already present."
          echo "Bump the upstream pin or data revision in Localize.Inflection.Provider and push a new tag to trigger a real upload."