upload-locales.yml

name: Upload Locales to Cloudflare R2

# Called synchronously from the CI workflow ahead of the test matrix:
# when the data version in priv/localize/localize_patch_version is not
# yet on R2, this generates and uploads it so the test jobs (and
# users) can download it. When the version is already published the
# run is a quick no-op. workflow_dispatch is the manual recovery path
# after a transient upload failure.
on:
  workflow_call:
  workflow_dispatch:

env:
  MIX_ENV: dev
  # Toolchain pins — also part of the dependency cache key, since
  # cached BEAMs are only valid for the OTP/Elixir that built them.
  # Kept in step with the developer toolchain (`mise current`).
  # ETF files are serialized with the term_to_binary `:deterministic`
  # option, so the generated bytes are reproducible across platforms
  # and OTP versions regardless of this pin; the pre-upload manifest
  # check is the backstop if that ever stops holding.
  ELIXIR_VERSION: "1.20.2"
  OTP_VERSION: "29.0.3"
  R2_BUCKET: content
  # Prefix within the bucket where locale ETF files are stored.
  R2_PREFIX: locales
  # Maximum concurrent uploads to stay within Cloudflare API limits.
  # R2 allows ~1000 PUT requests/second; we use a conservative batch
  # size to avoid transient rate-limit errors.
  BATCH_SIZE: 50

permissions:
  contents: read

jobs:
  generate-and-upload:
    name: Generate and upload locale ETFs
    runs-on: ubuntu-latest
    # Serialize publish runs so overlapping pushes cannot upload
    # concurrently; the second run in the queue sees the version
    # already on R2 and skips.
    concurrency:
      group: locale-publish
      cancel-in-progress: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      # Read the data version directly from the on-disk version
      # files before doing any expensive work. The version is
      # derived from `priv/localize/version` (CLDR release) and
      # `priv/localize/localize_patch_version` (Localize patch
      # counter). No Elixir compile is required for this step.
      - name: Read data version
        id: version
        run: |
          cldr_version=$(cat priv/localize/version | tr -d '[:space:]')
          patch_raw=$(cat priv/localize/localize_patch_version | tr -d '[:space:]')
          # `localize_patch_version` is stored as "{cldr_version}:{patch}".
          # If the recorded CLDR version matches the current one, take
          # the numeric patch; otherwise treat the patch as 0.
          recorded_cldr="${patch_raw%%:*}"
          recorded_patch="${patch_raw##*:}"
          if [ "${recorded_cldr}" = "${cldr_version}" ]; then
            patch_version="${recorded_patch}"
          else
            patch_version="0"
          fi
          data_version="v${cldr_version}.${patch_version}"
          echo "data_version=${data_version}" >> "$GITHUB_OUTPUT"
          echo "Data version: ${data_version}"

      - name: Install rclone
        run: |
          curl -fsSL https://rclone.org/install.sh | sudo bash

      - name: Configure rclone for Cloudflare R2
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
        run: |
          mkdir -p ~/.config/rclone
          cat > ~/.config/rclone/rclone.conf << EOF
          [r2]
          type = s3
          provider = Cloudflare
          access_key_id = ${R2_ACCESS_KEY_ID}
          secret_access_key = ${R2_SECRET_ACCESS_KEY}
          endpoint = https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com
          acl = private
          no_check_bucket = true
          EOF

      # Short-circuit: if the target R2 prefix already contains ETF
      # files for this data_version, there is nothing to do. This
      # makes the CI-invoked run a cheap no-op on every push that
      # does not change the CLDR or patch version.
      - name: Check if data version already exists on R2
        id: check_existing
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          existing=$(rclone lsf "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" --include "*.etf" 2>/dev/null | wc -l | tr -d '[:space:]')
          echo "Found ${existing} existing ETF file(s) at r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/"
          if [ "${existing}" -gt 0 ]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "::notice::Data version ${DATA_VERSION} already exists on R2 with ${existing} file(s); skipping generation and upload."
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Set up Elixir
        if: steps.check_existing.outputs.skip != 'true'
        uses: erlef/setup-beam@v1
        with:
          elixir-version: ${{ env.ELIXIR_VERSION }}
          otp-version: ${{ env.OTP_VERSION }}

      - name: Restore dependency cache
        if: steps.check_existing.outputs.skip != 'true'
        uses: actions/cache@v4
        with:
          path: |
            deps
            _build
          key: mix-locales-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-${{ hashFiles('mix.lock') }}
          restore-keys: |
            mix-locales-otp${{ env.OTP_VERSION }}-elixir${{ env.ELIXIR_VERSION }}-

      - name: Install dependencies
        if: steps.check_existing.outputs.skip != 'true'
        run: mix deps.get

      - name: Compile
        if: steps.check_existing.outputs.skip != 'true'
        run: mix compile

      - name: Generate locale ETFs
        if: steps.check_existing.outputs.skip != 'true'
        run: mix localize.generate_locales

      - name: Verify generated locales
        if: steps.check_existing.outputs.skip != 'true'
        run: |
          count=$(ls priv/localize/locales/*.etf 2>/dev/null | wc -l)
          echo "Generated ${count} locale ETF files"
          if [ "$count" -lt 100 ]; then
            echo "::error::Expected at least 100 locale files, got ${count}"
            exit 1
          fi

      # The committed manifest pins the exact bytes downloads are
      # verified against. If the files generated here do not match it
      # (for example a toolchain difference changed term serialization),
      # uploading them would make every download fail integrity
      # verification — fail loudly before touching R2 instead.
      # Generation rewrites the manifest in the working tree, so the
      # comparison must read the committed manifest from git.
      - name: Verify generated locales match the committed hash manifest
        if: steps.check_existing.outputs.skip != 'true'
        run: |
          git show HEAD:priv/localize/locale_hashes.etf > /tmp/committed_locale_hashes.etf
          mix run --no-start -e '
          hashes = File.read!("/tmp/committed_locale_hashes.etf") |> :erlang.binary_to_term()

          mismatches =
            Enum.filter(hashes, fn {locale, expected} ->
              actual = :crypto.hash(:sha256, File.read!("priv/localize/locales/#{locale}.etf"))
              actual != expected
            end)

          if mismatches == [] do
            IO.puts("All #{map_size(hashes)} generated files match the committed manifest.")
          else
            IO.puts("::error::Generated files do not match the committed locale_hashes.etf: #{inspect(Enum.map(mismatches, &elem(&1, 0)))}")
            IO.puts("Regenerate the manifest on the same toolchain as this workflow, or align the toolchains.")
            System.halt(1)
          end
          '

      - name: Upload locales to R2
        if: steps.check_existing.outputs.skip != 'true'
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          echo "Uploading to r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/"
          echo "Files: $(ls priv/localize/locales/*.etf | wc -l)"

          # Use rclone sync with bandwidth and concurrency limits.
          # --transfers: concurrent file transfers (R2 allows ~1000 PUT/s,
          #   we use a conservative limit via BATCH_SIZE).
          # --checkers: concurrent hash checks.
          # --s3-upload-concurrency: per-file multipart concurrency.
          rclone sync \
            priv/localize/locales/ \
            "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" \
            --include "*.etf" \
            --transfers ${BATCH_SIZE} \
            --checkers 8 \
            --s3-upload-concurrency 2 \
            --stats 10s \
            --stats-one-line \
            --verbose

      - name: Verify upload
        if: steps.check_existing.outputs.skip != 'true'
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          uploaded=$(rclone ls "r2:${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/" | wc -l)
          local_count=$(ls priv/localize/locales/*.etf | wc -l)
          echo "Uploaded: ${uploaded}, Local: ${local_count}"
          if [ "$uploaded" -ne "$local_count" ]; then
            echo "::warning::Upload count mismatch: ${uploaded} uploaded vs ${local_count} local"
          fi
          echo "Upload complete: ${uploaded} locale files to ${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/"

      - name: Skipped summary
        if: steps.check_existing.outputs.skip == 'true'
        env:
          DATA_VERSION: ${{ steps.version.outputs.data_version }}
        run: |
          echo "Skipped: data version ${DATA_VERSION} already present on R2 at ${R2_BUCKET}/${R2_PREFIX}/${DATA_VERSION}/"
          echo "Nothing was regenerated or uploaded. Bump the CLDR or Localize patch version and push to trigger a real upload."