rust-release.yml

name: Rust Release

# Triggered by rust-v* tags for Rust library releases
on:
  push:
    tags:
      - 'rust-v*'

permissions:
  actions: write
  contents: write
  id-token: write

env:
  CARGO_TERM_COLOR: always

jobs:
  build:
    name: Build ${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Linux GNU targets
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
            archive-name: libtokenizers-x86_64-unknown-linux-gnu.tar.gz
            lib-name: libtokenizers.so
          - target: aarch64-unknown-linux-gnu
            os: ubuntu-latest
            archive-name: libtokenizers-aarch64-unknown-linux-gnu.tar.gz
            lib-name: libtokenizers.so

          # Linux MUSL targets (for Alpine and other musl-based distros)
          - target: x86_64-unknown-linux-musl
            os: ubuntu-latest
            archive-name: libtokenizers-x86_64-unknown-linux-musl.tar.gz
            lib-name: libtokenizers.a
          - target: aarch64-unknown-linux-musl
            os: ubuntu-latest
            archive-name: libtokenizers-aarch64-unknown-linux-musl.tar.gz
            lib-name: libtokenizers.a

          # macOS targets
          - target: x86_64-apple-darwin
            os: macos-latest
            archive-name: libtokenizers-x86_64-apple-darwin.tar.gz
            lib-name: libtokenizers.dylib
          - target: aarch64-apple-darwin
            os: macos-latest
            archive-name: libtokenizers-aarch64-apple-darwin.tar.gz
            lib-name: libtokenizers.dylib

          # Windows targets
          - target: x86_64-pc-windows-msvc
            os: windows-latest
            archive-name: libtokenizers-x86_64-pc-windows-msvc.tar.gz
            lib-name: tokenizers.dll

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Setup Rust
      uses: ./.github/actions/setup-rust
      with:
        targets: ${{ matrix.target }}
        cache-key-suffix: ${{ matrix.target }}

    - name: Setup cross-compilation
      if: matrix.os == 'ubuntu-latest'
      uses: ./.github/actions/setup-cross-compilation

    - name: Build Rust library
      uses: ./.github/actions/build-rust-library
      with:
        target: ${{ matrix.target }}
        use-cross: ${{ matrix.os == 'ubuntu-latest' && 'true' || 'false' }}

    - name: Prepare release archive (Unix)
      if: matrix.os != 'windows-latest'
      shell: bash
      run: |
        set -euo pipefail

        # Create staging directory
        mkdir -p release-staging

        # Copy the library
        LIBRARY_PATH="target/${{ matrix.target }}/release/${{ matrix.lib-name }}"
        if [ -f "$LIBRARY_PATH" ]; then
          cp "$LIBRARY_PATH" "release-staging/${{ matrix.lib-name }}"
        else
          echo "Library not found at: $LIBRARY_PATH"
          exit 1
        fi

        # Generate SHA256 for the library only
        cd release-staging
        if [[ "${{ matrix.os }}" == "macos-latest" ]]; then
          shasum -a 256 "${{ matrix.lib-name }}" > "${{ matrix.lib-name }}.sha256"
        else
          sha256sum "${{ matrix.lib-name }}" > "${{ matrix.lib-name }}.sha256"
        fi

        # Copy README and LICENSE if they exist
        cd ..
        [ -f README.md ] && cp README.md release-staging/
        [ -f LICENSE.md ] && cp LICENSE.md release-staging/

        # Create the final archive with all contents
        cd release-staging
        tar -czf ../${{ matrix.archive-name }} ./*
        cd ..

        # Display archive contents for verification
        echo "Archive contents:"
        tar -tzf ${{ matrix.archive-name }}

    - name: Prepare artifacts (Windows)
      if: matrix.os == 'windows-latest'
      shell: pwsh
      run: |
        # Create staging directory
        New-Item -ItemType Directory -Force -Path release-staging

        # Copy the library
        $libraryPath = "target/${{ matrix.target }}/release/${{ matrix.lib-name }}"
        if (Test-Path $libraryPath) {
          Copy-Item -Path $libraryPath -Destination "release-staging/${{ matrix.lib-name }}"
        } else {
          Write-Error "Library not found at: $libraryPath"
          exit 1
        }

        # Generate SHA256 for the library only
        Set-Location release-staging
        $hash = Get-FileHash "${{ matrix.lib-name }}" -Algorithm SHA256
        "$($hash.Hash)  ${{ matrix.lib-name }}" | Out-File "${{ matrix.lib-name }}.sha256" -Encoding ascii

        # Copy README and LICENSE if they exist
        Set-Location ..
        if (Test-Path README.md) {
          Copy-Item -Path README.md -Destination release-staging/
        }
        if (Test-Path LICENSE.md) {
          Copy-Item -Path LICENSE.md -Destination release-staging/
        }

        # Create the final archive with all contents
        Set-Location release-staging
        tar -czf ..\${{ matrix.archive-name }} *
        Set-Location ..

        # Display archive contents for verification
        Write-Host "Archive contents:"
        tar -tzf ${{ matrix.archive-name }}

    - name: Upload build artifact
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.target }}-build
        path: ${{ matrix.archive-name }}

  release:
    name: Sign & Publish Release
    runs-on: ubuntu-latest
    needs: build
    env:
      BUCKET: releases
      PROJECT: ${{ github.event.repository.name }}
      VERSION: ${{ github.ref_name }}
      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: artifacts

      - name: Prepare release assets
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p dist
          find artifacts -name "*.tar.gz" -exec cp {} dist/ \;

          shopt -s nullglob
          archives=(dist/*.tar.gz)
          if [ "${#archives[@]}" -eq 0 ]; then
            echo "No release archives found"
            exit 1
          fi

          echo "Release archives:"
          ls -la dist/

      - name: Generate checksums
        shell: bash
        run: |
          set -euo pipefail
          cd dist
          shopt -s nullglob
          archives=( *.tar.gz )
          if [ "${#archives[@]}" -eq 0 ]; then
            echo "No release archives found for checksum generation"
            exit 1
          fi
          sha256sum -- "${archives[@]}" > SHA256SUMS
          while read -r checksum file; do
            printf "%s  %s\n" "${checksum}" "${file}" > "${file}.sha256"
          done < SHA256SUMS

      - name: Install cosign
        uses: sigstore/cosign-installer@v3

      - name: Sign artifacts (keyless)
        shell: bash
        run: |
          set -euo pipefail
          cd dist
          ISSUER="https://token.actions.githubusercontent.com"
          shopt -s nullglob
          archives=( *.tar.gz )
          for f in "${archives[@]}"; do
            cosign sign-blob --yes \
              --oidc-issuer "${ISSUER}" \
              --output-signature "${f}.sig" \
              --output-certificate "${f}.pem" \
              "${f}"
          done

          cosign sign-blob --yes \
            --oidc-issuer "${ISSUER}" \
            --output-signature "SHA256SUMS.sig" \
            --output-certificate "SHA256SUMS.pem" \
            "SHA256SUMS"

      - name: Verify signatures
        shell: bash
        env:
          REPO: ${{ github.repository }}
          TAG: ${{ github.ref_name }}
        run: |
          set -euo pipefail
          cd dist
          IDENTITY="https://github.com/${REPO}/.github/workflows/rust-release.yml@refs/tags/${TAG}"
          ISSUER="https://token.actions.githubusercontent.com"
          shopt -s nullglob
          archives=( *.tar.gz )

          sha256sum -c SHA256SUMS

          for f in "${archives[@]}"; do
            cosign verify-blob \
              --signature "${f}.sig" \
              --certificate "${f}.pem" \
              --certificate-identity "${IDENTITY}" \
              --certificate-oidc-issuer "${ISSUER}" \
              "${f}"
            echo "Verified ${f}"
          done

          cosign verify-blob \
            --signature "SHA256SUMS.sig" \
            --certificate "SHA256SUMS.pem" \
            --certificate-identity "${IDENTITY}" \
            --certificate-oidc-issuer "${ISSUER}" \
            "SHA256SUMS"
          echo "Verified SHA256SUMS"

      - name: Validate R2 settings
        shell: bash
        env:
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
        run: |
          set -euo pipefail
          for var in R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT; do
            if [ -z "${!var}" ]; then
              echo "Missing required secret: ${var}"
              exit 1
            fi
          done

      - name: Check release immutability in R2
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
        run: |
          set -euo pipefail
          PREFIX="${PROJECT}/${VERSION}/"
          immutability_check_err="/tmp/immutability-check.err"
          trap 'rm -f "${immutability_check_err}"' EXIT
          if ! existing_prefix_listing="$(aws s3api list-objects-v2 \
            --bucket "${BUCKET}" \
            --prefix "${PREFIX}" \
            --max-keys 1 \
            --endpoint-url "${R2_ENDPOINT}" \
            --output json 2>"${immutability_check_err}")"; then
            echo "Failed to check release immutability for s3://${BUCKET}/${PREFIX}." >&2
            cat "${immutability_check_err}" >&2
            exit 1
          fi
          rm -f "${immutability_check_err}"
          if jq -e '.Contents | type == "array" and length > 0' >/dev/null <<<"${existing_prefix_listing}"; then
            echo "Release prefix already exists: s3://${BUCKET}/${PREFIX}"
            echo "Refusing to overwrite immutable release artifacts."
            exit 1
          fi

      - name: Upload artifacts and publish release metadata to R2
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
        run: |
          set -euo pipefail
          releases_fetch_err="/tmp/releases-fetch.err"
          trap 'rm -f "${releases_fetch_err}"' EXIT
          aws s3 cp dist/ "s3://${BUCKET}/${PROJECT}/${VERSION}/" \
            --recursive \
            --endpoint-url "${R2_ENDPOINT}"

          RELEASE_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
          jq -n \
            --arg version "${VERSION}" \
            --arg date "${RELEASE_DATE}" \
            --arg checksums_url "${PROJECT}/${VERSION}/SHA256SUMS" \
            '{version: $version, date: $date, checksums_url: $checksums_url}' \
            > /tmp/latest.json

          aws s3 cp /tmp/latest.json "s3://${BUCKET}/${PROJECT}/latest.json" \
            --endpoint-url "${R2_ENDPOINT}" \
            --content-type "application/json" \
            --cache-control "public, max-age=60"

          # Maintain bounded project release history for clients that need more than latest.json.
          # This logic is externalized to a script so it can be tested locally.
          if ! aws s3 cp "s3://${BUCKET}/${PROJECT}/releases.json" /tmp/releases-existing.json \
            --endpoint-url "${R2_ENDPOINT}" 2>"${releases_fetch_err}"; then
            if grep -qiE '404|NoSuchKey' "${releases_fetch_err}"; then
              rm -f /tmp/releases-existing.json
            else
              echo "Failed to fetch existing releases.json from R2." >&2
              cat "${releases_fetch_err}" >&2
              exit 1
            fi
          fi
          rm -f "${releases_fetch_err}"

          ./scripts/build_releases_index.sh \
            --existing /tmp/releases-existing.json \
            --output /tmp/releases.json \
            --project "${PROJECT}" \
            --version "${VERSION}" \
            --date "${RELEASE_DATE}" \
            --max 100

          if ! jq -e '.releases | type == "array" and length > 0' /tmp/releases.json >/dev/null; then
            echo "Generated releases.json is missing or invalid." >&2
            exit 1
          fi

          ISSUER="https://token.actions.githubusercontent.com"
          IDENTITY="https://github.com/${GITHUB_REPOSITORY}/.github/workflows/rust-release.yml@refs/tags/${VERSION}"
          cosign sign-blob --yes \
            --oidc-issuer "${ISSUER}" \
            --output-signature /tmp/releases.json.sig \
            --output-certificate /tmp/releases.json.pem \
            /tmp/releases.json
          cosign verify-blob \
            --signature /tmp/releases.json.sig \
            --certificate /tmp/releases.json.pem \
            --certificate-identity "${IDENTITY}" \
            --certificate-oidc-issuer "${ISSUER}" \
            /tmp/releases.json

          aws s3 cp /tmp/releases.json "s3://${BUCKET}/${PROJECT}/releases.json" \
            --endpoint-url "${R2_ENDPOINT}" \
            --content-type "application/json" \
            --cache-control "public, max-age=60"
          aws s3 cp /tmp/releases.json.sig "s3://${BUCKET}/${PROJECT}/releases.json.sig" \
            --endpoint-url "${R2_ENDPOINT}" \
            --content-type "text/plain" \
            --cache-control "public, max-age=60"
          aws s3 cp /tmp/releases.json.pem "s3://${BUCKET}/${PROJECT}/releases.json.pem" \
            --endpoint-url "${R2_ENDPOINT}" \
            --content-type "text/plain" \
            --cache-control "public, max-age=60"

      - name: Purge release metadata from CDN cache
        if: vars.CF_ZONE_ID != '' && env.CLOUDFLARE_API_TOKEN != ''
        shell: bash
        env:
          CF_ZONE_ID: ${{ vars.CF_ZONE_ID }}
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          RELEASES_DOMAIN: ${{ vars.RELEASES_DOMAIN || 'releases.amikos.tech' }}
        run: |
          set -euo pipefail
          curl -sf -X POST \
            "https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/purge_cache" \
            -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
            -H "Content-Type: application/json" \
            --data "{\"files\":[\"https://${RELEASES_DOMAIN}/${PROJECT}/latest.json\",\"https://${RELEASES_DOMAIN}/${PROJECT}/releases.json\",\"https://${RELEASES_DOMAIN}/${PROJECT}/releases.json.sig\",\"https://${RELEASES_DOMAIN}/${PROJECT}/releases.json.pem\"]}"
          echo "CDN cache purged for ${PROJECT}/latest.json and ${PROJECT}/releases.json"

      - name: Create Release
        uses: softprops/action-gh-release@v2
        with:
          files: |
            dist/*.tar.gz
            dist/*.sha256
            dist/SHA256SUMS
            dist/*.sig
            dist/*.pem
          name: Rust Library ${{ github.ref_name }}
          body: |
            ## Rust Tokenizers Library Release

            This release contains pre-built tokenizers libraries for multiple platforms.

            ### Supported Platforms
            - Linux x86_64 (GNU)
            - Linux aarch64 (GNU)
            - Linux x86_64 (MUSL)
            - Linux aarch64 (MUSL)
            - macOS x86_64
            - macOS aarch64 (Apple Silicon)
            - Windows x86_64

            ### Installation
            Download the appropriate archive for your platform and extract the library file.

            ### Usage with Go bindings
            The Go bindings will automatically download these libraries when needed.
            You can also set `TOKENIZERS_LIB_PATH` environment variable to use a specific library.

            ### R2 Release Endpoint
            Release assets are also published to:
            `https://${{ vars.RELEASES_DOMAIN || 'releases.amikos.tech' }}/${{ github.event.repository.name }}/${{ github.ref_name }}/`
          fail_on_unmatched_files: true
          generate_release_notes: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}