Merge pull request #114 from amikos-tech/codex/evaluate-concurrent-to… #252
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration CI | |
| # Main integration CI that runs when both Rust and Go code changes | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| permissions: | |
| contents: read | |
| jobs: | |
| rust-test: | |
| name: Rust Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: ./.github/actions/setup-rust | |
| with: | |
| components: rustfmt, clippy | |
| - name: Check formatting | |
| run: cargo fmt -- --check | |
| - name: Run clippy | |
| run: cargo clippy -- -D warnings | |
| - name: Run Rust tests | |
| run: cargo test --verbose | |
| - name: Build library | |
| run: cargo build --release | |
| go-test: | |
| name: Go Tests | |
| runs-on: ${{ matrix.os }} | |
| needs: rust-test | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| go-version: ['1.24'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ matrix.go-version }} | |
| - name: Setup Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Install cargo-zigbuild (Linux/macOS) | |
| if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' | |
| run: cargo install cargo-zigbuild | |
| - name: Build library (macOS) | |
| if: matrix.os == 'macos-latest' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| LIB_PATH="$( | |
| cargo build --release --message-format=json | | |
| jq -r 'select(.reason=="compiler-artifact") | |
| | select(.target.kind[]=="cdylib") | |
| | .filenames[] | |
| | select(endswith(".dylib"))' | head -n1 | |
| )" | |
| echo "TOKENIZERS_LIB_PATH=$LIB_PATH" >> "$GITHUB_ENV" | |
| echo "Using $LIB_PATH" | |
| - name: Build library (Linux) | |
| if: matrix.os == 'ubuntu-latest' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ARGS=(--release --message-format=json) | |
| LIB_PATH="$( | |
| cargo zigbuild "${ARGS[@]}" | | |
| jq -r 'select(.reason=="compiler-artifact") | |
| | select(.target.kind[]=="cdylib") | |
| | .filenames[] | |
| | select(endswith(".so"))' | head -n1 | |
| )" | |
| echo "TOKENIZERS_LIB_PATH=$LIB_PATH" >> "$GITHUB_ENV" | |
| echo "Using $LIB_PATH" | |
| - name: Build library (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| # Build and capture output | |
| $buildOutput = cargo build --release --message-format=json | |
| if ($LASTEXITCODE -ne 0) { | |
| throw "Cargo build failed with exit code $LASTEXITCODE" | |
| } | |
| # Parse and find DLL | |
| $dll = $buildOutput ` | |
| | % { | |
| try { | |
| $_ | ConvertFrom-Json | |
| } catch { | |
| Write-Warning "Failed to parse JSON: $_" | |
| $null | |
| } | |
| } ` | |
| | ? { $_.reason -eq "compiler-artifact" -and $_.target.kind -contains "cdylib" } ` | |
| | % { $_.filenames } | ? { $_ -like "*.dll" } ` | |
| | Get-Item | Sort-Object LastWriteTime -Descending ` | |
| | Select-Object -ExpandProperty FullName -First 1 | |
| # Validate result | |
| if (-not $dll) { | |
| throw "No DLL found in build output" | |
| } | |
| if (-not (Test-Path $dll)) { | |
| throw "DLL path does not exist: $dll" | |
| } | |
| "TOKENIZERS_LIB_PATH=$dll" | Out-File -FilePath $env:GITHUB_ENV -Append | |
| Write-Host "Using DLL: $dll" | |
| - name: Check Library Path (Linux/macOS) | |
| if: matrix.os != 'windows-latest' | |
| shell: bash | |
| run: | | |
| if [ -z "$TOKENIZERS_LIB_PATH" ]; then | |
| echo "TOKENIZERS_LIB_PATH is not set. Ensure the build step succeeded." | |
| exit 1 | |
| fi | |
| file "$TOKENIZERS_LIB_PATH" || true | |
| - name: Check Library Path (Windows) | |
| if: matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| if (-not $env:TOKENIZERS_LIB_PATH) { | |
| Write-Error "TOKENIZERS_LIB_PATH is not set. Ensure the build step succeeded." | |
| exit 1 | |
| } | |
| Get-Item $env:TOKENIZERS_LIB_PATH | |
| - name: Run Go linting | |
| uses: golangci/golangci-lint-action@v6 | |
| with: | |
| version: latest | |
| args: --timeout=5m | |
| - name: Run Go tests | |
| run: | | |
| go mod tidy | |
| make test-ci | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| go-security: | |
| uses: ./.github/workflows/go-security.yml | |
| with: | |
| go-version: '1.24' | |
| integration-test: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: [rust-test, go-test, go-security] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.24' | |
| - name: Setup Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Build library | |
| run: cargo build --release | |
| - name: Test download functionality (mock) | |
| run: | | |
| export TOKENIZERS_LIB_PATH="$(pwd)/target/release/libtokenizers.so" | |
| go test -v -run "TestDownloadFunctionality|TestGetLibraryInfo" | |
| - name: Test library loading | |
| run: | | |
| export TOKENIZERS_LIB_PATH="$(pwd)/target/release/libtokenizers.so" | |
| go test -v -run "TestGetCachedLibraryPath|TestCacheDirectory" |