diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c45f35cb5..e3358ae1b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,12 +125,11 @@ jobs: ci_license_diff: name: License Diff - needs: [prepare, ci_changes, ci_check] + needs: [prepare, ci_changes] uses: ./.github/workflows/ci_license_diff.yml if: >- ${{ needs.prepare.outputs.is_pr == 'true' - && needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_dependencies == 'true' }} permissions: @@ -142,9 +141,9 @@ jobs: ci_rust: name: Rust - needs: [prepare, ci_changes, ci_check] + needs: [prepare, ci_changes] uses: ./.github/workflows/ci_rust.yml - if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_rust == 'true' }} + if: ${{ needs.ci_changes.outputs.run_rust == 'true' }} permissions: contents: read secrets: @@ -154,9 +153,9 @@ jobs: ci_go: name: Go - needs: [prepare, ci_changes, ci_check] + needs: [prepare, ci_changes] uses: ./.github/workflows/ci_go.yml - if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_go == 'true' }} + if: ${{ needs.ci_changes.outputs.run_go == 'true' }} permissions: contents: read secrets: @@ -164,9 +163,9 @@ jobs: ci_node: name: Node.js - needs: [prepare, ci_changes, ci_check] + needs: [prepare, ci_changes] uses: ./.github/workflows/ci_node.yml - if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_node == 'true' }} + if: ${{ needs.ci_changes.outputs.run_node == 'true' }} permissions: contents: read secrets: @@ -179,9 +178,9 @@ jobs: ci_python: name: Python - needs: [prepare, ci_changes, ci_check] + needs: [prepare, ci_changes] uses: ./.github/workflows/ci_python.yml - if: ${{ needs.ci_check.result == 'success' && ( needs.ci_changes.outputs.run_python == 'true' || needs.ci_changes.outputs.run_python_integration_langchain == 'true' ) }} + if: ${{ needs.ci_changes.outputs.run_python == 'true' || needs.ci_changes.outputs.run_python_integration_langchain == 'true' }} permissions: contents: read secrets: @@ -189,7 +188,7 @@ jobs: with: ref_type: ${{ github.ref_type }} ref_name: ${{ github.ref_name }} - run_package: ${{ ( needs.ci_changes.outputs.run_python == 'true' || needs.ci_changes.outputs.run_python_integration_langchain == 'true' ) }} + run_package: ${{ needs.ci_changes.outputs.run_python_package == 'true' }} run_integration_langchain: ${{ needs.ci_changes.outputs.run_python_integration_langchain == 'true' }} ci_required: diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index ab2cdc6fd..a8c686750 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -44,18 +44,27 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 10 strategy: - fail-fast: false + fail-fast: true matrix: include: - platform: linux-x86_64 runner: ubuntu-latest + shell: sh run_mock_tests: true - platform: linux-arm64 runner: ubuntu-24.04-arm + shell: sh run_mock_tests: false - platform: macos-arm64 runner: macos-15 + shell: sh run_mock_tests: false + - platform: windows-x86_64 + runner: windows-2022 + shell: pwsh + - platform: windows-ARM64 + runner: windows-11-arm + shell: pwsh permissions: contents: read @@ -66,41 +75,22 @@ jobs: persist-credentials: false - name: Validate installer + if: ${{ matrix.shell == 'sh' }} run: | sh -n install.sh sh -n scripts/test-install.sh sh -n scripts/test-install-mocks.sh - name: Run live installer tests + if: ${{ matrix.shell == 'sh' }} run: sh scripts/test-install.sh - name: Run mock-only installer tests - if: ${{ matrix.run_mock_tests }} + if: ${{ matrix.shell == 'sh' && matrix.run_mock_tests }} run: sh scripts/test-install-mocks.sh - install-script-windows: - name: Install script (Windows ${{ matrix.architecture }}) - if: ${{ inputs.run_installer }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - - architecture: x86_64 - runner: windows-2022 - - architecture: ARM64 - runner: windows-11-arm - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - name: Validate PowerShell installer syntax + if: ${{ matrix.shell == 'pwsh' }} shell: pwsh run: | $scripts = @('install.ps1', 'scripts/test-install.ps1') @@ -116,6 +106,7 @@ jobs: } - name: Run PowerShell installer tests + if: ${{ matrix.shell == 'pwsh' }} shell: pwsh run: ./scripts/test-install.ps1 @@ -164,7 +155,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-linux-amd64-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false diff --git a/.github/workflows/ci_go.yml b/.github/workflows/ci_go.yml index cf06434cb..bcabca159 100644 --- a/.github/workflows/ci_go.yml +++ b/.github/workflows/ci_go.yml @@ -25,11 +25,10 @@ jobs: name: Test (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - platform: linux-amd64 @@ -62,7 +61,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index 9f96ef560..7d7d8d4c5 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -15,7 +15,7 @@ on: required: true type: string run_package: - description: 'Whether to build package artifacts after tests pass' + description: 'Whether to build package artifacts' required: false default: true type: boolean @@ -44,22 +44,26 @@ jobs: name: Test (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 + - &linux-amd64 + platform: linux-amd64 runner: ubuntu-latest - - platform: linux-arm64 + - &linux-arm64 + platform: linux-arm64 runner: ubuntu-24.04-arm - - platform: macos-arm64 + - &macos-arm64 + platform: macos-arm64 runner: macos-15 - - platform: windows-amd64 + - &windows-amd64 + platform: windows-amd64 runner: windows-2022 - - platform: windows-arm64 + - &windows-arm64 + platform: windows-arm64 runner: windows-11-arm steps: @@ -77,11 +81,10 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false - save-if: false - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 with: @@ -113,26 +116,32 @@ jobs: Package: name: Package (${{ matrix.platform }}) - needs: [Test] - if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + if: ${{ inputs.run_package }} runs-on: ${{ matrix.runner }} - timeout-minutes: 30 + timeout-minutes: 60 permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 + - *linux-amd64 + - *linux-arm64 + - *macos-arm64 + - *windows-amd64 + - *windows-arm64 + - &package-linux-musl-amd64 + node_arch: x64 + platform: linux-musl-amd64 runner: ubuntu-latest - - platform: linux-arm64 + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + package_name: nemo-relay-node-linux-x64-musl + - &package-linux-musl-arm64 + node_arch: arm64 + platform: linux-musl-arm64 runner: ubuntu-24.04-arm - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm + container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a + package_name: nemo-relay-node-linux-arm64-musl steps: - name: Checkout @@ -143,27 +152,28 @@ jobs: uses: ./.github/actions/load-ci-tool-versions - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + if: ${{ !contains(matrix.platform, 'musl') }} with: cache: false toolchain: ${{ steps.ci-config.outputs.rust_version }} - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + if: ${{ !contains(matrix.platform, 'musl') }} with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false - save-if: false - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 - if: ${{ startsWith(matrix.platform, 'linux-') }} + if: ${{ startsWith(matrix.platform, 'linux-') && !contains(matrix.platform, 'musl') }} with: version: ${{ steps.ci-config.outputs.uv_version }} enable-cache: true cache-dependency-glob: ${{ env.NEMO_RELAY_CI_WORKSPACE }}/uv.lock - name: Install managed Python for Zig packaging - if: ${{ startsWith(matrix.platform, 'linux-') }} + if: ${{ startsWith(matrix.platform, 'linux-') && !contains(matrix.platform, 'musl') }} run: | set -e UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} @@ -173,21 +183,22 @@ jobs: node-version: ${{ steps.ci-config.outputs.node_version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + if: ${{ !contains(matrix.platform, 'musl') }} with: python-version: ${{ steps.ci-config.outputs.default_python_version }} - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 + if: ${{ !contains(matrix.platform, 'musl') }} with: tool: just@${{ steps.ci-config.outputs.just_version }} - name: Create packaging output directory - if: ${{ inputs.run_package }} + if: ${{ !contains(matrix.platform, 'musl') }} run: | set -e mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm" - name: Derive Node package version - if: ${{ inputs.run_package }} working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | set -e @@ -201,7 +212,7 @@ jobs: printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - name: Package Node - if: ${{ inputs.run_package }} + if: ${{ !contains(matrix.platform, 'musl') }} working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | set -e @@ -214,54 +225,14 @@ jobs: - name: Upload split Node npm package artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: ${{ inputs.run_package }} + if: ${{ !contains(matrix.platform, 'musl') }} with: name: node-npm-package-${{ matrix.platform }} path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm/*.tgz if-no-files-found: error - PackageMusllinux: - name: Package (musllinux-${{ matrix.node_arch }}) - needs: [Test] - if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - node_arch: x64 - platform: linux-musl-amd64 - runner: ubuntu-latest - container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a - - node_arch: arm64 - platform: linux-musl-arm64 - runner: ubuntu-24.04-arm - container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Load CI tool versions - id: ci-config - uses: ./.github/actions/load-ci-tool-versions - - - name: Derive Node package version - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} - run: | - set -euo pipefail - version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync("crates/node/package.json", "utf8")); if (!pkg.version) { throw new Error("crates/node/package.json missing version field"); } console.log(pkg.version);')" - if [[ "${{ inputs.ref_type }}" == "tag" ]]; then - version="${{ inputs.ref_name }}" - else - version="${version}+${GITHUB_SHA::8}" - fi - printf 'NEMO_RELAY_NODE_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - - name: Build musllinux Node package + if: ${{ contains(matrix.platform, 'musl') }} working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | set -euo pipefail @@ -272,7 +243,7 @@ jobs: --workdir "${NEMO_RELAY_CI_WORKSPACE}" \ --env NEMO_RELAY_RUST_VERSION="${{ steps.ci-config.outputs.rust_version }}" \ --env NEMO_RELAY_NODE_OUTPUT_DIR="$output_dir" \ - --env NEMO_RELAY_NODE_VERSION="${NEMO_RELAY_NODE_PACKAGE_VERSION}" \ + --env NEMO_RELAY_NODE_VERSION="${NEMO_RELAY_PACKAGE_VERSION}" \ --env NEMO_RELAY_NODE_PLATFORM="${{ matrix.platform }}" \ "${{ matrix.container_image }}" \ sh -ceu ' @@ -286,89 +257,16 @@ jobs: ' - name: Upload musllinux Node npm package artifact + if: ${{ contains(matrix.platform, 'musl') }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: node-npm-package-${{ matrix.platform }} path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/npm-musllinux/*.tgz if-no-files-found: error - PackageMusllinuxSmoke: - name: Package smoke (musllinux-${{ matrix.node_arch }}) - needs: [Package, PackageMusllinux] - if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' && needs.PackageMusllinux.result == 'success' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - node_arch: x64 - platform: linux-musl-amd64 - package_name: nemo-relay-node-linux-x64-musl - runner: ubuntu-latest - container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a - - node_arch: arm64 - platform: linux-musl-arm64 - package_name: nemo-relay-node-linux-arm64-musl - runner: ubuntu-24.04-arm - container_image: node:24-alpine3.22@sha256:191c9f0080fcbbc6547a85dc0ff7988072214a355aabdc1d2ec55a7dae5eea8a - - steps: - - name: Download Node metapackage artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: node-npm-package-linux-amd64 - path: node-metapackage/ - - - name: Download matching musllinux Node native package artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: node-npm-package-${{ matrix.platform }} - path: node-native/ - - - name: Install and load musllinux split Node package - run: | - set -euo pipefail - docker run --rm \ - --volume "${{ github.workspace }}:${{ github.workspace }}" \ - --workdir "${{ github.workspace }}" \ - --env NEMO_RELAY_NODE_PACKAGE_NAME="${{ matrix.package_name }}" \ - --env NEMO_RELAY_WORKSPACE="${{ github.workspace }}" \ - "${{ matrix.container_image }}" \ - sh -ceu ' - smoke_dir="$(mktemp -d)" - trap "rm -rf $smoke_dir" EXIT - cd "$smoke_dir" - npm init --yes >/dev/null - metapackage="$(find "$NEMO_RELAY_WORKSPACE/node-metapackage" -name "nemo-relay-node-npm-[0-9]*.tgz" -print -quit)" - native="$(find "$NEMO_RELAY_WORKSPACE/node-native" -name "*.tgz" -print -quit)" - test -n "$metapackage" - test -n "$native" - npm install --ignore-scripts "$native" "$metapackage" - node -e " - const assert = require(\"node:assert/strict\"); - const manifest = require(process.env.NEMO_RELAY_NODE_PACKAGE_NAME + \"/package.json\"); - assert.deepEqual(manifest.libc, [\"musl\"]); - for (const entrypoint of [ - \"nemo-relay-node\", - \"nemo-relay-node/typed\", - \"nemo-relay-node/plugin\", - \"nemo-relay-node/adaptive\", - \"nemo-relay-node/observability\", - \"nemo-relay-node/pii_redaction\", - \"nemo-relay-node/model_pricing\", - ]) { - require(entrypoint); - } - " - ' - PackageOpenClaw: name: Package OpenClaw plugin - needs: [Test] - if: ${{ inputs.run_openclaw && !cancelled() && needs.Test.result == 'success' }} + if: ${{ inputs.run_openclaw }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -388,7 +286,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-openclaw-package-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-linux-amd64-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false @@ -441,30 +339,30 @@ jobs: permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 - runner: ubuntu-latest - - platform: linux-arm64 - runner: ubuntu-24.04-arm - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm + - *linux-amd64 + - *linux-arm64 + - *macos-arm64 + - *windows-amd64 + - *windows-arm64 + - *package-linux-musl-amd64 + - *package-linux-musl-arm64 steps: - name: Checkout + if: ${{ !contains(matrix.platform, 'musl') }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - name: Load CI tool versions + if: ${{ !contains(matrix.platform, 'musl') }} id: ci-config uses: ./.github/actions/load-ci-tool-versions - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + if: ${{ !contains(matrix.platform, 'musl') }} with: node-version: ${{ steps.ci-config.outputs.node_version }} @@ -481,6 +379,7 @@ jobs: path: node-native/ - name: Install and load split Node package + if: ${{ !contains(matrix.platform, 'musl') }} run: | set -euo pipefail mkdir package-smoke @@ -508,3 +407,48 @@ jobs: require(entrypoint); } NODE + + - name: Install and load musllinux split Node package + if: ${{ contains(matrix.platform, 'musl') }} + run: | + set -euo pipefail + docker run --rm \ + --volume "${{ github.workspace }}:${{ github.workspace }}" \ + --workdir "${{ github.workspace }}" \ + --env NEMO_RELAY_NODE_PACKAGE_NAME="${{ matrix.package_name }}" \ + --env NEMO_RELAY_WORKSPACE="${{ github.workspace }}" \ + "${{ matrix.container_image }}" \ + sh -ceu ' + smoke_dir="$(mktemp -d)" + trap "rm -rf $smoke_dir" EXIT + cd "$smoke_dir" + npm init --yes >/dev/null + metapackage="$(find "$NEMO_RELAY_WORKSPACE/node-metapackage" -name "nemo-relay-node-npm-[0-9]*.tgz" -print -quit)" + native_count="$(find "$NEMO_RELAY_WORKSPACE/node-native" -name "nemo-relay-node-npm-*-*-[0-9]*.tgz" | wc -l)" + test "$native_count" -eq 1 + native="$(find "$NEMO_RELAY_WORKSPACE/node-native" -name "nemo-relay-node-npm-*-*-[0-9]*.tgz" -print -quit)" + test -n "$metapackage" + test -n "$native" + npm install --ignore-scripts "$native" "$metapackage" + node -e " + const assert = require(\"node:assert/strict\"); + const { readdirSync } = require(\"node:fs\"); + const manifest = require(process.env.NEMO_RELAY_NODE_PACKAGE_NAME + \"/package.json\"); + const nativePackages = readdirSync(\"node_modules\").filter( + (name) => name.startsWith(\"nemo-relay-node-\") && name !== \"nemo-relay-node\", + ); + assert.equal(nativePackages.length, 1); + assert.deepEqual(manifest.libc, [\"musl\"]); + for (const entrypoint of [ + \"nemo-relay-node\", + \"nemo-relay-node/typed\", + \"nemo-relay-node/plugin\", + \"nemo-relay-node/adaptive\", + \"nemo-relay-node/observability\", + \"nemo-relay-node/pii_redaction\", + \"nemo-relay-node/model_pricing\", + ]) { + require(entrypoint); + } + " + ' diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 6a7453610..e0487110f 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -15,7 +15,7 @@ on: required: true type: string run_package: - description: 'Whether to build package artifacts after tests pass' + description: 'Whether to build package artifacts' required: false default: true type: boolean @@ -44,22 +44,26 @@ jobs: name: Test (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 + - &linux-amd64 + platform: linux-amd64 runner: ubuntu-latest - - platform: linux-arm64 + - &linux-arm64 + platform: linux-arm64 runner: ubuntu-24.04-arm - - platform: macos-arm64 + - &macos-arm64 + platform: macos-arm64 runner: macos-15 - - platform: windows-amd64 + - &windows-amd64 + platform: windows-amd64 runner: windows-2022 - - platform: windows-arm64 + - &windows-arm64 + platform: windows-arm64 runner: windows-11-arm steps: @@ -77,11 +81,10 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false - save-if: false - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 with: @@ -163,24 +166,32 @@ jobs: working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: uv cache prune --ci - Package-musllinux: - name: Package (musllinux-${{ matrix.architecture }}) - needs: [Test] - if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + Package: + name: Package (${{ matrix.platform }}) + if: ${{ inputs.run_package }} runs-on: ${{ matrix.runner }} timeout-minutes: 45 permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - architecture: x86_64 + - *linux-amd64 + - *linux-arm64 + - *macos-arm64 + - *windows-amd64 + - *windows-arm64 + - &package-linux-musl-amd64 + platform: linux-musl-amd64 + architecture: x86_64 runner: ubuntu-latest - image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 - - architecture: aarch64 + container_image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 + - &package-linux-musl-arm64 + platform: linux-musl-arm64 + architecture: aarch64 runner: ubuntu-24.04-arm - image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 + container_image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 steps: - name: Checkout @@ -190,7 +201,138 @@ jobs: id: ci-config uses: ./.github/actions/load-ci-tool-versions - - name: Derive packaging tool versions + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + if: ${{ !contains(matrix.platform, 'musl') }} + with: + cache: false + toolchain: ${{ steps.ci-config.outputs.rust_version }} + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + if: ${{ !contains(matrix.platform, 'musl') }} + with: + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} + workspaces: . -> target + cache-all-crates: true + cache-bin: false + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + if: ${{ !contains(matrix.platform, 'musl') }} + with: + version: ${{ steps.ci-config.outputs.uv_version }} + enable-cache: true + cache-dependency-glob: ${{ env.NEMO_RELAY_CI_WORKSPACE }}/uv.lock + + - name: Set up Windows ARM Python + if: ${{ matrix.platform == 'windows-arm64' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + architecture: arm64 + + - name: Select setup-python interpreter for uv + if: ${{ matrix.platform == 'windows-arm64' }} + run: | + set -e + python - <<'PY' >> "$GITHUB_ENV" + import sys + print(f"UV_PYTHON={sys.executable}") + PY + + - name: Install managed Python + if: ${{ !contains(matrix.platform, 'musl') && matrix.platform != 'windows-arm64' }} + run: | + set -e + UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 + if: ${{ !contains(matrix.platform, 'musl') }} + with: + tool: just@${{ steps.ci-config.outputs.just_version }} + + - name: Create packaging output directory + if: ${{ !contains(matrix.platform, 'musl') }} + run: | + set -e + mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/wheels" + + - name: Derive Python package version + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -e + version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" + if [ -z "$version" ]; then + echo "Error: failed to read workspace version from Cargo.toml" >&2 + exit 1 + fi + sha="${GITHUB_SHA::8}" + if [ "${{ inputs.ref_type }}" = "tag" ]; then + version="${{ inputs.ref_name }}" + else + version="${version}+${sha}" + fi + printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + + - name: Package Python wheel + if: ${{ !contains(matrix.platform, 'musl') }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -e + just \ + --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ + package-python + + - name: Upload wheel artifact + if: ${{ !contains(matrix.platform, 'musl') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheel-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/wheels/*.whl + if-no-files-found: error + + - name: Package Python source distribution + if: ${{ matrix.platform == 'linux-amd64' }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -e + just \ + --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ + package-python-sdist + + - name: Upload Python source distribution artifact + if: ${{ matrix.platform == 'linux-amd64' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-sdist + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/sdists/nemo_relay-*.tar.gz + if-no-files-found: error + + - name: Package Python plugin SDK wheel + if: ${{ matrix.platform == 'linux-amd64' }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: | + set -e + just \ + --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ + --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ + package-python-plugin + + - name: Upload Python plugin SDK wheel artifact + if: ${{ matrix.platform == 'linux-amd64' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: plugin-wheel + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/plugin-wheels/*.whl + if-no-files-found: error + + - name: Prune uv cache + if: ${{ !contains(matrix.platform, 'musl') }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: uv cache prune --ci + + - name: Derive Maturin version + if: ${{ contains(matrix.platform, 'musl') }} run: | set -euo pipefail maturin_version="$(awk ' @@ -205,29 +347,18 @@ jobs: echo "Error: failed to read maturin version from uv.lock" >&2 exit 1 fi - version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" - if [[ -z "$version" ]]; then - echo "Error: failed to read workspace version from Cargo.toml" >&2 - exit 1 - fi - if [[ "${{ inputs.ref_type }}" == "tag" ]]; then - version="${{ inputs.ref_name }}" - else - version="${version}+${GITHUB_SHA::8}" - fi - { - printf 'NEMO_RELAY_MATURIN_VERSION=%s\n' "$maturin_version" - printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" - } >> "$GITHUB_ENV" + printf 'NEMO_RELAY_MATURIN_VERSION=%s\n' "$maturin_version" >> "$GITHUB_ENV" - - name: Create packaging output directory + - name: Create musllinux packaging output directory + if: ${{ contains(matrix.platform, 'musl') }} run: | set -euo pipefail mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels" - - name: Build and smoke test musllinux wheel + - name: Build musllinux wheel + if: ${{ contains(matrix.platform, 'musl') }} env: - MUSLLINUX_IMAGE: ${{ matrix.image }} + MUSLLINUX_IMAGE: ${{ matrix.container_image }} MUSLLINUX_INTERPRETER: /opt/python/cp311-cp311/bin/python MUSLLINUX_WHEEL_DIRECTORY: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels NEMO_RELAY_RUST_VERSION: ${{ steps.ci-config.outputs.rust_version }} @@ -253,14 +384,10 @@ jobs: --version "$NEMO_RELAY_PACKAGE_VERSION" \ --out "$MUSLLINUX_WHEEL_DIRECTORY" \ --interpreter "$MUSLLINUX_INTERPRETER" - wheel="$(find "$MUSLLINUX_WHEEL_DIRECTORY" -maxdepth 1 -name "*.whl" -print -quit)" - test -n "$wheel" - "$MUSLLINUX_INTERPRETER" -m venv /tmp/nemo-relay-musllinux-venv - /tmp/nemo-relay-musllinux-venv/bin/python -m pip install --no-deps "$wheel" - /tmp/nemo-relay-musllinux-venv/bin/python -c "import nemo_relay" ' - - name: Verify wheel platform tag + - name: Verify musllinux wheel platform tag + if: ${{ contains(matrix.platform, 'musl') }} run: | set -euo pipefail shopt -s nullglob @@ -270,61 +397,67 @@ jobs: exit 1 fi - - name: Upload wheel artifact + - name: Upload musllinux wheel artifact + if: ${{ contains(matrix.platform, 'musl') }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: wheel-musllinux-${{ matrix.architecture }} + name: wheel-${{ matrix.platform }} path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/musllinux-wheels/*.whl if-no-files-found: error - Package: - name: Package (${{ matrix.platform }}) - needs: [Test] - if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + PackageSmoke: + name: Package smoke (${{ matrix.platform }}) + needs: [Package] + if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' }} runs-on: ${{ matrix.runner }} timeout-minutes: 30 permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 + - *linux-amd64 + - *linux-arm64 + - *macos-arm64 + - *windows-amd64 + - *windows-arm64 + - *package-linux-musl-amd64 + - *package-linux-musl-arm64 + - platform: source-and-plugin runner: ubuntu-latest - - platform: linux-arm64 - runner: ubuntu-24.04-arm - - platform: macos-arm64 - runner: macos-15 - - platform: windows-amd64 - runner: windows-2022 - - platform: windows-arm64 - runner: windows-11-arm steps: - name: Checkout + if: ${{ !contains(matrix.platform, 'musl') }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Load CI tool versions + if: ${{ !contains(matrix.platform, 'musl') }} id: ci-config uses: ./.github/actions/load-ci-tool-versions - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + if: ${{ matrix.platform == 'source-and-plugin' }} with: cache: false toolchain: ${{ steps.ci-config.outputs.rust_version }} - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + if: ${{ matrix.platform == 'source-and-plugin' }} with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-linux-amd64-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false + save-if: false - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + if: ${{ !contains(matrix.platform, 'musl') }} with: version: ${{ steps.ci-config.outputs.uv_version }} - enable-cache: true - cache-dependency-glob: ${{ env.NEMO_RELAY_CI_WORKSPACE }}/uv.lock - name: Set up Windows ARM Python if: ${{ matrix.platform == 'windows-arm64' }} @@ -343,89 +476,92 @@ jobs: PY - name: Install managed Python - if: ${{ matrix.platform != 'windows-arm64' }} - run: | - set -e - UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + if: ${{ !contains(matrix.platform, 'musl') && matrix.platform != 'windows-arm64' }} + run: UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} - - uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8 # v2.77.6 + - name: Download matching Python wheel artifact + if: ${{ !contains(matrix.platform, 'musl') && matrix.platform != 'source-and-plugin' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - tool: just@${{ steps.ci-config.outputs.just_version }} - - - name: Create packaging output directory - run: | - set -e - mkdir -p "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/wheels" + name: wheel-${{ matrix.platform }} + path: python-wheel/ - - name: Derive Python package version - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + - name: Install and import Python wheel + if: ${{ !contains(matrix.platform, 'musl') && matrix.platform != 'source-and-plugin' }} run: | - set -e - version="$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n1)" - if [ -z "$version" ]; then - echo "Error: failed to read workspace version from Cargo.toml" >&2 - exit 1 - fi - sha="${GITHUB_SHA::8}" - if [ "${{ inputs.ref_type }}" = "tag" ]; then - version="${{ inputs.ref_name }}" + set -euo pipefail + wheel="$(find python-wheel -maxdepth 1 -name '*.whl' -print -quit)" + test -n "$wheel" + uv venv --python "$(uv python find)" "${RUNNER_TEMP}/nemo-relay-wheel-venv" + if [ "${{ runner.os }}" = "Windows" ]; then + venv_python="${RUNNER_TEMP}/nemo-relay-wheel-venv/Scripts/python.exe" else - version="${version}+${sha}" + venv_python="${RUNNER_TEMP}/nemo-relay-wheel-venv/bin/python" fi - printf 'NEMO_RELAY_PACKAGE_VERSION=%s\n' "$version" >> "$GITHUB_ENV" - - - name: Package Python wheel - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} - run: | - set -e - just \ - --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ - package-python + uv pip install --python "$venv_python" --no-deps "$wheel" + "$venv_python" -c "import nemo_relay" - - name: Upload wheel artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Download matching musllinux Python wheel artifact + if: ${{ contains(matrix.platform, 'musl') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: wheel-${{ matrix.platform }} - path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/wheels/*.whl - if-no-files-found: error + path: python-wheel/ - - name: Package Python source distribution - if: ${{ matrix.platform == 'linux-amd64' }} - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + - name: Install and import musllinux Python wheel + if: ${{ contains(matrix.platform, 'musl') }} + env: + MUSLLINUX_IMAGE: ${{ matrix.container_image }} run: | - set -e - just \ - --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ - package-python-sdist + set -euo pipefail + docker run --rm \ + --volume "${{ github.workspace }}:${{ github.workspace }}" \ + --workdir "${{ github.workspace }}" \ + "$MUSLLINUX_IMAGE" \ + /bin/sh -ec ' + wheel="$(find python-wheel -maxdepth 1 -name "*.whl" -print -quit)" + test -n "$wheel" + /opt/python/cp311-cp311/bin/python -m venv /tmp/nemo-relay-wheel-venv + /tmp/nemo-relay-wheel-venv/bin/python -m pip install --no-deps "$wheel" + /tmp/nemo-relay-wheel-venv/bin/python -c "import nemo_relay" + ' - - name: Upload Python source distribution artifact - if: ${{ matrix.platform == 'linux-amd64' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Download Python source distribution artifact + if: ${{ matrix.platform == 'source-and-plugin' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: python-sdist - path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/sdists/nemo_relay-*.tar.gz - if-no-files-found: error - - - name: Package Python plugin SDK wheel - if: ${{ matrix.platform == 'linux-amd64' }} - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} - run: | - set -e - just \ - --set output_dir "${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}" \ - --set ref_name "${NEMO_RELAY_PACKAGE_VERSION}" \ - package-python-plugin + path: python-sdist/ - - name: Upload Python plugin SDK wheel artifact - if: ${{ matrix.platform == 'linux-amd64' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Download Python plugin SDK wheel artifact + if: ${{ matrix.platform == 'source-and-plugin' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: plugin-wheel - path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/plugin-wheels/*.whl - if-no-files-found: error + path: plugin-wheel/ - - name: Prune uv cache - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} - run: uv cache prune --ci + - name: Build, install, and import Python source distribution + if: ${{ matrix.platform == 'source-and-plugin' }} + run: | + set -euo pipefail + sdist="$(find python-sdist -maxdepth 1 -name '*.tar.gz' -print -quit)" + test -n "$sdist" + uv venv --python "$(uv python find)" "${RUNNER_TEMP}/nemo-relay-sdist-venv" + uv pip install \ + --python "${RUNNER_TEMP}/nemo-relay-sdist-venv/bin/python" \ + --no-deps \ + "$sdist" + "${RUNNER_TEMP}/nemo-relay-sdist-venv/bin/python" -c "import nemo_relay" + + - name: Install and import Python plugin SDK wheel + if: ${{ matrix.platform == 'source-and-plugin' }} + run: | + set -euo pipefail + wheel="$(find plugin-wheel -maxdepth 1 -name '*.whl' -print -quit)" + test -n "$wheel" + uv venv --python "$(uv python find)" "${RUNNER_TEMP}/nemo-relay-plugin-venv" + uv pip install \ + --python "${RUNNER_TEMP}/nemo-relay-plugin-venv/bin/python" \ + "$wheel" + "${RUNNER_TEMP}/nemo-relay-plugin-venv/bin/python" -c \ + "import nemo_relay_plugin._proto.plugin_worker_pb2_grpc" diff --git a/.github/workflows/ci_rust.yml b/.github/workflows/ci_rust.yml index e6640a936..c39df95d5 100644 --- a/.github/workflows/ci_rust.yml +++ b/.github/workflows/ci_rust.yml @@ -7,7 +7,7 @@ on: workflow_call: inputs: run_package: - description: 'Whether to build package artifacts after tests pass' + description: 'Whether to build package artifacts' required: false default: true type: boolean @@ -31,7 +31,6 @@ jobs: name: Test (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 - continue-on-error: ${{ startsWith(matrix.platform, 'windows') }} permissions: contents: read services: @@ -56,7 +55,7 @@ jobs: --health-timeout 3s --health-retries 20 strategy: - fail-fast: false + fail-fast: true matrix: include: - platform: linux-amd64 @@ -105,7 +104,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false @@ -186,47 +185,53 @@ jobs: Package: name: Package (${{ matrix.platform }}) - needs: [Test] - if: ${{ inputs.run_package && !cancelled() && needs.Test.result == 'success' }} + if: ${{ inputs.run_package }} runs-on: ${{ matrix.runner }} timeout-minutes: 60 permissions: contents: read strategy: - fail-fast: false + fail-fast: true matrix: include: - - platform: linux-amd64 + - &package-linux-amd64 + platform: linux-amd64 runner: ubuntu-latest target: x86_64-unknown-linux-gnu build_image: quay.io/pypa/manylinux2014_x86_64@sha256:35baef377f64c2ae2e7ed647917ecd090c50f6e8b06fd605012661c2e954cc92 runtime_image: quay.io/pypa/manylinux2014_x86_64@sha256:35baef377f64c2ae2e7ed647917ecd090c50f6e8b06fd605012661c2e954cc92 - - platform: linux-arm64 + - &package-linux-arm64 + platform: linux-arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu build_image: quay.io/pypa/manylinux2014_aarch64@sha256:63f73ecd2be3ad7cf66a402c72c1594b3f9140e263ab6838896e93a8fbf53794 runtime_image: quay.io/pypa/manylinux2014_aarch64@sha256:63f73ecd2be3ad7cf66a402c72c1594b3f9140e263ab6838896e93a8fbf53794 - - platform: linux-musl-amd64 + - &package-linux-musl-amd64 + platform: linux-musl-amd64 runner: ubuntu-latest target: x86_64-unknown-linux-musl build_image: '' runtime_image: quay.io/pypa/musllinux_1_2_x86_64@sha256:7b54360b191356838f6e41d4b9820c69be4f3d8dc8e7bca26f2684d48e5c8633 - - platform: linux-musl-arm64 + - &package-linux-musl-arm64 + platform: linux-musl-arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl build_image: '' runtime_image: quay.io/pypa/musllinux_1_2_aarch64@sha256:274a947b4d5d745b56b868b0d052641e0798a652e3e65755987edf60547d1268 - - platform: macos-arm64 + - &package-macos-arm64 + platform: macos-arm64 runner: macos-15 target: aarch64-apple-darwin build_image: '' runtime_image: '' - - platform: windows-amd64 + - &package-windows-amd64 + platform: windows-amd64 runner: windows-2022 target: x86_64-pc-windows-msvc build_image: '' runtime_image: '' - - platform: windows-arm64 + - &package-windows-arm64 + platform: windows-arm64 runner: windows-11-arm target: aarch64-pc-windows-msvc build_image: '' @@ -248,11 +253,10 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: nemo-relay-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.ci-config.outputs.rust_version }} + shared-key: nemo-relay-rust-${{ matrix.platform }}-${{ steps.ci-config.outputs.rust_version }} workspaces: . -> target cache-all-crates: true cache-bin: false - save-if: false - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 with: @@ -314,26 +318,6 @@ jobs: cargo build --release --target "${{ matrix.target }}" -p nemo-relay-cli ' - - name: Verify CLI binary in its Linux runtime container - if: ${{ matrix.runtime_image != '' }} - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} - run: | - set -euo pipefail - docker run --rm \ - --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ - --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ - --env NEMO_RELAY_CI_WORKSPACE \ - --env NEMO_RELAY_CLI_PACKAGE_VERSION \ - "${{ matrix.runtime_image }}" \ - /bin/sh -ec ' - actual="$("$NEMO_RELAY_CI_WORKSPACE/target/${{ matrix.target }}/release/nemo-relay" --version)" - expected="nemo-relay $NEMO_RELAY_CLI_PACKAGE_VERSION" - if [ "$actual" != "$expected" ]; then - echo "Error: expected CLI version \"$expected\", got \"$actual\"" >&2 - exit 1 - fi - ' - - name: Stage CLI binary artifact working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: | @@ -377,62 +361,134 @@ jobs: --version "$version" \ --output-dir "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-packages" - - name: Install and run CLI wheel - if: ${{ !endsWith(matrix.target, '-musl') }} - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + - name: Upload CLI binary artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-binary-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli/* + if-no-files-found: error + + - name: Upload CLI wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-python-wheel-${{ matrix.platform }} + path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli-packages/*.whl + if-no-files-found: error + + PackageSmoke: + name: Package smoke (${{ matrix.platform }}) + needs: [Package] + if: ${{ inputs.run_package && !cancelled() && needs.Package.result == 'success' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: true + matrix: + include: + - *package-linux-amd64 + - *package-linux-arm64 + - *package-linux-musl-amd64 + - *package-linux-musl-arm64 + - *package-macos-arm64 + - *package-windows-amd64 + - *package-windows-arm64 + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Load CI tool versions + id: ci-config + uses: ./.github/actions/load-ci-tool-versions + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + if: ${{ matrix.runtime_image == '' }} + with: + version: ${{ steps.ci-config.outputs.uv_version }} + + - name: Set up Windows ARM Python + if: ${{ matrix.platform == 'windows-arm64' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ steps.ci-config.outputs.default_python_version }} + architecture: arm64 + + - name: Select setup-python interpreter for uv + if: ${{ matrix.platform == 'windows-arm64' }} + run: | + set -e + python - <<'PY' >> "$GITHUB_ENV" + import sys + print(f"UV_PYTHON={sys.executable}") + PY + + - name: Install managed Python + if: ${{ matrix.runtime_image == '' && matrix.platform != 'windows-arm64' }} + run: UV_PYTHON_DOWNLOADS=manual uv python install --managed-python ${{ steps.ci-config.outputs.default_python_version }} + + - name: Download matching CLI binary artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-binary-${{ matrix.platform }} + path: cli-binary/ + + - name: Download matching CLI wheel artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-python-wheel-${{ matrix.platform }} + path: cli-wheel/ + + - name: Run downloaded CLI binary and wheel + if: ${{ matrix.runtime_image == '' }} run: | set -euo pipefail - wheels=("${NEMO_RELAY_CI_WORKSPACE_TMP}"/cli-packages/*.whl) - python -m venv "${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv" + binary="$(find cli-binary -maxdepth 1 -type f -print -quit)" + wheel="$(find cli-wheel -maxdepth 1 -name '*.whl' -print -quit)" + test -n "$binary" + test -n "$wheel" + chmod +x "$binary" + binary_version="$("$binary" --version)" + uv venv --python "$(uv python find)" "${RUNNER_TEMP}/nemo-relay-cli-venv" if [ "${{ runner.os }}" = "Windows" ]; then - venv_python="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/Scripts/python.exe" - venv_cli="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/Scripts/nemo-relay.exe" + venv_python="${RUNNER_TEMP}/nemo-relay-cli-venv/Scripts/python.exe" + venv_cli="${RUNNER_TEMP}/nemo-relay-cli-venv/Scripts/nemo-relay.exe" else - venv_python="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/bin/python" - venv_cli="${NEMO_RELAY_CI_WORKSPACE_TMP}/cli-wheel-venv/bin/nemo-relay" + venv_python="${RUNNER_TEMP}/nemo-relay-cli-venv/bin/python" + venv_cli="${RUNNER_TEMP}/nemo-relay-cli-venv/bin/nemo-relay" fi - "$venv_python" -m pip install --force-reinstall --no-deps "${wheels[0]}" - actual="$("$venv_cli" --version)" - expected="nemo-relay $NEMO_RELAY_CLI_PACKAGE_VERSION" - if [[ "$actual" != "$expected" ]]; then - echo "Error: expected CLI version '$expected', got '$actual'" >&2 + uv pip install --python "$venv_python" --no-deps "$wheel" + wheel_version="$("$venv_cli" --version)" + if [ "$binary_version" != "$wheel_version" ]; then + echo "Error: CLI binary reports '$binary_version' but wheel reports '$wheel_version'" >&2 exit 1 fi - - name: Install and run CLI wheel on musllinux - if: ${{ endsWith(matrix.target, '-musl') }} - working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + - name: Run downloaded CLI binary and wheel in Linux runtime container + if: ${{ matrix.runtime_image != '' }} + env: + RUNTIME_IMAGE: ${{ matrix.runtime_image }} run: | set -euo pipefail docker run --rm \ - --volume "${{ env.NEMO_RELAY_CI_WORKSPACE }}:${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ - --workdir "${{ env.NEMO_RELAY_CI_WORKSPACE }}" \ - --env NEMO_RELAY_CI_WORKSPACE_TMP \ - --env NEMO_RELAY_CLI_PACKAGE_VERSION \ - "${{ matrix.runtime_image }}" \ + --volume "${{ github.workspace }}:${{ github.workspace }}" \ + --workdir "${{ github.workspace }}" \ + "$RUNTIME_IMAGE" \ /bin/sh -ec ' - wheel="$(find "$NEMO_RELAY_CI_WORKSPACE_TMP/cli-packages" -maxdepth 1 -name "*.whl" -print -quit)" + binary="$(find cli-binary -maxdepth 1 -type f -print -quit)" + wheel="$(find cli-wheel -maxdepth 1 -name "*.whl" -print -quit)" + test -n "$binary" test -n "$wheel" - /opt/python/cp311-cp311/bin/python -m venv /tmp/nemo-relay-cli-wheel-venv - /tmp/nemo-relay-cli-wheel-venv/bin/python -m pip install --force-reinstall --no-deps "$wheel" - actual="$(/tmp/nemo-relay-cli-wheel-venv/bin/nemo-relay --version)" - expected="nemo-relay $NEMO_RELAY_CLI_PACKAGE_VERSION" - if [ "$actual" != "$expected" ]; then - echo "Error: expected CLI version \"$expected\", got \"$actual\"" >&2 + chmod +x "$binary" + binary_version="$("$binary" --version)" + /opt/python/cp311-cp311/bin/python -m venv /tmp/nemo-relay-cli-venv + /tmp/nemo-relay-cli-venv/bin/python -m pip install --no-deps "$wheel" + wheel_version="$(/tmp/nemo-relay-cli-venv/bin/nemo-relay --version)" + if [ "$binary_version" != "$wheel_version" ]; then + echo "Error: CLI binary and wheel versions differ" >&2 exit 1 fi ' - - - name: Upload CLI binary artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cli-binary-${{ matrix.platform }} - path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli/* - if-no-files-found: error - - - name: Upload CLI wheel artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cli-python-wheel-${{ matrix.platform }} - path: ${{ env.NEMO_RELAY_CI_WORKSPACE_TMP }}/cli-packages/*.whl - if-no-files-found: error diff --git a/justfile b/justfile index a84c6c23c..18d33a74e 100644 --- a/justfile +++ b/justfile @@ -59,7 +59,11 @@ PY uv_python_executable() { ( cd "$NEMO_RELAY_REPO_ROOT" - uv python find + if [[ -n "${UV_PYTHON:-}" ]]; then + uv python find "$UV_PYTHON" + else + uv python find + fi ) } @@ -922,7 +926,7 @@ rust_source_coverage_supported() { local host host="$(rustc -vV | sed -n 's/^host: //p')" case "$host" in - aarch64-pc-windows-msvc) + aarch64-pc-windows-msvc|*-unknown-linux-musl) return 1 ;; *) @@ -1216,7 +1220,7 @@ test-rust: prepare_llvm_cov_workspace fi prepare_test_plugin_fixtures - cargo nextest run --workspace --profile ci --no-fail-fast + cargo nextest run --locked --workspace --profile ci --no-fail-fast cp "$NEMO_RELAY_REPO_ROOT/target/nextest/ci/rust_junit_report.xml" "$junit_out" if rust_source_coverage_supported; then cargo llvm-cov report \ diff --git a/scripts/package_node_musllinux.mjs b/scripts/package_node_musllinux.mjs index 1b2c3146b..4b16464e1 100755 --- a/scripts/package_node_musllinux.mjs +++ b/scripts/package_node_musllinux.mjs @@ -2,17 +2,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; -const ignoredDirectories = new Set([".git", ".venv", "node_modules", "target", "tmp"]); +const ignoredDirectories = new Set(['.git', '.venv', 'node_modules', 'target', 'tmp']); function command(name, args, cwd) { - const result = spawnSync(name, args, { cwd, stdio: "inherit" }); + const result = spawnSync(name, args, { cwd, stdio: 'inherit' }); if (result.status !== 0) { - throw new Error(`${name} ${args.join(" ")} failed with exit code ${result.status}`); + throw new Error(`${name} ${args.join(' ')} failed with exit code ${result.status}`); } } @@ -22,18 +22,18 @@ function argumentsFrom(args) { let platform; for (let index = 0; index < args.length; index += 2) { const value = args[index + 1]; - if (args[index] === "--version") { + if (args[index] === '--version') { version = value; - } else if (args[index] === "--out") { + } else if (args[index] === '--out') { output = value; - } else if (args[index] === "--platform") { + } else if (args[index] === '--platform') { platform = value; } else { throw new Error(`Unexpected argument: ${args[index]}`); } } if (!version || !output || !platform) { - throw new Error("Usage: package_node_musllinux.mjs --version VERSION --platform PLATFORM --out DIRECTORY"); + throw new Error('Usage: package_node_musllinux.mjs --version VERSION --platform PLATFORM --out DIRECTORY'); } return { output: resolve(output), platform, version }; } @@ -41,8 +41,8 @@ function argumentsFrom(args) { function main() { const { output, platform, version } = argumentsFrom(process.argv.slice(2)); const repository = process.cwd(); - const temporaryDirectory = mkdtempSync(join(tmpdir(), "nemo-relay-node-musllinux-")); - const sourceDirectory = join(temporaryDirectory, "source"); + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'nemo-relay-node-musllinux-')); + const sourceDirectory = join(temporaryDirectory, 'source'); try { mkdirSync(output, { recursive: true }); @@ -50,25 +50,25 @@ function main() { filter: (source) => !ignoredDirectories.has(basename(source)), recursive: true, }); - command("npm", ["install", "--workspace=nemo-relay-node", "--ignore-scripts"], sourceDirectory); - command("npm", ["run", "--workspace=nemo-relay-node", "build"], sourceDirectory); + command('npm', ['ci', '--workspace=nemo-relay-node', '--ignore-scripts'], sourceDirectory); + command('npm', ['run', '--workspace=nemo-relay-node', 'build'], sourceDirectory); command( - "python3", + 'python3', [ - "scripts/package-node-bin.py", - "--node-dir", - "crates/node", - "--platform", + 'scripts/package-node-bin.py', + '--node-dir', + 'crates/node', + '--platform', platform, - "--version", + '--version', version, - "--output-dir", + '--output-dir', output, ], sourceDirectory, ); - const packages = readdirSync(output).filter((entry) => entry.endsWith(".tgz")); + const packages = readdirSync(output).filter((entry) => entry.endsWith('.tgz')); if (packages.length !== 1) { throw new Error(`Expected one npm package artifact in ${output}, found ${packages.length}`); } diff --git a/scripts/package_python_musllinux.py b/scripts/package_python_musllinux.py index fdee52747..ce05bd344 100755 --- a/scripts/package_python_musllinux.py +++ b/scripts/package_python_musllinux.py @@ -82,6 +82,7 @@ def main() -> None: [ "maturin", "build", + "--locked", "--release", "--compatibility", "musllinux_1_2",