fix(engine): centralize telemetry timer management in runtime manager #11521
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust-CI | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| merge_group: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Cancel in-progress runs on new commits to same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Non-required test matrix. | |
| # - experimental folders run unpartitioned on all OSes (they are small). | |
| # - otap-dataflow runs on ARM and macOS, split into 3 partitions. | |
| experimental_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: | |
| - experimental/query_abstraction | |
| - experimental/query_engine | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| - windows-latest | |
| - macos-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: Free disk space (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: Free disk space (Windows) | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "C:\Android") { Remove-Item -Recurse -Force "C:\Android" } | |
| if (Test-Path "C:\SeleniumWebDrivers") { Remove-Item -Recurse -Force "C:\SeleniumWebDrivers" } | |
| if (Test-Path "C:\imagemagick") { Remove-Item -Recurse -Force "C:\imagemagick" } | |
| - name: install cargo-llvm-cov and nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-llvm-cov,cargo-nextest | |
| - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| if: runner.os == 'Linux' | |
| with: | |
| go-version: "1.26.2" | |
| - name: Build the test collector | |
| if: runner.os == 'Linux' | |
| run: make otelarrowcol | |
| - name: Run tests with coverage (Linux x86_64) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: cargo llvm-cov nextest --all-features --workspace --lcov --output-path lcov.info | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Run tests (Linux ARM64) | |
| if: matrix.os == 'ubuntu-24.04-arm' | |
| run: cargo nextest run --all-features --workspace | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Run tests (Windows) | |
| if: runner.os == 'Windows' | |
| run: cargo nextest run --all-features --workspace | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Run tests (macOS) | |
| if: runner.os == 'macOS' | |
| run: cargo nextest run --all-features --workspace | |
| working-directory: ./rust/${{ matrix.folder }} | |
| # NOTE: when adding or removing a codecov upload step anywhere in | |
| # the repo, update `codecov.notify.after_n_builds` in | |
| # .github/codecov.yaml so codecov waits for the new total. | |
| - name: Upload to codecov.io | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| fail_ci_if_error: true | |
| # otap-dataflow on ARM and macOS (non-required): build once per OS, | |
| # fan out tests. macOS uses the workspace default features. Linux | |
| # ARM uses a reduced feature set Full feature coverage and | |
| # validation tests are handled by the required x86 jobs. | |
| build_nonrequired: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-24.04-arm, macos-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/${{ matrix.folder }} | |
| - name: Free disk space (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: install nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-nextest | |
| - name: Build and archive tests (Complete) | |
| if: matrix.os != 'ubuntu-24.04-arm' | |
| run: cargo nextest archive --workspace --archive-file nextest-archive.tar.zst | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Build and archive tests (Incomplete) | |
| if: matrix.os == 'ubuntu-24.04-arm' | |
| run: | | |
| cargo nextest archive \ | |
| --no-default-features \ | |
| --features=crypto-ring,jemalloc,azure,aws,azure-monitor-exporter,contrib-processors,unsafe-optimizations \ | |
| --workspace --exclude otap-df-validation --exclude benchmarks \ | |
| --archive-file nextest-archive.tar.zst | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Upload nextest archive | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: nextest-archive-${{ matrix.os }} | |
| path: ./rust/${{ matrix.folder }}/nextest-archive.tar.zst | |
| retention-days: 1 | |
| test_nonrequired: | |
| needs: build_nonrequired | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-24.04-arm, macos-latest] | |
| partition: [1, 2, 3] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: install nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-nextest | |
| - name: Free disk space (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: Download nextest archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: nextest-archive-${{ matrix.os }} | |
| path: ./rust/${{ matrix.folder }} | |
| - name: Run tests (partition ${{ matrix.partition }}/3) | |
| run: cargo nextest run --profile ci --config-file .config/nextest.toml --archive-file nextest-archive.tar.zst --partition count:${{ matrix.partition }}/3 | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Write JUnit artifact metadata | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo '{"job": "${{ github.job }}", "os": "${{ matrix.os }}", "partition": "${{ matrix.partition }}", "folder": "${{ matrix.folder }}"}' \ | |
| > ./rust/${{ matrix.folder }}/target/nextest/ci/metadata.json | |
| - name: Upload JUnit XML results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: junit-xml-nonrequired-${{ matrix.os }}-${{ matrix.partition }} | |
| path: | | |
| ./rust/${{ matrix.folder }}/target/nextest/ci/junit.xml | |
| ./rust/${{ matrix.folder }}/target/nextest/ci/metadata.json | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| fmt: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: | |
| - otap-dataflow | |
| - experimental/query_abstraction | |
| - experimental/query_engine | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| - windows-latest | |
| - macos-latest | |
| exclude: | |
| - folder: otap-dataflow | |
| os: ubuntu-latest | |
| - folder: otap-dataflow | |
| os: windows-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| components: rustfmt | |
| - name: cargo fmt ${{ matrix.folder }} | |
| run: cargo fmt --all -- --check | |
| working-directory: ./rust/${{ matrix.folder }} | |
| # Checks that Pest grammar files are properly formatted using pestfmt. | |
| # Only runs if opl.pest has been modified in PRs, or on pushes to main. | |
| # To format locally: cargo install pest_fmt && pestfmt rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest | |
| pest-fmt: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: Check if opl.pest was modified | |
| id: check_pest | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| # For PRs, check if the file was changed | |
| git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest" && echo "changed=true" >> $GITHUB_OUTPUT || echo "changed=false" >> $GITHUB_OUTPUT | |
| else | |
| # For pushes to main, always check | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install pestfmt | |
| if: steps.check_pest.outputs.changed == 'true' | |
| run: cargo install pest_fmt | |
| - name: Check opl.pest formatting | |
| if: steps.check_pest.outputs.changed == 'true' | |
| run: | | |
| # Create a backup of the original file before formatting. | |
| # pestfmt modifies files in-place, so we need to preserve the original | |
| # to compare and detect if any formatting changes are needed. | |
| cp rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest /tmp/opl.pest.original | |
| # Run pestfmt to format the file | |
| pestfmt rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest | |
| # Compare with original | |
| if ! diff -q /tmp/opl.pest.original rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest > /dev/null; then | |
| echo "opl.pest is not formatted correctly" | |
| echo "" | |
| echo "Please run the following command to format it:" | |
| echo " pestfmt rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest" | |
| echo "" | |
| echo "Diff:" | |
| diff -u /tmp/opl.pest.original rust/otap-dataflow/crates/query-engine-languages/src/opl/opl.pest || true | |
| exit 1 | |
| else | |
| echo "opl.pest is correctly formatted" | |
| fi | |
| clippy: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: | |
| - otap-dataflow | |
| - experimental/query_abstraction | |
| - experimental/query_engine | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| - windows-latest | |
| - macos-latest | |
| exclude: | |
| - folder: otap-dataflow | |
| os: ubuntu-latest | |
| - folder: otap-dataflow | |
| os: windows-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/${{ matrix.folder }} | |
| - name: cargo clippy ${{ matrix.folder }} | |
| run: | | |
| cargo clippy --all-targets --all-features --workspace -- -D warnings | |
| working-directory: ./rust/${{ matrix.folder }} | |
| deny: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: | |
| - otap-dataflow | |
| - experimental/query_abstraction | |
| - experimental/query_engine | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: advisories | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check advisories | |
| manifest-path: ./rust/${{ matrix.folder }}/Cargo.toml | |
| - name: licenses | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check licenses | |
| manifest-path: ./rust/${{ matrix.folder }}/Cargo.toml | |
| - name: bans | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check bans | |
| manifest-path: ./rust/${{ matrix.folder }}/Cargo.toml | |
| - name: sources | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check sources | |
| manifest-path: ./rust/${{ matrix.folder }}/Cargo.toml | |
| docs: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: | |
| - otap-dataflow | |
| - experimental/query_abstraction | |
| - experimental/query_engine | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: cargo doc ${{ matrix.folder }} | |
| run: cargo doc --no-deps | |
| working-directory: ./rust/${{ matrix.folder }} | |
| structure_check: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Only otap-dataflow has a structure check | |
| folder: [otap-dataflow] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: cargo xtask ${{ matrix.folder }} | |
| run: cargo xtask structure-check | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: check telemetry macro usage | |
| run: ./scripts/check-direct-telemetry-macros.sh | |
| working-directory: ./rust/${{ matrix.folder }} | |
| compile_proto: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| version: "34.1" | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: cargo xtask compile proto | |
| run: cargo xtask compile-proto | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: check git diff after compile-proto execution | |
| run: git diff --exit-code rust/otap-dataflow/crates/pdata/src/proto | |
| # Verify the workspace builds with --no-default-features to catch broken | |
| # optional feature gates (e.g., dev-tools, crypto-*). | |
| no_default_features_check: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: cargo check --no-default-features --workspace | |
| run: cargo check --no-default-features --workspace | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: cargo check validation --no-default-features | |
| run: cargo check -p otap-df-validation --lib --no-default-features | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: cargo check benchmarks --no-default-features | |
| run: cargo check -p benchmarks --benches --no-default-features | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: cargo test otap integration targets --no-run | |
| run: cargo test -p otap-df-otap --tests --no-run | |
| working-directory: ./rust/${{ matrix.folder }} | |
| # Required builds for otap-dataflow: compile once per OS, create nextest archive. | |
| # Test binaries are uploaded as artifacts so test partitions don't re-compile. | |
| build_required: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: Install SymCrypt (Windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/install-symcrypt | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/${{ matrix.folder }} | |
| - name: Free disk space (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: Free disk space (Windows) | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "C:\Android") { Remove-Item -Recurse -Force "C:\Android" } | |
| if (Test-Path "C:\SeleniumWebDrivers") { Remove-Item -Recurse -Force "C:\SeleniumWebDrivers" } | |
| if (Test-Path "C:\imagemagick") { Remove-Item -Recurse -Force "C:\imagemagick" } | |
| - name: install nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-nextest | |
| - name: Build and archive tests (Incomplete) | |
| if: runner.os == 'Windows' | |
| run: cargo nextest archive --no-default-features --features=crypto-symcrypt,mimalloc,azure,aws,azure-monitor-exporter,contrib-processors,unsafe-optimizations --workspace --archive-file nextest-archive.tar.zst | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Build and archive tests (Complete) | |
| if: runner.os != 'Windows' | |
| run: cargo nextest archive --all-features --workspace --archive-file nextest-archive.tar.zst | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Upload nextest archive | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: nextest-archive-${{ matrix.os }} | |
| path: ./rust/${{ matrix.folder }}/nextest-archive.tar.zst | |
| retention-days: 1 | |
| # Required test partitions: download pre-built archive and run 1/3 of tests each. | |
| test_required: | |
| needs: build_required | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-latest, windows-latest] | |
| partition: [1, 2, 3] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: Install SymCrypt (Windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/install-symcrypt | |
| - name: install nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-nextest | |
| - name: Free disk space (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| if: runner.os == 'Linux' | |
| with: | |
| go-version: "1.26.2" | |
| - name: Build the test collector | |
| if: runner.os == 'Linux' | |
| run: make otelarrowcol | |
| - name: Download nextest archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: nextest-archive-${{ matrix.os }} | |
| path: ./rust/${{ matrix.folder }} | |
| - name: Run tests (partition ${{ matrix.partition }}/3) | |
| run: cargo nextest run --profile ci --config-file .config/nextest.toml --archive-file nextest-archive.tar.zst --partition count:${{ matrix.partition }}/3 | |
| working-directory: ./rust/${{ matrix.folder }} | |
| - name: Write JUnit artifact metadata | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo '{"job": "${{ github.job }}", "os": "${{ matrix.os }}", "partition": "${{ matrix.partition }}", "folder": "${{ matrix.folder }}"}' \ | |
| > ./rust/${{ matrix.folder }}/target/nextest/ci/metadata.json | |
| - name: Upload JUnit XML results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: junit-xml-required-${{ matrix.os }}-${{ matrix.partition }} | |
| path: | | |
| ./rust/${{ matrix.folder }}/target/nextest/ci/junit.xml | |
| ./rust/${{ matrix.folder }}/target/nextest/ci/metadata.json | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # Coverage runs unpartitioned on Linux x86_64 (does not block merge). | |
| # This compiles from scratch with coverage instrumentation and runs all tests. | |
| coverage: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/otap-dataflow | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: install cargo-llvm-cov and nextest | |
| uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2.75.28 | |
| with: | |
| tool: cargo-llvm-cov,cargo-nextest | |
| - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| with: | |
| go-version: "1.26.2" | |
| - name: Build the test collector | |
| run: make otelarrowcol | |
| - name: Run tests with coverage | |
| run: cargo llvm-cov nextest --profile ci --all-features --workspace --lcov --output-path lcov.info | |
| working-directory: ./rust/otap-dataflow | |
| # NOTE: when adding or removing a codecov upload step anywhere in | |
| # the repo, update `codecov.notify.after_n_builds` in | |
| # .github/codecov.yaml so codecov waits for the new total. | |
| - name: Upload to codecov.io | |
| uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| fail_ci_if_error: true | |
| # Required matrix combinations for fmt: otap-dataflow on ubuntu and windows | |
| fmt_required: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| components: rustfmt | |
| - name: cargo fmt ${{ matrix.folder }} | |
| run: cargo fmt --all -- --check | |
| working-directory: ./rust/${{ matrix.folder }} | |
| # Required matrix combinations for clippy: otap-dataflow on ubuntu and windows | |
| clippy_required: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| folder: [otap-dataflow] | |
| os: [ubuntu-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/${{ matrix.folder }} | |
| - name: Install OpenSSL (Windows) | |
| if: runner.os == 'Windows' | |
| # Required for the crypto-openssl feature (rustls-openssl -> openssl-sys). | |
| run: | | |
| vcpkg install openssl:x64-windows-static-md | |
| echo "OPENSSL_DIR=$env:VCPKG_INSTALLATION_ROOT\installed\x64-windows-static-md" >> $env:GITHUB_ENV | |
| shell: pwsh | |
| - name: Install SymCrypt (Windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/install-symcrypt | |
| - name: cargo clippy ${{ matrix.folder }} | |
| run: | | |
| cargo clippy --all-targets --all-features --workspace -- -D warnings | |
| working-directory: ./rust/${{ matrix.folder }} | |
| host-metrics-semconv: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: open-telemetry/semantic-conventions | |
| ref: v1.41.0 | |
| path: semantic-conventions | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| workspaces: ./rust/otap-dataflow | |
| - name: Run host metrics semconv drift check | |
| env: | |
| OTAP_HOST_METRICS_SEMCONV_REGISTRY: ${{ github.workspace }}/semantic-conventions/model | |
| run: | | |
| cargo test -p otap-df-core-nodes \ | |
| --features dev-tools,otap-df-otap/crypto-ring \ | |
| emitted_phase1_metric_shapes_match_weaver_semconv --lib -- --ignored | |
| working-directory: ./rust/otap-dataflow | |
| # Required matrix combinations for deny: otap-dataflow only | |
| deny_required: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: advisories | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check advisories | |
| manifest-path: ./rust/otap-dataflow/Cargo.toml | |
| - name: licenses | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check licenses | |
| manifest-path: ./rust/otap-dataflow/Cargo.toml | |
| - name: bans | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check bans | |
| manifest-path: ./rust/otap-dataflow/Cargo.toml | |
| - name: sources | |
| if: '!cancelled()' | |
| uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2.0.17 | |
| with: | |
| command: check sources | |
| manifest-path: ./rust/otap-dataflow/Cargo.toml | |
| # Required matrix combinations for docs: otap-dataflow only | |
| docs_required: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: cargo doc otap-dataflow | |
| run: cargo doc --no-deps | |
| working-directory: ./rust/otap-dataflow | |
| validate-configs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - name: Validate all otel_dataflow config files | |
| run: ./scripts/validate-configs.sh | |
| working-directory: ./rust/otap-dataflow | |
| document-build-options: | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Document Build Options | |
| run: | | |
| echo "## :bulb: Build Options" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Add the label \`cargobench\` to your PR to run micro benchmarks." >> $GITHUB_STEP_SUMMARY | |
| echo "- More info: [link to docs](https://github.com/open-telemetry/otel-arrow/blob/main/CONTRIBUTING.md#micro-benchmarks)" >> $GITHUB_STEP_SUMMARY | |
| # Publish test results from JUnit XML artifacts. | |
| # Runs after all test jobs and posts a summary to the workflow run. | |
| test-report: | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - test_required | |
| - test_nonrequired | |
| permissions: | |
| checks: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download all JUnit XML artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: junit-xml-* | |
| path: junit-results | |
| - name: Publish test results | |
| uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 | |
| if: always() | |
| with: | |
| name: Rust Test Results | |
| path: junit-results/**/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| # Pipeline performance test - validates that Rust changes don't regress performance. | |
| pipeline_perf_test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.14" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/lib/jvm /usr/share/dotnet /usr/share/swift /usr/local/.ghcup | |
| sudo rm -rf /usr/local/julia* /usr/local/lib/android /usr/local/share/chromium | |
| sudo rm -rf /opt/microsoft /opt/google /opt/az /usr/local/share/powershell | |
| - name: Build dataflow_engine | |
| run: | | |
| cd rust/otap-dataflow | |
| docker buildx build --load --build-context otel-arrow=../../ -f Dockerfile -t df_engine . | |
| cd ../.. | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --user --require-hashes -r tools/pipeline_perf_test/orchestrator/requirements.lock.txt | |
| python -m pip install --user --require-hashes -r tools/pipeline_perf_test/load_generator/requirements.lock.txt | |
| - name: Run pipeline performance test suite | |
| run: | | |
| cd tools/pipeline_perf_test | |
| python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/100klrps-docker.yaml | |
| # Aggregated status check - depends only on the required matrix combinations. | |
| # Add/remove jobs from the needs list to change what is required via PR, | |
| # rather than updating GitHub branch protection settings directly. | |
| rust-required-status-check: | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - test_required | |
| - fmt_required | |
| - clippy_required | |
| - deny_required | |
| - docs_required | |
| - structure_check | |
| - compile_proto | |
| - pest-fmt | |
| - no_default_features_check | |
| - pipeline_perf_test | |
| - host-metrics-semconv | |
| steps: | |
| - name: Check if all required jobs succeeded | |
| run: | | |
| if [[ "${{ needs.test_required.result }}" != "success" ]]; then | |
| echo "test_required failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.fmt_required.result }}" != "success" ]]; then | |
| echo "fmt_required failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.clippy_required.result }}" != "success" ]]; then | |
| echo "clippy_required failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.deny_required.result }}" != "success" ]]; then | |
| echo "deny_required failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.docs_required.result }}" != "success" ]]; then | |
| echo "docs_required failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.structure_check.result }}" != "success" ]]; then | |
| echo "structure_check failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.pest-fmt.result }}" != "success" ]]; then | |
| echo "pest-fmt failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.no_default_features_check.result }}" != "success" ]]; then | |
| echo "no_default_features_check failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.pipeline_perf_test.result }}" != "success" ]]; then | |
| echo "pipeline_perf_test failed or was cancelled" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.host-metrics-semconv.result }}" != "success" ]]; then | |
| echo "host-metrics-semconv failed or was cancelled" | |
| exit 1 | |
| fi | |
| echo "All required checks passed!" |