fix(engine): centralize telemetry timer management in runtime manager #5038
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This action runs the pipeline perf benchmarking suite on dedicated Oracle | |
| # bare-metal hardware when the 'pipelineperf' label is added to a PR. | |
| # Basic perf validation on ubuntu-latest is handled by Rust-CI (rust-ci.yml). | |
| # The results from this workflow do not update the charts. | |
| name: Pipeline Perf Dedicated | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| # Cancel in-progress runs for the same PR if a new commit is pushed | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Run on dedicated Oracle hardware when 'pipelineperf' label is present | |
| pipeline-perf-test-dedicated: | |
| if: contains(github.event.pull_request.labels.*.name, 'pipelineperf') | |
| runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24 | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0 | |
| with: | |
| egress-policy: audit | |
| - name: Disk usage diagnostics | |
| run: | | |
| echo "=== Overall disk usage ===" | |
| df -h | |
| echo "" | |
| echo "=== Largest directories under / (top 20) ===" | |
| sudo du -h --max-depth=3 / 2>/dev/null | sort -rh | head -20 | |
| echo "" | |
| echo "=== Docker disk usage ===" | |
| docker system df -v 2>/dev/null || echo "Docker not running or not available" | |
| echo "" | |
| echo "=== Docker images ===" | |
| docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null || true | |
| echo "" | |
| echo "=== Docker volumes ===" | |
| docker volume ls 2>/dev/null || true | |
| echo "" | |
| echo "=== Docker containers (all) ===" | |
| docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Size}}\t{{.Status}}' 2>/dev/null || true | |
| echo "" | |
| echo "=== /home usage ===" | |
| sudo du -h --max-depth=2 /home 2>/dev/null | sort -rh | head -20 | |
| echo "" | |
| echo "=== /var usage ===" | |
| sudo du -h --max-depth=2 /var 2>/dev/null | sort -rh | head -20 | |
| echo "" | |
| echo "=== /tmp usage ===" | |
| sudo du -h --max-depth=1 /tmp 2>/dev/null | sort -rh | head -10 | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.14" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Set up Python (non-OL8) | |
| if: ${{ steps.detect_os.outputs.id != 'ol' }} | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.14" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Build dataflow_engine | |
| run: | | |
| git submodule init | |
| git submodule update | |
| cd rust/otap-dataflow | |
| docker buildx build --load --build-context otel-arrow=../../ -f Dockerfile -t df_engine . | |
| cd ../.. | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --user --require-hashes -r tools/pipeline_perf_test/orchestrator/requirements.lock.txt | |
| python -m pip install --user --require-hashes -r tools/pipeline_perf_test/load_generator/requirements.lock.txt | |
| - name: Run pipeline performance test suite | |
| run: | | |
| cd tools/pipeline_perf_test | |
| python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/100klrps-docker.yaml | |
| - name: Post-run disk usage diagnostics | |
| if: always() | |
| run: | | |
| echo "=== Overall disk usage ===" | |
| df -h | |
| echo "" | |
| echo "=== Docker disk usage ===" | |
| docker system df -v 2>/dev/null || true |