Skip to content

fix(engine): centralize telemetry timer management in runtime manager #5038

fix(engine): centralize telemetry timer management in runtime manager

fix(engine): centralize telemetry timer management in runtime manager #5038

# This action runs the pipeline perf benchmarking suite on dedicated Oracle
# bare-metal hardware when the 'pipelineperf' label is added to a PR.
# Basic perf validation on ubuntu-latest is handled by Rust-CI (rust-ci.yml).
# The results from this workflow do not update the charts.
name: Pipeline Perf Dedicated
on:
pull_request:
branches:
- main
permissions:
contents: read
# Cancel in-progress runs for the same PR if a new commit is pushed
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
# Run on dedicated Oracle hardware when 'pipelineperf' label is present
pipeline-perf-test-dedicated:
if: contains(github.event.pull_request.labels.*.name, 'pipelineperf')
runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
with:
egress-policy: audit
- name: Disk usage diagnostics
run: |
echo "=== Overall disk usage ==="
df -h
echo ""
echo "=== Largest directories under / (top 20) ==="
sudo du -h --max-depth=3 / 2>/dev/null | sort -rh | head -20
echo ""
echo "=== Docker disk usage ==="
docker system df -v 2>/dev/null || echo "Docker not running or not available"
echo ""
echo "=== Docker images ==="
docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null || true
echo ""
echo "=== Docker volumes ==="
docker volume ls 2>/dev/null || true
echo ""
echo "=== Docker containers (all) ==="
docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Size}}\t{{.Status}}' 2>/dev/null || true
echo ""
echo "=== /home usage ==="
sudo du -h --max-depth=2 /home 2>/dev/null | sort -rh | head -20
echo ""
echo "=== /var usage ==="
sudo du -h --max-depth=2 /var 2>/dev/null | sort -rh | head -20
echo ""
echo "=== /tmp usage ==="
sudo du -h --max-depth=1 /tmp 2>/dev/null | sort -rh | head -10
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.14"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Set up Python (non-OL8)
if: ${{ steps.detect_os.outputs.id != 'ol' }}
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.14"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build dataflow_engine
run: |
git submodule init
git submodule update
cd rust/otap-dataflow
docker buildx build --load --build-context otel-arrow=../../ -f Dockerfile -t df_engine .
cd ../..
- name: Install dependencies
run: |
python -m pip install --user --require-hashes -r tools/pipeline_perf_test/orchestrator/requirements.lock.txt
python -m pip install --user --require-hashes -r tools/pipeline_perf_test/load_generator/requirements.lock.txt
- name: Run pipeline performance test suite
run: |
cd tools/pipeline_perf_test
python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/100klrps-docker.yaml
- name: Post-run disk usage diagnostics
if: always()
run: |
echo "=== Overall disk usage ==="
df -h
echo ""
echo "=== Docker disk usage ==="
docker system df -v 2>/dev/null || true