Skip to content

Commit 155b3c4

Browse files
cijothomascijothomas
authored
PerfTest - Idle run perf numbers for all available cores (open-telemetry#1744)
We were running with core-id=0, so df_engine only spun one thread. Adding one more test where we don't restrict cores, so it'll run on all cores (creating as many threads) and show memory/cpu. (the lab perf machine has 128 cores)
1 parent 7cffafe commit 155b3c4

5 files changed

Lines changed: 142 additions & 117 deletions

File tree

‎.github/workflows/pipeline-perf-test-continuous.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,10 +93,11 @@ jobs:
9393
python orchestrator/run_orchestrator.py --config "$config"
9494
done
9595
96-
- name: Run idle state performance test
96+
- name: Run idle state performance tests
9797
run: |
9898
cd tools/pipeline_perf_test
9999
python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/idle-state-docker.yaml
100+
python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/idle-state-allcores-docker.yaml
100101
101102
- name: Upload benchmark results for processing
102103
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -114,7 +115,7 @@ jobs:
114115
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
115116
with:
116117
name: benchmark-results-idle-state
117-
path: tools/pipeline_perf_test/results/idle_state/gh-actions-benchmark/*.json
118+
path: tools/pipeline_perf_test/results/idle_state*/gh-actions-benchmark/*.json
118119

119120
- name: Add benchmark link to job summary
120121
run: |

‎tools/pipeline_perf_test/readme.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ pip install -r orchestrator/requirements.txt
239239
Ensure you are in the otel-arrow/tools/pipeline_perf_test directory, then:
240240

241241
```shell
242-
python ./orchestrator/run_orchestrator.py --debug --config ./test_suites/integration/continuous/100klrps-docker.yaml
242+
python ./orchestrator/run_orchestrator.py --config ./test_suites/integration/continuous/100klrps-docker.yaml
243243
```
244244

245245
```shell

‎tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-allcores-docker.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Idle State Performance Test - All Cores
2+
# Measures baseline resource consumption with no active telemetry traffic using all available cores
3+
from_template:
4+
path: test_suites/integration/continuous/idle-state-template.yaml.j2
5+
variables:
6+
core_label: "All Cores"
7+
port: 8087
8+
result_subdir: "allcores"

‎tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-docker.yaml‎

Lines changed: 10 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,10 @@
1-
# Idle State Performance Test
2-
# Measures baseline resource consumption with no active telemetry traffic
3-
name: Continuous - Idle State Performance
4-
components:
5-
df-engine:
6-
deployment:
7-
docker:
8-
image: df_engine:latest
9-
network: testbed
10-
ports:
11-
- "8086:8080"
12-
volumes:
13-
- 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
14-
command:
15-
- "--pipeline"
16-
- "./config.yaml"
17-
- "--core-id-range"
18-
- "0-0"
19-
- "--http-admin-bind"
20-
- "0.0.0.0:8080"
21-
monitoring:
22-
docker_component:
23-
allocated_cores: 1
24-
prometheus:
25-
endpoint: http://localhost:8086/telemetry/metrics?format=prometheus&reset=false
26-
27-
tests:
28-
- name: Idle State Baseline
29-
steps:
30-
- name: Deploy Dataflow Engine
31-
action:
32-
component_action:
33-
phase: deploy
34-
target: df-engine
35-
hooks:
36-
run:
37-
pre:
38-
- render_template:
39-
template_path: 'test_suites/integration/templates/configs/engine/continuous/otlp-attr-otlp.yaml'
40-
output_path: ./test_suites/integration/configs/engine/config.rendered.yaml
41-
variables:
42-
backend_hostname: localhost
43-
post:
44-
- ready_check_http:
45-
url: http://localhost:8086/telemetry/metrics?reset=false
46-
method: GET
47-
expected_status_code: 200
48-
49-
- name: Wait for Startup Stabilization
50-
action:
51-
wait:
52-
delay_seconds: 5
53-
hooks:
54-
run:
55-
pre:
56-
- record_event:
57-
name: stabilization_start
58-
post:
59-
- record_event:
60-
name: stabilization_complete
61-
62-
- name: Monitor Engine
63-
action:
64-
component_action:
65-
phase: start_monitoring
66-
target: df-engine
67-
68-
- name: Observe Idle State
69-
action:
70-
wait:
71-
delay_seconds: 15
72-
hooks:
73-
run:
74-
pre:
75-
- record_event:
76-
name: observation_start
77-
post:
78-
- record_event:
79-
name: observation_stop
80-
81-
- name: Stop Monitoring
82-
action:
83-
component_action:
84-
phase: stop_monitoring
85-
target: df-engine
86-
87-
- name: Destroy Engine
88-
action:
89-
component_action:
90-
phase: destroy
91-
target: df-engine
92-
93-
- name: Run Report
94-
action:
95-
wait:
96-
delay_seconds: 0
97-
hooks:
98-
run:
99-
post:
100-
- print_container_logs: {}
101-
- sql_report:
102-
name: Idle State Performance Report
103-
report_config_file: ./test_suites/integration/configs/idle_state_report.yaml
104-
output:
105-
- format:
106-
template: {}
107-
destination:
108-
console: {}
109-
- format:
110-
template:
111-
path: ./test_suites/integration/templates/reports/gh-action-sqlreport.j2
112-
destination:
113-
file:
114-
directory: results/idle_state/gh-actions-benchmark
1+
# Idle State Performance Test - Single Core
2+
# Measures baseline resource consumption with no active telemetry traffic on 1 core
3+
from_template:
4+
path: test_suites/integration/continuous/idle-state-template.yaml.j2
5+
variables:
6+
core_label: "Single Core"
7+
core_range: "0-0"
8+
allocated_cores: 1
9+
port: 8086
10+
result_subdir: "1core"

‎tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Template for idle state performance test
2+
# This template is used to generate both single-core and full-cores tests
3+
name: Continuous - Idle State Performance - {{core_label}}
4+
components:
5+
df-engine:
6+
deployment:
7+
docker:
8+
image: df_engine:latest
9+
network: testbed
10+
ports:
11+
- "{{port}}:8080"
12+
volumes:
13+
- 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
14+
command:
15+
- "--pipeline"
16+
- "./config.yaml"
17+
{% if core_range is defined %}
18+
- "--core-id-range"
19+
- "{{core_range}}"
20+
{% endif %}
21+
- "--http-admin-bind"
22+
- "0.0.0.0:8080"
23+
monitoring:
24+
{% if allocated_cores is defined %}
25+
docker_component:
26+
allocated_cores: {{allocated_cores}}
27+
{% else %}
28+
docker_component: {}
29+
{% endif %}
30+
prometheus:
31+
endpoint: http://localhost:{{port}}/telemetry/metrics?format=prometheus&reset=false
32+
33+
tests:
34+
- name: Idle State Baseline - {{core_label}}
35+
steps:
36+
- name: Deploy Dataflow Engine
37+
action:
38+
component_action:
39+
phase: deploy
40+
target: df-engine
41+
hooks:
42+
run:
43+
pre:
44+
- render_template:
45+
template_path: 'test_suites/integration/templates/configs/engine/continuous/otlp-attr-otlp.yaml'
46+
output_path: ./test_suites/integration/configs/engine/config.rendered.yaml
47+
variables:
48+
backend_hostname: localhost
49+
post:
50+
- ready_check_http:
51+
url: http://localhost:{{port}}/telemetry/metrics?reset=false
52+
method: GET
53+
expected_status_code: 200
54+
55+
- name: Wait for Startup Stabilization
56+
action:
57+
wait:
58+
delay_seconds: 5
59+
hooks:
60+
run:
61+
pre:
62+
- record_event:
63+
name: stabilization_start
64+
post:
65+
- record_event:
66+
name: stabilization_complete
67+
68+
- name: Monitor Engine
69+
action:
70+
component_action:
71+
phase: start_monitoring
72+
target: df-engine
73+
74+
- name: Observe Idle State
75+
action:
76+
wait:
77+
delay_seconds: 15
78+
hooks:
79+
run:
80+
pre:
81+
- record_event:
82+
name: observation_start
83+
post:
84+
- record_event:
85+
name: observation_stop
86+
87+
- name: Stop Monitoring
88+
action:
89+
component_action:
90+
phase: stop_monitoring
91+
target: df-engine
92+
93+
- name: Destroy Engine
94+
action:
95+
component_action:
96+
phase: destroy
97+
target: df-engine
98+
99+
- name: Run Report
100+
action:
101+
wait:
102+
delay_seconds: 0
103+
hooks:
104+
run:
105+
post:
106+
- print_container_logs: {}
107+
- sql_report:
108+
name: Idle State Performance Report - {{core_label}}
109+
report_config_file: ./test_suites/integration/configs/idle_state_report.yaml
110+
output:
111+
- format:
112+
template: {}
113+
destination:
114+
console: {}
115+
- format:
116+
template:
117+
path: ./test_suites/integration/templates/reports/gh-action-sqlreport.j2
118+
destination:
119+
file:
120+
directory: results/idle_state_{{result_subdir}}/gh-actions-benchmark

0 commit comments

Comments
 (0)