From e3fdedccfdb2733bfb3117c856c604968c64e51e Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Tue, 10 Mar 2026 01:54:22 +0000 Subject: [PATCH 1/2] Add batchprocessor to perf tests --- .../integration/continuous/passthrough.yaml | 13 ++++++ .../configs/engine/batch/otlp-batch-otlp.yaml | 42 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml index 4a94ddcc4f..9b96a9d25f 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml @@ -77,3 +77,16 @@ tests: observation_interval: 60 signals_per_second: null max_batch_size: 512 + # TODO: Uncomment when https://github.com/open-telemetry/otel-arrow/issues/2194 is fixed. + # Batch processor crashes the engine when inbound slots are exhausted. + # - name: OTLP-BATCH-OTLP + # from_template: + # path: test_suites/integration/templates/test_steps/df-loadgen-steps-docker.yaml + # variables: + # result_dir: continuous_passthrough_batch + # engine_config_template: test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml + # loadgen_exporter_type: otlp + # backend_receiver_type: otlp + # observation_interval: 60 + # signals_per_second: null + # max_batch_size: 512 diff --git a/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml b/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml new file mode 100644 index 0000000000..49fb44d65e --- /dev/null +++ b/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml @@ -0,0 +1,42 @@ +version: otel_dataflow/v1 +policies: + channel_capacity: + control: + node: 100 + pipeline: 100 + pdata: 100 +engine: + telemetry: + logs: + level: info +groups: + default: + pipelines: + main: + nodes: + receiver: + type: urn:otel:receiver:otlp + config: + protocols: + grpc: + listening_addr: 0.0.0.0:4317 + request_compression: gzip + wait_for_result: true + batch: + type: urn:otel:processor:batch + config: + otlp: + min_size: 8192 + sizer: items + flush_timeout: 200ms + format: preserve + exporter: + type: urn:otel:exporter:otlp_grpc + config: + grpc_endpoint: http://{{backend_hostname}}:1235 + compression_method: gzip + connections: + - from: receiver + to: batch + - from: batch + to: exporter From a75240660b1e0d9804acb13f178476ffd26daf8e Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Tue, 10 Mar 2026 18:51:24 +0000 Subject: [PATCH 2/2] keep batch in the mix --- .../crates/otap/src/batch_processor.rs | 2 +- .../integration/continuous/passthrough.yaml | 25 +++++++++---------- .../configs/engine/batch/otlp-batch-otlp.yaml | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/rust/otap-dataflow/crates/otap/src/batch_processor.rs b/rust/otap-dataflow/crates/otap/src/batch_processor.rs index 270fa31b30..47c59731ab 100644 --- a/rust/otap-dataflow/crates/otap/src/batch_processor.rs +++ b/rust/otap-dataflow/crates/otap/src/batch_processor.rs @@ -195,7 +195,7 @@ const fn default_otap_max_size_items() -> Option { } const fn default_otap_sizer_items() -> Sizer { - Sizer::Bytes + Sizer::Items } const fn default_otlp_min_size_bytes() -> Option { diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml index 9b96a9d25f..4b7d701713 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml @@ -77,16 +77,15 @@ tests: observation_interval: 60 signals_per_second: null max_batch_size: 512 - # TODO: Uncomment when https://github.com/open-telemetry/otel-arrow/issues/2194 is fixed. - # Batch processor crashes the engine when inbound slots are exhausted. - # - name: OTLP-BATCH-OTLP - # from_template: - # path: test_suites/integration/templates/test_steps/df-loadgen-steps-docker.yaml - # variables: - # result_dir: continuous_passthrough_batch - # engine_config_template: test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml - # loadgen_exporter_type: otlp - # backend_receiver_type: otlp - # observation_interval: 60 - # signals_per_second: null - # max_batch_size: 512 + + - name: OTLP-BATCH-OTLP + from_template: + path: test_suites/integration/templates/test_steps/df-loadgen-steps-docker.yaml + variables: + result_dir: continuous_passthrough_batch + engine_config_template: test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml + loadgen_exporter_type: otlp + backend_receiver_type: otlp + observation_interval: 60 + signals_per_second: null + max_batch_size: 512 diff --git a/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml b/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml index 49fb44d65e..022faf17de 100644 --- a/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/templates/configs/engine/batch/otlp-batch-otlp.yaml @@ -27,7 +27,7 @@ groups: config: otlp: min_size: 8192 - sizer: items + sizer: bytes flush_timeout: 200ms format: preserve exporter: