From 0add6db8056e9b92ad8efcb630092ae2856472ca Mon Sep 17 00:00:00 2001 From: Rahul Jain Date: Sat, 25 Jul 2026 00:36:00 -0700 Subject: [PATCH 1/2] perf(pipeline): add sustained benchmark validation Add partition-ordered processing and exact delivery accounting, introduce the sustained benchmark job harness, and consolidate the public documentation and GitHub Pages experience. Co-Authored-By: claude-flow --- .github/workflows/pages.yml | 16 +- .github/workflows/performance-test.yml | 414 ++-- .gitignore | 16 + ARCHITECTURE.md | 787 +++----- BENCHMARKS.md | 550 +----- CHANGELOG.md | 14 +- PERFORMANCE_OPTIMIZATIONS.md | 437 +---- PERFORMANCE_TUNING_RESULTS.md | 285 +-- README.md | 155 +- ROADMAP.md | 100 +- SUPPORT.md | 15 +- benches/end_to_end_benchmark.rs | 328 ++-- benches/filter_benchmarks.rs | 42 + benches/transform_benchmarks.rs | 48 +- docker-compose.benchmark.yml | 49 +- docs/404.md | 20 + docs/CHANGELOG.md | 288 --- docs/COMPATIBILITY.md | 2 +- docs/CONFIG_SCHEMA.json | 95 +- docs/CONTRIBUTING.md | 5 +- docs/DELIVERY_GUARANTEES.md | 808 +------- docs/DEPLOYMENT.md | 1743 +---------------- docs/DOCKER.md | 533 +---- docs/DOCUMENTATION_INDEX.md | 37 - docs/IMPLEMENTATION_STATUS.md | 418 ++-- docs/KUBERNETES.md | 900 ++------- docs/OBSERVABILITY_QUICKSTART.md | 395 ++-- docs/OPERATIONS.md | 1429 ++------------ docs/PERFORMANCE.md | 827 ++------ docs/PERFORMANCE_TESTING.md | 547 ++---- docs/PERFORMANCE_TUNING_RESULTS.md | 7 + docs/QUICKSTART.md | 10 +- docs/QUICK_REFERENCE.md | 1 + docs/SECURITY_CONFIGURATION.md | 612 ++---- docs/TROUBLESHOOTING.md | 1467 ++------------ docs/UI_MINIKUBE_DEMO.md | 71 +- docs/USAGE.md | 12 +- docs/_config.yml | 90 +- docs/_includes/footer_custom.html | 4 + docs/_includes/head_custom.html | 16 + docs/_includes/mermaid_config.js | 1 - docs/assets/css/streamforge-site.css | 491 +++++ docs/assets/images/streamforge-favicon.png | Bin 0 -> 2972 bytes docs/assets/images/streamforge-mark.svg | 10 + .../assets/images/streamforge-social-card.png | Bin 0 -> 17343 bytes .../assets/images/streamforge-social-card.svg | 40 + docs/benchmarks/CI_PERFORMANCE_TESTING.md | 259 +-- docs/benchmarks/README.md | 124 +- docs/benchmarks/THROUGHPUT_TESTING.md | 378 +--- .../results/phase2-baseline-20260724.md | 202 ++ docs/development/demo-recording.md | 76 + docs/development/internal/README.md | 2 +- docs/index.md | 183 +- examples/benchmarks/README.md | 44 +- examples/production/README.md | 48 +- examples/redpanda/README.md | 4 +- scripts/benchmarks/benchmark_ingress_job.py | 209 ++ scripts/benchmarks/benchmark_job_common.py | 196 ++ scripts/benchmarks/benchmark_jobs.py | 52 + scripts/benchmarks/benchmark_metrics_job.py | 223 +++ scripts/benchmarks/benchmark_output_job.py | 147 ++ scripts/benchmarks/generate_json_test_data.sh | 148 +- scripts/benchmarks/run_throughput_test.sh | 770 +++++--- scripts/benchmarks/test_benchmark_helpers.py | 170 ++ scripts/benchmarks/throughput_results.py | 417 ++++ scripts/docs/check_internal_links.py | 91 + src/config.rs | 495 +++++ src/dlq.rs | 1 + src/filter/envelope_transform.rs | 194 +- src/filter_parser.rs | 568 ++++-- src/kafka/sink.rs | 264 ++- src/kafka/sink/delivery.rs | 160 ++ src/kafka/sink_tests.rs | 105 + src/lib.rs | 4 +- src/main.rs | 303 +-- src/observability/metrics.rs | 11 + src/observability/mod.rs | 2 +- src/observability/server.rs | 15 +- src/partition_pipeline.rs | 367 ++++ src/partitioner.rs | 79 +- src/processor.rs | 174 +- src/retry.rs | 1 + tests.disabled/common/mod.rs | 1 + 83 files changed, 7955 insertions(+), 12667 deletions(-) create mode 100644 docs/404.md delete mode 100644 docs/CHANGELOG.md delete mode 100644 docs/DOCUMENTATION_INDEX.md create mode 100644 docs/PERFORMANCE_TUNING_RESULTS.md create mode 100644 docs/_includes/footer_custom.html create mode 100644 docs/_includes/head_custom.html delete mode 100644 docs/_includes/mermaid_config.js create mode 100644 docs/assets/css/streamforge-site.css create mode 100644 docs/assets/images/streamforge-favicon.png create mode 100644 docs/assets/images/streamforge-mark.svg create mode 100644 docs/assets/images/streamforge-social-card.png create mode 100644 docs/assets/images/streamforge-social-card.svg create mode 100644 docs/benchmarks/results/phase2-baseline-20260724.md create mode 100644 docs/development/demo-recording.md create mode 100755 scripts/benchmarks/benchmark_ingress_job.py create mode 100755 scripts/benchmarks/benchmark_job_common.py create mode 100755 scripts/benchmarks/benchmark_jobs.py create mode 100755 scripts/benchmarks/benchmark_metrics_job.py create mode 100755 scripts/benchmarks/benchmark_output_job.py create mode 100644 scripts/benchmarks/test_benchmark_helpers.py create mode 100755 scripts/benchmarks/throughput_results.py create mode 100644 scripts/docs/check_internal_links.py create mode 100644 src/kafka/sink/delivery.rs create mode 100644 src/kafka/sink_tests.rs create mode 100644 src/partition_pipeline.rs diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 0f3d799..0a04e6d 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,10 +1,17 @@ +--- name: GitHub Pages -on: +on: # yamllint disable-line rule:truthy push: branches: [main] paths: - 'docs/**' + - 'scripts/docs/**' + - '.github/workflows/pages.yml' + pull_request: + paths: + - 'docs/**' + - 'scripts/docs/**' - '.github/workflows/pages.yml' workflow_dispatch: @@ -33,12 +40,19 @@ jobs: source: ./docs destination: ./_site + - name: Validate internal links + run: >- + python3 scripts/docs/check_internal_links.py + _site --baseurl /streamforge + - name: Upload artifact + if: github.event_name != 'pull_request' uses: actions/upload-pages-artifact@v3 deploy: name: Deploy needs: build + if: github.event_name != 'pull_request' runs-on: ubuntu-latest environment: name: github-pages diff --git a/.github/workflows/performance-test.yml b/.github/workflows/performance-test.yml index 388f6c7..c5228c6 100644 --- a/.github/workflows/performance-test.yml +++ b/.github/workflows/performance-test.yml @@ -1,341 +1,185 @@ -name: Performance Tests +--- +name: Performance Smoke Tests -# Manual trigger only - don't run on every commit -on: +# Manual, on-demand evidence only. Shared GitHub-hosted runners are noisy, so +# these measurements are not a performance regression gate. +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: messages: - description: 'Number of messages to test' + description: Number of messages for the throughput smoke run required: false - default: '100000' + default: "100000" + type: string partitions: - description: 'Number of partitions' + description: Number of Kafka partitions required: false - default: '8' + default: "8" + type: string threads: - description: 'Number of threads' + description: Number of Streamforge worker threads required: false - default: '8' + default: "8" + type: string + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" jobs: - # ==================== MICROBENCHMARKS ==================== - criterion-benchmarks: - name: Criterion Benchmarks + criterion-smoke: + name: Criterion smoke evidence runs-on: ubuntu-latest + timeout-minutes: 45 + steps: - - uses: actions/checkout@v4 + - name: Check out repository + uses: actions/checkout@v4 - - name: Install system dependencies + - name: Install native build dependencies run: | sudo apt-get update sudo apt-get install -y \ + clang \ + cmake \ + libclang-dev \ libsasl2-dev \ libssl-dev \ libzstd-dev \ - cmake \ - pkg-config \ - clang \ - libclang-dev + pkg-config - - name: Install Rust - uses: dtolnay/rust-toolchain@stable + - name: Install Rust 1.89.0 + uses: dtolnay/rust-toolchain@1.89.0 - - name: Cache cargo + - name: Cache Cargo build data uses: Swatinem/rust-cache@v2 - - name: Run Criterion benchmarks - run: cargo bench --bench filter_benchmarks --bench transform_benchmarks + - name: Prepare structured result directories + run: mkdir -p target/performance-results/criterion + + - name: Run all Criterion targets + run: | + FILTER_LOG=target/performance-results/criterion/filter_benchmarks.log + TRANSFORM_LOG=target/performance-results/criterion/transform_benchmarks.log + PIPELINE_LOG=target/performance-results/criterion/end_to_end_benchmark.log + cargo bench --bench filter_benchmarks -- --noplot \ + 2>&1 | tee "$FILTER_LOG" + cargo bench --bench transform_benchmarks -- --noplot \ + 2>&1 | tee "$TRANSFORM_LOG" + cargo bench --bench end_to_end_benchmark -- --noplot \ + 2>&1 | tee "$PIPELINE_LOG" - - name: Upload benchmark results + - name: Explain evidence scope + if: always() + run: | + { + echo "## Criterion smoke evidence" + echo + echo "All three repository Criterion targets were executed" \ + "on a shared GitHub-hosted runner." + echo "Results are diagnostic smoke evidence only;" \ + "this workflow applies no regression threshold." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Criterion reports and logs + if: always() uses: actions/upload-artifact@v4 with: - name: criterion-results - path: target/criterion/ + name: criterion-smoke-${{ github.run_id }} + if-no-files-found: warn + retention-days: 14 + path: | + target/criterion/ + target/performance-results/criterion/ - # ==================== THROUGHPUT TEST ==================== - throughput-test: - name: End-to-End Throughput Test + throughput-smoke: + name: Throughput smoke evidence runs-on: ubuntu-latest timeout-minutes: 30 + steps: - - uses: actions/checkout@v4 + - name: Check out repository + uses: actions/checkout@v4 - - name: Install system dependencies + - name: Install native and harness dependencies run: | sudo apt-get update sudo apt-get install -y \ + clang \ + cmake \ + curl \ + libclang-dev \ libsasl2-dev \ libssl-dev \ libzstd-dev \ - cmake \ - pkg-config \ - clang \ - libclang-dev \ - kafkacat \ - jq + pkg-config - - name: Install Rust - uses: dtolnay/rust-toolchain@stable + - name: Install Rust 1.89.0 + uses: dtolnay/rust-toolchain@1.89.0 - - name: Cache cargo + - name: Cache Cargo build data uses: Swatinem/rust-cache@v2 - - name: Build Streamforge + - name: Build release binary run: cargo build --release - - name: Start Kafka - run: | - docker-compose -f docker-compose.benchmark.yml up -d - sleep 30 - - - name: Wait for Kafka - run: | - for i in {1..30}; do - if nc -z localhost 9092; then - echo "Kafka is ready" - exit 0 - fi - echo "Waiting for Kafka... ($i/30)" - sleep 2 - done - echo "Kafka failed to start" - exit 1 - - - name: Create topics - run: | - docker exec benchmark-kafka kafka-topics \ - --create --topic test-${{ inputs.partitions }}p-input \ - --partitions ${{ inputs.partitions }} \ - --replication-factor 1 \ - --bootstrap-server localhost:9092 || true + - name: Start benchmark Kafka + run: docker compose -f docker-compose.benchmark.yml up -d --wait - docker exec benchmark-kafka kafka-topics \ - --create --topic test-${{ inputs.partitions }}p-output \ - --partitions ${{ inputs.partitions }} \ - --replication-factor 1 \ - --bootstrap-server localhost:9092 || true + - name: Prepare structured result directory + run: mkdir -p target/performance-results/throughput - - name: Generate test data + - name: Run one throughput repetition + env: + SMOKE_MESSAGES: ${{ inputs.messages }} + SMOKE_PARTITIONS: ${{ inputs.partitions }} + SMOKE_THREADS: ${{ inputs.threads }} run: | - cd benchmarks - ./generate_json_test_data.sh ${{ inputs.messages }} test_data.jsonl + scripts/benchmarks/run_throughput_test.sh \ + "$SMOKE_MESSAGES" \ + "$SMOKE_PARTITIONS" \ + "$SMOKE_THREADS" \ + 1 \ + 2>&1 | tee target/performance-results/throughput/harness.log - - name: Create test config - run: | - cat > config.json << EOF - { - "appid": "ci-perf-test", - "bootstrap": "localhost:9092", - "target_broker": "localhost:9092", - "input": "test-${{ inputs.partitions }}p-input", - "output": "test-${{ inputs.partitions }}p-output", - "threads": ${{ inputs.threads }}, - "offset": "earliest", - "observability": { - "metrics_enabled": true, - "metrics_port": 9090, - "lag_monitoring_enabled": true, - "lag_monitoring_interval_secs": 10 - }, - "routing": { - "routing_type": "filter", - "destinations": [ - { - "output": "test-${{ inputs.partitions }}p-output", - "key_transform": "/userId", - "headers": { - "x-processed": "true" - } - } - ] - } - } - EOF - - - name: Start Streamforge - run: | - ./target/release/streamforge > streamforge.log 2>&1 & - echo $! > streamforge.pid - sleep 5 - - - name: Send test messages - run: | - date +%s > start_time.txt - cat benchmarks/test_data.jsonl | docker exec -i benchmark-kafka \ - kafka-console-producer \ - --bootstrap-server localhost:9092 \ - --topic test-${{ inputs.partitions }}p-input \ - --batch-size 2000 - date +%s > end_time.txt - - - name: Wait for processing - run: | - for i in {1..60}; do - CONSUMED=$(curl -s http://localhost:9090/metrics | grep "^streamforge_messages_consumed_total " | awk '{print $2}') - PRODUCED=$(curl -s http://localhost:9090/metrics | grep "^streamforge_messages_produced_total" | awk '{print $2}') - echo "[$i] Consumed: $CONSUMED | Produced: $PRODUCED" - - if [[ "$CONSUMED" -ge "${{ inputs.messages }}" ]]; then - echo "โœ“ All messages processed!" - break - fi - sleep 5 - done - - - name: Collect metrics - run: | - curl -s http://localhost:9090/metrics > final_metrics.txt - - START=$(cat start_time.txt) - END=$(cat end_time.txt) - DURATION=$((END - START)) - - CONSUMED=$(grep "^streamforge_messages_consumed_total " final_metrics.txt | awk '{print $2}') - PRODUCED=$(grep "^streamforge_messages_produced_total" final_metrics.txt | awk -F'}' '{print $2}' | awk '{print $1}') - ERRORS=$(grep "^streamforge_processing_errors_total " final_metrics.txt | awk '{print $2}' | head -1) - - THROUGHPUT=$((CONSUMED / DURATION)) - - echo "=== PERFORMANCE TEST RESULTS ===" | tee -a $GITHUB_STEP_SUMMARY - echo "" | tee -a $GITHUB_STEP_SUMMARY - echo "**Configuration:**" | tee -a $GITHUB_STEP_SUMMARY - echo "- Messages: ${{ inputs.messages }}" | tee -a $GITHUB_STEP_SUMMARY - echo "- Partitions: ${{ inputs.partitions }}" | tee -a $GITHUB_STEP_SUMMARY - echo "- Threads: ${{ inputs.threads }}" | tee -a $GITHUB_STEP_SUMMARY - echo "" | tee -a $GITHUB_STEP_SUMMARY - echo "**Results:**" | tee -a $GITHUB_STEP_SUMMARY - echo "- Consumed: $CONSUMED" | tee -a $GITHUB_STEP_SUMMARY - echo "- Produced: $PRODUCED" | tee -a $GITHUB_STEP_SUMMARY - echo "- Errors: ${ERRORS:-0}" | tee -a $GITHUB_STEP_SUMMARY - echo "- Duration: ${DURATION}s" | tee -a $GITHUB_STEP_SUMMARY - echo "- **Throughput: ${THROUGHPUT} msg/s**" | tee -a $GITHUB_STEP_SUMMARY - echo "" | tee -a $GITHUB_STEP_SUMMARY - - - name: Stop Streamforge + - name: Collect structured throughput evidence if: always() + env: + SMOKE_MESSAGES: ${{ inputs.messages }} + SMOKE_PARTITIONS: ${{ inputs.partitions }} + SMOKE_THREADS: ${{ inputs.threads }} run: | - if [ -f streamforge.pid ]; then - kill $(cat streamforge.pid) || true - fi + mkdir -p \ + target/performance-results/throughput/docker - - name: Upload logs + COMPOSE_LOG=target/performance-results/throughput/docker/compose.log + docker compose -f docker-compose.benchmark.yml logs --no-color \ + > "$COMPOSE_LOG" 2>&1 || true + + { + echo "## Throughput smoke evidence" + echo + echo "- Messages: $SMOKE_MESSAGES" + echo "- Partitions: $SMOKE_PARTITIONS" + echo "- Threads: $SMOKE_THREADS" + echo "- Repetitions: 1" + echo "- Environment: shared GitHub-hosted runner" + echo + echo "This is diagnostic smoke evidence, not a regression gate." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload structured throughput results and logs if: always() uses: actions/upload-artifact@v4 with: - name: performance-test-logs - path: | - streamforge.log - final_metrics.txt - benchmarks/test_data.jsonl + name: throughput-smoke-${{ github.run_id }} + if-no-files-found: warn + retention-days: 14 + path: target/performance-results/throughput/ - - name: Stop Kafka + - name: Stop Streamforge and Kafka if: always() - run: | - docker-compose -f docker-compose.benchmark.yml down -v - - # ==================== LATENCY TEST ==================== - latency-test: - name: Latency Profile Test - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libsasl2-dev libssl-dev libzstd-dev cmake pkg-config clang libclang-dev - - - name: Build - run: cargo build --release - - - name: Start Kafka - run: | - docker-compose -f docker-compose.benchmark.yml up -d - sleep 30 - - - name: Run latency test - run: | - # Create small config for latency test - cat > config.json << EOF - { - "appid": "latency-test", - "bootstrap": "localhost:9092", - "input": "test-latency-input", - "output": "test-latency-output", - "threads": 4, - "observability": { - "metrics_enabled": true, - "metrics_port": 9090 - } - } - EOF - - # Create topics - docker exec benchmark-kafka kafka-topics \ - --create --topic test-latency-input --partitions 4 --replication-factor 1 \ - --bootstrap-server localhost:9092 || true - docker exec benchmark-kafka kafka-topics \ - --create --topic test-latency-output --partitions 4 --replication-factor 1 \ - --bootstrap-server localhost:9092 || true - - # Start Streamforge - ./target/release/streamforge > streamforge.log 2>&1 & - STREAMFORGE_PID=$! - sleep 5 - - # Generate and send 10K messages - cd benchmarks - ./generate_json_test_data.sh 10000 latency_test.jsonl - cat latency_test.jsonl | docker exec -i benchmark-kafka \ - kafka-console-producer --bootstrap-server localhost:9092 --topic test-latency-input - - # Wait for processing - sleep 10 - - # Extract latency metrics - curl -s http://localhost:9090/metrics > latency_metrics.txt - - echo "=== LATENCY RESULTS ===" | tee -a $GITHUB_STEP_SUMMARY - echo "" | tee -a $GITHUB_STEP_SUMMARY - grep "streamforge_processing_duration_seconds" latency_metrics.txt | grep -v "^#" | tee -a $GITHUB_STEP_SUMMARY - - # Cleanup - kill $STREAMFORGE_PID || true - - - name: Stop Kafka - if: always() - run: docker-compose -f docker-compose.benchmark.yml down -v - - # ==================== COMPARISON REPORT ==================== - performance-report: - name: Generate Performance Report - needs: [criterion-benchmarks, throughput-test, latency-test] - runs-on: ubuntu-latest - if: always() - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - - - name: Generate summary report - run: | - echo "# ๐Ÿš€ Performance Test Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Test run completed for:" >> $GITHUB_STEP_SUMMARY - echo "- SHA: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "- Branch: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY - echo "- Triggered by: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## ๐Ÿ“Š Test Configuration" >> $GITHUB_STEP_SUMMARY - echo "- Messages: ${{ inputs.messages }}" >> $GITHUB_STEP_SUMMARY - echo "- Partitions: ${{ inputs.partitions }}" >> $GITHUB_STEP_SUMMARY - echo "- Threads: ${{ inputs.threads }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "## ๐Ÿ“ฅ Artifacts" >> $GITHUB_STEP_SUMMARY - echo "- Criterion benchmark results" >> $GITHUB_STEP_SUMMARY - echo "- Performance test logs" >> $GITHUB_STEP_SUMMARY - echo "- Prometheus metrics" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "---" >> $GITHUB_STEP_SUMMARY - echo "*Note: CI performance results are indicative only. Production performance will vary based on hardware and network.*" >> $GITHUB_STEP_SUMMARY + run: docker compose -f docker-compose.benchmark.yml down -v diff --git a/.gitignore b/.gitignore index 44876c7..5e3e9e4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,19 @@ operator/target benchmarks/*.jsonl benchmarks/results/live_test_*/ benchmarks/test_data_*.jsonl + +# Codex local configuration +.codex/ + +# Claude Flow runtime data +.claude-flow/data/ +.claude-flow/logs/ + +# Environment variables +.env +.env.local +.env.*.local + +# Python caches +__pycache__/ +*.py[cod] diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9b269c6..d4bb54d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,637 +1,304 @@ -# Architecture Overview - -High-level architecture and design principles for Streamforge (formerly StreamForge). - ---- - -## System Overview - -Streamforge is a high-performance Kafka streaming toolkit that mirrors, filters, transforms, and routes messages between Kafka clusters with sub-microsecond latency. - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Source Kafka โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€>โ”‚ Streamforge โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€>โ”‚ Target Kafka โ”‚ -โ”‚ Cluster(s) โ”‚ โ”‚ Processing โ”‚ โ”‚ Cluster(s) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ”‚ Optional: - โ”œโ”€ Filter (44-145ns) - โ”œโ”€ Transform (810-1,633ns) - โ”œโ”€ Hash for deduplication - โ””โ”€ Route to multiple destinations -``` - ---- - -## Core Architecture - -### High-Level Design - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Streamforge โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Configuration Layer โ”‚ -โ”‚ โ”œโ”€ YAML/JSON Config Parser โ”‚ -โ”‚ โ””โ”€ Security Configuration (SSL/TLS, SASL) โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Consumer Layer โ”‚ -โ”‚ โ”œโ”€ Kafka StreamConsumer (rdkafka) โ”‚ -โ”‚ โ”œโ”€ Concurrent Batch Processing (80 parallel ops) โ”‚ -โ”‚ โ”œโ”€ At-least-once / At-most-once semantics โ”‚ -โ”‚ โ””โ”€ Manual/Auto commit strategies โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Processing Layer โ”‚ -โ”‚ โ”œโ”€ Message Processor (async trait) โ”‚ -โ”‚ โ”‚ โ”œโ”€ Single Destination Processor โ”‚ -โ”‚ โ”‚ โ””โ”€ Multi Destination Processor โ”‚ -โ”‚ โ”œโ”€ Filter Engine (custom DSL) โ”‚ -โ”‚ โ”‚ โ”œโ”€ Boolean Logic (AND/OR/NOT) โ”‚ -โ”‚ โ”‚ โ”œโ”€ Regex Matching โ”‚ -โ”‚ โ”‚ โ””โ”€ Array Operations โ”‚ -โ”‚ โ”œโ”€ Transform Engine (custom DSL) โ”‚ -โ”‚ โ”‚ โ”œโ”€ Field Mapping โ”‚ -โ”‚ โ”‚ โ”œโ”€ Object Construction โ”‚ -โ”‚ โ”‚ โ””โ”€ Arithmetic Operations โ”‚ -โ”‚ โ””โ”€ Hashing & Caching (optional) โ”‚ -โ”‚ โ”œโ”€ SHA256 hashing for deduplication โ”‚ -โ”‚ โ””โ”€ LRU/Redis cache backends โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Producer Layer โ”‚ -โ”‚ โ”œโ”€ Kafka Sink (FutureProducer) โ”‚ -โ”‚ โ”œโ”€ Custom Partitioning โ”‚ -โ”‚ โ”œโ”€ Compression (gzip/snappy/zstd/lz4) โ”‚ -โ”‚ โ””โ”€ Async message delivery โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Observability Layer โ”‚ -โ”‚ โ”œโ”€ Metrics (Stats Reporter) โ”‚ -โ”‚ โ”œโ”€ Tracing (tracing crate) โ”‚ -โ”‚ โ””โ”€ Error Handling โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +# Architecture + +StreamForge is a Rust-native Kafka data-plane service for selective replication: +consume records, evaluate routing rules, optionally transform message envelopes, +and produce to one or more destinations. + +This document describes the current architecture. Product boundaries and future +typed-envelope work are governed by `PROJECT_SPEC.md`. + +## System context + +```text +Source Kafka + | + v +rust-rdkafka StreamConsumer + | + v +selectable legacy batching or bounded source-partition worker lanes + | + v +MessageEnvelope (JSON value, optional key, headers, timestamp, source metadata) + | + +--> destination filter --> optional transform --> KafkaSink --> Target Kafka + +--> destination filter --> optional transform --> KafkaSink --> Target Kafka + | + +--> metrics, retry, DLQ, and offset-commit handling ``` ---- +The current data path parses payloads into `serde_json::Value`. A raw-byte +passthrough envelope is planned but is not part of the current runtime. -## Key Design Decisions +## Major layers -### 1. Rust Language Choice +### Configuration -**Rationale:** -- **Memory Safety**: Zero-cost abstractions, no garbage collection -- **Performance**: Native compilation, minimal runtime overhead -- **Concurrency**: Fearless concurrency with ownership model -- **Ecosystem**: Rich async ecosystem (Tokio, rdkafka) +`src/config.rs` parses YAML or JSON into typed configuration and validates +cross-field constraints. -**Benefits Achieved:** -- 40x faster filters/transforms vs Java JSLT -- 10x less memory (~50MB vs ~500MB) -- 2.5x higher throughput (25K+ msg/s vs 10K msg/s) -- Zero CVEs with Chainguard base images +Responsibilities include: -### 2. Custom DSL (No External Dependencies) +- source, destination, security, commit, retry, DLQ, cache, aggregation, and + observability settings; +- backward-compatible runtime performance defaults; +- mapping selected performance fields to librdkafka properties; +- preserving explicit `consumer_properties` and `producer_properties` as the + highest-precedence performance settings. StreamForge reapplies + `enable.auto.commit` and `enable.auto.offset.store` after raw consumer + properties because commit strategy is a validated reliability contract. -**Decision:** Build custom string-based filtering/transformation DSL instead of using JSLT/JavaScript/Rhai - -**Rationale:** -- JSLT (Java) and JavaScript engines have significant overhead -- Rhai (Rust scripting) adds ~300KB binary size and runtime complexity -- Custom DSL optimized for message streaming patterns -- Sub-microsecond performance critical for throughput -- Explicit syntax matches Kafka streaming patterns - -**Results:** -- Simple filters: 44-50ns (vs ~2,000ns for JSLT) -- Transforms: 810-1,633ns (vs ~40,000ns for JSLT) -- Zero external dependencies for core DSL -- Colon-delimited syntax (e.g., `/path,==,value`, `AND:cond1:cond2`) - -**v1.0 Gaps:** -- โŒ No formal grammar (EBNF) -- โŒ Parser lacks validation layer (errors found at runtime) -- โŒ No AST representation -- โŒ Error messages lack context -- โญ๏ธ Planned: Separate parser/AST/validator/evaluator in Phase 2 - -### 3. Async/Await Architecture - -**Decision:** Use Tokio async runtime for all I/O operations - -**Rationale:** -- Non-blocking I/O maximizes CPU utilization -- Concurrent message processing without thread-per-message overhead -- Efficient resource usage for high-throughput scenarios - -**Implementation:** -- `async fn process()` for message processing -- `buffer_unordered()` for concurrent batch processing -- 80 parallel operations (8 threads ร— 10 parallelism factor) - -### 4. Concurrent Batch Processing - -**Decision:** Process messages in batches with configurable concurrency - -**Configuration:** -```rust -BATCH_SIZE = 100 // Messages per batch -BATCH_FILL_TIMEOUT_MS = 100 // Max wait for batch -PARALLELISM_FACTOR = 10 // threads ร— 10 = concurrency -``` +`src/main.rs` applies the validated configuration to the consumer and processing +loop. -**Rationale:** -- Balance throughput and latency -- Efficient commit strategies -- Resource pooling (connections, buffers) +### Consumer and processing loop -**Results:** -- 132x improvement over sequential (83 โ†’ 11,000 msg/s) -- Perfect linear scaling (2.0x from 4 to 8 threads) -- Peak throughput: 34,517 msg/s sustained +`src/main.rs` owns the `StreamConsumer`, subscription, processing-mode +selection, and offset-commit coordination. `src/partition_pipeline.rs` owns the +bounded partition-ordered worker implementation. -### 5. Pluggable Delivery Semantics +The compatibility mode retains the historical batch barrier: -**Decision:** Support both at-least-once and at-most-once delivery - -**Configuration:** ```yaml -commit_strategy: - manual_commit: true # At-least-once - commit_mode: sync # Async or Sync - -dead_letter_queue: - enabled: true - topic: streamforge-dlq - max_retries: 3 -``` - -**Rationale:** -- Different use cases have different requirements -- Trade-off between throughput and guarantees -- Flexibility for users to choose - -**Performance:** -- At-least-once: 10,933 msg/s with full durability -- At-most-once: 11,200 msg/s (~3% overhead for guarantees) - -**v1.0 Gaps:** -- โŒ Commit semantics not formally documented -- โŒ Retry backoff policy undefined -- โŒ DLQ message format unspecified -- โŒ No integration tests for failure scenarios -- โญ๏ธ Planned: docs/DELIVERY_GUARANTEES.md + tests in Phase 1 - -### 6. Multi-Destination Routing - -**Decision:** Support content-based routing to multiple destinations - -**Architecture:** -```rust -pub struct DestinationProcessor { - sink: Arc, - filter: Arc, // Optional - transform: Arc, // Optional - name: String, -} - -pub struct MultiDestinationProcessor { - destinations: Vec, - routing_path: Option, -} +performance: + processing_mode: legacy_batch + consumer_batch_size: 100 + consumer_batch_timeout_ms: 100 + parallelism_factor: 10 ``` -**Rationale:** -- Single pipeline can serve multiple use cases -- Filter and transform per destination -- Efficient: share consumer, process once - -### 7. Cache-Based Deduplication - -**Decision:** Optional hash-based deduplication with pluggable cache backends - -**Supported Backends:** -- **LRU Cache** (in-memory, fast, bounded) -- **Redis** (distributed, persistent, shared) - -**Rationale:** -- Handle duplicate messages from upstream -- Configurable cache backend based on scale -- Async cache operations don't block processing - ---- - -## Data Flow - -### Single Destination Flow - -``` -1. Consumer reads message batch (100 messages, 100ms timeout) - โ†“ -2. Parse message key (permissive) and value (strict JSON) - โ†“ -3. Process batch concurrently (80 parallel operations) - โ”œโ”€ Apply filter (if configured) - โ”œโ”€ Apply transform (if configured) - โ””โ”€ Check cache/hash (if configured) - โ†“ -4. Send to Kafka sink (async) - โ†“ -5. Commit offsets (if manual commit mode) - โ”œโ”€ Retry with exponential backoff (3 attempts) - โ””โ”€ Halt on persistent failure (prevent data loss) -``` +Effective processing concurrency is the saturating product of `threads` and +`parallelism_factor`, with a minimum of one. -### Multi-Destination Flow +The opt-in partition-ordered mode detaches consumed records into owned messages +and routes every `(source topic, source partition)` to one of `threads` bounded +FIFO worker lanes: +```yaml +performance: + processing_mode: partition_ordered + worker_queue_capacity: 1024 ``` -1. Consumer reads message batch - โ†“ -2. Parse message - โ†“ -3. For each destination (in parallel): - โ”œโ”€ Evaluate destination-specific filter - โ”œโ”€ Apply destination-specific transform - โ”œโ”€ Check destination-specific cache - โ””โ”€ Send to destination sink - โ†“ -4. Collect results - โ”œโ”€ If any destination failed โ†’ halt (data integrity) - โ””โ”€ If all succeeded โ†’ commit offsets -``` - ---- - -## Component Details - -### Configuration Layer - -**File:** `src/config.rs` - -Responsibilities: -- Parse YAML/JSON configuration -- Validate configuration -- Apply security settings -- Provide defaults - -### Consumer Layer - -**File:** `src/main.rs` -Responsibilities: -- Create Kafka consumer -- Subscribe to topics -- Manage consumer groups -- Handle offset commits -- Implement commit retry logic +Records from one source partition enter one lane in consumption order. Different +lanes execute concurrently. This mode currently supports auto commit only; +manual commit requires a rebalance-aware completed-offset coordinator and is +rejected during configuration validation. -### Processing Layer +### Envelope -**Files:** `src/processor.rs`, `src/filter/`, `src/transform.rs` +`src/envelope.rs` defines `MessageEnvelope`. It carries: -Responsibilities: -- **MessageProcessor trait**: Define processing interface -- **SingleDestinationProcessor**: Single output processing -- **MultiDestinationProcessor**: Multi-output routing -- **Filter Engine**: Evaluate filter expressions (44-145ns) -- **Transform Engine**: Apply transformations (810-1,633ns) +- a JSON message value; +- an optional JSON key; +- headers; +- timestamp; +- source topic, partition, and offset metadata. -### Producer Layer +The JSON value is reference-counted for destination fan-out. Destinations without +a value transform retain the shared allocation. A destination with a transform +uses copy-on-write ownership: a uniquely owned value can be reused, while a +shared value is cloned only when mutation is required. -**Files:** `src/kafka/sink.rs`, `src/kafka/partitioner.rs` +### Filter and transform DSL -Responsibilities: -- Create Kafka producer (FutureProducer) -- Handle custom partitioning -- Apply compression -- Send messages asynchronously -- Handle producer errors +`src/filter_parser.rs`, `src/dsl/`, and `src/filter/` implement the DSL. -### Observability Layer +Supported execution forms include: -**Files:** `src/metrics.rs` +- legacy colon-delimited filters and transforms; +- function-style filters parsed into an AST; +- value, key, header, timestamp, array, string, and cache-aware operations. -Responsibilities: -- Track processed messages -- Track completed messages -- Track errors -- Report statistics (every 10 seconds) -- Tracing integration +Function-style filter construction lowers the parsed expression into a compiled +evaluation tree. JSON path segments, regexes, and typed array literals are +prepared once at construction rather than on every message. Key-template +transforms similarly tokenize placeholders and paths once. ---- +Function-style array `any` and `all` evaluation still clone each visited array +element into a temporary envelope. That boundary is intentionally left for a +later measured refactor. -## Performance Architecture +### Destination processing -### Throughput Optimization +`src/processor.rs` builds a runtime for each configured destination. -1. **Concurrent Batch Processing** - - Process 100 messages per batch - - 80 concurrent operations (8 threads ร— 10) - - Result: 132x throughput improvement +Each destination can have: -2. **Async I/O** - - Non-blocking Kafka I/O - - Tokio runtime for efficient scheduling - - Result: Maximize CPU utilization +- an optional filter; +- an optional value transform; +- key, header, and timestamp transforms; +- optional cache or aggregation behavior; +- an independent Kafka sink. -3. **Custom DSL** - - Zero-overhead parsing (compile-time) - - Sub-microsecond filter/transform - - Result: 40x faster than JSLT +An absent value transform remains `None`; no identity transform or transform +metric is executed. Multi-destination routing shares the incoming value until a +destination requires mutation. -4. **Efficient Memory Usage** - - ~50MB RAM footprint - - Zero garbage collection - - Result: 10x less memory than Java +### Producer and partitioning -### Latency Optimization +`src/kafka/sink.rs` wraps a rust-rdkafka `FutureProducer`, resolves output topic +templates, applies producer/security settings, serializes envelopes, and sends +records. -1. **Minimal Processing Overhead** - - Filters: 44-145ns per message - - Transforms: 810-1,633ns per message - - Total: < 2ยตs per message +Producer delivery is selectable: -2. **Batch Timeout** - - 100ms max wait for batch - - Ensures low-latency during low traffic - - Result: P99 latency < 150ms +- `acknowledged` is the compatibility default and waits for every record's + broker delivery result; +- `queued` uses librdkafka's nonblocking enqueue path, tracks delivery futures, + applies a configured pending-delivery bound, faults on asynchronous failure, + and drains on flush. ---- +Queued mode deliberately supports only auto commit, with message retries +disabled and the DLQ disabled. A delayed delivery failure cannot be associated +with the original envelope, so enabling queued mode with manual commits or +envelope-level recovery is rejected rather than weakening those contracts +silently. -## Scaling Architecture +`src/partitioner.rs` supplies explicit partition choices when StreamForge owns +the routing decision: -### Vertical Scaling +- a present key uses deterministic keyed hashing; +- field-based partitioning hashes the configured JSON field; +- an absent key with default partitioning returns no explicit partition, so + librdkafka selects the partition using its configured keyless behavior. -**Single Instance:** -- 4 threads โ†’ 10,933 msg/s -- 8 threads โ†’ 25,000-30,000 msg/s (linear scaling) -- Scales with CPU cores +An explicit JSON `null` key is still a present key and follows keyed hashing. -**Configuration:** -```yaml -threads: 8 # Number of consumer threads -``` - -### Horizontal Scaling +### Reliability -**Multiple Instances:** -- Kafka consumer groups -- Partitions distributed across instances -- Each instance processes subset of partitions - -**Example:** -``` -8 partitions, 2 instances: -- Instance 1: partitions 0-3 -- Instance 2: partitions 4-7 -``` +The runtime supports manual and automatic commit modes, retry policies, and a +dead-letter queue. Commit and failure semantics are defined in +`docs/DELIVERY_GUARANTEES.md`. -### Kubernetes Scaling +Exactly-once Kafka transactions are not implemented. -**Horizontal Pod Autoscaler (HPA):** -```yaml -minReplicas: 2 -maxReplicas: 10 -targetCPUUtilizationPercentage: 70 -``` +### State and aggregation -**Scaling triggers:** -- CPU utilization -- Custom metrics (lag, throughput) -- Message queue depth +`src/cache.rs` and `src/cache_backend.rs` provide local and Redis-backed caching. +`src/aggregation.rs` provides configured windowed aggregation subject to +validation constraints in `src/config.rs`. ---- +Stateful behavior must not silently change delivery guarantees. Broader +fault-tolerant state recovery remains future work. -## Security Architecture +### Observability -### Authentication +`src/metrics.rs` and `src/observability/` provide processing metrics, Prometheus +exposure, HTTP observability endpoints, and consumer-lag monitoring. -Supported mechanisms: -- **SASL/PLAIN** - Username/password (simple) -- **SASL/SCRAM-SHA-256** - Username/password (secure) -- **SASL/SCRAM-SHA-512** - Username/password (more secure) -- **SASL/GSSAPI** - Kerberos -- **Mutual TLS** - Certificate-based +Performance decisions should use completed-message rate, lag, error rate, +latency, CPU, and memory together. A microbenchmark result is not an end-to-end +Kafka service-level result. -### Encryption +## Phase 1 performance decisions -- **SSL/TLS** - Transport encryption -- **TLS 1.2/1.3** - Modern protocols -- **Certificate validation** - Hostname verification +Phase 1 deliberately uses low-risk changes that preserve the JSON envelope and +DSL contracts: -### Secrets Management +1. Delegate keyless default partition selection to librdkafka. +2. Skip absent value transforms. +3. Use copy-on-write values for actual destination transforms. +4. Compile function-style paths and regexes at filter construction. +5. Compile key-template placeholders and paths at transform construction. +6. Expose batching, fill timeout, concurrency, and selected Kafka tuning fields. +7. Add regression tests and steady-state Criterion benchmarks. -- **Environment variables** - For sensitive values -- **Kubernetes secrets** - For K8s deployments -- **File-based secrets** - Certificate files +No fixed throughput or latency is part of the architecture contract. See +`docs/PERFORMANCE.md` for the measurement method. ---- +## Phase 2 delivery and scheduling decisions -## Reliability Architecture +The first dedicated profile showed that per-record delivery waiting and the +100-record batch barrier were stronger candidates than SIMD. The resulting +opt-in path: -### Error Handling +1. replaces batch barriers with bounded partition-affine worker lanes; +2. makes `threads` the logical worker-lane count; +3. moves JSON parsing and envelope construction into those workers; +4. queues Kafka deliveries without awaiting each acknowledgement; +5. bounds and drains pending delivery futures; +6. exposes a delivery-acknowledgement metric separate from enqueue success; +7. keeps the legacy reliability behavior as the default. -1. **Parse Errors** - - Log with full context (topic, partition, offset, key) - - Count as error in metrics - - Handled per delivery semantics +The corrected Kafka harness warms the pipeline before timing and records input +publication, post-publication drain, and end-to-end completion independently. +This architecture is implemented but does not carry a throughput claim until a +new controlled benchmark is run. -2. **Processing Errors** - - Propagate to batch level - - Trigger commit failure handling +## Why SIMD is not in Phase 1 -3. **Commit Errors** - - Retry with exponential backoff (3 attempts) - - Halt on persistent failure (prevent data loss) +The current JSON filter path walks a heterogeneous tree and performs +pointer-heavy, branch-heavy operations. SIMD does not automatically accelerate +that representation. A SIMD implementation is justified only when profiling +identifies a stable, uniform kernel such as byte scanning, hashing, or +homogeneous numeric processing. -### Delivery Guarantees +The broader raw/lazy envelope design can avoid more work than vectorizing a +small part of the current parsed-JSON path. Because that design changes public +processing contracts, it remains in the later phase already defined by +`PROJECT_SPEC.md`. -**At-least-once:** -- Manual commits after successful processing -- Retry logic prevents message loss -- Duplicates possible on failure recovery +## Scaling model -**At-most-once:** -- Auto-commit mode -- Lower overhead (~3%) -- Message loss possible on failure +Vertical scaling is bounded by: ---- +- source partition parallelism; +- configured processing concurrency; +- CPU cost of parsing, filters, transforms, aggregation, and serialization; +- destination producer queues and broker/network latency; +- memory retained by in-flight messages. -## Deployment Architecture +Horizontal scaling uses Kafka consumer-group partition assignment. Adding +instances beyond the number of useful source partitions does not add consumer +parallelism. -### Docker +Ordering is preserved only within the constraints of Kafka partition ordering +and the configured processing/delivery behavior. Changing partitioning keys can +change ordering domains. -``` -streamforge:latest (20MB image) -โ”œโ”€ Chainguard base (minimal, zero CVEs) -โ”œโ”€ Static binary (no runtime dependencies) -โ””โ”€ Config via volume mount or env vars -``` +`partition_ordered` preserves source-partition processing/enqueue order during a +stable assignment. It does not force source and target partition identity, add +Kafka transactions, or fence work across a consumer-group rebalance. -### Kubernetes +## Module map -``` -Deployment -โ”œโ”€ ConfigMap (configuration) -โ”œโ”€ Secret (credentials) -โ”œโ”€ Service (metrics endpoint) -โ””โ”€ HPA (auto-scaling) -``` - -### Monitoring - -- **Metrics**: Built-in stats reporter -- **Logs**: Structured logging via tracing -- **Traces**: OpenTelemetry compatible - ---- - -## Module Organization (v1.0.0-alpha.1) - -``` +```text src/ -โ”œโ”€โ”€ main.rs # Entry point, tokio runtime setup -โ”œโ”€โ”€ lib.rs # Public API exports -โ”œโ”€โ”€ config.rs # YAML/JSON configuration parsing -โ”œโ”€โ”€ error.rs # Error types (โš ๏ธ currently string-based) -โ”‚ -โ”œโ”€โ”€ processor.rs # Message processing traits (~500 lines) -โ”œโ”€โ”€ filter_parser.rs # DSL parser (~1800 lines) -โ”‚ -โ”œโ”€โ”€ filter/ -โ”‚ โ”œโ”€โ”€ mod.rs # Filter and Transform traits -โ”‚ โ”œโ”€โ”€ envelope_filter.rs # Envelope-aware filters -โ”‚ โ””โ”€โ”€ envelope_transform.rs # Envelope transformations -โ”‚ -โ”œโ”€โ”€ kafka/ -โ”‚ โ”œโ”€โ”€ mod.rs # Kafka client abstractions -โ”‚ โ””โ”€โ”€ sink.rs # Producer wrapper (~300 lines) -โ”‚ -โ”œโ”€โ”€ envelope.rs # MessageEnvelope struct -โ”œโ”€โ”€ partitioner.rs # Partitioning strategies -โ”œโ”€โ”€ compression.rs # Compression codec support -โ”‚ -โ”œโ”€โ”€ cache.rs # Cache trait -โ”œโ”€โ”€ cache_backend.rs # Cache implementations (~600 lines) -โ”œโ”€โ”€ hash.rs # Hashing functions (MD5/SHA/Murmur) -โ”‚ -โ””โ”€โ”€ observability/ - โ”œโ”€โ”€ mod.rs # Observability exports - โ”œโ”€โ”€ metrics.rs # Prometheus metric definitions - โ”œโ”€โ”€ server.rs # HTTP metrics endpoint - โ””โ”€โ”€ lag_monitor.rs # Consumer lag tracking +โ”œโ”€โ”€ main.rs runtime setup, consumer loop, commits +โ”œโ”€โ”€ lib.rs public exports +โ”œโ”€โ”€ config.rs typed configuration and validation +โ”œโ”€โ”€ envelope.rs current JSON message envelope +โ”œโ”€โ”€ partition_pipeline.rs bounded source-partition worker lanes +โ”œโ”€โ”€ processor.rs destination runtime and routing +โ”œโ”€โ”€ filter_parser.rs DSL construction and compiled evaluator +โ”œโ”€โ”€ dsl/ function-style parser and AST +โ”œโ”€โ”€ filter/ filters and transforms +โ”œโ”€โ”€ kafka/sink.rs producer wrapper and serialization +โ”œโ”€โ”€ kafka/sink/delivery.rs bounded asynchronous delivery tracking +โ”œโ”€โ”€ partitioner.rs keyed and field partition decisions +โ”œโ”€โ”€ aggregation.rs windowed aggregation +โ”œโ”€โ”€ cache.rs cache interfaces +โ”œโ”€โ”€ cache_backend.rs cache implementations +โ”œโ”€โ”€ retry.rs retry policy +โ”œโ”€โ”€ dlq.rs dead-letter queue +โ”œโ”€โ”€ metrics.rs processing metrics +โ””โ”€โ”€ observability/ HTTP metrics and lag monitoring ``` -**Total:** ~15,638 lines of Rust code (as of v0.4.0) - -### Key Module Dependencies - -- **filter_parser.rs** โ†’ serde_json, regex (no AST layer yet) -- **processor.rs** โ†’ filter/, kafka/sink -- **kafka/sink.rs** โ†’ rdkafka, compression -- **observability/** โ†’ prometheus, axum -- **cache_backend.rs** โ†’ moka, redis (optional), dashmap - -## v1.0 Roadmap and Known Gaps - -### Phase 1: Core Engine Hardening (IN PROGRESS) - -**Critical gaps blocking v1.0:** - -1. **Error Type System** (`src/error.rs`) - - Currently: String-based errors (`anyhow::Error`) - - Needed: Typed error hierarchy with context - - Deliverable: Refactored `src/error.rs` + `docs/ERROR_HANDLING.md` - -2. **Delivery Semantics** (`src/processor.rs`) - - Currently: At-least-once implicit, no tests - - Needed: Explicit commit strategies, offset management tests - - Deliverable: `docs/DELIVERY_GUARANTEES.md` + integration tests - -3. **Retry and DLQ** (`src/retry.rs`, `src/dlq.rs`) - - Currently: Basic implementation, semantics undefined - - Needed: Retry policy (count, backoff), DLQ format - - Deliverable: Modules + metrics + tests - -4. **Integration Tests** (`tests/integration/`) - - Currently: Only unit tests (92 passing) - - Needed: End-to-end tests with Testcontainers - - Deliverable: 10+ integration scenarios, failure injection - -### Phase 2: DSL Stabilization - -**DSL gaps:** - -5. **Formal Grammar** (`docs/DSL_SPEC.md`) - - Currently: Informal syntax examples - - Needed: EBNF grammar, operator precedence, escaping rules - - Deliverable: Complete DSL specification - -6. **Parser Refactor** (`src/dsl/`) - - Currently: `filter_parser.rs` monolith - - Needed: Separate parser/AST/validator/evaluator - - Deliverable: `src/dsl/ast.rs`, `src/dsl/parser.rs`, `src/dsl/validator.rs` - -7. **Config Validation** (`src/bin/validate.rs`) - - Currently: No pre-deploy validation - - Needed: CLI tool to validate config files - - Deliverable: `streamforge validate config.yaml` command - -### Phase 3-6: See V1_PLAN.md - -**Phases:** -- Phase 3: Envelope/Enrichment/Runtime Maturity -- Phase 4: Operability and Deployment -- Phase 5: UI/Operator Polish -- Phase 6: v1.0 Release Readiness - -**Estimated total:** ~30 hours of autonomous execution - -## Future Architecture Considerations (Post v1.0) - -### Planned Improvements - -1. **Exactly-Once Semantics** - - Transactional producers (Kafka 3.3+) - - Idempotent writes - - EOS integration tests - -2. **Dynamic Reconfiguration** - - Reload config without restart - - Add/remove destinations at runtime - -3. **Advanced Routing** - - Content-based routing with complex rules - - Priority queues for message ordering - -4. **Enhanced Observability** - - Distributed tracing with trace IDs - - OpenTelemetry integration - - Grafana dashboard templates - -5. **Advanced Caching** - - Additional cache backends (Memcached, DynamoDB) - - TTL-based expiration - - Cache warming strategies - ---- - -## References - -### Documentation -- [Implementation Notes](docs/IMPLEMENTATION_NOTES.md) - Technical implementation details -- [Performance Guide](docs/PERFORMANCE.md) - Performance tuning -- [Scaling Guide](docs/SCALING.md) - Scaling strategies +## Verification boundaries -### Benchmarks -- [Concurrent Processing Results](benchmarks/results/CONCURRENT_PROCESSING_RESULTS.md) -- [Scaling Test Results](benchmarks/results/SCALING_TEST_RESULTS.md) -- [Comprehensive Benchmarks](benchmarks/results/BENCHMARKS.md) +Unit tests exercise configuration, DSL, transform, processor, and partitioning +semantics. Criterion targets cover isolated filter, transform, and end-to-end +code paths. Kafka integration results require a reproducible broker environment +and are not inferred from unit or microbenchmark success. -### External -- [Apache Kafka Documentation](https://kafka.apache.org/documentation/) -- [rdkafka-rust](https://github.com/fede1024/rust-rdkafka) -- [Tokio](https://tokio.rs/) +## Related documents ---- +- `PROJECT_SPEC.md` โ€” product scope and typed-envelope direction +- `ROADMAP.md` โ€” planned work +- `docs/IMPLEMENTATION_STATUS.md` โ€” verified capability status +- `docs/PERFORMANCE.md` โ€” tuning and benchmark method +- `docs/DELIVERY_GUARANTEES.md` โ€” commit and failure semantics -**Last Updated:** April 2026 -**Version:** 1.0.0-alpha.1 +**Last updated:** 2026-07-24 diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 701c57a..1ccbd33 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -1,545 +1,15 @@ # StreamForge Benchmarks -**Version:** 1.0.0 -**Last Updated:** 2026-04-18 +The canonical benchmark instructions are in +[`docs/PERFORMANCE_TESTING.md`](docs/PERFORMANCE_TESTING.md). -This document describes the benchmark infrastructure for StreamForge, including micro-benchmarks, end-to-end performance tests, and results analysis. +StreamForge distinguishes: ---- +- Criterion microbenchmarks for isolated code paths; +- synthetic pipeline benchmarks that do not include Kafka; +- Kafka-backed completion-throughput runs produced by + `scripts/benchmarks/run_throughput_test.sh`. -## Table of Contents - -1. [Overview](#overview) -2. [Micro-Benchmarks (Criterion)](#micro-benchmarks-criterion) -3. [End-to-End Performance Tests](#end-to-end-performance-tests) -4. [Running Benchmarks](#running-benchmarks) -5. [Benchmark Configurations](#benchmark-configurations) -6. [Results and Analysis](#results-and-analysis) -7. [Performance Targets](#performance-targets) - ---- - -## Overview - -StreamForge uses two types of benchmarks: - -### 1. Micro-Benchmarks (`benches/`) - -**Purpose:** Measure individual component performance (filters, transforms, parsers) -**Tool:** [Criterion.rs](https://github.com/bheisler/criterion.rs) -**Location:** `benches/*.rs` -**Run time:** Seconds -**Output:** Statistical analysis (mean, median, std dev) - -**Use when:** -- Optimizing a specific filter or transform -- Validating performance regression in DSL parser -- Comparing alternative implementations - -### 2. End-to-End Performance Tests (`examples/benchmarks/`, `scripts/benchmarks/`) - -**Purpose:** Measure full pipeline throughput, latency, and scaling -**Tool:** Custom scripts + Kafka + Prometheus -**Location:** `examples/benchmarks/*.yaml`, `scripts/benchmarks/*.sh` -**Run time:** Minutes to hours -**Output:** Throughput (msg/s), latency (p50/p95/p99), resource usage - -**Use when:** -- Validating production performance -- Testing scaling behavior (threads, partitions) -- Measuring end-to-end latency -- Load testing with realistic data - ---- - -## Micro-Benchmarks (Criterion) - -### Available Benchmarks - -Located in `benches/`: - -| File | Benchmarks | What It Measures | -|------|-----------|------------------| -| `filter_benchmarks.rs` | Simple filters, boolean logic, regex, arrays, parser | Filter evaluation performance | -| `transform_benchmarks.rs` | JSON extraction, CONSTRUCT, arrays, arithmetic, parser | Transform performance | - -### Running Micro-Benchmarks - -**Run all benchmarks:** -```bash -cargo bench -``` - -**Run specific benchmark:** -```bash -cargo bench filter/simple_numeric_gt -cargo bench transform/extract_field -``` - -**Run with output:** -```bash -cargo bench -- --verbose -``` - -**Compare against baseline:** -```bash -# Save baseline -cargo bench -- --save-baseline main - -# Make changes... - -# Compare -cargo bench -- --baseline main -``` - -### Example Output - -``` -filter/simple_numeric_gt - time: [145.23 ns 146.89 ns 148.72 ns] -filter/and_two_conditions - time: [312.45 ns 315.67 ns 319.12 ns] -filter/regex_email - time: [1.2341 ยตs 1.2456 ยตs 1.2598 ยตs] -``` - -### Interpreting Results - -- **Simple filters:** ~150ns (JSON path + comparison) -- **Boolean AND/OR:** ~300-400ns (multiple filters) -- **Regex:** ~1-2ยตs (regex compilation cached) -- **Array filters:** ~500-800ns (depends on array size) -- **Parser:** ~200-500ns (depends on complexity) - -**Throughput estimation:** -- Single thread, simple filter: ~6.6M operations/sec (1/150ns) -- Single thread, complex filter: ~2-3M operations/sec -- With JSON parsing overhead: ~100K-1M msg/sec per thread - ---- - -## End-to-End Performance Tests - -### Test Types - -#### 1. Throughput Tests - -**Goal:** Maximum messages per second -**Config:** `examples/benchmarks/throughput-8thread.yaml` -**Script:** `scripts/benchmarks/run_throughput_test.sh` - -**Configuration:** -- 8 threads on 8 partitions -- Large batches (5000 messages) -- Manual commit (5 second interval) -- zstd compression -- Passthrough (no filters) - -**Expected results:** -- **Target:** 30K+ msg/s sustained -- **Peak:** 35K+ msg/s - -#### 2. Latency Tests - -**Goal:** Minimum end-to-end latency -**Config:** `examples/benchmarks/latency-optimized.yaml` -**Script:** Custom timing measurement - -**Configuration:** -- 2 threads (low contention) -- Small batches (100 messages) -- Per-message commit -- No compression -- Passthrough - -**Expected results:** -- **p50:** < 5ms -- **p95:** < 10ms -- **p99:** < 20ms - -#### 3. Filter/Transform Performance - -**Goal:** DSL performance under load -**Config:** `examples/benchmarks/filter-transform.yaml` -**Script:** `scripts/benchmarks/run_throughput_test.sh` - -**Configuration:** -- 4 threads -- Various filters (simple, complex, regex, array) -- CONSTRUCT transforms -- Time-based commit (1 second) - -**Expected results:** -- **Simple filter:** ~50K msg/s -- **Complex filter:** ~20K msg/s -- **Regex filter:** ~10K msg/s -- **With CONSTRUCT:** ~15K msg/s - ---- - -## Running Benchmarks - -### Prerequisites - -**1. Start Kafka:** -```bash -docker-compose -f docker-compose.benchmark.yml up -d -``` - -**2. Create topics:** -```bash -# 8 partition topics for throughput tests -kafka-topics --create --topic test-8p-input --partitions 8 --replication-factor 1 --bootstrap-server localhost:9092 - -kafka-topics --create --topic test-8p-output --partitions 8 --replication-factor 1 --bootstrap-server localhost:9092 - -# Single partition topics for latency tests -kafka-topics --create --topic test-input --partitions 1 --replication-factor 1 --bootstrap-server localhost:9092 - -kafka-topics --create --topic test-output --partitions 1 --replication-factor 1 --bootstrap-server localhost:9092 -``` - -### Run Micro-Benchmarks - -```bash -# All benchmarks -cargo bench - -# Filter benchmarks only -cargo bench filter - -# Transform benchmarks only -cargo bench transform - -# Save baseline -cargo bench -- --save-baseline v1.0.0 - -# Generate HTML report -cargo bench -- --plotting-backend plotters -open target/criterion/report/index.html -``` - -### Run Throughput Tests - -**Quick test (200K messages):** -```bash -cd scripts/benchmarks -./run_throughput_test.sh -``` - -**Custom test (1M messages at 50K msg/s target):** -```bash -./run_throughput_test.sh 1000000 50000 -``` - -**With specific config:** -```bash -cd ../.. -cargo build --release -./target/release/streamforge --config examples/benchmarks/throughput-8thread.yaml -``` - -### Run Observability Tests - -**With Prometheus monitoring:** -```bash -cd scripts/benchmarks -./run_observability_test.sh -``` - -**Manual monitoring:** -```bash -# Terminal 1: Run StreamForge -cargo run --release -- --config examples/benchmarks/throughput-8thread.yaml - -# Terminal 2: Watch metrics -watch -n 1 'curl -s localhost:8080/metrics | grep -E "(messages_consumed|messages_produced|consumer_lag)"' - -# Terminal 3: Generate load -./scripts/benchmarks/generate_json_test_data.sh test-8p-input 100000 -``` - ---- - -## Benchmark Configurations - -All benchmark configs are in `examples/benchmarks/`: - -| Config | Threads | Batching | Commit | Use Case | -|--------|---------|----------|--------|----------| -| `throughput-8thread.yaml` | 8 | Large (5000) | Manual (5s) | Max throughput | -| `latency-optimized.yaml` | 2 | Small (100) | Per-message | Min latency | -| `filter-transform.yaml` | 4 | Medium (2000) | Time-based (1s) | DSL performance | - -### Customizing Configs - -**For higher throughput:** -```yaml -threads: 16 # More parallelism -performance: - batch_size: 10000 # Larger batches - linger_ms: 100 # More batching -commit_interval_ms: 10000 # Less frequent commits -``` - -**For lower latency:** -```yaml -threads: 1 # No contention -performance: - batch_size: 10 # Tiny batches - linger_ms: 0 # Send immediately -commit_strategy: "per-message" # Commit every message -``` - -**For testing filters:** -```yaml -routing: - routing_type: "filter" - destinations: - - output: "filtered" - filter: "YOUR_FILTER_HERE" - transform: "YOUR_TRANSFORM_HERE" -``` - ---- - -## Results and Analysis - -### Historical Results - -Benchmark results and analysis are in `docs/benchmarks/results/`: - -| Document | Content | -|----------|---------| -| `BENCHMARK_RESULTS.md` | Initial benchmark results | -| `BENCHMARKS.md` | Comprehensive benchmark analysis | -| `CONCURRENT_PROCESSING_RESULTS.md` | 132x improvement from concurrent processing | -| `SCALING_TEST_RESULTS.md` | Linear scaling validation (8 threads) | -| `DELIVERY_SEMANTICS_IMPLEMENTATION.md` | At-least-once vs at-most-once comparison | - -### Key Historical Results - -**Throughput improvements (0.x โ†’ 1.0):** -- **Sequential baseline:** 83 msg/s -- **Optimized sequential:** 3,000 msg/s (36x) -- **Concurrent (4 threads):** 10,933 msg/s (132x) -- **Concurrent (8 threads):** 25,000-30,000 msg/s sustained -- **Peak:** 34,517 msg/s - -**Scaling:** -- 4 threads โ†’ 8 threads: **2.0x improvement** (perfect linear scaling) -- Validates architecture scales with CPU cores - -**Delivery semantics:** -- **At-least-once (manual commit):** 10,933 msg/s -- **At-most-once (auto-commit):** 11,200 msg/s (<3% overhead) - -### Analyzing Your Results - -**1. Check throughput:** -```bash -# Messages consumed per second -curl -s localhost:8080/metrics | grep messages_consumed_total - -# Calculate rate -# (current - previous) / time_elapsed -``` - -**2. Check latency:** -```bash -# Processing duration histogram -curl -s localhost:8080/metrics | grep processing_duration_seconds - -# p95 latency -histogram_quantile(0.95, rate(streamforge_processing_duration_seconds_bucket[5m])) -``` - -**3. Check lag:** -```bash -curl -s localhost:8080/metrics | grep consumer_lag - -# Or via Kafka -kafka-consumer-groups --bootstrap-server localhost:9092 \ - --describe --group benchmark-8thread -``` - -**4. Check errors:** -```bash -curl -s localhost:8080/metrics | grep errors_total - -# Error rate -rate(streamforge_errors_total[5m]) -``` - ---- - -## Performance Targets - -### v1.0 Targets - -| Metric | Target | Config | -|--------|--------|--------| -| **Throughput (passthrough)** | 30K msg/s | 8 threads, 8 partitions, no filters | -| **Throughput (simple filter)** | 20K msg/s | 4 threads, JSON path filter | -| **Throughput (complex filter)** | 10K msg/s | 4 threads, AND + CONSTRUCT | -| **Latency (p95)** | < 10ms | Per-message commit, no batching | -| **Latency (p99)** | < 20ms | Per-message commit | -| **Filter evaluation** | < 200ns | Simple JSON path comparison | -| **Transform evaluation** | < 500ns | Simple extraction | -| **Parser** | < 300ns | Simple filter parse | - -### Scaling Targets - -| CPUs | Threads | Expected Throughput | -|------|---------|---------------------| -| 2 | 2 | ~7.5K msg/s | -| 4 | 4 | ~15K msg/s | -| 8 | 8 | ~30K msg/s | -| 16 | 16 | ~60K msg/s | - -**Assumptions:** -- Linear scaling with CPU cores -- 8+ partitions (no partition bottleneck) -- Simple filters or passthrough -- Adequate Kafka broker performance - ---- - -## Continuous Benchmarking - -### CI/CD Integration - -**GitHub Actions example:** -```yaml -name: Benchmark - -on: - push: - branches: [main] - pull_request: - -jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Run micro-benchmarks - run: cargo bench -- --save-baseline ${{ github.sha }} - - - name: Compare with main - if: github.event_name == 'pull_request' - run: | - cargo bench -- --baseline main - # Fail if > 10% regression -``` - -### Regression Detection - -**Compare baselines:** -```bash -# Save current as baseline -cargo bench -- --save-baseline current - -# Make changes... - -# Compare -cargo bench -- --baseline current - -# Look for regressions -# "Performance has regressed" = slower than baseline -# "Performance has improved" = faster than baseline -``` - -**Automatic regression check:** -```bash -#!/bin/bash -# benchmark-check.sh - -cargo bench -- --baseline main > bench-results.txt - -if grep -q "Performance has regressed" bench-results.txt; then - echo "โŒ Performance regression detected!" - exit 1 -else - echo "โœ… No performance regression" - exit 0 -fi -``` - ---- - -## Troubleshooting - -### Low Throughput - -**Check:** -1. CPU usage: `htop` or `top` -2. Consumer lag: `kafka-consumer-groups --describe` -3. Thread count: Match CPU cores -4. Batch sizes: Increase for throughput -5. Commit interval: Less frequent commits - -**Fix:** -```yaml -threads: 8 # Match CPU cores -performance: - batch_size: 5000 - linger_ms: 50 -commit_interval_ms: 10000 -``` - -### High Latency - -**Check:** -1. Batch sizes: Too large -2. Linger time: Too long -3. Commit strategy: Per-message vs batched -4. Filter complexity: Regex is slow - -**Fix:** -```yaml -threads: 2 # Reduce contention -performance: - batch_size: 10 - linger_ms: 0 -commit_strategy: "per-message" -``` - -### Inconsistent Results - -**Causes:** -- Background processes (close Chrome, Slack, etc.) -- CPU throttling (run on AC power) -- Insufficient warm-up (criterion does auto-warmup) -- Network latency (use local Kafka) - -**Fix:** -```bash -# Minimal system load -systemctl stop unnecessary-services - -# Fixed CPU frequency -sudo cpupower frequency-set --governor performance - -# Longer benchmark run -cargo bench -- --measurement-time 30 -``` - ---- - -## Next Steps - -- **Baseline:** Run `cargo bench` to establish v1.0 baseline -- **Monitor:** Run end-to-end tests weekly -- **Optimize:** Focus on regressions > 10% -- **Document:** Add results to `docs/benchmarks/results/` - ---- - -**Version:** 1.0.0 -**Last Updated:** 2026-04-18 -**See Also:** -- [Performance Tuning Guide](docs/PERFORMANCE_TUNING_RESULTS.md) -- [Deployment Guide](docs/DEPLOYMENT.md#performance-tuning) -- [Operations Runbook](docs/OPERATIONS.md#performance-optimization) +Historical reports under `docs/benchmarks/results/` are retained as historical +artifacts. They are not current baselines unless the workload, environment, +configuration, and measurement method match. diff --git a/CHANGELOG.md b/CHANGELOG.md index deef1f5..78bac1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,11 +131,11 @@ StreamForge v1.0.0 is the first production-ready release. This release focuses o - 3 complete example configurations in schema - **Production Examples** ([`examples/production/`](examples/production/)) - - `user-filtering.yaml`: Multi-destination routing (~50K msg/s) - - `cross-region-replication.yaml`: DR replication (~100K msg/s) - - `cdc-to-datalake.yaml`: Database CDC streaming (~20K msg/s) - - `multi-tenant-filtering.yaml`: Tenant routing (~30K msg/s) - - `pii-redaction.yaml`: Data masking and PII redaction (~15K msg/s) + - `user-filtering.yaml`: Multi-destination routing + - `cross-region-replication.yaml`: DR replication + - `cdc-to-datalake.yaml`: Database CDC streaming + - `multi-tenant-filtering.yaml`: Multi-tenant routing + - `pii-redaction.yaml`: Data minimization and pseudonymization - `README.md`: Production examples guide with tuning and deployment instructions #### Testing @@ -215,7 +215,7 @@ The `streamforge-validate` CLI will warn about deprecated syntax. - **Full generic `Envelope` implementation** deferred to v1.1 - Reason: 20-30 hours of work, high risk, touches entire codebase - v1.0: Documentation complete, runtime type awareness planned - - v1.1: Full implementation for 3-4x performance gains + - v1.1: Evaluate the implementation with the reproducible benchmark harness - See: [`docs/TYPED_ENVELOPE_DESIGN.md`](docs/TYPED_ENVELOPE_DESIGN.md) and [`docs/PHASE_3_PRAGMATIC_APPROACH.md`](docs/PHASE_3_PRAGMATIC_APPROACH.md) #### Parser Refactor @@ -398,7 +398,7 @@ See [`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md#performance-tuning) for guidance. ## What's Next (v1.1 Roadmap) ### Planned Features -1. **Typed Envelope System** - Generic `Envelope` for 3-4x performance +1. **Typed Envelope System** - Generic `Envelope` evaluated with the reproducible benchmark harness 2. **Parser Refactor** - Better error messages, AST-based validation 3. **Redis Cache Backend** - Distributed caching for enrichment 4. **Trace Correlation** - End-to-end message tracing with OpenTelemetry diff --git a/PERFORMANCE_OPTIMIZATIONS.md b/PERFORMANCE_OPTIMIZATIONS.md index 415487d..0849f2c 100644 --- a/PERFORMANCE_OPTIMIZATIONS.md +++ b/PERFORMANCE_OPTIMIZATIONS.md @@ -1,429 +1,16 @@ -# Performance Optimizations Summary +# Performance Optimizations -**Date:** 2026-04-18 -**Baseline Throughput:** 25K-45K msg/s -**Expected Throughput:** 70K-120K msg/s -**Expected Improvement:** +180-265% (2.8x-3.6x throughput) +This former optimization report was retired on 2026-07-24 because it mixed +unverified estimates, historical measurements, and current implementation +status. ---- +Use the maintained sources of truth: -## Phase 1: Quick Wins (Completed) +- `docs/PERFORMANCE.md` โ€” tuning guidance and optimization boundaries +- `docs/PERFORMANCE_TESTING.md` โ€” reproducible benchmark methodology +- `docs/IMPLEMENTATION_STATUS.md` โ€” verified implementation and test status +- `ROADMAP.md` โ€” planned performance work -### โœ… Task #5: Pre-resolve Prometheus Metrics -**Files Modified:** `src/processor.rs`, `src/observability/metrics.rs` - -**Changes:** -- Pre-resolve metrics with labels at processor construction time -- Store direct Counter/Histogram references instead of looking up in HashMap on every message -- Eliminated 15-20 HashMap lookups per message - -**Impact:** 5-12% throughput improvement - -**Details:** -```rust -// Before (hot path): -METRICS.filter_evaluations.with_label_values(&[name, "pass"]).inc(); - -// After (construction time): -let filter_pass_counter = METRICS - .filter_evaluations - .with_label_values(&[name.as_str(), labels::FILTER_RESULT_PASS]); - -// Hot path (no HashMap lookup): -self.filter_pass_counter.inc(); -``` - ---- - -### โœ… Task #8: Pre-parse JSON Paths -**Files Modified:** `src/filter/mod.rs`, `src/filter/envelope_transform.rs` - -**Changes:** -- Pre-parse JSON path segments at construction time -- Added `path_segments: Vec` field to all filter/transform structs -- Eliminated `path.trim_matches('/').split('/').collect()` allocation on every message - -**Structs Updated:** -1. JsonPathFilter -2. JsonPathTransform -3. RegexFilter -4. ObjectConstructTransform -5. ArrayFilter -6. ArrayMapTransform -7. ArithmeticTransform -8. HashTransform -9. CacheLookupTransform -10. CachePutTransform -11. KeyFromTransform -12. KeyHashTransform -13. KeyConstructTransform -14. HeaderFromTransform -15. TimestampFromTransform - -**Impact:** 3-8% throughput improvement - -**Pattern:** -```rust -// Before: -fn extract_value(&self, value: &Value) -> Option { - let parts: Vec<&str> = self.path.trim_matches('/').split('/').collect(); - // ... traverse with parts -} - -// After: -pub fn new(path: &str) -> Result { - let path_segments: Vec = path - .trim_matches('/') - .split('/') - .map(|s| s.to_string()) - .collect(); - Ok(Self { path: path.to_string(), path_segments }) -} - -fn extract_value(&self, value: &Value) -> Option { - // ... traverse with self.path_segments (no allocation) -} -``` - ---- - -### โœ… Task #9: Skip Retry Wrapper -**Files Modified:** `src/main.rs` - -**Changes:** -- When `max_attempts == 1`, use base processor directly -- Skip ProcessorWithRetry wrapper overhead on happy path - -**Impact:** 1-3% throughput improvement (for non-retry scenarios) - -**Code:** -```rust -let processor: Arc = if config.retry.max_attempts == 1 { - info!("Retry disabled (max_attempts=1) - using base processor directly"); - base_processor -} else { - Arc::new(ProcessorWithRetry::new( - base_processor, - retry_policy, - dlq, - config.appid.clone(), - )) as Arc -}; -``` - ---- - -### โœ… Task #6: Arc-wrapped Envelope Fields -**Files Modified:** `src/envelope.rs`, `src/processor.rs`, `src/kafka/sink.rs`, `src/dlq.rs`, `src/filter/envelope_transform.rs`, `src/main.rs` - -**Changes:** -- Wrapped `MessageEnvelope.value` in `Arc` -- Wrapped `MessageEnvelope.headers` in `Arc>>` -- Multi-destination cloning now just increments reference count instead of deep copying - -**Impact:** Major improvement for multi-destination routing (5KB value ร— 4 clones โ†’ 5KB value + 4 ref increments) - -**Before/After:** -```rust -// Before: -pub struct MessageEnvelope { - pub value: Value, // Deep clone for each destination - pub headers: HashMap>, // Deep clone -} - -// After: -pub struct MessageEnvelope { - pub value: Arc, // Cheap ref count increment - pub headers: Arc>>, // Cheap ref count increment -} -``` - -**Single-destination optimization:** -```rust -// Unwrap Arc to get owned Value (cheap if no other references) -let value_owned = Arc::try_unwrap(envelope.value) - .unwrap_or_else(|arc| (*arc).clone()); -``` - ---- - -### Skipped: Task #4 (Carry Raw Bytes) -**Status:** Not implemented (too invasive, requires architecture changes) - -**Reason:** Would require: -- MessageEnvelope to support both parsed Value and raw bytes -- Conditional parsing based on filter/transform requirements -- Sink to handle both parsed and raw payloads -- Major changes to the processing pipeline - -**Future Consideration:** For pure pass-through scenarios (no filter/transform), this could yield 30-40% improvement by skipping JSON parsing entirely. - ---- - -## Phase 2: Medium Effort (Completed) - -### โœ… Task #3: Concurrent Destination Processing -**Files Modified:** `src/processor.rs` - -**Changes:** -- Changed `MultiDestinationProcessor` from sequential to concurrent processing -- Uses `futures::join_all` to process all destinations in parallel -- Envelope cloning is cheap now (Arc-wrapped from Task #6) - -**Impact:** 15-25% improvement for multi-destination scenarios - -**Before:** -```rust -// Sequential processing -for dest in self.destinations.iter() { - dest.process(envelope.clone()).await?; -} -``` - -**After:** -```rust -// Concurrent processing -let futures: Vec<_> = self - .destinations - .iter() - .map(|dest| { - let env = envelope.clone(); - async move { (dest.name.clone(), dest.process(env).await) } - }) - .collect(); - -let results = futures::future::join_all(futures).await; -``` - ---- - -### โœ… Task #7: Thread-local Serialization Buffers -**Files Modified:** `src/kafka/sink.rs`, `src/dlq.rs` - -**Changes:** -- Added `thread_local! SERIALIZE_BUFFER` with 4KB pre-allocated capacity -- Reuse buffer instead of allocating new Vec on every serialization -- Updated 4 serialization call sites - -**Impact:** 3-7% improvement (reduces allocations on hot path) - -**Code:** -```rust -thread_local! { - static SERIALIZE_BUFFER: RefCell> = RefCell::new(Vec::with_capacity(4096)); -} - -fn serialize_to_vec(value: &T) -> Result> { - SERIALIZE_BUFFER.with(|buf_cell| { - let mut buf = buf_cell.borrow_mut(); - buf.clear(); - serde_json::to_writer(&mut *buf, value)?; - Ok(buf.clone()) - }) -} -``` - ---- - -### Skipped: Task #10 (Batch Metric Updates) -**Status:** Not implemented (complex in async context, Task #5 already addressed main bottleneck) - -**Reason:** -- Pre-resolution (Task #5) eliminated HashMap lookup overhead -- Batching would add complexity for marginal gain -- Async context makes batching coordination difficult - ---- - -## Phase 3: Major Refactors (Completed) - -### โœ… Task #2: Extract Shared JsonPath Resolver -**Files Created:** `src/jsonpath.rs` -**Files Modified:** `src/lib.rs` - -**Changes:** -- Created shared `JsonPath` struct with pre-parsed segments -- Consolidated duplicate `extract_value` implementations -- Added type-specific extraction methods (extract_string, extract_f64, etc.) -- Provided backward-compatible helper functions - -**Impact:** Code quality improvement, sets foundation for future optimizations - -**Features:** -```rust -pub struct JsonPath { - pub path: String, // Original path for error messages - pub segments: Vec, // Pre-parsed segments -} - -impl JsonPath { - pub fn new(path: &str) -> Self; - pub fn extract<'a>(&self, value: &'a Value) -> Option<&'a Value>; - pub fn extract_owned(&self, value: &Value) -> Option; - pub fn extract_string(&self, value: &Value) -> Option; - pub fn extract_f64(&self, value: &Value) -> Option; - pub fn extract_i64(&self, value: &Value) -> Option; - pub fn extract_bool(&self, value: &Value) -> Option; -} -``` - -**Tests:** 8 new tests, all passing - ---- - -### Skipped: Task #1 (Replace serde_json with simd-json) -**Status:** Not implemented (too invasive, moderate gain) - -**Reason:** -- Extremely invasive (affects 100+ files) -- Requires changing Value type throughout codebase -- simd-json requires mutable input buffers -- Parsing is not the main bottleneck (only happens once on input) -- We serialize more than parse, and thread-local buffers (Task #7) already optimized that - -**Expected Gain:** 10-15% on JSON parsing (but parsing isn't the bottleneck) - ---- - -## Summary of Completed Optimizations - -### Phase 1 (Quick Wins) -| Task | Status | Impact | Effort | -|------|--------|--------|--------| -| #5 Pre-resolve Prometheus metrics | โœ… | 5-12% | 1h | -| #8 Pre-parse JSON paths | โœ… | 3-8% | 2h | -| #9 Skip retry wrapper | โœ… | 1-3% | 0.5h | -| #6 Arc-wrapped envelope | โœ… | High (multi-dest) | 1.5h | -| #4 Carry raw bytes | โŒ | (skipped) | - | - -**Phase 1 Total:** +9-23% base improvement (single dest) to +25-45% (multi-dest) - -### Phase 2 (Medium Effort) -| Task | Status | Impact | Effort | -|------|--------|--------|--------| -| #3 Concurrent destinations | โœ… | 15-25% | 1h | -| #7 Thread-local buffers | โœ… | 3-7% | 1h | -| #10 Batch metrics | โŒ | (skipped) | - | - -**Phase 2 Total:** +18-32% improvement - -### Phase 3 (Major Refactors) -| Task | Status | Impact | Effort | -|------|--------|--------|--------| -| #2 Shared JsonPath resolver | โœ… | Code quality | 1h | -| #1 simd-json | โŒ | (skipped) | - | -| #4 Raw bytes | โŒ | (skipped) | - | - -**Phase 3 Total:** Foundation for future optimizations - ---- - -## Overall Impact - -### Conservative Estimate -- Phase 1: +15% (single dest) to +35% (multi-dest) -- Phase 2: +20% -- **Total: +40-60% throughput improvement** -- **From:** 25K-45K msg/s -- **To:** 50K-70K msg/s - -### Optimistic Estimate -- Phase 1: +25% (single dest) to +50% (multi-dest) -- Phase 2: +30% -- **Total: +60-80% throughput improvement** -- **From:** 25K-45K msg/s -- **To:** 70K-100K msg/s - -### Best Case (Multi-destination, High Arc Benefit) -- Phase 1: +45% -- Phase 2: +35% -- **Total: +90-110% throughput improvement** -- **From:** 25K-45K msg/s -- **To:** 80K-120K msg/s - ---- - -## Test Results - -**Build:** โœ… Success -**Tests:** โœ… 349 passing (341 existing + 8 new jsonpath tests) -**Warnings:** 4 (unused `path` fields kept for error messages) - ---- - -## Recommended Next Steps - -### Immediate (Production Ready) -1. **Benchmark actual throughput** on representative workload -2. **Monitor memory usage** with Arc-wrapped envelopes -3. **Verify concurrent processing** doesn't exceed connection limits - -### Future Optimizations (If Needed) -1. **Task #4 (Raw bytes pass-through):** For pure mirror scenarios, skip JSON parsing entirely - - Expected gain: +30-40% - - Effort: High (architecture changes) - - Use case: When 80%+ messages are pass-through without filter/transform - -2. **Task #1 (simd-json):** Replace serde_json with simd-json for faster parsing - - Expected gain: +10-15% on parsing - - Effort: Very high (invasive, 100+ file changes) - - Risk: High (API differences, potential bugs) - -3. **Compiled JsonPath:** Pre-compile paths to eliminate runtime string comparisons - - Expected gain: +5-8% - - Effort: Medium - - Builds on: Task #2 (JsonPath infrastructure already in place) - -4. **Zero-copy deserialization:** Use serde's zero-copy features - - Expected gain: +10-15% - - Effort: High - - Requires: Lifetime changes throughout codebase - -5. **Custom partitioner caching:** Cache partition count lookups - - Expected gain: +2-5% - - Effort: Low - ---- - -## Benchmarking Recommendations - -To measure actual improvement: - -```bash -# 1. Checkout baseline (before optimizations) -git checkout -cargo build --release - -# 2. Run benchmark (record baseline) -./benchmarks/performance-test.sh > baseline.txt - -# 3. Checkout optimized version -git checkout main -cargo build --release - -# 4. Run benchmark (record optimized) -./benchmarks/performance-test.sh > optimized.txt - -# 5. Compare -diff baseline.txt optimized.txt -``` - -**Key Metrics to Track:** -- Messages per second (throughput) -- P50, P95, P99 latency -- CPU usage % -- Memory usage (RSS) -- GC pressure (if any) -- Kafka producer queue size - ---- - -## Code Quality Impact - -- **Lines Added:** ~500 (jsonpath module, thread-local buffers, concurrent processing) -- **Lines Modified:** ~200 (Arc wrappers, pre-parsing, pre-resolution) -- **Lines Removed:** ~50 (duplicate code consolidated) -- **New Tests:** 8 (jsonpath module) -- **Test Coverage:** Maintained at 100% for modified code - -**No Breaking Changes:** All optimizations are internal implementation details. +Historical benchmark artifacts remain under `docs/benchmarks/results/` and must +not be treated as a current baseline unless their environment and method match a +new run. diff --git a/PERFORMANCE_TUNING_RESULTS.md b/PERFORMANCE_TUNING_RESULTS.md index c86ab9b..53da24a 100644 --- a/PERFORMANCE_TUNING_RESULTS.md +++ b/PERFORMANCE_TUNING_RESULTS.md @@ -1,279 +1,14 @@ -# Kafka Consumer Performance Tuning Results +# Performance Tuning Results -## Date: 2026-04-16 +This legacy report was retired on 2026-07-24. Its fixed throughput claims and +configuration examples no longer matched the executable configuration or the +current benchmark harness. -## Problem Identified +Current guidance is maintained in: -The `rdkafka` consumer was configured with `fetch_min_bytes=1`, causing: -- **Excessive round-trips** between consumer and broker -- **Backpressure** on the consumer side -- **Poor batching** - broker responding with single messages +- `docs/PERFORMANCE.md` for tuning +- `docs/PERFORMANCE_TESTING.md` for measurements and baselines +- `docs/IMPLEMENTATION_STATUS.md` for verified status -### Root Cause -```rust -// BEFORE (src/config.rs:878-880) - SLOW -fn default_fetch_min_bytes() -> u32 { - 1 // Broker responds immediately with even 1 byte -} - -fn default_fetch_max_wait_ms() -> u32 { - 100 // Only 100ms for broker to accumulate -} -``` - -This is a **classic anti-pattern** in Kafka consumers, discussed in: -- https://oneuptime.com/blog/post/2026-01-25-kafka-consumers-backpressure-rust/ -- https://www.reddit.com/r/rust/comments/1egfd1i/reimplemented_go_service_in_rust_throughput/ - ---- - -## Changes Made - -### Configuration Updates (src/config.rs) - -```rust -// AFTER - OPTIMIZED -fn default_fetch_min_bytes() -> u32 { - 65536 // 64KB - batch broker-side to reduce round-trips -} - -fn default_fetch_max_wait_ms() -> u32 { - 500 // Allow more time for broker to accumulate fetch_min_bytes -} -``` - -### How These Settings Work - -1. **`fetch_min_bytes: 65KB`** - - Broker accumulates ~65KB of messages before responding - - Batches multiple messages together - - Reduces network round-trips by 50-80% - -2. **`fetch_max_wait_ms: 500ms`** - - Maximum time broker waits to reach `fetch_min_bytes` - - Provides latency ceiling at low throughput - - Balances throughput vs latency - -### Consumer Configuration (src/main.rs:456-474) - -```rust -.set("fetch.min.bytes", p.fetch_min_bytes.to_string()) // 65KB -.set("fetch.wait.max.ms", p.fetch_max_wait_ms.to_string()) // 500ms -.set("max.partition.fetch.bytes", p.max_partition_fetch_bytes.to_string()) -.set("queued.max.messages.kbytes", p.queued_max_messages_kbytes.to_string()) // 512MB -``` - ---- - -## Benchmark Results - -### Test Configuration -- **Messages**: 50,000 JSON messages (~1KB each) -- **Partitions**: 8 -- **Threads**: 8 -- **Parallelism Factor**: 10 (80 concurrent produce operations) -- **Batch Size**: 1,000 messages per batch - -### Performance Metrics - -``` -Messages Consumed: 50,000 -Messages Produced: 50,000 -Total Duration: 4.54 seconds -Number of Batches: 50 -Avg Batch Duration: 90.74ms -Throughput: ~11,020 msg/s -Processing Errors: 0 -``` - -### Key Observations - -โœ… **High Throughput**: ~11K msg/s with complex JSON processing -โœ… **Zero Errors**: 100% success rate -โœ… **Efficient Batching**: ~1,000 messages per batch (matched config) -โœ… **Low Latency**: 90ms average batch processing time - ---- - -## Performance Analysis - -### Batch Processing Distribution - -From Prometheus metrics: -``` -le="0.1" โ†’ 44 batches (88%) // Under 100ms -le="0.25" โ†’ 50 batches (100%) // Under 250ms -``` - -**88% of batches processed in under 100ms** - excellent latency profile! - -### Consumer Efficiency - -The consumer is now efficiently: -1. **Batching broker-side** (fetch_min_bytes=64KB) -2. **Batching client-side** (batch_size=1000) -3. **Processing in parallel** (80 concurrent operations) - -This creates a **pipeline effect** where: -- Broker batches messages -- Consumer batches processing -- Producer batches writes -- All stages overlap (pipelining) - ---- - -## Comparison with Articles - -### Expected vs Actual - -Based on the articles you shared: - -| Metric | Expected Improvement | Actual Result | -|--------|---------------------|---------------| -| Throughput | 3-5x increase | โœ… Achieved high throughput | -| Network round-trips | 50-80% reduction | โœ… Batching working | -| Backpressure | Eliminated | โœ… Zero errors, smooth flow | -| CPU usage | Lower | โœ… Efficient batching | - ---- - -## Further Tuning Options - -If you need **even higher throughput**, you can tune these in your `config.json`: - -### Aggressive Tuning -```json -{ - "performance": { - "consumer_batch_size": 2000, // 2K messages per batch - "consumer_batch_timeout_ms": 50, // Lower timeout for faster batches - "parallelism_factor": 15, // 120 concurrent operations - - "fetch_min_bytes": 131072, // 128KB for larger broker batches - "fetch_max_wait_ms": 500, - "max_partition_fetch_bytes": 2097152, // 2MB per partition - "queued_max_messages_kbytes": 1048576 // 1GB pre-fetch buffer - } -} -``` - -### Conservative Tuning (Low Latency) -```json -{ - "performance": { - "consumer_batch_size": 500, - "consumer_batch_timeout_ms": 200, - "parallelism_factor": 8, - - "fetch_min_bytes": 32768, // 32KB (faster at low load) - "fetch_max_wait_ms": 250, // Lower ceiling for latency - "max_partition_fetch_bytes": 524288, - "queued_max_messages_kbytes": 262144 - } -} -``` - ---- - -## Trade-offs - -### Latency vs Throughput - -| Setting | Latency | Throughput | Use Case | -|---------|---------|------------|----------| -| `fetch_min_bytes=1` | Lowest (bad for HFT) | Lowest โŒ | Never use | -| `fetch_min_bytes=32KB` | Low | Good | Real-time apps | -| `fetch_min_bytes=64KB` | Medium | High โœ… | **Current (balanced)** | -| `fetch_min_bytes=128KB` | Higher | Higher | Batch processing | -| `fetch_min_bytes=1MB` | High | Highest | Data pipelines | - -### Memory Usage - -Higher `fetch_min_bytes` and `queued_max_messages_kbytes` use more memory: -- Current: ~512MB pre-fetch buffer (good for 8 partitions) -- Per partition buffer: 1MB default -- Total memory: `queued_max_messages_kbytes + (partitions * max_partition_fetch_bytes)` - ---- - -## Consumer Type Analysis - -### Yes, We Are Using StreamConsumer โœ… - -From `src/main.rs:3`: -```rust -use rdkafka::consumer::{Consumer, StreamConsumer}; -``` - -**StreamConsumer** is the correct choice because: -- โœ… Async/await friendly (Tokio integration) -- โœ… Built-in backpressure handling -- โœ… Efficient message streaming -- โœ… Works with `futures::stream::StreamExt` - -**NOT using** `BaseConsumer` which would require: -- Manual polling loops -- Blocking I/O -- Manual backpressure management - ---- - -## Recommendations - -### 1. Keep Current Defaults โœ… -The new defaults (64KB, 500ms) provide excellent balance: -- Good throughput (~11K msg/s) -- Acceptable latency (90ms avg) -- Zero data loss - -### 2. Monitor in Production -Watch these metrics: -``` -streamforge_consumer_lag # Should stay near 0 -streamforge_batch_processing_duration # p99 should be <200ms -streamforge_messages_in_flight # Should be steady, not growing -streamforge_processing_errors # Should be 0 -``` - -### 3. Tune Based on Workload - -**High Volume, Batch Processing:** -```yaml -performance: - fetch_min_bytes: 131072 # 128KB - fetch_max_wait_ms: 500 - consumer_batch_size: 2000 -``` - -**Real-time, Low Latency:** -```yaml -performance: - fetch_min_bytes: 32768 # 32KB - fetch_max_wait_ms: 200 - consumer_batch_size: 500 -``` - -### 4. Scale with Partitions -- **Rule of thumb**: `threads โ‰ˆ partitions` for best CPU utilization -- Each partition gets assigned to one consumer thread -- More partitions than threads = some threads handle multiple partitions - ---- - -## Conclusion - -โœ… **Problem Fixed**: Replaced `fetch_min_bytes=1` with `fetch_min_bytes=64KB` -โœ… **Performance Verified**: ~11K msg/s with zero errors -โœ… **Best Practices**: Following Kafka consumer tuning guidelines -โœ… **Production Ready**: Balanced defaults for most workloads - -The consumer fetch tuning changes have **eliminated the backpressure bottleneck** and enabled high-throughput message processing with `StreamConsumer`. - ---- - -## References - -1. [Kafka Consumers Backpressure in Rust](https://oneuptime.com/blog/post/2026-01-25-kafka-consumers-backpressure-rust/) -2. [Reddit: Reimplemented Go service in Rust - throughput discussion](https://www.reddit.com/r/rust/comments/1egfd1i/reimplemented_go_service_in_rust_throughput/) -3. [Confluent: Kafka Consumer Performance Tuning](https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#fetch.min.bytes) -4. [rdkafka Documentation](https://docs.rs/rdkafka/latest/rdkafka/) +Only results produced by the current deterministic harness, with its generated +manifest and configuration, qualify as a comparable Kafka-backed baseline. diff --git a/README.md b/README.md index 5b1a9e0..4289876 100644 --- a/README.md +++ b/README.md @@ -1,118 +1,101 @@ # StreamForge -> Selective replication for Kafka, with Redpanda as a compatibility target. Filter, transform, redact, and route data between topics and clusters without Kafka Connect. +> Selective replication for Kafka. Filter, transform, redact, and route records +> between topics and clusters without deploying Kafka Connect. -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Rust](https://img.shields.io/badge/rust-1.70%2B-orange.svg)](https://www.rust-lang.org) -[![Version](https://img.shields.io/badge/version-1.0.0-brightgreen.svg)](docs/CHANGELOG.md) -[![Kafka](https://img.shields.io/badge/broker-Kafka-black.svg)](#compatibility) -[![Redpanda](https://img.shields.io/badge/broker-Redpanda-red.svg)](#compatibility) +[![Version](https://img.shields.io/badge/version-1.0.0-36d1c4.svg)](CHANGELOG.md) [![CI](https://github.com/rahulbsw/streamforge/workflows/CI/badge.svg)](https://github.com/rahulbsw/streamforge/actions) +[![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-38a3ff.svg)](https://rahulbsw.github.io/streamforge/) +[![License](https://img.shields.io/badge/license-Apache--2.0-d5dde4.svg)](LICENSE) ---- +StreamForge moves only the records and fields that downstream systems need. +One source topic can feed analytics, lake, and lower-trust destinations with an +independent filter and transform for each route. -StreamForge helps data teams move only the records and fields downstream systems actually need. Instead of mirroring whole topics, StreamForge lets you filter, reshape, redact, and route messages before they land in analytics, lake, or lower-trust environments. +```text +Kafka source โ”€โ”€โ–บ filter โ”€โ”€โ–บ transform โ”€โ”€โ”ฌโ”€โ”€โ–บ analytics topic + โ””โ”€โ”€โ–บ PII-safe topic +``` -## Why Teams Use StreamForge +## Why StreamForge -- Replicate only analytics-safe fields instead of whole topics -- Split one source topic into multiple downstream topics -- Hash or drop PII before data crosses trust boundaries -- Keep the deployment surface small with a single binary, operator, and Helm chart +- Route records by payload, key, headers, and timestamps. +- Reshape events and remove or hash sensitive fields before delivery. +- Fan out one source topic into destination-specific representations. +- Run as a standalone Rust binary or through the Kubernetes operator. +- Observe delivery, errors, lag, retries, and dead-letter records with + Prometheus metrics. -## Watch StreamForge Deploy on Minikube +## Run the local demo -[![StreamForge UI demo on Minikube](docs/assets/demo/ui-minikube-demo-readme.gif)](docs/UI_MINIKUBE_DEMO.md) +Prerequisites: a Rust toolchain, Podman, and a Podman Compose provider. -This UI-driven demo shows the path most teams actually want to see first: install with Helm, create a pipeline in the browser, review the generated YAML, deploy the CRD, then verify transformed output on Kafka. +```bash +podman compose -f examples/redpanda/docker-compose.yml up -d -**[UI Demo](docs/UI_MINIKUBE_DEMO.md)** | **[5-Minute CLI Demo](#5-minute-demo)** | **[Examples](examples/README.md)** | **[Compatibility](#compatibility)** | **[Documentation Index](docs/DOCUMENTATION_INDEX.md)** +cargo run --quiet --bin streamforge-validate -- \ + examples/redpanda/selective-replication.yaml ---- +CONFIG_FILE=examples/redpanda/selective-replication.yaml \ + cargo run --release --bin streamforge +``` -## When to Use StreamForge +Keep StreamForge running, then follow the +[five-minute quickstart](docs/QUICKSTART.md) to create the topics, publish one +order, and inspect the two destination-specific outputs. -Use StreamForge when you need: -- selective replication to analytics or data lake pipelines -- PII-safe replication across environments -- topic fan-out with payload shaping -- a smaller operational footprint than Kafka Connect +## Choose a path -Do not position StreamForge as: -- a full replacement for MirrorMaker 2 active-active or offset-sync workflows -- a general-purpose stateful stream processor +| Goal | Start here | +| --- | --- | +| Understand the product boundary | [When to use StreamForge](docs/WHEN_TO_USE.md) | +| Build a selective replication pipeline | [Usage guide](docs/USAGE.md) | +| Learn the filter and transform language | [DSL reference](docs/ADVANCED_DSL_GUIDE.md) | +| Deploy with Podman or Kubernetes | [Deployment guide](docs/DEPLOYMENT.md) | +| Configure TLS and SASL | [Security configuration](docs/SECURITY_CONFIGURATION.md) | +| Operate and troubleshoot a pipeline | [Operations](docs/OPERATIONS.md) | +| Browse the complete public documentation | [StreamForge documentation](https://rahulbsw.github.io/streamforge/) | -For concrete usage patterns and configs, see [docs/USAGE.md](docs/USAGE.md) and [examples/README.md](examples/README.md). +## Deployment modes -## 5-Minute Demo +### Standalone -1. Start the local Redpanda demo broker: - ```bash - docker compose -f examples/redpanda/docker-compose.yml up -d - ``` -2. Validate the selective replication config: - ```bash - cargo run --quiet --bin streamforge-validate -- examples/redpanda/selective-replication.yaml - ``` -3. Run StreamForge with the same config: - ```bash - CONFIG_FILE=examples/redpanda/selective-replication.yaml \ - cargo run --release --bin streamforge - ``` - Leave StreamForge running in this terminal. -4. Open a second terminal and follow [docs/QUICKSTART.md](docs/QUICKSTART.md) to create the demo topics, produce a sample order, and inspect `analytics-orders` and `pii-safe-orders`. +Use the binary or container when configuration is managed directly by your +deployment system. Start with [Podman](docs/DOCKER.md). -If you want the Kubernetes + UI path instead of the local CLI path, use [docs/UI_MINIKUBE_DEMO.md](docs/UI_MINIKUBE_DEMO.md). +### Kubernetes -## Production Trust Signals +Use the operator and `StreamforgePipeline` custom resource when pipelines +should be managed declaratively. Start with +[Kubernetes](docs/KUBERNETES.md) or the +[Helm chart](helm/streamforge-operator/README.md). -- At-least-once delivery with retry and DLQ support -- Native Prometheus metrics and lag monitoring -- Kubernetes operator, Helm chart, and web UI -- Kafka-first examples for standalone configs and Kubernetes pipelines +## Compatibility and boundaries -## Compatibility +StreamForge targets Kafka-compatible brokers. Kafka is the primary target in +the current documentation; Redpanda is covered for the selective-replication +workflows exercised by this repository. -StreamForge is built for Kafka-compatible brokers. Kafka is the primary target in current docs and examples, and Redpanda is documented here as a compatibility target for the selective replication workflows covered in this repo. +StreamForge is not positioned as a replacement for MirrorMaker 2 active-active +replication and offset-sync workflows, or as a general-purpose stateful stream +processor. See [Compatibility](docs/COMPATIBILITY.md) for the tested scope. ---- +## Performance policy -## Core Capabilities - -- Content-based filtering across payload, key, headers, and timestamps -- Field extraction, reshaping, and PII hashing before downstream delivery -- Topic fan-out from one source topic to multiple destination topics -- At-least-once delivery with retry, DLQ handling, and observability hooks -- Standalone binary and Kubernetes operator deployment modes - -## Example Pipelines - -- [examples/configs/config.example.yaml](examples/configs/config.example.yaml) for a minimal standalone pipeline -- [examples/redpanda/selective-replication.yaml](examples/redpanda/selective-replication.yaml) for the validated local Redpanda selective replication demo -- [examples/production/pii-redaction.yaml](examples/production/pii-redaction.yaml) for analytics-safe redaction -- [examples/production/cdc-to-datalake.yaml](examples/production/cdc-to-datalake.yaml) for CDC-to-lake shaping -- [examples/pipelines/README.md](examples/pipelines/README.md) for operator-backed Kubernetes manifests - -## Deploy and Operate - -- [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for deployment patterns -- [docs/OPERATIONS.md](docs/OPERATIONS.md) for production runbooks -- [docs/OBSERVABILITY_QUICKSTART.md](docs/OBSERVABILITY_QUICKSTART.md) for Prometheus and lag monitoring -- [docs/SECURITY_CONFIGURATION.md](docs/SECURITY_CONFIGURATION.md) for TLS and SASL setup -- [helm/streamforge-operator/README.md](helm/streamforge-operator/README.md) for Helm-based installs - -## Learn More - -- [docs/QUICKSTART.md](docs/QUICKSTART.md) for the first local run -- [docs/USAGE.md](docs/USAGE.md) for deployment patterns and use cases -- [docs/YAML_CONFIGURATION.md](docs/YAML_CONFIGURATION.md) for config structure and format guidance -- [docs/ADVANCED_DSL_GUIDE.md](docs/ADVANCED_DSL_GUIDE.md) for the full filtering and transform DSL -- [docs/DOCUMENTATION_INDEX.md](docs/DOCUMENTATION_INDEX.md) for the broader doc set +Performance depends on message shape, partitions, broker configuration, +delivery guarantees, and hardware. The public documentation therefore provides +[measurement and tuning guidance](docs/PERFORMANCE.md), not a universal +throughput claim. Results are published only after a reproducible, like-for-like +comparison passes record-count and delivery validation. ## Contributing -Contribution and development setup are documented in [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md). +See the [contributing guide](docs/CONTRIBUTING.md) for development setup, +testing, and pull-request expectations. Use +[GitHub Discussions](https://github.com/rahulbsw/streamforge/discussions) for +questions and [GitHub Issues](https://github.com/rahulbsw/streamforge/issues) +for reproducible defects or feature proposals. ## License -Apache License 2.0. See [LICENSE](LICENSE) for details. +StreamForge is licensed under the [Apache License 2.0](LICENSE). diff --git a/ROADMAP.md b/ROADMAP.md index e2639a2..fcc8013 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,7 +6,7 @@ Vision and planned features for StreamForge. StreamForge aims to be the **fastest, most reliable, and easiest-to-use Kafka selective replication engine**. We focus on: -1. **Performance** - Rust-native speed (25K-45K msg/s sustained throughput) +1. **Performance** - Measured Rust-native efficiency on representative workloads 2. **Reliability** - Production-grade stability with typed errors, retry, and DLQ 3. **Usability** - Simple DSL, great documentation, validation CLI 4. **Security** - Enterprise-ready security features (SSL/TLS, SASL, Kerberos) @@ -24,9 +24,9 @@ StreamForge aims to be the **fastest, most reliable, and easiest-to-use Kafka se โœ… **Core Engine:** - Rust + rdkafka + tokio async runtime - At-least-once delivery semantics (documented and tested) -- Configurable threading model (linear scaling to 8+ threads) +- Configurable threading and in-flight processing model - Consumer/producer tuning knobs (exposed via `performance:` config block) -- Typed error system (14+ error types with recovery actions) +- Typed error system with recovery actions - Dead letter queue (DLQ) with error metadata headers - Exponential backoff retry policy (configurable max attempts, delays, jitter) @@ -38,29 +38,29 @@ StreamForge aims to be the **fastest, most reliable, and easiest-to-use Kafka se - **v2.1 Dollar shorthand** (concise field access) - Example: `"and($status == 'active', $tier == 'premium')"` - Dot notation: `$user.email`, `$data.nested.path` -- **v2.2 Transform evaluators** (35 functions) - - 14 string transforms: uppercase, lowercase, length, substring, split, join, replace, pad, trim, type conversions - - 21 date/time transforms: now, parse_date, format_date, add_days, year, month, day, hour, etc. +- **v2.2 Transform evaluators** + - String transforms: uppercase, lowercase, length, substring, split, join, replace, pad, trim, and type conversions + - Date/time transforms: now, parse, format, arithmetic, and component extraction - AST-based parser with position-tracked errors - Semantic validation pass before execution - Complete EBNF grammar specification (docs/DSL_SPEC.md) โœ… **Data Plane:** - Multi-destination routing with filter-based selection -- 40+ filter types (AND/OR/NOT, regex, array ops, key/header/timestamp filters, null/empty checks) -- 30+ transform types (extract, construct, arithmetic, hash, string ops, date/time ops) +- Filters for boolean logic, regex, arrays, keys, headers, timestamps, and null/empty checks +- Transforms for extraction, construction, arithmetic, hashing, strings, and date/time values - Envelope access (msg value, key, headers, timestamp, partition, offset, topic) - Compression support (gzip, snappy, zstd, lz4) -- Partitioning strategies (default, random, hash, field-based) +- Default keyed/keyless partitioning and field-based partitioning - Key transformation pipeline - Header manipulation - Timestamp control โœ… **Observability:** -- 60+ Prometheus metrics with per-destination tracking +- Prometheus metrics with per-destination tracking - Kafka consumer lag monitoring - Filter/transform operation tracking -- HTTP metrics endpoint (< 2% overhead) +- HTTP metrics endpoint - Structured logging (tracing with span IDs) - Grafana dashboard templates with alert rules @@ -69,32 +69,27 @@ StreamForge aims to be the **fastest, most reliable, and easiest-to-use Kafka se - Helm chart for Kubernetes Operator - Kubernetes CRD (StreamforgePipeline v1alpha1) - Web UI (Next.js with JWT auth) -- Chainguard distroless container images (~20MB) +- Chainguard distroless container images โœ… **Performance:** -- 25,000โ€“45,000 msg/s sustained throughput (JSON processing) -- 12ms p99 latency end-to-end -- 44โ€“50ns simple filter latency -- ~50MB memory footprint +- Configurable consumer batching, fill timeout, and processing concurrency +- Configuration-time compilation of function-style paths, regexes, and key templates +- Copy-on-write values for transformed destinations and shared values for passthrough destinations +- Keyless default partitioning delegated to librdkafka +- Criterion filter, transform, and end-to-end benchmark targets โœ… **Testing:** -- **333 unit tests passing** (0 failures, 0 warnings) - - 102 parser tests (v1 + v2 syntax) - - 15 dollar syntax tests - - 11 string transform tests - - 18 date/time transform tests - - 187 other tests (filters, transforms, core engine) +- Unit coverage for parser, filter, transform, routing, partitioning, configuration, and core modules - Integration test infrastructure (testcontainers-based) -- Comprehensive benchmarks (filter, transform, end-to-end) +- Criterion benchmarks for filter, transform, and end-to-end paths โœ… **Documentation:** -- **10,000+ lines across 42 documentation files** - Complete DSL reference (docs/ADVANCED_DSL_GUIDE.md, docs/DSL_SPEC.md) - Function-style DSL guide (docs/DSL_V2_FUNCTION_SYNTAX.md) - Production deployment guides (docs/DEPLOYMENT.md, docs/DOCKER.md, docs/KUBERNETES.md) -- Operations runbook (docs/OPERATIONS.md, 40 KB) -- Troubleshooting guide (docs/TROUBLESHOOTING.md, 70+ issues covered) -- 40+ real-world example configurations +- Operations runbook (docs/OPERATIONS.md) +- Troubleshooting guide (docs/TROUBLESHOOTING.md) +- Real-world example configurations - Delivery guarantees specification (docs/DELIVERY_GUARANTEES.md) - Error handling taxonomy (docs/ERROR_HANDLING.md) @@ -141,10 +136,48 @@ StreamForge aims to be the **fastest, most reliable, and easiest-to-use Kafka se ### Performance Enhancements -- [ ] Zero-copy optimizations for Envelope -- [ ] SIMD operations for bulk filtering -- [ ] Parallel message processing within partition -- [ ] Target: 60K+ messages/second (with zero-copy) +- [x] **Phase 1 hot-path hardening** + - Delegate keyless default partitioning to librdkafka + - Skip absent transforms and use copy-on-write for actual transforms + - Precompile function-style paths/regexes and key-template paths + - Expose runtime batching and concurrency controls + - Add focused regression tests and steady-state benchmarks +- [x] Establish the deterministic synthetic and Kafka-backed baseline framework + - Add stage-level JSON/envelope Criterion measurements + - Add isolated Kafka repetitions with structured environment/result manifests + - Record the initial local Phase 2 baseline +- [x] Capture a whole-process CPU profile on representative dedicated hardware + - AWS c7i.2xlarge passthrough profile captured 614 cycle samples with zero + lost samples + - Parsing was about 16.5% inclusive and serialization about 3.2%; the result + does not trigger the 30% raw/lazy-envelope threshold +- [x] Implement bounded queued delivery and source-partition worker lanes + - Keep legacy batching and acknowledged delivery as compatibility defaults + - Reject unsafe queued/manual-commit/retry/DLQ combinations + - Expose broker-delivery completion separately from enqueue completion +- [x] Correct the sustained Kafka harness measurement contract + - Start StreamForge and persistent ingress before the timed barrier + - Separate ingress, timed metrics, and post-window output-validation jobs + - Exclude startup, warm-up, drain, validation, and teardown + - Require exact counters/offsets and physical topic-file reclamation +- [x] Add single-destination produced accounting and focused regression tests +- [x] Pass the loopback-only local Podman sustained validation with exact + consumed, produced, delivered, output, and error counts +- [ ] Run the corrected legacy/partition-ordered and + acknowledged/queued live Kafka comparison matrix +- [ ] Produce a clean-worktree, matched Java/Rust comparison before publishing + a public throughput claim +- [ ] Replace ad hoc AWS host provisioning with cost-bounded Terraform and + ECS-on-EC2 benchmark jobs after the local comparison matrix passes +- [ ] Implement rebalance-aware completed-offset coordination before supporting + partition-ordered manual commits +- [ ] Profile transform-heavy and aggregation-heavy workloads +- [ ] Implement raw/lazy envelope paths where profiling confirms parse or + serialization cost +- [ ] Remove array-element cloning from function-style `any`/`all` evaluation +- [ ] Evaluate SIMD only for a profiled vectorizable kernel; the c7i + passthrough profile did not identify one +- [ ] Measure and tune aggregation data structures and timers ### Developer Experience @@ -258,7 +291,7 @@ Want to contribute? Here are high-impact areas: - Add integration tests for complex scenarios - Performance regression testing - Chaos engineering (failure injection) -- Load testing at scale (100K+ msg/s) +- Load testing at production-representative scale ### Community Contributions - Answer questions on GitHub Discussions @@ -296,5 +329,6 @@ We value community input and prioritize features based on user demand! --- -**Last Updated:** 2026-04-18 +**Last Updated:** 2026-07-25 + **Maintained By:** StreamForge Core Team diff --git a/SUPPORT.md b/SUPPORT.md index 95363af..763993d 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -10,8 +10,8 @@ Start with our comprehensive documentation: - **[Quick Start Guide](docs/QUICKSTART.md)** - Get up and running in 5 minutes - **[Usage Guide](docs/USAGE.md)** - 8 real-world use cases with complete examples -- **[Documentation Index](docs/DOCUMENTATION_INDEX.md)** - Complete guide to all documentation -- **[Quick Reference](docs/QUICK_REFERENCE.md)** - Handy reference card +- **[Documentation Site](https://github.datasierra.com/streamforge/)** - Curated product and operator documentation +- **[Configuration Guide](docs/YAML_CONFIGURATION.md)** - Pipeline configuration reference ### ๐Ÿ’ฌ Community Support @@ -42,9 +42,9 @@ To help us help you faster, please: ### 1. Check the Documentation -- Browse the [documentation index](docs/DOCUMENTATION_INDEX.md) +- Search the [documentation site](https://github.datasierra.com/streamforge/) - Search the [existing issues](https://github.com/rahulbsw/streamforge/issues) -- Check the [troubleshooting sections](docs/USAGE.md#troubleshooting) +- Check the [troubleshooting guide](docs/TROUBLESHOOTING.md) ### 2. Gather Information @@ -103,8 +103,7 @@ See [Quick Start Guide - Prerequisites](docs/QUICKSTART.md#prerequisites) ### Performance Issues - [Performance Tuning Guide](docs/PERFORMANCE.md) -- [Scaling Guide](docs/SCALING.md) -- [Benchmark Results](benchmarks/results/) +- [Operations Guide](docs/OPERATIONS.md) ### Security Configuration @@ -171,8 +170,8 @@ Please note that all interactions are governed by our [Code of Conduct](CODE_OF_ ### Official Documentation - [GitHub Repository](https://github.com/rahulbsw/streamforge) -- [Documentation Site](docs/) -- [Changelog](docs/CHANGELOG.md) +- [Documentation Site](https://github.datasierra.com/streamforge/) +- [Changelog](CHANGELOG.md) ### Related Projects - [Apache Kafka](https://kafka.apache.org/) diff --git a/benches/end_to_end_benchmark.rs b/benches/end_to_end_benchmark.rs index ac2ed3b..3ee25c4 100644 --- a/benches/end_to_end_benchmark.rs +++ b/benches/end_to_end_benchmark.rs @@ -1,196 +1,152 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use serde_json::json; -use std::sync::Arc; -use streamforge::filter::{Filter, JsonPathFilter, JsonPathTransform, Transform}; +use criterion::{ + black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput, +}; +use serde_json::Value; +use streamforge::filter::{Filter, JsonPathFilter, JsonPathTransform, RegexFilter, Transform}; use streamforge::MessageEnvelope; -/// Benchmark end-to-end message processing pipeline -fn benchmark_processing_pipeline(c: &mut Criterion) { - let mut group = c.benchmark_group("end_to_end"); - - // Test message - let test_message = json!({ - "user": { - "id": "user-123", - "email": "test@example.com", - "age": 30, - "active": true - }, - "order": { - "id": "order-456", - "amount": 99.99, - "status": "completed" - } - }); - - // Single filter + transform (common case) - group.bench_function("filter_transform_single", |b| { - let filter = JsonPathFilter::new("/user/active", "==", "true").unwrap(); - let transform = JsonPathTransform::new("/user/email").unwrap(); - - b.iter(|| { - let envelope = MessageEnvelope::new(test_message.clone()); - let passes = filter.evaluate(&envelope.value).unwrap(); - if passes { - let _ = transform - .transform(Arc::try_unwrap(envelope.value).unwrap_or_else(|arc| (*arc).clone())) - .unwrap(); - } - }); - }); - - // Multi-destination simulation (4 destinations) - group.bench_function("multi_destination_4", |b| { - let filters: Vec> = vec![ - Arc::new(JsonPathFilter::new("/user/active", "==", "true").unwrap()), - Arc::new(JsonPathFilter::new("/order/status", "==", "completed").unwrap()), - Arc::new(JsonPathFilter::new("/user/age", ">", "18").unwrap()), - Arc::new(JsonPathFilter::new("/order/amount", ">", "50").unwrap()), - ]; - - b.iter(|| { - let envelope = MessageEnvelope::new(test_message.clone()); - - // Simulate multi-destination processing with cheap Arc clones - for filter in &filters { - let env_clone = envelope.clone(); - let _ = black_box(filter.evaluate(&env_clone.value).unwrap()); - } - }); - }); - - // Heavy JSON path extraction (pre-parsing benefit) - group.bench_function("heavy_jsonpath_extraction", |b| { - let paths = vec![ - JsonPathTransform::new("/user/id").unwrap(), - JsonPathTransform::new("/user/email").unwrap(), - JsonPathTransform::new("/user/age").unwrap(), - JsonPathTransform::new("/order/id").unwrap(), - JsonPathTransform::new("/order/amount").unwrap(), - JsonPathTransform::new("/order/status").unwrap(), - ]; - - b.iter(|| { - let envelope = MessageEnvelope::new(test_message.clone()); - let value = Arc::try_unwrap(envelope.value).unwrap_or_else(|arc| (*arc).clone()); - - for path in &paths { - let _ = black_box(path.transform(value.clone()).unwrap()); - } - }); - }); - - group.finish(); +const PAYLOAD_SIZES: [(&str, usize); 3] = [("256b", 256), ("4kib", 4 * 1024), ("64kib", 64 * 1024)]; + +/// Build valid JSON at the requested size while retaining fields used by the +/// filter, regex, and transform stages. +fn synthetic_payload(target_bytes: usize) -> Vec { + const PREFIX: &str = concat!( + r#"{"user":{"id":"user-123","email":"alice@example.com","active":true},"#, + r#""event":{"kind":"order.created"},"padding":""# + ); + const SUFFIX: &str = "\"}"; + + assert!(target_bytes >= PREFIX.len() + SUFFIX.len()); + + let mut payload = Vec::with_capacity(target_bytes); + payload.extend_from_slice(PREFIX.as_bytes()); + payload.resize(target_bytes - SUFFIX.len(), b'x'); + payload.extend_from_slice(SUFFIX.as_bytes()); + debug_assert_eq!(payload.len(), target_bytes); + payload } -/// Benchmark envelope cloning performance (Arc benefit) -fn benchmark_envelope_cloning(c: &mut Criterion) { - let mut group = c.benchmark_group("envelope_cloning"); - - // Small message (1KB) - let small_msg = json!({ - "id": "test-123", - "value": "x".repeat(900) - }); - - // Large message (10KB) - let large_msg = json!({ - "id": "test-456", - "data": "x".repeat(9900) - }); - - for (name, msg) in [("small_1kb", small_msg), ("large_10kb", large_msg)] { - group.bench_with_input(BenchmarkId::new("clone", name), &msg, |b, msg| { - let envelope = MessageEnvelope::new(msg.clone()); - b.iter(|| { - let _ = black_box(envelope.clone()); - }); - }); - - group.bench_with_input(BenchmarkId::new("clone_4x", name), &msg, |b, msg| { - let envelope = MessageEnvelope::new(msg.clone()); - b.iter(|| { - // Simulate multi-destination (4 clones) - let e1 = envelope.clone(); - let e2 = envelope.clone(); - let e3 = envelope.clone(); - let e4 = envelope.clone(); - black_box((e1, e2, e3, e4)); - }); - }); +/// Synthetic, in-memory coverage of the CPU and allocation stages around the +/// current JSON envelope pipeline. Kafka/network latency is intentionally +/// excluded. +fn synthetic_pipeline(c: &mut Criterion) { + let mut group = c.benchmark_group("synthetic_pipeline"); + group.sample_size(10); + group.warm_up_time(std::time::Duration::from_millis(200)); + group.measurement_time(std::time::Duration::from_millis(500)); + + let path_filter = JsonPathFilter::new("/user/active", "==", "true").unwrap(); + let regex_filter = + RegexFilter::new("/user/email", r"^[a-z]+(?:\.[a-z]+)*@example\.com$").unwrap(); + let transform = JsonPathTransform::new("/user/email").unwrap(); + + for (size_name, target_bytes) in PAYLOAD_SIZES { + let payload = synthetic_payload(target_bytes); + let parsed: Value = serde_json::from_slice(&payload).unwrap(); + let envelope = MessageEnvelope::new(parsed.clone()); + + group.throughput(Throughput::Bytes(payload.len() as u64)); + + // Input allocation is excluded: this measures JSON bytes -> Value. + group.bench_with_input( + BenchmarkId::new("parse_bytes", size_name), + &payload, + |b, bytes| { + b.iter(|| { + black_box(serde_json::from_slice::(black_box(bytes.as_slice())).unwrap()) + }); + }, + ); + + // Value construction is excluded: this measures Value -> JSON bytes. + group.bench_with_input( + BenchmarkId::new("serialize_value", size_name), + &parsed, + |b, value| { + b.iter(|| black_box(serde_json::to_vec(black_box(value)).unwrap())); + }, + ); + + // Input allocation is excluded; both parse and serialization are timed. + group.bench_with_input( + BenchmarkId::new("parse_serialize_round_trip", size_name), + &payload, + |b, bytes| { + b.iter(|| { + let value: Value = serde_json::from_slice(black_box(bytes.as_slice())).unwrap(); + black_box(serde_json::to_vec(&value).unwrap()) + }); + }, + ); + + // Models current one-destination ingestion preparation. Payload creation + // is excluded; JSON parsing and envelope allocation are included. + group.bench_with_input( + BenchmarkId::new("one_destination_passthrough_preparation", size_name), + &payload, + |b, bytes| { + b.iter(|| { + let value: Value = serde_json::from_slice(black_box(bytes.as_slice())).unwrap(); + black_box(MessageEnvelope::new(value)) + }); + }, + ); + + // Fan-out reports destination operations. Envelope construction is + // excluded; each iteration times four Arc-backed envelope clones. + group.throughput(Throughput::Elements(4)); + group.bench_with_input( + BenchmarkId::new("four_destination_arc_fan_out", size_name), + &envelope, + |b, envelope| { + b.iter(|| { + black_box(( + envelope.clone(), + envelope.clone(), + envelope.clone(), + envelope.clone(), + )) + }); + }, + ); + + group.throughput(Throughput::Elements(1)); + + // Filter construction and JSON parsing are excluded. + group.bench_with_input( + BenchmarkId::new("one_path_filter", size_name), + &envelope, + |b, envelope| { + b.iter(|| black_box(path_filter.evaluate_envelope(black_box(envelope)).unwrap())); + }, + ); + + // Regex compilation and JSON parsing are excluded. + group.bench_with_input( + BenchmarkId::new("one_regex_filter", size_name), + &envelope, + |b, envelope| { + b.iter(|| black_box(regex_filter.evaluate_envelope(black_box(envelope)).unwrap())); + }, + ); + + // Transform consumes an owned Value. iter_batched performs the required + // deep clone as untimed setup, isolating the transform stage itself. + group.bench_with_input( + BenchmarkId::new("one_transform", size_name), + &parsed, + |b, value| { + b.iter_batched( + || value.clone(), + |owned| black_box(transform.transform(black_box(owned)).unwrap()), + BatchSize::SmallInput, + ); + }, + ); } group.finish(); } -/// Benchmark JSON path pre-parsing benefit -fn benchmark_jsonpath_preparsing(c: &mut Criterion) { - let mut group = c.benchmark_group("jsonpath"); - - let test_value = json!({ - "level1": { - "level2": { - "level3": { - "level4": { - "value": "target" - } - } - } - } - }); - - // Deep path extraction (benefits most from pre-parsing) - group.bench_function("deep_path_extraction", |b| { - let transform = JsonPathTransform::new("/level1/level2/level3/level4/value").unwrap(); - - b.iter(|| { - let _ = black_box(transform.transform(test_value.clone()).unwrap()); - }); - }); - - group.finish(); -} - -/// Benchmark filter evaluation with pre-resolved metrics -fn benchmark_filter_with_metrics(c: &mut Criterion) { - let mut group = c.benchmark_group("filter_metrics"); - - let test_value = json!({ - "age": 30, - "status": "active", - "verified": true - }); - - // Simple filter (pre-resolved metrics benefit) - group.bench_function("simple_filter_eval", |b| { - let filter = JsonPathFilter::new("/age", ">", "18").unwrap(); - - b.iter(|| { - let _ = black_box(filter.evaluate(&test_value).unwrap()); - }); - }); - - // Complex AND filter - group.bench_function("and_filter_eval", |b| { - use streamforge::filter::AndFilter; - - let filter1 = Box::new(JsonPathFilter::new("/age", ">", "18").unwrap()); - let filter2 = Box::new(JsonPathFilter::new("/status", "==", "active").unwrap()); - let and_filter = AndFilter::new(vec![filter1, filter2]); - - b.iter(|| { - let _ = black_box(and_filter.evaluate(&test_value).unwrap()); - }); - }); - - group.finish(); -} - -criterion_group!( - benches, - benchmark_processing_pipeline, - benchmark_envelope_cloning, - benchmark_jsonpath_preparsing, - benchmark_filter_with_metrics -); +criterion_group!(benches, synthetic_pipeline); criterion_main!(benches); diff --git a/benches/filter_benchmarks.rs b/benches/filter_benchmarks.rs index cd7962e..0641def 100644 --- a/benches/filter_benchmarks.rs +++ b/benches/filter_benchmarks.rs @@ -2,6 +2,7 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criteri use serde_json::{json, Value}; use streamforge::filter::*; use streamforge::filter_parser::parse_filter; +use streamforge::MessageEnvelope; fn create_test_message() -> Value { json!({ @@ -97,6 +98,46 @@ fn bench_regex_filter(c: &mut Criterion) { }); } +fn bench_function_style_filter_evaluation(c: &mut Criterion) { + let envelope = MessageEnvelope::new(create_test_message()); + let path_filter = parse_filter("field('/message/siteId') > 10000").unwrap(); + let regex_filter = + parse_filter("regex(field('/message/email'), '^[^@]+@example\\\\.com$')").unwrap(); + let multi_path_filter = + parse_filter("and(field('/message/siteId') > 10000, field('/message/status') == 'active')") + .unwrap(); + assert!(path_filter.evaluate_envelope(&envelope).unwrap()); + assert!(regex_filter.evaluate_envelope(&envelope).unwrap()); + assert!(multi_path_filter.evaluate_envelope(&envelope).unwrap()); + let mut group = c.benchmark_group("filter/function_style/evaluate"); + + group.bench_function("compiled_path_comparison", |b| { + b.iter(|| black_box(path_filter.evaluate_envelope(black_box(&envelope)).unwrap())) + }); + + group.bench_function("compiled_regex", |b| { + b.iter(|| { + black_box( + regex_filter + .evaluate_envelope(black_box(&envelope)) + .unwrap(), + ) + }) + }); + + group.bench_function("compiled_multi_path_and", |b| { + b.iter(|| { + black_box( + multi_path_filter + .evaluate_envelope(black_box(&envelope)) + .unwrap(), + ) + }) + }); + + group.finish(); +} + fn bench_array_filter(c: &mut Criterion) { let msg = create_test_message(); @@ -182,6 +223,7 @@ criterion_group!( bench_simple_filter, bench_boolean_logic, bench_regex_filter, + bench_function_style_filter_evaluation, bench_array_filter, bench_filter_parser, bench_filter_throughput diff --git a/benches/transform_benchmarks.rs b/benches/transform_benchmarks.rs index 8c13d3f..88ac81d 100644 --- a/benches/transform_benchmarks.rs +++ b/benches/transform_benchmarks.rs @@ -1,8 +1,11 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use criterion::{ + black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput, +}; use serde_json::{json, Value}; use std::collections::HashMap; use streamforge::filter::*; use streamforge::filter_parser::parse_transform; +use streamforge::MessageEnvelope; fn create_test_message() -> Value { json!({ @@ -138,6 +141,48 @@ fn bench_arithmetic_transform(c: &mut Criterion) { }); } +fn bench_key_template_transform(c: &mut Criterion) { + let envelope = MessageEnvelope::new(create_test_message()); + let single_path = KeyTemplateTransform::new("conference-{/message/confId}").unwrap(); + let multiple_paths = KeyTemplateTransform::new( + "site-{/message/siteId}/conference-{/message/confId}/status-{/message/status}", + ) + .unwrap(); + assert_eq!( + single_path + .transform_envelope(envelope.clone()) + .unwrap() + .key, + Some(json!("conference-12345")) + ); + assert_eq!( + multiple_paths + .transform_envelope(envelope.clone()) + .unwrap() + .key, + Some(json!("site-67890/conference-12345/status-active")) + ); + let mut group = c.benchmark_group("transform/key_template/evaluate"); + + group.bench_function("single_compiled_path", |b| { + b.iter_batched( + || envelope.clone(), + |input| black_box(single_path.transform_envelope(black_box(input)).unwrap()), + BatchSize::SmallInput, + ) + }); + + group.bench_function("multiple_compiled_paths", |b| { + b.iter_batched( + || envelope.clone(), + |input| black_box(multiple_paths.transform_envelope(black_box(input)).unwrap()), + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + fn bench_transform_parser(c: &mut Criterion) { c.bench_function("parser/simple_transform", |b| { b.iter(|| parse_transform(black_box("/message/confId"))) @@ -255,6 +300,7 @@ criterion_group!( bench_object_construction, bench_array_transform, bench_arithmetic_transform, + bench_key_template_transform, bench_transform_parser, bench_transform_throughput, bench_combined_operations diff --git a/docker-compose.benchmark.yml b/docker-compose.benchmark.yml index 3ccb1b0..f78314f 100644 --- a/docker-compose.benchmark.yml +++ b/docker-compose.benchmark.yml @@ -1,8 +1,10 @@ +--- version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:7.5.0 + image: >- + docker.io/confluentinc/cp-zookeeper@sha256:02f6c042bb9a7844382fc4cedc513a44585d8a5acae873fb9e510e3ca9dcabc6 container_name: benchmark-zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 @@ -11,20 +13,27 @@ services: - benchmark-net kafka: - image: confluentinc/cp-kafka:7.5.0 + image: >- + docker.io/confluentinc/cp-kafka@sha256:fbbb6fa11b258a88b83f54d4f0bddfcffbf2279f99d66a843486e3da7bdfbf41 container_name: benchmark-kafka depends_on: - zookeeper ports: - - "9092:9092" + - "127.0.0.1:9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: >- + PLAINTEXT://kafka:29092,PLAINTEXT_HOST://127.0.0.1:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >- + PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_DELETE_TOPIC_ENABLE: "true" + KAFKA_FILE_DELETE_DELAY_MS: 1000 + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000 + KAFKA_LOG_SEGMENT_BYTES: 268435456 KAFKA_NUM_PARTITIONS: 10 networks: - benchmark-net @@ -34,6 +43,36 @@ services: timeout: 10s retries: 10 + ingress-runner: + image: >- + docker.io/confluentinc/cp-kafka@sha256:fbbb6fa11b258a88b83f54d4f0bddfcffbf2279f99d66a843486e3da7bdfbf41 + container_name: benchmark-ingress + entrypoint: ["/bin/bash", "-lc"] + command: ["trap : TERM INT; sleep infinity & wait"] + read_only: true + tmpfs: + - /tmp + depends_on: + - kafka + networks: + - benchmark-net + + output-runner: + image: >- + docker.io/confluentinc/cp-kafka@sha256:fbbb6fa11b258a88b83f54d4f0bddfcffbf2279f99d66a843486e3da7bdfbf41 + container_name: benchmark-output + entrypoint: ["/bin/bash", "-lc"] + command: ["trap : TERM INT; sleep infinity & wait"] + read_only: true + tmpfs: + - /tmp + depends_on: + - kafka + networks: + - benchmark-net + networks: benchmark-net: + name: streamforge-benchmark-net driver: bridge + internal: true diff --git a/docs/404.md b/docs/404.md new file mode 100644 index 0000000..d90a439 --- /dev/null +++ b/docs/404.md @@ -0,0 +1,20 @@ +--- +title: Page not found +layout: default +permalink: /404.html +nav_exclude: true +search_exclude: true +description: The requested StreamForge documentation page could not be found. +--- + +
+
+

Route not found / 404

+

This path stops here.

+

The page may have moved during the documentation cleanup. Return to the product overview or search the documentation from the navigation.

+ Return to StreamForge +
+ +
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md deleted file mode 100644 index e53b79a..0000000 --- a/docs/CHANGELOG.md +++ /dev/null @@ -1,288 +0,0 @@ ---- -title: Changelog -nav_order: 13 ---- - -# Changelog - -All notable changes to StreamForge are documented here. -Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ---- - -## [1.0.0-alpha.1] - 2026-04-18 - v1.0 Hardening Phase - -### Changed -- **Version bumped to 1.0.0-alpha.1** - Start of v1.0 completion plan -- **DSL Strategy Finalized** - Keeping custom string-based DSL, removing Rhai -- **Branch Reorganization** - Created `improvement` branch for v1.0 work - -### Added -- **V1_PLAN.md** - Comprehensive 6-phase v1.0 completion roadmap -- **ARCHITECTURE.md Updates** - Added module organization and v1.0 gap analysis -- **Kubernetes UI Integration** - Helm chart now includes Web UI deployment -- **UI RBAC** - ClusterRole, ServiceAccount, and RoleBinding for UI - -### Planned (In Progress) -- Phase 0: Repository coherence (in progress) -- Phase 1: Core engine hardening (delivery semantics, retry/DLQ, error types) -- Phase 2: DSL stabilization (formal grammar, parser refactor, validation) -- Phase 3-6: See V1_PLAN.md for complete roadmap - ---- - -## [0.4.0] - 2026-04-03 - Observability, Envelopes & Release Pipeline - -### Added - -#### Prometheus Observability -- Prometheus metrics exporter on `/metrics` HTTP endpoint -- Grafana dashboard templates (`examples/streamforge_alerts.yml`) -- Per-destination throughput, latency, and error counters -- JVM-style process metrics (CPU, memory, file descriptors) -- `docs/OBSERVABILITY_QUICKSTART.md` and `docs/OBSERVABILITY_METRICS_DESIGN.md` - -#### Envelope Transforms -- `ENVELOPE:wrap` โ€” wraps the full message payload into a named field -- `ENVELOPE:unwrap` โ€” extracts an inner field as the new root -- `ENVELOPE:add_metadata` โ€” injects topic, partition, offset, timestamp headers -- Migration guide: `docs/ENVELOPE_MIGRATION_GUIDE.md` -- Design reference: `docs/ENVELOPE_FEATURE_DESIGN.md` -- Example configs: `examples/config.envelope-simple.yaml`, `examples/config.envelope-features.yaml` - -#### Multi-Architecture Release Pipeline -- Automated GitHub Actions release workflow (`release.yml`) -- Pre-built binaries for `linux-x86_64`, `linux-aarch64`, `macos-x86_64`, `macos-aarch64` -- Docker images published to GHCR (`ghcr.io/rahulbsw/streamforge`) -- Chainguard distroless base images for minimal attack surface -- Native ARM64 runner eliminates QEMU cross-compilation overhead - -### Fixed -- All clippy warnings resolved across main crate and operator -- Operator reconciler unused import removed -- 17 npm security vulnerabilities in UI dependencies resolved - ---- - -## [0.3.0] - 2026-04-01 - Concurrent Processing, Hash & Cache - -### Added - -#### Concurrent Processing -- Multi-threaded pipeline with configurable thread count (`threads: N`) -- Lock-free work queue for message dispatch -- Linear scaling validated: 8 threads โ†’ 25,000โ€“34,500 msg/s -- Concurrent consumer/producer architecture with Tokio - -#### Hash Transforms -- 5 hash algorithms: `MD5`, `SHA256`, `SHA512`, `MURMUR64`, `MURMUR128` -- DSL syntax: `HASH:algorithm,/path[,outputField]` -- Use cases: PII anonymization, deduplication, consistent partitioning -- Throughput: up to 10M ops/s (Murmur), 2M ops/s (SHA256) - -#### Cache Backends -- **Local cache** (Moka): TTL/TTI eviction, 50ns lookup, async-first -- **Redis cache**: connection pooling, key prefixes, auto-expiration -- **Kafka-backed cache**: compacted topic as distributed cache, warmup on start -- **Multi-level cache**: L1 (local) + L2 (Redis), automatic promotion -- DSL syntax: `CACHE_LOOKUP:/keyPath,cacheName,/outputField` -- Feature flags: `local-cache` (default), `redis-cache`, `all-caches` - -#### At-Least-Once Delivery -- Manual commit mode with async or sync commit options -- Configurable commit interval (batching) -- Dead Letter Queue (DLQ) with configurable topic -- Exponential backoff retry (initial 100ms โ†’ max 30s, multiplier 2.0) -- Backward compatible โ€” auto-commit remains the default - -#### Kubernetes & Helm -- Kubernetes Operator (`operator/`) for CRD-based pipeline management -- Helm chart (`helm/streamforge-operator/`) for operator deployment -- Kubernetes secret support for secure Kafka connections -- Web UI for operator pipeline management (`ui/`) - -### Performance -- 25,000โ€“34,500 msg/s at 8 threads (linear scaling) -- Memory: ~50MB (vs ~500MB Java MirrorMaker) -- Hash operations: 50nsโ€“1ยตs depending on algorithm -- Local cache lookup: 50ns p50, 100ns p99 - ---- - -## [0.2.0] - 2026-03-10 - Advanced DSL & YAML Support - -### Added - -#### YAML Configuration Support -- โœ… YAML format support (`.yaml`, `.yml` extensions) -- โœ… Automatic format detection based on file extension -- โœ… Backward compatible with JSON -- โœ… Multi-line strings for complex filters -- โœ… Inline comments for documentation -- โœ… Much more readable for complex configurations - -**Examples:** -```yaml -routing: - destinations: - # Users with valid email - - output: validated-users - description: Email validation pipeline - filter: "REGEX:/user/email,^[\\w\\.-]+@[\\w\\.-]+\\.\\w+$" -``` - -**Files:** -- `config.example.yaml` - Simple YAML example -- `config.multidest.yaml` - Multi-destination YAML -- `config.advanced.yaml` - Advanced YAML with all features -- `YAML_CONFIGURATION.md` - Complete YAML guide - -#### Array Operations -- โœ… `ARRAY_ALL` filter - Check if all elements match a condition -- โœ… `ARRAY_ANY` filter - Check if any element matches a condition -- โœ… `ARRAY_MAP` transform - Map over array elements -- โœ… Support for nested array element filtering -- โœ… Empty array handling - -**Examples:** -```json -"filter": "ARRAY_ALL:/users,/status,==,active" -"filter": "ARRAY_ANY:/tasks,/priority,==,high" -"transform": "ARRAY_MAP:/users,/id" -``` - -#### Regular Expressions -- โœ… `REGEX` filter for pattern matching -- โœ… Full regex syntax support -- โœ… Compiled patterns for optimal performance -- โœ… Case-sensitive matching - -**Examples:** -```json -"filter": "REGEX:/email,^[\\w\\.-]+@[\\w\\.-]+\\.\\w+$" -"filter": "REGEX:/version,^2\\." -"filter": "REGEX:/status,^(active|pending)$" -``` - -#### Arithmetic Operations -- โœ… `ARITHMETIC:ADD` - Addition -- โœ… `ARITHMETIC:SUB` - Subtraction -- โœ… `ARITHMETIC:MUL` - Multiplication -- โœ… `ARITHMETIC:DIV` - Division -- โœ… Support for path-to-path operations -- โœ… Support for path-to-constant operations -- โœ… Division by zero error handling - -**Examples:** -```json -"transform": "ARITHMETIC:ADD,/price,/tax" -"transform": "ARITHMETIC:MUL,/price,1.2" -"transform": "ARITHMETIC:SUB,/total,/discount" -"transform": "ARITHMETIC:DIV,/total,/count" -``` - -#### Documentation -- โœ… ADVANCED_DSL_GUIDE.md - Comprehensive DSL reference -- โœ… DSL_FEATURES.md - Feature summary and comparison -- โœ… config.advanced.example.json - Example configurations - -#### Tests -- โœ… 19 new test cases for array operations -- โœ… 8 new test cases for regular expressions -- โœ… 14 new test cases for arithmetic operations -- โœ… Parser tests for all new features -- โœ… 100% test pass rate (56 tests passing) - -### Changed -- Updated README.md with DSL capabilities section -- Updated IMPLEMENTATION_STATUS.md to reflect completed features -- Updated comparison table to show Rust advantages -- Removed JSLT/JavaScript from "Future Enhancements" - -### Performance -- Array operations: ~1-10ยตs (size dependent) -- Regular expressions: ~500ns-1ยตs (complexity dependent) -- Arithmetic operations: ~50ns -- Overall: 40x faster than Java JSLT - ---- - -## [0.1.0] - 2026-03-10 - Initial Release - -### Added - -#### Core Features -- โœ… Cross-cluster Kafka mirroring -- โœ… Async/await with Tokio runtime -- โœ… Custom partitioning (hash-based, field-based) -- โœ… Multi-destination routing -- โœ… Native Kafka compression (Gzip, Snappy, Zstd) -- โœ… Lock-free metrics with atomic operations - -#### Filtering & Transformation -- โœ… JSON Path filters with comparison operators - - Numeric: `>`, `>=`, `<`, `<=`, `==`, `!=` - - String: `==`, `!=` - - Boolean: `==`, `!=` -- โœ… Boolean logic (AND/OR/NOT) -- โœ… JSON Path transforms (field extraction) -- โœ… Object construction (CONSTRUCT) -- โœ… Per-destination filters and transforms - -#### Docker Support -- โœ… Multi-stage Dockerfile with Chainguard base images -- โœ… Static binary variant (Dockerfile.static) -- โœ… Docker Compose configuration -- โœ… ~20-30MB dynamic image size -- โœ… ~10-15MB static image size -- โœ… Non-root user execution -- โœ… Health checks included - -#### Configuration -- โœ… JSON-based configuration -- โœ… Single-destination mode -- โœ… Multi-destination routing mode -- โœ… Environment variable config path -- โœ… Consumer/producer property override - -#### Metrics -- โœ… Processed messages counter -- โœ… Filtered messages counter -- โœ… Completed messages counter -- โœ… Error counter -- โœ… Rate calculation -- โœ… Periodic reporting (10s interval) - -#### Documentation -- โœ… README.md - Project overview -- โœ… QUICKSTART.md - Getting started guide -- โœ… IMPLEMENTATION_NOTES.md - Architecture details -- โœ… ADVANCED_FILTERS.md - Boolean logic guide -- โœ… DOCKER.md - Docker deployment guide -- โœ… IMPLEMENTATION_STATUS.md - Feature tracking - -### Performance -- Memory usage: ~50MB (vs ~500MB Java) -- CPU efficiency: 2-3x better than Java -- Throughput: ~25K msg/s (vs ~10K Java) -- Latency p99: ~15ms (vs ~50ms Java) -- Filter evaluation: ~100ns per filter - ---- - -## Roadmap - -### Version 0.5.0 (Planned) -- [ ] Avro serialization support -- [ ] Schema registry integration -- [ ] Schema evolution handling -- [ ] String manipulation operations (`UPPER`, `LOWER`, `TRIM`, `SUBSTRING`) -- [ ] Date/time operations and format transforms -- [ ] Conditional transforms (`IF:condition,thenTransform,elseTransform`) - -### Version 1.0.0 (Planned) -- [ ] Exactly-once semantics (idempotent producer + transactional consumer) -- [ ] UDF support via WASM or Lua -- [ ] State management with RocksDB -- [ ] Production hardening and SLA documentation -- [ ] Comprehensive production case studies diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 0de507d..0f11afd 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -1,6 +1,6 @@ --- title: Compatibility -nav_order: 5 +nav_order: 8 --- # Compatibility diff --git a/docs/CONFIG_SCHEMA.json b/docs/CONFIG_SCHEMA.json index 7ea1ad38..5275d48 100644 --- a/docs/CONFIG_SCHEMA.json +++ b/docs/CONFIG_SCHEMA.json @@ -54,55 +54,76 @@ }, "performance": { "type": "object", - "description": "Performance tuning parameters", + "description": "Runtime batching and selected librdkafka performance settings. Explicit consumer_properties and producer_properties entries take precedence.", "properties": { - "fetch_min_bytes": { + "consumer_batch_size": { "type": "integer", - "description": "Minimum bytes to fetch from Kafka", + "description": "Maximum messages collected for one processing batch", "minimum": 1, - "maximum": 10485760, - "default": 1024 + "default": 100 }, - "fetch_max_wait_ms": { + "consumer_batch_timeout_ms": { "type": "integer", - "description": "Maximum wait time for fetch request (ms)", + "description": "Maximum wait for a partially filled legacy batch and idle queued-delivery flush delay in partition_ordered mode (ms)", "minimum": 1, - "maximum": 10000, "default": 100 }, - "max_partition_fetch_bytes": { + "parallelism_factor": { "type": "integer", - "description": "Maximum bytes per partition fetch", - "minimum": 1024, - "maximum": 10485760, - "default": 1048576 + "description": "Concurrent processing multiplier applied to threads", + "minimum": 1, + "default": 10 }, - "queue_buffering_max_ms": { + "processing_mode": { + "type": "string", + "description": "Runtime scheduling mode. partition_ordered uses bounded FIFO worker lanes keyed by source partition.", + "enum": ["legacy_batch", "partition_ordered"], + "default": "legacy_batch" + }, + "worker_queue_capacity": { "type": "integer", - "description": "Producer queue buffering time (ms)", - "minimum": 0, - "maximum": 10000, - "default": 5 + "description": "Per-worker bounded queue capacity in partition_ordered mode", + "minimum": 1, + "default": 1024 }, - "batch_size": { + "producer_delivery_mode": { + "type": "string", + "description": "acknowledged waits for each delivery result; queued tracks bounded asynchronous deliveries and is restricted to auto-commit with retries and DLQ disabled", + "enum": ["acknowledged", "queued"], + "default": "acknowledged" + }, + "producer_max_in_flight": { "type": "integer", - "description": "Producer batch size (number of messages)", + "description": "Maximum pending delivery futures in queued mode", "minimum": 1, - "maximum": 100000, - "default": 1000 + "default": 10000 }, - "linger_ms": { + "fetch_min_bytes": { + "type": "integer", + "description": "Maps to librdkafka fetch.min.bytes", + "minimum": 1, + "maximum": 4294967295 + }, + "fetch_max_wait_ms": { "type": "integer", - "description": "Producer linger time to allow batching (ms)", + "description": "Maps to librdkafka fetch.wait.max.ms", "minimum": 0, - "maximum": 1000, - "default": 10 + "maximum": 4294967295 }, - "compression": { - "type": "string", - "description": "Producer compression codec", - "enum": ["none", "gzip", "snappy", "lz4", "zstd"], - "default": "none" + "queue_buffering_max_ms": { + "type": "integer", + "description": "Maps to librdkafka queue.buffering.max.ms; linger_ms takes precedence when both are set", + "minimum": 0 + }, + "batch_size": { + "type": "integer", + "description": "Maximum producer batch message count; maps to librdkafka batch.num.messages", + "minimum": 1 + }, + "linger_ms": { + "type": "integer", + "description": "Maps to librdkafka linger.ms; takes precedence over queue_buffering_max_ms", + "minimum": 0 } } }, @@ -369,10 +390,12 @@ "offset": "latest", "threads": 4, "performance": { + "consumer_batch_size": 100, + "consumer_batch_timeout_ms": 100, + "parallelism_factor": 10, "fetch_min_bytes": 1024, "batch_size": 1000, - "linger_ms": 10, - "compression": "zstd" + "linger_ms": 10 }, "retry": { "max_attempts": 3, @@ -411,10 +434,12 @@ "offset": "earliest", "threads": 8, "performance": { + "consumer_batch_size": 500, + "consumer_batch_timeout_ms": 50, + "parallelism_factor": 10, "fetch_min_bytes": 10240, "batch_size": 5000, - "linger_ms": 50, - "compression": "zstd" + "linger_ms": 50 }, "kafka": { "security": { diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 43878ed..37c97e0 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,6 +1,6 @@ --- title: Contributing -nav_order: 12 +nav_order: 9 --- # Contributing Guide @@ -802,7 +802,8 @@ cargo clippy --fix ## License -Apache License 2.0 - See [LICENSE](../LICENSE) for details. +Apache License 2.0 - See +[LICENSE](https://github.com/rahulbsw/streamforge/blob/main/LICENSE) for details. Copyright 2025 Rahul Jain diff --git a/docs/DELIVERY_GUARANTEES.md b/docs/DELIVERY_GUARANTEES.md index c41fe29..a0035b5 100644 --- a/docs/DELIVERY_GUARANTEES.md +++ b/docs/DELIVERY_GUARANTEES.md @@ -1,757 +1,157 @@ -# Delivery Guarantees - -**Version:** 1.0.0-alpha.1 -**Status:** Specification Complete (Phase 1) -**Last Updated:** 2026-04-18 - ---- - -## Executive Summary - -StreamForge v1.0 provides **at-least-once delivery** by default with configurable commit strategies, retry policies, and dead letter queue handling. - -**Key Guarantees:** -- โœ… **At-least-once:** Every message processed at least once (duplicates possible on failure) -- โฑ๏ธ **Ordered processing:** Messages within a partition processed in order -- ๐Ÿ”„ **Retry with backoff:** Transient failures retried automatically -- ๐Ÿ’€ **DLQ for permanent failures:** Bad messages don't halt the pipeline -- ๐Ÿ“Š **Observable:** Metrics track every stage of delivery - -**NOT Guaranteed:** -- โŒ Exactly-once semantics (planned v1.1+) -- โŒ Cross-partition ordering -- โŒ Zero duplicates on failure recovery - ---- - -## Table of Contents - -- [Delivery Semantics](#delivery-semantics) -- [Commit Strategies](#commit-strategies) -- [Retry Policy](#retry-policy) -- [Dead Letter Queue](#dead-letter-queue) -- [Failure Scenarios](#failure-scenarios) -- [Configuration](#configuration) -- [Observability](#observability) - ---- - -## Delivery Semantics - -### At-Least-Once (Default) - -**Definition:** Every message is processed and delivered at least once. Duplicates may occur on failure recovery. - -**How it works:** -1. Consume message from Kafka -2. Process message (filter, transform, produce) -3. **Commit offset only after successful produce** -4. On failure: retry or send to DLQ, then retry step 3 - -**Duplicate scenarios:** -- Process succeeds, produce succeeds, **commit fails** โ†’ message reprocessed on restart -- Process succeeds, commit succeeds, **crash before persist** โ†’ message reprocessed on restart - -**Trade-off:** Guarantees no data loss, but allows duplicates. - ---- - -### At-Most-Once (Optional) - -**Definition:** Every message is processed at most once. Data loss possible on failure. - -**How it works:** -1. Consume message from Kafka -2. **Commit offset immediately** (before processing) -3. Process message (filter, transform, produce) -4. On failure: skip message (already committed) - -**Data loss scenarios:** -- Commit succeeds, **process fails** โ†’ message lost -- Commit succeeds, **produce fails** โ†’ message lost - -**Trade-off:** No duplicates, but data loss possible. - -**โš ๏ธ Not recommended for production** unless data loss is acceptable. - --- - -### Exactly-Once (Planned v1.1+) - -**Definition:** Every message is processed exactly once. No duplicates, no data loss. - -**Requirements:** -- Kafka 3.3+ with transactional producer -- Idempotent producer enabled -- Read-process-write transactions - -**Status:** Not implemented in v1.0. Use at-least-once with idempotency keys for now. - +title: Delivery guarantees +nav_order: 5 +parent: Usage Guide --- -## Commit Strategies - -### 1. Manual Commit After Batch (Default, Recommended) +# Delivery guarantees -**Configuration:** -```yaml -commit_strategy: - manual_commit: true - commit_mode: async - commit_interval: 100 # Commit every 100 messages - commit_timeout_ms: 5000 -``` +StreamForge exposes different reliability and scheduling controls. They are not +interchangeable: stronger throughput-oriented modes deliberately give up +features needed to associate a failed Kafka acknowledgement with its source +record. -**Behavior:** -- Batch consume 100 messages -- Process all messages in batch (with retries) -- Produce all messages to destination(s) -- **Commit offset of last message in batch** -- If any message fails permanently โ†’ send to DLQ, then commit - -**Guarantees:** -- โœ… At-least-once per batch -- โœ… High throughput (batch commits) -- โš ๏ธ Duplicates on batch-level failure - -**When offset is committed:** -```rust -// Pseudocode -for batch in consumer.consume_batch(100) { - let mut dlq_messages = Vec::new(); - - for msg in batch { - match process(msg) { - Ok(_) => {}, // Success - Err(e) if e.is_recoverable() => retry_with_backoff(msg)?, - Err(e) => dlq_messages.push((msg, e)), // DLQ - } - } - - // Send all DLQ messages - for (msg, err) in dlq_messages { - dlq.send(msg, err)?; - } - - // Commit offset after batch processed - consumer.commit()?; // โ† COMMIT HAPPENS HERE -} -``` +## What the modes mean -**Failure scenarios:** -1. **Process fails before produce:** - - Retry up to 3 times - - If exhausted โ†’ DLQ - - Offset NOT committed yet โ†’ message not lost +| Configuration | Processing and delivery behavior | Important limitation | +|---|---|---| +| Default auto commit + `acknowledged` delivery | Each send awaits Kafka's delivery result; Kafka commits offsets on its own interval | A commit is not coordinated with completion of application processing | +| Manual commit + `legacy_batch` + `acknowledged` delivery | A batch is committed only after every record in that batch completes successfully | A crash after delivery and before commit can produce duplicates | +| Auto commit + `partition_ordered` | Each source partition is assigned to a bounded FIFO worker lane | Explicit rebalance-safe offset coordination is not implemented | +| Auto commit + `queued` delivery | Processing returns after librdkafka accepts a record; acknowledgements are tracked in the background | Auto commit, one processing attempt, and a disabled DLQ are required | -2. **Produce succeeds, commit fails:** - - Offset NOT committed โ†’ batch reprocessed - - Messages produced again โ†’ **duplicates** +The default configuration uses auto commit for backward compatibility. Do not +describe the default as strictly at-least-once or exactly-once. -3. **Commit succeeds, crash before persist:** - - Offset committed but not persisted to Kafka - - On restart: batch reprocessed โ†’ **duplicates** +StreamForge does not implement Kafka transactions across source offsets and +destination records. Exactly-once delivery is therefore not a current +guarantee. ---- +## At-least-once operating profile -### 2. Manual Commit Per Message (Low Latency) +For pipelines that must not commit a batch before every destination send has +been acknowledged, use the legacy batch processor with manual commits: -**Configuration:** ```yaml commit_strategy: manual_commit: true commit_mode: sync - commit_interval: 1 # Commit every message -``` - -**Behavior:** -- Consume 1 message -- Process message (with retries) -- Produce message -- **Commit offset immediately** -- Next message -**Guarantees:** -- โœ… At-least-once per message -- โœ… Low latency (no batching) -- โš ๏ธ Lower throughput (commit overhead) +performance: + processing_mode: legacy_batch + producer_delivery_mode: acknowledged -**Trade-off:** Commits are expensive (~5ms), so throughput drops to ~200 msg/s per partition. - -**Use when:** -- Latency critical (real-time processing) -- Small message rate (<1K msg/s) -- Need fine-grained recovery - ---- - -### 3. Auto Commit (Kafka Default, Not Recommended) - -**Configuration:** -```yaml -commit_strategy: - manual_commit: false # Use Kafka auto-commit - auto_commit_interval_ms: 5000 -``` - -**Behavior:** -- Kafka commits offset every 5 seconds automatically -- **Independent of processing success** -- Messages processed may not be committed yet -- Committed messages may not be processed yet - -**Guarantees:** -- โš ๏ธ No guarantees (between at-most-once and at-least-once) -- โš ๏ธ Data loss possible (commit before process) -- โš ๏ธ Duplicates possible (process before commit) - -**Problems:** -``` -Timeline: -0s: Consume msg offset 100 -1s: Process msg 100 (takes 6 seconds) -5s: Auto-commit offset 100 โ† Message not processed yet! -6s: Process completes, produce succeeds -7s: Crash -Restart: Offset 100 already committed โ†’ message lost -``` - -**โš ๏ธ DO NOT USE for anything that requires delivery guarantees.** - ---- - -### 4. Time-Based Commit (Alternative) - -**Configuration:** -```yaml -commit_strategy: - manual_commit: true - commit_mode: async - commit_interval_ms: 30000 # Commit every 30 seconds -``` - -**Behavior:** -- Process messages continuously -- Commit offset every 30 seconds (last successfully processed message) - -**Guarantees:** -- โœ… At-least-once -- โš ๏ธ Up to 30 seconds of duplicates on failure - -**Use when:** -- High throughput (minimize commit overhead) -- Duplicates acceptable (have idempotency elsewhere) - ---- - -## Commit Strategy Comparison - -| Strategy | Throughput | Latency | Duplicates on Failure | Data Loss | Recommended | -|----------|------------|---------|----------------------|-----------|-------------| -| Manual (batch) | ~35K msg/s | ~100ms | ~100 messages | Never | โœ… **Yes** | -| Manual (per-msg) | ~200 msg/s | ~5ms | 1 message | Never | Low rate only | -| Auto commit | ~35K msg/s | Variable | Variable | Possible | โŒ **Never** | -| Time-based | ~40K msg/s | Variable | ~30s worth | Never | High throughput | - ---- - -## Retry Policy - -### Exponential Backoff - -**Default configuration:** -```yaml retry: max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 30000 - multiplier: 2.0 - jitter: 0.1 # 10% random jitter -``` - -**Retry schedule:** -- Attempt 1: Immediate (0ms) -- Attempt 2: 100ms + jitter (90-110ms) -- Attempt 3: 200ms + jitter (180-220ms) -- Attempt 4: 400ms + jitter (360-440ms) -- Failed: Send to DLQ - -**Which errors are retried:** -- โœ… `KafkaProducer { recoverable: true }` -- โœ… `KafkaConsumer { recoverable: true }` -- โœ… `OffsetCommit` (always) -- โœ… `Redis` (cache failures) -- โœ… `Io` (network failures) -- โŒ `MessageDeserialization` (bad data, not transient) -- โŒ `FilterEvaluation` (logic error, not transient) -- โŒ `Config` (cannot retry, needs restart) - -**Retry logic:** -```rust -async fn process_with_retry(msg: Message) -> Result<()> { - let mut delay = config.retry.initial_delay_ms; - - for attempt in 1..=config.retry.max_attempts { - match process_message(msg).await { - Ok(_) => return Ok(()), - Err(e) if e.is_recoverable() && attempt < config.retry.max_attempts => { - warn!("Attempt {} failed: {}, retrying in {}ms", attempt, e, delay); - sleep(Duration::from_millis(delay)).await; - delay = (delay * config.retry.multiplier as u64).min(config.retry.max_delay_ms); - } - Err(e) => return Err(e), // Not recoverable or exhausted - } - } - - Err(MirrorMakerError::RetryExhausted { - message: "Max retry attempts reached".into(), - attempts: config.retry.max_attempts, - last_error: "...".into(), - }) -} -``` - ---- - -## Dead Letter Queue - -### Purpose -Messages that **cannot be processed** (permanent failures) are sent to a DLQ to: -1. Prevent pipeline halt (skip bad messages) -2. Enable manual inspection and replay -3. Maintain observability (what's failing and why) - -### DLQ Message Format - -**Headers added:** -``` -x-streamforge-error: "JSON path not found: /user/email" -x-streamforge-error-type: "JsonPathNotFound" -x-streamforge-source-topic: "input-topic" -x-streamforge-source-partition: "3" -x-streamforge-source-offset: "12345" -x-streamforge-timestamp: "2026-04-18T10:30:00Z" -x-streamforge-pipeline: "my-pipeline" -x-streamforge-destination: "output-topic" -x-streamforge-filter: "/status,==,active" -x-streamforge-transform: "EXTRACT:/user/email,userEmail" -``` - -**Key and value:** Original message unchanged - -**Example DLQ message:** -```json -{ - "headers": { - "x-streamforge-error": "JSON path not found: /user/email", - "x-streamforge-error-type": "JsonPathNotFound", - "x-streamforge-source-topic": "events", - "x-streamforge-source-partition": "3", - "x-streamforge-source-offset": "12345", - "x-streamforge-timestamp": "2026-04-18T10:30:00.123Z" - }, - "key": {"userId": "user-123"}, - "value": { - "event": "login", - "timestamp": 1234567890, - "user": {"id": "user-123"} // Note: no "email" field - } -} -``` - -### DLQ Configuration - -```yaml -dead_letter_queue: +dlq: enabled: true - topic: "streamforge-dlq" - - # Include original headers - include_original_headers: true - - # Include stack trace in error header - include_stack_trace: false - - # DLQ producer settings (can be different from main) - brokers: "kafka-dlq:9092" - compression: "none" - - # Max retries to send to DLQ (if DLQ fails, halt pipeline) + topic: streamforge-dlq max_dlq_retries: 3 ``` -### Which Errors Go to DLQ - -โœ… **Sent to DLQ:** -- `MessageDeserialization` (bad JSON) -- `FilterEvaluation` (filter threw exception) -- `TransformEvaluation` (transform threw exception) -- `JsonPathNotFound` (missing field) -- `Compression` / `Decompression` (corrupt data) -- `RetryExhausted` (after max retries) - -โŒ **NOT sent to DLQ (halt instead):** -- `Config` (fix and restart) -- `DslParse` (fix and restart) -- `DeadLetterQueue` (cannot lose data if DLQ fails) - -### DLQ Failure Handling - -**What if DLQ produce fails?** - -```rust -match send_to_dlq(msg, error) { - Ok(_) => { - // DLQ succeeded, continue processing - metrics.dlq_messages_total.inc(); - } - Err(dlq_error) => { - // DLQ FAILED - this is critical - error!( - "CRITICAL: Failed to send message to DLQ: {}\n\ - Original error: {}\n\ - Message: {:?}", - dlq_error, error, msg - ); - - // Retry DLQ send (up to 3 attempts) - for attempt in 1..=3 { - match retry_dlq_send(msg, error) { - Ok(_) => break, - Err(e) if attempt == 3 => { - // DLQ exhausted - HALT PIPELINE - // Cannot lose data - return Err(MirrorMakerError::DeadLetterQueue { - message: "DLQ send exhausted".into(), - dlq_topic: config.dlq.topic.clone(), - }); - } - _ => sleep(Duration::from_secs(1)), - } - } - } -} -``` - -**Why halt on DLQ failure?** -- Cannot lose data -- DLQ failure indicates serious problem (DLQ topic missing, broker down) -- Better to halt and alert than silently drop messages - ---- - -## Failure Scenarios - -### Scenario 1: Produce Failure (Transient) - -**Sequence:** -1. Consume message offset 100 -2. Process succeeds -3. Produce fails (queue full) -4. **Retry:** Wait 100ms, retry produce -5. Produce succeeds -6. Commit offset 100 -7. Continue - -**Outcome:** โœ… Message delivered once, no duplicates - ---- - -### Scenario 2: Produce Failure (Exhausted) - -**Sequence:** -1. Consume message offset 100 -2. Process succeeds -3. Produce fails (queue full) -4. Retry 1: Fails (still full) -5. Retry 2: Fails (still full) -6. Retry 3: Fails (still full) -7. **Send to DLQ** with error metadata -8. Commit offset 100 -9. Continue with offset 101 +In this profile: -**Outcome:** โœ… Bad message in DLQ, pipeline continues +1. StreamForge consumes a bounded batch. +2. Records in the batch may process concurrently. +3. Each destination send waits for Kafka's delivery result. +4. Recoverable failures are retried according to the retry policy. +5. Errors whose recovery action is DLQ are acknowledged only after the DLQ send + succeeds. +6. StreamForge commits the consumer state only when the whole batch succeeds. +7. A failed batch or exhausted commit retry stops the pipeline. ---- - -### Scenario 3: Commit Failure (Transient) - -**Sequence:** -1. Consume messages offset 100-199 (batch of 100) -2. Process all messages -3. Produce all messages -4. Commit offset 199 fails (coordinator unavailable) -5. **Retry commit:** Wait 100ms, retry -6. Commit succeeds -7. Continue with offset 200 - -**Outcome:** โœ… Batch committed, no duplicates - ---- +This is an at-least-once operating profile, so downstream consumers must tolerate +duplicates. A record can be delivered and then replayed if StreamForge stops +before its source offset is committed. -### Scenario 4: Commit Failure (Exhausted) - -**Sequence:** -1. Consume messages offset 100-199 -2. Process and produce all messages successfully -3. Commit offset 199 fails -4. Retry 1: Fails -5. Retry 2: Fails -6. Retry 3: Fails (exhausted) -7. **HALT PIPELINE** (cannot continue without committing) -8. On restart: Re-consume from offset 100 - -**Outcome:** โš ๏ธ Duplicates (messages 100-199 produced twice) - -**Why halt:** If we continue without committing, on restart we'd reprocess from offset 0, creating many more duplicates. - ---- +`commit_interval_ms` is present in the configuration schema, but the current +legacy loop commits after successful processing batches. Size batches with the +`performance.consumer_batch_size` and +`performance.consumer_batch_timeout_ms` controls. -### Scenario 5: Crash After Produce, Before Commit +## Ordering -**Sequence:** -1. Consume messages offset 100-199 -2. Process and produce all successfully -3. About to commit offset 199 -4. **CRASH** (pod killed, OOM, etc.) -5. On restart: Consumer resumes from last committed offset (99) -6. Re-consume and reprocess messages 100-199 -7. **Duplicates produced** +- Kafka defines order within a source partition, not across partitions. +- `legacy_batch` processes records concurrently and does not promise completion + order within a batch. +- `partition_ordered` provides FIFO worker lanes for source partitions, but it + currently requires auto commit. +- Destination partition selection can change ordering. Preserve a stable key or + explicitly choose a suitable partitioning field when order matters. +- Adding source partitions can change the mapping of keyed records. -**Outcome:** โš ๏ธ Duplicates (messages 100-199 produced twice) - -**Why:** Kafka doesn't know about uncommitted work. This is inherent to at-least-once. - -**Mitigation:** Use idempotency keys in messages, or implement exactly-once (v1.1+). - ---- - -### Scenario 6: DLQ Failure - -**Sequence:** -1. Consume message offset 100 -2. Deserialization fails (bad JSON) -3. Send to DLQ โ†’ DLQ broker unavailable -4. Retry DLQ send: Attempt 1 fails -5. Retry DLQ send: Attempt 2 fails -6. Retry DLQ send: Attempt 3 fails -7. **HALT PIPELINE** - -**Outcome:** โŒ Pipeline stopped, message not lost (offset not committed) - -**Manual recovery:** -1. Fix DLQ topic/broker -2. Restart StreamForge -3. Message 100 reprocessed and sent to DLQ - ---- +Do not claim both strict per-partition processing order and the manual-commit +profile until offset coordination for `partition_ordered` is implemented and +verified. -### Scenario 7: Rebalance During Processing +## Queued delivery -**Sequence:** -1. Consume message offset 100 from partition 3 -2. Processing (takes 5 seconds) -3. **Rebalance:** Another consumer joins group -4. Partition 3 revoked from this consumer -5. Processing completes, but **commit fails** (no longer own partition) -6. New consumer for partition 3 starts from last committed offset (99) -7. Message 100 reprocessed - -**Outcome:** โš ๏ธ Duplicate (message 100 processed twice) - -**Why:** In-flight messages during rebalance are not committed. - -**Mitigation:** Process quickly (<5s) to minimize rebalance window. - ---- - -## Configuration - -### Recommended Production Config +Queued delivery is opt-in: ```yaml -# At-least-once with batch commit (high throughput) commit_strategy: - manual_commit: true - commit_mode: async - commit_interval: 100 # Batch size - commit_timeout_ms: 5000 + manual_commit: false -# Retry with exponential backoff -retry: - max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 30000 - multiplier: 2.0 - jitter: 0.1 +performance: + producer_delivery_mode: queued + producer_max_in_flight: 10000 -# Dead letter queue -dead_letter_queue: - enabled: true - topic: "streamforge-dlq" - brokers: "kafka:9092" - include_original_headers: true - max_dlq_retries: 3 - -# Error handling -error_handling: - missing_field_behavior: "error" # Send to DLQ - null_value_behavior: "passthrough" - cache_failure_behavior: "skip" # Don't halt on cache failures -``` - -### Low-Latency Config - -```yaml -# Per-message commit (low latency) -commit_strategy: - manual_commit: true - commit_mode: sync # Wait for commit to complete - commit_interval: 1 - -# Aggressive retry retry: - max_attempts: 5 - initial_delay_ms: 10 - max_delay_ms: 1000 + max_attempts: 1 -# Same DLQ config... +dlq: + enabled: false ``` -### High-Throughput Config +Configuration validation rejects queued delivery with manual commits, multiple +processing attempts, or an enabled DLQ. Delayed delivery failures cannot retain +the original envelope for retry or DLQ routing. -```yaml -# Large batch commit (maximize throughput) -commit_strategy: - manual_commit: true - commit_mode: async - commit_interval: 1000 # Large batch - -# Same retry config... -``` - ---- - -## Observability - -### Metrics - -```promql -# Messages consumed -rate(streamforge_messages_consumed_total[5m]) - -# Messages produced -rate(streamforge_messages_produced_total[5m]) +Use `streamforge_messages_delivered_total` to observe broker-acknowledged +deliveries. Enqueue or processor completion is not a delivery guarantee. -# Commit successes -rate(streamforge_commits_total{result="success"}[5m]) +## Dead-letter queue behavior -# Commit failures (should be near zero) -rate(streamforge_commits_total{result="failure"}[5m]) +The DLQ producer uses acknowledgements from all in-sync replicas and retries its +own send. A successful DLQ send allows the source record to count as processed; +an exhausted DLQ send returns an error and prevents the manual-commit batch from +committing. -# Retry attempts -rate(streamforge_retries_total[5m]) +DLQ records retain the original key and value and add `x-streamforge-*` error +metadata. Protect the DLQ like the source topic: it can contain original payloads +and headers. -# DLQ messages (should be low) -rate(streamforge_dlq_messages_total[5m]) +Before enabling a DLQ: -# Processing lag (commit lag) -streamforge_consumer_lag{partition="3"} -``` - -### Alerts - -```yaml -# High commit failure rate -- alert: HighCommitFailureRate - expr: rate(streamforge_commits_total{result="failure"}[5m]) > 0.01 - for: 5m - annotations: - summary: "High commit failure rate" - -# DLQ growing rapidly -- alert: DLQBacklog - expr: rate(streamforge_dlq_messages_total[1h]) > 100 - for: 10m - annotations: - summary: "DLQ receiving >100 msg/hour" - -# Lag increasing -- alert: ConsumerLagIncreasing - expr: streamforge_consumer_lag > 10000 - for: 5m - annotations: - summary: "Consumer lag > 10K messages" -``` +- create the topic with appropriate replication and retention; +- restrict read and write ACLs; +- alert on DLQ writes and delivery failures; +- test inspection, correction, and replay; +- make replay idempotent and preserve an audit trail. ---- +## Failure matrix -## Testing Delivery Guarantees - -### Integration Tests - -```rust -#[tokio::test] -async fn test_at_least_once_with_produce_retry() { - // Setup: Producer that fails first 2 attempts - let producer = MockProducer::new() - .fail_times(2) - .then_succeed(); - - // Act: Process message - process_message(msg, producer).await.unwrap(); - - // Assert: Message produced exactly once - assert_eq!(producer.send_count(), 1); - assert_eq!(producer.attempt_count(), 3); -} - -#[tokio::test] -async fn test_bad_message_goes_to_dlq() { - let dlq = MockDlq::new(); - let invalid_json = b"not valid json"; - - process_message(invalid_json, dlq).await.unwrap(); - - // Assert: Original message in DLQ - assert_eq!(dlq.message_count(), 1); - assert_eq!(dlq.messages()[0].value, invalid_json); - assert!(dlq.messages()[0].headers.contains_key("x-streamforge-error")); -} - -#[tokio::test] -async fn test_commit_failure_causes_duplicate() { - let consumer = MockConsumer::new() - .with_message(100, "test") - .commit_fails_once(); - - // First attempt: produce succeeds, commit fails - process_batch(&consumer).await.unwrap_err(); - - // Restart: reprocess from last committed offset - let consumer = MockConsumer::new() - .from_offset(100) // Offset not committed - .with_message(100, "test"); - - process_batch(&consumer).await.unwrap(); - - // Assert: Message produced twice - assert_eq!(producer.send_count(), 2); -} -``` +| Failure | Manual acknowledged profile | Likely result | +|---|---|---| +| Destination rejects a record | Processing retries, routes to DLQ, or fails | No source commit until the configured recovery succeeds | +| DLQ send fails | Batch fails | Source records remain eligible for replay | +| Destination succeeds, process stops before commit | Offset is not committed | Duplicate delivery is possible | +| Commit retries are exhausted | Pipeline stops | Successfully delivered records in the batch may replay | +| Consumer group rebalances during work | Ownership can change | Duplicate processing is possible | +| Queued delivery fails after enqueue | Failure is recorded asynchronously | Original record cannot be retried or sent to DLQ | ---- +## Production verification -## References +Exercise delivery behavior with failures, not only a steady-state load: -- **ERROR_HANDLING.md**: Error types and recovery actions -- **src/error.rs**: Error type implementation -- **PROJECT_SPEC.md ยง5**: Delivery guarantees requirements -- **V1_PLAN.md Phase 1**: Core engine hardening - ---- +1. stop the destination broker during processing; +2. interrupt StreamForge after delivery but before a manual commit; +3. make the DLQ unavailable; +4. add and remove a consumer to force a rebalance; +5. restart from the last committed offsets; +6. verify destination duplicates, missing records, DLQ records, and committed + offsets against the expected profile. -**Status:** Specification complete, implementation in progress -**Version:** 1.0.0-alpha.1 -**Phase:** 1 (Core Engine Hardening) +See [Observability](OBSERVABILITY_QUICKSTART.md) for the metrics endpoints and +[Troubleshooting](TROUBLESHOOTING.md) for incident procedures. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d6da596..696b22f 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,1683 +1,148 @@ -# StreamForge Deployment Guide - -**Version:** 1.0.0 -**Last Updated:** 2026-04-18 - -This guide covers deploying StreamForge in production environments using Docker, Kubernetes, Helm, and the Kubernetes Operator. - ---- - -## Table of Contents - -1. [Prerequisites](#prerequisites) -2. [Docker Deployment](#docker-deployment) -3. [Kubernetes Deployment](#kubernetes-deployment) -4. [Helm Chart Deployment](#helm-chart-deployment) -5. [Operator Deployment](#operator-deployment) -6. [Multi-Cluster Setup](#multi-cluster-setup) -7. [Production Best Practices](#production-best-practices) -8. [Security Hardening](#security-hardening) -9. [Monitoring and Observability](#monitoring-and-observability) -10. [Configuration Management](#configuration-management) - ---- - -## Prerequisites - -### Required -- Kafka cluster (2.8+) with accessible bootstrap servers -- Docker (20.10+) or Kubernetes (1.21+) -- Network connectivity between StreamForge and Kafka brokers -- TLS certificates (if using SSL/SASL) - -### Recommended -- Prometheus for metrics collection -- Grafana for dashboards -- Persistent volume for DLQ messages -- Redis for distributed caching (optional) - -### Resource Requirements - -**Minimum (Development):** -- CPU: 1 core -- Memory: 512 MB -- Disk: 1 GB - -**Production (per pipeline):** -- CPU: 2-4 cores -- Memory: 2-4 GB -- Disk: 10 GB (for logs, DLQ) -- Network: 1 Gbps - -**Scaling:** -- 1 core per ~20K msg/s throughput -- 1 GB memory per 100K msg/s for JSON processing -- Increase threads parameter for CPU-bound workloads - ---- - -## Docker Deployment - -### 1. Build Docker Image - -**Dockerfile:** -```dockerfile -FROM rust:1.75-slim as builder - -WORKDIR /app -COPY Cargo.toml Cargo.lock ./ -COPY src ./src -COPY benches ./benches - -# Build release binary -RUN cargo build --release --bin streamforge - -# Runtime image -FROM debian:bookworm-slim - -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - libsasl2-2 \ - libzstd1 \ - && rm -rf /var/lib/apt/lists/* - -# Copy binary -COPY --from=builder /app/target/release/streamforge /usr/local/bin/ - -# Create non-root user -RUN useradd -m -u 1000 streamforge -USER streamforge - -WORKDIR /app - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 - -EXPOSE 8080 - -ENTRYPOINT ["/usr/local/bin/streamforge"] -CMD ["--config", "/app/config.yaml"] -``` - -**Build:** -```bash -docker build -t streamforge:1.0.0 . -docker tag streamforge:1.0.0 streamforge:latest -``` - -### 2. Run with Docker - -**Simple run:** -```bash -docker run -d \ - --name streamforge \ - -v $(pwd)/config.yaml:/app/config.yaml:ro \ - -p 8080:8080 \ - streamforge:1.0.0 -``` - -**With environment variables:** -```bash -docker run -d \ - --name streamforge \ - -e KAFKA_BOOTSTRAP=kafka.example.com:9092 \ - -e RUST_LOG=info \ - -e RUST_BACKTRACE=1 \ - -v $(pwd)/config.yaml:/app/config.yaml:ro \ - -v $(pwd)/certs:/app/certs:ro \ - -p 8080:8080 \ - streamforge:1.0.0 -``` - -### 3. Docker Compose - -**docker-compose.yml:** -```yaml -version: '3.8' - -services: - streamforge: - image: streamforge:1.0.0 - container_name: streamforge - restart: unless-stopped - ports: - - "8080:8080" - volumes: - - ./config.yaml:/app/config.yaml:ro - - ./certs:/app/certs:ro - - streamforge-data:/app/data - environment: - RUST_LOG: info - RUST_BACKTRACE: 1 - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/health"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s - networks: - - kafka-network - depends_on: - - kafka - - # Optional: Local Kafka for testing - kafka: - image: docker.redpanda.com/redpandadata/redpanda:latest - command: - - redpanda - - start - - --smp - - '1' - - --reserve-memory - - 0M - - --overprovisioned - - --node-id - - '0' - - --kafka-addr - - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 - - --advertise-kafka-addr - - PLAINTEXT://kafka:29092,OUTSIDE://localhost:9092 - ports: - - "9092:9092" - - "29092:29092" - networks: - - kafka-network - - # Optional: Prometheus - prometheus: - image: prom/prometheus:latest - ports: - - "9090:9090" - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - - prometheus-data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - networks: - - kafka-network - - # Optional: Grafana - grafana: - image: grafana/grafana:latest - ports: - - "3000:3000" - volumes: - - grafana-data:/var/lib/grafana - - ./grafana/dashboards:/etc/grafana/provisioning/dashboards:ro - - ./grafana/datasources:/etc/grafana/provisioning/datasources:ro - environment: - GF_SECURITY_ADMIN_PASSWORD: admin - networks: - - kafka-network - -volumes: - streamforge-data: - prometheus-data: - grafana-data: - -networks: - kafka-network: - driver: bridge -``` - -**prometheus.yml:** -```yaml -global: - scrape_interval: 15s - -scrape_configs: - - job_name: 'streamforge' - static_configs: - - targets: ['streamforge:8080'] -``` - -**Start:** -```bash -docker-compose up -d -docker-compose logs -f streamforge -``` - --- - -## Kubernetes Deployment - -### 1. Namespace - -**namespace.yaml:** -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: streamforge - labels: - name: streamforge -``` - -### 2. ConfigMap - -**configmap.yaml:** -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: streamforge-config - namespace: streamforge -data: - config.yaml: | - appid: "streamforge-prod" - bootstrap: "kafka.kafka.svc.cluster.local:9092" - input: "source-topic" - offset: "latest" - threads: 4 - - # Performance tuning - performance: - fetch_min_bytes: 1024 - fetch_max_wait_ms: 100 - queue_buffering_max_ms: 5 - batch_size: 1000 - linger_ms: 10 - - # Retry and DLQ - retry: - max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 30000 - jitter: true - - dlq: - enabled: true - topic: "streamforge-dlq" - include_error_headers: true - - # Routing - routing: - routing_type: "filter" - destinations: - - output: "filtered-topic" - filter: "/status,==,active" - transform: "/data" - key_transform: "/user/id" - headers: - x-pipeline: "streamforge" - - # Observability - metrics: - enabled: true - port: 8080 - path: "/metrics" -``` - -### 3. Secret (for TLS/SASL) - -**secret.yaml:** -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: streamforge-kafka-certs - namespace: streamforge -type: Opaque -data: - ca.crt: - client.crt: - client.key: - sasl-password: -``` - -**Create from files:** -```bash -kubectl create secret generic streamforge-kafka-certs \ - --from-file=ca.crt=./certs/ca.crt \ - --from-file=client.crt=./certs/client.crt \ - --from-file=client.key=./certs/client.key \ - --from-literal=sasl-password="your-password" \ - -n streamforge -``` - -### 4. Deployment - -**deployment.yaml:** -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: streamforge - namespace: streamforge - labels: - app: streamforge - version: "1.0.0" -spec: - replicas: 2 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - app: streamforge - template: - metadata: - labels: - app: streamforge - version: "1.0.0" - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: streamforge - securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 1000 - - containers: - - name: streamforge - image: streamforge:1.0.0 - imagePullPolicy: IfNotPresent - - args: - - "--config" - - "/app/config.yaml" - - ports: - - name: metrics - containerPort: 8080 - protocol: TCP - - env: - - name: RUST_LOG - value: "info" - - name: RUST_BACKTRACE - value: "1" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - volumeMounts: - - name: config - mountPath: /app/config.yaml - subPath: config.yaml - readOnly: true - - name: certs - mountPath: /app/certs - readOnly: true - - name: data - mountPath: /app/data - - resources: - requests: - cpu: 1000m - memory: 2Gi - limits: - cpu: 2000m - memory: 4Gi - - livenessProbe: - httpGet: - path: /health - port: metrics - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - - readinessProbe: - httpGet: - path: /health - port: metrics - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 2 - - volumes: - - name: config - configMap: - name: streamforge-config - - name: certs - secret: - secretName: streamforge-kafka-certs - - name: data - emptyDir: {} - - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - app: streamforge - topologyKey: kubernetes.io/hostname -``` - -### 5. Service - -**service.yaml:** -```yaml -apiVersion: v1 -kind: Service -metadata: - name: streamforge - namespace: streamforge - labels: - app: streamforge -spec: - type: ClusterIP - ports: - - name: metrics - port: 8080 - targetPort: metrics - protocol: TCP - selector: - app: streamforge -``` - -### 6. ServiceAccount and RBAC - -**rbac.yaml:** -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: streamforge - namespace: streamforge ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: streamforge - namespace: streamforge -rules: -- apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: streamforge - namespace: streamforge -subjects: -- kind: ServiceAccount - name: streamforge - namespace: streamforge -roleRef: - kind: Role - name: streamforge - apiGroup: rbac.authorization.k8s.io -``` - -### 7. HorizontalPodAutoscaler - -**hpa.yaml:** -```yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: streamforge - namespace: streamforge -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: streamforge - minReplicas: 2 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - behavior: - scaleUp: - stabilizationWindowSeconds: 60 - policies: - - type: Percent - value: 50 - periodSeconds: 60 - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 25 - periodSeconds: 60 -``` - -### 8. Deploy to Kubernetes - -```bash -# Create namespace -kubectl apply -f namespace.yaml - -# Create secrets and config -kubectl apply -f secret.yaml -kubectl apply -f configmap.yaml - -# Create RBAC -kubectl apply -f rbac.yaml - -# Deploy application -kubectl apply -f deployment.yaml -kubectl apply -f service.yaml -kubectl apply -f hpa.yaml - -# Verify deployment -kubectl get pods -n streamforge -kubectl logs -f deployment/streamforge -n streamforge -kubectl get svc -n streamforge -``` - ---- - -## Helm Chart Deployment - -### 1. Install Helm Chart - -**Add repository:** -```bash -helm repo add streamforge https://streamforge.io/helm-charts -helm repo update -``` - -**Install:** -```bash -helm install streamforge streamforge/streamforge \ - --namespace streamforge \ - --create-namespace \ - --values values.yaml -``` - -### 2. Custom Values - -**values.yaml:** -```yaml -# Image configuration -image: - repository: streamforge - tag: "1.0.0" - pullPolicy: IfNotPresent - -# Replica count -replicaCount: 2 - -# Resources -resources: - requests: - cpu: 1000m - memory: 2Gi - limits: - cpu: 2000m - memory: 4Gi - -# Autoscaling -autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 10 - targetCPUUtilizationPercentage: 70 - targetMemoryUtilizationPercentage: 80 - -# Service -service: - type: ClusterIP - port: 8080 - -# Ingress (if needed) -ingress: - enabled: false - className: nginx - annotations: {} - hosts: - - host: streamforge.example.com - paths: - - path: / - pathType: Prefix - tls: [] - -# StreamForge configuration -config: - appid: "streamforge-prod" - bootstrap: "kafka.kafka.svc.cluster.local:9092" - input: "source-topic" - offset: "latest" - threads: 4 - - performance: - fetch_min_bytes: 1024 - fetch_max_wait_ms: 100 - queue_buffering_max_ms: 5 - batch_size: 1000 - linger_ms: 10 - - retry: - max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 30000 - jitter: true - - dlq: - enabled: true - topic: "streamforge-dlq" - include_error_headers: true - - routing: - routing_type: "filter" - destinations: - - output: "filtered-topic" - filter: "/status,==,active" - transform: "/data" - key_transform: "/user/id" - - metrics: - enabled: true - port: 8080 - -# Kafka TLS/SASL -kafka: - tls: - enabled: false - ca: "" - cert: "" - key: "" - sasl: - enabled: false - mechanism: "PLAIN" - username: "" - password: "" - -# Monitoring -monitoring: - enabled: true - serviceMonitor: - enabled: true - interval: 30s - -# Security -securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 1000 - -podSecurityContext: - runAsNonRoot: true - runAsUser: 1000 - -# Affinity -affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - app: streamforge - topologyKey: kubernetes.io/hostname -``` - -### 3. Upgrade - -```bash -helm upgrade streamforge streamforge/streamforge \ - --namespace streamforge \ - --values values.yaml \ - --wait -``` - -### 4. Rollback - -```bash -helm rollback streamforge -n streamforge -``` - ---- - -## Operator Deployment - -### 1. Install Operator - -```bash -kubectl apply -f https://streamforge.io/operator/install.yaml -``` - -**Or with Helm:** -```bash -helm install streamforge-operator streamforge/operator \ - --namespace streamforge-system \ - --create-namespace -``` - -### 2. Create StreamforgePipeline CRD - -**pipeline.yaml:** -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: user-filtering - namespace: streamforge -spec: - image: streamforge:1.0.0 - replicas: 2 - - resources: - requests: - cpu: 1000m - memory: 2Gi - limits: - cpu: 2000m - memory: 4Gi - - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 10 - targetCPU: 70 - targetMemory: 80 - - kafka: - bootstrap: "kafka.kafka.svc.cluster.local:9092" - tls: - enabled: false - sasl: - enabled: false - - pipeline: - appid: "user-filtering" - input: "users" - offset: "latest" - threads: 4 - - performance: - fetch_min_bytes: 1024 - batch_size: 1000 - linger_ms: 10 - - retry: - maxAttempts: 3 - initialDelay: 100ms - maxDelay: 30s - jitter: true - - dlq: - enabled: true - topic: "user-filtering-dlq" - - routing: - type: filter - destinations: - - output: "active-users" - filter: "/status,==,active" - transform: "/data" - keyTransform: "/user/id" - headers: - x-pipeline: "user-filtering" - - - output: "premium-users" - filter: "/tier,==,premium" - transform: "/data" - - monitoring: - enabled: true - serviceMonitor: true -``` - -### 3. Apply Pipeline - -```bash -kubectl apply -f pipeline.yaml - -# Check status -kubectl get streamforgepipeline -n streamforge -kubectl describe streamforgepipeline user-filtering -n streamforge - -# View generated resources -kubectl get deploy,svc,hpa -n streamforge -l pipeline=user-filtering -``` - -### 4. Update Pipeline - -```bash -# Edit in-place -kubectl edit streamforgepipeline user-filtering -n streamforge - -# Or apply updated YAML -kubectl apply -f pipeline.yaml -``` - -### 5. Delete Pipeline - -```bash -kubectl delete streamforgepipeline user-filtering -n streamforge -``` - +title: Deployment +nav_order: 6 +has_children: true --- -## Multi-Cluster Setup - -### Architecture +# Deploy StreamForge -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Cluster A โ”‚ โ”‚ Cluster B โ”‚ -โ”‚ (us-east-1) โ”‚ โ”‚ (us-west-2) โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Kafka A โ”‚ โ”‚ โ”‚ โ”‚ Kafka B โ”‚ โ”‚ -โ”‚ โ”‚ (source) โ”‚ โ”‚ โ”‚ โ”‚ (target) โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ -โ”‚ โ–ผ โ”‚ โ”‚ โ–ผ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ WAN โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚StreamForge โ”‚โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บโ”‚ โ”‚ Consumer โ”‚ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ Apps โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` +StreamForge runs as a native binary, a container, or a Kubernetes workload. The +repository also includes a Kubernetes operator and a local Helm chart. Choose +the smallest deployment model that fits your operating environment. -### 1. Cross-Cluster Replication +## Choose a deployment model -**Scenario:** Replicate from Kafka A (us-east-1) to Kafka B (us-west-2) +| Model | Best for | Start here | +|---|---|---| +| Native binary | Development, diagnostics, and controlled hosts | This page | +| Container | A single managed pipeline or container platform | [Podman](DOCKER.md) | +| Kubernetes Deployment | Teams that manage application manifests directly | [Kubernetes](KUBERNETES.md) | +| Kubernetes operator | Multiple declarative `StreamforgePipeline` resources | [Kubernetes](KUBERNETES.md) | -**config-us-east-1.yaml:** -```yaml -appid: "cross-region-replication" -bootstrap: "kafka-a.us-east-1.internal:9092" -input: "events" -offset: "earliest" # or "latest" for new messages only -threads: 8 +The Helm chart in this repository is installed from a local checkout. The +documentation does not assume that a public chart repository or supported +prebuilt image is available. -# Source cluster TLS/SASL -kafka: - security: - protocol: "SASL_SSL" - sasl_mechanism: "PLAIN" - sasl_username: "${KAFKA_A_USER}" - sasl_password: "${KAFKA_A_PASSWORD}" - ssl: - ca_location: "/certs/kafka-a-ca.crt" +## Production prerequisites -# Performance for cross-region -performance: - fetch_min_bytes: 10240 # 10 KB - larger batches - fetch_max_wait_ms: 500 - batch_size: 5000 - linger_ms: 50 - compression: "zstd" # compress for WAN +- Source and destination Kafka endpoints reachable from the StreamForge runtime +- Topics, partitions, replication, retention, and ACLs created by the Kafka + administrator +- A configuration file validated against the same StreamForge revision that + will be deployed +- Credentials supplied by the platform secret store, never committed with the + configuration +- CPU, memory, and replica counts established with a representative load test +- Prometheus access to the metrics endpoint through a private network path -retry: - max_attempts: 5 - initial_delay_ms: 500 - max_delay_ms: 60000 - jitter: true +Kafka and Kubernetes version compatibility is documented in +[Compatibility](COMPATIBILITY.md). -dlq: - enabled: true - topic: "cross-region-dlq" +## Build and validate -routing: - routing_type: "passthrough" - destinations: - - output: "events" - # Target cluster (Kafka B) - bootstrap: "kafka-b.us-west-2.internal:9092" - security: - protocol: "SASL_SSL" - sasl_mechanism: "PLAIN" - sasl_username: "${KAFKA_B_USER}" - sasl_password: "${KAFKA_B_PASSWORD}" - ssl: - ca_location: "/certs/kafka-b-ca.crt" - - # Optional: Filter for regional data - filter: "/region,==,us-east" - - # Preserve original keys and timestamps - partitioning: "default" - preserve_timestamp: true -``` +Build release binaries: -**Deploy:** ```bash -# Deploy in source cluster (us-east-1) -kubectl apply -f deployment-cross-region.yaml -n streamforge - -# Monitor lag -kubectl exec -it deployment/streamforge -n streamforge -- \ - kafka-consumer-groups --bootstrap-server kafka-a:9092 \ - --describe --group cross-region-replication +cargo build --release --locked \ + --bin streamforge \ + --bin streamforge-validate ``` -### 2. Active-Passive Failover +Validate the configuration before starting a rollout: -**Primary (Active):** -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: primary-pipeline - namespace: streamforge -spec: - replicas: 3 - kafka: - bootstrap: "kafka-primary.internal:9092" - pipeline: - input: "orders" - offset: "earliest" - routing: - destinations: - - output: "processed-orders" -``` - -**Secondary (Standby):** -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: secondary-pipeline - namespace: streamforge -spec: - replicas: 1 # standby mode - kafka: - bootstrap: "kafka-secondary.internal:9092" - pipeline: - input: "orders" - offset: "latest" # don't reprocess on failover - routing: - destinations: - - output: "processed-orders" -``` - -**Failover process:** ```bash -# 1. Detect primary failure -kubectl get pods -n streamforge | grep primary-pipeline - -# 2. Scale up secondary -kubectl scale deployment secondary-pipeline --replicas=3 -n streamforge - -# 3. Update DNS/load balancer to point to secondary Kafka - -# 4. Monitor lag -kubectl logs -f deployment/secondary-pipeline -n streamforge -``` - -### 3. Hub-and-Spoke Pattern - -**Hub cluster:** Aggregates from multiple sources - -```yaml -# Pipeline 1: us-east โ†’ hub -appid: "us-east-to-hub" -bootstrap: "kafka-us-east.internal:9092" -input: "regional-events" -routing: - destinations: - - output: "global-events" - bootstrap: "kafka-hub.internal:9092" - transform: "CONSTRUCT:region=us-east:data=/data" - -# Pipeline 2: eu-west โ†’ hub -appid: "eu-west-to-hub" -bootstrap: "kafka-eu-west.internal:9092" -input: "regional-events" -routing: - destinations: - - output: "global-events" - bootstrap: "kafka-hub.internal:9092" - transform: "CONSTRUCT:region=eu-west:data=/data" -``` - ---- - -## Production Best Practices - -### 1. High Availability - -**Multiple replicas:** -```yaml -spec: - replicas: 3 # minimum for HA - - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: streamforge - topologyKey: kubernetes.io/hostname -``` - -**Multiple availability zones:** -```yaml -spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: streamforge - topologyKey: topology.kubernetes.io/zone -``` - -**PodDisruptionBudget:** -```yaml -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: streamforge-pdb - namespace: streamforge -spec: - minAvailable: 2 - selector: - matchLabels: - app: streamforge -``` - -### 2. Resource Management - -**Set requests == limits for guaranteed QoS:** -```yaml -resources: - requests: - cpu: 2000m - memory: 4Gi - limits: - cpu: 2000m - memory: 4Gi -``` - -**Use ResourceQuotas:** -```yaml -apiVersion: v1 -kind: ResourceQuota -metadata: - name: streamforge-quota - namespace: streamforge -spec: - hard: - requests.cpu: "20" - requests.memory: "40Gi" - limits.cpu: "20" - limits.memory: "40Gi" - pods: "20" -``` - -### 3. Performance Tuning - -**Consumer tuning:** -```yaml -performance: - fetch_min_bytes: 10240 # Wait for 10 KB - fetch_max_wait_ms: 100 # Or 100 ms - max_partition_fetch_bytes: 1048576 # 1 MB per partition -``` - -**Producer tuning:** -```yaml -performance: - batch_size: 5000 # Batch up to 5000 messages - linger_ms: 50 # Wait 50 ms for batching - queue_buffering_max_ms: 100 - compression: "zstd" # Use zstd compression -``` - -**Threading:** -```yaml -threads: 8 # Match available CPU cores -``` - -### 4. Commit Strategy - -**For low latency (< 100 ms):** -```yaml -commit_strategy: "per-message" -``` - -**For high throughput (> 50K msg/s):** -```yaml -commit_strategy: "manual" -commit_interval_ms: 5000 # Commit every 5 seconds -``` - -**For balanced:** -```yaml -commit_strategy: "time-based" -commit_interval_ms: 1000 # Commit every 1 second +target/release/streamforge-validate config.yaml --fail-on-warnings ``` -### 5. DLQ Management +Start StreamForge by passing the configuration path through `CONFIG_FILE`: -**Enable DLQ:** -```yaml -dlq: - enabled: true - topic: "streamforge-dlq" - include_error_headers: true - max_retries: 3 -``` - -**Monitor DLQ:** ```bash -# Count DLQ messages -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group streamforge-dlq-monitor - -# Inspect DLQ message -kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic streamforge-dlq \ - --from-beginning \ - --property print.headers=true \ - --max-messages 1 +CONFIG_FILE=config.yaml RUST_LOG=info target/release/streamforge ``` -**Reprocess DLQ:** -```yaml -# Create reprocessing pipeline -appid: "dlq-reprocessor" -input: "streamforge-dlq" -offset: "earliest" -routing: - destinations: - - output: "original-topic" - # Fix the issue that caused DLQ - filter: "/error-type,!=,permanent" -``` +StreamForge also accepts JSON configuration. The filename extension determines +whether the runtime loads YAML or JSON. -### 6. Graceful Shutdown +## Minimal pipeline -**Kubernetes termination:** ```yaml -spec: - containers: - - name: streamforge - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "sleep 15"] - - terminationGracePeriodSeconds: 30 -``` - -**Signal handling:** -- StreamForge handles SIGTERM gracefully -- Stops consuming new messages -- Finishes processing in-flight messages -- Commits offsets -- Closes producers/consumers +appid: orders-replica +bootstrap: source-kafka.internal:9092 +target_broker: destination-kafka.internal:9092 +input: orders +output: orders-replica +offset: earliest +threads: 4 -### 7. Logging +commit_strategy: + manual_commit: true + commit_mode: sync -**Structured logging:** -```yaml -env: -- name: RUST_LOG - value: "streamforge=info,rdkafka=warn" -- name: RUST_LOG_FORMAT - value: "json" # for log aggregation +observability: + metrics_enabled: true + metrics_port: 9090 + lag_monitoring_enabled: true ``` -**Log aggregation:** -```yaml -# Sidecar for log shipping -- name: fluentd - image: fluent/fluentd:latest - volumeMounts: - - name: logs - mountPath: /var/log/streamforge -``` +Use [Security](SECURITY_CONFIGURATION.md) to add TLS or SASL. Review +[Delivery guarantees](DELIVERY_GUARANTEES.md) before selecting a commit or +producer delivery mode. ---- +## Network exposure -## Security Hardening +The metrics server listens on all interfaces when enabled and serves +`/metrics` and `/health` on the configured port. It does not provide +authentication or TLS. -### 1. TLS Configuration +Do not expose that port directly to the public internet. Restrict access with a +host firewall, container network, Kubernetes `ClusterIP` service and +`NetworkPolicy`, or a private load balancer. If remote access is required, +terminate authentication and TLS in a trusted private proxy. -**Enable TLS:** -```yaml -kafka: - security: - protocol: "SSL" - ssl: - ca_location: "/certs/ca.crt" - certificate_location: "/certs/client.crt" - key_location: "/certs/client.key" - key_password: "${SSL_KEY_PASSWORD}" -``` +Kafka listeners should likewise remain private wherever possible. Limit egress +to the required broker addresses and DNS, and grant only the topic and consumer +group permissions used by the pipeline. -**Kubernetes secret:** -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: kafka-tls - namespace: streamforge -type: kubernetes.io/tls -data: - ca.crt: - tls.crt: - tls.key: -``` +## Rollout sequence -### 2. SASL Authentication +1. Validate the configuration and confirm the source and destination topics. +2. Deploy one pipeline instance with production delivery settings. +3. Verify `/health`, broker connectivity, destination acknowledgements, error + counters, and consumer lag. +4. Produce a controlled test record and verify its key, value, headers, + timestamp, and destination. +5. Increase replicas only up to the useful source-partition parallelism. +6. Observe at least one consumer-group rebalance and a clean shutdown before + declaring the rollout complete. +7. Record the deployed image digest, configuration revision, and rollback + procedure. -**PLAIN:** -```yaml -kafka: - security: - protocol: "SASL_SSL" - sasl_mechanism: "PLAIN" - sasl_username: "${KAFKA_USER}" - sasl_password: "${KAFKA_PASSWORD}" -``` +## Updates and rollback -**SCRAM-SHA-512:** -```yaml -kafka: - security: - protocol: "SASL_SSL" - sasl_mechanism: "SCRAM-SHA-512" - sasl_username: "${KAFKA_USER}" - sasl_password: "${KAFKA_PASSWORD}" -``` +Treat configuration and image changes independently: -**OAuth:** -```yaml -kafka: - security: - protocol: "SASL_SSL" - sasl_mechanism: "OAUTHBEARER" - sasl_oauthbearer_config: | - client_id=${OAUTH_CLIENT_ID} - client_secret=${OAUTH_CLIENT_SECRET} - token_endpoint_url=${OAUTH_TOKEN_URL} -``` - -### 3. Secrets Management +- validate a new configuration before rollout; +- use immutable image tags or digests; +- retain the previous configuration and image reference; +- roll instances gradually to limit consumer-group churn; +- watch destination errors and lag throughout the change; +- roll back both image and configuration when their compatibility is uncertain. -**Use External Secrets Operator:** -```yaml -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: streamforge-kafka-credentials - namespace: streamforge -spec: - secretStoreRef: - name: aws-secrets-manager - kind: SecretStore - target: - name: kafka-credentials - data: - - secretKey: username - remoteRef: - key: streamforge/kafka/username - - secretKey: password - remoteRef: - key: streamforge/kafka/password -``` +Changing `appid` creates a different Kafka consumer group. Changing `offset` +only affects partitions without a committed offset for that group. Resetting +offsets can intentionally replay or skip records and should be performed only +with the pipeline stopped and an approved recovery plan. -**Or use HashiCorp Vault:** -```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: streamforge - namespace: streamforge - annotations: - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/role: "streamforge" - vault.hashicorp.com/agent-inject-secret-kafka: "secret/data/streamforge/kafka" -``` - -### 4. Network Policies - -**Restrict traffic:** -```yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: streamforge-netpol - namespace: streamforge -spec: - podSelector: - matchLabels: - app: streamforge - policyTypes: - - Ingress - - Egress - ingress: - - from: - - namespaceSelector: - matchLabels: - name: monitoring - ports: - - protocol: TCP - port: 8080 - egress: - - to: - - namespaceSelector: - matchLabels: - name: kafka - ports: - - protocol: TCP - port: 9092 - - protocol: TCP - port: 9093 - - to: - - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - protocol: UDP - port: 53 -``` - -### 5. Pod Security - -**PodSecurityPolicy (deprecated) or Pod Security Standards:** -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: streamforge - labels: - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/audit: restricted - pod-security.kubernetes.io/warn: restricted -``` - -**SecurityContext:** -```yaml -securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - seccompProfile: - type: RuntimeDefault -``` - -### 6. Image Security - -**Use distroless or minimal images:** -```dockerfile -FROM gcr.io/distroless/cc-debian11 -COPY --from=builder /app/target/release/streamforge / -USER nonroot:nonroot -ENTRYPOINT ["/streamforge"] -``` +## Readiness checklist -**Scan images:** -```bash -# Trivy -trivy image streamforge:1.0.0 - -# Grype -grype streamforge:1.0.0 -``` - ---- - -## Monitoring and Observability - -### 1. Prometheus Metrics - -**ServiceMonitor:** -```yaml -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: streamforge - namespace: streamforge -spec: - selector: - matchLabels: - app: streamforge - endpoints: - - port: metrics - interval: 30s - path: /metrics -``` - -**Key metrics to monitor:** -```promql -# Throughput -rate(streamforge_messages_consumed_total[5m]) -rate(streamforge_messages_produced_total[5m]) - -# Lag -streamforge_consumer_lag - -# Error rate -rate(streamforge_errors_total[5m]) - -# DLQ rate -rate(streamforge_dlq_messages_total[5m]) - -# Processing latency (p95) -histogram_quantile(0.95, rate(streamforge_processing_duration_seconds_bucket[5m])) - -# Retry rate -rate(streamforge_retries_total[5m]) -``` - -### 2. Grafana Dashboards - -**Dashboard JSON:** -```json -{ - "dashboard": { - "title": "StreamForge Pipeline", - "panels": [ - { - "title": "Message Throughput", - "targets": [{ - "expr": "rate(streamforge_messages_consumed_total[5m])" - }] - }, - { - "title": "Consumer Lag", - "targets": [{ - "expr": "streamforge_consumer_lag" - }] - }, - { - "title": "Error Rate", - "targets": [{ - "expr": "rate(streamforge_errors_total[5m])" - }] - } - ] - } -} -``` - -### 3. Alerting Rules - -**prometheus-rules.yaml:** -```yaml -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: streamforge-alerts - namespace: streamforge -spec: - groups: - - name: streamforge - interval: 30s - rules: - - alert: StreamForgeHighLag - expr: streamforge_consumer_lag > 100000 - for: 5m - labels: - severity: warning - annotations: - summary: "High consumer lag" - description: "Lag is {{ $value }} messages" - - - alert: StreamForgeHighErrorRate - expr: rate(streamforge_errors_total[5m]) > 10 - for: 2m - labels: - severity: critical - annotations: - summary: "High error rate" - description: "Error rate is {{ $value }}/s" - - - alert: StreamForgePodDown - expr: up{job="streamforge"} == 0 - for: 1m - labels: - severity: critical - annotations: - summary: "StreamForge pod is down" - - - alert: StreamForgeHighDLQRate - expr: rate(streamforge_dlq_messages_total[5m]) > 5 - for: 5m - labels: - severity: warning - annotations: - summary: "High DLQ rate" - description: "DLQ rate is {{ $value }}/s" -``` - -### 4. Distributed Tracing - -**Jaeger integration (future):** -```yaml -env: -- name: OTEL_EXPORTER_JAEGER_ENDPOINT - value: "http://jaeger-collector:14268/api/traces" -- name: OTEL_SERVICE_NAME - value: "streamforge" -``` - ---- - -## Configuration Management - -### 1. Environment-Specific Configs - -**Directory structure:** -``` -configs/ -โ”œโ”€โ”€ base/ -โ”‚ โ”œโ”€โ”€ config.yaml -โ”‚ โ””โ”€โ”€ kustomization.yaml -โ”œโ”€โ”€ dev/ -โ”‚ โ”œโ”€โ”€ config-patch.yaml -โ”‚ โ””โ”€โ”€ kustomization.yaml -โ”œโ”€โ”€ staging/ -โ”‚ โ”œโ”€โ”€ config-patch.yaml -โ”‚ โ””โ”€โ”€ kustomization.yaml -โ””โ”€โ”€ prod/ - โ”œโ”€โ”€ config-patch.yaml - โ””โ”€โ”€ kustomization.yaml -``` - -**base/kustomization.yaml:** -```yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- namespace.yaml -- deployment.yaml -- service.yaml -configMapGenerator: -- name: streamforge-config - files: - - config.yaml -``` - -**prod/kustomization.yaml:** -```yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: -- ../base -patchesStrategicMerge: -- config-patch.yaml -replicas: -- name: streamforge - count: 5 -``` - -**Deploy:** -```bash -kubectl apply -k configs/prod/ -``` - -### 2. ConfigMap Hot Reload - -StreamForge supports config hot reload (without restart): - -**Watch ConfigMap changes:** -```yaml -spec: - containers: - - name: config-watcher - image: jimmidyson/configmap-reload:latest - args: - - --volume-dir=/config - - --webhook-url=http://localhost:8080/reload - volumeMounts: - - name: config - mountPath: /config -``` - -### 3. Validation Before Deploy - -```bash -# Validate config locally -streamforge-validate configs/prod/config.yaml - -# Validate in CI/CD -docker run --rm -v $(pwd)/configs:/configs streamforge:1.0.0 \ - streamforge-validate /configs/prod/config.yaml --fail-on-warnings -``` - ---- - -## Next Steps - -- [Operations Guide](OPERATIONS.md) - Day-to-day operations -- [Troubleshooting](TROUBLESHOOTING.md) - Common issues and solutions -- [Performance Tuning](PERFORMANCE_TUNING_RESULTS.md) - Optimization guide -- [Monitoring](docs/monitoring/) - Dashboards and alerts - ---- +- Configuration validation passes without unreviewed warnings. +- Kafka TLS/SASL material is mounted read-only from a secret store. +- Metrics and health endpoints are private. +- Delivery mode and duplicate-handling expectations are documented. +- Source and destination topic capacity and partitioning are verified. +- Resource requests and limits come from measurements of the target workload. +- Consumer lag, delivery failures, processing errors, and restarts are alerted. +- Rollback and offset-recovery procedures have been tested. -**Document Version:** 1.0.0 -**Last Updated:** 2026-04-18 -**Feedback:** https://github.com/rahulbsw/streamforge/issues +Continue with [Operations](OPERATIONS.md) for day-two procedures and +[Troubleshooting](TROUBLESHOOTING.md) for incident diagnosis. diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 882a22b..658cf4b 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -1,490 +1,157 @@ --- -title: Docker -nav_order: 6 +title: Podman +nav_order: 1 parent: Deployment --- -# Docker Deployment Guide +# Run StreamForge with Podman -## Overview +The repository contains two container builds: -Two Dockerfile options are provided: +- `Dockerfile` builds the default Chainguard-based image. +- `Dockerfile.static` builds an x86-64 musl binary on a Chainguard static + runtime. -1. **`Dockerfile`** - Dynamic linking (recommended for most use cases) - - Runtime: `cgr.dev/chainguard/glibc-dynamic` - - Size: ~20-30MB - - Includes necessary shared libraries +Build and scan the exact image revision that you plan to deploy. Do not infer +current vulnerability status from a base-image brand or an unpinned `latest` +tag. -2. **`Dockerfile.static`** - Fully static binary (maximum security) - - Runtime: `cgr.dev/chainguard/static` - - Size: ~10-15MB - - No dependencies, ultra-minimal - -## Why Chainguard Images? - -- โœ… **Minimal attack surface** - Only essential components -- โœ… **Daily updates** - Automatic CVE patching -- โœ… **Non-root by default** - Enhanced security -- โœ… **SBOM included** - Software Bill of Materials -- โœ… **Signed with Sigstore** - Supply chain security -- โœ… **No CVEs** - Zero known vulnerabilities - -## Quick Start - -### 1. Build the Image - -**Dynamic version (recommended):** -```bash -docker build -t streamforge:latest . -``` - -**Static version:** -```bash -docker build -f Dockerfile.static -t streamforge:static . -``` - -### 2. Create Configuration - -```bash -# Copy example config -cp config.example.json config.json - -# Edit for your environment -vim config.json -``` - -### 3. Run the Container - -```bash -docker run -d \ - --name streamforge \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - -e RUST_LOG=info \ - --restart unless-stopped \ - streamforge:latest -``` - -### 4. Check Logs - -```bash -docker logs -f streamforge -``` - -## Docker Compose - -### Basic Usage - -```bash -# Start with your config -docker-compose up -d - -# View logs -docker-compose logs -f mirrormaker - -# Stop -docker-compose down -``` - -### With Local Kafka (for testing) +## Build ```bash -# Start Kafka + MirrorMaker -docker-compose --profile kafka up -d - -# Check all services -docker-compose --profile kafka ps +podman build --pull --tag streamforge:local . ``` -### Static Version +For the static x86-64 image: ```bash -# Use the static build -docker-compose --profile static up -d mirrormaker-static +podman build --pull \ + --file Dockerfile.static \ + --tag streamforge:static-local . ``` -## Configuration Options - -### Environment Variables +For a reproducible release, replace moving base-image tags with approved +digests in your release process and record the resulting StreamForge image +digest. -| Variable | Default | Description | -|----------|---------|-------------| -| `CONFIG_FILE` | `/app/config/config.json` | Path to config file | -| `RUST_LOG` | `info` | Log level (trace, debug, info, warn, error) | +## Prepare a configuration -### Volume Mounts +Build the local validation binary: ```bash -docker run -d \ - --name streamforge \ - -v $(pwd)/config.json:/app/config/config.json:ro \ # Config (read-only) - -v $(pwd)/logs:/app/logs \ # Logs (optional) - streamforge:latest +cp examples/configs/config.example.yaml streamforge.yaml +cargo build --release --locked --bin streamforge-validate +target/release/streamforge-validate streamforge.yaml --fail-on-warnings ``` -### Network Modes +Do not commit credentials in `streamforge.yaml`. If the file contains secrets, +render it from an approved secret store into a protected runtime path. -**Bridge mode (default):** -```bash -docker run --network bridge ... -``` +## Run without public exposure -**Host mode (for local Kafka):** ```bash -docker run --network host ... -``` - -**Custom network:** -```bash -docker network create kafka-network -docker run --network kafka-network ... -``` - -## Resource Limits - -### Recommended Settings - -```bash -docker run -d \ +podman run --detach \ --name streamforge \ - --cpus="2" \ - --memory="512m" \ - --memory-reservation="256m" \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - streamforge:latest -``` - -### In docker-compose.yml - -```yaml -deploy: - resources: - limits: - cpus: '2' - memory: 512M - reservations: - cpus: '1' - memory: 256M -``` - -## Health Checks - -### Built-in Health Check - -The Dockerfile includes a health check: - -```dockerfile -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD pgrep -f streamforge || exit 1 -``` - -### Check Health Status - -```bash -docker inspect --format='{{json .State.Health}}' streamforge | jq -``` - -## Logging - -### View Logs - -```bash -# Follow logs -docker logs -f streamforge - -# Last 100 lines -docker logs --tail 100 streamforge - -# With timestamps -docker logs -f --timestamps streamforge -``` - -### Structured Logging - -Set `RUST_LOG` for different verbosity: - -```bash -# Info level (default) -docker run -e RUST_LOG=info ... - -# Debug level -docker run -e RUST_LOG=debug ... - -# Module-specific -docker run -e RUST_LOG=streamforge::kafka=debug,streamforge::processor=trace ... -``` - -## Image Size Comparison - -| Image | Size | Security | Use Case | -|-------|------|----------|----------| -| Dynamic | ~25MB | High | Production (recommended) | -| Static | ~12MB | Highest | Maximum security | -| Java equivalent | ~200MB+ | Medium | Legacy | - -## Multi-Architecture Builds - -### Build for ARM64 - -```bash -docker buildx build \ - --platform linux/arm64 \ - -t streamforge:arm64 \ - . -``` - -### Multi-arch Manifest - -```bash -docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t streamforge:latest \ - --push \ - . -``` - -## Security Best Practices - -### 1. Run as Non-Root โœ… - -Both Dockerfiles use non-root user by default. - -```bash -# Verify -docker run --rm streamforge:latest id -# Should show: uid=65532(nonroot) gid=65532(nonroot) -``` - -### 2. Read-Only Root Filesystem - -```bash -docker run -d \ + --restart unless-stopped \ --read-only \ - --tmpfs /tmp \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - streamforge:latest + --tmpfs /tmp:rw,noexec,nosuid \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --mount type=bind,src="$(pwd)/streamforge.yaml",dst=/run/streamforge/config.yaml,readonly \ + --env CONFIG_FILE=/run/streamforge/config.yaml \ + --env RUST_LOG=info \ + --publish 127.0.0.1:9090:9090 \ + streamforge:local ``` -### 3. Drop Capabilities +Binding the published metrics port to `127.0.0.1` prevents remote network +access. Omit `--publish` when Prometheus shares a private Podman network with +StreamForge. -```bash -docker run -d \ - --cap-drop=ALL \ - --security-opt=no-new-privileges:true \ - streamforge:latest -``` +The metrics server has no authentication or TLS. Never publish it on +`0.0.0.0` on an internet-reachable host. -### 4. Complete Secure Configuration +## Verify ```bash -docker run -d \ - --name streamforge-secure \ - --read-only \ - --tmpfs /tmp:rw,noexec,nosuid,size=10m \ - --cap-drop=ALL \ - --security-opt=no-new-privileges:true \ - --cpus="2" \ - --memory="512m" \ - --pids-limit=100 \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - streamforge:latest -``` - -## Kubernetes Deployment - -### Basic Deployment - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: streamforge -spec: - replicas: 3 - selector: - matchLabels: - app: streamforge - template: - metadata: - labels: - app: streamforge - spec: - securityContext: - runAsNonRoot: true - runAsUser: 65532 - fsGroup: 65532 - containers: - - name: mirrormaker - image: streamforge:latest - imagePullPolicy: Always - env: - - name: CONFIG_FILE - value: /app/config/config.json - - name: RUST_LOG - value: info - resources: - requests: - memory: "256Mi" - cpu: "500m" - limits: - memory: "512Mi" - cpu: "2000m" - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - volumes: - - name: config - configMap: - name: mirrormaker-config +podman logs --tail 200 streamforge +curl --fail http://127.0.0.1:9090/health +curl --fail http://127.0.0.1:9090/metrics ``` -### ConfigMap - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: mirrormaker-config -data: - config.json: | - { - "appid": "streamforge", - "bootstrap": "kafka-broker:9092", - "input": "source-topic", - "output": "destination-topic", - "offset": "latest", - "threads": 4 - } -``` +`/health` proves that the HTTP process responds; it does not prove Kafka source +or destination health. Produce a controlled source record and verify the +destination independently before accepting a deployment. -## Troubleshooting +## Private container network -### Container Won't Start +Create a dedicated network when StreamForge and a private Kafka endpoint are +containerized on the same host: ```bash -# Check logs -docker logs streamforge - -# Run interactively -docker run --rm -it \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - streamforge:latest -``` - -### Config Validation - -```bash -# Test config file -docker run --rm \ - -v $(pwd)/config.json:/app/config/config.json:ro \ - streamforge:latest --help +podman network create streamforge-private +podman run --detach \ + --name streamforge \ + --network streamforge-private \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --mount type=bind,src="$(pwd)/streamforge.yaml",dst=/run/streamforge/config.yaml,readonly \ + --env CONFIG_FILE=/run/streamforge/config.yaml \ + streamforge:local ``` -### Network Issues +Attach only the required private Kafka and monitoring services to that network. +Do not use host networking as a generic connectivity fix. -```bash -# Test connectivity to Kafka -docker run --rm --network host nicolaka/netshoot \ - nc -zv kafka-broker 9092 -``` +## Resource controls -### Permission Issues +Set CPU and memory limits from a representative workload test: ```bash -# Check file permissions -ls -l config.json - -# Should be readable by all -chmod 644 config.json +podman update \ + --cpus 2 \ + --memory 1g \ + --memory-swap 1g \ + streamforge ``` -## Performance Monitoring +The values above demonstrate Podman syntax, not production sizing. Observe +consumer lag, broker-acknowledged deliveries, CPU throttling, memory, and +restarts before and after applying limits. -### Container Stats - -```bash -docker stats streamforge -``` +Memory use is affected by payload size, application batch size, processing +parallelism, worker queue capacity, destination fan-out, and queued producer +depth. -### Resource Usage +## Logs and shutdown ```bash -# CPU and memory -docker inspect streamforge | jq '.[0].HostConfig.Memory' - -# Current usage -docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}" streamforge -``` - -## CI/CD Integration - -### GitHub Actions Example - -```yaml -name: Build and Push Docker Image - -on: - push: - branches: [ main ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Build Docker image - run: docker build -t streamforge:${{ github.sha }} . - - - name: Run tests - run: docker run --rm streamforge:${{ github.sha }} cargo test - - - name: Push to registry - run: | - echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin - docker push streamforge:${{ github.sha }} +podman logs --follow streamforge +podman stop --time 30 streamforge ``` -## Best Practices Summary - -โœ… Use Chainguard base images for security -โœ… Multi-stage builds to minimize size -โœ… Run as non-root user (uid 65532) -โœ… Mount config as read-only -โœ… Set resource limits -โœ… Use health checks -โœ… Enable structured logging -โœ… Read-only root filesystem -โœ… Drop all capabilities -โœ… Regular image updates +Keep application logs on the container output stream. Avoid mounting a writable +host log directory unless retention, rotation, and permissions are managed by +the platform. -## Image Registry +After shutdown, verify the last committed source offsets and destination +records according to the selected +[delivery profile](DELIVERY_GUARANTEES.md). -### Push to Registry - -```bash -# Tag -docker tag streamforge:latest your-registry.com/streamforge:latest - -# Push -docker push your-registry.com/streamforge:latest -``` - -### Pull from Registry - -```bash -docker pull your-registry.com/streamforge:latest -``` +## Image publishing checklist -## Questions? +- Build from a reviewed source revision and locked dependencies. +- Use an immutable registry tag or digest. +- Generate an SBOM and retain it with the release. +- Scan the final image, including the current base layers. +- Sign the image according to the registry policy. +- Run as a non-root identity and verify it in the built image. +- Keep the root filesystem read-only and drop Linux capabilities. +- Mount configuration and certificate material read-only. +- Bind metrics only to loopback or a private container network. +- Test on every published CPU architecture. -See: -- `README.md` - Application overview -- `QUICKSTART.md` - Getting started -- `IMPLEMENTATION_NOTES.md` - Architecture details +Continue with [Security](SECURITY_CONFIGURATION.md) and +[Operations](OPERATIONS.md). diff --git a/docs/DOCUMENTATION_INDEX.md b/docs/DOCUMENTATION_INDEX.md deleted file mode 100644 index 76fcdd4..0000000 --- a/docs/DOCUMENTATION_INDEX.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Documentation Map -nav_order: 14 ---- - -# StreamForge Documentation Map - -Curated, task-oriented navigation for the most important StreamForge documentation. - -## Quick Navigation - -### Start Here -1. [Home](index.md) - Product overview and decision framing -2. [QUICKSTART.md](QUICKSTART.md) - Five-minute demo -3. [WHEN_TO_USE.md](WHEN_TO_USE.md) - StreamForge vs MM2 vs Arroyo - -### Build Pipelines -1. [USAGE.md](USAGE.md) - End-to-end pipeline patterns -2. [AGGREGATIONS.md](AGGREGATIONS.md) - Windowed derived metrics for selective replication pipelines -3. [ADVANCED_DSL_GUIDE.md](ADVANCED_DSL_GUIDE.md) - Full filter/transform reference -4. [YAML_CONFIGURATION.md](YAML_CONFIGURATION.md) - Author and review pipeline configs -5. [EXAMPLES.md](EXAMPLES.md) - Runnable configs and example packs - -### Run in Production -1. [SECURITY_CONFIGURATION.md](SECURITY_CONFIGURATION.md) -2. [KUBERNETES.md](KUBERNETES.md) -3. [OBSERVABILITY_QUICKSTART.md](OBSERVABILITY_QUICKSTART.md) -4. [DEPLOYMENT.md](DEPLOYMENT.md) - -### Compatibility -1. [COMPATIBILITY.md](COMPATIBILITY.md) -2. [WHEN_TO_USE.md](WHEN_TO_USE.md) - -### Reference -1. [DELIVERY_GUARANTEES.md](DELIVERY_GUARANTEES.md) -2. [SECURITY_CONFIGURATION.md](SECURITY_CONFIGURATION.md) -3. [DOCUMENTATION_INDEX.md](DOCUMENTATION_INDEX.md) - This curated doc map diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 782b9bc..96bd4ae 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,194 +1,228 @@ # Implementation Status -## โœ… Fully Implemented - -### Core Kafka Sink -- โœ… Cross-cluster mirroring (`KafkaSink`) -- โœ… Native Kafka compression (Gzip, Snappy, Zstd) -- โœ… Custom partitioning (hash-based, field-based) -- โœ… Multi-destination routing -- โœ… Async/await with Tokio -- โœ… Lock-free metrics - -### Filtering & Transformation (FULLY IMPLEMENTED!) -- โœ… JSON Path filters with comparison operators - - Numeric: `>`, `>=`, `<`, `<=`, `==`, `!=` - - String: `==`, `!=` - - Boolean: `==`, `!=` -- โœ… Boolean logic (AND/OR/NOT) -- โœ… Regular expressions (REGEX) -- โœ… Array operations (ARRAY_ALL, ARRAY_ANY) -- โœ… JSON Path transforms (field extraction) -- โœ… Object construction (CONSTRUCT) -- โœ… Array mapping (ARRAY_MAP) -- โœ… Arithmetic operations (ADD/SUB/MUL/DIV) -- โœ… Per-destination filters and transforms -- โœ… Zero external DSL dependencies -- โœ… High performance (~100ns per filter, 40x faster than JSLT) - -### Configuration -- โœ… JSON-based configuration -- โœ… Single-destination mode -- โœ… Multi-destination routing mode -- โœ… Filter and transform per destination -- โœ… Environment variable config file path -- โœ… Consumer/producer property override - -### Metrics -- โœ… Processed messages counter -- โœ… Filtered messages counter -- โœ… Completed messages counter -- โœ… Error counter -- โœ… Rate calculation -- โœ… Periodic reporting (10s interval) - -## โš ๏ธ Partially Implemented - -### Compression -- โœ… Gzip -- โœ… Snappy -- โœ… Zstd -- โŒ LZ4 (native Kafka LZ4 used, but custom implementation not needed) - -## โŒ Not Implemented - -### JSLT/JavaScript -- โŒ JSLT expression language -- โŒ JavaScript filters -- โŒ JavaScript transforms -- โŒ Runtime lambda compilation - -### Avro -- โŒ Avro serialization -- โŒ Schema inference -- โŒ Schema registry integration - -### Advanced Features -- โŒ Exactly-once semantics -- โŒ Dead letter queue -- โŒ Prometheus metrics exporter -- โŒ Health check HTTP endpoint -- โŒ Dynamic reconfiguration - -## ๐Ÿ“Š Feature Comparison - -| Feature | Java | Rust | Status | -|---------|------|------|--------| -| **Core** ||| -| Cross-cluster mirroring | โœ… | โœ… | Complete | -| Native compression | โœ… | โœ… | Complete | -| Custom partitioning | โœ… | โœ… | Complete | -| Multi-destination routing | โœ… | โœ… | Complete | -| **Filtering** ||| -| JSON path filters | โŒ | โœ… | **Rust better!** | -| Boolean logic (AND/OR/NOT) | โœ… | โœ… | Both (Rust 40x faster) | -| Regular expressions | โŒ | โœ… | **Rust better!** | -| Array operations | โŒ | โœ… | **Rust better!** | -| JSLT filters | โœ… | โŒ | Java only | -| JavaScript filters | โœ… | โŒ | Java only | -| Streaming filters | โœ… | โœ… | Both | -| **Transformation** ||| -| JSON path transforms | โŒ | โœ… | **Rust better!** | -| Object construction | โœ… | โœ… | Both (Rust 40x faster) | -| Array mapping | โŒ | โœ… | **Rust better!** | -| Arithmetic operations | โŒ | โœ… | **Rust better!** | -| JSLT transforms | โœ… | โŒ | Java only | -| JavaScript transforms | โœ… | โŒ | Java only | -| Field extraction | โœ… | โœ… | Both | -| **Serialization** ||| -| JSON | โœ… | โœ… | Both | -| Avro | โœ… | โŒ | Java only | -| Schema registry | โœ… | โŒ | Java only | -| **Performance** ||| -| Memory usage | High | **Low** | Rust 10x better | -| CPU efficiency | Moderate | **High** | Rust 3x better | -| Throughput | 10K msg/s | **25K msg/s** | Rust 2.5x better | -| Latency p99 | 50ms | **15ms** | Rust 3x better | - -## ๐Ÿ“ˆ What's Next? - -### Priority 1: Metrics Export -- Prometheus endpoint -- Grafana dashboard -- Custom metrics tags - -### Priority 2: Avro Support -- Schema inference -- Schema registry integration -- Avro serialization - -### Priority 3: Dead Letter Queue -- Failed message queue -- Retry logic -- Error tracking - -### Priority 4: Nested Transform Composition -- Compose transforms (e.g., ARRAY_MAP with CONSTRUCT) -- Chained transformations -- Complex data reshaping - -## ๐ŸŽฏ Current Capabilities - -**What you can do TODAY:** - -1. โœ… Mirror messages between Kafka clusters -2. โœ… Compress with Gzip/Snappy/Zstd -3. โœ… Partition by hash or field -4. โœ… Route to multiple destinations -5. โœ… Filter by numeric/string/boolean comparison -6. โœ… **Boolean logic (AND/OR/NOT)** -7. โœ… **Regular expression matching** -8. โœ… **Array filtering (ALL/ANY)** -9. โœ… Extract nested fields or objects -10. โœ… **Object construction** -11. โœ… **Array mapping** -12. โœ… **Arithmetic operations** -13. โœ… Per-destination filtering and transformation -14. โœ… Monitor with built-in metrics - -**What requires workarounds:** - -1. โš ๏ธ Avro โ†’ Use JSON for now or add feature -2. โš ๏ธ JSLT compatibility โ†’ Migrate expressions to custom DSL (40x faster!) -3. โš ๏ธ Nested transform composition โ†’ Apply transforms sequentially - -## ๐Ÿš€ Migration from Java - -### Easy Migrations (Drop-in Replacement) - -If your Java config uses: -- โœ… Basic mirroring (no filters) -- โœ… Gzip/Snappy/Zstd compression -- โœ… Hash or field partitioning -- โœ… Single or multi-destination routing - -โ†’ **Just migrate the config format!** - -### Medium Complexity - -If your Java config uses: -- โœ… Simple JSLT filters (numeric/string comparisons) -- โœ… Boolean logic (AND/OR/NOT) -- โœ… Field extraction transforms -- โœ… Object construction - -โ†’ **Convert JSLT to custom DSL syntax** (see ADVANCED_DSL_GUIDE.md) - **40x faster!** - -### High Complexity - -If your Java config uses: -- โœ… Array operations โ†’ **Now supported!** -- โœ… Regular expressions โ†’ **Now supported!** -- โœ… Arithmetic operations โ†’ **Now supported!** -- โŒ JavaScript filters/transforms โ†’ **Not supported** (use custom DSL instead) -- โŒ Avro serialization โ†’ **Not yet supported** -- โŒ Schema registry โ†’ **Not yet supported** - -โ†’ **Migrate most features** OR **wait for Avro support** - -## ๐Ÿ“ž Questions? - -- Filter syntax: See `ADVANCED_FILTERS.md` and `ADVANCED_DSL_GUIDE.md` -- Quick start: See `QUICKSTART.md` -- Architecture: See `IMPLEMENTATION_NOTES.md` -- Examples: See `config*.json` files +Verified source status as of 2026-07-25. This document records implemented +capabilities and known boundaries; it does not assert production throughput +without a reproducible workload and benchmark result. + +## Core data plane + +Implemented: + +- Kafka consume, process, and produce pipeline using Tokio and rust-rdkafka. +- Single- and multi-destination routing. +- Optional per-destination filters and transforms. +- Manual and automatic offset commit modes. +- Retry and dead-letter queue modules. +- Key, header, timestamp, and value-aware envelope operations. +- Default keyed partitioning and field-based partitioning. +- Native Kafka compression configuration. +- Local and Redis cache backends. +- Prometheus metrics, HTTP observability endpoints, and consumer-lag monitoring. +- Windowed aggregation with the constraints validated in configuration. + +Known boundaries: + +- Delivery is not exactly-once; transactional producer support is not + implemented. +- Payload processing uses JSON values. Avro and Schema Registry integration are + not implemented. +- Runtime configuration reload is not implemented. +- The generic raw/typed envelope described in `PROJECT_SPEC.md` remains planned. + +## Filter and transform DSL + +Implemented: + +- Legacy colon-delimited filters and transforms. +- Function-style filters with parsed AST input. +- JSON path comparisons and boolean composition. +- Regex, array, key, header, timestamp, null/empty, and string predicates. +- JSON path extraction, object construction, array mapping, arithmetic, string, + key, header, timestamp, hash, and cache transforms. +- Configuration-time compilation of function-style paths and regex patterns. +- Configuration-time tokenization of key-template placeholder paths. + +Known hot-path boundary: + +- Function-style array `any`/`all` evaluation still creates an envelope from a + cloned array element. This is a candidate for a later measured optimization. + +## Phase 1 performance hardening + +Implemented in the current source: + +- Keyless default partitioning delegates to librdkafka instead of forcing an + explicit partition. +- Keyed default partitioning and field-based routing remain explicit. +- Destinations without transforms skip identity-transform execution and keep the + existing shared value allocation. +- Actual transforms use copy-on-write value ownership. +- Function-style paths and regexes are compiled once. +- Key-template placeholder paths are compiled once. +- Runtime consumer batch size, batch fill timeout, and concurrency factor are + configurable with backward-compatible defaults. +- Selected consumer and producer performance fields map to librdkafka + properties, with explicit property maps taking precedence. +- Focused regression tests and steady-state Criterion benchmarks cover these + paths. + +Default runtime values remain: + +| Setting | Default | +|---|---:| +| Consumer batch size | `100` messages | +| Consumer batch fill timeout | `100` ms | +| Parallelism factor | `10` times `threads` | + +See `docs/PERFORMANCE.md` for the configuration and measurement contract. + +## Phase 2 delivery and scheduling implementation + +Implemented in the current source: + +- Backward-compatible `legacy_batch` and opt-in `partition_ordered` processing + modes. +- `partition_ordered` uses `threads` bounded FIFO worker lanes and stable source + topic/partition routing, with JSON parsing and processing inside the workers. +- Backward-compatible per-record `acknowledged` delivery and opt-in bounded + asynchronous `queued` delivery. +- Queued delivery tracks broker acknowledgements, applies configurable + backpressure, persists the first delivery failure, and drains during flush. +- A separate `streamforge_messages_delivered_total` metric distinguishes broker + acknowledgement from processor/enqueue completion. +- Validation rejects queued delivery with manual commits, retries, or DLQ, and + rejects partition-ordered processing with manual commits until explicit, + rebalance-aware completed-offset coordination is implemented. + +These modes are implemented and unit-tested. The partition-ordered/queued +combination has completed a valid sustained local Kafka run; the remaining +mode comparison matrix and a new AWS run are still pending. + +Default compatibility values remain: + +| Setting | Default | +|---|---:| +| Processing mode | `legacy_batch` | +| Worker queue capacity | `1024` per worker | +| Producer delivery mode | `acknowledged` | +| Producer maximum pending deliveries | `10000` | + +## Phase 2 benchmark and profiling foundation + +Implemented in the current tree: + +- A deterministic JSONL generator keyed by message count and seed. +- A Kafka-backed harness with isolated topics/groups; independent persistent + ingress, timed metrics/resource, and post-window output-validation jobs; a + shared monotonic barrier; exact counters and offsets; multiple repetitions; + and schema-version-3 environment/results manifests. +- Per-repetition Kafka metadata deletion plus verified physical partition-file + reclamation before the next repetition. +- A 24-case synthetic pipeline Criterion matrix covering JSON and envelope + stages across 256 B, 4 KiB, and 64 KiB payloads. +- A manual GitHub Actions workflow that runs all Criterion targets and one + Kafka-backed smoke repetition without imposing a noisy shared-runner gate. +- One canonical performance-testing contract and a dated baseline record + containing local and dedicated AWS x86_64 evidence. +- Whole-process `perf` capture with release debug information, forced frame + pointers, raw profile data, and zero-lost-sample verification. + +The Kafka harness uses auto commit, partition-ordered workers, queued producer +delivery, retries disabled, and DLQ disabled for its optimized passthrough +workload. Exact delivered/output counts verify that a benchmark run completed +without loss, but do not change or prove general delivery semantics. + +## Verification + +Verified for the current source on 2026-07-25 UTC: + +- `cargo test --all --no-fail-fast`: 474 passed, 0 failed, 30 ignored across + unit, integration, and documentation tests. +- Partition-worker tests verify same-partition FIFO order, cross-lane + concurrency, and bounded-queue backpressure. +- Queued-delivery tests verify successful acknowledgements, broker failures, + canceled futures, configuration safety constraints, and flush forwarding. +- `cargo clippy --all-targets --offline -- -D warnings`: passed. +- `cargo fmt --all -- --check`, benchmark Bash/Python syntax checks, + `git diff --check`, generated benchmark-config validation, and JSON schema + parsing: passed. +- `cargo build --release --bin streamforge`: passed before the local benchmark + preflight. +- Six sustained-harness unit tests, Python compilation, Bash syntax, compose + rendering, and `git diff --check`: passed. +- Single-destination produced accounting now increments the exact + destination-labelled counter and has focused regression tests. +- The loopback-only Podman harness passed three 120-second + partition-ordered/queued repetitions after a one-million-record untimed + warm-up. Each repetition reconciled exactly 24,000,000 timed records across + input offsets, consumed, produced, broker-delivered, output offsets, and the + independent output validator, with zero errors. +- Diagnostic local aggregate: median `199,604.900121 msg/s`, minimum + `199,576.121938`, maximum `199,677.648263`, coefficient of variation + `0.0214%`, median `1.721` StreamForge cores, and median peak RSS `128 MiB`. + Environment: Apple M4 Pro host, Podman 5.7.1 ARM64 VM with 4 vCPUs and + 6,144 MiB RAM, Kafka image pinned by digest, 8 partitions, and 8 threads. +- That aggregate is not a public baseline: the worktree was dirty and two runs + were ingress-limited. It is a sustained lower bound and is not directly + comparable to the superseded coupled-harness numbers or an unmatched Java + run. +- Kafka was published only on `127.0.0.1:9092`; the benchmark network was + internal; ingress and output runners exposed no ports. Final topic and disk + reclamation checks passed. +- No AWS rerun has been attempted after the valid local result. All resources + from the previous AWS attempt remain verified deleted. + +Previously verified on 2026-07-24: + +- `cargo test --all`: 462 passed, 0 failed, 30 ignored across unit, integration, + and documentation tests. +- All three Criterion targets completed and saved the `phase2-current` + baseline; the synthetic pipeline ran all 24 cases. +- Kafka-backed passthrough baseline: 10,000 deterministic messages, 4 + partitions, 4 threads, 3 repetitions, median `1,495.709902 msg/s`, exact + 10,000 consumed/output records per repetition, and zero processing errors. +- `cargo clippy --all-targets --offline -- -D warnings`: passed. +- `cargo fmt --all -- --check`: passed. +- `git diff --check`: passed. +- `docs/CONFIG_SCHEMA.json`: parsed successfully with `jq`. + +Previously verified on dedicated AWS x86_64 compute on 2026-07-25: + +- `cargo test --all --locked`: 462 passed, 0 failed, 30 ignored. +- All three Criterion targets completed with baseline name `aws-c7i-phase2`. +- Kafka-backed passthrough: 100,000 deterministic messages, 8 partitions, 8 + threads, 5 repetitions, median `6,254.848569 msg/s`, exact consumed/output + counts, and zero processing errors. +- A 200,000-message profiling repetition completed at `7,094.333485 msg/s`; + `perf` recorded 614 cycle samples with zero lost samples. +- Direct AWS service inventories confirmed both instances terminated, no live + volumes or public IPs, and deletion of the private S3 bucket, scheduler, IAM + objects, and isolated VPC/network resources. + +Those local and AWS rates were measured by the superseded coupled harness. They +include source publication and polling overhead and are not StreamForge capacity +claims or results for the new modes. See +`docs/benchmarks/results/phase2-baseline-20260724.md` for the full environment +and method. + +## Planned measured work + +The next performance work starts from the dedicated whole-process profile, not +from a blanket SIMD rewrite: + +1. Run the corrected live Kafka matrix for legacy/acknowledged, + partition-ordered/acknowledged, and partition-ordered/queued. +2. Run a clean-worktree, matched Java/Rust comparison with identical payloads, + partitions, acknowledgement semantics, warm-up, duration, and validation. +3. Provision the later AWS benchmark with cost-bounded Terraform and + ECS-on-EC2 jobs only after the local matrix passes; keep all endpoints + private or restricted to the user's IP. +4. Keep raw/lazy envelope work behind the existing 30% parse/serialization + threshold; the AWS passthrough profile measured about 16.5% parsing and 3.2% + serialization on overlapping inclusive stacks. +5. Profile transform-heavy and aggregation-heavy workloads separately. +6. Evaluate SIMD only if one of those profiles identifies a dominant + vectorizable kernel. + +Product boundaries and the typed-envelope direction remain governed by +`PROJECT_SPEC.md`. diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md index d64527e..be3966e 100644 --- a/docs/KUBERNETES.md +++ b/docs/KUBERNETES.md @@ -1,820 +1,280 @@ --- title: Kubernetes -nav_order: 7 +nav_order: 2 parent: Deployment --- -# Streamforge on Kubernetes +# Run StreamForge on Kubernetes -Complete guide for deploying and managing Streamforge pipelines on Kubernetes using the Operator pattern. +The repository includes a `StreamforgePipeline` custom resource, a Rust +operator, and a Helm chart at `helm/streamforge-operator`. Install from a local +checkout or a reviewed internal artifact; this guide does not assume a public +chart repository or public image tag. -## Table of Contents +## Current operator scope -- [Architecture](#architecture) -- [Quick Start](#quick-start) -- [Helm Chart](#helm-chart) -- [CRD & Operator](#crd--operator) -- [UI Options](#ui-options) -- [Examples](#examples) -- [Best Practices](#best-practices) +The operator currently: ---- - -## Architecture - -### Traditional Deployment vs Operator Pattern - -**โŒ Traditional Approach (Limitations):** -``` -User creates Deployment + ConfigMap manually -โ†’ Hard to manage multiple pipelines -โ†’ No dynamic updates -โ†’ Requires manual scaling -โ†’ No validation -``` - -**โœ… Operator Pattern (Recommended):** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Kubernetes Cluster โ”‚ -โ”‚ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Streamforge Operator โ”‚ โ”‚ -โ”‚ โ”‚ โ€ข Watches StreamforgePipeline CRDs โ”‚ โ”‚ -โ”‚ โ”‚ โ€ข Reconciles desired vs actual state โ”‚ โ”‚ -โ”‚ โ”‚ โ€ข Creates Deployment + ConfigMap โ”‚ โ”‚ -โ”‚ โ”‚ โ€ข Updates status โ”‚ โ”‚ -โ”‚ โ”‚ โ€ข Self-healing โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”‚ โ”‚ โ”‚ -โ”‚ โ”‚ manages โ”‚ -โ”‚ โ–ผ โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚Pipeline 1โ”‚ โ”‚Pipeline 2โ”‚ โ”‚Pipeline 3โ”‚ โ”‚ -โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ -โ”‚ โ”‚Deploymentโ”‚ โ”‚Deploymentโ”‚ โ”‚Deploymentโ”‚ โ”‚ -โ”‚ โ”‚ConfigMap โ”‚ โ”‚ConfigMap โ”‚ โ”‚ConfigMap โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Key Benefits - -โœ… **Dynamic Management**: Add/update/delete pipelines without affecting others -โœ… **Declarative**: Define pipelines as YAML resources -โœ… **Self-Healing**: Operator reconciles failures automatically -โœ… **Validation**: CRD validates specs before creation -โœ… **Status Tracking**: Real-time pipeline status -โœ… **GitOps Ready**: Perfect for ArgoCD, Flux - ---- - -## Quick Start - -### Prerequisites - -- Kubernetes 1.19+ -- Helm 3.0+ -- kubectl configured - -### 1. Install Operator - -```bash -# Add Helm repository (when published) -helm repo add streamforge https://rahulbsw.github.io/streamforge -helm repo update - -# Install operator -helm install streamforge-operator streamforge/streamforge-operator \ - --namespace streamforge-system \ - --create-namespace -``` - -### 2. Create Your First Pipeline - -```bash -kubectl apply -f - < rendered-streamforge.yaml ``` -Access: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/ - -### Option 2: Lens (Desktop App) โญ Recommended - -**Pros:** -- Best developer experience -- CRD support out-of-the-box -- Multi-cluster management -- Terminal, logs, port-forwarding built-in - -**Cons:** -- Desktop app (not web-based) -- Free for open source, paid for teams - -**Setup:** - -1. Download: https://k8slens.dev -2. Connect your cluster -3. Navigate to Custom Resources โ†’ streamforgepipelines - -Lens will show all pipelines with create/edit/delete options. +Review: -### Option 3: Headlamp (Web-based) +- `ClusterRole` permissions and cluster-wide watch scope; +- service accounts and image references; +- pod and container security contexts; +- resource requests and limits; +- whether any `Service`, `Ingress`, `NodePort`, or `LoadBalancer` would be + created; +- all generated configuration for secrets or public endpoints. -**Pros:** -- Web-based (self-hosted) -- Open source -- CRD support -- Modern UI +The UI is disabled above. Do not enable it until its authentication, RBAC, +secret handling, and private access path have completed a production security +review. -**Cons:** -- Requires deployment - -**Setup:** +## Install the local chart ```bash -helm repo add headlamp https://headlamp-k8s.github.io/headlamp/ -helm install headlamp headlamp/headlamp \ - --namespace headlamp \ - --create-namespace - -# Access -kubectl port-forward -n headlamp svc/headlamp 8080:80 -``` - -Access: http://localhost:8080 - -### Option 4: Streamforge UI (Built-in) โญ New! - -**Web UI for managing pipelines - included with the Helm chart!** - -**Features:** -- ๐Ÿ” JWT authentication -- ๐Ÿ“ Visual pipeline builder (form + YAML editor) -- ๐Ÿ“Š Real-time pipeline monitoring -- ๐Ÿ“‹ Live log viewer with auto-refresh -- ๐ŸŽฏ Direct CRD management via Kubernetes API -- ๐ŸŽจ Modern Next.js + React + Tailwind CSS - -**Architecture:** +kubectl create namespace streamforge-system -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Streamforge UI (Next.js) โ”‚ -โ”‚ โ€ข Pipeline form builder โ”‚ -โ”‚ โ€ข DSL syntax editor โ”‚ -โ”‚ โ€ข Real-time status monitoring โ”‚ -โ”‚ โ€ข Pod log viewer โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ”‚ Kubernetes API - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Kubernetes API Server โ”‚ -โ”‚ โ€ข RBAC authentication โ”‚ -โ”‚ โ€ข StreamforgePipeline CRD ops โ”‚ -โ”‚ โ€ข Pod logs access โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ”‚ watches/reconciles - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Streamforge Operator โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**Setup (via Helm):** - -```bash -# Install operator with UI enabled -helm install streamforge-operator ./helm/streamforge-operator \ +helm upgrade --install streamforge-operator ./helm/streamforge-operator \ --namespace streamforge-system \ - --create-namespace \ - --set ui.enabled=true - -# Access the UI -# Option 1: Minikube -minikube service streamforge-operator-ui -n streamforge-system - -# Option 2: Port forward -kubectl port-forward -n streamforge-system svc/streamforge-operator-ui 3001:3001 -# Access at: http://localhost:3001 - -# Option 3: Ingress -helm upgrade streamforge-operator ./helm/streamforge-operator \ - -n streamforge-system \ - --reuse-values \ - --set ui.ingress.enabled=true \ - --set ui.ingress.hosts[0].host=streamforge.example.com + --values private-values.yaml \ + --wait ``` -**Default credentials:** -- Username: `admin` -- Password: `admin` - -โš ๏ธ **Change these in production!** - -See [UI Demo on Minikube](UI_MINIKUBE_DEMO.md) for a full recorded walkthrough of the Helm install, UI pipeline creation, YAML preview, and transformed output verification. - -**UI Configuration:** - -```yaml -# custom-ui-values.yaml -ui: - enabled: true - replicas: 2 - - service: - type: LoadBalancer # or NodePort, ClusterIP - port: 3001 - - # Secure JWT secret - jwtSecret: "your-secure-random-secret-key" - - # Enable ingress - ingress: - enabled: true - className: nginx - annotations: - cert-manager.io/cluster-issuer: "letsencrypt-prod" - hosts: - - host: streamforge.example.com - paths: - - path: / - pathType: Prefix - tls: - - secretName: streamforge-tls - hosts: - - streamforge.example.com -``` - -See [ui/README.md](../ui/README.md) for more details. - -### Option 5: kubectl Plugin - -**Quick CLI management:** +Verify the controller: ```bash -# Install kubectl-streamforge plugin (planned v1.1) -kubectl krew install streamforge - -# Usage -kubectl streamforge create my-pipeline \ - --source kafka:9092/input \ - --dest kafka:9092/output \ - --replicas 2 - -kubectl streamforge list -kubectl streamforge logs my-pipeline -kubectl streamforge scale my-pipeline --replicas=4 -``` - ---- - -## Examples - -### Simple Mirror - -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: simple-mirror -spec: - source: - brokers: "kafka:9092" - topic: "events" - destinations: - - brokers: "kafka:9092" - topic: "events-backup" - replicas: 2 +kubectl get deployment,pods -n streamforge-system +kubectl logs -n streamforge-system \ + deployment/streamforge-operator \ + --tail=200 ``` -### Filtered Multi-Destination +The generated Deployment name can include Helm release-name expansion. Use +`kubectl get deployment -n streamforge-system` if the exact name differs. -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: filtered-routing -spec: - source: - brokers: "kafka-source:9092" - topic: "events" - destinations: - # Active events - - brokers: "kafka-target:9092" - topic: "active-events" - filter: "/status,==,active" - # High priority - - brokers: "kafka-priority:9092" - topic: "priority" - filter: "AND:/priority,==,high:/status,==,active" - replicas: 3 -``` +## Create a basic pipeline -### With Transformation +The operator's generated runtime configuration currently supports one +destination: ```yaml apiVersion: streamforge.io/v1alpha1 kind: StreamforgePipeline metadata: - name: transform-pipeline + name: orders-copy + namespace: streamforge-system spec: + appid: orders-copy source: - brokers: "kafka:9092" - topic: "raw-events" + brokers: source-kafka.kafka.svc.cluster.local:9092 + topic: orders + offset: earliest destinations: - - brokers: "kafka:9092" - topic: "processed" - transform: "CONSTRUCT:output,/user/id:userId,/event/type:eventType" - compression: "zstd" - replicas: 4 - threads: 8 -``` - -### Secure with SSL/SASL - -See [examples/pipelines/03-secure-transform.yaml](../examples/pipelines/03-secure-transform.yaml) - ---- - -## Best Practices - -### 1. Resource Management - -**Set resource limits:** -```yaml -spec: + - brokers: destination-kafka.kafka.svc.cluster.local:9092 + topic: orders-copy + replicas: 1 + threads: 4 + image: + repository: registry.internal/streamforge + tag: reviewed-release + pullPolicy: IfNotPresent resources: requests: - cpu: "200m" - memory: "256Mi" + cpu: 250m + memory: 256Mi limits: - cpu: "1000m" - memory: "512Mi" + cpu: 1 + memory: 1Gi ``` -**Adjust based on load:** -- Light: 100m CPU, 128Mi memory -- Medium: 500m CPU, 512Mi memory -- Heavy: 2000m CPU, 2Gi memory - -### 2. Scaling Strategy - -**Replicas = Kafka Partitions** - -If source topic has 10 partitions: -- Set replicas: 10 (one pod per partition) -- Or replicas: 5 (two partitions per pod) +Apply and inspect: -**Horizontal scaling:** ```bash -kubectl patch sfp my-pipeline -p '{"spec":{"replicas":10}}' --type=merge +kubectl apply -f pipeline.yaml +kubectl get streamforgepipelines -n streamforge-system +kubectl describe streamforgepipeline orders-copy -n streamforge-system +kubectl get deployment,pods,configmap -n streamforge-system \ + -l streamforge.io/pipeline=orders-copy ``` -**Vertical scaling:** -```bash -kubectl patch sfp my-pipeline --type=merge -p ' -spec: - resources: - limits: - memory: "2Gi" - threads: 8 -' -``` +The resource values demonstrate schema and Kubernetes syntax; establish +production sizing with the target workload. -### 3. Security +## Secure Kafka connections -**Use Secrets for credentials:** +Because the operator does not currently emit CR security fields into the +runtime configuration, do not put Kafka usernames or passwords directly in the +CR and assume they will be applied. -```bash -# Create secret -kubectl create secret generic kafka-creds \ - --from-literal=username=myuser \ - --from-literal=password=mypass +For TLS/SASL pipelines, use a directly managed Deployment with a complete +StreamForge configuration supplied from a Kubernetes `Secret`, or update and +verify the operator before using it. See [Security](SECURITY_CONFIGURATION.md). -# Reference in pipeline -``` +Never store a secret-bearing StreamForge configuration in a `ConfigMap`. -```yaml -spec: - source: - security: - sasl: - mechanism: SCRAM-SHA-256 - username: myuser - password: mypass # TODO: Support secret references in operator -``` +## Private metrics access -### 4. Monitoring +StreamForge listens on the configured metrics port on all pod interfaces. The +endpoint has no authentication or TLS. -**Enable Prometheus:** -```yaml -# values.yaml -monitoring: - enabled: true - serviceMonitor: - enabled: true -``` - -**Key metrics:** -- `streamforge_messages_consumed_total` -- `streamforge_messages_produced_total` -- `streamforge_lag_current` -- `streamforge_filter_duration_seconds` +If a metrics service is required, make it internal: -### 5. Naming Conventions - -``` ---- - -Examples: -- prod-mirror-events-backup -- staging-filter-logs-analytics -- dev-transform-users-warehouse +```yaml +apiVersion: v1 +kind: Service +metadata: + name: streamforge-metrics + namespace: streamforge-system +spec: + type: ClusterIP + selector: + streamforge.io/pipeline: orders-copy + ports: + - name: metrics + port: 9090 + targetPort: 9090 ``` -### 6. GitOps - -**Store pipelines in Git:** +Add a `NetworkPolicy` that permits ingress only from the monitoring namespace +and egress only to Kafka, DNS, and other required private services. Do not use a +public `LoadBalancer`, `NodePort`, or internet-facing `Ingress` for metrics or +the UI. -``` -pipelines/ -โ”œโ”€โ”€ prod/ -โ”‚ โ”œโ”€โ”€ critical-mirror.yaml -โ”‚ โ””โ”€โ”€ analytics-pipeline.yaml -โ”œโ”€โ”€ staging/ -โ”‚ โ””โ”€โ”€ test-pipeline.yaml -โ””โ”€โ”€ dev/ - โ””โ”€โ”€ dev-pipeline.yaml -``` +For temporary local inspection: -**Deploy with ArgoCD/Flux:** ```bash -# ArgoCD -argocd app create streamforge-pipelines \ - --repo https://github.com/myorg/pipelines \ - --path pipelines/prod \ - --dest-namespace default \ - --sync-policy automated +kubectl port-forward -n streamforge-system \ + deployment/orders-copy 9090:9090 +curl --fail http://127.0.0.1:9090/health ``` -### 7. Testing +## Scaling -**Test pipeline before production:** +Useful consumer parallelism is bounded by source partitions. Increase replicas +only while partitions remain available for assignment and watch the rebalance. -```yaml -apiVersion: streamforge.io/v1alpha1 -kind: StreamforgePipeline -metadata: - name: test-pipeline - namespace: dev -spec: - source: - brokers: "kafka-dev:9092" - topic: "test-input" - destinations: - - brokers: "kafka-dev:9092" - topic: "test-output" - replicas: 1 - logLevel: "debug" +```bash +kubectl patch streamforgepipeline orders-copy \ + --namespace streamforge-system \ + --type merge \ + --patch '{"spec":{"replicas":2}}' ``` ---- +Measure lag, broker-acknowledged deliveries, processing errors, CPU, memory, and +restart behavior after each change. More replicas do not divide a single hot +partition. -## Troubleshooting +## Upgrade and rollback -### Pipeline Not Starting +Render the new chart and diff it before applying: ```bash -# Check events -kubectl get events --sort-by='.lastTimestamp' | grep my-pipeline - -# Check operator logs -kubectl logs -n streamforge-system -l app.kubernetes.io/name=streamforge-operator +helm template streamforge-operator ./helm/streamforge-operator \ + --namespace streamforge-system \ + --values private-values.yaml > rendered-streamforge-next.yaml -# Check pod status -kubectl describe pod -l streamforge.io/pipeline=my-pipeline +kubectl diff --server-side -f rendered-streamforge-next.yaml ``` -### High Memory Usage +Then upgrade: ```bash -# Reduce threads -kubectl patch sfp my-pipeline -p '{"spec":{"threads":2}}' --type=merge - -# Increase memory limit -kubectl patch sfp my-pipeline --type=merge -p ' -spec: - resources: - limits: - memory: "1Gi" -' +helm upgrade streamforge-operator ./helm/streamforge-operator \ + --namespace streamforge-system \ + --values private-values.yaml \ + --wait ``` -### Lag Increasing +Retain the previous image digests and values file. A Helm rollback restores +chart state, but it does not undo Kafka records, consumer offsets, topic +changes, or externally managed secrets. -```bash -# Scale up -kubectl patch sfp my-pipeline -p '{"spec":{"replicas":6}}' --type=merge - -# Check consumer group lag -kafka-consumer-groups.sh --bootstrap-server kafka:9092 \ - --group streamforge-my-pipeline --describe -``` +## Removal ---- +Before uninstalling, decide whether pipeline custom resources and their +consumer groups must be retained. The chart is configured to keep the CRD by +default. -## Next Steps - -1. **Install Operator**: `helm install streamforge-operator` -2. **Enable UI**: `--set ui.enabled=true` -3. **Create First Pipeline**: Apply example YAML or use UI -4. **Monitor**: Enable Prometheus metrics -5. **Scale**: Test with production load +```bash +helm uninstall streamforge-operator --namespace streamforge-system +``` -## Contributing +Inventory remaining custom resources, deployments, config maps, service +accounts, cluster roles, cluster role bindings, services, secrets, and CRDs. +Deleting a namespace or custom resource is destructive and should follow an +approved data and offset-retention plan. -See [CONTRIBUTING.md](CONTRIBUTING.md) +## Production checklist -## License +- Images are private, immutable, scanned, and signed. +- Rendered cluster-scoped RBAC has been approved. +- UI remains disabled unless separately security-reviewed. +- No public service, ingress, listener, or security-group rule is created. +- Secret-bearing configuration is stored in a `Secret`, not a `ConfigMap`. +- Kafka TLS/SASL and ACLs are verified from the running pod. +- Metrics use `ClusterIP` plus restrictive network policy. +- Resource sizing comes from a representative workload. +- Delivery behavior is tested across a restart and rebalance. +- Rollback and teardown inventories have been rehearsed. -Apache License 2.0 +Continue with [Deployment](DEPLOYMENT.md), [Observability](OBSERVABILITY_QUICKSTART.md), +and [Troubleshooting](TROUBLESHOOTING.md). diff --git a/docs/OBSERVABILITY_QUICKSTART.md b/docs/OBSERVABILITY_QUICKSTART.md index ed6db15..9b1ace2 100644 --- a/docs/OBSERVABILITY_QUICKSTART.md +++ b/docs/OBSERVABILITY_QUICKSTART.md @@ -1,366 +1,193 @@ --- title: Observability -nav_order: 11 -parent: Deployment +nav_order: 1 +parent: Operations --- -# Observability Quickstart +# Observability -Get Prometheus metrics and Kafka lag monitoring running in 5 minutes. +StreamForge exposes Prometheus metrics and a simple process health endpoint. +Use them with Kafka consumer-group and destination-topic observations to monitor +the full pipeline. -## Quick Start - -### 1. Enable Metrics in Config - -Add to your `config.yaml`: +## Enable the endpoints ```yaml observability: metrics_enabled: true metrics_port: 9090 + metrics_path: /metrics lag_monitoring_enabled: true lag_monitoring_interval_secs: 30 ``` -### 2. Start Streamforge +Start StreamForge: ```bash -CONFIG_FILE=config.yaml ./streamforge -``` - -You'll see: -``` -โœ… Metrics registered successfully -๐Ÿ” Metrics server listening on http://0.0.0.0:9090 - Metrics endpoint: http://localhost:9090/metrics - Health endpoint: http://localhost:9090/health -โœ… Consumer lag monitoring started (interval: 30s) +CONFIG_FILE=config.yaml target/release/streamforge ``` -### 3. View Metrics +The HTTP server listens on all interfaces. It serves `/metrics` and `/health`; +the current server route is `/metrics` even if a different `metrics_path` value +is configured. -**Browser:** -``` -http://localhost:9090/metrics -``` +Test from the same private network: -**curl:** ```bash -curl http://localhost:9090/metrics +curl --fail http://streamforge.internal:9090/health +curl --fail http://streamforge.internal:9090/metrics ``` -**Sample Output:** -```prometheus -# HELP streamforge_messages_consumed_total Total messages consumed from source Kafka -# TYPE streamforge_messages_consumed_total counter -streamforge_messages_consumed_total 125000 - -# HELP streamforge_messages_produced_total Messages successfully produced to destinations -# TYPE streamforge_messages_produced_total counter -streamforge_messages_produced_total{destination="premium-events"} 45000 -streamforge_messages_produced_total{destination="standard-events"} 80000 - -# HELP streamforge_consumer_lag Consumer lag per partition -# TYPE streamforge_consumer_lag gauge -streamforge_consumer_lag{topic="input-topic",partition="0"} 1250 -streamforge_consumer_lag{topic="input-topic",partition="1"} 890 - -# HELP streamforge_processing_duration_seconds End-to-end processing latency per destination -# TYPE streamforge_processing_duration_seconds histogram -streamforge_processing_duration_seconds_bucket{destination="premium-events",le="0.001"} 35000 -streamforge_processing_duration_seconds_bucket{destination="premium-events",le="0.005"} 43000 -streamforge_processing_duration_seconds_bucket{destination="premium-events",le="0.01"} 44500 -streamforge_processing_duration_seconds_bucket{destination="premium-events",le="+Inf"} 45000 -streamforge_processing_duration_seconds_sum{destination="premium-events"} 67.5 -streamforge_processing_duration_seconds_count{destination="premium-events"} 45000 -``` +`/health` returns `OK` when the HTTP process responds. It is not a readiness +check for source consumption or destination delivery. + +## Keep the endpoint private + +The metrics server does not provide TLS or authentication. Do not expose it to +the public internet. -## Prometheus Setup +- In Kubernetes, use a `ClusterIP` service and restrict ingress to the + monitoring namespace with a `NetworkPolicy`. +- In Docker, publish the port only on a private interface or scrape it through + a private container network. +- If a proxy is required, add authentication and TLS there. -### Add Scrape Config +Metrics labels and operational values can reveal topic names and traffic +patterns. Apply the same access controls used for other production telemetry. -Edit `prometheus.yml`: +## Prometheus scrape configuration ```yaml scrape_configs: - - job_name: 'streamforge' + - job_name: streamforge static_configs: - - targets: ['localhost:9090'] + - targets: + - streamforge.internal:9090 scrape_interval: 15s scrape_timeout: 10s ``` -### Start Prometheus +For Kubernetes, a `ServiceMonitor` can select the private metrics service when +the Prometheus Operator is installed. -```bash -docker run -d \ - -p 9091:9090 \ - -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml \ - prom/prometheus -``` - -Access Prometheus UI: `http://localhost:9091` +## Useful metrics -## Quick Queries +### Pipeline flow -### Message Throughput ```promql -# Messages per second rate(streamforge_messages_consumed_total[5m]) - -# Per destination -sum(rate(streamforge_messages_produced_total[5m])) by (destination) ``` -### Error Rate ```promql -# Errors per second -rate(streamforge_processing_errors_total[5m]) - -# Error percentage -rate(streamforge_processing_errors_total[5m]) / -rate(streamforge_messages_consumed_total[5m]) * 100 +sum by (destination) ( + rate(streamforge_messages_delivered_total[5m]) +) ``` -### Consumer Lag -```promql -# Total lag -sum(streamforge_consumer_lag) +`streamforge_messages_delivered_total` counts successful Kafka delivery +acknowledgements. Prefer it over enqueue or processor completion when measuring +delivery. -# Per partition -streamforge_consumer_lag +### Errors -# Lag increasing (alert!) -delta(streamforge_consumer_lag[5m]) > 1000 +```promql +sum by (type) ( + rate(streamforge_processing_errors_total[5m]) +) ``` -### Processing Latency ```promql -# P99 latency -histogram_quantile(0.99, - rate(streamforge_processing_duration_seconds_bucket[5m]) +sum by (destination) ( + rate(streamforge_filter_errors_total[5m]) ) - -# Average latency -rate(streamforge_processing_duration_seconds_sum[5m]) / -rate(streamforge_processing_duration_seconds_count[5m]) ``` -### Filter Effectiveness ```promql -# Pass rate percentage -rate(streamforge_filter_evaluations_total{result="pass"}[5m]) / -rate(streamforge_filter_evaluations_total[5m]) * 100 +sum by (destination) ( + rate(streamforge_transform_errors_total[5m]) +) +``` -# Messages filtered out per destination -rate(streamforge_messages_filtered_total[5m]) +### Lag + +```promql +sum(streamforge_consumer_lag) ``` -## Grafana Dashboard +```promql +max by (topic, partition) (streamforge_consumer_lag) +``` -### Quick Dashboard JSON +Lag metrics appear after the consumer has partition assignments and the lag +monitor completes a collection interval. -Create a dashboard with these panels: +### Processing latency -**Panel 1: Message Throughput** -```json -{ - "title": "Message Throughput", - "targets": [{ - "expr": "rate(streamforge_messages_consumed_total[5m])", - "legendFormat": "Consumed" - }, { - "expr": "sum(rate(streamforge_messages_produced_total[5m]))", - "legendFormat": "Produced" - }] -} +```promql +histogram_quantile( + 0.95, + sum by (le, destination) ( + rate(streamforge_processing_duration_seconds_bucket[5m]) + ) +) ``` -**Panel 2: Consumer Lag** -```json -{ - "title": "Consumer Lag by Partition", - "targets": [{ - "expr": "streamforge_consumer_lag", - "legendFormat": "{{topic}}-{{partition}}" - }] -} -``` +### Saturation -**Panel 3: Error Rate** -```json -{ - "title": "Error Rate", - "targets": [{ - "expr": "rate(streamforge_processing_errors_total[5m])", - "legendFormat": "{{type}}" - }] -} +```promql +streamforge_messages_in_flight ``` -**Panel 4: Processing Latency** -```json -{ - "title": "Processing Latency (P50, P95, P99)", - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(streamforge_processing_duration_seconds_bucket[5m]))", - "legendFormat": "P50" - }, - { - "expr": "histogram_quantile(0.95, rate(streamforge_processing_duration_seconds_bucket[5m]))", - "legendFormat": "P95" - }, - { - "expr": "histogram_quantile(0.99, rate(streamforge_processing_duration_seconds_bucket[5m]))", - "legendFormat": "P99" - } - ] -} -``` +Pair application metrics with container CPU, throttling, memory, restart, and +network metrics from the runtime platform. -Import to Grafana: -```bash -# Coming soon: Pre-built dashboard JSON -# Check examples/grafana-dashboard.json -``` +## Alert strategy -## Alerting Rules +Use workload-specific service objectives rather than copied numeric thresholds. +At minimum, detect: -### prometheus-alerts.yml +- StreamForge unavailable; +- source traffic present while broker-acknowledged delivery stops; +- sustained consumer-lag growth; +- processing errors or DLQ traffic; +- repeated restarts; +- memory approaching its limit; +- sustained CPU throttling; +- abnormal processing-latency changes. + +Example availability rule: ```yaml groups: - - name: streamforge_alerts - interval: 30s + - name: streamforge rules: - # High error rate - - alert: StreamforgeHighErrorRate - expr: rate(streamforge_processing_errors_total[5m]) > 10 - for: 2m - labels: - severity: warning - annotations: - summary: "High error rate in Streamforge" - description: "Error rate is {{ $value }} errors/sec" - - # Consumer lag increasing - - alert: StreamforgeConsumerLagIncreasing - expr: delta(streamforge_consumer_lag[5m]) > 10000 - for: 5m - labels: - severity: warning - annotations: - summary: "Consumer lag increasing" - description: "Lag increased by {{ $value }} in 5 minutes" - - # High latency - - alert: StreamforgeHighLatency - expr: | - histogram_quantile(0.99, - rate(streamforge_processing_duration_seconds_bucket[5m]) - ) > 1.0 - for: 5m - labels: - severity: warning - annotations: - summary: "P99 latency above 1 second" - - # Service down - - alert: StreamforgeDown + - alert: StreamForgeUnavailable expr: up{job="streamforge"} == 0 - for: 1m + for: 2m labels: severity: critical annotations: - summary: "Streamforge service is down" -``` - -## Testing Locally - -### 1. Generate Load - -```bash -# Terminal 1: Start Streamforge -CONFIG_FILE=examples/config.with-observability.yaml ./streamforge - -# Terminal 2: Produce test messages -kafka-console-producer.sh --topic input-topic --bootstrap-server localhost:9092 -``` - -### 2. Watch Metrics - -```bash -# Watch metrics update -watch -n 2 'curl -s http://localhost:9090/metrics | grep streamforge_messages' - -# Check specific metric -curl -s http://localhost:9090/metrics | grep streamforge_consumer_lag + summary: StreamForge metrics endpoint is unavailable ``` -### 3. Verify Lag Monitoring - -```bash -# Check lag metrics are updating -curl -s http://localhost:9090/metrics | grep consumer_lag - -# Example output: -# streamforge_consumer_lag{topic="input-topic",partition="0"} 0 -# streamforge_consumer_lag{topic="input-topic",partition="1"} 0 -``` - -## Troubleshooting - -### Metrics endpoint not accessible - -**Check if server started:** -```bash -netstat -an | grep 9090 -# Should show: tcp4 0 0 *.9090 *.* LISTEN -``` - -**Check logs:** -``` -2026-04-03T10:00:00Z INFO streamforge: Metrics server listening on http://0.0.0.0:9090 -``` - -### No lag metrics - -**Possible causes:** -1. No partitions assigned yet (consumer just started) -2. Lag monitoring disabled in config -3. Consumer group has no committed offsets - -**Check:** -```bash -# Wait 30 seconds for first lag check -sleep 30 - -# Check metrics -curl http://localhost:9090/metrics | grep consumer_lag -``` - -### Metrics not updating - -**Verify:** -1. Messages are being consumed (check logs) -2. Metrics are being incremented (check counter values) -3. Prometheus is scraping (check Prometheus UI โ†’ Targets) - -## Next Steps +Choose the `for` duration and severity according to the pipeline objective. -- [Full Design Document](OBSERVABILITY_METRICS_DESIGN.md) - Complete metrics reference -- [Prometheus Documentation](https://prometheus.io/docs/) -- [Grafana Dashboard Tutorial](https://grafana.com/docs/grafana/latest/dashboards/) -- See `examples/config.with-observability.yaml` for full config example +## Validate the signal path -## Summary +1. Confirm Prometheus reports the target as healthy. +2. Produce a controlled source record. +3. Observe the consumed counter. +4. Verify the destination record with an independent Kafka consumer. +5. Observe the delivered counter for that destination. +6. Confirm lag reflects the committed consumer-group position. +7. Send an intentionally rejected test record in a non-production pipeline and + verify error and DLQ monitoring. +8. Stop the test instance and verify the availability alert. -You now have: -- โœ… Prometheus metrics exposed on `:9090/metrics` -- โœ… Kafka consumer lag monitoring -- โœ… Per-destination metrics (throughput, errors, latency) -- โœ… Filter and transform operation tracking -- โœ… Health check endpoint +If metrics disagree with Kafka offsets or destination records, treat Kafka as +the delivery source of truth and investigate instrumentation before publishing +performance results. -**Total setup time:** < 5 minutes ๐Ÿš€ +See [Operations](OPERATIONS.md) for incident workflows and +[Delivery guarantees](DELIVERY_GUARANTEES.md) for counter interpretation in +acknowledged and queued modes. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index a69517d..a23b85d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,1362 +1,199 @@ -# StreamForge Operations Runbook - -**Version:** 1.0.0 -**Last Updated:** 2026-04-18 - -This runbook provides operational procedures for running StreamForge in production. - --- - -## Table of Contents - -1. [Daily Operations](#daily-operations) -2. [Monitoring and Alerting](#monitoring-and-alerting) -3. [Scaling Operations](#scaling-operations) -4. [Incident Response](#incident-response) -5. [Capacity Planning](#capacity-planning) -6. [Maintenance Windows](#maintenance-windows) -7. [Backup and Recovery](#backup-and-recovery) -8. [Performance Optimization](#performance-optimization) -9. [Common Operational Tasks](#common-operational-tasks) - +title: Operations +nav_order: 7 +has_children: true --- -## Daily Operations +# Operate StreamForge -### Morning Health Check +This runbook focuses on signals and procedures that apply to both container and +Kubernetes deployments. Establish alert thresholds from the normal behavior and +service objectives of each pipeline; hard-coded global thresholds are not +meaningful across different payloads, brokers, and traffic patterns. -**1. Check pod status:** -```bash -kubectl get pods -n streamforge -kubectl get hpa -n streamforge -``` - -Expected output: -``` -NAME READY STATUS RESTARTS AGE -streamforge-7c8f9d4b6-abc12 1/1 Running 0 2d -streamforge-7c8f9d4b6-def34 1/1 Running 0 2d -streamforge-7c8f9d4b6-ghi56 1/1 Running 0 1d -``` +## First-response checklist -**2. Check consumer lag:** -```bash -# Via metrics endpoint -curl http://streamforge.streamforge.svc:8080/metrics | grep consumer_lag +1. Confirm the process or pod is running and inspect recent restarts. +2. Query the private `/health` endpoint. +3. Check source consumer-group assignment and lag with Kafka tooling. +4. Compare consumed records, broker-acknowledged deliveries, processing errors, + and in-flight work. +5. Inspect recent configuration or image changes. +6. Check CPU, memory, network, and Kafka broker health. +7. Sample the DLQ without copying sensitive payloads into tickets or logs. -# Or via Kafka directly -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` +For Kubernetes: -Acceptable lag: -- **< 1000 messages:** Healthy -- **1000-10000 messages:** Monitor -- **> 10000 messages:** Investigate (scale up or tune) - -**3. Check error rates:** -```promql -rate(streamforge_errors_total[5m]) -``` - -Acceptable error rate: -- **< 1/s:** Normal (transient errors) -- **1-10/s:** Monitor -- **> 10/s:** Investigate (check DLQ and logs) - -**4. Check DLQ:** ```bash -kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic streamforge-dlq \ - --property print.headers=true \ - --max-messages 10 +kubectl get pods -n streamforge -o wide +kubectl logs -n streamforge deployment/streamforge --tail=200 +kubectl top pods -n streamforge +kubectl get events -n streamforge --sort-by=.lastTimestamp ``` -Review recent DLQ messages: -- Parse error headers (`x-streamforge-error-type`) -- Identify patterns (bad data format, config issues) -- Update filters/transforms if needed - -### Weekly Tasks +For Kafka: -**1. Review metrics trends:** -- Throughput (consumed/produced per second) -- Processing latency (p50, p95, p99) -- Error rates over time -- Resource utilization (CPU, memory) - -**2. Check for updates:** -```bash -helm repo update -helm search repo streamforge -``` - -**3. Review DLQ accumulation:** ```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group streamforge-dlq-consumer -``` - -If DLQ is growing: -- Investigate root cause (bad data, config error) -- Fix issue in pipeline config -- Reprocess DLQ messages if needed - -**4. Capacity planning check:** -- Review growth trends -- Predict when scaling will be needed -- Plan for capacity additions - -### Monthly Tasks - -**1. Upgrade StreamForge:** -```bash -helm upgrade streamforge streamforge/streamforge \ - --namespace streamforge \ - --values values.yaml \ - --version 1.0.1 -``` - -**2. Security audit:** -- Review access logs -- Rotate credentials (Kafka passwords, TLS certs) -- Update TLS certificates before expiration - -**3. Performance benchmarking:** -- Run load tests -- Compare with baseline metrics -- Identify performance degradation - -**4. Disaster recovery test:** -- Verify backups are working -- Test failover procedures -- Update runbooks based on findings - ---- - -## Monitoring and Alerting - -### Key Metrics - -#### Throughput Metrics - -**Messages consumed per second:** -```promql -rate(streamforge_messages_consumed_total[5m]) -``` - -**Messages produced per second:** -```promql -rate(streamforge_messages_produced_total[5m]) -``` - -**Baseline:** Establish during initial deployment (e.g., 50K msg/s) - -#### Lag Metrics - -**Consumer lag:** -```promql -streamforge_consumer_lag -``` - -**Lag increase rate:** -```promql -deriv(streamforge_consumer_lag[10m]) +kafka-consumer-groups --bootstrap-server kafka.internal:9092 \ + --describe --group PIPELINE_APPID ``` -**Alert thresholds:** -- Warning: lag > 10000 -- Critical: lag > 100000 or lag growing for 10 minutes +## Core service indicators -#### Error Metrics +Monitor these together: -**Error rate:** -```promql -rate(streamforge_errors_total[5m]) -``` +| Signal | What it answers | +|---|---| +| `streamforge_messages_consumed_total` | Is StreamForge receiving source records? | +| `streamforge_messages_delivered_total` | Are records being acknowledged by destination Kafka? | +| `streamforge_processing_errors_total` | Are parse, processing, or Kafka errors occurring? | +| `streamforge_consumer_lag` | Is the pipeline keeping up with each source partition? | +| `streamforge_messages_in_flight` | Is application work accumulating? | +| `streamforge_processing_duration_seconds` | Is per-destination processing latency changing? | +| Process restarts and resource use | Is the runtime unhealthy or constrained? | -**Error by type:** -```promql -rate(streamforge_errors_total{error_type="FilterEvaluation"}[5m]) -rate(streamforge_errors_total{error_type="ProducerTimeout"}[5m]) -``` +Use Kafka destination offsets or an independent consumer to verify end-to-end +delivery. Application counters alone do not prove that every expected source +record reached the intended destination. -**Alert thresholds:** -- Warning: error rate > 1/s -- Critical: error rate > 10/s +The metrics endpoint has no authentication. Keep it private as described in +[Deployment](DEPLOYMENT.md). -#### DLQ Metrics +## Alert design -**DLQ message rate:** -```promql -rate(streamforge_dlq_messages_total[5m]) -``` - -**DLQ accumulation:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group streamforge | grep dlq -``` +Create alerts from service objectives and observed baselines: -**Alert thresholds:** -- Warning: DLQ rate > 1/s -- Critical: DLQ rate > 10/s or DLQ lag > 1000 +- page when the process is unavailable or broker acknowledgements stop while + source traffic continues; +- alert when lag grows for a sustained interval rather than on a universal + message-count threshold; +- alert on processing errors and any DLQ traffic relative to input volume; +- alert before memory exhaustion or sustained CPU throttling; +- alert on repeated restarts and consumer-group rebalance loops; +- use latency percentiles only after confirming the histogram has traffic. -#### Latency Metrics +Example PromQL: -**Processing duration (p95):** ```promql -histogram_quantile(0.95, rate(streamforge_processing_duration_seconds_bucket[5m])) +up{job="streamforge"} == 0 ``` -**Processing duration (p99):** ```promql -histogram_quantile(0.99, rate(streamforge_processing_duration_seconds_bucket[5m])) +sum by (type) (rate(streamforge_processing_errors_total[5m])) ``` -**Alert thresholds:** -- Warning: p95 > 100ms -- Critical: p95 > 500ms - -#### Resource Metrics - -**CPU usage:** ```promql -rate(container_cpu_usage_seconds_total{pod=~"streamforge.*"}[5m]) +sum(streamforge_consumer_lag) ``` -**Memory usage:** ```promql -container_memory_usage_bytes{pod=~"streamforge.*"} -``` - -**Alert thresholds:** -- Warning: CPU > 70% or Memory > 80% -- Critical: CPU > 90% or Memory > 95% - -### Alert Rules - -**Critical Alerts (page immediately):** - -1. **Pipeline Down:** - ```promql - up{job="streamforge"} == 0 - ``` - **Action:** Check pod status, review logs, restart if needed - -2. **High Error Rate:** - ```promql - rate(streamforge_errors_total[5m]) > 10 - ``` - **Action:** Check logs for error patterns, review recent config changes - -3. **Consumer Lag Critical:** - ```promql - streamforge_consumer_lag > 100000 - ``` - **Action:** Scale up replicas, increase threads, check Kafka performance - -4. **Memory Exhaustion:** - ```promql - container_memory_usage_bytes / container_spec_memory_limit_bytes > 0.95 - ``` - **Action:** Increase memory limits, check for memory leak - -**Warning Alerts (investigate during business hours):** - -1. **Consumer Lag Warning:** - ```promql - streamforge_consumer_lag > 10000 - ``` - **Action:** Monitor lag trend, prepare to scale if increasing - -2. **DLQ Accumulation:** - ```promql - rate(streamforge_dlq_messages_total[5m]) > 1 - ``` - **Action:** Review DLQ messages, identify data quality issues - -3. **High Latency:** - ```promql - histogram_quantile(0.95, rate(streamforge_processing_duration_seconds_bucket[5m])) > 0.1 - ``` - **Action:** Check filter/transform complexity, review performance - -4. **Pod Restarts:** - ```promql - rate(kube_pod_container_status_restarts_total{pod=~"streamforge.*"}[1h]) > 0 - ``` - **Action:** Review pod logs, check for OOMKilled or CrashLoopBackOff - -### Dashboard Layout - -**Overview Dashboard:** -- Throughput (consumed/produced) -- Consumer lag -- Error rate -- DLQ rate -- Pod count and health - -**Performance Dashboard:** -- Processing latency (p50, p95, p99) -- CPU usage by pod -- Memory usage by pod -- Network I/O - -**Error Analysis Dashboard:** -- Errors by type -- Error rate over time -- DLQ messages by error type -- Retry attempts histogram - ---- - -## Scaling Operations - -### Horizontal Scaling (Add Replicas) - -**When to scale up:** -- Consumer lag > 10000 and growing -- CPU usage > 70% sustained -- Throughput needs to increase - -**Scale up:** -```bash -# Manual -kubectl scale deployment streamforge --replicas=5 -n streamforge - -# Or update HPA -kubectl edit hpa streamforge -n streamforge -``` - -**Verify:** -```bash -kubectl get pods -n streamforge -kubectl get hpa streamforge -n streamforge - -# Check consumer group rebalancing -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` - -**Expected behavior:** -- Pods start and become Ready -- Consumer group rebalances -- Partitions redistributed across consumers -- Lag decreases over 5-10 minutes - -**When to scale down:** -- Lag < 100 sustained -- CPU usage < 30% sustained -- Traffic decreased - -**Scale down:** -```bash -kubectl scale deployment streamforge --replicas=2 -n streamforge -``` - -**Caution:** Gradual scale-down to avoid lag spikes during rebalancing. - -### Vertical Scaling (Increase Resources) - -**When to scale up:** -- Memory usage > 80% sustained -- CPU limits hit frequently -- OOMKilled events - -**Update resources:** -```bash -kubectl patch deployment streamforge -n streamforge --patch ' -spec: - template: - spec: - containers: - - name: streamforge - resources: - requests: - cpu: "2000m" - memory: "4Gi" - limits: - cpu: "4000m" - memory: "8Gi" -' -``` - -**Or via Helm:** -```bash -helm upgrade streamforge streamforge/streamforge \ - --namespace streamforge \ - --reuse-values \ - --set resources.requests.cpu=2000m \ - --set resources.requests.memory=4Gi \ - --set resources.limits.cpu=4000m \ - --set resources.limits.memory=8Gi -``` - -**Verify:** -```bash -kubectl get pods -n streamforge -kubectl describe pod streamforge- -n streamforge | grep -A5 Requests -``` - -### Thread Scaling (Increase Parallelism) - -**When to increase threads:** -- CPU usage < 50% but lag is high -- Many CPU cores available -- Filter/transform logic is CPU-bound - -**Update config:** -```yaml -threads: 8 # increase from 4 -``` - -**Apply:** -```bash -kubectl edit configmap streamforge-config -n streamforge -kubectl rollout restart deployment/streamforge -n streamforge -``` - -**Verify:** -```bash -kubectl logs -f deployment/streamforge -n streamforge | grep "threads" -``` - -**Rule of thumb:** -- 1 thread per CPU core -- Max 16 threads (diminishing returns) -- Monitor CPU usage after change - -### Autoscaling Configuration - -**HPA based on CPU:** -```yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: streamforge -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: streamforge - minReplicas: 2 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 -``` - -**HPA based on custom metric (consumer lag):** -```yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: streamforge -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: streamforge - minReplicas: 2 - maxReplicas: 10 - metrics: - - type: Pods - pods: - metric: - name: streamforge_consumer_lag - target: - type: AverageValue - averageValue: "5000" -``` - -**Scale-up behavior:** -- Stabilization: 60 seconds -- Max scale rate: 50% per minute - -**Scale-down behavior:** -- Stabilization: 300 seconds (5 minutes) -- Max scale rate: 25% per minute - ---- - -## Incident Response - -### High Consumer Lag - -**Symptoms:** -- Lag > 100000 messages -- Lag growing steadily -- Alert fired: "StreamForgeHighLag" - -**Investigation:** - -1. **Check throughput:** - ```bash - curl http://streamforge:8080/metrics | grep messages_consumed_total - curl http://streamforge:8080/metrics | grep messages_produced_total - ``` - -2. **Check replicas:** - ```bash - kubectl get deployment streamforge -n streamforge - ``` - -3. **Check CPU/memory:** - ```bash - kubectl top pods -n streamforge - ``` - -4. **Check errors:** - ```bash - kubectl logs deployment/streamforge -n streamforge | grep ERROR - ``` - -**Resolution:** - -**If CPU saturated (> 80%):** -- Scale up replicas: `kubectl scale deployment streamforge --replicas=N` -- Increase threads in config -- Optimize filters/transforms - -**If memory saturated (> 90%):** -- Increase memory limits -- Reduce batch size -- Check for memory leaks (restart pods) - -**If Kafka is slow:** -- Check Kafka broker health -- Increase `fetch_max_wait_ms` -- Increase `fetch_min_bytes` - -**If throughput is limited:** -- Increase producer batch size -- Reduce `linger_ms` -- Enable compression - -### High Error Rate - -**Symptoms:** -- Error rate > 10/s -- Alert fired: "StreamForgeHighErrorRate" -- DLQ accumulating rapidly - -**Investigation:** - -1. **Check error types:** - ```bash - kubectl logs deployment/streamforge -n streamforge | grep ERROR | tail -50 - ``` - -2. **Check DLQ headers:** - ```bash - kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic streamforge-dlq \ - --property print.headers=true \ - --max-messages 5 - ``` - -3. **Check recent config changes:** - ```bash - kubectl describe configmap streamforge-config -n streamforge - ``` - -**Common error types and resolutions:** - -**FilterEvaluation errors:** -- **Cause:** Bad data format (missing fields, wrong types) -- **Fix:** Update filter to handle missing fields (e.g., add default value) -- **Example:** Change `/status,==,active` to `OR:/status,==,active:/status,==,null` - -**ProducerTimeout errors:** -- **Cause:** Kafka producer timeout (slow brokers, network issues) -- **Fix:** Increase retry attempts, check Kafka health -- **Config:** Increase `max_delay_ms` in retry config - -**SerializationError:** -- **Cause:** Invalid JSON in transform output -- **Fix:** Review transform logic, add validation -- **Example:** Ensure CONSTRUCT generates valid JSON - -**ConnectionError:** -- **Cause:** Kafka broker unreachable -- **Fix:** Check network, DNS, Kafka broker status -- **Recovery:** Will auto-retry with exponential backoff - -### Pod Crashes (CrashLoopBackOff) - -**Symptoms:** -- Pods restarting frequently -- Status: CrashLoopBackOff -- Alert fired: "StreamForgePodDown" - -**Investigation:** - -1. **Check pod status:** - ```bash - kubectl get pods -n streamforge - kubectl describe pod streamforge- -n streamforge - ``` - -2. **Check logs:** - ```bash - kubectl logs streamforge- -n streamforge --previous - ``` - -3. **Check events:** - ```bash - kubectl get events -n streamforge --sort-by='.lastTimestamp' - ``` - -**Common causes:** - -**OOMKilled (Out of Memory):** -- **Symptom:** Last State: Terminated, Reason: OOMKilled -- **Fix:** Increase memory limits -- **Config:** - ```yaml - resources: - limits: - memory: 8Gi - ``` - -**Config error:** -- **Symptom:** Logs show "invalid config" or "parse error" -- **Fix:** Validate config with `streamforge-validate` -- **Check:** Run `kubectl logs` to see exact error - -**Kafka connection failure:** -- **Symptom:** Logs show "Failed to connect to Kafka" -- **Fix:** Check bootstrap servers, TLS certs, SASL credentials -- **Test:** Use `kafka-console-consumer` to verify connectivity - -**Missing secret:** -- **Symptom:** Logs show "secret not found" or volume mount error -- **Fix:** Create missing secret -- **Check:** `kubectl get secret -n streamforge` - -### DLQ Overflow - -**Symptoms:** -- DLQ lag > 1000 -- DLQ rate > 10/s sustained -- Disk usage increasing - -**Investigation:** - -1. **Count DLQ messages:** - ```bash - kafka-run-class kafka.tools.GetOffsetShell \ - --broker-list kafka:9092 \ - --topic streamforge-dlq - ``` - -2. **Sample DLQ messages:** - ```bash - kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic streamforge-dlq \ - --property print.headers=true \ - --max-messages 20 - ``` - -3. **Identify error patterns:** - - Group by `x-streamforge-error-type` header - - Identify common source topics - - Check for data quality issues - -**Resolution:** - -**If error is in config:** -- Fix filter/transform logic -- Deploy updated config -- Reprocess DLQ messages - -**If error is in data:** -- Fix upstream data producer -- Add data validation at source -- Optionally skip bad messages (update filter) - -**Reprocess DLQ:** -```yaml -# Create DLQ reprocessing pipeline -appid: "dlq-reprocessor" -input: "streamforge-dlq" -offset: "earliest" -threads: 1 -routing: - destinations: - - output: "original-topic" - filter: "/error-type,!=,permanent" # Skip permanent failures - transform: "/original-data" # Extract original message -``` - -**Purge DLQ (if data is bad and not recoverable):** -```bash -kafka-delete-records --bootstrap-server kafka:9092 \ - --offset-json-file delete-dlq.json -``` - -delete-dlq.json: -```json -{ - "partitions": [ - {"topic": "streamforge-dlq", "partition": 0, "offset": 1000} - ] -} -``` - -### Performance Degradation - -**Symptoms:** -- Processing latency increased (p95 > 200ms, was 50ms) -- Throughput decreased (20K msg/s, was 50K msg/s) -- No obvious errors - -**Investigation:** - -1. **Check metrics history:** - - Compare current vs baseline (1 week ago) - - Identify when degradation started - -2. **Check resource usage:** - ```bash - kubectl top pods -n streamforge - ``` - -3. **Check Kafka performance:** - ```bash - kafka-broker-api-versions --bootstrap-server kafka:9092 - # Check broker response time - ``` - -4. **Check for config changes:** - ```bash - kubectl get configmap streamforge-config -n streamforge -o yaml - ``` - -**Common causes:** - -**Increased message size:** -- **Symptom:** Same throughput (msg/s) but higher latency -- **Fix:** Increase `fetch_min_bytes`, tune compression - -**Complex filters/transforms added:** -- **Symptom:** CPU usage increased -- **Fix:** Optimize DSL expressions, increase threads - -**Kafka broker issues:** -- **Symptom:** High fetch latency -- **Fix:** Scale Kafka brokers, add partitions - -**Network congestion:** -- **Symptom:** High network I/O wait -- **Fix:** Increase network bandwidth, enable compression - ---- - -## Capacity Planning - -### Throughput Estimation - -**Formula:** -``` -Max throughput (msg/s) = (CPU cores ร— threads per core ร— single-thread throughput) / message size factor +sum(rate(streamforge_messages_delivered_total[5m])) by (destination) ``` -**Baseline single-thread throughput:** -- Passthrough (no filters): ~100K msg/s -- Simple filters (JSON path): ~50K msg/s -- Complex transforms (CONSTRUCT): ~20K msg/s -- Regex filters: ~10K msg/s - -**Message size factor:** -- Small (< 1 KB): 1.0x -- Medium (1-10 KB): 0.8x -- Large (10-100 KB): 0.5x -- Very large (> 100 KB): 0.2x - -**Example:** -- 4 CPU cores -- 4 threads per core = 16 threads total -- Complex transforms (~20K msg/s per thread) -- Medium messages (1-10 KB): 0.8x factor - -Max throughput = 4 ร— 4 ร— 20000 ร— 0.8 = 256K msg/s - -### Resource Requirements - -**Per replica:** - -| Throughput | CPU Request | CPU Limit | Memory Request | Memory Limit | -|------------|-------------|-----------|----------------|--------------| -| 10K msg/s | 500m | 1000m | 1 Gi | 2 Gi | -| 50K msg/s | 1000m | 2000m | 2 Gi | 4 Gi | -| 100K msg/s | 2000m | 4000m | 4 Gi | 8 Gi | -| 200K msg/s | 4000m | 8000m | 8 Gi | 16 Gi | - -**Partitions:** -- One consumer per partition (max) -- If replicas > partitions, some replicas will be idle -- Recommended: partitions โ‰ฅ replicas ร— 2 - -**Example:** -- Target: 100K msg/s -- Partitions: 16 -- Replicas: 4 (leaves headroom for scaling to 16) -- Resources per replica: 2 CPU / 4 Gi - -### Growth Planning +## Scaling safely -**Monthly review:** +Kafka source partitions bound useful consumer parallelism for one consumer +group. Adding replicas beyond the available partitions leaves consumers idle. -1. **Measure current usage:** - ```promql - avg_over_time(rate(streamforge_messages_consumed_total[1d])[30d]) - ``` +Scale only after identifying the constraint: -2. **Calculate growth rate:** - ``` - Growth rate = (Current - Last month) / Last month - ``` - -3. **Project future needs:** - ``` - Projected throughput (3 months) = Current ร— (1 + growth_rate)^3 - ``` - -4. **Plan capacity additions:** - - If projected > 80% of max capacity: add replicas - - If projected > 200% of max capacity: add partitions - -**Example:** -- Current: 50K msg/s -- Last month: 40K msg/s -- Growth rate: (50K - 40K) / 40K = 25% per month -- Projected (3 months): 50K ร— 1.25^3 = 97.7K msg/s -- Current max: 100K msg/s (80% threshold = 80K) -- **Action:** Plan to add 2 replicas in 2 months - -### Kafka Partition Scaling - -**When to add partitions:** -- Consumer lag high even with max replicas -- Throughput > (partitions ร— per-partition throughput) -- Need more parallelism - -**Add partitions:** -```bash -kafka-topics --bootstrap-server kafka:9092 \ - --alter --topic source-topic \ - --partitions 32 -``` - -**Considerations:** -- **Cannot decrease partitions** (Kafka limitation) -- Rebalancing will occur (temporary lag spike) -- Keyed messages may redistribute (breaks ordering) -- DLQ topic should match partition count - -**Best practice:** -- Start with 16 partitions -- Double when needed (16 โ†’ 32 โ†’ 64) -- Max 128 partitions per topic (broker limits) - ---- +- **CPU saturated:** add CPU or replicas, then remeasure. +- **CPU available and destination waits dominate:** test bounded processing + concurrency. +- **Memory pressure:** reduce batch size, worker queue bounds, or queued producer + depth before increasing memory. +- **One hot partition:** inspect key distribution; more replicas cannot divide a + single partition. +- **Kafka latency or errors:** fix broker or network capacity before increasing + application concurrency. -## Maintenance Windows +Every replica change triggers a consumer-group rebalance. Scale gradually and +watch lag, duplicates, and destination errors through the rebalance. -### Planned Upgrades +See [Performance](PERFORMANCE.md) for a controlled tuning method. -**Pre-upgrade checklist:** -1. Review changelog for breaking changes -2. Backup current config -3. Test in dev/staging environment -4. Schedule during low-traffic window -5. Prepare rollback plan +## Configuration changes -**Upgrade procedure:** +StreamForge loads its configuration at process start. Apply a changed +`ConfigMap` with a rolling restart; do not assume hot reload: ```bash -# 1. Backup config -kubectl get configmap streamforge-config -n streamforge -o yaml > config-backup.yaml - -# 2. Update Helm chart -helm repo update -helm upgrade streamforge streamforge/streamforge \ - --namespace streamforge \ - --values values.yaml \ - --version 1.1.0 - -# 3. Monitor rollout -kubectl rollout status deployment/streamforge -n streamforge - -# 4. Verify health -kubectl get pods -n streamforge -curl http://streamforge:8080/metrics | grep up - -# 5. Check lag -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` - -**Rollback procedure (if upgrade fails):** -```bash -# 1. Rollback Helm release -helm rollback streamforge -n streamforge - -# 2. Verify rollback -kubectl rollout status deployment/streamforge -n streamforge - -# 3. Restore config if needed -kubectl apply -f config-backup.yaml -``` - -### Config Updates - -**Zero-downtime config update:** - -```bash -# 1. Edit ConfigMap -kubectl edit configmap streamforge-config -n streamforge - -# 2. Rolling restart +target/release/streamforge-validate config.yaml --fail-on-warnings +kubectl apply -f streamforge-config.yaml kubectl rollout restart deployment/streamforge -n streamforge - -# 3. Monitor rollout (one pod at a time) kubectl rollout status deployment/streamforge -n streamforge - -# 4. Check logs for errors -kubectl logs -f deployment/streamforge -n streamforge -``` - -**High-risk config changes:** -- Changing consumer group ID (will re-consume from offset) -- Changing partition routing (breaks keyed ordering) -- Changing DLQ topic (old DLQ orphaned) - -**For high-risk changes:** -1. Deploy as new pipeline with new appid -2. Run in parallel with old pipeline -3. Verify correctness -4. Switch traffic to new pipeline -5. Retire old pipeline - -### Kafka Cluster Maintenance - -**Broker rolling restart:** - -StreamForge will auto-reconnect to Kafka brokers: -- Retry connection errors -- Consumer rebalances automatically -- Producer retries failed sends - -**Monitor during Kafka maintenance:** -```bash -watch kubectl logs deployment/streamforge -n streamforge --tail=20 ``` -**Expected behavior:** -- Connection errors logged (normal during restart) -- Retry attempts visible in logs -- Consumer lag may spike temporarily (catchup after restart) +Treat these as high-risk: -**Kafka version upgrade:** -- Test StreamForge with new Kafka version in dev first -- Check rdkafka compatibility matrix -- Update bootstrap servers if endpoints changed +- changing `appid`, because it selects a different consumer group; +- changing `offset` or resetting committed offsets; +- changing keys or partitioning; +- switching between acknowledged and queued delivery; +- enabling auto commit on a reliability-sensitive pipeline; +- disabling the DLQ or changing its topic; +- changing filter or transform logic. ---- +For high-risk changes, deploy a separate pipeline identity, compare controlled +outputs, and define how to retire or replay the previous pipeline. -## Backup and Recovery +## Incident procedures -### Configuration Backup +### Lag is growing -**Backup all resources:** -```bash -kubectl get configmap,secret,deployment,service,hpa -n streamforge -o yaml > streamforge-backup.yaml -``` +1. Determine whether all expected partitions are assigned. +2. Compare input rate with broker-acknowledged delivery rate. +3. Inspect CPU throttling, memory pressure, and destination latency. +4. Break lag down by partition to detect skew. +5. Check processing errors and DLQ traffic. +6. Scale or tune one control at a time, then confirm that lag is recovering. -**Backup to Git:** -```bash -# Export to Git repo -mkdir -p backups/$(date +%Y-%m-%d) -kubectl get configmap streamforge-config -n streamforge -o yaml > backups/$(date +%Y-%m-%d)/config.yaml -git add backups/ -git commit -m "Backup StreamForge config" -git push -``` +### Deliveries stop -**Automated backup (CronJob):** -```yaml -apiVersion: batch/v1 -kind: CronJob -metadata: - name: streamforge-backup - namespace: streamforge -spec: - schedule: "0 2 * * *" # Daily at 2 AM - jobTemplate: - spec: - template: - spec: - containers: - - name: backup - image: bitnami/kubectl:latest - command: - - /bin/sh - - -c - - | - kubectl get configmap,secret,deployment -n streamforge -o yaml > /backup/streamforge-$(date +%Y-%m-%d).yaml - # Upload to S3 or Git - volumeMounts: - - name: backup - mountPath: /backup - restartPolicy: OnFailure - volumes: - - name: backup - persistentVolumeClaim: - claimName: backup-pvc -``` +1. Check destination broker reachability, authentication, ACLs, and topic + existence. +2. Inspect processing errors and librdkafka logs. +3. Confirm the producer queue is not saturated. +4. In queued mode, check delivered counters rather than enqueue completion. +5. Preserve source offsets and avoid resets until the recovery consequence is + understood. -### Offset Backup +### Process restarts or is killed -**Current offsets are managed by Kafka** (consumer group state). +1. Inspect the previous container logs and termination reason. +2. Check configuration validation, secret mounts, and certificate expiry. +3. Check for OOM kills and CPU throttling. +4. Confirm the metrics port can bind and is not already in use. +5. Restart only after preserving enough diagnostics to identify recurrence. -**View current offsets:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group > offsets-backup.txt -``` +### DLQ traffic appears -**Reset offsets (disaster recovery):** -```bash -# Reset to earliest -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --group \ - --reset-offsets --to-earliest --topic source-topic \ - --execute +1. Record the error type, source topic, partition, offset, and deployed + configuration revision. +2. Determine whether the failure is data-specific or affects all records. +3. Correct the producer data or pipeline expression. +4. Test replay in an isolated topic. +5. Replay with an idempotency strategy and verify the destination. -# Reset to specific offset -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --group \ - --reset-offsets --to-offset 1000 --topic source-topic:0 \ - --execute +## Offset recovery -# Reset to timestamp -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --group \ - --reset-offsets --to-datetime 2026-04-18T00:00:00.000 --topic source-topic \ - --execute -``` +Offset changes can replay or skip data. Stop every consumer in the group before +changing offsets, preview the Kafka command when supported, record the old +offsets, and obtain approval for the exact topic, partition, and target offset. -**Note:** Stop all consumers before resetting offsets. +Never reset to `latest` as a generic incident fix. It intentionally skips the +backlog. -### Disaster Recovery - -**Scenario 1: Namespace deleted** - -```bash -# 1. Recreate namespace -kubectl create namespace streamforge - -# 2. Restore resources -kubectl apply -f streamforge-backup.yaml - -# 3. Verify -kubectl get pods -n streamforge -``` - -**Scenario 2: Config lost** - -```bash -# 1. Restore from backup -kubectl apply -f backups/2026-04-18/config.yaml - -# 2. Restart pods -kubectl rollout restart deployment/streamforge -n streamforge -``` +## Maintenance -**Scenario 3: Kafka data loss (topic deleted)** - -- **Source topic deleted:** StreamForge will error (topic not found), fix Kafka -- **Destination topic deleted:** Recreate topic, StreamForge will auto-recover -- **DLQ topic deleted:** Recreate, but messages lost (not recoverable) - -**Best practice:** Enable Kafka topic auto-create or pre-create topics. - ---- - -## Performance Optimization - -### Tuning for Throughput - -**Goal:** Maximize messages per second - -**Config changes:** -```yaml -threads: 8 # Match CPU cores - -performance: - fetch_min_bytes: 10240 # Larger batches - fetch_max_wait_ms: 100 # Don't wait long - batch_size: 5000 # Large producer batches - linger_ms: 50 # Allow batching - queue_buffering_max_ms: 100 - compression: "zstd" # Fast compression - -# Manual commit for throughput -commit_strategy: "manual" -commit_interval_ms: 5000 # Commit every 5 seconds -``` - -**Resource allocation:** -```yaml -resources: - requests: - cpu: 4000m - memory: 8Gi - limits: - cpu: 4000m - memory: 8Gi -``` - -**Scale replicas:** -```bash -kubectl scale deployment streamforge --replicas=8 -n streamforge -``` - -### Tuning for Latency - -**Goal:** Minimize end-to-end latency - -**Config changes:** -```yaml -threads: 2 # Fewer threads, less contention - -performance: - fetch_min_bytes: 1 # Don't wait for data - fetch_max_wait_ms: 10 # Short wait - batch_size: 100 # Small batches - linger_ms: 0 # Send immediately - queue_buffering_max_ms: 1 - -# Per-message commit for low latency -commit_strategy: "per-message" -``` - -**Resource allocation:** -```yaml -resources: - limits: - cpu: 2000m - memory: 2Gi -``` - -**Trade-off:** Lower throughput (10-20K msg/s) for lower latency (< 10ms p95). - -### Tuning for Efficiency - -**Goal:** Minimize resource usage (cost optimization) - -**Config changes:** -```yaml -threads: 4 # Moderate threading - -performance: - fetch_min_bytes: 5120 # Medium batches - fetch_max_wait_ms: 500 # Wait for batches - batch_size: 2000 - linger_ms: 100 # Batch aggressively - compression: "zstd" - -commit_strategy: "manual" -commit_interval_ms: 10000 # Infrequent commits -``` - -**Resource allocation:** -```yaml -resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi -``` - -**Scale down aggressively:** -```yaml -autoscaling: - minReplicas: 1 - maxReplicas: 5 - targetCPU: 80 # Allow higher utilization -``` - ---- - -## Common Operational Tasks - -### View Logs - -**Tail logs:** -```bash -kubectl logs -f deployment/streamforge -n streamforge -``` - -**Logs from specific pod:** -```bash -kubectl logs streamforge-7c8f9d4b6-abc12 -n streamforge -``` - -**Logs from previous crashed pod:** -```bash -kubectl logs streamforge-7c8f9d4b6-abc12 -n streamforge --previous -``` - -**Search logs for errors:** -```bash -kubectl logs deployment/streamforge -n streamforge | grep ERROR -``` - -**Export logs:** -```bash -kubectl logs deployment/streamforge -n streamforge --since=1h > logs.txt -``` - -### Restart Pods - -**Rolling restart (zero downtime):** -```bash -kubectl rollout restart deployment/streamforge -n streamforge -``` - -**Force restart single pod:** -```bash -kubectl delete pod streamforge-7c8f9d4b6-abc12 -n streamforge -``` - -**Restart all pods:** -```bash -kubectl delete pods -l app=streamforge -n streamforge -``` - -### Update Config - -**Edit ConfigMap:** -```bash -kubectl edit configmap streamforge-config -n streamforge -``` - -**Or apply from file:** -```bash -kubectl apply -f config.yaml -``` - -**Reload config (if hot-reload enabled):** -```bash -curl -X POST http://streamforge:8080/reload -``` - -**Or restart pods:** -```bash -kubectl rollout restart deployment/streamforge -n streamforge -``` - -### Check Consumer Group - -**Describe group:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` - -Output: -``` -GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID -appid source-topic 0 1000 1050 50 consumer-1 -appid source-topic 1 1200 1200 0 consumer-2 -``` - -### Reset Consumer Offsets - -**Reset to latest:** -```bash -kubectl scale deployment streamforge --replicas=0 -n streamforge - -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --group \ - --reset-offsets --to-latest --topic source-topic \ - --execute - -kubectl scale deployment streamforge --replicas=3 -n streamforge -``` - -**Reset to specific timestamp:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --group \ - --reset-offsets --to-datetime 2026-04-18T12:00:00.000 --topic source-topic \ - --execute -``` - -### Test Config Locally - -**Validate config:** -```bash -streamforge-validate config.yaml -``` - -**Run locally:** -```bash -docker run --rm \ - -v $(pwd)/config.yaml:/app/config.yaml:ro \ - streamforge:1.0.0 \ - --config /app/config.yaml -``` - -### Export Metrics - -**Scrape metrics:** -```bash -curl http://streamforge:8080/metrics -``` - -**Export to file:** -```bash -curl http://streamforge:8080/metrics > metrics.txt -``` - -**Query specific metric:** -```bash -curl -s http://streamforge:8080/metrics | grep consumer_lag -``` - ---- - -## Contact and Escalation - -**On-call rotation:** See PagerDuty schedule - -**Escalation path:** -1. On-call engineer (initial response) -2. Platform team lead (if unresolved in 30 minutes) -3. SRE manager (if critical and unresolved in 1 hour) - -**Documentation:** -- [Troubleshooting Guide](TROUBLESHOOTING.md) -- [Architecture](ARCHITECTURE.md) -- [Performance Tuning](PERFORMANCE_TUNING_RESULTS.md) - -**Support channels:** -- Slack: #streamforge-support -- Email: streamforge-oncall@example.com -- GitHub Issues: https://github.com/rahulbsw/streamforge/issues - ---- +- Validate backup copies of configuration without storing secrets in Git. +- Track certificate and credential rotation dates. +- Test clean shutdown, destination outage, DLQ outage, and consumer rebalance. +- Re-run workload tests after Kafka, StreamForge, instance, filter, transform, + or partition changes. +- Keep image digests and configuration revisions in the deployment record. +- Review ACLs and private-network controls regularly. -**Document Version:** 1.0.0 -**Last Updated:** 2026-04-18 +Continue with [Observability](OBSERVABILITY_QUICKSTART.md), +[Delivery guarantees](DELIVERY_GUARANTEES.md), and +[Troubleshooting](TROUBLESHOOTING.md). diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index e26454c..7b8bc12 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -1,717 +1,194 @@ --- title: Performance -nav_order: 9 -parent: Deployment +nav_order: 4 +parent: Operations --- -# Performance Guide +# Performance -Comprehensive guide for optimizing StreamForge performance. +StreamForge performance depends on payload size, partition count, broker and +network latency, filter and transform complexity, destination fan-out, delivery +semantics, and available CPU and memory. -## Table of Contents +No headline throughput result is published here. A result belongs in public +documentation only when it comes from a reproducible end-to-end comparison, +uses the same workload and delivery guarantees as the comparison target, and +improves the approved baseline without correctness regressions. -- [Performance Overview](#performance-overview) -- [Benchmarks](#benchmarks) -- [Configuration Tuning](#configuration-tuning) -- [Best Practices](#best-practices) -- [Monitoring](#monitoring) -- [Troubleshooting](#troubleshooting) -- [Advanced Optimization](#advanced-optimization) +## Runtime controls -## Performance Overview - -### Key Metrics - -| Metric | Typical Value | Excellent Value | -|--------|---------------|-----------------| -| Throughput | 10K-25K msg/s | 50K+ msg/s | -| Latency (p50) | 5-10ms | <5ms | -| Latency (p99) | 15-30ms | <15ms | -| Memory Usage | 50-100MB | <50MB | -| CPU Usage | 50-100% | 200-400% (multi-core) | - -### Performance Characteristics - -**Filter Performance:** -- Simple comparison: ~100ns -- Boolean logic (AND/OR/NOT): ~100-300ns -- Regular expressions: ~500ns-1ยตs -- Array operations: ~1-10ยตs (size dependent) - -**Transform Performance:** -- JSON path extraction: ~50-100ns -- Object construction: ~200-500ns -- Array mapping: ~1-10ยตs (size dependent) -- Arithmetic: ~50ns - -**Overall Overhead:** -- Per-message processing: ~2-10ยตs -- Network I/O: Dominant factor (>99% of time) - -## Benchmarks - -### Throughput Tests - -**Configuration:** -- Message size: 1KB -- Partitions: 10 -- Replicas: 3 -- Hardware: 4 CPU cores, 8GB RAM - -**Results:** - -| Scenario | Throughput | CPU | Memory | -|----------|------------|-----|--------| -| Simple mirroring (no filter) | 45K msg/s | 150% | 45MB | -| With simple filter | 42K msg/s | 180% | 48MB | -| With boolean logic (3 conditions) | 38K msg/s | 200% | 50MB | -| With regex filter | 35K msg/s | 220% | 52MB | -| With array operations | 30K msg/s | 250% | 60MB | -| Multi-destination (5 topics) | 40K msg/s | 300% | 65MB | - -### Latency Tests - -**Configuration:** -- Message size: 1KB -- Batch size: 100 -- Linger: 10ms - -**Results:** - -| Percentile | Simple | With Filter | Multi-Dest | -|------------|--------|-------------|------------| -| p50 | 3ms | 4ms | 5ms | -| p95 | 8ms | 10ms | 12ms | -| p99 | 12ms | 15ms | 20ms | -| p99.9 | 25ms | 30ms | 40ms | - -### Performance Characteristics - -**Streamforge performance at 10K msg/s baseline workload (1KB messages):** - -| Metric | Performance | Capability | -|--------|-------------|------------| -| **Throughput** | 25,000 msg/s | High-volume sustained processing | -| **CPU Usage** | 120% (4 cores) | Efficient multi-core utilization | -| **Memory** | 50MB | Minimal memory footprint | -| **Latency (p99)** | 15ms | Consistent low latency | -| **Startup** | 0.1s | Rapid deployment and recovery | -| **Scalability** | Linear | Predictable resource growth | - -## Configuration Tuning - -### Basic Configuration - -**Minimal (Low Throughput):** -```json -{ - "threads": 2, - "consumer_properties": { - "fetch.min.bytes": "1", - "fetch.wait.max.ms": "100" - }, - "producer_properties": { - "batch.size": "16384", - "linger.ms": "0" - } -} -``` - -**Balanced (Recommended):** -```json -{ - "threads": 4, - "consumer_properties": { - "fetch.min.bytes": "1048576", - "fetch.wait.max.ms": "500", - "max.poll.records": "500" - }, - "producer_properties": { - "batch.size": "65536", - "linger.ms": "10", - "compression.type": "gzip" - } -} -``` - -**High Throughput:** -```json -{ - "threads": 8, - "consumer_properties": { - "fetch.min.bytes": "1048576", - "fetch.wait.max.ms": "500", - "max.poll.records": "1000", - "max.partition.fetch.bytes": "1048576" - }, - "producer_properties": { - "batch.size": "131072", - "linger.ms": "10", - "buffer.memory": "67108864", - "compression.type": "snappy", - "max.in.flight.requests.per.connection": "5" - } -} -``` - -**Low Latency:** -```json -{ - "threads": 4, - "consumer_properties": { - "fetch.min.bytes": "1", - "fetch.wait.max.ms": "0", - "max.poll.records": "100" - }, - "producer_properties": { - "batch.size": "16384", - "linger.ms": "0", - "acks": "1" - } -} -``` - -### Thread Configuration - -**Rule of thumb:** -- Start with: `threads = CPU cores` -- Low throughput: `threads = 2-4` -- High throughput: `threads = CPU cores * 2` -- Very high throughput: `threads = CPU cores * 2-4` - -**Testing:** -```bash -# Measure with different thread counts -for threads in 2 4 8 16; do - echo "Testing with $threads threads..." - # Update config and run - # Monitor throughput -done -``` - -### Consumer Tuning - -**fetch.min.bytes:** -- Low latency: `1` (don't wait for data) -- Balanced: `1048576` (1MB) -- High throughput: `2097152` (2MB) - -**fetch.wait.max.ms:** -- Low latency: `0-100` -- Balanced: `500` -- High throughput: `1000` - -**max.poll.records:** -- Low memory: `100-200` -- Balanced: `500` -- High throughput: `1000-2000` - -**session.timeout.ms:** -- Stable network: `10000` (10s) -- Unreliable network: `30000` (30s) -- Very unreliable: `60000` (60s) - -### Producer Tuning - -**batch.size:** -- Low latency: `16384` (16KB) -- Balanced: `65536` (64KB) -- High throughput: `131072` (128KB) - -**linger.ms:** -- Low latency: `0-1` -- Balanced: `10` -- High throughput: `20-50` - -**compression.type:** -- Fastest: `snappy` -- Balanced: `gzip` -- Best compression: `zstd` -- None: `none` - -**acks:** -- Fastest: `0` (no acknowledgment) -- Balanced: `1` (leader acknowledgment) -- Most durable: `all` (all replicas) - -### Compression Selection - -**Benchmarks (1KB messages):** - -| Type | Compression Ratio | CPU Usage | Throughput | -|------|-------------------|-----------|------------| -| None | 1.0x | Low | 50K msg/s | -| Snappy | 2.5x | Medium | 45K msg/s | -| Gzip | 4.0x | High | 35K msg/s | -| Zstd | 4.5x | Medium-High | 40K msg/s | - -**Recommendations:** -- Network bandwidth limited โ†’ Use `zstd` or `gzip` -- CPU limited โ†’ Use `snappy` or `none` -- Balanced โ†’ Use `snappy` -- Storage limited โ†’ Use `zstd` - -## Best Practices - -### 1. Filter Optimization - -**โŒ Inefficient:** -```json -{ - "filter": "REGEX:/message,.*complex.*pattern.*with.*many.*terms.*" -} -``` - -**โœ… Efficient:** -```json -{ - "filter": "AND:/message/type,==,complex:/message/hasPattern,==,true" -} -``` - -**Guidelines:** -- Use simple comparisons when possible -- Avoid complex regex patterns -- Put cheaper filters first in AND logic -- Use NOT sparingly (still evaluates inner filter) - -### 2. Transform Optimization - -**โŒ Inefficient:** -```json -{ - "transform": "CONSTRUCT:f1=/a/b/c/d/e:f2=/a/b/c/d/f:f3=/a/b/c/d/g" -} -``` - -**โœ… Efficient:** -```json -{ - "transform": "/a/b/c/d" -} -``` - -**Guidelines:** -- Extract parent object when possible -- Avoid redundant field extraction -- Use array operations efficiently -- Minimize arithmetic operations - -### 3. Partitioning Strategy - -**Hash Partitioning (Default):** -```json -{ - "partition": null -} -``` -- Pros: Even distribution -- Cons: No ordering guarantees -- Use: When order doesn't matter - -**Field Partitioning:** -```json -{ - "partition": "/userId" -} -``` -- Pros: Maintains ordering per key -- Cons: Potential hotspots -- Use: When ordering important - -**Hotspot Prevention:** -```json -{ - "filter": "NOT:/userId,==,very-active-user" -} -``` -- Filter out high-volume keys -- Use separate topics for hot keys -- Monitor partition distribution - -### 4. Multi-Destination Efficiency - -**โŒ Inefficient:** -```json -{ - "destinations": [ - {"filter": "REGEX:/type,.*"}, - {"filter": "REGEX:/type,.*"}, - {"filter": "REGEX:/type,.*"} - ] -} -``` - -**โœ… Efficient:** -```json -{ - "destinations": [ - {"filter": "/type,==,a"}, - {"filter": "/type,==,b"}, - {"filter": "/type,==,c"} - ] -} -``` - -**Guidelines:** -- Limit destinations to <10 for best performance -- Use mutually exclusive filters when possible -- Order by match probability (most likely first) -- Combine related destinations - -### 5. Resource Management - -**Memory:** -```json -{ - "consumer_properties": { - "max.poll.records": "500", - "fetch.max.bytes": "52428800" - }, - "producer_properties": { - "buffer.memory": "33554432" - } -} -``` - -**CPU:** -- Match threads to available cores -- Leave 1-2 cores for OS -- Monitor CPU saturation -- Use CPU affinity in containers - -**Network:** -- Compression for bandwidth-limited networks -- Increase batch sizes for high-latency networks -- Use local Kafka clusters when possible -- Monitor network saturation - -### 6. Container Deployment - -**Docker Resource Limits:** -```bash -docker run -d \ - --cpus="4" \ - --memory="512m" \ - --memory-reservation="256m" \ - streamforge:latest -``` - -**Kubernetes Resource Limits:** ```yaml -resources: - requests: - memory: "256Mi" - cpu: "1000m" - limits: - memory: "512Mi" - cpu: "4000m" -``` - -## Monitoring - -### Built-in Metrics - -The application reports metrics every 10 seconds: - -``` -Stats: processed=10000 (1000.0/s), filtered=100 (10.0/s), - completed=9900 (990.0/s), errors=0 (0.0/s) -``` - -**Key Metrics:** -- `processed`: Total messages read -- `filtered`: Messages rejected by filters -- `completed`: Messages successfully sent -- `errors`: Failed sends - -**Rates:** -- Monitor `completed/s` for throughput -- Watch `errors/s` for issues -- Check `filtered/s` for filter effectiveness - -### System Metrics - -**CPU:** -```bash -# Overall CPU -top -p $(pgrep streamforge) - -# Per-thread CPU -ps -eLo pid,tid,pcpu,comm | grep streamforge -``` - -**Memory:** -```bash -# Memory usage -ps aux | grep streamforge - -# Detailed memory -pmap $(pgrep streamforge) -``` - -**Network:** -```bash -# Network traffic -iftop -f "port 9092" +threads: 4 -# Per-process -nethogs -``` +performance: + consumer_batch_size: 100 + consumer_batch_timeout_ms: 100 + parallelism_factor: 10 -### Kafka Metrics + processing_mode: legacy_batch + worker_queue_capacity: 1024 -**Consumer Lag:** -```bash -kafka-consumer-groups.sh \ - --bootstrap-server kafka:9092 \ - --group streamforge \ - --describe -``` + producer_delivery_mode: acknowledged + producer_max_in_flight: 10000 -**Topic Metrics:** -```bash -kafka-run-class.sh kafka.tools.JmxTool \ - --object-name kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec + fetch_min_bytes: 65536 + fetch_max_wait_ms: 500 + batch_size: 1000 + linger_ms: 10 ``` -### Alerting - -**Key Alerts:** -1. Consumer lag > 10000 messages -2. Error rate > 1% -3. Throughput dropped > 50% -4. CPU usage > 90% -5. Memory usage > 80% - -## Troubleshooting - -### Low Throughput +These values illustrate the schema; they are not recommended production sizing. -**Symptoms:** -- Throughput < expected -- CPU usage < 50% +| Field | Default | Effect | +|---|---:|---| +| `consumer_batch_size` | `100` | Maximum records collected for one application batch | +| `consumer_batch_timeout_ms` | `100` | Maximum legacy batch fill wait; queued-delivery drain delay in partition-ordered mode | +| `parallelism_factor` | `10` | Processing concurrency multiplier applied to `threads` | +| `processing_mode` | `legacy_batch` | Legacy batch barrier or bounded partition worker lanes | +| `worker_queue_capacity` | `1024` | Per-worker input bound in `partition_ordered` mode | +| `producer_delivery_mode` | `acknowledged` | Await Kafka acknowledgement or track delivery after enqueue | +| `producer_max_in_flight` | `10000` | Bound for queued delivery futures | -**Diagnosis:** -```bash -# Check consumer lag -kafka-consumer-groups.sh --describe +Effective legacy processing concurrency is: -# Check producer metrics -# Enable debug logging -RUST_LOG=debug +```text +max(1, threads ร— parallelism_factor) ``` -**Solutions:** -1. Increase thread count -2. Increase batch size -3. Increase linger.ms -4. Check network latency -5. Verify partition count - -### High CPU Usage +Configuration validation enforces reliability constraints: -**Symptoms:** -- CPU usage > 90% -- Throughput plateaued +- `partition_ordered` requires auto commit; +- `queued` delivery requires auto commit; +- `queued` delivery requires `retry.max_attempts: 1`; +- `queued` delivery requires `dlq.enabled: false`. -**Diagnosis:** -```bash -# CPU profiling -perf record -p $(pgrep streamforge) -perf report +Review [Delivery guarantees](DELIVERY_GUARANTEES.md) before using either mode. -# Check filter complexity -# Review regex patterns -``` +## Kafka client mappings -**Solutions:** -1. Reduce thread count -2. Simplify filters -3. Optimize regex patterns -4. Reduce destinations -5. Scale horizontally +| Performance field | librdkafka property | +|---|---| +| `fetch_min_bytes` | `fetch.min.bytes` | +| `fetch_max_wait_ms` | `fetch.wait.max.ms` | +| `batch_size` | `batch.num.messages` | +| `linger_ms` | `linger.ms` | +| `queue_buffering_max_ms` | `queue.buffering.max.ms` | -### High Memory Usage +`batch_size` is a message count. Configure librdkafka `batch.size` through +`producer_properties` when a byte limit is needed. -**Symptoms:** -- Memory usage > expected -- OOM errors +`linger.ms` and `queue.buffering.max.ms` are aliases. If both performance fields +are set, `linger_ms` wins. Explicit `consumer_properties` and +`producer_properties` override generated performance properties. -**Diagnosis:** -```bash -# Memory profiling -valgrind --tool=massif ./streamforge - -# Check message sizes -# Review batch sizes -``` - -**Solutions:** -1. Reduce max.poll.records -2. Reduce buffer.memory -3. Reduce fetch.max.bytes -4. Check for memory leaks -5. Increase container limits - -### High Latency - -**Symptoms:** -- p99 latency > 50ms -- Slow message delivery - -**Diagnosis:** -```bash -# Network latency -ping kafka-broker - -# Kafka latency -kafka-run-class.sh kafka.tools.JmxTool -``` - -**Solutions:** -1. Reduce linger.ms -2. Reduce batch.size -3. Set fetch.wait.max.ms=0 -4. Use acks=1 -5. Optimize network path - -## Advanced Optimization - -### CPU Pinning - -```bash -# Pin to specific CPUs -taskset -c 0-3 ./streamforge +```yaml +performance: + fetch_min_bytes: 65536 + batch_size: 1000 + +consumer_properties: + fetch.min.bytes: "1" + +producer_properties: + batch.num.messages: "500" + batch.size: "65536" +``` + +## Tuning procedure + +1. Define the correctness and delivery profile. +2. Fix the source and destination topology, topic partitions, replication, + acknowledgements, and security settings. +3. Use representative payload sizes, keys, headers, filters, transforms, and + fan-out. +4. Warm the runtime and brokers before measurement. +5. Record broker-acknowledged completions, end-to-end latency, lag, errors, CPU, + memory, network, and destination offsets. +6. Run multiple trials and report variance. +7. Change one control at a time. +8. Retain a change only if it improves the target without violating reliability, + latency, error, or resource objectives. + +Useful experiments: + +- increase application batch size for steady traffic, then check latency and + memory; +- reduce the batch timeout for low-volume latency; +- increase processing concurrency only while work is I/O-bound and bounded + queues remain healthy; +- compare the legacy batch scheduler with partition-ordered lanes using a + workload whose delivery constraints permit auto commit; +- sweep producer linger and queued depth only with the queued-mode reliability + limitations explicitly accepted; +- use simple comparisons instead of regex when they express the same rule; +- avoid transforms on passthrough destinations; +- inspect key distribution before adding partitions or replicas. + +## Partitioning + +- A present key, including explicit JSON `null`, is hashed to an explicit target + partition. +- An absent key delegates partition choice to librdkafka. +- Field partitioning selects an explicit partition from the configured JSON + field. + +Low-cardinality or skewed keys can create hot partitions. Measure per-partition +lag and delivery rate rather than relying only on totals. -# Docker with CPU affinity -docker run --cpuset-cpus="0-3" streamforge:latest -``` +## Benchmarks -### Huge Pages +Focused Criterion suites are available: ```bash -# Enable huge pages -echo 512 > /proc/sys/vm/nr_hugepages - -# Run with huge pages -MALLOC_MMAP_THRESHOLD_=131072 ./streamforge +cargo bench --bench filter_benchmarks +cargo bench --bench transform_benchmarks +cargo bench --bench end_to_end_benchmark ``` -### Network Optimization +Microbenchmarks isolate code paths. They do not include Kafka brokers, network, +consumer commits, scheduling, or destination acknowledgement and must not be +presented as end-to-end message throughput. -```bash -# Increase socket buffers -sysctl -w net.core.rmem_max=16777216 -sysctl -w net.core.wmem_max=16777216 - -# TCP tuning -sysctl -w net.ipv4.tcp_window_scaling=1 -sysctl -w net.ipv4.tcp_rmem="4096 87380 16777216" -sysctl -w net.ipv4.tcp_wmem="4096 65536 16777216" -``` +An end-to-end result record should include: -### Profiling +- source revision and clean/dirty worktree state; +- instance or host type, CPU architecture, core allocation, and memory; +- Kafka versions, broker topology, storage, and network placement; +- topic partitions, replication, and retention; +- payload distribution and total records; +- complete StreamForge configuration with secrets redacted; +- warm-up, run duration, repetitions, and aggregation method; +- source-produced count, source-consumed count, destination-acknowledged count, + and independently observed destination count; +- latency percentiles, lag, errors, CPU, memory, and network; +- setup, runtime, and teardown cost; +- confirmation that no benchmark service was exposed publicly. -**CPU Profiling:** -```bash -# Install perf -# Start profiling -perf record -g -p $(pgrep streamforge) +Reject a run if counters are inconsistent, the destination count is incomplete, +the comparison uses different delivery semantics, or any resource remains after +the teardown audit. -# Generate flamegraph -perf script | stackcollapse-perf.pl | flamegraph.pl > flamegraph.svg -``` +## Optimization priorities -**Memory Profiling:** -```bash -# Using valgrind -valgrind --tool=massif --massif-out-file=massif.out ./streamforge +The current JSON pipeline parses payloads into `serde_json::Value`, walks the +tree for filters and transforms, and serializes destination values. SIMD by +itself is unlikely to improve pointer-heavy tree traversal. -# Analyze -ms_print massif.out -``` - -### Load Testing +Profile before changing the representation. Candidate work should be evaluated +in this order: -**Generate Load:** -```bash -# Using kafka-producer-perf-test -kafka-producer-perf-test.sh \ - --topic test \ - --num-records 1000000 \ - --record-size 1024 \ - --throughput 10000 \ - --producer-props bootstrap.servers=kafka:9092 -``` +1. preserve raw Kafka bytes for routes that do not need JSON; +2. parse lazily according to selected operations; +3. reduce array-element and destination serialization copies; +4. isolate vectorizable byte scanning, hashing, or numeric kernels; +5. measure the full pipeline again after each change. -**Measure Performance:** -```bash -# Monitor throughput -watch -n 1 'docker logs mirrormaker 2>&1 | tail -1' - -# Measure latency -kafka-consumer-perf-test.sh \ - --topic output \ - --bootstrap-server kafka:9092 \ - --messages 100000 -``` +## Production checklist -## Performance Checklist - -### Pre-Production - -- [ ] Benchmark with production-like data -- [ ] Load test at 2x expected throughput -- [ ] Verify latency under load -- [ ] Test failure scenarios -- [ ] Profile CPU and memory usage -- [ ] Validate filter performance -- [ ] Check network bandwidth -- [ ] Monitor consumer lag -- [ ] Test with different thread counts -- [ ] Verify compression benefits - -### Production - -- [ ] Set up monitoring -- [ ] Configure alerting -- [ ] Tune based on metrics -- [ ] Monitor consumer lag -- [ ] Track error rates -- [ ] Review logs regularly -- [ ] Plan for scaling -- [ ] Document configuration -- [ ] Set resource limits -- [ ] Regular performance reviews - -## Summary - -### Quick Wins - -1. **Enable compression** โ†’ 2-4x bandwidth reduction -2. **Tune thread count** โ†’ Match CPU cores -3. **Optimize batch size** โ†’ Balance latency/throughput -4. **Simplify filters** โ†’ Use simple comparisons -5. **Monitor metrics** โ†’ Identify bottlenecks - -### Performance Targets - -| Environment | Throughput | Latency p99 | CPU | Memory | -|-------------|------------|-------------|-----|--------| -| Development | 5K msg/s | 50ms | <100% | <100MB | -| Staging | 15K msg/s | 30ms | <200% | <150MB | -| Production | 25K+ msg/s | 15ms | <400% | <200MB | - -### Next Steps - -- Test with your specific workload -- Measure before optimizing -- Optimize bottlenecks first -- Monitor continuously -- Iterate and improve - -For more information: -- [USAGE.md](USAGE.md) - Use cases and patterns -- [CONTRIBUTING.md](CONTRIBUTING.md) - Development setup -- [ADVANCED_DSL_GUIDE.md](ADVANCED_DSL_GUIDE.md) - Filter optimization +- Benchmark the exact delivery profile used in production. +- Verify destination records independently of application counters. +- Monitor lag, delivery errors, CPU, memory, and partition balance. +- Establish resource limits from measured use. +- Repeat the workload after any broker, instance, partition, filter, transform, + fan-out, security, or version change. +- Publish numerical comparisons only after the result contract is satisfied. diff --git a/docs/PERFORMANCE_TESTING.md b/docs/PERFORMANCE_TESTING.md index a24e7e3..e52c9ae 100644 --- a/docs/PERFORMANCE_TESTING.md +++ b/docs/PERFORMANCE_TESTING.md @@ -1,449 +1,214 @@ -# Performance Testing Guide - -## Overview - -Streamforge includes multiple levels of performance testing: -1. **Microbenchmarks** (Criterion) - Fast, run in CI -2. **End-to-End Throughput Tests** - Manual, requires Kafka -3. **CI Performance Tests** - Manual trigger only +--- +title: Performance Testing +nav_order: 10 +parent: Deployment +--- -## 1. Microbenchmarks (Criterion) +# Performance Testing -### What They Test +StreamForge separates microbenchmarks from Kafka-backed throughput tests. +Results from one layer must not be presented as results from another. -- Filter evaluation performance -- JSON Path extraction speed -- Transform operation overhead -- Header manipulation cost +| Layer | Target | Kafka | Purpose | +|---|---|---:|---| +| Criterion microbenchmarks | `filter_benchmarks`, `transform_benchmarks` | No | Isolate DSL construction and steady-state evaluation | +| Criterion synthetic pipeline | `end_to_end_benchmark` | No | Isolate parse, envelope, filter, transform, and serialization stages | +| Sustained Kafka harness | `run_throughput_test.sh` | Yes | Measure a warmed, fixed-duration pipeline with independent correctness checks | -### Running Locally +## Criterion ```bash -# Run all benchmarks -cargo bench - -# Run specific benchmark -cargo bench --bench filter_benchmarks -cargo bench --bench transform_benchmarks - -# With verbose output -cargo bench -- --verbose +cargo bench --bench filter_benchmarks -- --noplot +cargo bench --bench transform_benchmarks -- --noplot +cargo bench --bench end_to_end_benchmark -- --noplot ``` -### Results Location - -``` -target/criterion/ -โ”œโ”€โ”€ filter_benchmarks/ -โ”‚ โ””โ”€โ”€ report/index.html -โ””โ”€โ”€ transform_benchmarks/ - โ””โ”€โ”€ report/index.html -``` - -Open `target/criterion/report/index.html` in browser for detailed charts. - -### CI Integration - -**Status**: โœ… Builds benchmarks, โŒ Doesn't run them - -```yaml -# .github/workflows/ci.yml -rust-benchmarks: - name: Rust - Benchmarks - runs-on: ubuntu-latest - steps: - - name: Build benchmarks - run: cargo bench --no-run -``` - -**Why not run?** -- Takes 5-10 minutes -- Results would be noisy (shared CI runners) -- Not critical for PR validation - -## 2. End-to-End Throughput Tests - -### What They Test - -- Real Kafka integration -- Message consumption rate -- Transformation throughput -- Producer performance -- Consumer lag behavior -- Observability overhead - -### Running Locally - -#### Quick Test (Automated) +Save and compare the same target: ```bash -cd benchmarks - -# Run with 100K messages, 30K msg/s target -./run_throughput_test.sh 100000 30000 - -# Run with 500K messages, 50K msg/s target -./run_throughput_test.sh 500000 50000 +cargo bench --bench end_to_end_benchmark -- \ + --save-baseline current --noplot +cargo bench --bench end_to_end_benchmark -- \ + --baseline current --noplot ``` -#### Manual Test (Full Control) - -```bash -# 1. Start Kafka -docker-compose -f docker-compose.benchmark.yml up -d - -# 2. Create topics -kafka-topics --create --topic test-input --partitions 16 --replication-factor 1 --bootstrap-server localhost:9092 -kafka-topics --create --topic test-output --partitions 16 --replication-factor 1 --bootstrap-server localhost:9092 - -# 3. Generate test data -cd benchmarks -./generate_json_test_data.sh 200000 test_data.jsonl - -# 4. Configure Streamforge -cat > config.json << EOF -{ - "appid": "perf-test", - "bootstrap": "localhost:9092", - "input": "test-input", - "output": "test-output", - "threads": 16, - "observability": { - "metrics_enabled": true, - "metrics_port": 9090, - "lag_monitoring_enabled": true, - "lag_monitoring_interval_secs": 10 - } -} -EOF - -# 5. Start Streamforge -./target/release/streamforge - -# 6. In another terminal, send messages -cat benchmarks/test_data.jsonl | kafka-console-producer \ - --bootstrap-server localhost:9092 \ - --topic test-input \ - --batch-size 2000 - -# 7. Monitor metrics -watch -n 2 'curl -s http://localhost:9090/metrics | grep consumed_total' -``` +Criterion artifacts are written below `target/criterion/`. These tests do not +include broker, network, consumer, commit, or delivery-acknowledgement costs. -### Expected Performance +## Sustained Kafka harness -| Environment | Partitions | Threads | Throughput | Notes | -|-------------|------------|---------|------------|-------| -| macOS (laptop) | 8 | 8 | 6,700 msg/s | Validated | -| macOS (laptop) | 16 | 16 | 11,890 msg/s | Validated | -| Linux (server) | 16 | 16 | ~15,000 msg/s | Estimated | -| Linux (dedicated) | 32 | 32 | ~30,000 msg/s | Estimated | -| Production (clustered) | 64+ | 32+ | 50,000+ msg/s | With Kafka cluster | - -### Results Location - -``` -benchmarks/results/throughput_test_/ -โ”œโ”€โ”€ REPORT.md # Performance summary -โ”œโ”€โ”€ streamforge.log # Application logs -โ”œโ”€โ”€ metrics_before.txt # Prometheus metrics before -โ”œโ”€โ”€ metrics_after.txt # Prometheus metrics after -โ””โ”€โ”€ producer_output.txt # Kafka producer output -``` - -## 3. CI Performance Tests (GitHub Actions) - -### Manual Workflow Trigger - -Performance tests are **NOT** run automatically on every commit. They must be triggered manually: +The supported local runtime is Podman. Build StreamForge, then start the private +benchmark environment: ```bash -# Via GitHub UI: -# 1. Go to Actions tab -# 2. Select "Performance Tests" workflow -# 3. Click "Run workflow" -# 4. Configure parameters: -# - Messages: 100000 -# - Partitions: 8 -# - Threads: 8 - -# Via GitHub CLI: -gh workflow run performance-test.yml \ - -f messages=100000 \ - -f partitions=16 \ - -f threads=16 +cargo build --release --bin streamforge +podman compose -f docker-compose.benchmark.yml up -d ``` -### What It Tests - -1. **Criterion Benchmarks** - Microbenchmarks -2. **Throughput Test** - End-to-end with Kafka -3. **Latency Test** - Latency distribution - -### Limitations +Kafka is published only on `127.0.0.1:9092`. The compose network is internal, +and the ingress and output runners publish no host ports. -**CI Environment:** -- Ubuntu-latest runners -- 2 CPU cores, 7GB RAM -- Shared VM (variable load) -- No dedicated resources +Run the harness: -**Expected Results:** -- Lower than production -- High variance between runs -- **For comparison only**, not absolute benchmarks - -**Why Not Run on Every PR?** -- Takes 15-30 minutes -- Results are not reliable for comparison -- CI runners not suitable for performance testing -- Would significantly slow down CI feedback loop +```bash +scripts/benchmarks/run_throughput_test.sh \ + [dataset_records] [partitions] [threads] [repetitions] +``` + +Defaults are 10,000 deterministic dataset records, 8 partitions, 8 threads, +and 3 repetitions. The dataset is replayed for the configured duration; it is +not the timed record count. + +| Variable | Default | Meaning | +|---|---:|---| +| `BENCHMARK_DURATION_SECONDS` | `180` | Shared measured window per repetition | +| `BENCHMARK_WARMUP_MESSAGES` | `10000` | Untimed full-path warm-up count | +| `BENCHMARK_INGRESS_TARGET_RATE` | `0` | Open-loop ingress msg/s; `0` is unbounded | +| `BENCHMARK_STARTUP_TIMEOUT` | `120` | Readiness and physical cleanup timeout | +| `BENCHMARK_DRAIN_TIMEOUT` | `180` | Producer flush and post-window drain timeout | +| `BENCHMARK_POLL_INTERVAL_MS` | `500` | Metrics/resource sample interval | +| `BENCHMARK_METRICS_PORT` | `19090` | Loopback-only StreamForge metrics port | +| `BENCHMARK_RESULTS_ROOT` | `target/performance-results/throughput` | Artifact root | +| `BENCHMARK_PROCESSING_MODE` | `partition_ordered` | `legacy_batch` or `partition_ordered` | +| `BENCHMARK_DELIVERY_MODE` | `queued` | `acknowledged` or bounded `queued` | +| `BENCHMARK_MAX_IN_FLIGHT` | `10000` | Queued delivery bound | +| `CONTAINER_RUNTIME` | `podman` | Container CLI | -### When to Run +`legacy_batch` plus `queued` is rejected because it lacks a safe final delivery +drain. Queued delivery also retains the product configuration safety checks +documented in [Delivery guarantees](DELIVERY_GUARANTEES.md). -โœ… **Run manually when:** -- Major performance optimization completed -- Investigating performance regression -- Before release (validation) -- After infrastructure changes +### Job model -โŒ **Don't run for:** -- Every commit -- Minor bug fixes -- Documentation changes -- Regular PRs +Every repetition uses three independent jobs: -## 4. Continuous Performance Monitoring (Recommended) +1. The ingress job starts one persistent Kafka producer, settles it, publishes + warm-up records through that same process, then sends deterministic records + for the shared duration. Producer startup is outside measurement. +2. The metrics validator samples destination-specific consumed, produced, + broker-delivered, and error counters plus StreamForge CPU and RSS. It owns + the timed output-delivery rate. +3. The output validator starts only after the measured window. It consumes the + exact expected output count independently, so validation cannot reduce the + timed throughput. -For production-grade performance tracking, use a dedicated performance testing environment: +StreamForge and all three jobs must be ready before the controller releases one +monotonic-clock barrier. The harness requires two stable, exact warm-up samples +before releasing it. -### Option A: Dedicated Perf Environment +### Measurement contract -```yaml -# Separate performance testing server -- Dedicated hardware (not shared) -- Linux (Ubuntu 22.04 LTS) -- 16+ CPU cores -- 32GB+ RAM -- SSD storage -- Kafka cluster (3+ brokers) -``` +The primary rate is: -**Schedule:** -- Nightly performance tests -- Compare against baseline -- Alert on regressions > 10% - -### Option B: Nightly GitHub Actions - -```yaml -# .github/workflows/nightly-perf.yml -on: - schedule: - - cron: '0 2 * * *' # 2 AM UTC daily - workflow_dispatch: - -jobs: - performance-baseline: - runs-on: ubuntu-latest - # ... run throughput tests - # ... compare with previous results - # ... alert if regression detected +```text +destination-specific broker-delivered delta +------------------------------------------------ +actual monotonic metrics-sample window in seconds ``` -### Option C: External Service +The metrics sample immediately after the configured deadline defines the +actual window. Its observation delay is bounded by the polling interval and is +included in the denominator. Startup, broker readiness, consumer assignment, +warm-up, drain, output validation, and teardown are excluded. -Use services like: -- **Bencher.dev** - Continuous benchmarking -- **Conbench** - Benchmark tracking -- **Custom Grafana** - Historical tracking +Ingress is paced in batches when `BENCHMARK_INGRESS_TARGET_RATE` is non-zero. +Choose a rate high enough to establish the intended load but low enough to +avoid spending the run measuring an overloaded local broker. Treat an +ingress-limited result as a sustainable lower bound, not an engine ceiling. -## 5. Performance Regression Detection +CPU and RSS summaries use samples from the measured window only. Drain and the +post-window output validator do not contribute to those resource statistics. -### Tracking Baseline +### Pass criteria -```bash -# Run baseline test -./benchmarks/run_throughput_test.sh 100000 30000 - -# Save results -cp benchmarks/results/throughput_test_*/REPORT.md \ - benchmarks/baselines/baseline_$(date +%Y%m%d).md +A repetition passes only when all of these values exactly equal the ingress +job's timed record count: -# Compare with baseline -diff benchmarks/baselines/baseline_20260401.md \ - benchmarks/results/throughput_test_latest/REPORT.md -``` +- source-topic end-offset delta; +- StreamForge consumed delta; +- destination-labelled produced delta; +- destination-labelled broker-delivered delta; +- destination-topic end-offset delta; +- independently consumed output count. -### Regression Criteria +The processing-error delta must be zero. Any missing metric, decreasing +counter, failed process, premature output validator, timeout, or count mismatch +fails the run. -**Alert if:** -- Throughput drops > 10% -- Latency P99 increases > 20% -- Error rate > 0.1% -- Consumer lag > 1000 messages +After each repetition, the harness deletes its two topics and waits for both +Kafka metadata deletion and physical partition-directory reclamation before +starting the next repetition. This prevents retained benchmark data from +changing later runs or filling the broker disk. -### Git Bisect for Regressions +### Artifacts -```bash -# Find commit that caused regression -git bisect start -git bisect bad HEAD -git bisect good v1.0.0 +Each repetition contains: -# For each commit: -cargo build --release -./benchmarks/run_throughput_test.sh 100000 30000 -# Mark good/bad based on results +- generated StreamForge configuration; +- ingress, metrics, and output-validator results; +- StreamForge and job logs; +- CSV metric/resource samples; +- an exact-accounting result. -git bisect reset -``` +The schema-version-3 aggregate records median, minimum, maximum, mean, median +absolute deviation, and coefficient of variation. It also records the dataset +SHA-256, source revision and dirty state, host CPU/OS/Rust version, Kafka image +digest, Podman version, VM CPU/memory allocation, and all run manifests. -## 6. Profiling and Optimization +The aggregate rejects public use when fewer than three repetitions were run, +the measured duration is below 120 seconds, the worktree is dirty, or one or +more runs are ingress-limited. -### CPU Profiling +Stop and remove only the disposable benchmark environment: ```bash -# Install flamegraph -cargo install flamegraph +podman compose -f docker-compose.benchmark.yml down -v +``` -# Profile Streamforge -cargo flamegraph --bin streamforge +## Comparison and publication rules -# Open flamegraph.svg in browser -``` +Compare results only when all material inputs match: -### Memory Profiling +- reviewed source revision and clean/dirty state; +- payload bytes, count, seed, and dataset hash; +- StreamForge configuration and delivery semantics; +- Kafka image, topology, storage, partitions, and replication; +- host architecture, Podman VM CPU/memory, and toolchain; +- warm-up, duration, repetitions, ingress rate, and aggregation method. -```bash -# Install valgrind -sudo apt-get install valgrind +Use at least three 120-second repetitions on dedicated hardware. Shared CI is +smoke evidence only. Do not compare the sustained output-delivery rate with a +startup-inclusive completion rate, a Criterion operation time, or a benchmark +using different acknowledgement and correctness guarantees. -# Profile memory -valgrind --tool=massif ./target/release/streamforge +No numerical result should appear on the public performance page unless its +aggregate is publication-eligible and it improves the approved matched +baseline without correctness or resource regressions. -# Analyze -ms_print massif.out.* -``` +## Profiling and SIMD gates -### Async Profiling +Profile the whole process under the Kafka workload before changing the payload +representation or adding SIMD. -```bash -# Install tokio-console -cargo install tokio-console +Consider a raw/lazy envelope only when parse plus serialization accounts for at +least 30% of sampled data-plane CPU. Consider SIMD only when profiling finds a +stable vectorizable kernel such as structural byte scanning or homogeneous +numeric processing. Require: -# Run with tokio-console enabled -RUSTFLAGS="--cfg tokio_unstable" cargo build --release -./target/release/streamforge +- at least 15% improvement in the confirmed kernel; +- no more than 3% regression for small-message workloads; +- correctness coverage on `aarch64` and `x86_64`; +- a scalar fallback and unchanged public behavior. -# In another terminal -tokio-console -``` +Pointer-heavy `serde_json::Value` traversal and configuration-time regex +compilation are not standalone SIMD targets. -## 7. Best Practices - -### DO: -โœ… Run benchmarks before and after optimization -โœ… Use consistent test environment -โœ… Warm up before measuring -โœ… Run multiple iterations -โœ… Document test conditions -โœ… Compare against baseline -โœ… Profile before optimizing - -### DON'T: -โŒ Run performance tests on laptop during other work -โŒ Compare results from different machines -โŒ Optimize without measuring first -โŒ Trust single-run results -โŒ Run performance tests in CI for every commit -โŒ Mix load testing with other benchmarks - -## 8. Troubleshooting - -### Low Throughput - -**Symptoms**: Throughput < 5,000 msg/s - -**Check:** -1. Partition count matches thread count -2. Kafka is on dedicated machine (not localhost) -3. Producer tool (use kafka-producer-perf-test, not console-producer) -4. Consumer fetch settings (increase fetch.min.bytes) -5. CPU usage (should be 70-90%) -6. Network bandwidth - -### High Latency - -**Symptoms**: P99 > 100ms - -**Check:** -1. System load (other processes) -2. Garbage collection (though Rust doesn't have GC) -3. Disk I/O (check with iostat) -4. Network latency (ping Kafka broker) -5. Transform complexity -6. Logging level (debug logs add overhead) - -### Consumer Lag - -**Symptoms**: Lag keeps increasing - -**Check:** -1. Throughput < production rate -2. Error rate (errors slow processing) -3. Partition rebalancing -4. Kafka broker health -5. Thread count (increase if CPU available) - -## 9. Performance Testing Checklist - -Before running performance tests: - -- [ ] Clean Kafka topics (no old data) -- [ ] Restart Kafka (clean state) -- [ ] Build release mode (`cargo build --release`) -- [ ] Close other applications -- [ ] Consistent network conditions -- [ ] Document test configuration -- [ ] Monitor system resources during test -- [ ] Save results with timestamp -- [ ] Compare with baseline -- [ ] Document any anomalies - -## 10. Interpreting Results - -### Good Results - -โœ… Throughput within 10% of target -โœ… P99 latency < 25ms -โœ… Zero errors -โœ… Consumer lag returns to 0 -โœ… CPU usage 70-90% -โœ… Memory stable - -### Investigate If - -โš ๏ธ Throughput < 80% of expected -โš ๏ธ P99 latency > 50ms -โš ๏ธ Error rate > 0% -โš ๏ธ Consumer lag keeps growing -โš ๏ธ CPU usage < 50% or > 95% -โš ๏ธ Memory keeps growing - -## Summary - -| Test Type | When to Run | Where | Purpose | -|-----------|-------------|-------|---------| -| **Microbenchmarks** | During development | Local | Validate optimization | -| **Throughput Tests** | Before PR / Release | Local | Validate end-to-end performance | -| **CI Perf Tests** | Manual trigger | GitHub Actions | Regression detection | -| **Nightly Tests** | Scheduled | Dedicated server | Continuous monitoring | -| **Load Tests** | Pre-production | Staging environment | Production validation | +## CI ---- +`.github/workflows/performance-test.yml` runs Criterion and a Kafka smoke test +on manual dispatch. It intentionally has no regression threshold because +GitHub-hosted runners are shared and variable. -**For more details:** -- [Throughput Testing Guide](../benchmarks/THROUGHPUT_TESTING.md) -- [Observability Test Guide](../benchmarks/OBSERVABILITY_TEST_GUIDE.md) -- [Benchmark Results](../benchmarks/results/BENCHMARKS.md) +Historical records are retained under `docs/benchmarks/results/`. They are not +current baselines unless their workload, environment, configuration, delivery +semantics, and measurement contract match. diff --git a/docs/PERFORMANCE_TUNING_RESULTS.md b/docs/PERFORMANCE_TUNING_RESULTS.md new file mode 100644 index 0000000..2de6d20 --- /dev/null +++ b/docs/PERFORMANCE_TUNING_RESULTS.md @@ -0,0 +1,7 @@ +# Performance Tuning Results + +This compatibility page replaces the retired fixed-result report. + +- See [Performance](PERFORMANCE.md) for current tuning guidance. +- See [Performance Testing](PERFORMANCE_TESTING.md) for reproducible baselines. +- See [Implementation Status](IMPLEMENTATION_STATUS.md) for verified status. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 7a121de..f180265 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -14,7 +14,7 @@ In five minutes, run StreamForge locally and replicate one source topic into: ## 1. Start Redpanda ```bash -docker compose -f examples/redpanda/docker-compose.yml up -d +podman compose -f examples/redpanda/docker-compose.yml up -d ``` ## 2. Validate the Demo Config @@ -37,7 +37,7 @@ Leave StreamForge running in this terminal. Open a second terminal for the remai Create the demo topics: ```bash -docker compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ +podman compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ rpk topic create raw-orders analytics-orders pii-safe-orders ``` @@ -46,21 +46,21 @@ Produce one order that matches both destinations: ```bash printf '%s\n' \ '{"order_id":"ord-1001","customer":{"id":"cust-42","email":"alice@example.com"},"amount":125,"region":"us","created_at":"2026-05-12T15:04:05Z"}' \ - | docker compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ + | podman compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ rpk topic produce raw-orders ``` Verify the analytics-shaped payload: ```bash -docker compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ +podman compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ rpk topic consume analytics-orders -n 1 --offset start ``` Verify the PII-safe summary payload: ```bash -docker compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ +podman compose -f examples/redpanda/docker-compose.yml exec -T redpanda \ rpk topic consume pii-safe-orders -n 1 --offset start ``` diff --git a/docs/QUICK_REFERENCE.md b/docs/QUICK_REFERENCE.md index a8c7f6a..6493228 100644 --- a/docs/QUICK_REFERENCE.md +++ b/docs/QUICK_REFERENCE.md @@ -122,6 +122,7 @@ Useful metrics: - `streamforge_messages_consumed_total` - `streamforge_messages_produced_total` +- `streamforge_messages_delivered_total` - `streamforge_messages_filtered_total` - `streamforge_consumer_lag` - `streamforge_processing_duration_seconds` diff --git a/docs/SECURITY_CONFIGURATION.md b/docs/SECURITY_CONFIGURATION.md index 73f0808..69c45cd 100644 --- a/docs/SECURITY_CONFIGURATION.md +++ b/docs/SECURITY_CONFIGURATION.md @@ -1,563 +1,201 @@ --- title: Security -nav_order: 10 -parent: Deployment +nav_order: 2 +parent: Operations --- -# Security Configuration Guide +# Security -Complete guide for securing Kafka connections with SSL/TLS encryption and SASL authentication. +StreamForge maps its top-level `security` configuration to librdkafka for both +the source consumer and destination producer. Secure the configuration file, +Kafka authorization, network path, container, and observability endpoint as one +system. -## Table of Contents +## Supported configuration -- [Overview](#overview) -- [Security Protocols](#security-protocols) -- [SSL/TLS Encryption](#ssltls-encryption) -- [SASL Authentication](#sasl-authentication) -- [Cloud Provider Examples](#cloud-provider-examples) -- [Troubleshooting](#troubleshooting) -- [Best Practices](#best-practices) +The runtime schema accepts: ---- - -## Overview - -StreamForge supports all standard Kafka security features: +- `PLAINTEXT` +- `SSL` +- `SASL_PLAINTEXT` +- `SASL_SSL` +- SASL mechanisms `PLAIN`, `SCRAM-SHA-256`, `SCRAM-SHA-512`, `GSSAPI`, and + `OAUTHBEARER` -| Feature | Support | Use Case | -|---------|---------|----------| -| **SSL/TLS** | โœ… Full | Encrypted connections | -| **Mutual TLS** | โœ… Full | Certificate-based authentication | -| **SASL/PLAIN** | โœ… Full | Username/password (simple) | -| **SASL/SCRAM-SHA-256** | โœ… Full | Username/password (secure) | -| **SASL/SCRAM-SHA-512** | โœ… Full | Username/password (more secure) | -| **SASL/GSSAPI** | โœ… Full | Kerberos authentication | -| **SASL/OAUTHBEARER** | โœ… Full | OAuth 2.0 token authentication | +Actual availability also depends on the linked librdkafka build and broker +configuration. Validate the mechanism in the target environment before +production use. ---- - -## Security Protocols +Use `SSL` or `SASL_SSL` on untrusted networks. `PLAINTEXT` and +`SASL_PLAINTEXT` do not protect message data in transit. -Kafka supports four security protocols: +## TLS -### 1. PLAINTEXT (Default) -No encryption, no authentication. **Not recommended for production.** - -```yaml -# No security configuration needed -appid: mirrormaker -bootstrap: kafka:9092 -``` - -### 2. SSL -Encryption only, optional certificate-based authentication (mutual TLS). +Broker verification: ```yaml security: protocol: SSL ssl: - ca_location: /path/to/ca-cert.pem -``` - -### 3. SASL_PLAINTEXT -Authentication without encryption. **Not recommended for production.** - -```yaml -security: - protocol: SASL_PLAINTEXT - sasl: - mechanism: PLAIN - username: user - password: pass -``` - -### 4. SASL_SSL (Recommended) -Both encryption (SSL) and authentication (SASL). - -```yaml -security: - protocol: SASL_SSL - ssl: - ca_location: /path/to/ca-cert.pem - sasl: - mechanism: SCRAM-SHA-256 - username: user - password: pass -``` - ---- - -## SSL/TLS Encryption - -### Simple SSL (One-Way TLS) - -Client verifies broker's certificate: - -```yaml -security: - protocol: SSL - ssl: - # CA certificate to verify broker - ca_location: /path/to/ca-cert.pem - - # Verify broker hostname (recommended) + ca_location: /run/streamforge/tls/ca.pem endpoint_identification_algorithm: https ``` -**Use Case:** Basic encryption for data in transit. - -### Mutual TLS (mTLS) - -Both client and broker verify each other: +Mutual TLS: ```yaml security: protocol: SSL ssl: - # CA certificate to verify broker - ca_location: /path/to/ca-cert.pem - - # Client certificate for authentication - certificate_location: /path/to/client-cert.pem - key_location: /path/to/client-key.pem - key_password: optional-key-password - - # Verify broker hostname + ca_location: /run/streamforge/tls/ca.pem + certificate_location: /run/streamforge/tls/client.pem + key_location: /run/streamforge/tls/client-key.pem endpoint_identification_algorithm: https ``` -**Use Case:** Certificate-based authentication, high security environments. +Mount CA, certificate, and private-key files read-only. Restrict the private key +to the StreamForge runtime identity. Do not disable hostname verification as a +production workaround. -### Generating SSL Certificates +## SASL over TLS -```bash -# Generate CA certificate -openssl req -new -x509 -keyout ca-key.pem -out ca-cert.pem -days 365 - -# Generate client key and certificate -openssl req -new -keyout client-key.pem -out client-cert-req.pem -days 365 -openssl x509 -req -in client-cert-req.pem -CA ca-cert.pem -CAkey ca-key.pem \ - -CAcreateserial -out client-cert.pem -days 365 -``` - ---- - -## SASL Authentication - -### SASL/PLAIN - -Simple username/password authentication: - -```yaml -security: - protocol: SASL_SSL # Always use SSL with PLAIN - ssl: - ca_location: /path/to/ca-cert.pem - sasl: - mechanism: PLAIN - username: your-username - password: your-password -``` - -**Pros:** -- โœ… Simple to configure -- โœ… Works with most Kafka brokers - -**Cons:** -- โš ๏ธ Password transmitted in plain text (must use SSL!) -- โš ๏ธ Less secure than SCRAM - -**Use Case:** Development, testing, Confluent Cloud. - -### SASL/SCRAM-SHA-256 - -Secure Challenge-Response Authentication Mechanism: - -```yaml -security: - protocol: SASL_SSL - ssl: - ca_location: /path/to/ca-cert.pem - sasl: - mechanism: SCRAM-SHA-256 - username: your-username - password: your-password -``` - -**Pros:** -- โœ… Password never transmitted over network -- โœ… Mutual authentication -- โœ… Replay attack protection - -**Use Case:** Modern Kafka clusters, AWS MSK, production environments. - -### SASL/SCRAM-SHA-512 - -More secure variant of SCRAM: +SCRAM example: ```yaml security: protocol: SASL_SSL ssl: - ca_location: /path/to/ca-cert.pem - sasl: - mechanism: SCRAM-SHA-512 # Changed from SHA-256 - username: your-username - password: your-password -``` - -**Use Case:** High-security environments requiring stronger hashing. - -### SASL/GSSAPI (Kerberos) - -Enterprise authentication with Kerberos: - -```yaml -security: - protocol: SASL_SSL - ssl: - ca_location: /path/to/ca-cert.pem + ca_location: /run/streamforge/tls/ca.pem + endpoint_identification_algorithm: https sasl: - mechanism: GSSAPI - kerberos_service_name: kafka - kerberos_principal: client@EXAMPLE.COM - kerberos_keytab: /path/to/client.keytab + mechanism: SCRAM-SHA-512 + username: rendered-at-runtime + password: rendered-at-runtime ``` -**Prerequisites:** -1. Install Kerberos libraries: - ```bash - # Ubuntu/Debian - apt-get install libkrb5-dev - - # RHEL/CentOS - yum install krb5-devel - ``` - -2. Configure `/etc/krb5.conf`: - ```ini - [libdefaults] - default_realm = EXAMPLE.COM - - [realms] - EXAMPLE.COM = { - kdc = kdc.example.com - admin_server = admin.example.com - } - ``` - -3. Test Kerberos: - ```bash - kinit -kt /path/to/client.keytab client@EXAMPLE.COM - klist # Verify ticket - ``` +PLAIN transmits credentials inside the TLS session and must not be used without +TLS. GSSAPI requires a compatible librdkafka build and Kerberos environment. +OAUTHBEARER token lifecycle must be tested for the exact client and broker; a +static token in a long-running file is not a rotation strategy. -**Use Case:** Enterprise Hadoop/Kafka clusters, legacy systems. +## Secret injection ---- +StreamForge does not interpolate `${ENVIRONMENT_VARIABLE}` placeholders in +configuration values. A secret manager or entrypoint must render a protected +configuration file before StreamForge starts. -## Cloud Provider Examples +Safe patterns include: -### Confluent Cloud +- mounting a complete secret-bearing configuration from a Kubernetes `Secret`; +- rendering into a memory-backed volume from an approved secret sidecar; +- mounting a protected host file into a container read-only; +- rotating the rendered file and performing a controlled restart. -```yaml -appid: mirrormaker-confluent -bootstrap: pkc-xxxxx.us-east-1.aws.confluent.cloud:9092 -input: source-topic -output: destination-topic +Do not: -security: - protocol: SASL_SSL - sasl: - mechanism: PLAIN - username: - password: -``` +- commit credentials, tokens, private keys, or a rendered configuration; +- put a secret-bearing configuration in a Kubernetes `ConfigMap`; +- print the configuration in CI logs or diagnostics; +- pass passwords on a shell command line; +- use example or default credentials. -**How to get credentials:** -1. Go to Confluent Cloud Console -2. Select your cluster -3. API Keys โ†’ Create Key -4. Copy API Key (username) and Secret (password) +Ensure temporary rendered files are excluded from backups and removed according +to the platform secret-handling policy. -### AWS MSK (Managed Streaming for Kafka) +## Different source and destination credentials -#### Option 1: SASL/SCRAM +The top-level `security` block is applied to both Kafka clients. When the source +and destination require different settings, explicit `consumer_properties` and +`producer_properties` can override the generated librdkafka properties: ```yaml -appid: mirrormaker-msk -bootstrap: b-1.msk-cluster.xxxxx.kafka.us-east-1.amazonaws.com:9096 -input: source-topic -output: destination-topic - security: protocol: SASL_SSL - sasl: - mechanism: SCRAM-SHA-512 - username: - password: -``` - -**How to set up:** -1. Create secret in AWS Secrets Manager -2. Associate secret with MSK cluster -3. Use secret values as username/password - -#### Option 2: IAM Authentication - -```yaml -appid: mirrormaker-msk-iam -bootstrap: b-1.msk-cluster.xxxxx.kafka.us-east-1.amazonaws.com:9098 -input: source-topic -output: destination-topic + ssl: + ca_location: /run/streamforge/tls/ca.pem + endpoint_identification_algorithm: https -# For IAM auth, use custom properties consumer_properties: - security.protocol: SASL_SSL - sasl.mechanism: AWS_MSK_IAM - sasl.jaas.config: software.amazon.msk.auth.iam.IAMLoginModule required; - sasl.client.callback.handler.class: software.amazon.msk.auth.iam.IAMClientCallbackHandler + sasl.mechanism: SCRAM-SHA-512 + sasl.username: rendered-source-user + sasl.password: rendered-source-password producer_properties: - security.protocol: SASL_SSL - sasl.mechanism: AWS_MSK_IAM - sasl.jaas.config: software.amazon.msk.auth.iam.IAMLoginModule required; - sasl.client.callback.handler.class: software.amazon.msk.auth.iam.IAMClientCallbackHandler + sasl.mechanism: SCRAM-SHA-512 + sasl.username: rendered-destination-user + sasl.password: rendered-destination-password ``` -### Azure Event Hubs (Kafka Protocol) +These values are still secrets and require the same protected rendering process. +Validate the full configuration without exposing it in logs. -```yaml -appid: mirrormaker-eventhubs -bootstrap: .servicebus.windows.net:9093 -input: source-topic -output: destination-topic +Do not copy Java callback-handler or JAAS properties into this Rust client. +Provider-specific authentication is supported only when the linked librdkafka +client and StreamForge configuration have been explicitly tested for that +provider. -security: - protocol: SASL_SSL - sasl: - mechanism: PLAIN - username: $ConnectionString - password: Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey= -``` +## Kafka authorization ---- +Grant only the resources used by a pipeline: -## Troubleshooting +- source topic `READ` and metadata access; +- consumer group access for the configured `appid`; +- destination topic `WRITE` and metadata access; +- DLQ topic `WRITE` when enabled; +- any additional permissions required by the broker's authorization model. -### SSL Certificate Issues +Use a separate principal per environment and, where practical, per pipeline. +Avoid wildcard topic and consumer-group grants. -**Problem:** `SSL handshake failed` +## Network controls -**Solutions:** -1. Verify CA certificate path: - ```bash - openssl verify -CAfile ca-cert.pem broker-cert.pem - ``` +- Keep Kafka listeners on private subnets or cluster networks. +- Restrict StreamForge egress to Kafka, DNS, secret services, and required + telemetry. +- Do not create public broker listeners for troubleshooting. +- Keep `/metrics` and `/health` private; they have no authentication or TLS. +- Prefer loopback port forwarding or a private monitoring network for + diagnostics. -2. Check certificate expiration: - ```bash - openssl x509 -in ca-cert.pem -noout -dates - ``` +If temporary remote access is unavoidable, allow only the operator's verified +IP at the network boundary and remove the rule immediately after use. -3. Disable hostname verification (testing only): - ```yaml - ssl: - endpoint_identification_algorithm: "" - ``` +## Containers and Kubernetes -### SASL Authentication Issues +- Run as a non-root identity. +- Drop Linux capabilities and disable privilege escalation. +- Use a read-only root filesystem with an explicit temporary filesystem. +- Mount configuration and key material read-only. +- Use `ClusterIP` services and restrictive `NetworkPolicy`. +- Avoid `NodePort`, public `LoadBalancer`, and internet-facing `Ingress`. +- Review service-account and operator RBAC from rendered manifests. -**Problem:** `Authentication failed` +The current Kubernetes operator mounts referenced secrets but does not emit CR +security fields into its generated runtime configuration. Use a directly +managed Deployment for secured Kafka connections until that path is implemented +and verified. See [Kubernetes](KUBERNETES.md). -**Solutions:** -1. Verify credentials are correct -2. Check SASL mechanism matches broker configuration -3. For SCRAM, ensure user exists on broker: - ```bash - kafka-configs.sh --bootstrap-server kafka:9092 \ - --describe --entity-type users - ``` +## Verification -### Kerberos Issues +Before production: -**Problem:** `GSSAPI authentication failed` +1. validate the rendered configuration without printing it; +2. confirm the process identity can read only the required files; +3. verify broker hostname validation and certificate chain; +4. verify source read, group, destination write, and DLQ permissions separately; +5. confirm an unauthorized topic access is denied; +6. test credential and certificate rotation; +7. confirm metrics and health are unreachable from outside the private network; +8. inspect logs and diagnostic bundles for secret leakage. -**Solutions:** -1. Verify Kerberos ticket: - ```bash - klist -e - ``` +Useful certificate checks: -2. Check keytab: - ```bash - klist -kt client.keytab - ``` - -3. Test kinit: - ```bash - kinit -kt client.keytab client@EXAMPLE.COM - ``` - -4. Check service name matches broker configuration - -### Connection Timeout - -**Problem:** Connection times out - -**Solutions:** -1. Verify broker hostname/port -2. Check firewall rules allow port (9093, 9094, etc.) -3. Verify security group rules (cloud providers) -4. Test with openssl: - ```bash - openssl s_client -connect kafka:9093 - ``` - ---- - -## Best Practices - -### 1. Always Use Encryption - -โœ… **Do:** -```yaml -security: - protocol: SASL_SSL # SSL encryption enabled -``` - -โŒ **Don't:** -```yaml -security: - protocol: SASL_PLAINTEXT # No encryption! -``` - -### 2. Secure Credential Storage - -โœ… **Do:** Use environment variables or secret management ```bash -export KAFKA_USERNAME="myuser" -export KAFKA_PASSWORD="mypass" +openssl x509 -in /run/streamforge/tls/ca.pem -noout -subject -issuer -dates +openssl verify \ + -CAfile /run/streamforge/tls/ca.pem \ + /run/streamforge/tls/client.pem ``` -โŒ **Don't:** Store passwords in configuration files -```yaml -sasl: - password: "plaintext-password-in-git" # Bad! -``` - -### 3. Use Strong Authentication - -**Security Ranking:** -1. ๐Ÿฅ‡ Mutual TLS (mTLS) - Best -2. ๐Ÿฅˆ SASL/SCRAM-SHA-512 - Very Good -3. ๐Ÿฅ‰ SASL/SCRAM-SHA-256 - Good -4. โš ๏ธ SASL/PLAIN - Acceptable with SSL -5. โŒ PLAINTEXT - Never use in production - -### 4. Certificate Management - -- โœ… Rotate certificates regularly (90 days recommended) -- โœ… Use separate certificates for each service -- โœ… Monitor certificate expiration -- โœ… Keep private keys secure (chmod 600) - -### 5. Network Security - -- โœ… Use VPN or private networks for Kafka traffic -- โœ… Restrict broker access with firewall rules -- โœ… Use separate security groups for Kafka -- โœ… Enable VPC peering for cross-account access (AWS) - -### 6. Monitoring - -Monitor these metrics: -- Authentication failures -- SSL handshake errors -- Certificate expiration warnings -- Connection timeouts - -### 7. Testing - -Test security configuration before production: - -```bash -# Test SSL connection -openssl s_client -connect kafka:9093 -CAfile ca-cert.pem - -# Test with kafkacat -kafkacat -b kafka:9093 -L \ - -X security.protocol=SASL_SSL \ - -X sasl.mechanism=SCRAM-SHA-256 \ - -X sasl.username=user \ - -X sasl.password=pass - -# Test with MirrorMaker -CONFIG_FILE=examples/config.security-sasl-scram.yaml cargo run -``` - ---- - -## Configuration Examples - -All security examples are in the `examples/` folder: - -- **[config.security-ssl.yaml](../examples/configs/config.security-ssl.yaml)** - SSL/TLS encryption -- **[config.security-sasl-plain.yaml](../examples/configs/config.security-sasl-plain.yaml)** - SASL/PLAIN authentication -- **[config.security-sasl-scram.yaml](../examples/configs/config.security-sasl-scram.yaml)** - SASL/SCRAM authentication -- **[config.security-kerberos.yaml](../examples/configs/config.security-kerberos.yaml)** - Kerberos authentication - ---- - -## References - -### Official Documentation -- [Kafka Security](https://kafka.apache.org/documentation/#security) -- [librdkafka Configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) -- [Confluent Security](https://docs.confluent.io/platform/current/security/index.html) - -### Cloud Provider Guides -- [AWS MSK Security](https://docs.aws.amazon.com/msk/latest/developerguide/security.html) -- [Azure Event Hubs Kafka](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-for-kafka-ecosystem-overview) -- [Confluent Cloud](https://docs.confluent.io/cloud/current/security/index.html) - ---- - -## Quick Reference - -### Security Configuration Template - -```yaml -security: - # Protocol: PLAINTEXT | SSL | SASL_PLAINTEXT | SASL_SSL - protocol: SASL_SSL - - # SSL Configuration (for SSL or SASL_SSL) - ssl: - ca_location: /path/to/ca-cert.pem - certificate_location: /path/to/client-cert.pem # Optional (mTLS) - key_location: /path/to/client-key.pem # Optional (mTLS) - key_password: key-password # Optional - endpoint_identification_algorithm: https # Optional - - # SASL Configuration (for SASL_PLAINTEXT or SASL_SSL) - sasl: - # Mechanism: PLAIN | SCRAM-SHA-256 | SCRAM-SHA-512 | GSSAPI | OAUTHBEARER - mechanism: SCRAM-SHA-256 - - # For PLAIN/SCRAM - username: your-username - password: your-password - - # For GSSAPI (Kerberos) - kerberos_service_name: kafka - kerberos_principal: client@REALM - kerberos_keytab: /path/to/keytab - - # For OAUTHBEARER - oauthbearer_token: your-token -``` - ---- - -**Need Help?** Open an issue on GitHub. +Continue with [Docker](DOCKER.md), [Kubernetes](KUBERNETES.md), and +[Deployment](DEPLOYMENT.md). diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5d76ef7..dbe84a3 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1,1399 +1,240 @@ -# StreamForge Troubleshooting Guide - -**Version:** 1.0.0 -**Last Updated:** 2026-04-18 - -This guide covers common issues, symptoms, causes, and solutions for StreamForge operations. - ---- - -## Table of Contents - -1. [Quick Diagnosis](#quick-diagnosis) -2. [Startup Issues](#startup-issues) -3. [Performance Issues](#performance-issues) -4. [Data Issues](#data-issues) -5. [Connectivity Issues](#connectivity-issues) -6. [Resource Issues](#resource-issues) -7. [Configuration Issues](#configuration-issues) -8. [Kafka Issues](#kafka-issues) -9. [Debug Commands](#debug-commands) - --- - -## Quick Diagnosis - -### Health Check Commands - -```bash -# 1. Check pod status -kubectl get pods -n streamforge - -# 2. Check logs -kubectl logs -f deployment/streamforge -n streamforge --tail=50 - -# 3. Check metrics -curl http://streamforge:8080/metrics | grep -E "(up|error|lag)" - -# 4. Check consumer group -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group - -# 5. Check resource usage -kubectl top pods -n streamforge -``` - -### Common Symptoms Quick Reference - -| Symptom | Likely Cause | Quick Fix | -|---------|--------------|-----------| -| Pod stuck in `CrashLoopBackOff` | Config error or missing secret | Check logs, validate config | -| Consumer lag growing | Insufficient replicas or CPU | Scale up | -| High error rate | Bad data or config mismatch | Check DLQ headers | -| Zero throughput | Kafka connection failure | Check connectivity | -| High memory usage | Large messages or memory leak | Increase limits, restart | -| Slow processing | Complex filters or transforms | Optimize DSL, add threads | - +title: Troubleshooting +nav_order: 3 +parent: Operations --- -## Startup Issues +# Troubleshooting -### Issue: Pod stuck in `Pending` +Start with evidence from the running revision. Avoid changing offsets, scaling, +or relaxing security controls until the failure mode is clear. -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -streamforge-7c8f9d4b6-abc12 0/1 Pending 0 5m -``` +## Collect a safe diagnostic snapshot -**Diagnosis:** ```bash -kubectl describe pod streamforge-7c8f9d4b6-abc12 -n streamforge -``` - -**Common causes:** - -**1. Insufficient resources:** +kubectl get pods -n streamforge -o wide +kubectl describe pod -n streamforge STREAMFORGE_POD +kubectl logs -n streamforge STREAMFORGE_POD --previous --tail=200 +kubectl top pod -n streamforge STREAMFORGE_POD +kubectl get events -n streamforge --sort-by=.lastTimestamp ``` -Events: - Warning FailedScheduling 5m default-scheduler 0/3 nodes are available: insufficient cpu. -``` -**Solution:** -- Reduce resource requests -- Add more nodes to cluster -- Remove resource limits temporarily ```bash -kubectl patch deployment streamforge -n streamforge --patch ' -spec: - template: - spec: - containers: - - name: streamforge - resources: - requests: - cpu: 500m - memory: 1Gi -' -``` - -**2. ImagePullBackOff:** -``` -Events: - Warning Failed 5m kubelet Failed to pull image "streamforge:1.0.0": rpc error: code = Unknown -``` -**Solution:** -- Check image exists: `docker pull streamforge:1.0.0` -- Check image pull secret: `kubectl get secret -n streamforge` -- Use correct image registry - -**3. PVC not bound:** -``` -Events: - Warning FailedMount 5m kubelet Unable to attach or mount volumes -``` -**Solution:** -- Check PVC status: `kubectl get pvc -n streamforge` -- Create missing PV/PVC -- Remove volume if not needed - -### Issue: Pod stuck in `Init:Error` - -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -streamforge-7c8f9d4b6-abc12 0/1 Init:Error 0 2m +kafka-consumer-groups --bootstrap-server kafka.internal:9092 \ + --describe --group PIPELINE_APPID ``` -**Diagnosis:** ```bash -kubectl logs streamforge-7c8f9d4b6-abc12 -c init-container -n streamforge -kubectl describe pod streamforge-7c8f9d4b6-abc12 -n streamforge +curl --fail --silent http://PRIVATE_STREAMFORGE_ADDRESS:9090/health +curl --fail --silent http://PRIVATE_STREAMFORGE_ADDRESS:9090/metrics ``` -**Common causes:** - -**Init container failed:** -- Check init container logs -- Verify dependencies (e.g., Kafka must be reachable) -- Fix init script +Redact passwords, tokens, certificates, message values, and sensitive headers +before sharing configuration, logs, metrics labels, or DLQ samples. -**Solution:** -Remove init container if not essential, or fix the init logic. +## Process does not start -### Issue: Pod `CrashLoopBackOff` +### Configuration error -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -streamforge-7c8f9d4b6-abc12 0/1 CrashLoopBackOff 5 5m -``` +Validate the same file mounted in the workload: -**Diagnosis:** ```bash -# Check current logs -kubectl logs streamforge-7c8f9d4b6-abc12 -n streamforge - -# Check previous crashed instance -kubectl logs streamforge-7c8f9d4b6-abc12 -n streamforge --previous +target/release/streamforge-validate config.yaml --fail-on-warnings ``` -**Common causes:** - -**1. Config parse error:** -``` -ERROR Failed to parse config: invalid YAML at line 10 -``` -**Solution:** -```bash -# Validate config -streamforge-validate config.yaml +Confirm that `CONFIG_FILE` points to an existing readable `.yaml`, `.yml`, or +`.json` file. A missing path causes the current binary to fall back to a built-in +test configuration, so treat a โ€œconfig file not foundโ€ warning as a deployment +failure. -# Fix ConfigMap -kubectl edit configmap streamforge-config -n streamforge +### Kafka connection or authentication error -# Restart -kubectl rollout restart deployment/streamforge -n streamforge -``` +Verify: -**2. Missing environment variable:** -``` -ERROR Environment variable KAFKA_BOOTSTRAP not set -``` -**Solution:** -```bash -kubectl set env deployment/streamforge KAFKA_BOOTSTRAP=kafka:9092 -n streamforge -``` +- bootstrap hostname and port resolve from the workload; +- egress policy allows the broker and DNS; +- the configured security protocol matches the listener; +- certificate paths exist inside the container; +- the CA and client certificates are current; +- SASL mechanism and credentials match the broker; +- ACLs allow source reads, consumer-group access, and destination writes. -**3. Kafka connection failure:** -``` -ERROR Failed to connect to Kafka broker at kafka:9092: Connection refused -``` -**Solution:** -- Check Kafka is running -- Verify bootstrap servers in config -- Check network policies -- Test connectivity: `kubectl exec -it streamforge-xxx -n streamforge -- ping kafka` +Do not disable hostname verification or expose Kafka publicly to bypass a +connection problem. See [Security](SECURITY_CONFIGURATION.md). -**4. OOMKilled (Out of Memory):** -```bash -kubectl describe pod streamforge-xxx -n streamforge -``` -``` -Last State: Terminated - Reason: OOMKilled - Exit Code: 137 -``` -**Solution:** -```bash -# Increase memory limits -kubectl patch deployment streamforge -n streamforge --patch ' -spec: - template: - spec: - containers: - - name: streamforge - resources: - limits: - memory: 4Gi -' -``` +### Metrics server cannot bind -### Issue: Container exits immediately with code 1 +The server binds the configured port on `0.0.0.0`. Check for a port conflict: -**Diagnosis:** ```bash -kubectl logs streamforge-xxx -n streamforge --previous +kubectl logs -n streamforge STREAMFORGE_POD | rg "metrics|bind" ``` -**Common causes:** +Use a private `ClusterIP` service or local port-forward. The endpoint has no +built-in authentication. -**Invalid filter/transform syntax:** -``` -ERROR Failed to parse filter: /status,==,active,extra-arg -``` -**Solution:** -- Fix DSL syntax -- Use `streamforge-validate` to check config -- Review docs/DSL_SPEC.md for correct syntax +## Kubernetes workload problems -**Permission denied (TLS certs):** -``` -ERROR Failed to read TLS certificate: Permission denied -``` -**Solution:** -```bash -# Check file permissions in secret -kubectl describe secret kafka-tls -n streamforge - -# Ensure securityContext allows reading -kubectl patch deployment streamforge -n streamforge --patch ' -spec: - template: - spec: - securityContext: - fsGroup: 1000 -' -``` +### `Pending` ---- +Use `kubectl describe pod` and events to distinguish insufficient resources, +unbound volumes, scheduling constraints, and missing service accounts. Adjust +only the constraint reported by the scheduler. -## Performance Issues +### `ImagePullBackOff` -### Issue: High Consumer Lag +Verify the exact repository, tag or digest, registry credentials, node +architecture, and image pull policy. The repository contains Dockerfiles and a +local Helm chart; do not assume an image tag has been published. -**Symptoms:** -- Lag > 10000 messages -- Lag growing over time -- Alert: "StreamForgeHighLag" +### `CrashLoopBackOff` -**Diagnosis:** -```bash -# Check lag -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group +Inspect previous logs and the termination reason: -# Check throughput -curl http://streamforge:8080/metrics | grep messages_consumed_total +- exit during startup: configuration, secret mount, or Kafka initialization; +- `OOMKilled`: memory limit or bounded-queue sizing; +- repeated readiness failures: remember that `/health` reports HTTP process + health, not end-to-end Kafka delivery; +- immediate operator-created pod failures: compare the generated `ConfigMap` + with the operator CR and runtime schema. -# Check CPU usage -kubectl top pods -n streamforge -``` - -**Common causes:** - -**1. Insufficient parallelism (too few replicas):** -``` -Partitions: 16 -Replicas: 2 -Result: 14 partitions idle, only 2 being consumed -``` -**Solution:** -```bash -kubectl scale deployment streamforge --replicas=8 -n streamforge -``` - -**2. CPU saturation:** -``` -CPU: 1900m/2000m (95%) -``` -**Solution:** -```bash -# Increase CPU limits -kubectl patch deployment streamforge -n streamforge --patch ' -spec: - template: - spec: - containers: - - name: streamforge - resources: - limits: - cpu: 4000m -' -``` +## Records are consumed but not delivered -**3. Complex filters/transforms:** -``` -Filter: REGEX:/email,.*@[a-z]+\.[a-z]{2,}$ -Transform: CONSTRUCT with 20 fields -``` -**Solution:** -- Simplify filters (use KEY_PREFIX instead of REGEX) -- Move complex logic upstream -- Increase threads: -```yaml -threads: 8 # increase from 4 -``` +1. Compare `streamforge_messages_consumed_total` with + `streamforge_messages_delivered_total`. +2. Check `streamforge_processing_errors_total` by `type`. +3. Check destination topic, ACL, broker availability, and partition metadata. +4. Inspect filter-fail and filtered counters; a filter can intentionally remove + records. +5. Sample the destination with an independent Kafka consumer. +6. If using queued delivery, remember that enqueue completion precedes broker + acknowledgement. -**4. Kafka broker slow:** -``` -Fetch wait time: 500ms (high) -``` -**Solution:** -- Scale Kafka brokers -- Add partitions to topic -- Tune `fetch_max_wait_ms`: -```yaml -performance: - fetch_max_wait_ms: 50 # reduce wait time -``` +For a single-destination pipeline, use broker acknowledgements and destination +Kafka offsets as the authoritative delivery signals. -### Issue: High Processing Latency +## Consumer lag grows -**Symptoms:** -- p95 latency > 100ms (was 20ms) -- Slow end-to-end processing +Break the lag down by partition: -**Diagnosis:** ```bash -curl http://streamforge:8080/metrics | grep processing_duration +kafka-consumer-groups --bootstrap-server kafka.internal:9092 \ + --describe --group PIPELINE_APPID ``` -**Common causes:** +Then check: -**1. Large batch sizes:** -```yaml -performance: - batch_size: 10000 # too large -``` -**Solution:** -```yaml -performance: - batch_size: 500 # smaller for lower latency - linger_ms: 0 # send immediately -``` +- whether every partition has an active owner; +- source key skew and hot partitions; +- CPU throttling and memory pressure; +- processing duration and error rate; +- destination broker latency and retry activity; +- batch, worker queue, and producer in-flight bounds; +- repeated group rebalances. -**2. Slow producer (destination Kafka slow):** -``` -ERROR Producer timeout after 30000ms -``` -**Solution:** -- Check destination Kafka health -- Increase producer timeout -- Use async producer (default) +More replicas help only while unassigned source partitions remain. Change one +tuning value at a time and confirm that lag begins to recover. -**3. Commit overhead:** -```yaml -commit_strategy: "per-message" # high overhead -``` -**Solution:** -```yaml -commit_strategy: "time-based" -commit_interval_ms: 1000 # commit every 1 second -``` - -### Issue: Low Throughput - -**Symptoms:** -- Throughput < 10K msg/s (expected 50K msg/s) -- CPU usage low (< 30%) - -**Diagnosis:** -```bash -# Check threading -kubectl logs deployment/streamforge -n streamforge | grep "threads" - -# Check batch sizes -kubectl logs deployment/streamforge -n streamforge | grep "batch" -``` +## Duplicate records -**Common causes:** +Duplicates are expected in an at-least-once profile when delivery succeeds but +the source offset is not committed before a crash or rebalance. -**1. Too few threads:** -```yaml -threads: 1 # only using 1 CPU core -``` -**Solution:** -```yaml -threads: 8 # match CPU cores -``` +Check for: -**2. Small batches:** -```yaml -performance: - fetch_min_bytes: 1 # wait for 1 byte - batch_size: 10 # small producer batches -``` -**Solution:** -```yaml -performance: - fetch_min_bytes: 10240 # 10 KB - batch_size: 5000 # large batches - linger_ms: 50 # allow batching -``` - -**3. Too many commits:** -```yaml -commit_strategy: "per-message" -``` -**Solution:** -```yaml -commit_strategy: "manual" -commit_interval_ms: 5000 # commit every 5 seconds -``` - -**4. Compression overhead:** -```yaml -performance: - compression: "gzip" # slow -``` -**Solution:** -```yaml -performance: - compression: "zstd" # faster - # or - compression: "none" # no compression overhead -``` - ---- +- restarts between destination delivery and offset commit; +- commit failures; +- consumer-group rebalances; +- manual offset resets; +- producer retries after an ambiguous acknowledgement; +- multiple pipelines writing the same destination. -## Data Issues +Use a stable event identifier and make downstream processing idempotent. See +[Delivery guarantees](DELIVERY_GUARANTEES.md). -### Issue: Messages going to DLQ +## Records enter the DLQ -**Symptoms:** -- DLQ accumulating messages -- Error rate > 1/s -- Alert: "StreamForgeHighDLQRate" +Read only enough metadata to classify the problem: -**Diagnosis:** ```bash -# Sample DLQ messages -kafka-console-consumer --bootstrap-server kafka:9092 \ +kafka-console-consumer --bootstrap-server kafka.internal:9092 \ --topic streamforge-dlq \ --property print.headers=true \ - --max-messages 10 - -# Check error types -kubectl logs deployment/streamforge -n streamforge | grep "Sending to DLQ" -``` - -**Common error types:** - -**1. FilterEvaluation error:** -``` -Headers: - x-streamforge-error-type: FilterEvaluation - x-streamforge-filter: /status,==,active - x-streamforge-source-topic: users -``` - -**Cause:** Message missing `/status` field or field is not a string. - -**Solution:** -```yaml -# Make filter more lenient -filter: "OR:/status,==,active:/status,==,null" - -# Or skip messages without field -filter: "AND:EXISTS:/status:/status,==,active" -``` - -**2. TransformError:** -``` -Headers: - x-streamforge-error-type: TransformError - x-streamforge-transform: /user/nonexistent -``` - -**Cause:** Transform path does not exist in message. - -**Solution:** -```yaml -# Use default value -transform: "EXTRACT:/user/id,user-id,default-value" - -# Or use CONSTRUCT with fallback -transform: "CONSTRUCT:id=/user/id:name=/user/name:fallback=unknown" -``` - -**3. SerializationError:** -``` -Headers: - x-streamforge-error-type: SerializationError -``` - -**Cause:** Transform produced invalid JSON. - -**Solution:** -- Review transform logic -- Validate transform output -- Use simpler transform (e.g., /data instead of CONSTRUCT) - -**4. ProducerTimeout:** -``` -Headers: - x-streamforge-error-type: RetryExhausted - x-streamforge-retry-attempts: 3 -``` - -**Cause:** Destination Kafka slow or unavailable, retries exhausted. - -**Solution:** -- Check destination Kafka health -- Increase retry attempts: -```yaml -retry: - max_attempts: 5 - max_delay_ms: 60000 -``` - -### Issue: Missing Messages (Data Loss) - -**Symptoms:** -- Messages consumed but not produced -- No DLQ entries -- No errors logged - -**Diagnosis:** -```bash -# Check filter logic -kubectl logs deployment/streamforge -n streamforge | grep "filtered out" - -# Check transform logic -kubectl logs deployment/streamforge -n streamforge | grep "transform result: null" - -# Compare consume vs produce counts -curl http://streamforge:8080/metrics | grep messages_consumed_total -curl http://streamforge:8080/metrics | grep messages_produced_total -``` - -**Common causes:** - -**1. Overly restrictive filter:** -```yaml -filter: "/status,==,active" -# If most messages have status != "active", they're filtered out -``` - -**Solution:** -- Review filter logic -- Check sample messages to verify filter correctness -- Add logging to see filtered messages: -```yaml -# In dev/staging, enable debug logging -env: -- name: RUST_LOG - value: "streamforge=debug" -``` - -**2. Transform returns null:** -```yaml -transform: "/user/optional-field" -# If field doesn't exist, transform returns null, message skipped -``` - -**Solution:** -```yaml -transform: "EXTRACT:/user/optional-field,field,default-value" -``` - -**3. Partition mismatch:** -```yaml -partitioning: "field:/user/region" -# If /user/region doesn't exist, message sent to partition -1 (error) -``` - -**Solution:** -- Use default partitioning -- Or ensure partition key field always exists - -### Issue: Duplicate Messages - -**Symptoms:** -- Same message ID appears multiple times in destination -- At-least-once delivery expected but duplicates excessive - -**Diagnosis:** -```bash -# Check consumer group stability -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group - -# Check for rebalances -kubectl logs deployment/streamforge -n streamforge | grep "rebalance" -``` - -**Common causes:** - -**1. Consumer group rebalancing:** -- Pod restarts trigger rebalance -- New replicas trigger rebalance -- Partitions redistributed, some messages re-consumed - -**Solution:** -- Reduce pod churn (avoid frequent restarts) -- Use stable replica count -- Commit more frequently: -```yaml -commit_strategy: "time-based" -commit_interval_ms: 1000 # commit every 1 second -``` - -**2. Producer retries:** -- Producer sends message -- Kafka acknowledges -- Acknowledgment lost (network blip) -- Producer retries, message duplicated - -**Solution:** -- This is expected with at-least-once semantics -- Use idempotent producer (enabled by default in rdkafka) -- Implement deduplication downstream (use message ID) - -**3. Manual offset reset:** -- Offsets reset to earlier position -- Messages re-consumed - -**Solution:** -- Avoid manual offset resets -- If needed, reset to specific timestamp, not "earliest" - ---- - -## Connectivity Issues - -### Issue: Cannot connect to Kafka - -**Symptoms:** -``` -ERROR Failed to connect to Kafka broker: Connection refused -``` - -**Diagnosis:** -```bash -# Test connectivity from pod -kubectl exec -it streamforge-xxx -n streamforge -- nc -zv kafka 9092 - -# Check DNS resolution -kubectl exec -it streamforge-xxx -n streamforge -- nslookup kafka - -# Check network policies -kubectl get networkpolicy -n streamforge -``` - -**Common causes:** - -**1. Wrong bootstrap servers:** -```yaml -bootstrap: "kafka:9092" # but Kafka is at kafka.kafka.svc:9092 + --max-messages 1 ``` -**Solution:** -```yaml -bootstrap: "kafka.kafka.svc.cluster.local:9092" -``` +Common categories: -**2. Network policy blocking traffic:** -```bash -kubectl describe networkpolicy -n streamforge -``` +- JSON parse failure: verify the producer contract and tombstone handling; +- filter or transform failure: validate the expression against a representative + payload; +- producer failure: inspect destination connectivity, ACLs, and broker health; +- DLQ delivery failure: restore the DLQ topic or broker before restarting a + manual-commit pipeline. -**Solution:** -- Add egress rule for Kafka: -```yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: streamforge-netpol - namespace: streamforge -spec: - podSelector: - matchLabels: - app: streamforge - policyTypes: - - Egress - egress: - - to: - - namespaceSelector: - matchLabels: - name: kafka - ports: - - protocol: TCP - port: 9092 -``` +Do not purge or replay the DLQ as a diagnostic step. Correct the cause, test the +replay into an isolated destination, then execute an approved replay plan. -**3. Kafka not running:** -```bash -kubectl get pods -n kafka -``` +## Memory pressure -**Solution:** -- Start Kafka cluster -- Wait for Kafka to be ready +Inspect the container limit and workload shape. Memory scales with message size, +batch size, processing concurrency, per-worker queue capacity, destination +fan-out, and queued producer depth. -**4. TLS certificate error:** -``` -ERROR SSL handshake failed: certificate verify failed -``` - -**Solution:** -- Check TLS config: -```yaml -kafka: - ssl: - ca_location: "/certs/ca.crt" # must exist -``` -- Verify secret mounted: -```bash -kubectl exec -it streamforge-xxx -n streamforge -- ls -la /certs -``` -- Check certificate validity: -```bash -kubectl exec -it streamforge-xxx -n streamforge -- openssl x509 -in /certs/ca.crt -noout -dates -``` - -### Issue: SASL authentication failure - -**Symptoms:** -``` -ERROR SASL authentication failed: Invalid credentials -``` - -**Diagnosis:** -```bash -# Check SASL config -kubectl get configmap streamforge-config -n streamforge -o yaml - -# Check credentials -kubectl get secret kafka-credentials -n streamforge -o yaml -``` - -**Common causes:** - -**1. Wrong SASL mechanism:** -```yaml -kafka: - security: - sasl_mechanism: "PLAIN" # but Kafka uses SCRAM-SHA-512 -``` - -**Solution:** -```yaml -kafka: - security: - sasl_mechanism: "SCRAM-SHA-512" -``` - -**2. Incorrect username/password:** -```yaml -sasl_username: "${KAFKA_USER}" # env var not set -``` - -**Solution:** -```bash -kubectl set env deployment/streamforge KAFKA_USER=myuser KAFKA_PASSWORD=mypass -n streamforge -``` - -**3. Secret not mounted:** -```bash -kubectl exec -it streamforge-xxx -n streamforge -- env | grep KAFKA -``` +Reduce bounded concurrency controls before increasing them: -**Solution:** -```yaml -spec: - containers: - - name: streamforge - envFrom: - - secretRef: - name: kafka-credentials -``` - ---- - -## Resource Issues - -### Issue: Out of Memory (OOMKilled) - -**Symptoms:** -``` -Last State: Terminated - Reason: OOMKilled - Exit Code: 137 -``` - -**Diagnosis:** -```bash -kubectl describe pod streamforge-xxx -n streamforge -kubectl top pod streamforge-xxx -n streamforge -``` - -**Common causes:** - -**1. Memory limit too low:** -```yaml -resources: - limits: - memory: 512Mi # too small -``` - -**Solution:** -```yaml -resources: - limits: - memory: 4Gi -``` - -**2. Large messages:** -``` -Average message size: 10 MB -Batch size: 1000 -Total: 10 GB in memory -``` - -**Solution:** ```yaml performance: - batch_size: 100 # reduce batch size - fetch_max_bytes: 10485760 # 10 MB limit -``` - -**3. Memory leak (rare):** -- Memory usage grows over time -- Not correlated with load - -**Solution:** -- Restart pods periodically -- Report issue to StreamForge GitHub - -### Issue: CPU Throttling - -**Symptoms:** -- CPU usage at limit (100%) -- Slow processing despite high CPU request - -**Diagnosis:** -```bash -kubectl top pods -n streamforge - -# Check throttling -kubectl exec -it streamforge-xxx -n streamforge -- cat /sys/fs/cgroup/cpu/cpu.stat -``` - -**Common causes:** - -**1. CPU limit too low:** -```yaml -resources: - limits: - cpu: 1000m # 1 core, but workload needs 4 -``` - -**Solution:** -```yaml -resources: - limits: - cpu: 4000m -``` - -**2. Set requests == limits (guaranteed QoS):** -```yaml -resources: - requests: - cpu: 2000m - limits: - cpu: 4000m # can throttle -``` - -**Solution:** -```yaml -resources: - requests: - cpu: 2000m - limits: - cpu: 2000m # guaranteed, no throttling -``` - -### Issue: Disk Space Full - -**Symptoms:** -``` -ERROR Failed to write log: No space left on device -``` - -**Diagnosis:** -```bash -kubectl exec -it streamforge-xxx -n streamforge -- df -h -``` - -**Common causes:** - -**1. Excessive logging:** -```yaml -env: -- name: RUST_LOG - value: "debug" # too verbose -``` - -**Solution:** -```yaml -env: -- name: RUST_LOG - value: "info" -``` - -**2. DLQ messages accumulating locally (if local DLQ):** - -**Solution:** -- Send DLQ to Kafka topic (default) -- Increase volume size - -**3. Persistent volume full:** -```bash -kubectl get pvc -n streamforge -``` - -**Solution:** -- Increase PVC size (if storage class supports expansion) -- Clean up old data - ---- - -## Configuration Issues - -### Issue: Invalid DSL Syntax - -**Symptoms:** -``` -ERROR Failed to parse filter: unexpected token at position 10 -``` - -**Diagnosis:** -```bash -streamforge-validate config.yaml -``` - -**Common syntax errors:** - -**1. Missing colon:** -```yaml -filter: "AND/status,==,active/age,>,18" # wrong -filter: "AND:/status,==,active:/age,>,18" # correct -``` - -**2. Unescaped special characters:** -```yaml -filter: 'REGEX:/email,.*@.*\.com' # wrong (. not escaped) -filter: 'REGEX:/email,.*@.*\\.com' # correct -``` - -**3. Wrong operator:** -```yaml -filter: "/age,>=,18" # wrong (>= not supported) -filter: "/age,>,17" # correct (use > instead) -``` - -**4. Mismatched quotes:** -```yaml -filter: "REGEX:/name,^(John|Jane)" # wrong (unclosed parenthesis) -filter: 'REGEX:/name,^(John|Jane)$' # correct -``` - -**Solution:** -- Use `streamforge-validate` before deploying -- Review docs/DSL_SPEC.md for syntax -- Test config locally first - -### Issue: Deprecated Syntax Warning - -**Symptoms:** -``` -WARNING Deprecated syntax: KEY_SUFFIX is deprecated, use KEY_MATCHES instead -``` - -**Diagnosis:** -```bash -streamforge-validate config.yaml -``` - -**Solution:** -```yaml -# Old (deprecated) -filter: "KEY_SUFFIX:-prod" - -# New -filter: 'KEY_MATCHES:.*-prod$' -``` - -**Migration guide:** docs/DSL_SPEC.md (Backward Compatibility section) - -### Issue: Config not reloading - -**Symptoms:** -- Updated ConfigMap -- Pods still using old config - -**Diagnosis:** -```bash -kubectl get configmap streamforge-config -n streamforge -o yaml -kubectl exec -it streamforge-xxx -n streamforge -- cat /app/config.yaml -``` - -**Causes:** - -**1. ConfigMap not propagated:** -- Kubernetes propagates ConfigMap updates eventually (up to 60 seconds) - -**Solution:** -```bash -# Force restart -kubectl rollout restart deployment/streamforge -n streamforge -``` - -**2. Hot-reload not enabled:** -- StreamForge requires restart for config changes - -**Solution:** -- Always restart after ConfigMap update - ---- - -## Kafka Issues - -### Issue: Consumer group lag not decreasing - -**Symptoms:** -- StreamForge running, no errors -- Lag stays at 10000, not decreasing - -**Diagnosis:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` - -**Common causes:** - -**1. More replicas than partitions:** -``` -Partitions: 4 -Replicas: 8 -Result: 4 replicas consume, 4 are idle -``` - -**Solution:** -- Scale replicas to match partitions: `kubectl scale deployment streamforge --replicas=4` -- Or add more partitions: `kafka-topics --alter --partitions 8` - -**2. Consumer group rebalancing:** -``` -Consumer rebalancing... -``` -**Solution:** -- Wait for rebalance to complete (30-60 seconds) -- Reduce pod churn - -**3. Kafka brokers overloaded:** -``` -Fetch latency: 5000ms -``` -**Solution:** -- Scale Kafka brokers -- Tune Kafka performance - -### Issue: Topic does not exist - -**Symptoms:** -``` -ERROR Topic 'nonexistent-topic' does not exist -``` - -**Diagnosis:** -```bash -kafka-topics --bootstrap-server kafka:9092 --list -``` - -**Solution:** - -**Option 1: Create topic** -```bash -kafka-topics --bootstrap-server kafka:9092 \ - --create --topic output-topic \ - --partitions 16 \ - --replication-factor 3 -``` - -**Option 2: Enable auto-create** -```yaml -# Kafka broker config -auto.create.topics.enable=true -``` - -**Option 3: Fix topic name in config** -```yaml -routing: - destinations: - - output: "output-topic" # ensure spelling is correct -``` - -### Issue: Partition count mismatch - -**Symptoms:** -- Some partitions have high lag -- Others have zero lag -- Unbalanced consumption - -**Diagnosis:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group -``` - -**Cause:** -- Producer uses key-based partitioning -- Keys are skewed (e.g., 80% have key "default") -- Most messages go to one partition - -**Solution:** - -**Option 1: Use random partitioning** -```yaml -partitioning: "random" -``` - -**Option 2: Use field-based partitioning with uniform distribution** -```yaml -partitioning: "field:/user/id" # if user IDs are uniformly distributed -``` - -**Option 3: Add more partitions** -```bash -kafka-topics --bootstrap-server kafka:9092 \ - --alter --topic source-topic \ - --partitions 32 -``` - ---- - -## Debug Commands - -### Enable Debug Logging - -**Temporarily (current pod):** -```bash -kubectl exec -it streamforge-xxx -n streamforge -- kill -USR1 1 -# Toggles debug logging for duration of pod lifetime -``` - -**Permanently (all pods):** -```bash -kubectl set env deployment/streamforge RUST_LOG=streamforge=debug -n streamforge -``` - -**Restore info logging:** -```bash -kubectl set env deployment/streamforge RUST_LOG=streamforge=info -n streamforge -``` - -### Inspect Message Contents - -**Sample source topic:** -```bash -kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic source-topic \ - --property print.key=true \ - --property print.headers=true \ - --property print.timestamp=true \ - --max-messages 10 -``` - -**Sample destination topic:** -```bash -kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic dest-topic \ - --property print.key=true \ - --max-messages 10 -``` - -**Sample DLQ:** -```bash -kafka-console-consumer --bootstrap-server kafka:9092 \ - --topic streamforge-dlq \ - --property print.headers=true \ - --max-messages 10 -``` - -### Profile Performance - -**CPU profiling:** -```bash -kubectl exec -it streamforge-xxx -n streamforge -- kill -SIGUSR2 1 -# Outputs CPU profile to /tmp/cpu-profile.txt -kubectl cp streamforge-xxx:/tmp/cpu-profile.txt ./cpu-profile.txt -n streamforge -``` - -**Memory profiling:** -```bash -kubectl exec -it streamforge-xxx -n streamforge -- cat /proc/$(pgrep streamforge)/status + consumer_batch_size: 50 + parallelism_factor: 2 + worker_queue_capacity: 256 + producer_max_in_flight: 1000 ``` -### Test Filters/Transforms Locally +These are diagnostic examples, not universal production values. Re-test latency, +lag, and delivery behavior after each change. -**Test config:** -```bash -# Use dry-run mode (if available) -docker run --rm \ - -v $(pwd)/config.yaml:/app/config.yaml:ro \ - streamforge:1.0.0 \ - --config /app/config.yaml \ - --dry-run -``` - -**Validate config:** -```bash -streamforge-validate config.yaml --verbose -``` - -### Force Consumer Rebalance - -**Restart single pod:** -```bash -kubectl delete pod streamforge-xxx -n streamforge -``` +## CPU is high or throughput regresses -**Restart all pods:** -```bash -kubectl rollout restart deployment/streamforge -n streamforge -``` +Profile the target workload before choosing an optimization. Check: -**Force rebalance by changing group ID:** -```yaml -appid: "streamforge-prod-v2" # new group ID -offset: "latest" # start from latest to avoid reprocessing -``` +- JSON payload size and parsing cost; +- regex and compound filters; +- transforms and destination fan-out; +- compression; +- Kafka wait time; +- CPU throttling; +- changes in broker, network, image, or configuration. -### Check Kafka Broker Health +Use [Performance](PERFORMANCE.md) to create a controlled comparison. Do not use +an old headline throughput number as an expected value. -**Broker API versions:** -```bash -kafka-broker-api-versions --bootstrap-server kafka:9092 -``` +## Offset recovery -**Topic metadata:** -```bash -kafka-topics --bootstrap-server kafka:9092 \ - --describe --topic source-topic -``` +Before changing offsets: -**Consumer group state:** -```bash -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group \ - --state -``` +1. stop all consumers in the group; +2. record current offsets and log-end offsets; +3. identify the exact topic and partitions; +4. preview the proposed reset; +5. document whether records will replay or be skipped; +6. obtain approval and execute once; +7. restart and verify destination results. -### Capture Metrics Snapshot +An offset reset is a data operation, not a routine restart procedure. -**Export all metrics:** -```bash -curl http://streamforge:8080/metrics > metrics-$(date +%s).txt -``` +## Escalation bundle -**Query specific metrics:** -```bash -curl -s http://streamforge:8080/metrics | grep -E "(lag|error|duration)" -``` +Provide: ---- - -## Getting Help - -### Check Documentation - -- [DSL Specification](DSL_SPEC.md) - Filter/transform syntax -- [Deployment Guide](DEPLOYMENT.md) - Deployment options -- [Operations Guide](OPERATIONS.md) - Day-to-day operations -- [Architecture](ARCHITECTURE.md) - System design - -### Enable Verbose Logging - -```yaml -env: -- name: RUST_LOG - value: "streamforge=debug,rdkafka=info" -- name: RUST_BACKTRACE - value: "full" -``` - -### Collect Diagnostic Bundle - -```bash -#!/bin/bash -# collect-diagnostics.sh - -mkdir -p diagnostics/$(date +%Y-%m-%d) -cd diagnostics/$(date +%Y-%m-%d) - -# Pod status -kubectl get pods -n streamforge -o wide > pods.txt - -# Logs -kubectl logs deployment/streamforge -n streamforge --tail=1000 > logs.txt - -# Config -kubectl get configmap streamforge-config -n streamforge -o yaml > config.yaml - -# Metrics -curl http://streamforge:8080/metrics > metrics.txt - -# Consumer group -kafka-consumer-groups --bootstrap-server kafka:9092 \ - --describe --group > consumer-group.txt - -# Events -kubectl get events -n streamforge --sort-by='.lastTimestamp' > events.txt - -# Resource usage -kubectl top pods -n streamforge > resources.txt - -echo "Diagnostics collected in diagnostics/$(date +%Y-%m-%d)/" -``` - -### Report Issues - -**GitHub Issues:** https://github.com/rahulbsw/streamforge/issues - -**Include:** -- StreamForge version -- Kubernetes version -- Kafka version -- Config file (redact sensitive data) -- Logs (last 100 lines) -- Error messages -- Steps to reproduce - ---- - -## Issue Decision Tree - -``` -Is StreamForge running? - โ”œโ”€ No โ†’ Check startup issues - โ”‚ โ””โ”€ CrashLoopBackOff? โ†’ Check logs for config errors - โ”‚ โ””โ”€ Pending? โ†’ Check resource availability - โ”‚ โ””โ”€ ImagePullBackOff? โ†’ Check image registry - โ”‚ - โ””โ”€ Yes โ†’ Check metrics - โ”œโ”€ High lag? โ†’ Check performance issues - โ”‚ โ””โ”€ CPU high? โ†’ Scale up or add threads - โ”‚ โ””โ”€ CPU low? โ†’ Increase batch sizes - โ”‚ - โ”œโ”€ High errors? โ†’ Check DLQ headers - โ”‚ โ””โ”€ FilterEvaluation? โ†’ Fix filter logic - โ”‚ โ””โ”€ ProducerTimeout? โ†’ Check destination Kafka - โ”‚ - โ”œโ”€ Zero throughput? โ†’ Check connectivity - โ”‚ โ””โ”€ Kafka connection error? โ†’ Check network - โ”‚ โ””โ”€ SASL error? โ†’ Check credentials - โ”‚ - โ””โ”€ Duplicates? โ†’ Check commit strategy - โ””โ”€ Frequent rebalances? โ†’ Reduce pod churn - โ””โ”€ Manual offset reset? โ†’ Avoid resets -``` - ---- +- StreamForge commit, image tag, and digest; +- redacted configuration and validation output; +- deployment revision and recent changes; +- source and destination Kafka versions and topology; +- pod status, termination reason, and relevant logs; +- consumer-group assignment and lag by partition; +- a bounded metrics snapshot; +- exact reproduction steps and timestamps. -**Document Version:** 1.0.0 -**Last Updated:** 2026-04-18 -**Feedback:** https://github.com/rahulbsw/streamforge/issues +Open a GitHub issue only after removing credentials and payload data. Link to the +repository through the โ€œStreamForge on GitHubโ€ navigation item. diff --git a/docs/UI_MINIKUBE_DEMO.md b/docs/UI_MINIKUBE_DEMO.md index 7f8b25d..03fa8d3 100644 --- a/docs/UI_MINIKUBE_DEMO.md +++ b/docs/UI_MINIKUBE_DEMO.md @@ -1,55 +1,50 @@ --- -title: UI Demo on Minikube +title: UI preview nav_order: 3 --- -# UI Demo on Minikube +# UI preview on Minikube -This walkthrough shows the public-facing StreamForge UI flow on a local Minikube cluster: +This archived two-minute recording shows the StreamForge operator and UI +running on a local Minikube cluster. It covers Helm installation, pipeline +authoring, generated YAML review, and CRD deployment. -1. Install the operator and UI with Helm -2. Open the UI and sign in -3. Create a pipeline in form mode -4. Show the generated YAML before deployment -5. Deploy the CRD to Kubernetes -6. Produce a sample event to Kafka -7. Consume the transformed event from the analytics topic +{: .important } +The recording is a UI preview, not current end-to-end proof of a transformed +destination. Its UI-created mirror destination and separately verified +analytics destination do not represent one continuous pipeline. Use the +[local quickstart](QUICKSTART.md) for the currently reproducible data-path +demonstration. -