qa-txpool-performance-test.yml

name: QA - TxPool performance test

on:
  workflow_call:
  workflow_dispatch:
  push:
    branches:
      - "release/3.*"
  schedule:
    # Every Sunday at 00:00 UTC
    - cron: "0 0 * * 0"

jobs:
  tx_pool_assertoor_test:
    runs-on: [self-hosted, qa, X64, long-running]
    env:
      ERIGON_QA_PATH: /home/qarunner/erigon-qa
      ENCLAVE_NAME: "kurtosis-run-${{ github.run_id }}"

    steps:
      - name: Fast checkout git repository
        uses: actions/checkout@v6

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_USERNAME }}
          password: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}

      - name: Docker build current branch
        run: |
          docker build -t test/erigon:current .

      - name: Run self hosted Kurtosis + assertoor tests
        id: erigon_kurtosis_test
        uses: erigontech/kurtosis-assertoor-github-action@v1.1.7
        with:
          enclave_name: ${{ env.ENCLAVE_NAME }}
          ethereum_package_args: ".github/workflows/kurtosis/txpool-assertoor.io"
          kurtosis_extra_args: --verbosity detailed --cli-log-level trace
          persistent_logs: "true"
          clean_docker: "true"

      - name: Parse Kurtosis output log and create JSON
        if: always()
        id: parse_kurtosis_output
        run: |
          # Find the folder starting with 'assertoor--'
          assertoor_folder=$(find ${{ runner.temp }}/${{ env.ENCLAVE_NAME }}/dump -type d -name 'assertoor--*' | head -n 1)

          # Exit if no assertoor folder is found
          if [ -z "$assertoor_folder" ]; then
            echo "No directory starting with 'assertoor--' found. Skipping."
            exit 0
          fi

          # Parse the output.log file within the assertoor-- folder
          output_log="${assertoor_folder}/output.log"
          echo "Parsing file: $output_log"

          # Create an empty JSON file if there are no results
          output_json="${{ github.workspace }}/outputs_kurtosis.json"
          echo "{}" > $output_json

          # Create a temporary directory
          tmp_dir=$(mktemp -d)

          # Parse the output.log file to find lines with msg="outputs_json"
          while IFS= read -r line; do
            if [[ "$line" == *'msg="outputs_json:'* ]]; then
              # Extract the JSON object and unescape the quotes
              json_str=$(echo "$line" | sed 's/.*msg="outputs_json: \(.*\)".*/\1/' | sed 's/\\"/"/g')
              
              # Extract the task value from the log line
              task=$(echo "$line" | sed -n 's/.*task=\([^ ]*\).*/\1/p')
              
              # Skip if task is empty
              if [ -n "$task" ]; then
                # Create or append to task file
                if [ -f "$tmp_dir/$task" ]; then
                  echo "," >> "$tmp_dir/$task"
                fi
                echo "$json_str" >> "$tmp_dir/$task"
              fi
            fi
          done < "$output_log"

          # Write the JSON object to the final file
          echo "{" > $output_json
          first_task=true
          for task_file in "$tmp_dir"/*; do
            [ -f "$task_file" ] || continue
            
            task=$(basename "$task_file")
            
            # Add comma for all but first task
            if [ "$first_task" = true ]; then
              first_task=false
            else
              echo "," >> $output_json
            fi
            
            # Write the task array to the file
            echo "\"$task\": [$(cat "$task_file")]" >> $output_json
          done
          echo "}" >> $output_json

          echo "outputs_kurtosis.json: $output_json"

          echo "success=true" >> $GITHUB_OUTPUT

          # Clean up
          rm -rf "$tmp_dir"
          echo "cleaned up"

      - name: Upload outputs_kurtosis.json as artifact
        if: always() && steps.parse_kurtosis_output.outputs.success == 'true'
        id: upload_outputs_kurtosis_json
        uses: actions/upload-artifact@v6
        with:
          name: outputs_kurtosis.json
          path: outputs_kurtosis.json

      - name: Create HDR plots PNG artifacts
        id: create_hdr_plots
        if: always() && steps.parse_kurtosis_output.outputs.success == 'true'
        run: |
          # Check if the JSON file exists
          output_json="${{ github.workspace }}/outputs_kurtosis.json"
          if [ ! -f "$output_json" ]; then
            echo "JSON file not found, skipping"
            exit 0
          fi

          # Check if tx_pool_latency_analysis array exists in the JSON
          if ! jq -e '.tx_pool_latency_analysis' "$output_json" > /dev/null 2>&1; then
            echo "tx_pool_latency_analysis array not found in JSON, skipping"
            exit 0
          fi

          # Get the array length
          array_length=$(jq '.tx_pool_latency_analysis | length' "$output_json")

          # Loop through each element in the array
          for ((i=0; i<array_length; i++)); do
            # Get tx_count for this element
            tx_count=$(jq -r ".tx_pool_latency_analysis[$i].tx_count" "$output_json")
            
            # Check if tx_pool_latency_hdr_plot exists for this element
            if jq -e ".tx_pool_latency_analysis[$i].tx_pool_latency_hdr_plot" "$output_json" > /dev/null 2>&1; then
              # Extract the plot data to a CSV file
              csv_file="tx_pool_latency_analysis_${i}_${tx_count}.csv"
              
              # Extract the plot data, replace literal '\n' with actual newlines and '\t' with actual tabs,
              # and remove all lines starting with '#'
              jq -r ".tx_pool_latency_analysis[$i].tx_pool_latency_hdr_plot" "$output_json" | \
                sed 's/\\n/\n/g; s/\\t/\t/g' | grep -v "^#" > "$csv_file"
              
              # Call the Python script to process the CSV file
              python3 $ERIGON_QA_PATH/test_system/qa-tests/tx-pool/hdr_plot.py \
                --input_file "$csv_file" \
                --output_file "tx_pool_latency_analysis_${i}_${tx_count}.png"
              
              # Delete the CSV file after processing
              rm -f "$csv_file"
              
              echo "Processed tx_pool_latency_analysis[$i] with tx_count=$tx_count"
            else
              echo "tx_pool_latency_hdr_plot not found for element $i, skipping"
            fi
          done
          # Set an output to indicate we processed at least one element
          echo "processed=true" >> $GITHUB_OUTPUT

      - name: Create throughput plots PNG artifacts
        id: create_throughput_plots
        if: always() && steps.parse_kurtosis_output.outputs.success == 'true'
        run: |
          # Check if the JSON file exists
          output_json="${{ github.workspace }}/outputs_kurtosis.json"
          if [ ! -f "$output_json" ]; then
            echo "JSON file not found, skipping"
            exit 0
          fi

          # Check if tx_pool_throughput_analysis array exists in the JSON
          if ! jq -e '.tx_pool_throughput_analysis' "$output_json" > /dev/null 2>&1; then
            echo "tx_pool_throughput_analysis array not found in JSON, skipping"
            exit 0
          fi

          # Get the array length
          array_length=$(jq '.tx_pool_throughput_analysis | length' "$output_json")

          # Loop through each element in the array
          for ((i=0; i<array_length; i++)); do
            # Check if throughput_measures exists for this element
            if jq -e ".tx_pool_throughput_analysis[$i].throughput_measures" "$output_json" > /dev/null 2>&1; then
              # Extract the throughput measures data to a CSV file
              csv_file="tx_pool_throughput_analysis_${i}.csv"

              # Extract the throughput_measures array and convert to CSV format
              # Each measure contains load_tps, processed_tps, not_received_p2p_event_rate and coordinated_omission_event_rate
              echo "load_tps,processed_tps,not_received_p2p_event_rate,coordinated_omission_event_rate" > "$csv_file"
              jq -r ".tx_pool_throughput_analysis[$i].throughput_measures[] | [.load_tps, .processed_tps, .not_received_p2p_event_rate, .coordinated_omission_event_rate] | @csv" "$output_json" >> "$csv_file"

              # Call the Python script to process the CSV file
              python3 $ERIGON_QA_PATH/test_system/qa-tests/tx-pool/throughput_plot.py \
                --input_file "$csv_file" \
                --output_file "tx_pool_throughput_analysis_${i}.png"

              # Delete the CSV file after processing
              rm -f "$csv_file"

              echo "Processed tx_pool_throughput_analysis[$i]"
            else
              echo "throughput_measures not found for element $i, skipping"
            fi
          done
          # Set an output to indicate we processed at least one element
          echo "processed=true" >> $GITHUB_OUTPUT

      - name: Upload latency PNG images as artifacts
        if: always() && steps.create_hdr_plots.outputs.processed == 'true'
        id: upload_latency_plots
        uses: actions/upload-artifact@v6
        with:
          name: tx-pool-latency-plots
          path: tx_pool_latency_analysis_*.png
          if-no-files-found: ignore

      - name: Upload throughput PNG images as artifacts
        if: always() && steps.create_throughput_plots.outputs.processed == 'true'
        id: upload_throughput_plots
        uses: actions/upload-artifact@v6
        with:
          name: tx-pool-throughput-plots
          path: tx_pool_throughput_analysis_*.png
          if-no-files-found: ignore

      - name: Clean up PNG files
        if: always() && (steps.create_hdr_plots.outputs.processed == 'true' || steps.create_throughput_plots.outputs.processed == 'true')
        run: |
          # Remove all generated PNG files
          rm -f tx_pool_latency_analysis_*.png
          rm -f tx_pool_throughput_analysis_*.png

      - name: Save test results
        if: always()
        env:
          TEST_RESULT: ${{ steps.erigon_kurtosis_test.outputs.test_result || 'failure' }}
        run: |
          python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
            --repo erigon \
            --commit $(git rev-parse HEAD) \
            --branch ${{ github.ref_name }} \
            --test_name txpool-performance \
            --runner ${{ runner.name }} \
            --outcome $TEST_RESULT \
            --result_file ${{ github.workspace }}/outputs_kurtosis.json

      - name: Report results in run summary
        if: always()
        run: |
          output_json="${{ github.workspace }}/outputs_kurtosis.json"

          # Start building the summary
          echo "# TxPool Performance Test Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Check if JSON file exists
          if [ ! -f "$output_json" ]; then
            echo "❌ **No test results found** - JSON output file is missing" >> $GITHUB_STEP_SUMMARY
            exit 0
          fi

          # Extract test result status
          test_result="${{ steps.erigon_kurtosis_test.outputs.test_result || 'failure' }}"
          if [ "$test_result" = "success" ]; then
            echo "✅ **Overall Test Status:** PASSED" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ **Overall Test Status:** FAILED" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY

          # Report throughput analysis
          echo "## 📊 Throughput Analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          if jq -e '.tx_pool_throughput_analysis' "$output_json" > /dev/null 2>&1; then
            echo "| Max TPS | Missed Events % | Coordinated Omission Events % | Starting TPS | Ending TPS | Increment TPS | Duration (s) |" >> $GITHUB_STEP_SUMMARY
            echo "|---------|-----------------|-------------------------------|--------------|------------|---------------|--------------|" >> $GITHUB_STEP_SUMMARY
            
            throughput_length=$(jq '.tx_pool_throughput_analysis | length' "$output_json")
            for ((i=0; i<throughput_length; i++)); do
              max_tps=$(jq -r ".tx_pool_throughput_analysis[$i].max_tps" "$output_json")
              missed_p2p_event_rate=$(jq -r ".tx_pool_throughput_analysis[$i].missed_p2p_event_rate" "$output_json")
              coordinated_omission_event_rate=$(jq -r ".tx_pool_throughput_analysis[$i].coordinated_omission_event_rate" "$output_json")
              starting_tps=$(jq -r ".tx_pool_throughput_analysis[$i].starting_tps" "$output_json")
              ending_tps=$(jq -r ".tx_pool_throughput_analysis[$i].ending_tps" "$output_json")
              increment_tps=$(jq -r ".tx_pool_throughput_analysis[$i].increment_tps" "$output_json")
              duration_s=$(jq -r ".tx_pool_throughput_analysis[$i].duration_s" "$output_json")
              
              # Format percentages from decimal to percentage format
              if [ "$missed_p2p_event_rate" != "null" ] && [ -n "$missed_p2p_event_rate" ] && [[ "$missed_p2p_event_rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                missed_p2p_event_count_percentage=$(printf "%.2f%%" $(echo "scale=6; $missed_p2p_event_rate * 100" | bc -l))
              else
                missed_p2p_event_count_percentage="N/A"
              fi
              
              if [ "$coordinated_omission_event_rate" != "null" ] && [ -n "$coordinated_omission_event_rate" ] && [[ "$coordinated_omission_event_rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                coordinated_omission_event_count_percentage=$(printf "%.2f%%" $(echo "scale=6; $coordinated_omission_event_rate * 100" | bc -l))
              else
                coordinated_omission_event_count_percentage="N/A"
              fi
              
              echo "| $max_tps | $missed_p2p_event_count_percentage | $coordinated_omission_event_count_percentage | $starting_tps | $ending_tps | $increment_tps | $duration_s |" >> $GITHUB_STEP_SUMMARY
            done
          else
            echo "⚠️ No throughput analysis data available" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY

          # Report latency analysis
          echo "## 🕐 Latency Analysis" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          if jq -e '.tx_pool_latency_analysis' "$output_json" > /dev/null 2>&1; then
            echo "| TX Count | Min (μs) | Max (ms) | Coordinated Omission Events % | Missed Events % | Duplicated Events % |" >> $GITHUB_STEP_SUMMARY
            echo "|----------|----------|----------|-------------------------------|-----------------|---------------------|" >> $GITHUB_STEP_SUMMARY
            
            latency_length=$(jq '.tx_pool_latency_analysis | length' "$output_json")
            for ((i=0; i<latency_length; i++)); do
              tx_count=$(jq -r ".tx_pool_latency_analysis[$i].tx_count" "$output_json")
              min_us=$(jq -r ".tx_pool_latency_analysis[$i].min_latency_mus" "$output_json")
              max_us=$(jq -r ".tx_pool_latency_analysis[$i].max_latency_mus" "$output_json")
              missed_p2p_event_rate=$(jq -r ".tx_pool_latency_analysis[$i].missed_p2p_event_rate" "$output_json")
              coor_omission_event_rate=$(jq -r ".tx_pool_latency_analysis[$i].coordinated_omission_event_rate" "$output_json")
              duplicated_p2p_event_rate=$(jq -r ".tx_pool_latency_analysis[$i].duplicated_p2p_event_rate" "$output_json")
              
              # Convert max from microseconds to milliseconds, handle null/invalid values
              if [ "$max_us" != "null" ] && [ -n "$max_us" ] && [[ "$max_us" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                max_ms=$(echo "scale=2; $max_us / 1000" | bc -l)
              else
                max_ms="N/A"
              fi
              
              # Format percentages from decimal to percentage format
              if [ "$missed_p2p_event_rate" != "null" ] && [ -n "$missed_p2p_event_rate" ] && [[ "$missed_p2p_event_rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                missed_p2p_event_count_percentage=$(printf "%.2f%%" $(echo "scale=6; $missed_p2p_event_rate * 100" | bc -l))
              else
                missed_p2p_event_count_percentage="N/A"
              fi

              if [ "$coor_omission_event_rate" != "null" ] && [ -n "$coor_omission_event_rate" ] && [[ "$coor_omission_event_rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                coor_omission_event_count_percentage=$(printf "%.2f%%" $(echo "scale=6; $coor_omission_event_rate * 100" | bc -l))
              else
                coor_omission_event_count_percentage="N/A"
              fi
              
              if [ "$duplicated_p2p_event_rate" != "null" ] && [ -n "$duplicated_p2p_event_rate" ] && [[ "$duplicated_p2p_event_rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then
                duplicated_p2p_event_count_percentage=$(printf "%.2f%%" $(echo "scale=6; $duplicated_p2p_event_rate * 100" | bc -l))
              else
                duplicated_p2p_event_count_percentage="N/A"
              fi
              
              echo "| $tx_count | $min_us | $max_ms | $coor_omission_event_count_percentage | $missed_p2p_event_count_percentage | $duplicated_p2p_event_count_percentage |" >> $GITHUB_STEP_SUMMARY
            done
          else
            echo "⚠️ No latency analysis data available" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY

          # Add artifact links
          echo "## 📈 Artifacts" >> $GITHUB_STEP_SUMMARY

          # Add link to raw JSON results if available
          if [ -f "$output_json" ]; then
            echo "- [Raw JSON Results](${{ steps.upload_outputs_kurtosis_json.outputs.artifact-url }})" >> $GITHUB_STEP_SUMMARY
          fi

          # Add link to latency plots if they were generated
          if [ "${{ steps.upload_latency_plots.outputs.artifact-url }}" != "" ]; then
            echo "- [Latency Distribution Plots](${{ steps.upload_latency_plots.outputs.artifact-url }})" >> $GITHUB_STEP_SUMMARY
          fi

          # Add link to throughput plots if they were generated
          if [ "${{ steps.upload_throughput_plots.outputs.artifact-url }}" != "" ]; then
            echo "- [Throughput Analysis Plots](${{ steps.upload_throughput_plots.outputs.artifact-url }})" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Clean test results
        uses: gacts/run-and-post-run@v1.4.3
        if: always()
        with:
          post: |
            rm -rf kurtosis_artifacts
            rm -f outputs_kurtosis.json