test(plugin): retry-until-valid loop + per-plugin CI matrix

- run.sh: segments flush asynchronously, so a single shot can validate before they
  all arrive (this failed in CI). Each attempt now fires traffic, waits 20s, then
  validates; retried up to 10 times.
- plugin-test.yml: one matrix job PER scenario (auto-discovered) so results show per
  plugin in the checks UI, mirroring apache/skywalking-java's plugin-test.

Author: wu-sheng

.github/workflows/plugin-test.yml

@@ -1,39 +1,51 @@
 name: Plugin Test
 
-# Per-plugin tests: run each plugin scenario against a dedicated mock collector and assert the
-# REAL emitted segments, version-by-version (see test/plugin/). Heavier than unit tests (Docker +
-# a Java collector build), so it runs only when the agent, a plugin, or the framework changes.
+# Per-plugin tests: each plugin scenario runs against a dedicated mock collector and asserts the REAL
+# emitted segments, version-by-version (see test/plugin/). One matrix job PER PLUGIN so results show
+# per plugin in the checks UI (like apache/skywalking-java's plugin-test). Heavier than unit tests
+# (Docker + a Java collector build), so it runs only when the agent, a plugin, or the framework changes.
 on:
   push:
     branches: [main]
-    paths:
-      - 'src/**'
-      - 'test/plugin/**'
-      - '.github/workflows/plugin-test.yml'
+    paths: ['src/**', 'test/plugin/**', '.github/workflows/plugin-test.yml']
   pull_request:
-    paths:
-      - 'src/**'
-      - 'test/plugin/**'
-      - '.github/workflows/plugin-test.yml'
+    paths: ['src/**', 'test/plugin/**', '.github/workflows/plugin-test.yml']
 
 concurrency:
   group: plugin-test-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  # Discover the scenarios so each becomes its own matrix entry (no need to edit this file per plugin).
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      scenarios: ${{ steps.list.outputs.scenarios }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: list
+        run: |
+          scenarios=$(ls -d test/plugin/scenarios/*/ | xargs -n1 basename | jq -R . | jq -cs .)
+          echo "scenarios=$scenarios" >> "$GITHUB_OUTPUT"
+          echo "scenarios: $scenarios"
+
   plugin-test:
+    needs: discover
     runs-on: ubuntu-latest
     timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        scenario: ${{ fromJSON(needs.discover.outputs.scenarios) }}
+    name: ${{ matrix.scenario }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive   # the protocol-v3 codegen source
-
-      # Needed to build the mock collector from the pinned agent-test-tool commit.
       - uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: '11'
-
-      - name: Run plugin scenarios (build collector + per-version build/run/validate)
-        run: bash test/plugin/run.sh
+          cache: maven            # cache the mock-collector build deps
+      - name: Run ${{ matrix.scenario }} (build collector + per-version build/run/validate)
+        run: bash test/plugin/run.sh ${{ matrix.scenario }}

test/plugin/run.sh

@@ -51,23 +51,21 @@ ensure_collector() {
   rm -rf "$work" "$ctx"
 }
 
-# Wait until the collector has received at least one segment, then validate.
+# Segments flush to the collector asynchronously (CAP dispatches on background threads, and the
+# agent batches), so a single shot can validate before everything arrives. Each attempt FIRES
+# TRAFFIC, WAITS, then VALIDATES; we retry the whole loop until it passes or the attempts run out.
+VALIDATE_ATTEMPTS="${VALIDATE_ATTEMPTS:-10}"
+VALIDATE_WAIT="${VALIDATE_WAIT:-20}"
+
+drive_traffic() {
+  for _ in 1 2 3 4 5; do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/cap" >/dev/null 2>&1 || true; done
+}
+
 validate() {
   local expected="$1"
-  for _ in $(seq 1 30); do
-    if curl -sf --max-time 10 "$COLLECTOR_HTTP/receiveData" 2>/dev/null | grep -q "segmentId:"; then
-      break
-    fi
-    sleep 2
-  done
   local code; code="$(curl -s -o /tmp/plugin-validate.out -w '%{http_code}' --max-time 20 \
     -X POST --data-binary @"$expected" "$COLLECTOR_HTTP/dataValidate")"
-  if [ "$code" = "200" ]; then
-    return 0
-  fi
-  log "VALIDATION FAILED (HTTP $code):"
-  sed -n '/cause by/,$p' /tmp/plugin-validate.out | head -40
-  return 1
+  [ "$code" = "200" ]
 }
 
 run_case() {
@@ -79,13 +77,20 @@ run_case() {
     TFM="$tfm" CAP_VERSION="$ver" DOTNET_VERSION="$dotnet" \
       docker compose up -d --build --quiet-pull
   )
-  local rc=0
-  # health-gate the app, then drive the entry endpoint a few times
+  # health-gate the app
   for _ in $(seq 1 30); do
     curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/healthCheck" >/dev/null 2>&1 && break || sleep 2
   done
-  for _ in $(seq 1 5); do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/cap" >/dev/null 2>&1 || true; sleep 1; done
-  validate "$scen/config/expectedData.yaml" || rc=1
+  local rc=1
+  for attempt in $(seq 1 "$VALIDATE_ATTEMPTS"); do
+    drive_traffic
+    sleep "$VALIDATE_WAIT"
+    if validate "$scen/config/expectedData.yaml"; then rc=0; break; fi
+    log "attempt $attempt/$VALIDATE_ATTEMPTS: not valid yet, retrying"
+  done
+  if [ "$rc" -ne 0 ]; then
+    log "VALIDATION FAILED:"; sed -n '/cause by/,$p' /tmp/plugin-validate.out | head -40
+  fi
   ( cd "$scen" && docker compose down -v >/dev/null 2>&1 || true )
   return $rc
 }