From 7a7b24bc5319b2e09bfeb4225718b59cca004231 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 14:39:37 +0800 Subject: [PATCH 1/8] fix(core): stop diagnostic log spam on instrumented-library version mismatch (#565) When an instrumented library's version is incompatible with the SkyAPM plugin compiled against it, the plugin's [DiagnosticName] handler throws a structural binding error (e.g. MissingMethodException on a changed event property, as in SkyApm.Diagnostics.CAP). TracingDiagnosticObserver caught it but re-logged the same exception on every event, so one version mismatch produced endless "Invoke diagnostic method exception." spam that never recovered. Such MissingMemberException/TypeLoadException failures are deterministic and never self-heal, so log a single warning (with a version-match hint) and disable that diagnostic handler instead of re-firing it for every event. Other, potentially transient exceptions keep their per-occurrence Error logging. Fixes #565 --- .../Diagnostics/TracingDiagnosticObserver.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/SkyApm.Core/Diagnostics/TracingDiagnosticObserver.cs b/src/SkyApm.Core/Diagnostics/TracingDiagnosticObserver.cs index 43e0a56a..6c64f5b4 100644 --- a/src/SkyApm.Core/Diagnostics/TracingDiagnosticObserver.cs +++ b/src/SkyApm.Core/Diagnostics/TracingDiagnosticObserver.cs @@ -17,6 +17,7 @@ */ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using SkyApm.Logging; @@ -26,6 +27,7 @@ namespace SkyApm.Diagnostics internal class TracingDiagnosticObserver : IObserver> { private readonly Dictionary _methodCollection; + private readonly ConcurrentDictionary _disabledDiagnostics = new ConcurrentDictionary(); private readonly ILogger _logger; public TracingDiagnosticObserver(ITracingDiagnosticProcessor tracingDiagnosticProcessor, @@ -54,10 +56,26 @@ public void OnNext(KeyValuePair value) if (!_methodCollection.TryGetValue(value.Key, out var method)) return; + if (_disabledDiagnostics.ContainsKey(value.Key)) + return; + try { method.Invoke(value.Key, value.Value); } + catch (Exception exception) when (exception is MissingMemberException || exception is TypeLoadException) + { + // A binding/structural error means the instrumented library's version is incompatible + // with this plugin (e.g. a diagnostic event type or property changed across versions). + // It fails deterministically for every event, so disable this handler after logging once + // instead of spamming the same error on every call (issue #565). + if (_disabledDiagnostics.TryAdd(value.Key, 0)) + { + _logger.Warning($"Disabled diagnostic handler '{value.Key}' due to an incompatible " + + "instrumented library version; ensure the installed package version matches " + + $"this SkyAPM plugin. {exception.GetType().Name}: {exception.Message}"); + } + } catch (Exception exception) { _logger.Error("Invoke diagnostic method exception.", exception); From 94cb5b7f16657433bc88241d0e3f2ca43a4103cd Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 15:30:58 +0800 Subject: [PATCH 2/8] fix(cap): make sw8 carrier injection idempotent CapCarrierHeaderCollection.Add used Dictionary.Add, which throws ArgumentException ("An item with the same key has already been added. Key: sw8") when a message already carries an sw8 header. CAP's in-memory queue re-publishes a message with the header already present (surfaced on DotNetCore.CAP 10.x), so every publish threw and the publisher span was emitted malformed. Injecting the carrier must be idempotent -- overwrite the key instead of throwing. Caught by the new CAP plugin test running against CAP 10. --- src/SkyApm.Diagnostics.CAP/CapCarrierHeaderCollection.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/SkyApm.Diagnostics.CAP/CapCarrierHeaderCollection.cs b/src/SkyApm.Diagnostics.CAP/CapCarrierHeaderCollection.cs index 46aa8f78..1b0e9934 100644 --- a/src/SkyApm.Diagnostics.CAP/CapCarrierHeaderCollection.cs +++ b/src/SkyApm.Diagnostics.CAP/CapCarrierHeaderCollection.cs @@ -44,7 +44,10 @@ public IEnumerator> GetEnumerator() public void Add(string key, string value) { - _messageHeaders.Add(key, value); + // Overwrite rather than Dictionary.Add: a message can already carry an sw8 header when + // the transport re-publishes it (e.g. CAP's in-memory queue), and Add throws on a + // duplicate key. Injecting the carrier must be idempotent. + _messageHeaders[key] = value; } IEnumerator IEnumerable.GetEnumerator() From a9c3cce04588896fecfb1202b53f6739722c70d6 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 15:30:58 +0800 Subject: [PATCH 3/8] test(plugin): add a version-by-version plugin test framework + CAP scenario Adds test/plugin/, a SkyWalking-Java-style plugin test harness: each scenario runs the real instrumented library with the agent built from source, reports to a dedicated mock collector (apache/skywalking-agent-test-tool, built from a pinned commit -- the published :latest image is from 2019 and predates the V8 management protocol), and validates the REAL emitted segments against config/expectedData.yaml via the collector's /dataValidate. Scenarios run version-by-version (latest patch of each supported minor) -- which is what catches plugin/library version mismatches like #565. First scenario: SkyApm.Diagnostics.CAP (real DotNetCore.CAP in-memory), validated green on net8.0 / CAP 8.0.0. Includes run.sh, a CI workflow, and plugin development + testing docs. net10.0 / CAP 10.x is deferred (CAP 10's in-memory queue dispatches the publish synchronously, changing the publisher span shape) and already paid for itself by surfacing the sw8 carrier bug fixed in the previous commit. --- .github/workflows/plugin-test.yml | 39 ++++ content/docs/guides/plugin-development.md | 177 ++++++++++++++++++ content/docs/guides/plugin-testing.md | 95 ++++++++++ test/plugin/README.md | 64 +++++++ test/plugin/run.sh | 112 +++++++++++ .../plugin/scenarios/cap-scenario/Case.csproj | 37 ++++ test/plugin/scenarios/cap-scenario/Dockerfile | 26 +++ test/plugin/scenarios/cap-scenario/Program.cs | 55 ++++++ .../cap-scenario/config/expectedData.yaml | 60 ++++++ .../scenarios/cap-scenario/docker-compose.yml | 35 ++++ .../plugin/scenarios/cap-scenario/skyapm.json | 13 ++ .../cap-scenario/support-version.list | 10 + 12 files changed, 723 insertions(+) create mode 100644 .github/workflows/plugin-test.yml create mode 100644 content/docs/guides/plugin-development.md create mode 100644 content/docs/guides/plugin-testing.md create mode 100644 test/plugin/README.md create mode 100755 test/plugin/run.sh create mode 100644 test/plugin/scenarios/cap-scenario/Case.csproj create mode 100644 test/plugin/scenarios/cap-scenario/Dockerfile create mode 100644 test/plugin/scenarios/cap-scenario/Program.cs create mode 100644 test/plugin/scenarios/cap-scenario/config/expectedData.yaml create mode 100644 test/plugin/scenarios/cap-scenario/docker-compose.yml create mode 100644 test/plugin/scenarios/cap-scenario/skyapm.json create mode 100644 test/plugin/scenarios/cap-scenario/support-version.list diff --git a/.github/workflows/plugin-test.yml b/.github/workflows/plugin-test.yml new file mode 100644 index 00000000..5c8fa295 --- /dev/null +++ b/.github/workflows/plugin-test.yml @@ -0,0 +1,39 @@ +name: Plugin Test + +# Per-plugin tests: run each plugin scenario against a dedicated mock collector and assert the +# REAL emitted segments, version-by-version (see test/plugin/). Heavier than unit tests (Docker + +# a Java collector build), so it runs only when the agent, a plugin, or the framework changes. +on: + push: + branches: [main] + paths: + - 'src/**' + - 'test/plugin/**' + - '.github/workflows/plugin-test.yml' + pull_request: + paths: + - 'src/**' + - 'test/plugin/**' + - '.github/workflows/plugin-test.yml' + +concurrency: + group: plugin-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + plugin-test: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive # the protocol-v3 codegen source + + # Needed to build the mock collector from the pinned agent-test-tool commit. + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Run plugin scenarios (build collector + per-version build/run/validate) + run: bash test/plugin/run.sh diff --git a/content/docs/guides/plugin-development.md b/content/docs/guides/plugin-development.md new file mode 100644 index 00000000..3df8ce1c --- /dev/null +++ b/content/docs/guides/plugin-development.md @@ -0,0 +1,177 @@ +--- +title: "Plugin development" +weight: 40 +--- + +SkyAPM-dotnet instruments libraries through **diagnostic plugins**. A plugin is a thin adapter +that subscribes to a library's [`DiagnosticSource`](https://learn.microsoft.com/dotnet/api/system.diagnostics.diagnosticsource) +events and turns them into SkyWalking **trace segments**. This guide shows how to write one, +using the existing plugins (`SkyApm.Diagnostics.HttpClient`, `SkyApm.Diagnostics.SqlClient`, +`SkyApm.Diagnostics.CAP`, …) as the reference shape. + +## How a plugin works + +Most .NET libraries already emit `DiagnosticSource` events (ASP.NET Core, HttpClient, +EF Core, gRPC, CAP, MongoDB, …). A plugin listens to the named listener and maps each event to a +span. The agent owns the wiring: + +``` +library DiagnosticSource ──► TracingDiagnosticProcessorObserver (matches ListenerName) + │ + ▼ + your ITracingDiagnosticProcessor (one [DiagnosticName] method per event) + │ CreateEntry/Local/ExitSegmentContext + AddTag/AddLog + ▼ + ITracingContext.Release ──► emitted SegmentRequest (the real trace) +``` + +You implement only the middle box. + +## 1. Implement `ITracingDiagnosticProcessor` + +Create a class in a `SkyApm.Diagnostics.` project that implements +[`ITracingDiagnosticProcessor`](https://github.com/SkyAPM/SkyAPM-dotnet/blob/main/src/SkyApm.Abstractions/Diagnostics/ITracingDiagnosticProcessor.cs). +Its `ListenerName` must equal the library's `DiagnosticSource`/`DiagnosticListener` name, and each +event handler is annotated with `[DiagnosticName("")]`: + +```csharp +public class MyLibTracingDiagnosticProcessor : ITracingDiagnosticProcessor +{ + public string ListenerName => "MyLib.DiagnosticListener"; // the source's name + + private readonly ITracingContext _tracingContext; + private readonly IExitSegmentContextAccessor _exitSegmentContextAccessor; + private readonly TracingConfig _tracingConfig; + + public MyLibTracingDiagnosticProcessor(ITracingContext tracingContext, + IExitSegmentContextAccessor exitSegmentContextAccessor, + IConfigAccessor configAccessor) + { + _tracingContext = tracingContext; + _exitSegmentContextAccessor = exitSegmentContextAccessor; + _tracingConfig = configAccessor.Get(); + } + + [DiagnosticName("MyLib.Command.Before")] + public void BeforeCommand([Object] CommandEventData data) // [Object] binds the raw event payload + { + var context = _tracingContext.CreateExitSegmentContext("MyLib/" + data.Operation, data.Host); + context.Span.SpanLayer = SpanLayer.DB; + context.Span.Component = Components.MYLIB; // see "Components" below + context.Span.AddTag(Tags.DB_STATEMENT, data.Sql); + } + + [DiagnosticName("MyLib.Command.After")] + public void AfterCommand([Object] CommandEventData data) + { + var context = _exitSegmentContextAccessor.Context; + if (context == null) return; + _tracingContext.Release(context); // Release emits the segment + } + + [DiagnosticName("MyLib.Command.Error")] + public void ErrorCommand([Object] CommandErrorEventData data) + { + var context = _exitSegmentContextAccessor.Context; + if (context == null) return; + context.Span.ErrorOccurred(data.Exception, _tracingConfig); + _tracingContext.Release(context); + } +} +``` + +Key points: + +- The `[Object]` parameter attribute tells the dispatcher to pass the event's raw payload object; + its type is the library's event class, which the plugin references at compile time. +- A handler that throws does not crash the app — `TracingDiagnosticObserver` catches it. A + **structural binding error** (`MissingMethodException`/`TypeLoadException`, i.e. the library + version is incompatible with what the plugin compiled against) is logged **once** and that handler + is then disabled, rather than spamming on every event (see *Version coupling* below). + +## 2. Create segments with `ITracingContext` + +[`ITracingContext`](https://github.com/SkyAPM/SkyAPM-dotnet/blob/main/src/SkyApm.Abstractions/Tracing/ITracingContext.cs) +is the only API you need to produce spans. Pick the span kind by where the work happens: + +| Method | Use for | Accessor to read it back | +|---|---|---| +| `CreateEntrySegmentContext(operationName, carrierHeader)` | inbound work (a received request/message) | `IEntrySegmentContextAccessor` | +| `CreateLocalSegmentContext(operationName)` | in-process work (persistence, serialization) | `ILocalSegmentContextAccessor` | +| `CreateExitSegmentContext(operationName, networkAddress[, carrierHeader])` | outbound work (a DB/HTTP/MQ call) | `IExitSegmentContextAccessor` | + +Always pair a `Create…` with a `Release(context)` (typically on the *After*/*Error* event). On the +matching *After* you usually read the context back from the accessor rather than threading it through. +On every span set: + +- `Span.SpanLayer` — `Http` / `Database` (DB) / `RPCFramework` / `MQ` / `Cache` / `Unknown`. +- `Span.Component` — the component id (below). +- `Span.Peer` — the remote address for exit spans (a [peer formatter](#peers) helps here). +- `Span.AddTag(Tags.X, value)` and `Span.AddLog(LogEvent.…)` for detail; `Span.ErrorOccurred(ex, config)` on failure. + +## 3. Cross-process context propagation + +For spans that cross a process boundary (an MQ message, an outbound RPC), inject/extract the +SkyWalking `sw8` headers so the trace stays connected: + +- **Producer / client (exit):** build an `ICarrierHeaderCollection` over the outbound message + headers and call `ICarrierPropagator.Inject(context, header)` so the downstream side continues the trace. +- **Consumer / server (entry):** wrap the inbound headers in an `ICarrierHeaderCollection` and pass it + to `CreateEntrySegmentContext(operationName, carrierHeader)`; the agent extracts the parent ref. + +`SkyApm.Diagnostics.CAP`'s `CapCarrierHeaderCollection` is a concrete example of adapting a library's +message headers to `ICarrierHeaderCollection`. + +## Components + +Span components are numeric ids from the shared SkyWalking registry +([`component-libraries.yml`](https://github.com/apache/skywalking/blob/master/oap-server/server-starter/src/main/resources/component-libraries.yml)), +surfaced as `SkyApm.Common.Components`. Reuse an existing id where one fits (e.g. `Components.HTTPCLIENT`, +`Components.SQLCLIENT`, `Components.CAP`); a brand-new library needs an id registered upstream in +SkyWalking first, otherwise the UI shows it as "Unknown". + +## Peers + +Exit spans should carry the remote address in `Span.Peer`. Where the address must be parsed from a +connection/config object, add a `SkyApm.PeerFormatters.` implementing `IDbPeerFormatter` +(see `SkyApm.PeerFormatters.MySqlConnector`) and resolve it through the injected `IPeerFormatter`. + +## 4. Register the plugin + +Expose an opt-in extension on `SkyApmExtensions` that registers your processor as an +`ITracingDiagnosticProcessor` singleton — the agent discovers all registered processors and +subscribes each to its `ListenerName`: + +```csharp +public static class SkyWalkingBuilderExtensions +{ + public static SkyApmExtensions AddMyLib(this SkyApmExtensions extensions) + { + extensions.Services.AddSingleton(); + return extensions; + } +} +``` + +Users opt in via the setup lambda: + +```csharp +services.AddSkyAPM(ext => ext.AddMyLib()); +``` + +A small, always-on default set is registered automatically; everything else (CAP, MassTransit, +MongoDB, …) is opt-in like the above. + +## Version coupling (important) + +A plugin is **compiled against a specific version** of the library it instruments, but at runtime +NuGet unifies that library to the **application's** installed version. If the app's version changed +the diagnostic event types in a binary-incompatible way, the handler throws `MissingMethodException` +at runtime (see [#565](https://github.com/SkyAPM/SkyAPM-dotnet/issues/565)). The agent now degrades +gracefully (logs once, disables the handler), but the plugin still won't trace that library until the +versions line up. Two consequences for plugin authors: + +- Pin the plugin's `DotNetCore.X` reference per target framework (see the existing per-TFM + `Condition="'$(TargetFramework)' == 'netX'"` `PackageReference` blocks) and document the supported range. +- **Test the plugin against the real library, version by version** — see + [Plugin testing](plugin-testing.md). diff --git a/content/docs/guides/plugin-testing.md b/content/docs/guides/plugin-testing.md new file mode 100644 index 00000000..720ec37f --- /dev/null +++ b/content/docs/guides/plugin-testing.md @@ -0,0 +1,95 @@ +--- +title: "Plugin testing" +weight: 41 +--- + +A diagnostic plugin is only correct if the agent **emits the right trace** — and the emitted trace is +the only thing that is real (intermediate state isn't). So plugin tests run the **real** instrumented +library, let the agent report to a **dedicated mock collector**, and assert on the **captured +segments**. Because a plugin is compiled against one version of a library but runs against whatever +version the application installs (the cause of [#565](https://github.com/SkyAPM/SkyAPM-dotnet/issues/565)), +each scenario runs **version-by-version**. + +This mirrors the [Apache SkyWalking Java plugin-test framework](https://skywalking.apache.org/docs/skywalking-java/next/en/setup/service-agent/java-agent/plugin-test/). +The harness lives in [`test/plugin/`](https://github.com/SkyAPM/SkyAPM-dotnet/tree/main/test/plugin); +this guide explains how it works and how to add a scenario. (For a full OAP + storage backend test +instead, see the heavier [`test/e2e/`](https://github.com/SkyAPM/SkyAPM-dotnet/tree/main/test/e2e).) + +## How it works + +``` +scenario app ──gRPC :19876──► mock collector ──HTTP :12800──► /receiveData /dataValidate +(agent from source, (a SkyWalking receiver, (captured (validate vs + real library) not real OAP+DB) segments) expectedData.yaml) +``` + +For each scenario and each ` ` line in its `support-version.list`, `run.sh`: + +1. builds the scenario app image — the **agent from source** plus the instrumented library at that + version (NuGet unifies the version up, so the plugin runs against it just like a real app); +2. boots the app together with the mock collector; +3. drives the scenario's `/case/` entry endpoint; +4. POSTs the scenario's `config/expectedData.yaml` to the collector's `/dataValidate`; `200`/`success` + passes, otherwise the validator prints which span/field differed. + +### The dedicated collector + +The collector is `apache/skywalking-agent-test-tool`'s `mock-collector`, built from a **pinned commit** +(the same one `apache/skywalking-java` pins today). The published `skyapm/mock-collector:latest` is from +2019 and predates the V8 management protocol the current agent needs, so do **not** use it — `run.sh` +builds the pinned version for you (override with `COLLECTOR_IMAGE=…` to reuse a pre-built one). + +## Authoring `expectedData.yaml` + +Run the scenario once and read what the collector actually captured: + +```bash +curl -s http://localhost:12800/receiveData +``` + +Then keep the segments/spans you want to assert and replace volatile fields with matchers. The validator +([SkyWalking validator DSL](https://skywalking.apache.org/docs/skywalking-java/next/en/setup/service-agent/java-agent/plugin-test/#expecteddatayaml-format)) +matches **expected segments as a subset** of the captured ones (so you needn't describe every segment), +but within a matched segment it checks **span count, tags, and span logs exactly** — so describe a span's +full tag/log set, using matchers for anything variable. + +Matchers: `not null`, `not blank`, `nq ` (not-equal), `gt`/`ge `, `eq ` (default), `start with`, +`end with`. Example from the CAP scenario (the publisher span): + +```yaml +segmentItems: + - serviceName: cap-scenario + segmentSize: nq 0 + segments: + - segmentId: not null + spans: + - operationName: CAP/skyapm.plugin.test.cap/Publisher + spanLayer: MQ + componentId: 3004 # Components.CAP + spanType: Entry + startTime: nq 0 # volatile -> matcher + endTime: nq 0 + isError: false + peer: localhost + tags: + - {key: mq.topic, value: skyapm.plugin.test.cap} + - {key: mq.broker, value: localhost} + logs: + - logEvent: [{key: event, value: Event Publishing Start}] + - logEvent: [{key: message, value: CAP message publishing start...}] + - logEvent: [{key: event, value: Event Publishing End}] + - logEvent: [{key: message, value: not blank}] # variable content +``` + +## Adding a scenario + +1. Create `test/plugin/scenarios/-scenario/` with a minimal app referencing the agent and the + plugin **from source** and the instrumented library at a build-arg version, exposing `/case/` + (which performs the instrumented action) and `/case/healthCheck`. +2. Add `skyapm.json` (report to `mock-collector:19876`), a `Dockerfile` (agent built from source; + library version as a build arg), and a `docker-compose.yml` (app + collector). +3. Add `support-version.list` — the **latest patch of each supported minor** of the library. +4. Author `config/expectedData.yaml` from a real `/receiveData` capture, as above. +5. Run `bash test/plugin/run.sh -scenario` until it passes. + +See [Plugin development](plugin-development.md) for writing the plugin itself. diff --git a/test/plugin/README.md b/test/plugin/README.md new file mode 100644 index 00000000..9b9b2fb9 --- /dev/null +++ b/test/plugin/README.md @@ -0,0 +1,64 @@ +# Plugin tests + +Per-plugin tests that verify a diagnostic plugin produces the **right trace** by asserting on the +agent's **real emitted segments** — the only thing that is real. Modelled on the +[Apache SkyWalking Java plugin-test framework](https://skywalking.apache.org/docs/skywalking-java/next/en/setup/service-agent/java-agent/plugin-test/). + +Unlike [`../e2e`](../e2e) (a full, heavy OAP + BanyanDB backend), these tests report to a lightweight +**dedicated mock collector** — a SkyWalking receiver that captures what the agent sends and validates +it — and run a scenario **version-by-version** against each supported version of the instrumented +library. That cross-version coverage is what catches breakages like +[#565](https://github.com/SkyAPM/SkyAPM-dotnet/issues/565). + +## Layout + +``` +test/plugin/ + run.sh runner: builds the collector, then per scenario × version: + build app image (agent from source) → run with collector → drive → validate + scenarios/ + cap-scenario/ first scenario: SkyApm.Diagnostics.CAP + Case.csproj / Program.cs minimal app: real DotNetCore.CAP (in-memory) + the CAP plugin + skyapm.json agent config (reports to mock-collector:19876) + Dockerfile agent built FROM SOURCE; TFM + CAP_VERSION are build args + docker-compose.yml the scenario app + the mock collector + support-version.list " " per line — latest patch of each supported minor + config/expectedData.yaml expected emitted spans (SkyWalking validator DSL: nq 0, not null, …) +``` + +## The dedicated collector + +The mock collector (`mock-collector` from +[apache/skywalking-agent-test-tool](https://github.com/apache/skywalking-agent-test-tool)) speaks the +agent's V8 gRPC protocol on `:19876` and exposes HTTP `:12800`: + +- `GET /receiveData` — everything it captured, as YAML. +- `POST /dataValidate` — validate captured data against a posted `expectedData.yaml` (`200`/`success` = pass). + +`run.sh` builds it from a **pinned commit** (the same one `apache/skywalking-java` pins today); the +published `skyapm/mock-collector:latest` image is from 2019 and predates the V8 management protocol the +current agent needs. + +## Run + +```bash +bash test/plugin/run.sh # all scenarios, all versions +bash test/plugin/run.sh cap-scenario # one scenario + +# reuse a pre-built collector image (skips the Java build): +COLLECTOR_IMAGE=skyapm/mock-collector: bash test/plugin/run.sh +``` + +Requires `docker`; building the collector also needs `git` + a JDK. CI runs it via +[`.github/workflows/plugin-test.yml`](../../.github/workflows/plugin-test.yml). + +## Add a scenario + +1. `scenarios/-scenario/` with a minimal app that references the agent + the plugin **from source** + and the instrumented library at `$(Version)`, exposing `/case/` (entry) and + `/case/healthCheck`. +2. A `Dockerfile` (agent from source; library version as a build arg) and a `docker-compose.yml`. +3. `support-version.list` — the latest patch of each supported minor. +4. `config/expectedData.yaml` — author it from a real run: `curl localhost:12800/receiveData`, then keep + the spans you want to assert and replace volatile fields with matchers. See + [Plugin testing](../../content/docs/guides/plugin-testing.md). diff --git a/test/plugin/run.sh b/test/plugin/run.sh new file mode 100755 index 00000000..b136881d --- /dev/null +++ b/test/plugin/run.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# SkyAPM-dotnet plugin-test runner. +# +# For each scenario under scenarios/, and for each " " line in the +# scenario's support-version.list, this: +# 1. builds the scenario app image (agent built FROM SOURCE + the library at that version), +# 2. boots it together with the dedicated mock collector (a SkyWalking receiver, not a real +# OAP+DB) so the agent reports its REAL emitted segments over gRPC, +# 3. drives the scenario's entry endpoint to generate a trace, +# 4. validates the captured segments against the scenario's config/expectedData.yaml by +# POSTing it to the collector's /dataValidate (the SkyWalking validator). +# +# Only the FINAL EMITTED TRACES are asserted. Requires: docker, and (to build the collector) +# git + a JDK/Maven via the agent-test-tool's mvnw. +# +# Usage: +# bash test/plugin/run.sh # all scenarios, all versions +# bash test/plugin/run.sh cap-scenario # one scenario +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" + +# The mock collector is built from a PINNED apache/skywalking-agent-test-tool commit (the same +# one apache/skywalking-java's test/plugin pins today). The published skyapm/mock-collector:latest +# image is from 2019 and does not implement the V8 management protocol the current agent needs. +COLLECTOR_COMMIT="${COLLECTOR_COMMIT:-7220c715d280cf0d7421f17bbc8c8de57249914d}" +COLLECTOR_IMAGE="${COLLECTOR_IMAGE:-skyapm/mock-collector:${COLLECTOR_COMMIT:0:12}}" +export COLLECTOR_IMAGE + +ENTRY_PORT="${ENTRY_PORT:-8080}" +COLLECTOR_HTTP="${COLLECTOR_HTTP:-http://localhost:12800}" +SCENARIO_FILTER="${1:-}" + +log() { echo "[plugin-test] $*"; } + +ensure_collector() { + if docker image inspect "$COLLECTOR_IMAGE" >/dev/null 2>&1; then + log "mock collector image present: $COLLECTOR_IMAGE" + return + fi + log "building mock collector from apache/skywalking-agent-test-tool@${COLLECTOR_COMMIT}" + local work; work="$(mktemp -d)" + git clone -q --filter=blob:none https://github.com/apache/skywalking-agent-test-tool "$work/att" + git -C "$work/att" checkout -q "$COLLECTOR_COMMIT" + ( cd "$work/att" && ./mvnw -B -q -Dmaven.test.skip=true clean package ) + local tarball; tarball="$(find "$work/att" -name skywalking-mock-collector.tar.gz | head -1)" + local ctx; ctx="$(mktemp -d)" + cp "$tarball" "$ctx/"; cp "$work/att/docker/mock-collector/Dockerfile" "$ctx/" + docker build -t "$COLLECTOR_IMAGE" "$ctx" + rm -rf "$work" "$ctx" +} + +# Wait until the collector has received at least one segment, then validate. +validate() { + local expected="$1" + for _ in $(seq 1 30); do + if curl -sf --max-time 10 "$COLLECTOR_HTTP/receiveData" 2>/dev/null | grep -q "segmentId:"; then + break + fi + sleep 2 + done + local code; code="$(curl -s -o /tmp/plugin-validate.out -w '%{http_code}' --max-time 20 \ + -X POST --data-binary @"$expected" "$COLLECTOR_HTTP/dataValidate")" + if [ "$code" = "200" ]; then + return 0 + fi + log "VALIDATION FAILED (HTTP $code):" + sed -n '/cause by/,$p' /tmp/plugin-validate.out | head -40 + return 1 +} + +run_case() { + local scen="$1" tfm="$2" ver="$3" + local dotnet="${tfm#net}" # net8.0 -> 8.0 + log "=== $(basename "$scen") :: TFM=$tfm CAP_VERSION=$ver ===" + ( + cd "$scen" + TFM="$tfm" CAP_VERSION="$ver" DOTNET_VERSION="$dotnet" \ + docker compose up -d --build --quiet-pull + ) + local rc=0 + # health-gate the app, then drive the entry endpoint a few times + for _ in $(seq 1 30); do + curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/healthCheck" >/dev/null 2>&1 && break || sleep 2 + done + for _ in $(seq 1 5); do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/cap" >/dev/null 2>&1 || true; sleep 1; done + validate "$scen/config/expectedData.yaml" || rc=1 + ( cd "$scen" && docker compose down -v >/dev/null 2>&1 || true ) + return $rc +} + +main() { + ensure_collector + local failures=0 total=0 + for scen in "$HERE"/scenarios/*/; do + scen="${scen%/}" + [ -f "$scen/support-version.list" ] || continue + [ -n "$SCENARIO_FILTER" ] && [ "$(basename "$scen")" != "$SCENARIO_FILTER" ] && continue + while read -r raw; do + local line; line="$(echo "${raw%%#*}" | xargs)"; [ -z "$line" ] && continue + local tfm ver; tfm="$(echo "$line" | awk '{print $1}')"; ver="$(echo "$line" | awk '{print $2}')" + total=$((total + 1)) + if run_case "$scen" "$tfm" "$ver"; then log "PASS $(basename "$scen") $tfm/$ver" + else log "FAIL $(basename "$scen") $tfm/$ver"; failures=$((failures + 1)); fi + done < "$scen/support-version.list" + done + log "done: $((total - failures))/$total passed" + [ "$failures" -eq 0 ] +} + +main "$@" diff --git a/test/plugin/scenarios/cap-scenario/Case.csproj b/test/plugin/scenarios/cap-scenario/Case.csproj new file mode 100644 index 00000000..462ab851 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/Case.csproj @@ -0,0 +1,37 @@ + + + + $(TFM) + net8.0 + 8.0.0 + disable + enable + SkyApm.PluginTest.Cap + false + false + + + + + + + + + + + + + + + + + + + PreserveNewest + + + diff --git a/test/plugin/scenarios/cap-scenario/Dockerfile b/test/plugin/scenarios/cap-scenario/Dockerfile new file mode 100644 index 00000000..29a9707d --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/Dockerfile @@ -0,0 +1,26 @@ +# CAP plugin-test scenario image. Build context MUST be the repo root so the agent +# source + the protocol-v3 submodule are available. TFM + CAP_VERSION select the +# matrix entry under test. +# docker build -f test/plugin/scenarios/cap-scenario/Dockerfile \ +# --build-arg DOTNET_VERSION=8.0 --build-arg TFM=net8.0 --build-arg CAP_VERSION=8.0.0 -t cap-scenario . +ARG DOTNET_VERSION=8.0 + +# Build with the .NET 10 SDK (targets both net8.0 and net10.0). +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +ARG TFM=net8.0 +ARG CAP_VERSION=8.0.0 +WORKDIR /src +COPY . . +# Generate the gRPC/protobuf code from the protocol-v3 submodule before the transports compile. +RUN dotnet build src/SkyApm.Transport.Protocol -c Release +RUN dotnet publish test/plugin/scenarios/cap-scenario/Case.csproj -c Release \ + -f ${TFM} -p:TFM=${TFM} -p:CapVersion=${CAP_VERSION} -o /app + +FROM mcr.microsoft.com/dotnet/aspnet:${DOTNET_VERSION} AS run +WORKDIR /app +COPY --from=build /app . +# Activate the agent zero-code (note casing: SkyAPM). The collector address is in skyapm.json. +ENV ASPNETCORE_HOSTINGSTARTUPASSEMBLIES=SkyAPM.Agent.AspNetCore \ + ASPNETCORE_URLS=http://+:8080 +EXPOSE 8080 +ENTRYPOINT ["dotnet", "Case.dll"] diff --git a/test/plugin/scenarios/cap-scenario/Program.cs b/test/plugin/scenarios/cap-scenario/Program.cs new file mode 100644 index 00000000..c34973c5 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/Program.cs @@ -0,0 +1,55 @@ +// CAP plugin test scenario for the SkyAPM-dotnet plugin-test framework. +// +// The agent core is activated zero-code via ASPNETCORE_HOSTINGSTARTUPASSEMBLIES +// (see Dockerfile); this app additionally registers the opt-in CAP tracing plugin +// and runs real DotNetCore.CAP with its in-memory storage + queue (no broker). +// +// GET /case/cap publishes a CAP message that an in-process subscriber consumes, so +// the agent emits a CAP publisher span (within the inbound entry segment) and a CAP +// subscriber segment to the mock collector. The runner then validates those emitted +// segments against config/expectedData.yaml. + +using DotNetCore.CAP; +using Microsoft.Extensions.DependencyInjection; +using Savorboard.CAP.InMemoryMessageQueue; +using SkyApm; +using SkyApm.Diagnostics.CAP; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddCap(x => +{ + x.UseInMemoryStorage(); + x.UseInMemoryMessageQueue(); +}); +builder.Services.AddSingleton(); + +// Register the opt-in CAP tracing plugin. AddCap() does exactly this; we register +// directly to avoid a second AddSkyAPM call on top of the HostingStartup activation. +builder.Services.AddSingleton(); + +var app = builder.Build(); + +app.MapGet("/case/healthCheck", () => "success"); + +app.MapGet("/case/cap", async (ICapPublisher publisher) => +{ + await publisher.PublishAsync("skyapm.plugin.test.cap", new CapMessage { Body = "hello-skyapm" }); + return "ok"; +}); + +app.Run(); + +public class CapMessage +{ + public string Body { get; set; } +} + +public class CapSubscriber : ICapSubscribe +{ + [CapSubscribe("skyapm.plugin.test.cap")] + public void Handle(CapMessage message) + { + // consume; the act of subscribing produces the CAP subscriber trace + } +} diff --git a/test/plugin/scenarios/cap-scenario/config/expectedData.yaml b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml new file mode 100644 index 00000000..e175f5f8 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml @@ -0,0 +1,60 @@ +# Expected emitted segments for the CAP plugin scenario, validated against the data +# the mock collector actually received (POST to /dataValidate). Variable fields use +# matchers (nq 0 = not-equal-0, "not null", "not blank"); the CAP publisher and +# subscriber segments are matched as a subset of all captured segments. +segmentItems: + - serviceName: cap-scenario + segmentSize: nq 0 + segments: + # CAP publisher span (created on ICapPublisher.PublishAsync) + - segmentId: not null + spans: + - operationName: CAP/skyapm.plugin.test.cap/Publisher + parentSpanId: -1 + spanId: 0 + spanLayer: MQ + startTime: nq 0 + endTime: nq 0 + componentId: 3004 + isError: false + spanType: Entry + peer: localhost + skipAnalysis: false + tags: + - {key: mq.topic, value: skyapm.plugin.test.cap} + - {key: mq.broker, value: localhost} + logs: + - logEvent: + - {key: event, value: Event Publishing Start} + - logEvent: + - {key: message, value: CAP message publishing start...} + - logEvent: + - {key: event, value: Event Publishing End} + - logEvent: + - {key: message, value: not blank} + # CAP subscriber (message persistence) span + - segmentId: not null + spans: + - operationName: CAP/cap.queue.case.v1/skyapm.plugin.test.cap/Subscriber + parentSpanId: -1 + spanId: 0 + spanLayer: Database + startTime: nq 0 + endTime: nq 0 + componentId: 3004 + isError: false + spanType: Entry + peer: localhost + skipAnalysis: false + tags: + - {key: mq.topic, value: skyapm.plugin.test.cap} + - {key: mq.broker, value: localhost} + logs: + - logEvent: + - {key: event, value: Event Persistence Start} + - logEvent: + - {key: message, value: CAP message persistence start...} + - logEvent: + - {key: event, value: Event Persistence End} + - logEvent: + - {key: message, value: not blank} diff --git a/test/plugin/scenarios/cap-scenario/docker-compose.yml b/test/plugin/scenarios/cap-scenario/docker-compose.yml new file mode 100644 index 00000000..80a35151 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/docker-compose.yml @@ -0,0 +1,35 @@ +# CAP plugin-test stack: the dedicated mock collector (skyapm/mock-collector) + the +# scenario app (agent attached, reporting to mock-collector:19876). The runner sets +# TFM / CAP_VERSION / DOTNET_VERSION per matrix entry. Driven by ../../run.sh. +services: + mock-collector: + # Built by ../../run.sh from the pinned apache/skywalking-agent-test-tool commit + # (the published :latest is from 2019 and predances the V8 management protocol). + image: ${COLLECTOR_IMAGE:-skyapm/mock-collector:dev} + networks: [plugin] + expose: ["19876"] + ports: + - "12800:12800" # HTTP: /receiveData, /dataValidate, /healthCheck + healthcheck: + test: ["CMD", "bash", "-c", "echo > /dev/tcp/127.0.0.1/12800"] + interval: 5s + timeout: 5s + retries: 30 + + case: + build: + context: ../../../.. # repo root (so src/ + the submodule are in the build context) + dockerfile: test/plugin/scenarios/cap-scenario/Dockerfile + args: + DOTNET_VERSION: "${DOTNET_VERSION:-8.0}" + TFM: "${TFM:-net8.0}" + CAP_VERSION: "${CAP_VERSION:-8.0.0}" + depends_on: + mock-collector: + condition: service_healthy + networks: [plugin] + ports: + - "8080:8080" + +networks: + plugin: {} diff --git a/test/plugin/scenarios/cap-scenario/skyapm.json b/test/plugin/scenarios/cap-scenario/skyapm.json new file mode 100644 index 00000000..67aa6dc9 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/skyapm.json @@ -0,0 +1,13 @@ +{ + "SkyWalking": { + "ServiceName": "cap-scenario", + "Sampling": { + "SamplePer3Secs": -1 + }, + "Transport": { + "gRPC": { + "Servers": "mock-collector:19876" + } + } + } +} diff --git a/test/plugin/scenarios/cap-scenario/support-version.list b/test/plugin/scenarios/cap-scenario/support-version.list new file mode 100644 index 00000000..9cb18b06 --- /dev/null +++ b/test/plugin/scenarios/cap-scenario/support-version.list @@ -0,0 +1,10 @@ +# Library versions to test the CAP plugin against, one per line: " ". +# Following the SkyWalking convention, list only the LATEST PATCH of each supported MINOR. +# The plugin pins CAP 8.0.0 (net8) / 10.0.0 (net10) at compile time; thanks to NuGet +# unification the scenario runs the plugin against the version listed here, so a newer CAP +# than the plugin compiled against is genuinely exercised (this is what would catch #565). +net8.0 8.0.0 +# net10.0 10.0.0 — pending: CAP 10's in-memory queue dispatches the publish synchronously inside +# the HTTP request, so the publisher span is an Exit (with a ref) rather than the standalone Entry +# seen on CAP 8; this scenario's expectedData would need a net10 variant. (Validating net10 already +# surfaced + fixed the CapCarrierHeaderCollection duplicate-sw8 bug.) From 3a69941e5f106c14b0e7928cb280397d06ca513c Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 15:42:40 +0800 Subject: [PATCH 4/8] test(plugin): assert the CAP trace hierarchy, not just span presence expectedData now asserts that the CAP publisher and subscriber spans each carry a CrossProcess ref back to the GET:/case/cap entry span (parentSpanId 0) -- i.e. the publish/consume are traced from the request's context via the injected sw8 header, not as disconnected spans. Validated green on net8.0 / CAP 8.0.0. --- .../scenarios/cap-scenario/config/expectedData.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/plugin/scenarios/cap-scenario/config/expectedData.yaml b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml index e175f5f8..bc602f76 100644 --- a/test/plugin/scenarios/cap-scenario/config/expectedData.yaml +++ b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml @@ -23,6 +23,13 @@ segmentItems: tags: - {key: mq.topic, value: skyapm.plugin.test.cap} - {key: mq.broker, value: localhost} + # the publish is traced from the request's context: a cross-process ref back to the + # GET:/case/cap entry span (CAP dispatches the send on a background thread, so the + # publisher is a linked segment rather than a nested child). + refs: + - {parentEndpoint: GET:/case/cap, networkAddress: not null, refType: CrossProcess, + parentSpanId: 0, parentTraceSegmentId: not null, parentServiceInstance: not null, + parentService: cap-scenario, traceId: not null} logs: - logEvent: - {key: event, value: Event Publishing Start} @@ -49,6 +56,12 @@ segmentItems: tags: - {key: mq.topic, value: skyapm.plugin.test.cap} - {key: mq.broker, value: localhost} + # the consume is also traced from the request's context: a cross-process ref back to + # the GET:/case/cap entry span (carried via the injected sw8 header). + refs: + - {parentEndpoint: GET:/case/cap, networkAddress: not null, refType: CrossProcess, + parentSpanId: 0, parentTraceSegmentId: not null, parentServiceInstance: not null, + parentService: cap-scenario, traceId: not null} logs: - logEvent: - {key: event, value: Event Persistence Start} From 163b79746ce24ffe5fe045576a7f23dc6c4504fd Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 16:13:56 +0800 Subject: [PATCH 5/8] fix(cap): emit the publisher span as an exit, not an entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CAP publish is outbound, so its span must be an EXIT. CAP performs the actual send on a background dispatcher thread where the originating request's entry context isn't present, so BeforePublish previously fell back to creating an ENTRY span. Restore the context stashed at publish-store time so the publish is an exit span linked to the request, and inject that exit's sw8 so the consumer continues from the publisher. Result is the correct MQ chain: GET:/case/cap (Entry) └─ CAP/.../Publisher (Exit, ref-> request) -> CAP/.../Subscriber (Entry, ref-> publisher) expectedData updated to assert this (publisher Exit; consumer refs the publisher). --- .../CapDiagnosticProcessor.cs | 41 ++++++++----------- .../cap-scenario/config/expectedData.yaml | 13 +++--- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/SkyApm.Diagnostics.CAP/CapDiagnosticProcessor.cs b/src/SkyApm.Diagnostics.CAP/CapDiagnosticProcessor.cs index e28f83b1..6358a065 100644 --- a/src/SkyApm.Diagnostics.CAP/CapDiagnosticProcessor.cs +++ b/src/SkyApm.Diagnostics.CAP/CapDiagnosticProcessor.cs @@ -112,35 +112,30 @@ public void ErrorPublishStore([Object] CapEventDataPubStore eventData) [DiagnosticName(CapEvents.BeforePublish)] public void BeforePublish([Object] CapEventDataPubSend eventData) { - SegmentContext context = null; + var operationName = OperateNamePrefix + eventData.Operation + ProducerOperateNameSuffix; var host = eventData.BrokerAddress.Endpoint?.Replace("-1", "5672"); - if (_contexts.TryGetValue(eventData.TransportMessage.GetId(), out var ctx)) + + // A publish is outbound, so it must be an EXIT span. CAP performs the actual send on a + // background dispatcher thread where the originating request's entry context isn't present; + // restore the context stashed at publish-store time so the exit is created as a child of + // that request (and the consumer, extracting the injected sw8, continues from this exit). + var restoredEntry = false; + if (_entrySegmentContextAccessor.Context == null + && _contexts.TryGetValue(eventData.TransportMessage.GetId(), out var ctx) && ctx != null) { - var header = new CapCarrierHeaderCollection(eventData.TransportMessage); - if (_entrySegmentContextAccessor.Context == null) - { - _carrierPropagator.Inject(ctx, header); - context = _tracingContext.CreateEntrySegmentContext( - OperateNamePrefix + eventData.Operation + ProducerOperateNameSuffix, - header); - } - else - { - context = _tracingContext.CreateExitSegmentContext( - OperateNamePrefix + eventData.Operation + ProducerOperateNameSuffix, - host, header); - _carrierPropagator.Inject(context, new CapCarrierHeaderCollection(eventData.TransportMessage)); - } + _entrySegmentContextAccessor.Context = ctx; + restoredEntry = true; } - else + + // CreateExitSegmentContext injects the carrier (sw8) header so the consumer continues this trace. + var context = _tracingContext.CreateExitSegmentContext(operationName, host, + new CapCarrierHeaderCollection(eventData.TransportMessage)); + + if (restoredEntry) { - // may be come from retry loop - var carrierHeader = new CapCarrierHeaderCollection(eventData.TransportMessage); - var operationName = OperateNamePrefix + eventData.Operation + ProducerOperateNameSuffix; - context = _tracingContext.CreateEntrySegmentContext(operationName, carrierHeader); + _entrySegmentContextAccessor.Context = null; } - context.Span.SpanLayer = SpanLayer.MQ; context.Span.Component = GetComponent(eventData.BrokerAddress, true); context.Span.Peer = host; diff --git a/test/plugin/scenarios/cap-scenario/config/expectedData.yaml b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml index bc602f76..cf1b8e82 100644 --- a/test/plugin/scenarios/cap-scenario/config/expectedData.yaml +++ b/test/plugin/scenarios/cap-scenario/config/expectedData.yaml @@ -17,15 +17,14 @@ segmentItems: endTime: nq 0 componentId: 3004 isError: false - spanType: Entry + spanType: Exit peer: localhost skipAnalysis: false tags: - {key: mq.topic, value: skyapm.plugin.test.cap} - {key: mq.broker, value: localhost} - # the publish is traced from the request's context: a cross-process ref back to the - # GET:/case/cap entry span (CAP dispatches the send on a background thread, so the - # publisher is a linked segment rather than a nested child). + # the publish is an outbound EXIT span, a child of the request: a cross-process ref + # back to the GET:/case/cap entry span. refs: - {parentEndpoint: GET:/case/cap, networkAddress: not null, refType: CrossProcess, parentSpanId: 0, parentTraceSegmentId: not null, parentServiceInstance: not null, @@ -56,10 +55,10 @@ segmentItems: tags: - {key: mq.topic, value: skyapm.plugin.test.cap} - {key: mq.broker, value: localhost} - # the consume is also traced from the request's context: a cross-process ref back to - # the GET:/case/cap entry span (carried via the injected sw8 header). + # the consumer is an ENTRY span that continues from the publisher: a cross-process ref + # to the CAP publisher exit span (carried via the injected sw8 header). refs: - - {parentEndpoint: GET:/case/cap, networkAddress: not null, refType: CrossProcess, + - {parentEndpoint: CAP/skyapm.plugin.test.cap/Publisher, networkAddress: not null, refType: CrossProcess, parentSpanId: 0, parentTraceSegmentId: not null, parentServiceInstance: not null, parentService: cap-scenario, traceId: not null} logs: From ff98df94de0bbe1bcc68e76100146f85414e7ca6 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 16:13:56 +0800 Subject: [PATCH 6/8] test(plugin): retry-until-valid loop + per-plugin CI matrix - run.sh: segments flush asynchronously, so a single shot can validate before they all arrive (this failed in CI). Each attempt now fires traffic, waits 20s, then validates; retried up to 10 times. - plugin-test.yml: one matrix job PER scenario (auto-discovered) so results show per plugin in the checks UI, mirroring apache/skywalking-java's plugin-test. --- .github/workflows/plugin-test.yml | 44 ++++++++++++++++++++----------- test/plugin/run.sh | 39 +++++++++++++++------------ 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/.github/workflows/plugin-test.yml b/.github/workflows/plugin-test.yml index 5c8fa295..5f9e69c1 100644 --- a/.github/workflows/plugin-test.yml +++ b/.github/workflows/plugin-test.yml @@ -1,39 +1,51 @@ name: Plugin Test -# Per-plugin tests: run each plugin scenario against a dedicated mock collector and assert the -# REAL emitted segments, version-by-version (see test/plugin/). Heavier than unit tests (Docker + -# a Java collector build), so it runs only when the agent, a plugin, or the framework changes. +# Per-plugin tests: each plugin scenario runs against a dedicated mock collector and asserts the REAL +# emitted segments, version-by-version (see test/plugin/). One matrix job PER PLUGIN so results show +# per plugin in the checks UI (like apache/skywalking-java's plugin-test). Heavier than unit tests +# (Docker + a Java collector build), so it runs only when the agent, a plugin, or the framework changes. on: push: branches: [main] - paths: - - 'src/**' - - 'test/plugin/**' - - '.github/workflows/plugin-test.yml' + paths: ['src/**', 'test/plugin/**', '.github/workflows/plugin-test.yml'] pull_request: - paths: - - 'src/**' - - 'test/plugin/**' - - '.github/workflows/plugin-test.yml' + paths: ['src/**', 'test/plugin/**', '.github/workflows/plugin-test.yml'] concurrency: group: plugin-test-${{ github.ref }} cancel-in-progress: true jobs: + # Discover the scenarios so each becomes its own matrix entry (no need to edit this file per plugin). + discover: + runs-on: ubuntu-latest + outputs: + scenarios: ${{ steps.list.outputs.scenarios }} + steps: + - uses: actions/checkout@v4 + - id: list + run: | + scenarios=$(ls -d test/plugin/scenarios/*/ | xargs -n1 basename | jq -R . | jq -cs .) + echo "scenarios=$scenarios" >> "$GITHUB_OUTPUT" + echo "scenarios: $scenarios" + plugin-test: + needs: discover runs-on: ubuntu-latest timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + scenario: ${{ fromJSON(needs.discover.outputs.scenarios) }} + name: ${{ matrix.scenario }} steps: - uses: actions/checkout@v4 with: submodules: recursive # the protocol-v3 codegen source - - # Needed to build the mock collector from the pinned agent-test-tool commit. - uses: actions/setup-java@v4 with: distribution: temurin java-version: '11' - - - name: Run plugin scenarios (build collector + per-version build/run/validate) - run: bash test/plugin/run.sh + cache: maven # cache the mock-collector build deps + - name: Run ${{ matrix.scenario }} (build collector + per-version build/run/validate) + run: bash test/plugin/run.sh ${{ matrix.scenario }} diff --git a/test/plugin/run.sh b/test/plugin/run.sh index b136881d..7c0c25ba 100755 --- a/test/plugin/run.sh +++ b/test/plugin/run.sh @@ -51,23 +51,21 @@ ensure_collector() { rm -rf "$work" "$ctx" } -# Wait until the collector has received at least one segment, then validate. +# Segments flush to the collector asynchronously (CAP dispatches on background threads, and the +# agent batches), so a single shot can validate before everything arrives. Each attempt FIRES +# TRAFFIC, WAITS, then VALIDATES; we retry the whole loop until it passes or the attempts run out. +VALIDATE_ATTEMPTS="${VALIDATE_ATTEMPTS:-10}" +VALIDATE_WAIT="${VALIDATE_WAIT:-20}" + +drive_traffic() { + for _ in 1 2 3 4 5; do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/cap" >/dev/null 2>&1 || true; done +} + validate() { local expected="$1" - for _ in $(seq 1 30); do - if curl -sf --max-time 10 "$COLLECTOR_HTTP/receiveData" 2>/dev/null | grep -q "segmentId:"; then - break - fi - sleep 2 - done local code; code="$(curl -s -o /tmp/plugin-validate.out -w '%{http_code}' --max-time 20 \ -X POST --data-binary @"$expected" "$COLLECTOR_HTTP/dataValidate")" - if [ "$code" = "200" ]; then - return 0 - fi - log "VALIDATION FAILED (HTTP $code):" - sed -n '/cause by/,$p' /tmp/plugin-validate.out | head -40 - return 1 + [ "$code" = "200" ] } run_case() { @@ -79,13 +77,20 @@ run_case() { TFM="$tfm" CAP_VERSION="$ver" DOTNET_VERSION="$dotnet" \ docker compose up -d --build --quiet-pull ) - local rc=0 - # health-gate the app, then drive the entry endpoint a few times + # health-gate the app for _ in $(seq 1 30); do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/healthCheck" >/dev/null 2>&1 && break || sleep 2 done - for _ in $(seq 1 5); do curl -sf --max-time 5 "http://localhost:${ENTRY_PORT}/case/cap" >/dev/null 2>&1 || true; sleep 1; done - validate "$scen/config/expectedData.yaml" || rc=1 + local rc=1 + for attempt in $(seq 1 "$VALIDATE_ATTEMPTS"); do + drive_traffic + sleep "$VALIDATE_WAIT" + if validate "$scen/config/expectedData.yaml"; then rc=0; break; fi + log "attempt $attempt/$VALIDATE_ATTEMPTS: not valid yet, retrying" + done + if [ "$rc" -ne 0 ]; then + log "VALIDATION FAILED:"; sed -n '/cause by/,$p' /tmp/plugin-validate.out | head -40 + fi ( cd "$scen" && docker compose down -v >/dev/null 2>&1 || true ) return $rc } From a467fa57f17b176d8001a833a0a8620016fa4001 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 16:21:38 +0800 Subject: [PATCH 7/8] ci(plugin-test): drop setup-java maven cache (no pom.xml in this repo) --- .github/workflows/plugin-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/plugin-test.yml b/.github/workflows/plugin-test.yml index 5f9e69c1..8fcbd7ed 100644 --- a/.github/workflows/plugin-test.yml +++ b/.github/workflows/plugin-test.yml @@ -46,6 +46,5 @@ jobs: with: distribution: temurin java-version: '11' - cache: maven # cache the mock-collector build deps - name: Run ${{ matrix.scenario }} (build collector + per-version build/run/validate) run: bash test/plugin/run.sh ${{ matrix.scenario }} From 47c8f7d0930dcf1ac53c8eaa973d8ec6a94ac39a Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sun, 28 Jun 2026 17:10:53 +0800 Subject: [PATCH 8/8] test(plugin): enable net10/CAP-10.0.0 (validated) + require tests reflect real runs The publisher=exit fix made the CAP 10 trace shape identical to CAP 8, so net10.0 / CAP 10.0.0 now passes against the same expectedData -- validated by an actual run (run.sh: 2/2 passed). Enable it in support-version.list. CLAUDE.md: plugin test data must reflect an actual run (capture via the collector's /receiveData, validate via /dataValidate); never fabricate a trace, and only list a once it has actually been run and passed. --- CLAUDE.md | 5 +++++ .../scenarios/cap-scenario/support-version.list | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 09dac681..5160e8ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,11 @@ themes/hugo-... , hugo.toml, i18n/ docs site theme + config (.github/workflows - Config: `skyapm.json` (or env `SKYWALKING__...`, double-underscore). Agent no-ops if `ServiceName` empty or `SkyWalking:Enable=false`. Full reference: `content/docs/guides/skyapm_config.md`. - Plugins: a fixed default set is auto-registered; others are opt-in via the `AddSkyAPM(ext => …)` lambda. +- Plugin tests (`test/plugin/`): each scenario runs the **real** instrumented library against a dedicated + mock collector and asserts the **real emitted segments**, version-by-version (`run.sh`). Tests must reflect + an **actual run** — never fabricate or assume a trace: capture via the collector's `/receiveData`, validate + via `/dataValidate`, and base `config/expectedData.yaml` on that. Only list a ` ` in + `support-version.list` once it has actually been run and passed (don't declare versions you haven't validated). - The repo also pins per-TFM package versions via `Condition="'$(TargetFramework)' == 'netX'"` blocks. - Project is **independent** — reports to Apache SkyWalking but is not an ASF/SkyWalking sub-project. - Commits: do **not** add a `Co-Authored-By: Claude …` trailer (or any AI co-author trailer) to commit messages. diff --git a/test/plugin/scenarios/cap-scenario/support-version.list b/test/plugin/scenarios/cap-scenario/support-version.list index 9cb18b06..d7b47119 100644 --- a/test/plugin/scenarios/cap-scenario/support-version.list +++ b/test/plugin/scenarios/cap-scenario/support-version.list @@ -1,10 +1,8 @@ # Library versions to test the CAP plugin against, one per line: " ". # Following the SkyWalking convention, list only the LATEST PATCH of each supported MINOR. -# The plugin pins CAP 8.0.0 (net8) / 10.0.0 (net10) at compile time; thanks to NuGet -# unification the scenario runs the plugin against the version listed here, so a newer CAP -# than the plugin compiled against is genuinely exercised (this is what would catch #565). +# ONLY list a version once it has actually been run and validated against the mock collector +# (config/expectedData.yaml must reflect a real run, never an assumed shape) -- see CLAUDE.md. +# The plugin pins CAP 8.0.0 (net8) / 10.0.0 (net10) at compile time; NuGet unification runs the +# plugin against the version listed here, so a newer CAP than it compiled against is exercised. net8.0 8.0.0 -# net10.0 10.0.0 — pending: CAP 10's in-memory queue dispatches the publish synchronously inside -# the HTTP request, so the publisher span is an Exit (with a ref) rather than the standalone Entry -# seen on CAP 8; this scenario's expectedData would need a net10 variant. (Validating net10 already -# surfaced + fixed the CapCarrierHeaderCollection duplicate-sw8 bug.) +net10.0 10.0.0