From fb0e84f42fc7ecac195bbce8544ed090c5035454 Mon Sep 17 00:00:00 2001 From: Erwan LE TUTOUR Date: Tue, 22 Sep 2026 08:28:49 +0200 Subject: [PATCH 1/3] feat: expose assault metrics via Micrometer/Prometheus (#46) Add an optional quarkus-goblin-metrics module that surfaces the assault activity as Micrometer metrics on any backend: - new AssaultObserver SPI on the engine, notified on every recorded assault and on every active-state change (observers errors are never propagated); reused later by the OpenTelemetry and assertion work. - GoblinMetricsObserver bean registers goblin_active (functional gauge), goblin_assaults_total (counter, tags type+source) and goblin_latency_injected_seconds (latency histogram, tag source). - source tag derived from the history identifier: server, rest-client, webclient. Documentation: new metrics page, README sections, compatibility note and roadmap tick; integration tests cover the live Prometheus scrape. --- README.md | 21 ++ ROADMAP.md | 4 +- docs/modules/ROOT/pages/compatibility.adoc | 8 +- docs/modules/ROOT/pages/how-it-works.adoc | 11 + docs/modules/ROOT/pages/index.adoc | 1 + docs/modules/ROOT/pages/metrics.adoc | 81 ++++++ integration-tests/README.md | 1 + integration-tests/pom.xml | 5 + .../src/main/resources/application.properties | 1 + .../it/GoblinMetricsIntegrationTest.java | 237 ++++++++++++++++++ metrics/README.md | 35 +++ metrics/pom.xml | 59 +++++ .../goblin/metrics/GoblinMetricsObserver.java | 102 ++++++++ .../metrics/GoblinMetricsObserverTest.java | 104 ++++++++ pom.xml | 1 + .../io/quarkiverse/goblin/AssaultEngine.java | 29 +++ .../quarkiverse/goblin/AssaultObserver.java | 29 +++ 17 files changed, 726 insertions(+), 3 deletions(-) create mode 100644 docs/modules/ROOT/pages/metrics.adoc create mode 100644 integration-tests/src/test/java/io/quarkiverse/goblin/it/GoblinMetricsIntegrationTest.java create mode 100644 metrics/README.md create mode 100644 metrics/pom.xml create mode 100644 metrics/src/main/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserver.java create mode 100644 metrics/src/test/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserverTest.java create mode 100644 runtime/src/main/java/io/quarkiverse/goblin/AssaultObserver.java diff --git a/README.md b/README.md index 3cd0c81..78edff8 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Quarkus has excellent resilience primitives (MicroProfile Fault Tolerance, Mutin - **Response body injection** -- Truncate or inflate the response entity (`TRUNCATE` keeps the first N%, `INFLATE` pads it) to break strict JSON clients and length-validating consumers - **Response header injection** -- Set or remove headers on emitted responses (`SET` forces the value, replacing an existing header or adding it when absent; `REMOVE` deletes it when present) - **Client-side assaults** -- Inject latency and exceptions into outgoing MicroProfile / Quarkus REST Client calls (`quarkus-rest-client`) and Vert.x `WebClient` calls (`GoblinWebClient.enable(...)`, opt-in at client creation) +- **Metrics** -- Optional `quarkus-goblin-metrics` module exposing assault activity as Micrometer / Prometheus metrics (`goblin_assaults_total`, `goblin_latency_injected_seconds`, `goblin_active`, see the [Metrics guide](docs/modules/ROOT/pages/metrics.adoc)) - **Multiple types simultaneously** -- Enable latency + exception together for slow failure simulation - **Targeting** -- By package, by annotation, by percentage of requests - **Dev UI** -- Toggle assaults, edit config, view history -- all in real time @@ -98,6 +99,25 @@ quarkus.goblin.target.level=100 # quarkus.goblin.target.exclude-annotations=org.eclipse.microprofile.faulttolerance.Timeout ``` +## Metrics (optional) + +Add the `quarkus-goblin-metrics` module to expose the assault activity through Micrometer, so it shows up in your +Prometheus / Grafana dashboards: + +```xml + + io.quarkiverse.goblin + quarkus-goblin-metrics + ${goblin.version} + +``` + +Metrics are scraped at the standard Prometheus endpoint `/q/metrics`: + +- `goblin_assaults_total` -- counter of every fired assault, tagged by `type` and `source` (`server`, `rest-client`, `webclient`) +- `goblin_latency_injected_seconds` -- timer of the delays actually injected, tagged by `source` (sum/count/max; see the [guide](docs/modules/ROOT/pages/metrics.adoc) for histogram tuning) +- `goblin_active` -- gauge, `1` while the engine is active, `0` otherwise + ## Dev UI The Chaos Dashboard provides: @@ -124,6 +144,7 @@ All changes apply instantly with WARN logs in the console and are persisted to ` Each module carries its own README for contributors: - [runtime](runtime/README.md) -- the assault abstraction and how to add a new assault (the extension SPI) +- [metrics](metrics/README.md) -- optional Micrometer / Prometheus metrics for assault activity - [runtime-dev](runtime-dev/README.md) -- the Dev UI JSON-RPC backend (dev mode only) - [deployment](deployment/README.md) -- build steps, bean registration, and Dev UI wiring - [integration-tests](integration-tests/README.md) -- the `@QuarkusTest` suite and how to extend it diff --git a/ROADMAP.md b/ROADMAP.md index 508e2ac..3fdc171 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -51,8 +51,8 @@ Current status: **preview** (v0.2.0) ## v0.3.0 -- Dev UI & Observability -- [ ] **Micrometer/Prometheus metrics** - Expose assault counters and latency histograms via Micrometer so they appear in existing Prometheus/Grafana dashboards. Metrics: `goblin_assaults_total` (tagged by type), `goblin_latency_injected_seconds` (histogram), `goblin_active` (gauge). +- [x] **Micrometer/Prometheus metrics** + Expose assault counters and latency histograms via Micrometer so they appear in existing Prometheus/Grafana dashboards. Metrics: `goblin_assaults_total` (tagged by type), `goblin_latency_injected_seconds` (histogram), `goblin_active` (gauge). Delivered as the optional `quarkus-goblin-metrics` module (#46). - [ ] **OpenTelemetry tracing integration** Create an OTel span for each injected assault, with attributes for assault type, target method, and injected value. Link the assault span to the parent request span for end-to-end trace correlation. diff --git a/docs/modules/ROOT/pages/compatibility.adoc b/docs/modules/ROOT/pages/compatibility.adoc index 6d05747..2333158 100644 --- a/docs/modules/ROOT/pages/compatibility.adoc +++ b/docs/modules/ROOT/pages/compatibility.adoc @@ -24,4 +24,10 @@ Client-side assaults are implemented as a standard JAX-RS `ClientRequestFilter` == Vert.x WebClient -The same client-side assaults apply to outgoing `io.vertx.ext.web.client.WebClient` calls, but Vert.x 4.x exposes no public interceptor hook on `WebClient`. Goblin therefore attaches its interceptor explicitly, at the point where the application creates its client, via `GoblinWebClient.enable(webClient)`. The interceptor relies on Vert.x's internal `WebClientInternal.addInterceptor` mechanism (the same one used by Vert.x's own `OAuth2WebClient`, `CachingWebClient` and `WebClientSession` decorators), so it works with any client backed by Vert.x's default `WebClientImpl`; a custom decorator that does not extend the base client fails fast with an `IllegalArgumentException`. A plain `WebClient` is protected by Goblin's opt-in default: without `enable(...)` it is never touched. \ No newline at end of file +The same client-side assaults apply to outgoing `io.vertx.ext.web.client.WebClient` calls, but Vert.x 4.x exposes no public interceptor hook on `WebClient`. Goblin therefore attaches its interceptor explicitly, at the point where the application creates its client, via `GoblinWebClient.enable(webClient)`. The interceptor relies on Vert.x's internal `WebClientInternal.addInterceptor` mechanism (the same one used by Vert.x's own `OAuth2WebClient`, `CachingWebClient` and `WebClientSession` decorators), so it works with any client backed by Vert.x's default `WebClientImpl`; a custom decorator that does not extend the base client fails fast with an `IllegalArgumentException`. A plain `WebClient` is protected by Goblin's opt-in default: without `enable(...)` it is never touched. + +== Micrometer / Prometheus + +Observability is opt-in through the `quarkus-goblin-metrics` module (see xref:metrics.adoc[]). It depends on +`quarkus-micrometer-registry-prometheus` and exposes the assault metrics on `/q/metrics`; it co-exists with any other +Micrometer-based monitoring and is inert when absent from the classpath. \ No newline at end of file diff --git a/docs/modules/ROOT/pages/how-it-works.adoc b/docs/modules/ROOT/pages/how-it-works.adoc index 09cd9e1..221185b 100644 --- a/docs/modules/ROOT/pages/how-it-works.adoc +++ b/docs/modules/ROOT/pages/how-it-works.adoc @@ -48,6 +48,17 @@ TIP: Add `.goblin-state.json` to your `.gitignore` to avoid committing local cha If the state file is corrupted or unreadable, Goblin logs a warning and falls back to the configuration from `application.properties`. +== Observability: the AssaultObserver SPI + +Every recorded assault and every active-state change is broadcast to the registered `AssaultObserver` beans +(`io.quarkiverse.goblin.AssaultObserver`, default no-op methods). Observers run synchronously on the request path: +the engine calls each bean and skips/logs any failing observer so observability can never break an assault. The engine +injects `Instance`, so any `@ApplicationScoped` implementation is picked up automatically; outside +the CDI container (plain unit tests) the notification simply does nothing. + +The optional `quarkus-goblin-metrics` module is the first consumer (see xref:metrics.adoc[]); OpenTelemetry tracing +and post-assault assertions will plug into the same hook. + == Adding your own assault type Goblin ships with four built-in assault types and lets you register your own. Implement the `io.quarkiverse.goblin.assault.Assault` interface, annotate the class `@ApplicationScoped`, and it is discovered automatically at build time -- no other wiring required: diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 06940f9..9c08d29 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -71,6 +71,7 @@ You should see a response time well above your normal baseline. * xref:configuration-reference.adoc[] -- every configuration key and the startup/Dev UI validation rules. * xref:dev-ui.adoc[] -- the Chaos Dashboard, the Assault History panel, and the Markdown report export. * xref:how-it-works.adoc[] -- JAX-RS filter mechanics, runtime config modification, state persistence, and adding your own assault type. +* xref:metrics.adoc[] -- optional Micrometer / Prometheus metrics for assault activity (`quarkus-goblin-metrics`). * xref:compatibility.adoc[] -- coexistence with MicroProfile Fault Tolerance and both JAX-RS runtimes. * xref:troubleshooting.adoc[] -- common issues and their fixes. * xref:end-to-end-example.adoc[] -- prove that your `@Timeout` and `@Fallback` actually work. diff --git a/docs/modules/ROOT/pages/metrics.adoc b/docs/modules/ROOT/pages/metrics.adoc new file mode 100644 index 0000000..fef07ab --- /dev/null +++ b/docs/modules/ROOT/pages/metrics.adoc @@ -0,0 +1,81 @@ += Metrics (Micrometer / Prometheus) + +:page-aliases: metrics + +:toc: left +:toclevels: 3 + +include::../partials/attributes.adoc[] + +Goblin can surface its assault activity as Micrometer metrics, so the chaos you inject shows up in your existing Prometheus / Grafana dashboards alongside your application's regular metrics. + +The integration is **optional** and lives in the `quarkus-goblin-metrics` module. Adding it to the classpath is enough: it observes the assaults through the engine's observer SPI and never alters the assault behavior. + +== Adding the dependency + +[source,xml] +---- + + io.quarkiverse.goblin + quarkus-goblin-metrics + ${goblin.version} + +---- + +The module brings in `quarkus-micrometer-registry-prometheus`, so `/q/metrics` is available immediately. Any other Micrometer backend (e.g. a custom `MeterRegistry`) picks up the same meters: the metrics are registered against the application's `MeterRegistry` bean. + +== Exposed metrics + +[cols="1,2,3"] +|=== +| Metric +| Description +| Tags + +| `goblin_assaults_total` +| Counter of every fired assault, incremented once per assault (including client-side ones). +| `type` (e.g. `latency`, `exception`, `http-status`, `response-body-truncate`) and `source` (`server`, `rest-client`, `webclient`) + +| `goblin_latency_injected_seconds` +| Timer of the delays actually injected (only incremented when latency was really applied). Exposed as a histogram: `_sum` / `_count` plus `_bucket` distribution series. Custom bucket boundaries can be set with `quarkus.micrometer.export.prometheus.prometheus.bucket-boundaries`. +| `source` + +| `goblin_active` +| Gauge reading `1` while the engine is active (the master toggle), `0` otherwise. +| +|=== + +`source` is derived from the recorded history identifier: server-side assaults (`SampleResource.hello`) are tagged `server`, outgoing REST Client calls tag `rest-client`, and outgoing Vert.x WebClient calls tag `webclient`. + +TIP: Reduction: summarize by `type` only with `sum(goblin_assaults_total) by (type)`; watch a specific source with `sum(rate(goblin_latency_injected_seconds_count[5m])) by (source)`. + +== Reading the latency histogram + +Micrometer's Prometheus export keeps `_sum` / `_count` **cumulative**, but `_max` (and any quantile you derive from the `_bucket` series) only reflects the **current scrape interval**. When no latency is injected in a given interval, `_max` therefore drops to `0` -- this is the standard Prometheus/Micrometer behavior, identical to Quarkus's own `http_server_requests_seconds_max`, not a broken probe. There is no `_min` line: the histogram replaces it, and a running minimum over the buckets requires PromQL. + +Percentile and min/max over a rolling window, e.g. per source: + +[source,promql] +---- +histogram_quantile(0.99, sum(rate(goblin_latency_injected_seconds_bucket[5m])) by (le, source)) # p99 of injected latency +min_over_time(max_over_time(goblin_latency_injected_seconds_max[10m])[10m:]) # persistent max if the interval gaps bother you +---- + +Because the histogram is enabled by default, on every scrape you get the `_bucket` series (`goblin_latency_injected_seconds_bucket{source="..."}`, with boundaries from ~1 ms up to 30 s), which is also what Grafana heatmaps expect. + +== Assault signals for dashboards + +Typical alerts: + +* `goblin_assaults_total > 0` -- chaos was injected (expect it while testing). +* `goblin_active == 0` -- the engine is off, none of the other signals are changing. +* Sudden spikes on `goblin_latency_injected_seconds` -- latency assault is degrading the injected delays. + +These metrics are also the foundation for the planned *post-assault assertions* (issue #50): declaring "inject 500 ms latency, expect a fallback to fire" becomes verifiable from the fault-tolerance signals combined with these metrics. + +== Internals + +The `quarkus-goblin-metrics` module is a plain jar containing one `@ApplicationScoped` bean +(`io.quarkiverse.goblin.metrics.GoblinMetricsObserver`) that implements the engine's `AssaultObserver` SPI +(`io.quarkiverse.goblin.AssaultObserver`, default no-op methods). The observer is fed by the engine on every recorded +assault and on every active-state change, so it has zero coupling to the JAX-RS filters or the client interceptor. \ No newline at end of file diff --git a/integration-tests/README.md b/integration-tests/README.md index 0a4b80d..09c2c04 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -18,6 +18,7 @@ REST Client calls with latency and exceptions, and `GoblinWebClientIntegrationTe | `GoblinIntegrationTest` | Endpoint basics, each assault type (latency, exception, HTTP status, dependency degradation, response body truncate/inflate, response header set/remove), target-level percentage behavior | | `GoblinClientAssaultIntegrationTest` | Client-side latency and exception on outgoing REST Client calls (incl. interplay with the target level and isolation from incoming-request assaults) | | `GoblinWebClientIntegrationTest` | Client-side latency and exception on outgoing Vert.x WebClient calls armed with `GoblinWebClient.enable(...)`, against the same 8081 test-port endpoint | +| `GoblinMetricsIntegrationTest` | The optional `quarkus-goblin-metrics` module end-to-end: meters registered with the expected names/tags on the live Prometheus registry (server, REST Client and WebClient sources), settled-step counter/timer values, and the `goblin.active` gauge | | `GoblinJsonRPCServiceTest` | The Dev UI JSON-RPC contract (status, toggles, editors, history, Markdown report, response body and response header config self-service) | | `AbstractPackageTargetingTest` + `ExcludePackageTargetingTest`, `IncludeNonMatchingPackageTargetingTest`, `IncludeMatchingPackageTargetingTest`, `ExcludeOverridesIncludeTargetingTest` | Package-based targeting via `include-packages` / `exclude-packages` | diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1c65de3..e35957f 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -29,6 +29,11 @@ ${project.version} provided + + io.quarkiverse.goblin + quarkus-goblin-metrics + ${project.version} + io.quarkus quarkus-rest diff --git a/integration-tests/src/main/resources/application.properties b/integration-tests/src/main/resources/application.properties index 581d652..d6d3bcd 100644 --- a/integration-tests/src/main/resources/application.properties +++ b/integration-tests/src/main/resources/application.properties @@ -3,4 +3,5 @@ quarkus.goblin.assault.type=latency quarkus.goblin.assault.latency.min-milliseconds=100 quarkus.goblin.assault.latency.max-milliseconds=500 quarkus.goblin.target.level=100 +quarkus.micrometer.export.prometheus.step=PT1S quarkus.rest-client.sample-client.url=http://localhost:8081 diff --git a/integration-tests/src/test/java/io/quarkiverse/goblin/it/GoblinMetricsIntegrationTest.java b/integration-tests/src/test/java/io/quarkiverse/goblin/it/GoblinMetricsIntegrationTest.java new file mode 100644 index 0000000..9ca522d --- /dev/null +++ b/integration-tests/src/test/java/io/quarkiverse/goblin/it/GoblinMetricsIntegrationTest.java @@ -0,0 +1,237 @@ +package io.quarkiverse.goblin.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import jakarta.inject.Inject; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.quarkiverse.goblin.AssaultEngine; +import io.quarkiverse.goblin.AssaultEngine.AssaultRecord; +import io.quarkiverse.goblin.MutableAssaultConfig; +import io.quarkiverse.goblin.metrics.GoblinMetricsObserver; +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; + +/** + * End-to-end coverage of the optional Micrometer integration: assault activity surfaced through the + * {@code quarkus-goblin-metrics} module is registered against the application's {@link MeterRegistry} with the + * expected names and tags, reflected on the live Prometheus backend and the {@code goblin.active} gauge follows the + * engine state. + *

+ * The test application runs with a short Prometheus step ({@code quarkus.micrometer.export.prometheus.step=PT1S}), + * because Prometheus-backed counters and timers only report the value of the latest settled step. The assertions poll + * the registry briefly until the increment is visible; precise accumulation behavior is covered on a plain registry in + * {@code GoblinMetricsObserverTest}. + */ +@QuarkusTest +public class GoblinMetricsIntegrationTest { + + @Inject + AssaultEngine engine; + + @Inject + MeterRegistry registry; + + @Inject + PrometheusMeterRegistry prometheusRegistry; + + @BeforeEach + void start() { + engine.setActive(true); + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setLatencyEnabled(false); + cfg.setExceptionEnabled(false); + cfg.setHttpStatusEnabled(false); + cfg.setDependencyDegradationEnabled(false); + cfg.setClientLatencyEnabled(false); + cfg.setClientExceptionEnabled(false); + cfg.setResponseBodyEnabled(false); + cfg.setResponseHeaderEnabled(false); + cfg.setLatencyMinMs(100); + cfg.setLatencyMaxMs(100); + cfg.setTargetLevel(100); + engine.clearHistory(); + } + + @Test + public void serverLatencyAssaultExposesMetrics() { + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setLatencyEnabled(true); + + RestAssured.given() + .get("/api/hello") + .then() + .statusCode(200); + + assertHistory("latency", p -> p.equals("SampleResource.hello")); + awaitCounter("latency", "server"); + awaitTimerCount("server"); + assertEquals(1, activeGauge(), "engine must be active during the assault"); + } + + @Test + public void webClientAssaultExposesMetrics() { + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setClientLatencyEnabled(true); + + RestAssured.given() + .get("/api/web-proxy") + .then() + .statusCode(200); + + assertHistory("latency", p -> p.startsWith("WebClient")); + awaitCounter("latency", "webclient"); + awaitTimerCount("webclient"); + } + + @Test + public void restClientAssaultExposesMetrics() { + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setClientLatencyEnabled(true); + + RestAssured.given() + .get("/api/proxy") + .then() + .statusCode(200); + + assertHistory("latency", p -> p.startsWith("REST-Client")); + awaitCounter("latency", "rest-client"); + awaitTimerCount("rest-client"); + } + + @Test + public void exceptionAssaultIsCounted() { + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setClientExceptionEnabled(true); + cfg.setExceptionType("java.lang.IllegalStateException"); + cfg.setExceptionMessage("downstream is in trouble"); + + RestAssured.given() + .get("/api/web-proxy") + .then() + .statusCode(500); + + assertHistory("exception", p -> p.startsWith("WebClient")); + awaitCounter("exception", "webclient"); + } + + @Test + public void latencyTimerIsExportedAsPrometheusHistogram() { + MutableAssaultConfig cfg = engine.getMutableConfig(); + cfg.setLatencyEnabled(true); + + RestAssured.given() + .get("/api/hello") + .then() + .statusCode(200); + + assertHistory("latency", p -> p.equals("SampleResource.hello")); + awaitTimerCount("server"); + + String scrape = awaitScrapeContaining("goblin_latency_injected_seconds_bucket"); + assertTrue(scrape.contains("goblin_latency_injected_seconds_count{source=\"server\""), + "expected a cumulative count line in the scrape: " + scrape); + var infBucket = java.util.regex.Pattern + .compile("goblin_latency_injected_seconds_bucket\\{source=\"server\",le=\"\\+Inf\",} ([0-9.]+)") + .matcher(scrape); + assertTrue(infBucket.find() && Double.parseDouble(infBucket.group(1)) >= 1, + "expected at least one record in the +Inf bucket of the server latency histogram: " + scrape); + } + + @Test + public void engineDeactivationIsReflectedInGauge() { + engine.setActive(false); + assertEquals(0, activeGauge(), "deactivating the engine must set goblin.active to 0"); + + engine.setActive(true); + assertEquals(1, activeGauge(), "reactivating the engine must set goblin.active to 1"); + } + + private void assertHistory(String type, java.util.function.Predicate method) { + List history = engine.getHistory(); + assertTrue(history.stream().anyMatch(r -> r.type().equals(type) && method.test(r.method())), + "expected a '" + type + "' assault in the history, got: " + history); + } + + private void awaitCounter(String type, String source) { + assertNotNull(registry.find(GoblinMetricsObserver.TOTAL_METRIC) + .tag("type", type) + .tag("source", source) + .counter(), "no goblin.assaults.total counter registered for " + source + "/" + type); + long deadline = System.currentTimeMillis() + 3_000; + while (System.currentTimeMillis() < deadline) { + long count = totalCount(type, source); + if (count >= 1) { + return; + } + sleepQuietly(50); + } + assertEquals(1, totalCount(type, source), "goblin.assaults.total for " + source + "/" + type + " must be incremented"); + } + + private void awaitTimerCount(String source) { + assertNotNull(registry.find(GoblinMetricsObserver.LATENCY_METRIC) + .tag("source", source) + .timer(), "no goblin.latency.injected.seconds timer registered for " + source); + long deadline = System.currentTimeMillis() + 3_000; + while (System.currentTimeMillis() < deadline) { + long count = timerCount(source); + if (count >= 1) { + return; + } + sleepQuietly(50); + } + assertEquals(1, timerCount(source), "goblin.latency.injected.seconds for " + source + " must be recorded"); + } + + private long totalCount(String type, String source) { + Counter counter = registry.find(GoblinMetricsObserver.TOTAL_METRIC) + .tag("type", type) + .tag("source", source) + .counter(); + return counter == null ? 0 : (long) counter.count(); + } + + private long timerCount(String source) { + Timer timer = registry.find(GoblinMetricsObserver.LATENCY_METRIC) + .tag("source", source) + .timer(); + return timer == null ? 0 : timer.count(); + } + + private double activeGauge() { + assertNotNull(registry.find(GoblinMetricsObserver.ACTIVE_METRIC).gauge(), + "goblin.active gauge must be registered"); + return registry.get(GoblinMetricsObserver.ACTIVE_METRIC).gauge().value(); + } + + private String awaitScrapeContaining(String fragment) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline) { + String scrape = prometheusRegistry.scrape(); + if (scrape.contains(fragment)) { + return scrape; + } + sleepQuietly(50); + } + throw new AssertionError("the Prometheus scrape never contained: " + fragment); + } + + private static void sleepQuietly(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} \ No newline at end of file diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 0000000..5488e40 --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,35 @@ +# Goblin Metrics + +Optional Micrometer / Prometheus metrics for the Goblin chaos engineering extension. + +## What it does + +One `@ApplicationScoped` bean (`GoblinMetricsObserver`) implements the engine's `AssaultObserver` SPI and registers +three meters on the application's `MeterRegistry`: + +- `goblin.assaults.total` -- counter tagged by `type` + `source` (`server`, `rest-client`, `webclient`) +- `goblin.latency.injected.seconds` -- timer of the delays actually injected, tagged by `source` +- `goblin.active` -- functional gauge over `AssaultEngine.isActive()` + +The module depends on `quarkus-micrometer-registry-prometheus` so `/q/metrics` works out of the box; any other +Micrometer backend receives the same meters. + +## The AssaultObserver SPI + +`io.quarkiverse.goblin.AssaultObserver` (in `runtime`) is the notification hook the engine fires on every recorded +assault and active-state change. Micrometer metrics is its first consumer; OpenTelemetry tracing and post-assault +assertions will reuse it. Observers run on the request path -- implementations must not throw, and the engine guards +against a failing observer without breaking the assault. + +## Adding a new metric + +Add a meter in `GoblinMetricsObserver` and derive it from the `AssaultRecord` (method, type, latencyMs, timestamp) +or engine state. Keep the name and tag conventions: dots for Micrometer (Prometheus normalizes to `_`), lower-case +tag values. + +## Testing + +- `GoblinMetricsObserverTest` -- unit test on a `SimpleMeterRegistry` (immediate values). +- `GoblinMetricsIntegrationTest` (in `integration-tests`) -- end-to-end with a real Prometheus backend; read carefully: + Prometheus-backed counters/timers are step-based, so the suite asserts the settled step value (polling) and runs with + `quarkus.micrometer.export.prometheus.step=PT1S`. \ No newline at end of file diff --git a/metrics/pom.xml b/metrics/pom.xml new file mode 100644 index 0000000..720adbe --- /dev/null +++ b/metrics/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + io.quarkiverse.goblin + quarkus-goblin-parent + 999-SNAPSHOT + + + quarkus-goblin-metrics + Goblin - Metrics + Optional Micrometer / Prometheus metrics for the Goblin chaos engineering extension + + + 25 + 25 + UTF-8 + + + + + io.quarkiverse.goblin + quarkus-goblin + ${project.version} + + + io.quarkus + quarkus-micrometer-registry-prometheus + + + org.junit.jupiter + junit-jupiter + test + + + + + + + io.smallrye + jandex-maven-plugin + 3.6.0 + + + make-index + + jandex + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + \ No newline at end of file diff --git a/metrics/src/main/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserver.java b/metrics/src/main/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserver.java new file mode 100644 index 0000000..d3a7d49 --- /dev/null +++ b/metrics/src/main/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserver.java @@ -0,0 +1,102 @@ +package io.quarkiverse.goblin.metrics; + +import java.util.concurrent.TimeUnit; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.quarkiverse.goblin.AssaultEngine; +import io.quarkiverse.goblin.AssaultObserver; +import io.quarkus.runtime.StartupEvent; + +/** + * Optional Micrometer integration exposing the assault activity for Prometheus and other metric backends. + *

+ * The following metrics are registered against the application's {@link MeterRegistry} when the + * {@code quarkus-goblin-metrics} dependency is present: + *

    + *
  • {@code goblin.assaults.total} -- counter of every fired assault, tagged with its {@code type} and {@code source} + * ({@code server}, {@code rest-client}, {@code webclient});
    • + *
  • {@code goblin.latency.injected.seconds} -- timer of the delays actually injected, tagged with {@code source};
    • + *
  • {@code goblin.active} -- gauge that lazily mirrors {@link AssaultEngine#isActive()} (1 when active, 0 otherwise).
    • + *
+ * All metrics are derived from the {@link AssaultObserver} notifications fired by the engine, so this module only needs + * to be on the classpath -- it never alters the assault behavior. + */ +@ApplicationScoped +public class GoblinMetricsObserver implements AssaultObserver { + + public static final String TOTAL_METRIC = "goblin.assaults.total"; + public static final String LATENCY_METRIC = "goblin.latency.injected.seconds"; + public static final String ACTIVE_METRIC = "goblin.active"; + public static final String TAG_TYPE = "type"; + public static final String TAG_SOURCE = "source"; + static final String SOURCE_SERVER = "server"; + static final String SOURCE_REST_CLIENT = "rest-client"; + static final String SOURCE_WEB_CLIENT = "webclient"; + + private final MeterRegistry registry; + private final AssaultEngine engine; + + /** + * Creates the observer and registers the active-state gauge as a functional gauge over the shared + * {@link AssaultEngine}, so its value always reflects the current engine state without any bookkeeping. + * + * @param registry the application's {@link MeterRegistry} + * @param engine the shared {@link AssaultEngine} + */ + public GoblinMetricsObserver(MeterRegistry registry, AssaultEngine engine) { + this.registry = registry; + this.engine = engine; + Gauge.builder(ACTIVE_METRIC, engine, e -> e.isActive() ? 1.0 : 0.0).register(registry); + } + + void init(@Observes StartupEvent event) { + // The gauge is functional, so there is nothing to initialise; this observer method merely anchors the bean in + // the application context (a bean declaring observer methods is never pruned as unused). + } + + @Override + public void onActiveChange(boolean active) { + // The gauge reads the engine state lazily. + } + + @Override + public void onAssault(AssaultEngine.AssaultRecord record) { + String source = sourceOf(record.method()); + Counter.builder(TOTAL_METRIC) + .tags(TAG_TYPE, record.type(), TAG_SOURCE, source) + .register(registry) + .increment(); + if (record.latencyMs() > 0) { + Timer.builder(LATENCY_METRIC) + .tags(TAG_SOURCE, source) + .publishPercentileHistogram() + .register(registry) + .record(record.latencyMs(), TimeUnit.MILLISECONDS); + } + } + + /** + * Derives the assault source tag from the history identifier produced by the engine. + * + * @param method the history identifier (e.g. {@code "SampleResource.hello"}, {@code "REST-Client GET ..."}, + * {@code "WebClient GET ..."}) + * @return the source tag value + */ + static String sourceOf(String method) { + if (method != null) { + if (method.startsWith("REST-Client ")) { + return SOURCE_REST_CLIENT; + } + if (method.startsWith("WebClient ")) { + return SOURCE_WEB_CLIENT; + } + } + return SOURCE_SERVER; + } +} \ No newline at end of file diff --git a/metrics/src/test/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserverTest.java b/metrics/src/test/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserverTest.java new file mode 100644 index 0000000..ae4b23d --- /dev/null +++ b/metrics/src/test/java/io/quarkiverse/goblin/metrics/GoblinMetricsObserverTest.java @@ -0,0 +1,104 @@ +package io.quarkiverse.goblin.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import io.quarkiverse.goblin.AssaultEngine; + +class GoblinMetricsObserverTest { + + private final MeterRegistry registry = new SimpleMeterRegistry(); + private final AssaultEngine engine = new AssaultEngine(); + private final GoblinMetricsObserver observer = new GoblinMetricsObserver(registry, engine); + + @Test + void activeGaugeFollowsEngineState() { + engine.setActive(true); + assertEquals(1, registry.get(GoblinMetricsObserver.ACTIVE_METRIC).gauge().value()); + + engine.setActive(false); + assertEquals(0, registry.get(GoblinMetricsObserver.ACTIVE_METRIC).gauge().value()); + } + + @Test + void serverLatencyAssaultIsCountedAndTimed() { + observer.onAssault(record("SampleResource.hello", "latency", 250)); + + assertEquals(1, totalCount("latency", "server")); + assertNull(totalCounter("latency", "rest-client")); + assertNull(totalCounter("latency", "webclient")); + + Timer timer = registry.find(GoblinMetricsObserver.LATENCY_METRIC) + .tag("source", "server").timer(); + assertNotNull(timer); + assertEquals(1, timer.count()); + assertEquals(0.25, timer.totalTime(TimeUnit.SECONDS), 0.001); + } + + @Test + void nonLatencyAssaultIsCountedButNotTimed() { + observer.onAssault(record("SampleResource.hello", "http-status", 0)); + + assertEquals(1, totalCount("http-status", "server")); + assertNull(registry.find(GoblinMetricsObserver.LATENCY_METRIC).tag("source", "server").timer()); + } + + @Test + void restClientSourceIsDetected() { + observer.onAssault(record("REST-Client GET http://localhost:8081/api/hello", "latency", 100)); + + assertEquals(1, totalCount("latency", "rest-client")); + Timer timer = registry.find(GoblinMetricsObserver.LATENCY_METRIC) + .tag("source", "rest-client").timer(); + assertNotNull(timer); + assertEquals(1, timer.count()); + } + + @Test + void webClientSourceIsDetected() { + observer.onAssault(record("WebClient GET http://localhost:8081/api/hello", "latency", 100)); + + assertEquals(1, totalCount("latency", "webclient")); + } + + @Test + void unknownMethodFallsBackToServer() { + observer.onAssault(record(null, "exception", 0)); + observer.onAssault(record("", "exception", 0)); + + assertEquals(2, totalCount("exception", "server")); + } + + @Test + void sourceDetectionIsUnitTestable() { + assertEquals("server", GoblinMetricsObserver.sourceOf("SampleResource.hello")); + assertEquals("server", GoblinMetricsObserver.sourceOf("")); + assertEquals("rest-client", GoblinMetricsObserver.sourceOf("REST-Client GET http://x")); + assertEquals("webclient", GoblinMetricsObserver.sourceOf("WebClient GET http://x")); + } + + private AssaultEngine.AssaultRecord record(String method, String type, long latencyMs) { + return new AssaultEngine.AssaultRecord(method, type, System.currentTimeMillis(), latencyMs, "snapshot"); + } + + private double totalCount(String type, String source) { + var counter = totalCounter(type, source); + assertNotNull(counter, "expected a goblin.assaults.total counter for " + source + "/" + type); + return counter.count(); + } + + private io.micrometer.core.instrument.Counter totalCounter(String type, String source) { + return registry.find(GoblinMetricsObserver.TOTAL_METRIC) + .tag("type", type) + .tag("source", source) + .counter(); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6757e20..0f8261a 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ runtime runtime-dev deployment + metrics integration-tests docs diff --git a/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java b/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java index 018a569..046a71c 100644 --- a/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java +++ b/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java @@ -6,9 +6,12 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; import org.jboss.logging.Logger; @@ -29,6 +32,9 @@ public class AssaultEngine { private final ConcurrentHashMap assaultCounts = new ConcurrentHashMap<>(); private volatile long countersSinceEpoch = System.currentTimeMillis(); + @Inject + Instance observers; + public static void setStaticConfig(GoblinConfig config) { staticConfig = config; } @@ -78,6 +84,7 @@ public boolean isActive() { public void setActive(boolean active) { this.active = active; + notifyObservers(observer -> observer.onActiveChange(active)); } public boolean shouldAssault() { @@ -151,6 +158,28 @@ public void recordAssault(String method, String type, long latencyMs) { } totalAssaultCount.incrementAndGet(); assaultCounts.computeIfAbsent(type, k -> new AtomicLong()).incrementAndGet(); + notifyObservers(observer -> observer.onAssault(record)); + } + + /** + * Notifies every registered {@link AssaultObserver} of an assault or engine state change. Observers run on the + * request path, so a failing observer is logged and skipped rather than propagated: observability must never break + * an assault. Outside the CDI container (plain unit test, no injected observers) the notification is a no-op. + * + * @param action the notification to broadcast to each observer + */ + private void notifyObservers(Consumer action) { + Instance current = observers; + if (current == null) { + return; + } + for (AssaultObserver observer : current) { + try { + action.accept(observer); + } catch (RuntimeException e) { + LOG.debugf("Goblin: assault observer ignored the notification: %s", e.getMessage()); + } + } } /** diff --git a/runtime/src/main/java/io/quarkiverse/goblin/AssaultObserver.java b/runtime/src/main/java/io/quarkiverse/goblin/AssaultObserver.java new file mode 100644 index 0000000..8440703 --- /dev/null +++ b/runtime/src/main/java/io/quarkiverse/goblin/AssaultObserver.java @@ -0,0 +1,29 @@ +package io.quarkiverse.goblin; + +/** + * Observability hook fired by the {@link AssaultEngine} whenever an assault is recorded or the engine is activated or + * deactivated. Implementations are discovered through the CDI container and notified synchronously -- they must never + * throw, as the notification happens on the request path. + *

+ * This SPI is the integration point for optional modules such as Micrometer metrics, OpenTelemetry tracing or + * post-assault assertions. Every method has a default no-op implementation so a partial implementation is trivially + * safe to register. + */ +public interface AssaultObserver { + + /** + * Called after an assault has been recorded in the engine history. + * + * @param record the assault record that was just added + */ + default void onAssault(AssaultEngine.AssaultRecord record) { + } + + /** + * Called whenever the engine is activated or deactivated. + * + * @param active the new engine state + */ + default void onActiveChange(boolean active) { + } +} \ No newline at end of file From 1c11be54442be3086d5100e8136b2df7606cad10 Mon Sep 17 00:00:00 2001 From: Erwan LE TUTOUR Date: Tue, 22 Sep 2026 08:30:25 +0200 Subject: [PATCH 2/3] docs: list Micrometer metrics in the 0.3.0 release notes --- docs/modules/ROOT/pages/release-notes.adoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/modules/ROOT/pages/release-notes.adoc b/docs/modules/ROOT/pages/release-notes.adoc index 0223e7e..30d2eb3 100644 --- a/docs/modules/ROOT/pages/release-notes.adoc +++ b/docs/modules/ROOT/pages/release-notes.adoc @@ -2,6 +2,21 @@ include::../partials/attributes.adoc[] +[[version-0-3]] +== 0.3.x + +[cols="1,1"] +|=== +|Version |Date + +|0.3.0 +|In development +|=== + +=== New Features + +* Assault metrics and latency histograms via Micrometer / Prometheus (https://github.com/quarkiverse/quarkus-goblin/pull/51[#51]). + [[version-0-2]] == 0.2.x From 4d21980e18c16c2ce181bf5b75275cb49fb32f99 Mon Sep 17 00:00:00 2001 From: Erwan LE TUTOUR Date: Tue, 22 Sep 2026 08:40:53 +0200 Subject: [PATCH 3/3] test: prove the observer failure-isolation contract Add dedicated engine unit tests showing that an observer throwing on notification never breaks recordAssault or setActive, and that healthy observers still receive their notifications alongside failing ones. Also clarify in the metrics docs that Micrometer names keep dots while the Prometheus export sanitizes them to underscores. --- docs/modules/ROOT/pages/metrics.adoc | 2 + .../io/quarkiverse/goblin/AssaultEngine.java | 10 ++ .../quarkiverse/goblin/AssaultEngineTest.java | 143 ++++++++++++++++++ 3 files changed, 155 insertions(+) diff --git a/docs/modules/ROOT/pages/metrics.adoc b/docs/modules/ROOT/pages/metrics.adoc index fef07ab..906e3b6 100644 --- a/docs/modules/ROOT/pages/metrics.adoc +++ b/docs/modules/ROOT/pages/metrics.adoc @@ -45,6 +45,8 @@ The module brings in `quarkus-micrometer-registry-prometheus`, so `/q/metrics` i | |=== +NOTE: Meter names are declared with Micrometer's dotted convention (e.g. `goblin.assaults.total`), but the Prometheus export sanitizes them to underscores -- `goblin_assaults_total` in `/q/metrics`. Other backends (JVM, statsd, ...) keep the dotted names, so write PromQL against the underscored form and refer to the dotted names when consuming the tags/names elsewhere. + `source` is derived from the recorded history identifier: server-side assaults (`SampleResource.hello`) are tagged `server`, outgoing REST Client calls tag `rest-client`, and outgoing Vert.x WebClient calls tag `webclient`. TIP: Reduction: summarize by `type` only with `sum(goblin_assaults_total) by (type)`; watch a specific source with `sum(rate(goblin_latency_injected_seconds_count[5m])) by (source)`. diff --git a/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java b/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java index 046a71c..7ef6218 100644 --- a/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java +++ b/runtime/src/main/java/io/quarkiverse/goblin/AssaultEngine.java @@ -137,6 +137,16 @@ void setMutableConfigForTests(MutableAssaultConfig config) { this.mutableConfig = config; } + /** + * Installs the observers used by {@link #recordAssault(String, String, long)} and {@link #setActive(boolean)}. + * Package-private for unit tests; the production lifecycle relies on CDI injection of the {@code observers} field. + * + * @param observers the observers to notify + */ + void setObserversForTests(Instance observers) { + this.observers = observers; + } + public List getHistory() { return List.copyOf(history); } diff --git a/runtime/src/test/java/io/quarkiverse/goblin/AssaultEngineTest.java b/runtime/src/test/java/io/quarkiverse/goblin/AssaultEngineTest.java index c3add85..5865808 100644 --- a/runtime/src/test/java/io/quarkiverse/goblin/AssaultEngineTest.java +++ b/runtime/src/test/java/io/quarkiverse/goblin/AssaultEngineTest.java @@ -2,11 +2,16 @@ import static org.junit.jupiter.api.Assertions.*; +import java.lang.annotation.Annotation; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.util.TypeLiteral; + import org.junit.jupiter.api.Test; class AssaultEngineTest { @@ -199,4 +204,142 @@ void shouldAssaultClientFalseWhenServerToggleOnlyAndLevel100() throws Exception assertFalse(engine.shouldAssaultClient()); } + + @Test + void failingObserverDoesNotBreakTheAssaultFlow() { + AssaultEngine engine = new AssaultEngine(); + List received = new ArrayList<>(); + AssaultObserver failing = new AssaultObserver() { + @Override + public void onAssault(AssaultEngine.AssaultRecord record) { + throw new RuntimeException("observer exploded"); + } + }; + AssaultObserver healthy = new AssaultObserver() { + @Override + public void onAssault(AssaultEngine.AssaultRecord record) { + received.add(record); + } + }; + engine.setObserversForTests(new FakeInstance(failing, healthy)); + + assertDoesNotThrow(() -> engine.recordAssault("hello", "latency", 150)); + + assertEquals(1, engine.getHistory().size(), "the assault must be recorded despite the failing observer"); + assertEquals(1, engine.getTotalAssaultCount(), "the counter must be incremented despite the failing observer"); + assertEquals(1, received.size(), "healthy observers must still be notified alongside the failing one"); + assertEquals("hello", received.get(0).method()); + assertEquals(150, received.get(0).latencyMs()); + + assertDoesNotThrow(() -> engine.setActive(false)); + assertFalse(engine.isActive(), "the state change must apply despite the failing observer"); + } + + @Test + void failingObserversDoNotBlockFullyFailingNotifications() { + AssaultEngine engine = new AssaultEngine(); + AssaultObserver failing = new AssaultObserver() { + @Override + public void onAssault(AssaultEngine.AssaultRecord record) { + throw new RuntimeException("onAssault exploded"); + } + + @Override + public void onActiveChange(boolean active) { + throw new RuntimeException("onActiveChange exploded"); + } + }; + engine.setObserversForTests(new FakeInstance(failing)); + + assertDoesNotThrow(() -> engine.recordAssault("hello", "exception")); + assertDoesNotThrow(() -> engine.setActive(true)); + + assertEquals(1, engine.getHistory().size()); + assertTrue(engine.isActive()); + } + + private static final class FakeInstance implements Instance { + + private final List observers; + + FakeInstance(AssaultObserver... observers) { + this.observers = List.of(observers); + } + + @Override + public AssaultObserver get() { + return observers.get(0); + } + + @Override + public Instance select(Annotation... qualifiers) { + return this; + } + + @Override + public Instance select(Class subtype, Annotation... qualifiers) { + return (Instance) (Instance) FakeInstance.this; + } + + @Override + public Instance select(TypeLiteral subtype, Annotation... qualifiers) { + return (Instance) (Instance) FakeInstance.this; + } + + @Override + public boolean isUnsatisfied() { + return observers.isEmpty(); + } + + @Override + public boolean isAmbiguous() { + return false; + } + + @Override + public void destroy(AssaultObserver instance) { + } + + @Override + public Instance.Handle getHandle() { + return new FakeHandle(observers.get(0)); + } + + @Override + public Iterable> handles() { + return (Iterable>) (Iterable) observers.stream().map(FakeHandle::new).toList(); + } + + @Override + public Iterator iterator() { + return observers.iterator(); + } + } + + private static final class FakeHandle implements Instance.Handle { + + private final AssaultObserver observer; + + FakeHandle(AssaultObserver observer) { + this.observer = observer; + } + + @Override + public AssaultObserver get() { + return observer; + } + + @Override + public jakarta.enterprise.inject.spi.Bean getBean() { + return null; + } + + @Override + public void destroy() { + } + + @Override + public void close() { + } + } }