feat: production-ready 1.0.0

- Fix drain_loop recursion bug (only slept 500ms, never recursed)
- Restructure shutdown: drain in-flight before deleting bulkheads
- Drain monitors bulkhead occupancy, exits early when idle
- Emit gate_not_ready telemetry event on mark_not_ready
- Add uptime_seconds and node to health report VM info
- Add gate_enabled config (false skips startup gating for dev/test)
- Add health_severity config (critical returns 503 when unhealthy)
- Add drain_poll_interval config (configurable drain polling)
- Add unknown config key warnings on startup
- Convert all binary strings to sigil syntax
- Fix unused logger.hrl, catch→try/catch, direct fun reference
- Add xref ignores for public API exports
- Suppress atom exhaustion warnings (bounded dep set)

New test suites (29 new tests, 58 total):
- nova_resilience_shutdown_SUITE (7 tests)
- nova_resilience_gate_SUITE (9 tests)
- nova_resilience_deadline_plugin_SUITE (7 tests)
- nova_resilience_telemetry_SUITE (5 tests)

New guides:
- Circuit Breakers & Bulkheads
- Deadline Propagation
- Telemetry

Updated guides: Getting Started, Adapters, Shutdown, README

Author: Taure

.gitignore

@@ -6,3 +6,4 @@ rebar.lock
 *.ez
 erl_crash.dump
 .rebar3/
+rebar3.crashdump

README.md

@@ -2,7 +2,19 @@
 
 Production-grade resilience patterns for [Nova](https://github.com/novaframework/nova) web applications.
 
-Bridges Nova and [Seki](https://github.com/Taure/seki) to provide dependency health checking, Kubernetes-ready probes, circuit breakers, bulkheads, and ordered graceful shutdown — all via declarative configuration.
+Bridges Nova and [Seki](https://github.com/Taure/seki) to provide dependency health checking, Kubernetes-ready probes, circuit breakers, bulkheads, deadline propagation, and ordered graceful shutdown — all via declarative configuration.
+
+## Features
+
+- **Health endpoints** — `/health`, `/ready`, `/live` for Kubernetes probes
+- **Startup gating** — traffic held until critical dependencies are healthy
+- **Circuit breakers** — stop calling failing dependencies, allow recovery
+- **Bulkheads** — limit concurrent requests per dependency
+- **Retry** — configurable retry with exponential backoff and jitter
+- **Deadline propagation** — per-request timeouts via headers or defaults
+- **Graceful shutdown** — ordered teardown with drain, priority groups, and LB coordination
+- **Telemetry** — events for all resilience operations (calls, breakers, shutdown, health)
+- **Pluggable adapters** — built-in support for pgo, kura, brod, or custom
 
 ## Quick start
 
@@ -16,7 +28,7 @@ Add to your deps:
 ]}.
 ```
 
-Add to your app's `applications`:
+Add to your `.app.src` applications:
 
 ```erlang
 {applications, [kernel, stdlib, nova, seki, nova_resilience]}.
@@ -30,7 +42,7 @@ Register health routes in your Nova config:
 ]}.
 ```
 
-Configure dependencies:
+Configure dependencies in `sys.config`:
 
 ```erlang
 {nova_resilience, [
@@ -40,33 +52,40 @@ Configure dependencies:
           adapter => pgo,
           pool => default,
           critical => true,
+          breaker => #{failure_threshold => 5, wait_duration => 30000},
+          bulkhead => #{max_concurrent => 25},
           shutdown_priority => 2}
     ]}
 ]}.
 ```
 
-That's it. Your app now has `/health`, `/ready`, and `/live` endpoints, automatic startup gating, and ordered shutdown.
+That's it. Your app now has `/health`, `/ready`, and `/live` endpoints, automatic startup gating, circuit breakers, bulkheads, and ordered shutdown.
 
-## What it does
+## How it works
 
 ### Startup
 
-1. App starts, nova_resilience provisions health checks for each dependency
-2. `/ready` returns **503** until all critical dependencies are healthy
-3. Kubernetes readiness probe detects this and holds traffic
-4. Once all critical deps respond, `/ready` returns **200** and traffic flows
+1. App starts, nova_resilience provisions seki primitives for each dependency
+2. Health checks run — `/ready` returns **503** until all critical deps are healthy
+3. Kubernetes readiness probe holds traffic until ready
+4. Once healthy, `/ready` returns **200** and traffic flows
 
 ### Running
 
-Execute calls through the resilience stack:
+Wrap calls to external dependencies through the resilience stack:
 
 ```erlang
 case nova_resilience:call(primary_db, fun() ->
-    pgo:query(<<"SELECT * FROM users WHERE id = $1">>, [Id])
+    pgo:query(~"SELECT * FROM users WHERE id = $1", [Id])
 end) of
-    {ok, #{rows := Rows}} -> {json, #{users => Rows}};
-    {error, circuit_open} -> {json, 503, #{}, #{error => <<"db unavailable">>}};
-    {error, bulkhead_full} -> {json, 503, #{}, #{error => <<"overloaded">>}}
+    {ok, #{rows := Rows}} ->
+        {json, #{users => Rows}};
+    {error, circuit_open} ->
+        {json, 503, #{}, #{error => ~"db unavailable"}};
+    {error, bulkhead_full} ->
+        {json, 503, #{}, #{error => ~"overloaded"}};
+    {error, deadline_exceeded} ->
+        {json, 504, #{}, #{error => ~"timeout"}}
 end.
 ```
 
@@ -75,42 +94,96 @@ end.
 On SIGTERM (or application stop):
 
 1. `/ready` immediately returns **503** (load balancer stops sending traffic)
-2. Waits `shutdown_delay` for in-flight LB health checks to propagate
-3. Tears down dependencies in `shutdown_priority` order
-4. Nova drains HTTP connections and stops
+2. Waits `shutdown_delay` for LB health checks to propagate
+3. Drains in-flight requests (monitors bulkhead occupancy)
+4. Tears down dependencies in `shutdown_priority` order
+5. Nova drains HTTP connections and stops
 
 No manual `prep_stop` calls needed — shutdown is fully automatic.
 
 ## Health endpoints
 
 | Endpoint | Purpose | Response |
 |----------|---------|----------|
-| `GET /health` | Full health report | `{"status":"healthy","dependencies":{...},"vm":{...}}` |
+| `GET /health` | Full diagnostic report | `{"status":"healthy","dependencies":{...},"vm":{...}}` |
 | `GET /ready` | Kubernetes readiness probe | 200 when ready, 503 when not |
 | `GET /live` | Kubernetes liveness probe | 200 if process is responsive |
 
+The `/health` endpoint returns per-dependency status with circuit breaker state, bulkhead occupancy, and VM metrics (memory, process count, run queue, uptime, node).
+
 ## Configuration
 
+### Application environment
+
 ```erlang
 {nova_resilience, [
-    {dependencies, [...]},          %% List of dependency configs
-    {health_check_interval, 10000}, %% ms between health checks
-    {vm_checks, true},              %% Include BEAM VM in health report
-    {gate_timeout, 30000},          %% Max ms to wait for deps on startup
-    {shutdown_delay, 5000},         %% ms to wait after marking not-ready
-    {shutdown_drain_timeout, 15000},%% Max ms to drain per priority group
-    {health_prefix, <<"">>}         %% Prefix for health routes
+    {dependencies, [...]},            %% List of dependency configs
+    {health_check_interval, 10000},   %% ms between health checks
+    {vm_checks, true},                %% Include BEAM VM info in health report
+    {gate_enabled, true},             %% false to skip startup gating (dev/test)
+    {gate_timeout, 30000},            %% Max ms to wait for deps on startup
+    {gate_check_interval, 1000},      %% ms between gate readiness checks
+    {health_severity, info},          %% critical: /health returns 503 when unhealthy
+    {shutdown_delay, 5000},           %% ms to wait after marking not-ready
+    {shutdown_drain_timeout, 15000},  %% Max ms to drain per priority group
+    {drain_poll_interval, 100},       %% ms between drain occupancy polls
+    {health_prefix, ~""}              %% Prefix for health routes (e.g. ~"/internal")
 ]}.
 ```
 
+Unknown config keys are logged as warnings on startup to catch typos.
+
+### Dependency config
+
+```erlang
+#{
+    name => atom(),                    %% Required — unique identifier
+    type => database | kafka | custom, %% Optional — infers adapter
+    adapter => pgo | kura | brod | module(), %% Optional — inferred from type
+    critical => boolean(),             %% Default: false — gates /ready
+    shutdown_priority => non_neg_integer(), %% Default: 10 — lower = first
+    default_timeout => pos_integer(),  %% Default deadline in ms
+    health_check => {module(), function()}, %% Override adapter health check
+
+    %% Circuit breaker
+    breaker => #{
+        failure_threshold => pos_integer(),
+        wait_duration => pos_integer(),
+        slow_call_duration => pos_integer(),
+        half_open_requests => pos_integer()
+    },
+
+    %% Concurrency limiter
+    bulkhead => #{
+        max_concurrent => pos_integer()
+    },
+
+    %% Retry with backoff
+    retry => #{
+        max_attempts => pos_integer(),
+        base_delay => non_neg_integer(),
+        max_delay => non_neg_integer()
+    }
+}
+```
+
 ## Built-in adapters
 
-| Type | Adapter | Auto health check |
-|------|---------|-------------------|
-| `database` | `pgo` (default) | `SELECT 1` via pgo |
-| `database` | `kura` | `SELECT 1` via kura repo |
-| `kafka` | `brod` | `brod:get_partitions_count/2` |
-| any | custom module | Implement `nova_resilience_adapter` behaviour |
+| Type | Adapter | Health check | Shutdown |
+|------|---------|-------------|----------|
+| `database` | `pgo` (default) | `SELECT 1` via pgo pool | no-op |
+| `database` | `kura` | `SELECT 1` via kura repo | no-op |
+| `kafka` | `brod` | `brod:get_partitions_count/2` | `brod:stop_client/1` |
+| any | custom module | `nova_resilience_adapter` behaviour | custom |
+
+## Guides
+
+- [Getting Started](guides/getting-started.md) — Installation and basic setup
+- [Circuit Breakers & Bulkheads](guides/resilience-patterns.md) — Protecting dependencies
+- [Deadline Propagation](guides/deadlines.md) — Per-request timeout budgets
+- [Adapters](guides/adapters.md) — Built-in and custom adapters
+- [Graceful Shutdown](guides/shutdown.md) — Ordered teardown and Kubernetes integration
+- [Telemetry](guides/telemetry.md) — Observability and monitoring
 
 ## License
 

guides/adapters.md

@@ -32,17 +32,26 @@ The `repo` field is required — it's the kura repo module that implements `kura
 
 ### brod (default for `kafka` type)
 
-Health check calls `brod:get_partitions_count/2` to verify broker connectivity.
+Health check calls `brod:get_partitions_count/2` to verify broker connectivity. Shutdown calls `brod:stop_client/1`.
 
 ```erlang
 #{name => events,
   type => kafka,
   client => my_brod_client,
-  topic => <<"events">>}
+  topic => ~"events"}
 ```
 
 Both `client` and `topic` are required.
 
+## Adapter resolution
+
+nova_resilience resolves adapters in this order:
+
+1. Explicit `adapter` field → use that module
+2. `type => database` → `nova_resilience_adapter_pgo`
+3. `type => kafka` → `nova_resilience_adapter_brod`
+4. No type or `type => custom` → no adapter (no automatic health check)
+
 ## Custom adapters
 
 Implement the `nova_resilience_adapter` behaviour:
@@ -54,16 +63,19 @@ Implement the `nova_resilience_adapter` behaviour:
 -export([health_check/1, wrap_call/2, shutdown/1]).
 
 health_check(#{pool := Pool}) ->
-    case eredis:q(Pool, [<<"PING">>]) of
-        {ok, <<"PONG">>} -> ok;
+    case eredis:q(Pool, [~"PING"]) of
+        {ok, ~"PONG"} -> ok;
         {error, Reason} -> {error, Reason}
     end.
 
 wrap_call(_Config, Fun) ->
+    %% Called around every nova_resilience:call/2,3
+    %% Use for logging, tracing, connection checkout, etc.
     Fun().
 
-shutdown(_Config) ->
-    ok.
+shutdown(#{pool := Pool}) ->
+    %% Called during graceful shutdown
+    eredis:stop(Pool).
 ```
 
 Then reference it in your config:
@@ -77,6 +89,41 @@ Then reference it in your config:
   shutdown_priority => 0}
 ```
 
+### Behaviour callbacks
+
+| Callback | Return | Purpose |
+|----------|--------|---------|
+| `health_check(Config)` | `ok \| {error, Reason}` | Called periodically to check dependency health |
+| `wrap_call(Config, Fun)` | `term()` | Wraps every call through the resilience stack |
+| `shutdown(Config)` | `ok` | Called during graceful shutdown |
+
+The `Config` parameter is the full dependency config map, so you can pass any fields you need.
+
+### wrap_call examples
+
+**Connection checkout:**
+
+```erlang
+wrap_call(#{pool := Pool}, Fun) ->
+    case pool:checkout(Pool) of
+        {ok, Conn} ->
+            try Fun()
+            after pool:checkin(Pool, Conn)
+            end;
+        {error, _} = Err ->
+            Err
+    end.
+```
+
+**Distributed tracing:**
+
+```erlang
+wrap_call(#{name := Name}, Fun) ->
+    otel_tracer:with_span(Name, #{kind => client}, fun(_Ctx) ->
+        Fun()
+    end).
+```
+
 ## Overriding health checks
 
 Any dependency can override the adapter's health check with a custom `{Module, Function}` tuple:
@@ -88,7 +135,23 @@ Any dependency can override the adapter's health check with a custom `{Module, F
   health_check => {my_app_health, deep_db_check}}
 ```
 
-The function must return `ok | {error, Reason}`.
+The function must take zero arguments and return `ok | {error, Reason}`:
+
+```erlang
+-module(my_app_health).
+-export([deep_db_check/0]).
+
+deep_db_check() ->
+    case pgo:query(~"SELECT count(*) FROM pg_stat_activity") of
+        #{rows := [[Count]]} when Count < 100 -> ok;
+        #{rows := [[Count]]} -> {error, {too_many_connections, Count}};
+        {error, Reason} -> {error, Reason}
+    end.
+```
+
+## Soft dependencies
+
+All built-in adapters are soft dependencies — they're only loaded when used. Your application only needs to include the adapter libraries it actually uses (pgo, kura, brod).
 
 ## Runtime registration
 
@@ -103,11 +166,9 @@ nova_resilience:register_dependency(inventory_service, #{
     breaker => #{failure_threshold => 5, wait_duration => 30000}
 }).
 
-%% Then use it
 nova_resilience:call(inventory_service, fun() ->
     httpc:request("http://inventory:8080/api/stock")
 end).
 
-%% Unregister when no longer needed
 nova_resilience:unregister_dependency(inventory_service).
 ```