Improve CollectEndorsementsView's handling of signers #1226 (#1895)

Signed-off-by: Angelo De Caro <adc@zurich.ibm.com>

Author: adecaro

docs/drivers/metrics.md

@@ -42,6 +42,21 @@ Caller  →  Metrics Wrapper  →  Concrete Driver Service
 This keeps business logic free of monitoring concerns and guarantees that any new driver
 automatically gets the same metrics by wrapping its services at construction time.
 
+### Pitfall: `LabelNames` must include `network`, `channel`, `namespace`
+
+`NewTMSProvider` (`token/core/common/metrics/provider.go`) wraps the underlying `Provider` so that
+*every* metric it creates is bound to fixed `network`/`channel`/`namespace` label values via
+`.With(...)` before the metric is returned — the caller never supplies these three values itself.
+
+Because of this, any `CounterOpts`/`GaugeOpts`/`HistogramOpts` passed to a TMS-scoped provider's
+`NewCounter`/`NewGauge`/`NewHistogram` **must declare `"network", "channel", "namespace"` as
+`LabelNames`**, even though nothing in the wrapper code ever passes values for them explicitly.
+Forgetting them creates a Prometheus vector with 0 label names while `NewTMSProvider` immediately
+calls `.With(...)` with 3 values, which panics at runtime ("inconsistent label cardinality") the
+first time the metric is used — not at registration time, so it can slip past a quick smoke test.
+This exact mistake shipped in `token/services/identity/metrics.go` and crashed the DVP/DLog
+integration suite inside `SignerRouter.Register`; see that file for the corrected `LabelNames`.
+
 ## Wrapped Services
 
 Five driver services are wrapped:

docs/imgs/storage_db.puml

@@ -8,21 +8,24 @@ package "Transaction & Audit Store (TTXDB / AuditDB)" {
     entity "Requests" as requests {
         * tx_id : TEXT <<PK, NOT NULL>>
         --
-        request : BYTEA
-        status : INT
-        status_message : TEXT
-        application_metadata : JSONB
-        public_metadata : JSONB
-        pp_hash : BYTEA
+        request : BYTEA <<NOT NULL>>
+        status : INT <<NOT NULL>>
+        status_message : TEXT <<NOT NULL>>
+        application_metadata : JSONB <<NOT NULL>>
+        public_metadata : JSONB <<NOT NULL>>
+        pp_hash : BYTEA <<NOT NULL>>
+        recovery_claimed_by : TEXT
+        recovery_claim_expires_at : TIMESTAMP
+        stored_at : TIMESTAMP <<NOT NULL>>
     }
 
     entity "Transactions" as txs {
         * id : CHAR(36) <<PK>>
         --
         tx_id : TEXT <<FK, NOT NULL>>
-        action_type : INT
-        sender_eid : TEXT
-        recipient_eid : TEXT
+        action_type : INT <<NOT NULL>>
+        sender_eid : TEXT <<NOT NULL>>
+        recipient_eid : TEXT <<NOT NULL>>
         token_type : TEXT <<NOT NULL>>
         amount : NUMERIC(78,0) <<NOT NULL>>
         stored_at : TIMESTAMP <<NOT NULL>>
@@ -32,7 +35,7 @@ package "Transaction & Audit Store (TTXDB / AuditDB)" {
         * id : CHAR(36) <<PK>>
         --
         tx_id : TEXT <<FK, NOT NULL>>
-        enrollment_id : TEXT
+        enrollment_id : TEXT <<NOT NULL>>
         token_type : TEXT <<NOT NULL>>
         amount : NUMERIC(78,0) <<NOT NULL>>
         stored_at : TIMESTAMP <<NOT NULL>>
@@ -41,10 +44,10 @@ package "Transaction & Audit Store (TTXDB / AuditDB)" {
     entity "Endorsements" as tx_ends {
         * id : CHAR(36) <<PK>>
         --
-        tx_id : TEXT
-        endorser : BYTEA
-        sigma : BYTEA
-        stored_at : TIMESTAMP
+        tx_id : TEXT <<NOT NULL>>
+        endorser : BYTEA <<NOT NULL>>
+        sigma : BYTEA <<NOT NULL>>
+        stored_at : TIMESTAMP <<NOT NULL>>
     }
 
     txs }o--|| requests : "tx_id"
@@ -56,11 +59,19 @@ package "Endorser Store (EndorserDB)" {
     entity "Validations" as req_vals {
         * tx_id : TEXT <<PK, NOT NULL>>
         --
-        request : BYTEA
-        metadata : BYTEA
-        pp_hash : BYTEA
+        request : BYTEA <<NOT NULL>>
+        metadata : BYTEA <<NOT NULL>>
+        pp_hash : BYTEA <<NOT NULL>>
+        status : INT <<NOT NULL>>
+        status_message : TEXT <<NOT NULL>>
         stored_at : TIMESTAMP <<NOT NULL>>
     }
+
+    note right of req_vals
+      Self-contained: no FK to Requests.
+      Endorser nodes track validation
+      status directly on this table.
+    end note
 }
 
 package "Token Store (TokenDB)" {
@@ -112,11 +123,11 @@ package "Token Store (TokenDB)" {
     }
 
     entity "TokenLocks" as tkn_locks {
-        * tx_id : TEXT <<PK, NOT NULL>>
-        * idx : INT <<PK, NOT NULL>>
+        * tx_id : TEXT <<PK, NOT NULL, FK>>
+        * idx : INT <<PK, NOT NULL, FK>>
         --
-        consumer_tx_id : TEXT
-        created_at : TIMESTAMP
+        consumer_tx_id : TEXT <<NOT NULL>>
+        created_at : TIMESTAMP <<NOT NULL>>
     }
 
     entity "TokenSKICleanups" as tkn_ski_cleanups {
@@ -129,7 +140,7 @@ package "Token Store (TokenDB)" {
 
     tkn_own }o--|| tokens : "(tx_id, idx)"
     tkn_crts }o--|| tokens : "(tx_id, idx)"
-    tkn_locks |o--o| tokens : "(tx_id, idx)"
+    tkn_locks }o--|| tokens : "(tx_id, idx)"
     tkn_ski_cleanups }o--|| tokens : "(tx_id, idx)"
 }
 
@@ -140,8 +151,9 @@ package "Wallet & Identity Store (WalletDB / IdentityDB)" {
         * role_id : INT <<PK, NOT NULL>>
         --
         meta : BYTEA
-        enrollment_id : TEXT
+        enrollment_id : TEXT <<NOT NULL>>
         created_at : TIMESTAMP
+        conf_id : TEXT <<FK, NOT NULL>>
     }
 
     entity "IdentityConfigurations" as id_cfgs {
@@ -151,36 +163,38 @@ package "Wallet & Identity Store (WalletDB / IdentityDB)" {
         --
         conf : BYTEA
         raw : BYTEA
+        conf_id : TEXT <<UNIQUE, NOT NULL>>
     }
 
     entity "IdentityInfo" as id_info {
         * identity_hash : TEXT <<PK, NOT NULL>>
         --
-        identity : BYTEA
-        identity_audit_info : BYTEA
+        identity : BYTEA <<NOT NULL>>
+        identity_audit_info : BYTEA <<NOT NULL>>
         token_metadata : BYTEA
         token_metadata_audit_info : BYTEA
     }
 
     entity "IdentitySigners" as id_signers {
         * identity_hash : TEXT <<PK, NOT NULL>>
         --
-        identity : BYTEA
+        identity : BYTEA <<NOT NULL>>
         info : BYTEA
     }
 
-    wallets }o--|| id_info : "identity_hash"
-    wallets }o--|| id_signers : "identity_hash"
+    wallets }o..o{ id_info : "identity_hash (no FK constraint)"
+    wallets }o..o{ id_signers : "identity_hash (no FK constraint)"
+    wallets }o--|| id_cfgs : "conf_id"
 }
 
 entity "KeyStore" as key_store {
     * key : TEXT <<PK, NOT NULL>>
     --
-    val : BYTEA
+    val : BYTEA <<NOT NULL>>
 }
 
 ' Relationships between packages
-tkn_own }o--|| wallets : "wallet_id"
-tokens }o--|| wallets : "owner_wallet_id"
+tkn_own }o..o{ wallets : "wallet_id (no FK constraint)"
+tokens }o..o{ wallets : "owner_wallet_id (no FK constraint)"
 
-@enduml
+@enduml

docs/services/identity.md

@@ -141,6 +141,27 @@ func (d *Base) NewWalletService(...) (*wallet.Service, error) {
 }
 ```
 
+### SignerRouter (conf_id-pinned fast path)
+
+`GetSigner`'s default resolution path is a **fallback deserializer**: a linear scan across every `KeyManager` registered under the identity's type, each probed with a cryptographic sign+verify to find the one that actually matches. `SignerRouter` (`token/services/identity/signer_router.go`) is an optional fast path that skips this scan-and-probe entirely: it resolves the `conf_id` an identity was bound under (via a `ConfIDResolver`) and dispatches straight to the single `KeyManager` registered for that `conf_id`.
+
+*   **Wiring**: a driver builds a `SignerRouter` with `identity.NewSignerRouter(m *Metrics)`, registers `KeyManager`s against their `conf_id` with `Register`, sets a `ConfIDResolver` with `SetConfIDResolver`, and attaches it to the `Provider` with `Provider.SetSignerRouter`. See `token/core/fabtoken/v1/driver/ws.go` and the zkatdlog equivalent.
+*   **Fallback semantics**: `Resolve` returns `ok=false` (never an error) whenever routing cannot be attempted (no resolver set, no `conf_id` mapping, no `KeyManager` registered for it) or the routed `KeyManager` itself fails — callers always fall back to the probing deserializer in that case, never treating it as a hard failure.
+*   **Probe-free deserialization**: when the registered `KeyManager` also implements `idriver.ProbeFreeSignerDeserializer`, `Resolve` calls `DeserializeSignerNoProbe` directly, skipping the cryptographic probe that the fallback path relies on to catch a mismatched `KeyManager`. This is only safe because the `conf_id` already pins the identity to exactly one `KeyManager`.
+
+#### Metrics
+
+`identity.Metrics` (`token/services/identity/metrics.go`) instruments both `Provider.GetSigner` and `SignerRouter`, sharing one `Metrics` instance built with `identity.NewMetrics(provider)` (a `nil` provider yields a `disabled.Provider`-backed noop):
+
+| Metric | Type | Labels | Purpose |
+|:-------|:-----|:-------|:--------|
+| `identity_signer_resolutions_total` | Counter | `network`, `channel`, `namespace`, `outcome` = `cache` \| `routed` \| `fallback` | How each `GetSigner` call was ultimately resolved. |
+| `identity_get_signer_duration_seconds` | Histogram | `network`, `channel`, `namespace`, `path` = `cache` \| `routed` \| `fallback` | `GetSigner` wall-clock time by resolution path; compares the latency saved by skipping the probe. |
+| `identity_signer_router_registrations_total` | Counter | `network`, `channel`, `namespace` | `conf_id`→`KeyManager` bindings registered with the `SignerRouter`. A near-zero count in production means routing is never populated and every call falls back. |
+| `identity_signer_router_no_probe_errors_total` | Counter | `network`, `channel`, `namespace` | Failures of the probe-free deserialization path — since that path skips the cryptographic check, a non-zero count is worth investigating as a `conf_id` routing bug. |
+
+> **Note:** `provider` here is a `NewTMSProvider`-wrapped `Provider` (see [Driver Metrics](../drivers/metrics.md#pitfall-labelnames-must-include-network-channel-namespace)), which binds `network`/`channel`/`namespace` on every metric via `.With(...)` before returning it. Every `CounterOpts`/`HistogramOpts` above must therefore declare those three as `LabelNames` in addition to its own label(s), or the metric panics with "inconsistent label cardinality" on first use. This is exactly the bug that crashed the DVP/DLog integration suite in `SignerRouter.Register` before it was fixed.
+
 ## Identity Types
 
 The Identity Service leverages a wrapper called **TypedIdentity** to support various identity schemes uniformly. 

docs/services/storage.md

@@ -28,16 +28,15 @@ The [`ttxdb/auditdb`](./storage/ttxdb.md) manages the token request lifecycle an
 The following tables track the lifecycle of token requests from assembly to finality. 
 `AuditDB` uses the same schema but is isolated for compliance reporting.
 
-*   **Requests**: Tracks high-level token request state. Contains marshaled requests, current status (Pending, Confirmed, Deleted, Orphan), and application/public metadata.
+*   **Requests**: Tracks high-level token request state. Contains marshaled requests, current status (Pending, Confirmed, Deleted, Orphan), application/public metadata, and the `recovery_claimed_by`/`recovery_claim_expires_at` lease columns used by the Transaction Recovery Service to atomically claim batches of pending transactions.
 *   **Transactions**: Records individual actions (Issue, Transfer, Redeem) within a request, including sender/recipient IDs and amounts (stored as `NUMERIC(78, 0)`).
 *   **Movements**: Aggregates net value changes per enrollment ID (amounts stored as `NUMERIC(78, 0)`). Used to efficiently calculate balances and history.
 *   **Endorsements**: Collects digital signatures from participants and auditors required for transaction finality.
 
 ### Endorser Store (EndorserDB)
-The [`endorserdb`](./storage/endorserdb.md) manages validation records created during the token request endorsement process. It shares the physical database with TTXDB but provides a separate interface for validation-specific operations.
+The [`endorserdb`](./storage/endorserdb.md) manages validation records created during the token request endorsement process. It shares the physical database with TTXDB but owns its own, self-contained table — it does not write to the Requests table.
 
-*   **Validations**: Stores cryptographic validation metadata produced during the request verification phase. The Validations table is self-contained and stores the token request data directly (along with pp_hash) for efficient retrieval. When a validation record is created (typically by endorser nodes), it atomically creates entries in both the Requests table (for foreign key integrity and status tracking) and the Validations table (with embedded token request for direct access), ensuring that non-owner nodes can properly track and recover transactions.
-*   **Requests** (shared): The endorserdb uses the Requests table to track validation status. While the table is shared with TTXDB, the endorserdb interface provides methods specifically for managing validation-related status updates.
+*   **Validations**: Stores cryptographic validation metadata produced during the request verification phase. The table is self-contained: it stores the token request data, metadata, and pp_hash directly (along with its own `status`/`status_message` columns), so a validation record can be created and its status tracked without any foreign key into the Requests table. This lets non-owner (endorser) nodes track and recover transactions independently of the TTXDB/AuditDB Requests row for the same `tx_id`.
 
 ### Token Store (TokenDB)
 This store serves as the authoritative registry for all tokens (UTXOs) known to the node.
@@ -166,10 +165,9 @@ The `ttxdb` serves as the central repository for the lifecycle of token requests
 
 ### Endorser Store (EndorserDB)
 The `endorserdb` manages validation records for token requests during the endorsement process. It is used by the **Endorsement Service** to:
-*   **Validations**: Store validation metadata and token requests validated by endorser nodes.
-*   **Requests** (shared): Track the status of validated token requests.
+*   **Validations**: Store validation metadata, token requests, and their own status validated by endorser nodes, in a single self-contained table.
 
-The endorserdb shares the same physical database as ttxdb but provides a separate, focused interface for validation-specific operations, improving modularity and separation of concerns.
+The endorserdb shares the same physical database as ttxdb but owns its own table and interface for validation-specific operations, improving modularity and separation of concerns.
 
 ### Token Store (TokenDB)
 The `tokendb` is the registry for the current state of all tokens (UTXOs) known to the node. It is used by the **Selector Service** and **Vault Service** to:

docs/services/storage/endorserdb.md

@@ -9,6 +9,7 @@ The endorser database stores validation records that track token requests valida
 - Token request data
 - Validation metadata
 - Public parameters hash
+- Status and status message
 - Timestamp
 
 ## Architecture
@@ -116,30 +117,23 @@ status, message, err := store.GetStatus(ctx, txID)
 
 ## Database Schema
 
-The endorserdb uses two tables:
+The endorserdb uses a single, self-contained table:
 
 ### VALIDATIONS Table
 Stores validation records created during endorsement:
 - `tx_id`: Transaction identifier (primary key)
-- `request`: Token request data
-- `metadata`: Validation metadata (JSON)
-- `pp_hash`: Public parameters hash
-- `stored_at`: Timestamp
+- `request`: Token request data (NOT NULL)
+- `metadata`: Validation metadata (NOT NULL)
+- `pp_hash`: Public parameters hash (NOT NULL)
+- `status`: Validation status (NOT NULL)
+- `status_message`: Status message (NOT NULL)
+- `stored_at`: Timestamp (NOT NULL)
 
-### REQUESTS Table
-Stores token request status (shared with ttxdb):
-- `tx_id`: Transaction identifier (primary key)
-- `request`: Token request data
-- `status`: Transaction status
-- `status_message`: Status message
-- `application_metadata`: Application metadata (JSON)
-- `public_metadata`: Public metadata (JSON)
-- `pp_hash`: Public parameters hash
-- `stored_at`: Timestamp
+There is no foreign key to the Requests table — a validation record can be created and have its status tracked entirely independently of any TTXDB/AuditDB Requests row for the same `tx_id`.
 
 ## Relationship with TTXDB
 
-The endorserdb was created by extracting validation-related functionality from the Token Transaction Database (ttxdb). While both services share the same physical database and some tables (like REQUESTS), they provide separate interfaces:
+The endorserdb was created by extracting validation-related functionality from the Token Transaction Database (ttxdb). Both services share the same physical database, but each owns its own table (ttxdb owns Requests/Transactions/Movements/Endorsements, endorserdb owns Validations) and interface:
 
 - **ttxdb**: Manages token transactions, movements, and token requests
 - **endorserdb**: Manages validation records and their status