debug: E2EE pubkey routing diagnostics (#518)

* debug: add E2EE pubkey routing diagnostics

Log available pubkeys and provider counts when E2EE routing fails,
and log pubkey_to_providers state after each provider pool refresh.
This will help diagnose why E2EE requests fail despite successful
attestation fetches from backends.

* fix PR review comments + add org_id to no-usage error logs

- Drop mappings lock before logging (reduce contention)
- Use chars().take() instead of byte slicing (UTF-8 safe)
- Log only pubkey prefix, not full value
- Limit pubkey summaries to first 10 entries
- Add organization_id and model_id to "no usage stats" error logs

* test: add E2EE pubkey routing integration test

Tests the full register → attestation → pubkey store → route by key flow.
Verifies: correct key routes, wrong key fails, no-key routes normally.

* fix: cargo fmt

Author: Evrard-Nil

crates/api/src/routes/models.rs

@@ -111,7 +111,7 @@ pub async fn list_models(
                     model.input_modalities.clone(),
                     model.output_modalities.clone(),
                 ),
-                inference_url: model.inference_url.clone(),
+                inference_url: None, // Redacted: internal infrastructure URL, admin-only
             },
         })
         .collect();
@@ -214,7 +214,7 @@ pub async fn get_model_by_name(
                 model.input_modalities,
                 model.output_modalities,
             ),
-            inference_url: model.inference_url,
+            inference_url: None, // Redacted: internal infrastructure URL, admin-only
         },
     };
 

crates/services/src/completions/mod.rs

@@ -160,31 +160,35 @@ where
         )
         .entered();
 
-        let (input_tokens, output_tokens, cache_read_tokens, chat_id) =
-            match (&self.last_usage_stats, &self.last_chat_id) {
-                (Some(usage), Some(chat_id)) => (
-                    usage.prompt_tokens,
-                    usage.completion_tokens,
-                    usage.cached_tokens(),
-                    chat_id.clone(),
-                ),
-                (None, None) => {
-                    tracing::error!("Stream ended but no usage stats and no chat_id available");
-                    return;
-                }
-                (None, Some(chat_id)) => {
-                    tracing::error!(%chat_id, "Stream ended but no usage stats available");
-                    return;
-                }
-                (Some(usage), None) => {
-                    tracing::error!(
-                        prompt_tokens = usage.prompt_tokens,
-                        completion_tokens = usage.completion_tokens,
-                        "Stream ended but no chat_id available"
-                    );
-                    return;
-                }
-            };
+        let (input_tokens, output_tokens, cache_read_tokens, chat_id) = match (
+            &self.last_usage_stats,
+            &self.last_chat_id,
+        ) {
+            (Some(usage), Some(chat_id)) => (
+                usage.prompt_tokens,
+                usage.completion_tokens,
+                usage.cached_tokens(),
+                chat_id.clone(),
+            ),
+            (None, None) => {
+                tracing::error!(%organization_id, %model_id, "Stream ended but no usage stats and no chat_id available");
+                return;
+            }
+            (None, Some(chat_id)) => {
+                tracing::error!(%chat_id, %organization_id, %model_id, "Stream ended but no usage stats available");
+                return;
+            }
+            (Some(usage), None) => {
+                tracing::error!(
+                    prompt_tokens = usage.prompt_tokens,
+                    completion_tokens = usage.completion_tokens,
+                    %organization_id,
+                    %model_id,
+                    "Stream ended but no chat_id available"
+                );
+                return;
+            }
+        };
 
         if input_tokens == 0 && output_tokens == 0 {
             return;

crates/services/src/inference_provider_pool/mod.rs

@@ -584,9 +584,30 @@ impl InferenceProviderPool {
             Some(p) => p,
             None => {
                 if let Some(pub_key) = model_pub_key {
+                    let (available_pubkeys, model_provider_count) = {
+                        let mappings = self.provider_mappings.read().await;
+                        let pubkeys: Vec<String> = mappings
+                            .pubkey_to_providers
+                            .keys()
+                            .map(|k| {
+                                let prefix: String = k.chars().take(16).collect();
+                                format!("{}...({})", prefix, k.len())
+                            })
+                            .collect();
+                        let count = mappings
+                            .model_to_providers
+                            .get(model_id)
+                            .map(|v| v.len())
+                            .unwrap_or(0);
+                        (pubkeys, count)
+                    };
+                    let model_pub_key_prefix: String = pub_key.chars().take(16).collect();
                     tracing::warn!(
                         model_id = %model_id,
-                        model_pub_key = %pub_key,
+                        model_pub_key_prefix = %model_pub_key_prefix,
+                        model_pub_key_len = pub_key.len(),
+                        available_pubkeys = ?available_pubkeys,
+                        model_provider_count = model_provider_count,
                         operation = operation_name,
                         "No provider found for model public key"
                     );
@@ -1400,6 +1421,27 @@ impl InferenceProviderPool {
             }
         }
 
+        // Log pubkey mapping state for debugging E2EE routing issues
+        let (pubkey_count, pubkey_summaries) = {
+            let mappings = self.provider_mappings.read().await;
+            let count = mappings.pubkey_to_providers.len();
+            let summaries: Vec<String> = mappings
+                .pubkey_to_providers
+                .iter()
+                .take(10)
+                .map(|(k, v)| {
+                    let prefix: String = k.chars().take(16).collect();
+                    format!("{}...({}chars,{}providers)", prefix, k.len(), v.len())
+                })
+                .collect();
+            (count, summaries)
+        };
+        info!(
+            pubkey_mapping_count = pubkey_count,
+            pubkey_summaries = ?pubkey_summaries,
+            "pubkey_to_providers state after update"
+        );
+
         // Update the URL→provider cache
         *self.inference_url_providers.write().await = new_url_cache;
 
@@ -1999,6 +2041,53 @@ mod tests {
         }
     }
 
+    /// Test that E2EE routing via pubkey works end-to-end after register_provider.
+    /// This exercises: register_provider → fetch attestation → store pubkey → route by pubkey.
+    #[tokio::test]
+    async fn test_e2ee_pubkey_routing_after_register() {
+        use inference_providers::mock::MockProvider;
+
+        let pool = InferenceProviderPool::new(None, ExternalProvidersConfig::default());
+        let model_id = "test-e2ee-model".to_string();
+
+        // Register provider (fetches attestation, stores pubkeys)
+        let mock_provider = Arc::new(MockProvider::new());
+        pool.register_provider(model_id.clone(), mock_provider)
+            .await;
+
+        // The mock provider returns this ECDSA key
+        let ecdsa_key = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+        // Test 1: routing WITHOUT pubkey should work
+        let result: Result<((), _), _> = pool
+            .retry_with_fallback(&model_id, "test_op", None, |_provider| async { Ok(()) })
+            .await;
+        assert!(result.is_ok(), "Routing without pubkey should succeed");
+
+        // Test 2: routing WITH correct pubkey should work
+        let result: Result<((), _), _> = pool
+            .retry_with_fallback(&model_id, "test_op", Some(ecdsa_key), |_provider| async {
+                Ok(())
+            })
+            .await;
+        assert!(
+            result.is_ok(),
+            "Routing with correct ECDSA pubkey should succeed, got: {:?}",
+            result.err()
+        );
+
+        // Test 3: routing with WRONG pubkey should fail
+        let result: Result<((), _), _> = pool
+            .retry_with_fallback(
+                &model_id,
+                "test_op",
+                Some("deadbeef00000000deadbeef00000000deadbeef00000000deadbeef00000000deadbeef00000000deadbeef00000000deadbeef00000000deadbeef00000000"),
+                |_provider| async { Ok(()) },
+            )
+            .await;
+        assert!(result.is_err(), "Routing with wrong pubkey should fail");
+    }
+
     #[tokio::test]
     async fn test_sync_external_providers() {
         let pool = InferenceProviderPool::new(