feat: add account webhook metrics and metrics-only mode (#2378)

This commmit adds two statsd counters in the account webhooks endpoint for
 - number of events received
 - number of events processed, with event type labels

A metrics-only mode for the endpoint is also introduced.

Author: chenba

docs/src/config.md

@@ -106,6 +106,7 @@ The following configuration options are available.
 | <span id="SYNC_TOKENSERVER__STATSD_LABEL"></span>SYNC_TOKENSERVER__STATSD_LABEL | syncstorage.tokenserver | StatsD metrics label prefix |
 | <span id="SYNC_TOKENSERVER__TOKEN_DURATION"></span>SYNC_TOKENSERVER__TOKEN_DURATION | 3600 | Token TTL (1 hour) |
 | <span id="SYNC_TOKENSERVER__FXA_WEBHOOK_ENABLED"></span>SYNC_TOKENSERVER__FXA_WEBHOOK_ENABLED | false | Enable the FxA webhook endpoint. When disabled, the route is not registered. |
+| <span id="SYNC_TOKENSERVER__FXA_WEBHOOK_METRICS_ONLY"></span>SYNC_TOKENSERVER__FXA_WEBHOOK_METRICS_ONLY | false | Run the FxA webhook handler in metrics-only mode. Received events are counted but not processed. Only used if `FXA_WEBHOOK_ENABLED` is true. |
 
 ### Tokenserver+FxA Integration
 

syncserver/src/tokenserver/extractors.rs

@@ -1345,6 +1345,7 @@ mod tests {
             token_duration: TOKEN_DURATION,
             set_verifiers: Vec::new(),
             fxa_webhook_enabled: false,
+            fxa_webhook_metrics_only: false,
         }
     }
 
@@ -1367,6 +1368,7 @@ mod tests {
             token_duration: TOKEN_DURATION,
             set_verifiers,
             fxa_webhook_enabled: true,
+            fxa_webhook_metrics_only: false,
         }
     }
 

syncserver/src/tokenserver/handlers.rs

@@ -8,6 +8,7 @@ use serde_json::Value;
 use tokio::time::timeout;
 use utoipa::ToSchema;
 
+use syncserver_common::Metrics;
 use tokenserver_auth::{MakeTokenPlaintext, Tokenlib, TokenserverOrigin};
 use tokenserver_common::{NodeType, TokenserverError};
 use tokenserver_db::{
@@ -277,27 +278,42 @@ async fn update_user(
     }
 }
 
+// statsd counter names
+const METRIC_EVENTS_RECEIVED: &str = "webhook.events_received";
+const METRIC_EVENTS_PROCESSED: &str = "webhook.events_processed";
+
 pub async fn handle_fxa_events(
     FxaWebhookToken(claims): FxaWebhookToken,
     DbWrapper(mut db): DbWrapper,
     state: Data<super::ServerState>,
 ) -> Result<HttpResponse, TokenserverError> {
+    let metrics = Metrics::from(&state.metrics);
+
+    let Some(events) = claims.events.as_object() else {
+        return Ok(HttpResponse::Ok().finish());
+    };
+
+    // Count every event received.
+    metrics.count(METRIC_EVENTS_RECEIVED, events.len() as i64);
+
+    if state.fxa_webhook_metrics_only {
+        return Ok(HttpResponse::Ok().finish());
+    }
+
     let service_id = db
         .get_service_id(tokenserver_db::params::GetServiceId {
             service: SYNC_SERVICE_NAME.to_owned(),
         })
         .await?
         .id;
-    let Some(events) = claims.events.as_object() else {
-        return Ok(HttpResponse::Ok().finish());
-    };
 
     for event_type in events.keys() {
         let email = format!("{}@{}", claims.sub, state.fxa_email_domain);
         match event_type.as_str() {
             "https://schemas.accounts.firefox.com/event/delete-user" => {
                 info!("Processing account delete for {}", email);
                 db.retire_user(RetireUser { service_id, email }).await?;
+                metrics.incr_with_tag(METRIC_EVENTS_PROCESSED, "event_type", "delete_user");
             }
             "https://schemas.accounts.firefox.com/event/password-change" => {
                 if let Some(change_time_ms) = events[event_type]
@@ -312,6 +328,7 @@ pub async fn handle_fxa_events(
                         keys_changed_at: None,
                     })
                     .await?;
+                    metrics.incr_with_tag(METRIC_EVENTS_PROCESSED, "event_type", "password_change");
                 }
             }
             _ => {
@@ -444,6 +461,7 @@ mod tests {
             token_duration: 3600,
             set_verifiers,
             fxa_webhook_enabled: true,
+            fxa_webhook_metrics_only: false,
         }
     }
 
@@ -580,6 +598,31 @@ mod tests {
         assert_eq!(retire_user_calls[0].email, "quux@api.accounts.firefox.com");
     }
 
+    #[actix_web::test]
+    async fn test_metrics_only_mode_skips_processing() {
+        let verifier =
+            SETVerifierImpl::new(&test_jwk(), "testo", "https://accounts.firefox.com/").unwrap();
+        let (pool, call_log) = MockDbPool::with_capture();
+        let mut state = make_state_with_db_pool(vec![verifier], Box::new(pool));
+        state.fxa_webhook_metrics_only = true;
+        let app = make_app_from_state(state).await;
+        let token = make_set(
+            "quux",
+            "testo",
+            json!({"https://schemas.accounts.firefox.com/event/delete-user": {}}),
+            3600,
+            TEST_PRIVATE_KEY_PEM,
+        );
+        let req = TestRequest::post()
+            .uri("/1.0/webhooks/fxa/events")
+            .insert_header(("Authorization", format!("Bearer {token}")))
+            .to_request();
+        let resp = test::call_service(&app, req).await;
+        // no database mutation in metrics-only mode
+        assert_eq!(resp.status(), 200);
+        assert_eq!(call_log.retire_user.lock().unwrap().len(), 0);
+    }
+
     #[actix_web::test]
     async fn test_password_change_event() {
         let verifier =

syncserver/src/tokenserver/mod.rs

@@ -34,6 +34,7 @@ pub struct ServerState {
     pub token_duration: u64,
     pub set_verifiers: Vec<SETVerifierImpl>,
     pub fxa_webhook_enabled: bool,
+    pub fxa_webhook_metrics_only: bool,
 }
 
 impl ServerState {
@@ -115,6 +116,7 @@ impl ServerState {
             token_duration: settings.token_duration,
             set_verifiers,
             fxa_webhook_enabled: settings.fxa_webhook_enabled,
+            fxa_webhook_metrics_only: settings.fxa_webhook_metrics_only,
         })
     }
 

tokenserver-settings/src/lib.rs

@@ -71,6 +71,10 @@ pub struct Settings {
     /// Whether to enable the FxA webhook endpoint.
     /// Defaults to false.
     pub fxa_webhook_enabled: bool,
+    /// Whether the FxA webhook handler runs in metrics-only mode. When enabled, received events
+    /// are counted but not processed.
+    /// Defaults to false.
+    pub fxa_webhook_metrics_only: bool,
 }
 
 impl Default for Settings {
@@ -100,6 +104,7 @@ impl Default for Settings {
             init_node_url: None,
             init_node_capacity: 100000,
             fxa_webhook_enabled: false,
+            fxa_webhook_metrics_only: false,
         }
     }
 }