From 59a10fcfc3d1dce642e9523b9916e1bcd2389182 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 12 Dec 2024 11:00:07 -0800 Subject: [PATCH 001/168] [nexus] Webhook API skeleton This commit adds (unimplemented) public API endpoints for managing Nexus webhooks, as described in [RFD 364][1]. [1]: https://rfd.shared.oxide.computer/rfd/364#_external_api --- Cargo.lock | 3 + nexus/external-api/output/nexus_tags.txt | 10 + nexus/external-api/src/lib.rs | 87 +++ nexus/src/external_api/http_entrypoints.rs | 165 ++++++ nexus/types/Cargo.toml | 3 +- nexus/types/src/external_api/params.rs | 23 + nexus/types/src/external_api/views.rs | 95 ++++ openapi/nexus.json | 581 +++++++++++++++++++++ uuid-kinds/src/lib.rs | 2 + 9 files changed, 968 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 41dacdffcb2..3480bc8711f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6388,6 +6388,7 @@ dependencies = [ "test-strategy", "thiserror 1.0.69", "update-engine", + "url", "uuid", ] @@ -10368,6 +10369,7 @@ dependencies = [ "schemars_derive", "serde", "serde_json", + "url", "uuid", ] @@ -12835,6 +12837,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 82767e399c0..2528dfeffe4 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -243,6 +243,16 @@ API operations found with tag "system/status" OPERATION ID METHOD URL PATH ping GET /v1/ping +API operations found with tag "system/webhooks" +OPERATION ID METHOD URL PATH +webhook_create POST /experimental/v1/webhooks +webhook_delete DELETE /experimental/v1/webhooks/{webhook_id} +webhook_delivery_list GET /experimental/v1/webhooks/{webhook_id}/deliveries +webhook_delivery_resend POST /experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend +webhook_secrets_add POST /experimental/v1/webhooks/{webhook_id}/secrets +webhook_secrets_list GET /experimental/v1/webhooks/{webhook_id}/secrets +webhook_view GET /experimental/v1/webhooks/{webhook_id} + API operations found with tag "vpcs" OPERATION ID METHOD URL PATH internet_gateway_create POST /v1/internet-gateways diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index d76192072a8..2ed50fb3b3d 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -158,6 +158,12 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; url = "http://docs.oxide.computer/api/vpcs" } }, + "system/webhooks" = { + description = "Webhooks deliver notifications for audit log events and fault management alerts.", + external_docs = { + url = "http://docs.oxide.computer/api/webhooks" + } + }, "system/probes" = { description = "Probes for testing network connectivity", external_docs = { @@ -3242,6 +3248,87 @@ pub trait NexusExternalApi { rqctx: RequestContext, params: TypedBody, ) -> Result, HttpError>; + + // Webhooks (experimental) + + /// Get the configuration for a webhook. + #[endpoint { + method = GET, + path = "/experimental/v1/webhooks/{webhook_id}", + tags = ["system/webhooks"], + }] + async fn webhook_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Create a new webhook receiver. + #[endpoint { + method = POST, + path = "/experimental/v1/webhooks", + tags = ["system/webhooks"], + }] + async fn webhook_create( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError>; + + /// Delete a webhook receiver. + #[endpoint { + method = DELETE, + path = "/experimental/v1/webhooks/{webhook_id}", + tags = ["system/webhooks"], + }] + async fn webhook_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// List the IDs of secrets for a webhook receiver. + #[endpoint { + method = GET, + path = "/experimental/v1/webhooks/{webhook_id}/secrets", + tags = ["system/webhooks"], + }] + async fn webhook_secrets_list( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a secret to a webhook. + #[endpoint { + method = POST, + path = "/experimental/v1/webhooks/{webhook_id}/secrets", + tags = ["system/webhooks"], + }] + async fn webhook_secrets_add( + rqctx: RequestContext, + path_params: Path, + params: TypedBody, + ) -> Result, HttpError>; + + /// List delivery attempts to a webhook receiver. + #[endpoint { + method = GET, + path = "/experimental/v1/webhooks/{webhook_id}/deliveries", + tags = ["system/webhooks"], + }] + async fn webhook_delivery_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + /// Request re-delivery of a webhook event. + #[endpoint { + method = POST, + path = "/experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend", + tags = ["system/webhooks"], + }] + async fn webhook_delivery_resend( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; } /// Perform extra validations on the OpenAPI spec. diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 191c304216d..ca7ffeff9b6 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6926,4 +6926,169 @@ impl NexusExternalApi for NexusExternalApiImpl { .instrument_dropshot_handler(&rqctx, handler) .await } + + async fn webhook_view( + rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_create( + rqctx: RequestContext, + _params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_delete( + rqctx: RequestContext, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_secrets_list( + rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Add a secret to a webhook. + async fn webhook_secrets_add( + rqctx: RequestContext, + _path_params: Path, + _params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_delivery_list( + rqctx: RequestContext, + _path_params: Path, + _query_params: Query, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_delivery_resend( + rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } } diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index edf22679ef4..a6893f94cf3 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -30,7 +30,7 @@ openssl.workspace = true oxql-types.workspace = true oxnet.workspace = true parse-display.workspace = true -schemars = { workspace = true, features = ["chrono", "uuid1"] } +schemars = { workspace = true, features = ["chrono", "uuid1", "url"] } serde.workspace = true serde_json.workspace = true serde_with.workspace = true @@ -42,6 +42,7 @@ thiserror.workspace = true newtype-uuid.workspace = true update-engine.workspace = true uuid.workspace = true +url = { workspace = true, features = ["serde"] } api_identity.workspace = true gateway-client.workspace = true diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 8294e23e2f2..e80784cf997 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -28,6 +28,7 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::{net::IpAddr, str::FromStr}; +use url::Url; use uuid::Uuid; macro_rules! path_param { @@ -92,6 +93,7 @@ path_param!(CertificatePath, certificate, "certificate"); id_path_param!(SupportBundlePath, support_bundle, "support bundle"); id_path_param!(GroupPath, group_id, "group"); +id_path_param!(WebhookPath, webhook_id, "webhook"); // TODO: The hardware resources should be represented by its UUID or a hardware // ID that can be used to deterministically generate the UUID. @@ -2300,3 +2302,24 @@ pub struct DeviceAccessTokenRequest { pub device_code: String, pub client_id: Uuid, } + +// Webhooks + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookCreate { + pub name: String, + pub endpoint: Url, + pub secrets: Vec, + pub events: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookSecret { + pub secret: String, +} + +#[derive(Deserialize, JsonSchema)] +pub struct WebhookDeliveryPath { + pub webhook_id: Uuid, + pub delivery_id: Uuid, +} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 3430d06f724..f3ef7300a3f 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -17,6 +17,7 @@ use omicron_common::api::external::{ IdentityMetadata, InstanceState, Name, ObjectIdentity, RoleName, SimpleIdentityOrName, }; +use omicron_uuid_kinds::{EventUuid, WebhookUuid}; use oxnet::{Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -25,6 +26,7 @@ use std::collections::BTreeSet; use std::fmt; use std::net::IpAddr; use strum::{EnumIter, IntoEnumIterator}; +use url::Url; use uuid::Uuid; use super::params::PhysicalDiskKind; @@ -1027,3 +1029,96 @@ pub struct OxqlQueryResult { /// Tables resulting from the query, each containing timeseries. pub tables: Vec, } + +// WEBHOOKS + +/// The configuration for a webhook. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct Webhook { + pub id: WebhookUuid, + pub name: String, + pub endpoint: Url, + pub secrets: Vec, + // XXX(eliza): should eventually be an enum? + pub events: Vec, + // TODO(eliza): roles? +} + +/// A list of the IDs of secrets associated with a webhook. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookSecrets { + pub secrets: Vec, +} + +/// The public ID of a secret key assigned to a webhook. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookSecretId { + pub id: String, +} + +/// A delivery attempt for a webhook event. + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookDelivery { + /// The UUID of this delivery attempt. + pub id: Uuid, + + /// The UUID of the webhook receiver that this event was delivered to. + pub webhook_id: WebhookUuid, + + /// The event class. + pub event: String, + + /// The UUID of the event. + pub event_id: EventUuid, + + /// The state of the delivery attempt. + pub state: WebhookDeliveryState, + + /// The time at which the webhook delivery was attempted, or `null` if + /// webhook delivery has not yet been attempted (`state` is "pending"). + pub time_sent: Option>, + + /// Describes the response returned by the receiver endpoint. + /// + /// This is present if the webhook has been delivered successfully, or if the + /// endpoint returned an HTTP error (`state` is "delivered" or + /// "failed_http_error"). This is `null` if the webhook has not yet been + /// delivered, or if the endpoint was unreachable (`state` is "pending" or + /// "failed_unreachable"). + pub response: Option, + + /// The UUID of a previous delivery attempt that this is a repeat of, if + /// this was a resending of a previous delivery. If this is the first time + /// this event has been delivered, this is `null`. + pub resent_for: Option, +} + +/// The state of a webhook delivery attempt. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WebhookDeliveryState { + /// The webhook event has not yet been delivered. + Pending, + /// The webhook event has been delivered successfully. + Delivered, + /// A webhook request was sent to the endpoint, and it + /// returned a HTTP error status code indicating an error. + FailedHttpError, + /// The webhook request could not be sent to the receiver endpoint. + FailedUnreachable, +} + +/// The response received from a webhook receiver endpoint. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookDeliveryResponse { + /// The HTTP status code returned from the webhook endpoint. + pub status: u16, + /// The response time of the webhook endpoint, in milliseconds. + pub duration_ms: usize, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookDeliveryId { + pub delivery_id: Uuid, +} diff --git a/openapi/nexus.json b/openapi/nexus.json index c94b5970c08..1d032d41ad0 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -624,6 +624,314 @@ } } }, + "/experimental/v1/webhooks": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Create a new webhook receiver.", + "operationId": "webhook_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/experimental/v1/webhooks/{webhook_id}": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "Get the configuration for a webhook.", + "operationId": "webhook_view", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/webhooks" + ], + "summary": "Delete a webhook receiver.", + "operationId": "webhook_delete", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/experimental/v1/webhooks/{webhook_id}/deliveries": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List delivery attempts to a webhook receiver.", + "operationId": "webhook_delivery_list", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Request re-delivery of a webhook event.", + "operationId": "webhook_delivery_resend", + "parameters": [ + { + "in": "path", + "name": "delivery_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "webhook_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryId" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/experimental/v1/webhooks/{webhook_id}/secrets": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List the IDs of secrets for a webhook receiver.", + "operationId": "webhook_secrets_list", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecrets" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Add a secret to a webhook.", + "operationId": "webhook_secrets_add", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecret" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecretId" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/login/{silo_name}/saml/{provider_name}": { "post": { "tags": [ @@ -21758,10 +22066,18 @@ } } }, + "TypedUuidForEventKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForSupportBundleKind": { "type": "string", "format": "uuid" }, + "TypedUuidForWebhookKind": { + "type": "string", + "format": "uuid" + }, "UninitializedSled": { "description": "A sled that has not been added to an initialized rack yet", "type": "object", @@ -23172,6 +23488,264 @@ } } }, + "Webhook": { + "description": "The configuration for a webhook.", + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "format": "uri" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "$ref": "#/components/schemas/TypedUuidForWebhookKind" + }, + "name": { + "type": "string" + }, + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookSecretId" + } + } + }, + "required": [ + "endpoint", + "events", + "id", + "name", + "secrets" + ] + }, + "WebhookCreate": { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "format": "uri" + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "secrets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "endpoint", + "events", + "name", + "secrets" + ] + }, + "WebhookDelivery": { + "description": "A delivery attempt for a webhook event.", + "type": "object", + "properties": { + "event": { + "description": "The event class.", + "type": "string" + }, + "event_id": { + "description": "The UUID of the event.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForEventKind" + } + ] + }, + "id": { + "description": "The UUID of this delivery attempt.", + "type": "string", + "format": "uuid" + }, + "resent_for": { + "nullable": true, + "description": "The UUID of a previous delivery attempt that this is a repeat of, if this was a resending of a previous delivery. If this is the first time this event has been delivered, this is `null`.", + "type": "string", + "format": "uuid" + }, + "response": { + "nullable": true, + "description": "Describes the response returned by the receiver endpoint.\n\nThis is present if the webhook has been delivered successfully, or if the endpoint returned an HTTP error (`state` is \"delivered\" or \"failed_http_error\"). This is `null` if the webhook has not yet been delivered, or if the endpoint was unreachable (`state` is \"pending\" or \"failed_unreachable\").", + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDeliveryResponse" + } + ] + }, + "state": { + "description": "The state of the delivery attempt.", + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDeliveryState" + } + ] + }, + "time_sent": { + "nullable": true, + "description": "The time at which the webhook delivery was attempted, or `null` if webhook delivery has not yet been attempted (`state` is \"pending\").", + "type": "string", + "format": "date-time" + }, + "webhook_id": { + "description": "The UUID of the webhook receiver that this event was delivered to.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForWebhookKind" + } + ] + } + }, + "required": [ + "event", + "event_id", + "id", + "state", + "webhook_id" + ] + }, + "WebhookDeliveryId": { + "type": "object", + "properties": { + "delivery_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "delivery_id" + ] + }, + "WebhookDeliveryResponse": { + "description": "The response received from a webhook receiver endpoint.", + "type": "object", + "properties": { + "duration_ms": { + "description": "The response time of the webhook endpoint, in milliseconds.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "status": { + "description": "The HTTP status code returned from the webhook endpoint.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "duration_ms", + "status" + ] + }, + "WebhookDeliveryResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookDelivery" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "WebhookDeliveryState": { + "description": "The state of a webhook delivery attempt.", + "oneOf": [ + { + "description": "The webhook event has not yet been delivered.", + "type": "string", + "enum": [ + "pending" + ] + }, + { + "description": "The webhook event has been delivered successfully.", + "type": "string", + "enum": [ + "delivered" + ] + }, + { + "description": "A webhook request was sent to the endpoint, and it returned a HTTP error status code indicating an error.", + "type": "string", + "enum": [ + "failed_http_error" + ] + }, + { + "description": "The webhook request could not be sent to the receiver endpoint.", + "type": "string", + "enum": [ + "failed_unreachable" + ] + } + ] + }, + "WebhookSecret": { + "type": "object", + "properties": { + "secret": { + "type": "string" + } + }, + "required": [ + "secret" + ] + }, + "WebhookSecretId": { + "description": "The public ID of a secret key assigned to a webhook.", + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + "WebhookSecrets": { + "description": "A list of the IDs of secrets associated with a webhook.", + "type": "object", + "properties": { + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookSecretId" + } + } + }, + "required": [ + "secrets" + ] + }, "NameOrIdSortMode": { "description": "Supported set of sort modes for scanning by name or id", "oneOf": [ @@ -23407,6 +23981,13 @@ { "name": "system/update" }, + { + "name": "system/webhooks", + "description": "Webhooks deliver notifications for audit log events and fault management alerts.", + "externalDocs": { + "url": "http://docs.oxide.computer/api/webhooks" + } + }, { "name": "vpcs", "description": "Virtual Private Clouds (VPCs) provide isolated network environments for managing and deploying services.", diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index 1d6dc600226..126af803fc3 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -57,6 +57,7 @@ impl_typed_uuid_kind! { DemoSaga => "demo_saga", Downstairs => "downstairs", DownstairsRegion => "downstairs_region", + Event => "event", ExternalIp => "external_ip", Instance => "instance", LoopbackAddress => "loopback_address", @@ -76,5 +77,6 @@ impl_typed_uuid_kind! { UpstairsSession => "upstairs_session", Vnic => "vnic", Volume => "volume", + Webhook => "webhook", Zpool => "zpool", } From 4ece574fd74e7f06049ca2c073be7793f5e23044 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Dec 2024 13:09:17 -0800 Subject: [PATCH 002/168] naming consistency edits from @augustuswm Co-authored-by: Augustus Mayo --- nexus/external-api/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 2ed50fb3b3d..0243d3326e3 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3251,7 +3251,7 @@ pub trait NexusExternalApi { // Webhooks (experimental) - /// Get the configuration for a webhook. + /// Get the configuration for a webhook receiver. #[endpoint { method = GET, path = "/experimental/v1/webhooks/{webhook_id}", @@ -3295,7 +3295,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Add a secret to a webhook. + /// Add a secret to a webhook receiver. #[endpoint { method = POST, path = "/experimental/v1/webhooks/{webhook_id}/secrets", From 6e8f8acd0659410d72e1fa581f2d63cd19d9f5b6 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 19 Dec 2024 11:40:59 -0800 Subject: [PATCH 003/168] fix cargo workspace hack --- Cargo.lock | 1 + workspace-hack/Cargo.toml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3480bc8711f..d1d81133a41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7591,6 +7591,7 @@ dependencies = [ "toml_edit 0.22.22", "tracing", "unicode-xid", + "url", "usdt", "usdt-impl", "uuid", diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index c53cf44a114..3f35bb0d96b 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -105,7 +105,7 @@ reqwest = { version = "0.12.9", features = ["blocking", "cookies", "json", "rust rsa = { version = "0.9.6", features = ["serde", "sha2"] } rustls = { version = "0.23.19", features = ["ring"] } rustls-webpki = { version = "0.102.8", default-features = false, features = ["aws_lc_rs", "ring", "std"] } -schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "url", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.24", features = ["serde"] } serde = { version = "1.0.217", features = ["alloc", "derive", "rc"] } @@ -128,6 +128,7 @@ toml = { version = "0.7.8" } toml_datetime = { version = "0.6.8", default-features = false, features = ["serde"] } toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.22", features = ["serde"] } tracing = { version = "0.1.40", features = ["log"] } +url = { version = "2.5.3", features = ["serde"] } usdt = { version = "0.5.0" } usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] } uuid = { version = "1.12.0", features = ["serde", "v4"] } @@ -225,7 +226,7 @@ reqwest = { version = "0.12.9", features = ["blocking", "cookies", "json", "rust rsa = { version = "0.9.6", features = ["serde", "sha2"] } rustls = { version = "0.23.19", features = ["ring"] } rustls-webpki = { version = "0.102.8", default-features = false, features = ["aws_lc_rs", "ring", "std"] } -schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "url", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.24", features = ["serde"] } serde = { version = "1.0.217", features = ["alloc", "derive", "rc"] } @@ -251,6 +252,7 @@ toml_datetime = { version = "0.6.8", default-features = false, features = ["serd toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.22", features = ["serde"] } tracing = { version = "0.1.40", features = ["log"] } unicode-xid = { version = "0.2.6" } +url = { version = "2.5.3", features = ["serde"] } usdt = { version = "0.5.0" } usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] } uuid = { version = "1.12.0", features = ["serde", "v4"] } From 478b2cee01692ca1ae16cc4707a7f611a19ec500 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 12:28:03 -0800 Subject: [PATCH 004/168] update to match RFD 538 --- nexus/external-api/output/nexus_tags.txt | 3 +- nexus/external-api/src/lib.rs | 14 ++- nexus/src/external_api/http_entrypoints.rs | 24 ++++ nexus/types/src/external_api/params.rs | 35 +++++- nexus/types/src/external_api/views.rs | 18 ++- openapi/nexus.json | 123 +++++++++++++++++++-- 6 files changed, 203 insertions(+), 14 deletions(-) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 2528dfeffe4..a1a70157ff5 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -248,9 +248,10 @@ OPERATION ID METHOD URL PATH webhook_create POST /experimental/v1/webhooks webhook_delete DELETE /experimental/v1/webhooks/{webhook_id} webhook_delivery_list GET /experimental/v1/webhooks/{webhook_id}/deliveries -webhook_delivery_resend POST /experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend +webhook_delivery_resend POST /experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend webhook_secrets_add POST /experimental/v1/webhooks/{webhook_id}/secrets webhook_secrets_list GET /experimental/v1/webhooks/{webhook_id}/secrets +webhook_update PUT /experimental/v1/webhooks/{webhook_id} webhook_view GET /experimental/v1/webhooks/{webhook_id} API operations found with tag "vpcs" diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 0243d3326e3..6687a25226c 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3273,6 +3273,18 @@ pub trait NexusExternalApi { params: TypedBody, ) -> Result, HttpError>; + /// Update the configuration of an existing webhook receiver. + #[endpoint { + method = PUT, + path = "/experimental/v1/webhooks/{webhook_id}", + tags = ["system/webhooks"], + }] + async fn webhook_update( + rqctx: RequestContext, + path_params: Path, + params: TypedBody, + ) -> Result; + /// Delete a webhook receiver. #[endpoint { method = DELETE, @@ -3322,7 +3334,7 @@ pub trait NexusExternalApi { /// Request re-delivery of a webhook event. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend", + path = "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend", tags = ["system/webhooks"], }] async fn webhook_delivery_resend( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index ca7ffeff9b6..7e252bf834c 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6973,6 +6973,30 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn webhook_update( + rqctx: RequestContext, + _path_params: Path, + _params: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn webhook_delete( rqctx: RequestContext, _path_params: Path, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index e80784cf997..a1359c524ed 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2307,10 +2307,43 @@ pub struct DeviceAccessTokenRequest { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookCreate { + /// An identifier for this webhook receiver, which must be unique. pub name: String, + + /// The URL that webhook notification requests should be sent to pub endpoint: Url, + + /// A non-empty list of secret keys used to sign webhook payloads. pub secrets: Vec, + + /// A list of webhook event classes to subscribe to. + /// + /// If this list is empty or is not included in the request body, the + /// webhook will not be subscribed to any events. + #[serde(default)] pub events: Vec, + + /// If `true`, liveness probe requests will not be sent to this webhook receiver. + #[serde(default)] + pub disable_probes: bool, +} + +/// Parameters to update a webhook configuration. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookUpdate { + /// An identifier for this webhook receiver, which must be unique. + pub name: String, + + /// The URL that webhook notification requests should be sent to + pub endpoint: Url, + + /// A list of webhook event classes to subscribe to. + /// + /// If this list is empty, the webhook will not be subscribed to any events. + pub events: Vec, + + /// If `true`, liveness probe requests will not be sent to this webhook receiver. + pub disable_probes: bool, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -2321,5 +2354,5 @@ pub struct WebhookSecret { #[derive(Deserialize, JsonSchema)] pub struct WebhookDeliveryPath { pub webhook_id: Uuid, - pub delivery_id: Uuid, + pub event_id: Uuid, } diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index f3ef7300a3f..9114f270eea 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1035,13 +1035,22 @@ pub struct OxqlQueryResult { /// The configuration for a webhook. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct Webhook { + /// The UUID of this webhook receiver. pub id: WebhookUuid, + /// The identifier assigned to this webhook receiver upon creation. pub name: String, + /// The URL that webhook notification requests are sent to. pub endpoint: Url, + /// The UUID of the user associated with this webhook receiver for + /// role-based ccess control. + pub actor_id: Uuid, + // A list containing the IDs of the secret keys used to sign payloads sent + // to this receiver. pub secrets: Vec, - // XXX(eliza): should eventually be an enum? + /// The list of event classes to which this receiver is subscribed. pub events: Vec, - // TODO(eliza): roles? + /// If `true`, liveness probe requests are not sent to this receiver. + pub disable_probes: bool, } /// A list of the IDs of secrets associated with a webhook. @@ -1067,7 +1076,7 @@ pub struct WebhookDelivery { pub webhook_id: WebhookUuid, /// The event class. - pub event: String, + pub event_class: String, /// The UUID of the event. pub event_id: EventUuid, @@ -1107,6 +1116,9 @@ pub enum WebhookDeliveryState { FailedHttpError, /// The webhook request could not be sent to the receiver endpoint. FailedUnreachable, + /// A connection to the receiver endpoint was successfully established, but + /// no response was received within the delivery timeout. + FailedTimeout, } /// The response received from a webhook receiver endpoint. diff --git a/openapi/nexus.json b/openapi/nexus.json index 1d032d41ad0..7e502167145 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -666,7 +666,7 @@ "tags": [ "system/webhooks" ], - "summary": "Get the configuration for a webhook.", + "summary": "Get the configuration for a webhook receiver.", "operationId": "webhook_view", "parameters": [ { @@ -699,6 +699,46 @@ } } }, + "put": { + "tags": [ + "system/webhooks" + ], + "summary": "Update the configuration of an existing webhook receiver.", + "operationId": "webhook_update", + "parameters": [ + { + "in": "path", + "name": "webhook_id", + "description": "ID of the webhook", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, "delete": { "tags": [ "system/webhooks" @@ -799,7 +839,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}/deliveries/{delivery_id}/resend": { + "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend": { "post": { "tags": [ "system/webhooks" @@ -809,7 +849,7 @@ "parameters": [ { "in": "path", - "name": "delivery_id", + "name": "event_id", "required": true, "schema": { "type": "string", @@ -888,7 +928,7 @@ "tags": [ "system/webhooks" ], - "summary": "Add a secret to a webhook.", + "summary": "Add a secret to a webhook receiver.", "operationId": "webhook_secrets_add", "parameters": [ { @@ -23492,20 +23532,37 @@ "description": "The configuration for a webhook.", "type": "object", "properties": { + "actor_id": { + "description": "The UUID of the user associated with this webhook receiver for role-based ccess control.", + "type": "string", + "format": "uuid" + }, + "disable_probes": { + "description": "If `true`, liveness probe requests are not sent to this receiver.", + "type": "boolean" + }, "endpoint": { + "description": "The URL that webhook notification requests are sent to.", "type": "string", "format": "uri" }, "events": { + "description": "The list of event classes to which this receiver is subscribed.", "type": "array", "items": { "type": "string" } }, "id": { - "$ref": "#/components/schemas/TypedUuidForWebhookKind" + "description": "The UUID of this webhook receiver.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForWebhookKind" + } + ] }, "name": { + "description": "The identifier assigned to this webhook receiver upon creation.", "type": "string" }, "secrets": { @@ -23516,6 +23573,8 @@ } }, "required": [ + "actor_id", + "disable_probes", "endpoint", "events", "id", @@ -23526,20 +23585,30 @@ "WebhookCreate": { "type": "object", "properties": { + "disable_probes": { + "description": "If `true`, liveness probe requests will not be sent to this webhook receiver.", + "default": false, + "type": "boolean" + }, "endpoint": { + "description": "The URL that webhook notification requests should be sent to", "type": "string", "format": "uri" }, "events": { + "description": "A list of webhook event classes to subscribe to.\n\nIf this list is empty or is not included in the request body, the webhook will not be subscribed to any events.", + "default": [], "type": "array", "items": { "type": "string" } }, "name": { + "description": "An identifier for this webhook receiver, which must be unique.", "type": "string" }, "secrets": { + "description": "A non-empty list of secret keys used to sign webhook payloads.", "type": "array", "items": { "type": "string" @@ -23548,7 +23617,6 @@ }, "required": [ "endpoint", - "events", "name", "secrets" ] @@ -23557,7 +23625,7 @@ "description": "A delivery attempt for a webhook event.", "type": "object", "properties": { - "event": { + "event_class": { "description": "The event class.", "type": "string" }, @@ -23613,7 +23681,7 @@ } }, "required": [ - "event", + "event_class", "event_id", "id", "state", @@ -23705,6 +23773,13 @@ "enum": [ "failed_unreachable" ] + }, + { + "description": "A connection to the receiver endpoint was successfully established, but no response was received within the delivery timeout.", + "type": "string", + "enum": [ + "failed_timeout" + ] } ] }, @@ -23746,6 +23821,38 @@ "secrets" ] }, + "WebhookUpdate": { + "description": "Parameters to update a webhook configuration.", + "type": "object", + "properties": { + "disable_probes": { + "description": "If `true`, liveness probe requests will not be sent to this webhook receiver.", + "type": "boolean" + }, + "endpoint": { + "description": "The URL that webhook notification requests should be sent to", + "type": "string", + "format": "uri" + }, + "events": { + "description": "A list of webhook event classes to subscribe to.\n\nIf this list is empty, the webhook will not be subscribed to any events.", + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "description": "An identifier for this webhook receiver, which must be unique.", + "type": "string" + } + }, + "required": [ + "disable_probes", + "endpoint", + "events", + "name" + ] + }, "NameOrIdSortMode": { "description": "Supported set of sort modes for scanning by name or id", "oneOf": [ From c730c63d663cc91700405431c8dc471939837edb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 14:12:23 -0800 Subject: [PATCH 005/168] also do event classes APIs --- nexus/external-api/output/nexus_tags.txt | 2 + nexus/external-api/src/lib.rs | 24 ++++ nexus/src/external_api/http_entrypoints.rs | 48 +++++++ nexus/types/src/external_api/params.rs | 19 +++ nexus/types/src/external_api/views.rs | 10 ++ openapi/nexus.json | 138 +++++++++++++++++++++ 6 files changed, 241 insertions(+) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index a1a70157ff5..5bd2c372b2c 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -249,6 +249,8 @@ webhook_create POST /experimental/v1/webhooks webhook_delete DELETE /experimental/v1/webhooks/{webhook_id} webhook_delivery_list GET /experimental/v1/webhooks/{webhook_id}/deliveries webhook_delivery_resend POST /experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend +webhook_event_class_list GET /experimental/v1/webhook-events/classes +webhook_event_class_view GET /experimental/v1/webhook-events/classes/{name} webhook_secrets_add POST /experimental/v1/webhooks/{webhook_id}/secrets webhook_secrets_list GET /experimental/v1/webhooks/{webhook_id}/secrets webhook_update PUT /experimental/v1/webhooks/{webhook_id} diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 6687a25226c..1f4aef61a63 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3251,6 +3251,30 @@ pub trait NexusExternalApi { // Webhooks (experimental) + /// List webhook event classes + #[endpoint { + method = GET, + path = "/experimental/v1/webhook-events/classes", + tags = ["system/webhooks"], + }] + async fn webhook_event_class_list( + rqctx: RequestContext, + query_params: Query< + PaginationParams, + >, + ) -> Result>, HttpError>; + + /// Fetch details on an event class by name. + #[endpoint { + method = GET, + path ="/experimental/v1/webhook-events/classes/{name}", + tags = ["system/webhooks"], + }] + async fn webhook_event_class_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + /// Get the configuration for a webhook receiver. #[endpoint { method = GET, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 7e252bf834c..e538615cfe3 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6927,6 +6927,54 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn webhook_event_class_list( + rqctx: RequestContext, + _query_params: Query< + PaginationParams, + >, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn webhook_event_class_view( + rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn webhook_view( rqctx: RequestContext, _path_params: Path, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index a1359c524ed..f167bdc522c 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2305,6 +2305,25 @@ pub struct DeviceAccessTokenRequest { // Webhooks +/// Query params for listing webhook event classes. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct EventClassFilter { + /// An optional glob pattern for filtering event class names. + pub filter: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct EventClassPage { + /// The last webhook event class returned by a previous page. + pub last_seen: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct EventClassSelector { + /// The name of the event class. + pub name: String, +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookCreate { /// An identifier for this webhook receiver, which must be unique. diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 9114f270eea..e36d26bc214 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1032,6 +1032,16 @@ pub struct OxqlQueryResult { // WEBHOOKS +/// A webhook event class. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct EventClass { + /// The name of the event class. + pub name: String, + + /// A description of what this event class represents. + pub description: String, +} + /// The configuration for a webhook. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct Webhook { diff --git a/openapi/nexus.json b/openapi/nexus.json index 7e502167145..3ba09107098 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -624,6 +624,105 @@ } } }, + "/experimental/v1/webhook-events/classes": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List webhook event classes", + "operationId": "webhook_event_class_list", + "parameters": [ + { + "in": "query", + "name": "filter", + "description": "An optional glob pattern for filtering event class names.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EventClassResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/experimental/v1/webhook-events/classes/{name}": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "Fetch details on an event class by name.", + "operationId": "webhook_event_class_view", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of the event class.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EventClass" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/experimental/v1/webhooks": { "post": { "tags": [ @@ -14821,6 +14920,45 @@ "request_id" ] }, + "EventClass": { + "description": "A webhook event class.", + "type": "object", + "properties": { + "description": { + "description": "A description of what this event class represents.", + "type": "string" + }, + "name": { + "description": "The name of the event class.", + "type": "string" + } + }, + "required": [ + "description", + "name" + ] + }, + "EventClassResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/EventClass" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, "ExternalIp": { "oneOf": [ { From d5ed2ba4ee769395c3befb825aba38747171471e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 14:24:15 -0800 Subject: [PATCH 006/168] apparently this is also a thing we need to do --- nexus/tests/output/uncovered-authz-endpoints.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 8a639f1224c..711de0d0993 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -1,10 +1,16 @@ API endpoints with no coverage in authz tests: probe_delete (delete "/experimental/v1/probes/{probe}") +webhook_delete (delete "/experimental/v1/webhooks/{webhook_id}") probe_list (get "/experimental/v1/probes") probe_view (get "/experimental/v1/probes/{probe}") support_bundle_download (get "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_download_file (get "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") support_bundle_index (get "/experimental/v1/system/support-bundles/{support_bundle}/index") +webhook_event_class_list (get "/experimental/v1/webhook-events/classes") +webhook_event_class_view (get "/experimental/v1/webhook-events/classes/{name}") +webhook_view (get "/experimental/v1/webhooks/{webhook_id}") +webhook_delivery_list (get "/experimental/v1/webhooks/{webhook_id}/deliveries") +webhook_secrets_list (get "/experimental/v1/webhooks/{webhook_id}/secrets") ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") @@ -15,7 +21,11 @@ device_auth_request (post "/device/auth") device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") probe_create (post "/experimental/v1/probes") +webhook_create (post "/experimental/v1/webhooks") +webhook_delivery_resend (post "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend") +webhook_secrets_add (post "/experimental/v1/webhooks/{webhook_id}/secrets") login_saml (post "/login/{silo_name}/saml/{provider_name}") login_local (post "/v1/login/{silo_name}/local") logout (post "/v1/logout") networking_switch_port_lldp_config_update (post "/v1/system/hardware/switch-port/{port}/lldp/config") +webhook_update (put "/experimental/v1/webhooks/{webhook_id}") From 5d4f13b92a8241c65c196e0cc839ebaf2a89988a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Dec 2024 13:08:22 -0800 Subject: [PATCH 007/168] [nexus] start DB model for webhooks --- schema/crdb/add-webhooks/README.adoc | 40 ++++++++ schema/crdb/add-webhooks/up01.sql | 10 ++ schema/crdb/add-webhooks/up02.sql | 13 +++ schema/crdb/add-webhooks/up03.sql | 5 + schema/crdb/add-webhooks/up04.sql | 10 ++ schema/crdb/add-webhooks/up05.sql | 4 + schema/crdb/add-webhooks/up06.sql | 12 +++ schema/crdb/add-webhooks/up07.sql | 5 + schema/crdb/add-webhooks/up08.sql | 7 ++ schema/crdb/add-webhooks/up09.sql | 9 ++ schema/crdb/add-webhooks/up10.sql | 27 ++++++ schema/crdb/add-webhooks/up11.sql | 4 + schema/crdb/dbinit.sql | 131 +++++++++++++++++++++++++++ 13 files changed, 277 insertions(+) create mode 100644 schema/crdb/add-webhooks/README.adoc create mode 100644 schema/crdb/add-webhooks/up01.sql create mode 100644 schema/crdb/add-webhooks/up02.sql create mode 100644 schema/crdb/add-webhooks/up03.sql create mode 100644 schema/crdb/add-webhooks/up04.sql create mode 100644 schema/crdb/add-webhooks/up05.sql create mode 100644 schema/crdb/add-webhooks/up06.sql create mode 100644 schema/crdb/add-webhooks/up07.sql create mode 100644 schema/crdb/add-webhooks/up08.sql create mode 100644 schema/crdb/add-webhooks/up09.sql create mode 100644 schema/crdb/add-webhooks/up10.sql create mode 100644 schema/crdb/add-webhooks/up11.sql diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc new file mode 100644 index 00000000000..f3ea209b559 --- /dev/null +++ b/schema/crdb/add-webhooks/README.adoc @@ -0,0 +1,40 @@ +# Overview + +This migration adds initial tables required for webhook delivery. + +## Upgrade steps + +The individual transactions in this upgrade do the following: + +* *Webhook receivers*: +** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores +the receiver endpoints that receive webhook events. +** *Receiver secrets*: +*** `up02.sql` creates the `omicron.public.webhook_rx_secret` table, which +associates webhook receivers with secret keys and their IDs. +*** `up03.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, +for looking up all secrets associated with a receiver. +** *Receiver subscriptions*: +*** `up04.sql` creates the `omicron.public.webhook_rx_subscription` table, which +associates a webhook receiver with multiple event classes that the receiver is +subscribed to. +*** `up05.sql` creates an index `lookup_webhook_subscriptions_by_rx` for +looking up all event classes that a receiver ID is subscribed to. +* *Webhook message dispatching and delivery attempts*: +** *Dispatch table*: +*** `up06.sql` creates the table `omicron.public.webhook_msg_dispatch`, which +tracks the webhook messages that have been dispatched to receivers. +*** `up07.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up +entries in `omicron.public.webhook_msg_dispatch` by receiver ID. +*** `up08.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook +messages (entries in `omicron.public.webhook_msg_dispatch` where the +`time_completed` field has not been set). +** *Delivery attempts*: +*** `up09.sql` creates the enum `omicron.public.webhook_msg_delivery_result`, +representing the potential outcomes of a webhook delivery attempt. +*** `up10.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`, +which records each individual delivery attempt for a webhook message in the +`webhook_msg_dispatch` table. +*** `up11.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on +`omicron.public.webhook_msg_delivery_attempt`, for looking up all attempts to +deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql new file mode 100644 index 00000000000..58799496b05 --- /dev/null +++ b/schema/crdb/add-webhooks/up01.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + id UUID PRIMARY KEY, + -- A human-readable identifier for this webhook receiver. + name STRING(63) NOT NULL, + -- URL of the endpoint webhooks are delivered to. + endpoint STRING(512) NOT NULL, + -- TODO(eliza): how do we track which roles are assigned to a webhook? + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ +); diff --git a/schema/crdb/add-webhooks/up02.sql b/schema/crdb/add-webhooks/up02.sql new file mode 100644 index 00000000000..df945f4299e --- /dev/null +++ b/schema/crdb/add-webhooks/up02.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- ID of this secret. + signature_id STRING(63) NOT NULL, + -- Secret value. + secret BYTES NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + + PRIMARY KEY (signature_id, rx_id) +); diff --git a/schema/crdb/add-webhooks/up03.sql b/schema/crdb/add-webhooks/up03.sql new file mode 100644 index 00000000000..5a799088577 --- /dev/null +++ b/schema/crdb/add-webhooks/up03.sql @@ -0,0 +1,5 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx +ON omicron.public.webhook_rx_secret ( + rx_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/add-webhooks/up04.sql b/schema/crdb/add-webhooks/up04.sql new file mode 100644 index 00000000000..7911418a786 --- /dev/null +++ b/schema/crdb/add-webhooks/up04.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which this receiver is subscribed. + event_class STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) +); diff --git a/schema/crdb/add-webhooks/up05.sql b/schema/crdb/add-webhooks/up05.sql new file mode 100644 index 00000000000..4ffe7cbce05 --- /dev/null +++ b/schema/crdb/add-webhooks/up05.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx +ON omicron.public.webhook_rx_subscription ( + rx_id +); diff --git a/schema/crdb/add-webhooks/up06.sql b/schema/crdb/add-webhooks/up06.sql new file mode 100644 index 00000000000..08a44f54e38 --- /dev/null +++ b/schema/crdb/add-webhooks/up06.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( + -- UUID of this dispatch. + id UUID PRIMARY KEY, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + payload JSONB NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, +); diff --git a/schema/crdb/add-webhooks/up07.sql b/schema/crdb/add-webhooks/up07.sql new file mode 100644 index 00000000000..4cc13a67cc8 --- /dev/null +++ b/schema/crdb/add-webhooks/up07.sql @@ -0,0 +1,5 @@ +-- Index for looking up all webhook messages dispatched to a receiver ID +CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +ON omicron.public.webhook_msg_dispatch ( + rx_id +); diff --git a/schema/crdb/add-webhooks/up08.sql b/schema/crdb/add-webhooks/up08.sql new file mode 100644 index 00000000000..d7cb44b173a --- /dev/null +++ b/schema/crdb/add-webhooks/up08.sql @@ -0,0 +1,7 @@ +-- Index for looking up all currently in-flight webhook messages, and ordering +-- them by their creation times. +CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight +ON omicron.public.webhook_msg_dispatch ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql new file mode 100644 index 00000000000..00e5cb3e7b2 --- /dev/null +++ b/schema/crdb/add-webhooks/up09.sql @@ -0,0 +1,9 @@ +CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + -- The delivery attempt succeeded. + 'succeeded' +); diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql new file mode 100644 index 00000000000..19f87bf459e --- /dev/null +++ b/schema/crdb/add-webhooks/up10.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( + id UUID PRIMARY KEY, + -- Foreign key into `omicron.public.webhook_msg_dispatch`. + dispatch_id UUID NOT NULL, + result omicron.public.webhook_msg_delivery_result NOT NULL, + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable', no response data is + -- present. + (result = 'failed_unreachable') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) +); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql new file mode 100644 index 00000000000..2a32f10969f --- /dev/null +++ b/schema/crdb/add-webhooks/up11.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg +ON omicron.public.webhook_msg_delivery_attempts ( + dispatch_id +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1ee7b985aff..4b60a591604 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4799,6 +4799,137 @@ CREATE UNIQUE INDEX IF NOT EXISTS one_record_per_volume_resource_usage on omicro region_snapshot_snapshot_id ); +/* + * WEBHOOKS + */ + + +/* + * Webhook receivers, receiver secrets, and receiver subscriptions. + */ + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + id UUID PRIMARY KEY, + -- A human-readable identifier for this webhook receiver. + name STRING(63) NOT NULL, + -- URL of the endpoint webhooks are delivered to. + endpoint STRING(512) NOT NULL, + -- TODO(eliza): how do we track which roles are assigned to a webhook? + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ, + time_deleted TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- ID of this secret. + signature_id STRING(63) NOT NULL, + -- Secret value. + secret BYTES NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + + PRIMARY KEY (signature_id, rx_id) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx +ON omicron.public.webhook_rx_secret ( + rx_id +) WHERE + time_deleted IS NULL; + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which this receiver is subscribed. + event_class STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx +ON omicron.public.webhook_rx_subscription ( + rx_id +); + +/* + * Webhook message dispatching and delivery attempts. + */ + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( + -- UUID of this dispatch. + id UUID PRIMARY KEY, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + payload JSONB NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, +); + +-- Index for looking up all webhook messages dispatched to a receiver ID +CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +ON omicron.public.webhook_msg_dispatch ( + rx_id +); + +-- Index for looking up all currently in-flight webhook messages, and ordering +-- them by their creation times. +CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight +ON omicron.public.webhook_msg_dispatch ( + time_created, id +) WHERE + time_completed IS NULL; + +CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + -- The delivery attempt succeeded. + 'succeeded' +); + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( + id UUID PRIMARY KEY, + -- Foreign key into `omicron.public.webhook_msg_dispatch`. + dispatch_id UUID NOT NULL, + result omicron.public.webhook_msg_delivery_result NOT NULL, + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable', no response data is + -- present. + (result = 'failed_unreachable') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg +ON omicron.public.webhook_msg_delivery_attempts ( + dispatch_id +); + /* * Keep this at the end of file so that the database does not contain a version * until it is fully populated. From 5bf263325e3c5a87f3abe7db22148d8a3d0bdfb8 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Dec 2024 13:32:15 -0800 Subject: [PATCH 008/168] message queue --- schema/crdb/add-webhooks/README.adoc | 23 ++++++++++++++----- schema/crdb/add-webhooks/up06.sql | 16 +++++--------- schema/crdb/add-webhooks/up07.sql | 13 +++++++---- schema/crdb/add-webhooks/up08.sql | 14 ++++++------ schema/crdb/add-webhooks/up09.sql | 19 +++++++++------- schema/crdb/add-webhooks/up10.sql | 30 ++++--------------------- schema/crdb/add-webhooks/up11.sql | 11 ++++++---- schema/crdb/add-webhooks/up12.sql | 9 ++++++++ schema/crdb/add-webhooks/up13.sql | 27 +++++++++++++++++++++++ schema/crdb/add-webhooks/up14.sql | 4 ++++ schema/crdb/dbinit.sql | 33 +++++++++++++++++++++++++++- 11 files changed, 132 insertions(+), 67 deletions(-) create mode 100644 schema/crdb/add-webhooks/up12.sql create mode 100644 schema/crdb/add-webhooks/up13.sql create mode 100644 schema/crdb/add-webhooks/up14.sql diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index f3ea209b559..c913776b737 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -20,21 +20,32 @@ associates a webhook receiver with multiple event classes that the receiver is subscribed to. *** `up05.sql` creates an index `lookup_webhook_subscriptions_by_rx` for looking up all event classes that a receiver ID is subscribed to. +*** `up06.sql` creates an index `lookup_webhook_rxs_for_event` on +`omicron.public.webhook_rx_subscription` for looking up all receivers subscribed +to a particular event class. +* *Webhook message queue*: +** `up07.sql` creates the `omicron.public.webhook_msg` table, which contains the +queue of un-dispatched webhook events. The dispatcher operates on entries in +this queue, dispatching the event to receivers and generating the payload for +each receiver. +** `up08.sql` creates the `lookup_undispatched_webhook_msgs` index on +`omicron.public.webhook_msg` for looking up webhook messages which have not yet been +dispatched and ordering by their creation times. * *Webhook message dispatching and delivery attempts*: ** *Dispatch table*: -*** `up06.sql` creates the table `omicron.public.webhook_msg_dispatch`, which +*** `up09.sql` creates the table `omicron.public.webhook_msg_dispatch`, which tracks the webhook messages that have been dispatched to receivers. -*** `up07.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up +*** `up10.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up entries in `omicron.public.webhook_msg_dispatch` by receiver ID. -*** `up08.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook +*** `up11.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook messages (entries in `omicron.public.webhook_msg_dispatch` where the `time_completed` field has not been set). ** *Delivery attempts*: -*** `up09.sql` creates the enum `omicron.public.webhook_msg_delivery_result`, +*** `up12.sql` creates the enum `omicron.public.webhook_msg_delivery_result`, representing the potential outcomes of a webhook delivery attempt. -*** `up10.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`, +*** `up13.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`, which records each individual delivery attempt for a webhook message in the `webhook_msg_dispatch` table. -*** `up11.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on +*** `up14.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on `omicron.public.webhook_msg_delivery_attempt`, for looking up all attempts to deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up06.sql b/schema/crdb/add-webhooks/up06.sql index 08a44f54e38..407424440c6 100644 --- a/schema/crdb/add-webhooks/up06.sql +++ b/schema/crdb/add-webhooks/up06.sql @@ -1,12 +1,6 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( - -- UUID of this dispatch. - id UUID PRIMARY KEY, - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - payload JSONB NOT NULL, - time_created TIMESTAMPTZ NOT NULL, - -- If this is set, then this webhook message has either been delivered - -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, +-- Look up all webhook receivers subscribed to an event class. This is used by +-- the dispatcher to determine who is interested in a particular event. +CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event +ON omicron.public.webhook_rx_subscription ( + event_class ); diff --git a/schema/crdb/add-webhooks/up07.sql b/schema/crdb/add-webhooks/up07.sql index 4cc13a67cc8..ba667e930f5 100644 --- a/schema/crdb/add-webhooks/up07.sql +++ b/schema/crdb/add-webhooks/up07.sql @@ -1,5 +1,10 @@ --- Index for looking up all webhook messages dispatched to a receiver ID -CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx -ON omicron.public.webhook_msg_dispatch ( - rx_id +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- Set when dispatch entries have been created for this event. + time_dispatched TIMESTAMPTZ, + -- The class of event that this is. + event_class STRING(512) NOT NULL, + -- Actual event data. The structure of this depends on the event class. + event JSONB NOT NULL ); diff --git a/schema/crdb/add-webhooks/up08.sql b/schema/crdb/add-webhooks/up08.sql index d7cb44b173a..e18c2cea1a5 100644 --- a/schema/crdb/add-webhooks/up08.sql +++ b/schema/crdb/add-webhooks/up08.sql @@ -1,7 +1,7 @@ --- Index for looking up all currently in-flight webhook messages, and ordering --- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight -ON omicron.public.webhook_msg_dispatch ( - time_created, id -) WHERE - time_completed IS NULL; +-- Look up webhook messages in need of dispatching. +-- +-- This is used by the message dispatcher when looking for messages to dispatch. +CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_msgs +ON omicron.public.webhook_msg ( + id, time_created +) WHERE time_dispatched IS NULL; diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql index 00e5cb3e7b2..08a44f54e38 100644 --- a/schema/crdb/add-webhooks/up09.sql +++ b/schema/crdb/add-webhooks/up09.sql @@ -1,9 +1,12 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( - -- The delivery attempt failed with an HTTP error. - 'failed_http_error', - -- The delivery attempt failed because the receiver endpoint was - -- unreachable. - 'failed_unreachable', - -- The delivery attempt succeeded. - 'succeeded' +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( + -- UUID of this dispatch. + id UUID PRIMARY KEY, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + payload JSONB NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, ); diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql index 19f87bf459e..4cc13a67cc8 100644 --- a/schema/crdb/add-webhooks/up10.sql +++ b/schema/crdb/add-webhooks/up10.sql @@ -1,27 +1,5 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( - id UUID PRIMARY KEY, - -- Foreign key into `omicron.public.webhook_msg_dispatch`. - dispatch_id UUID NOT NULL, - result omicron.public.webhook_msg_delivery_result NOT NULL, - response_status INT2, - response_duration INTERVAL, - time_created TIMESTAMPTZ NOT NULL, - - CONSTRAINT response_iff_not_unreachable CHECK ( - ( - -- If the result is 'succeedeed' or 'failed_http_error', response - -- data must be present. - (result = 'succeeded' OR result = 'failed_http_error') AND ( - response_status IS NOT NULL AND - response_duration IS NOT NULL - ) - ) OR ( - -- If the result is 'failed_unreachable', no response data is - -- present. - (result = 'failed_unreachable') AND ( - response_status IS NULL AND - response_duration IS NULL - ) - ) - ) +-- Index for looking up all webhook messages dispatched to a receiver ID +CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +ON omicron.public.webhook_msg_dispatch ( + rx_id ); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql index 2a32f10969f..d7cb44b173a 100644 --- a/schema/crdb/add-webhooks/up11.sql +++ b/schema/crdb/add-webhooks/up11.sql @@ -1,4 +1,7 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_msg_delivery_attempts ( - dispatch_id -); +-- Index for looking up all currently in-flight webhook messages, and ordering +-- them by their creation times. +CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight +ON omicron.public.webhook_msg_dispatch ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up12.sql b/schema/crdb/add-webhooks/up12.sql new file mode 100644 index 00000000000..00e5cb3e7b2 --- /dev/null +++ b/schema/crdb/add-webhooks/up12.sql @@ -0,0 +1,9 @@ +CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + -- The delivery attempt succeeded. + 'succeeded' +); diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/add-webhooks/up13.sql new file mode 100644 index 00000000000..19f87bf459e --- /dev/null +++ b/schema/crdb/add-webhooks/up13.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( + id UUID PRIMARY KEY, + -- Foreign key into `omicron.public.webhook_msg_dispatch`. + dispatch_id UUID NOT NULL, + result omicron.public.webhook_msg_delivery_result NOT NULL, + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable', no response data is + -- present. + (result = 'failed_unreachable') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) +); diff --git a/schema/crdb/add-webhooks/up14.sql b/schema/crdb/add-webhooks/up14.sql new file mode 100644 index 00000000000..2a32f10969f --- /dev/null +++ b/schema/crdb/add-webhooks/up14.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg +ON omicron.public.webhook_msg_delivery_attempts ( + dispatch_id +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4b60a591604..933dc0ac3ff 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4840,7 +4840,7 @@ ON omicron.public.webhook_rx_secret ( ) WHERE time_deleted IS NULL; -CREATE TABLE IF NOT EXISTS omicron.public.webhook_subscription ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, @@ -4856,6 +4856,37 @@ ON omicron.public.webhook_rx_subscription ( rx_id ); +-- Look up all webhook receivers subscribed to an event class. This is used by +-- the dispatcher to determine who is interested in a particular event. +CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event +ON omicron.public.webhook_rx_subscription ( + event_class +); + +/* + * Webhook message queue. + */ + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- Set when dispatch entries have been created for this event. + time_dispatched TIMESTAMPTZ, + -- The class of event that this is. + event_class STRING(512) NOT NULL, + -- Actual event data. The structure of this depends on the event class. + event JSONB NOT NULL +); + +-- Look up webhook messages in need of dispatching. +-- +-- This is used by the message dispatcher when looking for messages to dispatch. +CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_msgs +ON omicron.public.webhook_msg ( + id, time_created +) WHERE time_dispatched IS NULL; + + /* * Webhook message dispatching and delivery attempts. */ From 52efc087ec091eab485e3c1c6585cd62d895cb87 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Dec 2024 14:24:39 -0800 Subject: [PATCH 009/168] more diesel plumbing --- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 75 ++++++++++++++++++++++ nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/webhook_msg_delivery.rs | 30 +++++++++ schema/crdb/dbinit.sql | 2 +- 5 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 nexus/db-model/src/webhook_msg_delivery.rs diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bec35c233c6..bd8c2afd3e9 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -64,6 +64,7 @@ mod switch_interface; mod switch_port; mod v2p_mapping; mod vmm_state; +mod webhook_msg_delivery; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. @@ -129,6 +130,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; +pub use self::webhook_msg_delivery::*; pub use address_lot::*; pub use allow_list::*; pub use bfd::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f734d1e88f3..a7cac7f6d54 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2129,3 +2129,78 @@ table! { region_snapshot_snapshot_id -> Nullable, } } + +table! { + webhook_rx (id) { + id -> Uuid, + name -> Text, + endpoint -> Text, + time_created -> Timestamptz, + time_modified -> Nullable, + time_deleted -> Nullable, + } +} + +table! { + webhook_rx_secret (rx_id, signature_id) { + rx_id -> Uuid, + signature_id -> Text, + secret -> Binary, + time_created -> Timestamptz, + time_deleted -> Nullable, + } +} + +allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_secret); +joinable!(webhook_rx_secret -> webhook_rx (rx_id)); + +table! { + webhook_rx_subscription (rx_id, event_class) { + rx_id -> Uuid, + event_class -> Text, + time_created -> Timestamptz, + } +} + +allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_subscription); +joinable!(webhook_rx_subscription -> webhook_rx (rx_id)); + +table! { + webhook_msg (id) { + id -> Uuid, + time_created -> Timestamptz, + time_dispatched -> Nullable, + event_class -> Text, + event -> Jsonb, + } +} + +table! { + webhook_msg_dispatch (id) { + id -> Uuid, + rx_id -> Uuid, + payload -> Jsonb, + time_created -> Timestamptz, + time_completed -> Nullable, + } +} + +allow_tables_to_appear_in_same_query!(webhook_rx, webhook_msg_dispatch); +joinable!(webhook_msg_dispatch -> webhook_rx (rx_id)); + +table! { + webhook_msg_delivery_attempt (id) { + id -> Uuid, + dispatch_id -> Uuid, + result -> crate::WebhookDeliveryResultEnum, + response_status -> Nullable, + response_duration -> Nullable, + time_created -> Timestamptz, + } +} + +allow_tables_to_appear_in_same_query!( + webhook_msg_dispatch, + webhook_msg_delivery_attempt +); +joinable!(webhook_msg_delivery_attempt -> webhook_msg_dispatch (dispatch_id)); diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 150f4d092df..a360eba7eb7 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(122, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(123, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(123, "add-webhooks"), KnownVersion::new(122, "tuf-artifact-replication"), KnownVersion::new(121, "dataset-to-crucible-dataset"), KnownVersion::new(120, "rendezvous-debug-dataset"), diff --git a/nexus/db-model/src/webhook_msg_delivery.rs b/nexus/db-model/src/webhook_msg_delivery.rs new file mode 100644 index 00000000000..d9724ef9a5f --- /dev/null +++ b/nexus/db-model/src/webhook_msg_delivery.rs @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use serde::Deserialize; +use serde::Serialize; + +impl_enum_type!( + #[derive(SqlType, Debug, Clone)] + #[diesel(postgres_type(name = "webhook_msg_delivery_result", schema = "public"))] + pub struct WebhookDeliveryResultEnum; + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, + )] + #[diesel(sql_type = WebhookDeliveryResultEnum)] + pub enum WebhookDeliveryResult; + + FailedHttpError => b"failed_http_error" + FailedUnreachable => b"failed_unreachable" + Succeeded => b"succeeded" +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 933dc0ac3ff..97a1571d374 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4972,7 +4972,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '122.0.0', NULL) + (TRUE, NOW(), NOW(), '123.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From 72754d74323c8afbe41454b0a0d3d945481279c5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 14:48:58 -0800 Subject: [PATCH 010/168] terminology tweaks --- nexus/db-model/src/lib.rs | 4 +-- nexus/db-model/src/schema.rs | 18 ++++++------ nexus/db-model/src/webhook_msg_delivery.rs | 2 +- schema/crdb/add-webhooks/README.adoc | 22 +++++++------- schema/crdb/add-webhooks/up07.sql | 2 +- schema/crdb/add-webhooks/up08.sql | 4 +-- schema/crdb/add-webhooks/up09.sql | 2 +- schema/crdb/add-webhooks/up10.sql | 2 +- schema/crdb/add-webhooks/up11.sql | 4 +-- schema/crdb/add-webhooks/up12.sql | 2 +- schema/crdb/add-webhooks/up13.sql | 8 ++--- schema/crdb/add-webhooks/up14.sql | 4 +-- schema/crdb/dbinit.sql | 34 +++++++++++----------- 13 files changed, 54 insertions(+), 54 deletions(-) diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bd8c2afd3e9..bfaaed1b1f7 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -64,7 +64,7 @@ mod switch_interface; mod switch_port; mod v2p_mapping; mod vmm_state; -mod webhook_msg_delivery; +mod webhook_event_delivery; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. @@ -130,7 +130,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; -pub use self::webhook_msg_delivery::*; +pub use self::webhook_event_delivery::*; pub use address_lot::*; pub use allow_list::*; pub use bfd::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index a7cac7f6d54..20f2ec2c927 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2166,7 +2166,7 @@ allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_subscription); joinable!(webhook_rx_subscription -> webhook_rx (rx_id)); table! { - webhook_msg (id) { + webhook_event (id) { id -> Uuid, time_created -> Timestamptz, time_dispatched -> Nullable, @@ -2176,7 +2176,7 @@ table! { } table! { - webhook_msg_dispatch (id) { + webhook_delivery (id) { id -> Uuid, rx_id -> Uuid, payload -> Jsonb, @@ -2185,13 +2185,13 @@ table! { } } -allow_tables_to_appear_in_same_query!(webhook_rx, webhook_msg_dispatch); -joinable!(webhook_msg_dispatch -> webhook_rx (rx_id)); +allow_tables_to_appear_in_same_query!(webhook_rx, webhook_delivery); +joinable!(webhook_delivery -> webhook_rx (rx_id)); table! { - webhook_msg_delivery_attempt (id) { + webhook_event_delivery_attempt (id) { id -> Uuid, - dispatch_id -> Uuid, + delivery_id -> Uuid, result -> crate::WebhookDeliveryResultEnum, response_status -> Nullable, response_duration -> Nullable, @@ -2200,7 +2200,7 @@ table! { } allow_tables_to_appear_in_same_query!( - webhook_msg_dispatch, - webhook_msg_delivery_attempt + webhook_delivery, + webhook_event_delivery_attempt ); -joinable!(webhook_msg_delivery_attempt -> webhook_msg_dispatch (dispatch_id)); +joinable!(webhook_event_delivery_attempt -> webhook_delivery (delivery_id)); diff --git a/nexus/db-model/src/webhook_msg_delivery.rs b/nexus/db-model/src/webhook_msg_delivery.rs index d9724ef9a5f..2f25c0dbc7d 100644 --- a/nexus/db-model/src/webhook_msg_delivery.rs +++ b/nexus/db-model/src/webhook_msg_delivery.rs @@ -8,7 +8,7 @@ use serde::Serialize; impl_enum_type!( #[derive(SqlType, Debug, Clone)] - #[diesel(postgres_type(name = "webhook_msg_delivery_result", schema = "public"))] + #[diesel(postgres_type(name = "webhook_event_delivery_result", schema = "public"))] pub struct WebhookDeliveryResultEnum; #[derive( diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index c913776b737..9eff8b16ea6 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -24,28 +24,28 @@ looking up all event classes that a receiver ID is subscribed to. `omicron.public.webhook_rx_subscription` for looking up all receivers subscribed to a particular event class. * *Webhook message queue*: -** `up07.sql` creates the `omicron.public.webhook_msg` table, which contains the +** `up07.sql` creates the `omicron.public.webhook_event` table, which contains the queue of un-dispatched webhook events. The dispatcher operates on entries in this queue, dispatching the event to receivers and generating the payload for each receiver. -** `up08.sql` creates the `lookup_undispatched_webhook_msgs` index on -`omicron.public.webhook_msg` for looking up webhook messages which have not yet been +** `up08.sql` creates the `lookup_undispatched_webhook_events` index on +`omicron.public.webhook_event` for looking up webhook messages which have not yet been dispatched and ordering by their creation times. * *Webhook message dispatching and delivery attempts*: ** *Dispatch table*: -*** `up09.sql` creates the table `omicron.public.webhook_msg_dispatch`, which +*** `up09.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. *** `up10.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up -entries in `omicron.public.webhook_msg_dispatch` by receiver ID. -*** `up11.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook -messages (entries in `omicron.public.webhook_msg_dispatch` where the +entries in `omicron.public.webhook_delivery` by receiver ID. +*** `up11.sql` creates an index `webhook_deliverey_in_flight` for looking up all currently in-flight webhook +messages (entries in `omicron.public.webhook_delivery` where the `time_completed` field has not been set). ** *Delivery attempts*: -*** `up12.sql` creates the enum `omicron.public.webhook_msg_delivery_result`, +*** `up12.sql` creates the enum `omicron.public.webhook_event_delivery_result`, representing the potential outcomes of a webhook delivery attempt. -*** `up13.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`, +*** `up13.sql` creates the table `omicron.public.webhook_event_delivery_attempt`, which records each individual delivery attempt for a webhook message in the -`webhook_msg_dispatch` table. +`webhook_delivery` table. *** `up14.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on -`omicron.public.webhook_msg_delivery_attempt`, for looking up all attempts to +`omicron.public.webhook_event_delivery_attempt`, for looking up all attempts to deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up07.sql b/schema/crdb/add-webhooks/up07.sql index ba667e930f5..99e00dfaebc 100644 --- a/schema/crdb/add-webhooks/up07.sql +++ b/schema/crdb/add-webhooks/up07.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( id UUID PRIMARY KEY, time_created TIMESTAMPTZ NOT NULL, -- Set when dispatch entries have been created for this event. diff --git a/schema/crdb/add-webhooks/up08.sql b/schema/crdb/add-webhooks/up08.sql index e18c2cea1a5..c64ae2db0df 100644 --- a/schema/crdb/add-webhooks/up08.sql +++ b/schema/crdb/add-webhooks/up08.sql @@ -1,7 +1,7 @@ -- Look up webhook messages in need of dispatching. -- -- This is used by the message dispatcher when looking for messages to dispatch. -CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_msgs -ON omicron.public.webhook_msg ( +CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_events +ON omicron.public.webhook_event ( id, time_created ) WHERE time_dispatched IS NULL; diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql index 08a44f54e38..02d13a552bf 100644 --- a/schema/crdb/add-webhooks/up09.sql +++ b/schema/crdb/add-webhooks/up09.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of this dispatch. id UUID PRIMARY KEY, -- UUID of the webhook receiver (foreign key into diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql index 4cc13a67cc8..412930194f8 100644 --- a/schema/crdb/add-webhooks/up10.sql +++ b/schema/crdb/add-webhooks/up10.sql @@ -1,5 +1,5 @@ -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx -ON omicron.public.webhook_msg_dispatch ( +ON omicron.public.webhook_delivery ( rx_id ); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql index d7cb44b173a..b7f9d57c8c1 100644 --- a/schema/crdb/add-webhooks/up11.sql +++ b/schema/crdb/add-webhooks/up11.sql @@ -1,7 +1,7 @@ -- Index for looking up all currently in-flight webhook messages, and ordering -- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight -ON omicron.public.webhook_msg_dispatch ( +CREATE INDEX IF NOT EXISTS webhook_deliverey_in_flight +ON omicron.public.webhook_delivery ( time_created, id ) WHERE time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up12.sql b/schema/crdb/add-webhooks/up12.sql index 00e5cb3e7b2..87d21c7df82 100644 --- a/schema/crdb/add-webhooks/up12.sql +++ b/schema/crdb/add-webhooks/up12.sql @@ -1,4 +1,4 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( +CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_delivery_result as ENUM ( -- The delivery attempt failed with an HTTP error. 'failed_http_error', -- The delivery attempt failed because the receiver endpoint was diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/add-webhooks/up13.sql index 19f87bf459e..89f3cff55b6 100644 --- a/schema/crdb/add-webhooks/up13.sql +++ b/schema/crdb/add-webhooks/up13.sql @@ -1,8 +1,8 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event_delivery_attempt ( id UUID PRIMARY KEY, - -- Foreign key into `omicron.public.webhook_msg_dispatch`. - dispatch_id UUID NOT NULL, - result omicron.public.webhook_msg_delivery_result NOT NULL, + -- Foreign key into `omicron.public.webhook_delivery`. + delivery_id UUID NOT NULL, + result omicron.public.webhook_event_delivery_result NOT NULL, response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, diff --git a/schema/crdb/add-webhooks/up14.sql b/schema/crdb/add-webhooks/up14.sql index 2a32f10969f..17853efbcba 100644 --- a/schema/crdb/add-webhooks/up14.sql +++ b/schema/crdb/add-webhooks/up14.sql @@ -1,4 +1,4 @@ CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_msg_delivery_attempts ( - dispatch_id +ON omicron.public.webhook_event_delivery_attempts ( + delivery_id ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 97a1571d374..77df8e4eebf 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4864,10 +4864,10 @@ ON omicron.public.webhook_rx_subscription ( ); /* - * Webhook message queue. + * Webhook event message queue. */ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( id UUID PRIMARY KEY, time_created TIMESTAMPTZ NOT NULL, -- Set when dispatch entries have been created for this event. @@ -4878,11 +4878,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg ( event JSONB NOT NULL ); --- Look up webhook messages in need of dispatching. +-- Look up webhook events in need of dispatching. -- --- This is used by the message dispatcher when looking for messages to dispatch. -CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_msgs -ON omicron.public.webhook_msg ( +-- This is used by the message dispatcher when looking for events to dispatch. +CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_events +ON omicron.public.webhook_event ( id, time_created ) WHERE time_dispatched IS NULL; @@ -4891,7 +4891,7 @@ ON omicron.public.webhook_msg ( * Webhook message dispatching and delivery attempts. */ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of this dispatch. id UUID PRIMARY KEY, -- UUID of the webhook receiver (foreign key into @@ -4906,19 +4906,19 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx -ON omicron.public.webhook_msg_dispatch ( +ON omicron.public.webhook_delivery ( rx_id ); -- Index for looking up all currently in-flight webhook messages, and ordering -- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight -ON omicron.public.webhook_msg_dispatch ( +CREATE INDEX IF NOT EXISTS webhook_deliverey_in_flight +ON omicron.public.webhook_delivery ( time_created, id ) WHERE time_completed IS NULL; -CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( +CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_delivery_result as ENUM ( -- The delivery attempt failed with an HTTP error. 'failed_http_error', -- The delivery attempt failed because the receiver endpoint was @@ -4928,11 +4928,11 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( 'succeeded' ); -CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event_delivery_attempt ( id UUID PRIMARY KEY, - -- Foreign key into `omicron.public.webhook_msg_dispatch`. - dispatch_id UUID NOT NULL, - result omicron.public.webhook_msg_delivery_result NOT NULL, + -- Foreign key into `omicron.public.webhook_delivery`. + delivery_id UUID NOT NULL, + result omicron.public.webhook_event_delivery_result NOT NULL, response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, @@ -4957,8 +4957,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( ); CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_msg_delivery_attempts ( - dispatch_id +ON omicron.public.webhook_event_delivery_attempts ( + delivery_id ); /* From fa75a2f7aa7bdc94e2e177559b28720d747d9769 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 14:55:40 -0800 Subject: [PATCH 011/168] change tracking of delivery attempts --- nexus/db-model/src/lib.rs | 4 ++-- nexus/db-model/src/schema.rs | 6 +++--- ...hook_msg_delivery.rs => webhook_delivery.rs} | 2 +- schema/crdb/add-webhooks/README.adoc | 6 +++--- schema/crdb/add-webhooks/up09.sql | 2 ++ schema/crdb/add-webhooks/up10.sql | 2 +- schema/crdb/add-webhooks/up11.sql | 2 +- schema/crdb/add-webhooks/up12.sql | 2 +- schema/crdb/add-webhooks/up13.sql | 9 ++++++--- schema/crdb/add-webhooks/up14.sql | 2 +- schema/crdb/dbinit.sql | 17 +++++++++++------ 11 files changed, 32 insertions(+), 22 deletions(-) rename nexus/db-model/src/{webhook_msg_delivery.rs => webhook_delivery.rs} (89%) diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bfaaed1b1f7..608fb1383d8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -64,7 +64,7 @@ mod switch_interface; mod switch_port; mod v2p_mapping; mod vmm_state; -mod webhook_event_delivery; +mod webhook_delivery; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. @@ -130,7 +130,6 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; -pub use self::webhook_event_delivery::*; pub use address_lot::*; pub use allow_list::*; pub use bfd::*; @@ -232,6 +231,7 @@ pub use vpc_firewall_rule::*; pub use vpc_route::*; pub use vpc_router::*; pub use vpc_subnet::*; +pub use webhook_delivery::*; pub use zpool::*; // TODO: The existence of both impl_enum_type and impl_enum_wrapper is a diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 20f2ec2c927..ef1e00b308c 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2189,7 +2189,7 @@ allow_tables_to_appear_in_same_query!(webhook_rx, webhook_delivery); joinable!(webhook_delivery -> webhook_rx (rx_id)); table! { - webhook_event_delivery_attempt (id) { + webhook_delivery_attempt (id) { id -> Uuid, delivery_id -> Uuid, result -> crate::WebhookDeliveryResultEnum, @@ -2201,6 +2201,6 @@ table! { allow_tables_to_appear_in_same_query!( webhook_delivery, - webhook_event_delivery_attempt + webhook_delivery_attempt ); -joinable!(webhook_event_delivery_attempt -> webhook_delivery (delivery_id)); +joinable!(webhook_delivery_attempt -> webhook_delivery (delivery_id)); diff --git a/nexus/db-model/src/webhook_msg_delivery.rs b/nexus/db-model/src/webhook_delivery.rs similarity index 89% rename from nexus/db-model/src/webhook_msg_delivery.rs rename to nexus/db-model/src/webhook_delivery.rs index 2f25c0dbc7d..1133813a404 100644 --- a/nexus/db-model/src/webhook_msg_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -8,7 +8,7 @@ use serde::Serialize; impl_enum_type!( #[derive(SqlType, Debug, Clone)] - #[diesel(postgres_type(name = "webhook_event_delivery_result", schema = "public"))] + #[diesel(postgres_type(name = "webhook_delivery_result", schema = "public"))] pub struct WebhookDeliveryResultEnum; #[derive( diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index 9eff8b16ea6..7a51cafd7c7 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -41,11 +41,11 @@ entries in `omicron.public.webhook_delivery` by receiver ID. messages (entries in `omicron.public.webhook_delivery` where the `time_completed` field has not been set). ** *Delivery attempts*: -*** `up12.sql` creates the enum `omicron.public.webhook_event_delivery_result`, +*** `up12.sql` creates the enum `omicron.public.webhook_delivery_result`, representing the potential outcomes of a webhook delivery attempt. -*** `up13.sql` creates the table `omicron.public.webhook_event_delivery_attempt`, +*** `up13.sql` creates the table `omicron.public.webhook_delivery_attempt`, which records each individual delivery attempt for a webhook message in the `webhook_delivery` table. *** `up14.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on -`omicron.public.webhook_event_delivery_attempt`, for looking up all attempts to +`omicron.public.webhook_delivery_attempt`, for looking up all attempts to deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql index 02d13a552bf..c2ee13acdb0 100644 --- a/schema/crdb/add-webhooks/up09.sql +++ b/schema/crdb/add-webhooks/up09.sql @@ -1,6 +1,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of this dispatch. id UUID PRIMARY KEY, + --- UUID of the event (foreign key into `omicron.public.webhook_event`). + event_id UUID NOT NULL, -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql index 412930194f8..0e67ca550f9 100644 --- a/schema/crdb/add-webhooks/up10.sql +++ b/schema/crdb/add-webhooks/up10.sql @@ -1,5 +1,5 @@ -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx ON omicron.public.webhook_delivery ( - rx_id + rx_id, event_id ); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql index b7f9d57c8c1..3b4532836d7 100644 --- a/schema/crdb/add-webhooks/up11.sql +++ b/schema/crdb/add-webhooks/up11.sql @@ -1,4 +1,4 @@ --- Index for looking up all currently in-flight webhook messages, and ordering +-- Index for looking up all currently in-flight webhook deliveries, and ordering -- them by their creation times. CREATE INDEX IF NOT EXISTS webhook_deliverey_in_flight ON omicron.public.webhook_delivery ( diff --git a/schema/crdb/add-webhooks/up12.sql b/schema/crdb/add-webhooks/up12.sql index 87d21c7df82..dbe765f9060 100644 --- a/schema/crdb/add-webhooks/up12.sql +++ b/schema/crdb/add-webhooks/up12.sql @@ -1,4 +1,4 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_delivery_result as ENUM ( +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( -- The delivery attempt failed with an HTTP error. 'failed_http_error', -- The delivery attempt failed because the receiver endpoint was diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/add-webhooks/up13.sql index 89f3cff55b6..4bc92ef3d5b 100644 --- a/schema/crdb/add-webhooks/up13.sql +++ b/schema/crdb/add-webhooks/up13.sql @@ -1,12 +1,15 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_event_delivery_attempt ( - id UUID PRIMARY KEY, +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- Foreign key into `omicron.public.webhook_delivery`. delivery_id UUID NOT NULL, - result omicron.public.webhook_event_delivery_result NOT NULL, + -- attempt number. + attempt INT2 NOT NULL, + result omicron.public.webhook_delivery_result NOT NULL, response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, + PRIMARY KEY (delivery_id, attempt), + CONSTRAINT response_iff_not_unreachable CHECK ( ( -- If the result is 'succeedeed' or 'failed_http_error', response diff --git a/schema/crdb/add-webhooks/up14.sql b/schema/crdb/add-webhooks/up14.sql index 17853efbcba..05ec2ba0c95 100644 --- a/schema/crdb/add-webhooks/up14.sql +++ b/schema/crdb/add-webhooks/up14.sql @@ -1,4 +1,4 @@ CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_event_delivery_attempts ( +ON omicron.public.webhook_delivery_attempts ( delivery_id ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 77df8e4eebf..2fe1a4cc559 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4894,6 +4894,8 @@ ON omicron.public.webhook_event ( CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of this dispatch. id UUID PRIMARY KEY, + --- UUID of the event (foreign key into `omicron.public.webhook_event`). + event_id UUID NOT NULL, -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, @@ -4907,7 +4909,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx ON omicron.public.webhook_delivery ( - rx_id + rx_id, event_id, ); -- Index for looking up all currently in-flight webhook messages, and ordering @@ -4918,7 +4920,7 @@ ON omicron.public.webhook_delivery ( ) WHERE time_completed IS NULL; -CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_delivery_result as ENUM ( +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( -- The delivery attempt failed with an HTTP error. 'failed_http_error', -- The delivery attempt failed because the receiver endpoint was @@ -4928,15 +4930,18 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_delivery_result as ENUM ( 'succeeded' ); -CREATE TABLE IF NOT EXISTS omicron.public.webhook_event_delivery_attempt ( - id UUID PRIMARY KEY, +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- Foreign key into `omicron.public.webhook_delivery`. delivery_id UUID NOT NULL, - result omicron.public.webhook_event_delivery_result NOT NULL, + -- attempt number. + attempt INT2 NOT NULL, + result omicron.public.webhook_delivery_result NOT NULL, response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, + PRIMARY KEY (delivery_id, attempt), + CONSTRAINT response_iff_not_unreachable CHECK ( ( -- If the result is 'succeedeed' or 'failed_http_error', response @@ -4957,7 +4962,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_event_delivery_attempt ( ); CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_event_delivery_attempts ( +ON omicron.public.webhook_delivery_attempts ( delivery_id ); From f9e5fc924678f9baeefab2fd86f1c43ddf3f0a22 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 14:56:29 -0800 Subject: [PATCH 012/168] s/deliverey/delivery --- schema/crdb/add-webhooks/README.adoc | 2 +- schema/crdb/add-webhooks/up11.sql | 2 +- schema/crdb/dbinit.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index 7a51cafd7c7..6f724c75f16 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -37,7 +37,7 @@ dispatched and ordering by their creation times. tracks the webhook messages that have been dispatched to receivers. *** `up10.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up entries in `omicron.public.webhook_delivery` by receiver ID. -*** `up11.sql` creates an index `webhook_deliverey_in_flight` for looking up all currently in-flight webhook +*** `up11.sql` creates an index `webhook_delivery_in_flight` for looking up all currently in-flight webhook messages (entries in `omicron.public.webhook_delivery` where the `time_completed` field has not been set). ** *Delivery attempts*: diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql index 3b4532836d7..b3c36debbbd 100644 --- a/schema/crdb/add-webhooks/up11.sql +++ b/schema/crdb/add-webhooks/up11.sql @@ -1,6 +1,6 @@ -- Index for looking up all currently in-flight webhook deliveries, and ordering -- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_deliverey_in_flight +CREATE INDEX IF NOT EXISTS webhook_delivery_in_flight ON omicron.public.webhook_delivery ( time_created, id ) WHERE diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 2fe1a4cc559..7c820d9a706 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4914,7 +4914,7 @@ ON omicron.public.webhook_delivery ( -- Index for looking up all currently in-flight webhook messages, and ordering -- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_deliverey_in_flight +CREATE INDEX IF NOT EXISTS webhook_delivery_in_flight ON omicron.public.webhook_delivery ( time_created, id ) WHERE From 432057a5ae71a4795e6133f7c6aeb874203e02d2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 15:12:31 -0800 Subject: [PATCH 013/168] add 'failed_timeout' delivery result --- nexus/db-model/src/webhook_delivery.rs | 1 + schema/crdb/add-webhooks/up12.sql | 3 +++ schema/crdb/add-webhooks/up13.sql | 6 +++--- schema/crdb/dbinit.sql | 9 ++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 1133813a404..6819b5a2409 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -26,5 +26,6 @@ impl_enum_type!( FailedHttpError => b"failed_http_error" FailedUnreachable => b"failed_unreachable" + FailedTimeout => b"failed_timeout" Succeeded => b"succeeded" ); diff --git a/schema/crdb/add-webhooks/up12.sql b/schema/crdb/add-webhooks/up12.sql index dbe765f9060..06ce3110625 100644 --- a/schema/crdb/add-webhooks/up12.sql +++ b/schema/crdb/add-webhooks/up12.sql @@ -4,6 +4,9 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( -- The delivery attempt failed because the receiver endpoint was -- unreachable. 'failed_unreachable', + --- The delivery attempt connected successfully but no response was received + -- within the timeout. + 'failed_timeout', -- The delivery attempt succeeded. 'succeeded' ); diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/add-webhooks/up13.sql index 4bc92ef3d5b..3497555f786 100644 --- a/schema/crdb/add-webhooks/up13.sql +++ b/schema/crdb/add-webhooks/up13.sql @@ -19,9 +19,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( response_duration IS NOT NULL ) ) OR ( - -- If the result is 'failed_unreachable', no response data is - -- present. - (result = 'failed_unreachable') AND ( + -- If the result is 'failed_unreachable' or 'failed_timeout', no + -- response data is present. + (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( response_status IS NULL AND response_duration IS NULL ) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 7c820d9a706..6345241ee91 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4926,6 +4926,9 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( -- The delivery attempt failed because the receiver endpoint was -- unreachable. 'failed_unreachable', + --- The delivery attempt connected successfully but no response was received + -- within the timeout. + 'failed_timeout', -- The delivery attempt succeeded. 'succeeded' ); @@ -4951,9 +4954,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( response_duration IS NOT NULL ) ) OR ( - -- If the result is 'failed_unreachable', no response data is - -- present. - (result = 'failed_unreachable') AND ( + -- If the result is 'failed_unreachable' or 'failed_timeout', no + -- response data is present. + (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( response_status IS NULL AND response_duration IS NULL ) From 52d0f246a65586963ca1cc8bd3ae4ea90c844cdb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 15:45:13 -0800 Subject: [PATCH 014/168] ag --- schema/crdb/add-webhooks/up09.sql | 2 +- schema/crdb/dbinit.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql index c2ee13acdb0..41c99934134 100644 --- a/schema/crdb/add-webhooks/up09.sql +++ b/schema/crdb/add-webhooks/up09.sql @@ -10,5 +10,5 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( time_created TIMESTAMPTZ NOT NULL, -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, + time_completed TIMESTAMPTZ ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 6345241ee91..8c834e09344 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4903,7 +4903,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( time_created TIMESTAMPTZ NOT NULL, -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, + time_completed TIMESTAMPTZ ); -- Index for looking up all webhook messages dispatched to a receiver ID From 1aa95c3fd5e04703db1a808c5721af54055401ec Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 2 Jan 2025 15:48:04 -0800 Subject: [PATCH 015/168] ag --- schema/crdb/dbinit.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 8c834e09344..edd050d23cd 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4909,7 +4909,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx ON omicron.public.webhook_delivery ( - rx_id, event_id, + rx_id, event_id ); -- Index for looking up all currently in-flight webhook messages, and ordering From 75ac3501c73b005fdce21a71b0f8e3741dc65258 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 3 Jan 2025 10:17:23 -0800 Subject: [PATCH 016/168] more models for webhook delivery --- nexus/db-model/src/instance.rs | 81 +----------------------- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 6 +- nexus/db-model/src/serde_time_delta.rs | 87 ++++++++++++++++++++++++++ nexus/db-model/src/webhook_delivery.rs | 75 ++++++++++++++++++++++ nexus/db-model/src/webhook_event.rs | 41 ++++++++++++ schema/crdb/add-webhooks/up09.sql | 8 ++- schema/crdb/add-webhooks/up13.sql | 15 +++++ schema/crdb/dbinit.sql | 23 ++++++- uuid-kinds/src/lib.rs | 4 +- 10 files changed, 257 insertions(+), 85 deletions(-) create mode 100644 nexus/db-model/src/serde_time_delta.rs create mode 100644 nexus/db-model/src/webhook_event.rs diff --git a/nexus/db-model/src/instance.rs b/nexus/db-model/src/instance.rs index 19886c5f677..002543f1ef6 100644 --- a/nexus/db-model/src/instance.rs +++ b/nexus/db-model/src/instance.rs @@ -8,6 +8,7 @@ use super::{ }; use crate::collection::DatastoreAttachTargetConfig; use crate::schema::{disk, external_ip, instance}; +use crate::serde_time_delta::optional_time_delta; use chrono::{DateTime, TimeDelta, Utc}; use db_macros::Resource; use diesel::expression::{is_aggregate, ValidGrouping}; @@ -453,86 +454,6 @@ impl InstanceAutoRestart { .and(dsl::updater_id.is_null()) } } - -/// It's just a type with the same representation as a `TimeDelta` that -/// implements `Serialize` and `Deserialize`, because `chrono`'s `Deserialize` -/// implementation for this type is not actually for `TimeDelta`, but for the -/// `rkyv::Archived` wrapper type (see [here]). While `chrono` *does* provide a -/// `Serialize` implementation that we could use with this type, it's preferable -/// to provide our own `Serialize` as well as `Deserialize`, since a future -/// semver-compatible change in `chrono` could change the struct's internal -/// representation, quietly breaking our ability to round-trip it. So, let's -/// just derive both traits for this thing, which we control. -/// -/// If you feel like this is unfortunate...yeah, I do too. -/// -/// [here]: https://docs.rs/chrono/latest/chrono/struct.TimeDelta.html#impl-Deserialize%3CTimeDelta,+__D%3E-for-%3CTimeDelta+as+Archive%3E::Archived -#[derive(Copy, Clone, Debug, Serialize, Deserialize)] -struct SerdeTimeDelta { - secs: i64, - nanos: i32, -} - -impl From for SerdeTimeDelta { - fn from(delta: TimeDelta) -> Self { - Self { secs: delta.num_seconds(), nanos: delta.subsec_nanos() } - } -} - -impl TryFrom for TimeDelta { - type Error = &'static str; - fn try_from( - SerdeTimeDelta { secs, nanos }: SerdeTimeDelta, - ) -> Result { - // This is a bit weird: `chrono::TimeDelta`'s getter for - // nanoseconds (`TimeDelta::subsec_nanos`) returns them as an i32, - // with the sign coming from the seconds part, but when constructing - // a `TimeDelta`, it takes them as a `u32` and panics if they're too - // big. So, we take the absolute value here, because what the serialize - // impl saw may have had its sign bit set, but the constructor will get - // mad if we give it something with that bit set. Hopefully that made - // sense? - let nanos = nanos.unsigned_abs(); - TimeDelta::new(secs, nanos).ok_or("time delta out of range") - } -} -mod optional_time_delta { - use super::*; - use serde::{Deserializer, Serializer}; - - pub(super) fn deserialize<'de, D>( - deserializer: D, - ) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let val = Option::::deserialize(deserializer)?; - match val { - None => return Ok(None), - Some(delta) => delta - .try_into() - .map_err(|e| { - ::custom(format!( - "{e}: {val:?}" - )) - }) - .map(Some), - } - } - - pub(super) fn serialize( - td: &Option, - serializer: S, - ) -> Result - where - S: Serializer, - { - td.as_ref() - .map(|&delta| SerdeTimeDelta::from(delta)) - .serialize(serializer) - } -} - /// The parts of an Instance that can be directly updated after creation. #[derive(Clone, Debug, AsChangeset, Serialize, Deserialize)] #[diesel(table_name = instance, treat_none_as_null = true)] diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 608fb1383d8..bc8bda74d93 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -60,11 +60,13 @@ mod producer_endpoint; mod project; mod rendezvous_debug_dataset; mod semver_version; +mod serde_time_delta; mod switch_interface; mod switch_port; mod v2p_mapping; mod vmm_state; mod webhook_delivery; +mod webhook_event; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index ef1e00b308c..836a17ece9b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2178,8 +2178,10 @@ table! { table! { webhook_delivery (id) { id -> Uuid, + event_id -> Uuid, rx_id -> Uuid, payload -> Jsonb, + attempts -> Int2, time_created -> Timestamptz, time_completed -> Nullable, } @@ -2189,9 +2191,9 @@ allow_tables_to_appear_in_same_query!(webhook_rx, webhook_delivery); joinable!(webhook_delivery -> webhook_rx (rx_id)); table! { - webhook_delivery_attempt (id) { - id -> Uuid, + webhook_delivery_attempt (delivery_id, attempt) { delivery_id -> Uuid, + attempt -> Int2, result -> crate::WebhookDeliveryResultEnum, response_status -> Nullable, response_duration -> Nullable, diff --git a/nexus/db-model/src/serde_time_delta.rs b/nexus/db-model/src/serde_time_delta.rs new file mode 100644 index 00000000000..49b9b7239dd --- /dev/null +++ b/nexus/db-model/src/serde_time_delta.rs @@ -0,0 +1,87 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use chrono::TimeDelta; +use serde::Deserialize; +use serde::Serialize; + +/// It's just a type with the same representation as a `TimeDelta` that +/// implements `Serialize` and `Deserialize`, because `chrono`'s `Deserialize` +/// implementation for this type is not actually for `TimeDelta`, but for the +/// `rkyv::Archived` wrapper type (see [here]). While `chrono` *does* provide a +/// `Serialize` implementation that we could use with this type, it's preferable +/// to provide our own `Serialize` as well as `Deserialize`, since a future +/// semver-compatible change in `chrono` could change the struct's internal +/// representation, quietly breaking our ability to round-trip it. So, let's +/// just derive both traits for this thing, which we control. +/// +/// If you feel like this is unfortunate...yeah, I do too. +/// +/// [here]: https://docs.rs/chrono/latest/chrono/struct.TimeDelta.html#impl-Deserialize%3CTimeDelta,+__D%3E-for-%3CTimeDelta+as+Archive%3E::Archived +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] +pub(crate) struct SerdeTimeDelta { + secs: i64, + nanos: i32, +} + +impl From for SerdeTimeDelta { + fn from(delta: TimeDelta) -> Self { + Self { secs: delta.num_seconds(), nanos: delta.subsec_nanos() } + } +} + +impl TryFrom for TimeDelta { + type Error = &'static str; + fn try_from( + SerdeTimeDelta { secs, nanos }: SerdeTimeDelta, + ) -> Result { + // This is a bit weird: `chrono::TimeDelta`'s getter for + // nanoseconds (`TimeDelta::subsec_nanos`) returns them as an i32, + // with the sign coming from the seconds part, but when constructing + // a `TimeDelta`, it takes them as a `u32` and panics if they're too + // big. So, we take the absolute value here, because what the serialize + // impl saw may have had its sign bit set, but the constructor will get + // mad if we give it something with that bit set. Hopefully that made + // sense? + let nanos = nanos.unsigned_abs(); + TimeDelta::new(secs, nanos).ok_or("time delta out of range") + } +} + +pub(crate) mod optional_time_delta { + use super::*; + use serde::{Deserializer, Serializer}; + + pub(crate) fn deserialize<'de, D>( + deserializer: D, + ) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let val = Option::::deserialize(deserializer)?; + match val { + None => return Ok(None), + Some(delta) => delta + .try_into() + .map_err(|e| { + ::custom(format!( + "{e}: {val:?}" + )) + }) + .map(Some), + } + } + + pub(crate) fn serialize( + td: &Option, + serializer: S, + ) -> Result + where + S: Serializer, + { + td.as_ref() + .map(|&delta| SerdeTimeDelta::from(delta)) + .serialize(serializer) + } +} diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 6819b5a2409..0999c17cc1c 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -3,6 +3,14 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; +use crate::schema::{webhook_delivery, webhook_delivery_attempt}; +use crate::serde_time_delta::optional_time_delta; +use crate::typed_uuid::DbTypedUuid; +use crate::SqlU8; +use chrono::{DateTime, TimeDelta, Utc}; +use omicron_uuid_kinds::{ + WebhookDeliveryKind, WebhookEventKind, WebhookReceiverKind, +}; use serde::Deserialize; use serde::Serialize; @@ -29,3 +37,70 @@ impl_enum_type!( FailedTimeout => b"failed_timeout" Succeeded => b"succeeded" ); + +/// A webhook delivery dispatch entry. +#[derive( + Clone, + Queryable, + Debug, + Selectable, + Serialize, + Deserialize, + Insertable, + PartialEq, +)] +#[diesel(table_name = webhook_delivery)] +pub struct WebhookDelivery { + /// ID of this dispatch entry. + pub id: DbTypedUuid, + + /// ID of the event dispatched to this receiver (foreign key into + /// `webhook_event`). + pub event_id: DbTypedUuid, + + /// ID of the receiver to whcih this event is dispatched (foreign key into + /// `webhook_rx`). + pub rx_id: DbTypedUuid, + + /// The data payload as sent to this receiver. + pub payload: serde_json::Value, + + /// Attempt count + pub attempts: SqlU8, + + /// The time at which this dispatch entry was created. + pub time_created: DateTime, + + /// The time at which the webhook message was either delivered successfully + /// or permanently failed. + pub time_completed: Option>, +} + +/// An individual delivery attempt for a [`WebhookDelivery`]. +#[derive( + Clone, + Queryable, + Debug, + Selectable, + Serialize, + Deserialize, + Insertable, + PartialEq, +)] +#[diesel(table_name = webhook_delivery_attempt)] +pub struct WebhookDeliveryAttempt { + /// ID of the delivery entry (foreign key into `webhook_delivery`). + pub delivery_id: DbTypedUuid, + + /// Attempt number (retry count). + pub attempt: SqlU8, + + pub result: WebhookDeliveryResult, + + pub response_status: Option, + + #[serde(with = "optional_time_delta")] + pub response_duration: Option, + + pub time_created: DateTime, +} diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs new file mode 100644 index 00000000000..d0f469400ef --- /dev/null +++ b/nexus/db-model/src/webhook_event.rs @@ -0,0 +1,41 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::schema::webhook_event; +use crate::typed_uuid::DbTypedUuid; +use chrono::{DateTime, Utc}; +use omicron_uuid_kinds::WebhookEventKind; +use serde::{Deserialize, Serialize}; + +/// A webhook event. +#[derive( + Clone, + Queryable, + Debug, + Selectable, + Serialize, + Deserialize, + Insertable, + PartialEq, +)] +#[diesel(table_name = webhook_event)] +pub struct WebhookEvent { + /// ID of the event. + pub id: DbTypedUuid, + + /// The time this event was created. + pub time_created: DateTime, + + /// The time at which this event was dispatched by creating entries in the + /// `webhook_delivery` table. + /// + /// If this is `None`, this event has yet to be dispatched. + pub time_dispatched: Option>, + + /// The class of this event. + pub event_class: String, + + /// The event's data payload. + pub event: serde_json::Value, +} diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql index 41c99934134..74569aed13c 100644 --- a/schema/crdb/add-webhooks/up09.sql +++ b/schema/crdb/add-webhooks/up09.sql @@ -7,8 +7,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, payload JSONB NOT NULL, + + --- Delivery attempt count. Starts at 0. + attempts INT2 NOT NULL, + time_created TIMESTAMPTZ NOT NULL, -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ + time_completed TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0) ); diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/add-webhooks/up13.sql index 3497555f786..04e3acbf9db 100644 --- a/schema/crdb/add-webhooks/up13.sql +++ b/schema/crdb/add-webhooks/up13.sql @@ -4,12 +4,27 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- attempt number. attempt INT2 NOT NULL, result omicron.public.webhook_delivery_result NOT NULL, + -- A status code > 599 would be Very Surprising, so rather than using an + -- INT4 to store a full unsigned 16-bit number in the database, we'll use a + -- signed 16-bit integer with a check constraint that it's unsigned. response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, PRIMARY KEY (delivery_id, attempt), + -- Attempt numbers start at 1 + CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), + + -- Ensure response status codes are not negative. + -- We could be more prescriptive here, and also check that they're >= 100 + -- and <= 599, but some servers may return weird stuff, and we'd like to be + -- able to record that they did that. + CONSTRAINT response_status_is_unsigned CHECK ( + (response_status IS NOT NULL AND response_status >= 0) OR + (response_status IS NULL) + ), + CONSTRAINT response_iff_not_unreachable CHECK ( ( -- If the result is 'succeedeed' or 'failed_http_error', response diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index edd050d23cd..a2cf0cada40 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4900,10 +4900,16 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, payload JSONB NOT NULL, + + --- Delivery attempt count. Starts at 0. + attempts INT2 NOT NULL, + time_created TIMESTAMPTZ NOT NULL, -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ + time_completed TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0) ); -- Index for looking up all webhook messages dispatched to a receiver ID @@ -4939,12 +4945,27 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- attempt number. attempt INT2 NOT NULL, result omicron.public.webhook_delivery_result NOT NULL, + -- A status code > 599 would be Very Surprising, so rather than using an + -- INT4 to store a full unsigned 16-bit number in the database, we'll use a + -- signed 16-bit integer with a check constraint that it's unsigned. response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, PRIMARY KEY (delivery_id, attempt), + -- Attempt numbers start at 1 + CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), + + -- Ensure response status codes are not negative. + -- We could be more prescriptive here, and also check that they're >= 100 + -- and <= 599, but some servers may return weird stuff, and we'd like to be + -- able to record that they did that. + CONSTRAINT response_status_is_unsigned CHECK ( + (response_status IS NOT NULL AND response_status >= 0) OR + (response_status IS NULL) + ), + CONSTRAINT response_iff_not_unreachable CHECK ( ( -- If the result is 'succeedeed' or 'failed_http_error', response diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index 126af803fc3..b5a707ab9be 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -77,6 +77,8 @@ impl_typed_uuid_kind! { UpstairsSession => "upstairs_session", Vnic => "vnic", Volume => "volume", - Webhook => "webhook", + WebhookEvent => "webhook_event", + WebhookReceiver => "webhook_receiver", + WebhookDelivery => "webhook_delivery", Zpool => "zpool", } From 59c98a026e4c70e9d161584562434c395290f819 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 3 Jan 2025 10:33:46 -0800 Subject: [PATCH 017/168] models for receivers --- nexus/db-model/src/lib.rs | 3 ++ nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_rx.rs | 51 +++++++++++++++++++++++++++++++ schema/crdb/add-webhooks/up01.sql | 1 + schema/crdb/dbinit.sql | 1 + 5 files changed, 57 insertions(+) create mode 100644 nexus/db-model/src/webhook_rx.rs diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bc8bda74d93..bfa01596c95 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -67,6 +67,7 @@ mod v2p_mapping; mod vmm_state; mod webhook_delivery; mod webhook_event; +mod webhook_rx; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. @@ -234,6 +235,8 @@ pub use vpc_route::*; pub use vpc_router::*; pub use vpc_subnet::*; pub use webhook_delivery::*; +pub use webhook_event::*; +pub use webhook_rx::*; pub use zpool::*; // TODO: The existence of both impl_enum_type and impl_enum_wrapper is a diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 836a17ece9b..0f12a4aee2d 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2134,6 +2134,7 @@ table! { webhook_rx (id) { id -> Uuid, name -> Text, + description -> Text, endpoint -> Text, time_created -> Timestamptz, time_modified -> Nullable, diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs new file mode 100644 index 00000000000..c1fda0a25fd --- /dev/null +++ b/nexus/db-model/src/webhook_rx.rs @@ -0,0 +1,51 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::schema::{webhook_rx, webhook_rx_secret, webhook_rx_subscription}; +use crate::typed_uuid::DbTypedUuid; +use chrono::{DateTime, Utc}; +use db_macros::Resource; +use omicron_uuid_kinds::WebhookReceiverKind; +use serde::{Deserialize, Serialize}; + +/// A webhook receiver configuration. +#[derive( + Clone, + Debug, + Queryable, + Selectable, + Resource, + Insertable, + Serialize, + Deserialize, +)] +#[diesel(table_name = webhook_rx)] +pub struct WebhookReceiver { + #[diesel(embed)] + identity: WebhookReceiverIdentity, + + pub endpoint: String, +} + +#[derive( + Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, +)] +#[diesel(table_name = webhook_rx_secret)] +pub struct WebhookRxSecret { + pub rx_id: DbTypedUuid, + pub signature_id: String, + pub secret: Vec, + pub time_created: DateTime, + pub time_deleted: Option>, +} + +#[derive( + Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, +)] +#[diesel(table_name = webhook_rx_subscription)] +pub struct WebhookRxSubscription { + pub rx_id: DbTypedUuid, + pub event_class: String, + pub time_created: DateTime, +} diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql index 58799496b05..989683281eb 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/add-webhooks/up01.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( id UUID PRIMARY KEY, -- A human-readable identifier for this webhook receiver. name STRING(63) NOT NULL, + description STRING(512) NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- TODO(eliza): how do we track which roles are assigned to a webhook? diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a2cf0cada40..f136d15e602 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4812,6 +4812,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( id UUID PRIMARY KEY, -- A human-readable identifier for this webhook receiver. name STRING(63) NOT NULL, + description STRING(512) NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- TODO(eliza): how do we track which roles are assigned to a webhook? From 9fdd77a358b7ba986769ff1d50fe7d9bc1bc95df Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 3 Jan 2025 10:46:50 -0800 Subject: [PATCH 018/168] s/delivery_attempts/delivery_attempt --- schema/crdb/add-webhooks/README.adoc | 2 +- schema/crdb/add-webhooks/up14.sql | 4 ++-- schema/crdb/dbinit.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index 6f724c75f16..ffcb1538a24 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -46,6 +46,6 @@ representing the potential outcomes of a webhook delivery attempt. *** `up13.sql` creates the table `omicron.public.webhook_delivery_attempt`, which records each individual delivery attempt for a webhook message in the `webhook_delivery` table. -*** `up14.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on +*** `up14.sql` creates an index `lookup_webhook_delivery_attempt_for_msg` on `omicron.public.webhook_delivery_attempt`, for looking up all attempts to deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up14.sql b/schema/crdb/add-webhooks/up14.sql index 05ec2ba0c95..06ad7d4f937 100644 --- a/schema/crdb/add-webhooks/up14.sql +++ b/schema/crdb/add-webhooks/up14.sql @@ -1,4 +1,4 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_delivery_attempts ( +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempt_for_msg +ON omicron.public.webhook_delivery_attempt ( delivery_id ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f136d15e602..b8bf06f8590 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4986,8 +4986,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( ) ); -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg -ON omicron.public.webhook_delivery_attempts ( +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempt_for_msg +ON omicron.public.webhook_delivery_attempt ( delivery_id ); From b8d1efa6b5f34a89e819f3f9405453ee18448a24 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 3 Jan 2025 11:43:10 -0800 Subject: [PATCH 019/168] add probes to receiver models --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_rx.rs | 2 +- schema/crdb/add-webhooks/up01.sql | 2 ++ schema/crdb/dbinit.sql | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 0f12a4aee2d..aa4ab2b5b9d 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2136,6 +2136,7 @@ table! { name -> Text, description -> Text, endpoint -> Text, + probes_enabled -> Bool, time_created -> Timestamptz, time_modified -> Nullable, time_deleted -> Nullable, diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index c1fda0a25fd..a5a459845dc 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; pub struct WebhookReceiver { #[diesel(embed)] identity: WebhookReceiverIdentity, - + pub probes_enabled: bool, pub endpoint: String, } diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql index 989683281eb..c7b8d207470 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/add-webhooks/up01.sql @@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( description STRING(512) NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, + -- Whether or not liveness probes are sent to this receiver. + probes_enabled BOOL NOT NULL, -- TODO(eliza): how do we track which roles are assigned to a webhook? time_created TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index b8bf06f8590..f5828ca71e3 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4815,6 +4815,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( description STRING(512) NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, + -- Whether or not liveness probes are sent to this receiver. + probes_enabled BOOL NOT NULL, -- TODO(eliza): how do we track which roles are assigned to a webhook? time_created TIMESTAMPTZ NOT NULL, time_modified TIMESTAMPTZ, From c66825256ea15fad96cd4d74d6a41037a97a3bdd Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 3 Jan 2025 16:38:08 -0800 Subject: [PATCH 020/168] turn event class globs into `SIMILAR TO` patterns needs testing, but i'd like to do that after finishing the DB queries that use it... --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_rx.rs | 84 ++++++++++++++++++- nexus/db-queries/src/db/datastore/mod.rs | 1 + .../src/db/datastore/webhook_event.rs | 59 +++++++++++++ schema/crdb/add-webhooks/up04.sql | 11 ++- schema/crdb/dbinit.sql | 11 ++- 6 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 nexus/db-queries/src/db/datastore/webhook_event.rs diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index aa4ab2b5b9d..1f7af596a84 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2160,6 +2160,7 @@ table! { webhook_rx_subscription (rx_id, event_class) { rx_id -> Uuid, event_class -> Text, + similar_to -> Text, time_created -> Timestamptz, } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index a5a459845dc..081ae909ce5 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -6,7 +6,7 @@ use crate::schema::{webhook_rx, webhook_rx_secret, webhook_rx_subscription}; use crate::typed_uuid::DbTypedUuid; use chrono::{DateTime, Utc}; use db_macros::Resource; -use omicron_uuid_kinds::WebhookReceiverKind; +use omicron_uuid_kinds::{WebhookReceiverKind, WebhookReceiverUuid}; use serde::{Deserialize, Serialize}; /// A webhook receiver configuration. @@ -47,5 +47,87 @@ pub struct WebhookRxSecret { pub struct WebhookRxSubscription { pub rx_id: DbTypedUuid, pub event_class: String, + pub similar_to: String, pub time_created: DateTime, } + +impl WebhookRxSubscription { + pub fn new(rx_id: WebhookReceiverUuid, event_class: String) -> Self { + fn seg2regex(segment: &str, similar_to: &mut String) { + match segment { + // Match one segment (i.e. any number of segment characters) + "*" => similar_to.push_str("[a-zA-Z0-9\\_\\-]+"), + // Match any number of segments + "**" => similar_to.push('%'), + // Match the literal segment. + // Because `_` his a metacharacter in Postgres' SIMILAR TO + // regexes, we've gotta go through and escape them. + s => { + for s in s.split_inclusive('_') { + // Handle the fact that there might not be a `_` in the + // string at all + if let Some(s) = s.strip_suffix('_') { + similar_to.push_str(s); + similar_to.push_str("\\_"); + } else { + similar_to.push_str(s); + } + } + } + } + } + + // The subscription's regex will always be at least as long as the event class. + let mut similar_to = String::with_capacity(event_class.len()); + let mut segments = event_class.split('.'); + if let Some(segment) = segments.next() { + seg2regex(segment, &mut similar_to); + for segment in segments { + similar_to.push('.'); // segment separator + seg2regex(segment, &mut similar_to); + } + } else { + // TODO(eliza): we should probably validate that the event class has + // at least one segment... + }; + + // `_` is a metacharacter in Postgres' SIMILAR TO regexes, so escape + // them. + + Self { + rx_id: DbTypedUuid(rx_id), + event_class, + similar_to, + time_created: Utc::now(), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_event_class_glob_to_regex() { + const CASES: &[(&str, &str)] = &[ + ("foo.bar", "foo.bar"), + ("foo.*.bar", "foo.[a-zA-Z0-9\\_\\-]+.bar"), + ("foo.*", "foo.[a-zA-Z0-9\\_\\-]+"), + ("*.foo", "[a-zA-Z0-9\\_\\-]+.foo"), + ("foo.**.bar", "foo.%.bar"), + ("foo.**", "foo.%"), + ("foo_bar.baz", "foo\\_bar.baz"), + ("foo_bar.*.baz", "foo\\_bar.[a-zA-Z0-9\\_\\-]+.baz"), + ]; + let rx_id = WebhookReceiverUuid::new_v4(); + for (class, regex) in CASES { + let subscription = + WebhookRxSubscription::new(rx_id, dbg!(class).to_string()); + assert_eq!( + dbg!(regex), + dbg!(&subscription.similar_to), + "event class {class:?} should produce the regex {regex:?}" + ); + } + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b2d9f8f2471..b8883aaec61 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -109,6 +109,7 @@ mod vmm; mod volume; mod volume_repair; mod vpc; +mod webhook_event; mod zpool; pub use address_lot::AddressLotCreateResult; diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs new file mode 100644 index 00000000000..e29469d327f --- /dev/null +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods for webhook events and event delivery dispatching. + +use super::DataStore; +use crate::db::pool::DbConnection; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::prelude::*; +use diesel::result::OptionalExtension; +use omicron_uuid_kinds::{GenericUuid, WebhookEventUuid}; + +use crate::db::model::WebhookEvent; +use crate::db::schema::webhook_event::dsl as event_dsl; + +impl DataStore { + /// Select the next webhook event in need of dispatching. + /// + /// This performs a `SELECT ... FOR UPDATE SKIP LOCKED` on the + /// `webhook_event` table, returning the oldest webhook event which has not + /// yet been dispatched to receivers and which is not actively being + /// dispatched in another transaction. + // NOTE: it would be kinda nice if this query could also select the + // webhook receivers subscribed to this event, but I am not totally sure + // what the CRDB semantics of joining on another table in a `SELECT ... FOR + // UPDATE SKIP LOCKED` query are. We don't want to inadvertantly also lock + // the webhook receivers... + pub async fn webhook_event_select_for_dispatch( + &self, + conn: &async_bb8_diesel::Connection, + ) -> Result, diesel::result::Error> { + event_dsl::webhook_event + .filter(event_dsl::time_dispatched.is_null()) + .order_by(event_dsl::time_created.asc()) + .limit(1) + .for_update() + .skip_locked() + .select(WebhookEvent::as_select()) + .get_result_async(conn) + .await + .optional() + } + + /// Mark the webhook event with the provided UUID as dispatched. + pub async fn webhook_event_set_dispatched( + &self, + event_id: &WebhookEventUuid, + conn: &async_bb8_diesel::Connection, + ) -> Result<(), diesel::result::Error> { + diesel::update(event_dsl::webhook_event) + .filter(event_dsl::id.eq(event_id.into_untyped_uuid())) + .filter(event_dsl::time_dispatched.is_null()) + .set(event_dsl::time_dispatched.eq(diesel::dsl::now)) + .execute_async(conn) + .await + .map(|_| ()) // this should always be 1... + } +} diff --git a/schema/crdb/add-webhooks/up04.sql b/schema/crdb/add-webhooks/up04.sql index 7911418a786..e5c6b433ca8 100644 --- a/schema/crdb/add-webhooks/up04.sql +++ b/schema/crdb/add-webhooks/up04.sql @@ -2,8 +2,17 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- An event class to which this receiver is subscribed. + -- An event class (or event class glob) to which this receiver is subscribed. event_class STRING(512) NOT NULL, + -- The event class or event classs glob transformed into a patteern for use + -- in SQL `SIMILAR TO` clauses. + -- + -- This is a bit interesting: users specify event class globs as sequences + -- of dot-separated segments which may be `*` to match any one segment or + -- `**` to match any number of segments. In order to match webhook events to + -- subscriptions within the database, we transform these into patterns that + -- can be used with a `SIMILAR TO` clause. + similar_to STRING(512) NOT NULL, time_created TIMESTAMPTZ NOT NULL, PRIMARY KEY (rx_id, event_class) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f5828ca71e3..da797de653e 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4847,8 +4847,17 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- An event class to which this receiver is subscribed. + -- An event class (or event class glob) to which this receiver is subscribed. event_class STRING(512) NOT NULL, + -- The event class or event classs glob transformed into a patteern for use + -- in SQL `SIMILAR TO` clauses. + -- + -- This is a bit interesting: users specify event class globs as sequences + -- of dot-separated segments which may be `*` to match any one segment or + -- `**` to match any number of segments. In order to match webhook events to + -- subscriptions within the database, we transform these into patterns that + -- can be used with a `SIMILAR TO` clause. + similar_to STRING(512) NOT NULL, time_created TIMESTAMPTZ NOT NULL, PRIMARY KEY (rx_id, event_class) From 58ab5d2171b6355a819a5e6c5107ddaa89eb5e62 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 7 Jan 2025 17:23:39 -0800 Subject: [PATCH 021/168] rudimentary receiver create gotta redo this to take the params but need to rebase --- common/src/api/external/mod.rs | 1 + nexus/auth/src/authz/api_resources.rs | 9 ++ nexus/db-model/src/schema.rs | 5 +- nexus/db-model/src/webhook_rx.rs | 76 ++++++++--- nexus/db-queries/src/db/datastore/mod.rs | 1 + .../db-queries/src/db/datastore/webhook_rx.rs | 122 ++++++++++++++++++ schema/crdb/add-webhooks/up01.sql | 10 +- schema/crdb/dbinit.sql | 11 +- 8 files changed, 206 insertions(+), 29 deletions(-) create mode 100644 nexus/db-queries/src/db/datastore/webhook_rx.rs diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fd5cbe38042..b3236060c8d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1045,6 +1045,7 @@ pub enum ResourceType { Probe, ProbeNetworkInterface, LldpLinkConfig, + WebhookReceiver, } // IDENTITY METADATA diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 745a699cf2b..b27ef3fa791 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -1041,3 +1041,12 @@ authz_resource! { roles_allowed = false, polar_snippet = FleetChild, } + +authz_resource! { + name = "WebhookReceiver", + parent = "Fleet", + primary_key = { uuid_kind = WebhookReceiverKind }, + roles_allowed = false, + polar_snippet = FleetChild, + +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 1f7af596a84..f000820c7a5 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2135,11 +2135,12 @@ table! { id -> Uuid, name -> Text, description -> Text, - endpoint -> Text, - probes_enabled -> Bool, time_created -> Timestamptz, time_modified -> Nullable, time_deleted -> Nullable, + rcgen -> Int8, + endpoint -> Text, + probes_enabled -> Bool, } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 081ae909ce5..4d8b27da6d5 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -2,12 +2,17 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::collection::DatastoreCollectionConfig; use crate::schema::{webhook_rx, webhook_rx_secret, webhook_rx_subscription}; use crate::typed_uuid::DbTypedUuid; +use crate::Generation; use chrono::{DateTime, Utc}; use db_macros::Resource; +use omicron_common::api::external::Error; use omicron_uuid_kinds::{WebhookReceiverKind, WebhookReceiverUuid}; use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use uuid::Uuid; /// A webhook receiver configuration. #[derive( @@ -23,11 +28,31 @@ use serde::{Deserialize, Serialize}; #[diesel(table_name = webhook_rx)] pub struct WebhookReceiver { #[diesel(embed)] - identity: WebhookReceiverIdentity, + pub identity: WebhookReceiverIdentity, pub probes_enabled: bool, pub endpoint: String, + + /// child resource generation number, per RFD 192 + pub rcgen: Generation, +} + +impl DatastoreCollectionConfig for WebhookReceiver { + type CollectionId = Uuid; + type GenerationNumberColumn = webhook_rx::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type CollectionIdColumn = webhook_rx_secret::dsl::rx_id; +} + +impl DatastoreCollectionConfig for WebhookReceiver { + type CollectionId = Uuid; + type GenerationNumberColumn = webhook_rx::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type CollectionIdColumn = webhook_rx_subscription::dsl::rx_id; } +// TODO(eliza): should deliveries/delivery attempts also be treated as children +// of a webhook receiver? + #[derive( Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, )] @@ -46,13 +71,23 @@ pub struct WebhookRxSecret { #[diesel(table_name = webhook_rx_subscription)] pub struct WebhookRxSubscription { pub rx_id: DbTypedUuid, + #[diesel(embed)] + pub glob: WebhookGlob, + pub time_created: DateTime, +} + +#[derive( + Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, +)] +#[diesel(table_name = webhook_rx_subscription)] +pub struct WebhookGlob { pub event_class: String, pub similar_to: String, - pub time_created: DateTime, } -impl WebhookRxSubscription { - pub fn new(rx_id: WebhookReceiverUuid, event_class: String) -> Self { +impl FromStr for WebhookGlob { + type Err = Error; + fn from_str(event_class: &str) -> Result { fn seg2regex(segment: &str, similar_to: &mut String) { match segment { // Match one segment (i.e. any number of segment characters) @@ -63,6 +98,7 @@ impl WebhookRxSubscription { // Because `_` his a metacharacter in Postgres' SIMILAR TO // regexes, we've gotta go through and escape them. s => { + // TODO(eliza): validate what characters are in the segment... for s in s.split_inclusive('_') { // Handle the fact that there might not be a `_` in the // string at all @@ -87,19 +123,19 @@ impl WebhookRxSubscription { seg2regex(segment, &mut similar_to); } } else { - // TODO(eliza): we should probably validate that the event class has - // at least one segment... + return Err(Error::invalid_value( + "event_class", + "must not be empty", + )); }; - // `_` is a metacharacter in Postgres' SIMILAR TO regexes, so escape - // them. + Ok(Self { event_class: event_class.to_string(), similar_to }) + } +} - Self { - rx_id: DbTypedUuid(rx_id), - event_class, - similar_to, - time_created: Utc::now(), - } +impl WebhookRxSubscription { + pub fn new(rx_id: WebhookReceiverUuid, glob: WebhookGlob) -> Self { + Self { rx_id: DbTypedUuid(rx_id), glob, time_created: Utc::now() } } } @@ -119,13 +155,17 @@ mod test { ("foo_bar.baz", "foo\\_bar.baz"), ("foo_bar.*.baz", "foo\\_bar.[a-zA-Z0-9\\_\\-]+.baz"), ]; - let rx_id = WebhookReceiverUuid::new_v4(); for (class, regex) in CASES { - let subscription = - WebhookRxSubscription::new(rx_id, dbg!(class).to_string()); + let glob = match WebhookGlob::from_str(dbg!(class)) { + Ok(glob) => glob, + Err(error) => panic!( + "event class glob {class:?} should produce the regex + {regex:?}, but instead failed to parse: {error}" + ), + }; assert_eq!( dbg!(regex), - dbg!(&subscription.similar_to), + dbg!(&glob.similar_to), "event class {class:?} should produce the regex {regex:?}" ); } diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b8883aaec61..aad45576e01 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -110,6 +110,7 @@ mod volume; mod volume_repair; mod vpc; mod webhook_event; +mod webhook_rx; mod zpool; pub use address_lot::AddressLotCreateResult; diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs new file mode 100644 index 00000000000..3880d3261f6 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -0,0 +1,122 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods for webhook receiver management. + +use super::DataStore; +use crate::authz; +use crate::context::OpContext; +use crate::db::collection_insert::AsyncInsertError; +use crate::db::collection_insert::DatastoreCollection; +use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; +use crate::db::error::ErrorHandler; +use crate::db::model::WebhookGlob; +use crate::db::model::WebhookReceiver; +use crate::db::model::WebhookRxSubscription; +use crate::db::pool::DbConnection; +use crate::db::TransactionError; +use async_bb8_diesel::AsyncConnection; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::prelude::*; +use diesel::result::OptionalExtension; +use nexus_types::identity::Resource; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; +use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; + +impl DataStore { + pub async fn webhook_rx_create( + &self, + opctx: &OpContext, + receiver: &WebhookReceiver, + subscriptions: &[WebhookGlob], + ) -> CreateResult { + use crate::db::schema::webhook_rx::dsl; + // TODO(eliza): someday we gotta allow creating webhooks with more + // restrictive permissions... + opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let rx_id = WebhookReceiverUuid::from_untyped_uuid(receiver.id()); + self.transaction_retry_wrapper("webhook_rx_create") + .transaction(&conn, |conn| { + let receiver = receiver.clone(); + async move { + diesel::insert_into(dsl::webhook_rx) + .values(receiver) + // .on_conflict(dsl::id) + // .do_update() + // .set(dsl::time_modified.eq(dsl::time_modified)) + // .returning(WebhookReceiver::as_returning()) + .execute_async(&conn) + .await?; + // .map_err(|e| { + // if retryable(&e) { + // return TransactionError::Database(e); + // }; + // TransactionError::CustomError(public_error_from_diesel( + // e, + // ErrorHandler::Conflict( + // ResourceType::WebhookReceiver, + // receiver.identity.name.as_str(), + // ), + // )) + // })?; + for glob in subscriptions { + match self + .webhook_add_subscription_on_conn( + WebhookRxSubscription::new(rx_id, glob.clone()), + &conn, + ) + .await + { + Ok(_) => {} + Err(AsyncInsertError::CollectionNotFound) => {} // we just created it? + Err(AsyncInsertError::DatabaseError(e)) => { + return Err(e); + } + } + } + // TODO(eliza): secrets go here... + Ok(()) + } + }) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::WebhookReceiver, + receiver.name().as_str(), + ), + ) + })?; + Ok(receiver.clone()) + } + + async fn webhook_add_subscription_on_conn( + &self, + subscription: WebhookRxSubscription, + conn: &async_bb8_diesel::Connection, + ) -> Result { + use crate::db::schema::webhook_rx_subscription::dsl; + let rx_id = subscription.rx_id.into_untyped_uuid(); + let subscription: WebhookRxSubscription = + WebhookReceiver::insert_resource( + rx_id, + diesel::insert_into(dsl::webhook_rx_subscription) + .values(subscription) + .on_conflict((dsl::rx_id, dsl::event_class)) + .do_update() + .set(dsl::time_created.eq(diesel::dsl::now)), + ) + .insert_and_get_result_async(conn) + .await?; + Ok(subscription) + } +} diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql index c7b8d207470..523222e0986 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/add-webhooks/up01.sql @@ -1,13 +1,15 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + /* Identity metadata (resource) */ id UUID PRIMARY KEY, - -- A human-readable identifier for this webhook receiver. name STRING(63) NOT NULL, description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Child resource generation + rcgen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- Whether or not liveness probes are sent to this receiver. probes_enabled BOOL NOT NULL, - -- TODO(eliza): how do we track which roles are assigned to a webhook? - time_created TIMESTAMPTZ NOT NULL, - time_deleted TIMESTAMPTZ ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index da797de653e..bf6030d9953 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4809,18 +4809,19 @@ CREATE UNIQUE INDEX IF NOT EXISTS one_record_per_volume_resource_usage on omicro */ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + /* Identity metadata (resource) */ id UUID PRIMARY KEY, - -- A human-readable identifier for this webhook receiver. name STRING(63) NOT NULL, description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Child resource generation + rcgen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- Whether or not liveness probes are sent to this receiver. probes_enabled BOOL NOT NULL, - -- TODO(eliza): how do we track which roles are assigned to a webhook? - time_created TIMESTAMPTZ NOT NULL, - time_modified TIMESTAMPTZ, - time_deleted TIMESTAMPTZ ); CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( From e859f99118ae47878ebb72b29ad23c9c15bfb5b0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 8 Jan 2025 09:33:13 -0800 Subject: [PATCH 022/168] prefix all webhook UUID kinds with webhook --- nexus/types/src/external_api/views.rs | 8 ++++---- uuid-kinds/src/lib.rs | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index e36d26bc214..ef46dc1587a 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -17,7 +17,7 @@ use omicron_common::api::external::{ IdentityMetadata, InstanceState, Name, ObjectIdentity, RoleName, SimpleIdentityOrName, }; -use omicron_uuid_kinds::{EventUuid, WebhookUuid}; +use omicron_uuid_kinds::{WebhookEventUuid, WebhookReceiverUuid}; use oxnet::{Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -1046,7 +1046,7 @@ pub struct EventClass { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct Webhook { /// The UUID of this webhook receiver. - pub id: WebhookUuid, + pub id: WebhookReceiverUuid, /// The identifier assigned to this webhook receiver upon creation. pub name: String, /// The URL that webhook notification requests are sent to. @@ -1083,13 +1083,13 @@ pub struct WebhookDelivery { pub id: Uuid, /// The UUID of the webhook receiver that this event was delivered to. - pub webhook_id: WebhookUuid, + pub webhook_id: WebhookReceiverUuid, /// The event class. pub event_class: String, /// The UUID of the event. - pub event_id: EventUuid, + pub event_id: WebhookEventUuid, /// The state of the delivery attempt. pub state: WebhookDeliveryState, diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index b5a707ab9be..f03a7049da8 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -57,7 +57,6 @@ impl_typed_uuid_kind! { DemoSaga => "demo_saga", Downstairs => "downstairs", DownstairsRegion => "downstairs_region", - Event => "event", ExternalIp => "external_ip", Instance => "instance", LoopbackAddress => "loopback_address", From e4194dc28ff2179c341cc8ccf006dfb85f7222de Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 8 Jan 2025 11:26:06 -0800 Subject: [PATCH 023/168] blargh trailing commas --- schema/crdb/add-webhooks/up01.sql | 2 +- schema/crdb/dbinit.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql index 523222e0986..76c519d9727 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/add-webhooks/up01.sql @@ -11,5 +11,5 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- Whether or not liveness probes are sent to this receiver. - probes_enabled BOOL NOT NULL, + probes_enabled BOOL NOT NULL ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index bf6030d9953..ae64daf97b0 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4821,7 +4821,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL, -- Whether or not liveness probes are sent to this receiver. - probes_enabled BOOL NOT NULL, + probes_enabled BOOL NOT NULL ); CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( From ac82a7103c3f3a112204d844abedda29acaa25d0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 8 Jan 2025 11:27:39 -0800 Subject: [PATCH 024/168] fix webhook rx model not being selectable --- nexus/db-model/src/schema.rs | 4 ++-- nexus/db-model/src/webhook_rx.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f000820c7a5..285fdf57154 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2136,11 +2136,11 @@ table! { name -> Text, description -> Text, time_created -> Timestamptz, - time_modified -> Nullable, + time_modified -> Timestamptz, time_deleted -> Nullable, - rcgen -> Int8, endpoint -> Text, probes_enabled -> Bool, + rcgen -> Int8, } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 4d8b27da6d5..dddb99cd669 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -29,8 +29,8 @@ use uuid::Uuid; pub struct WebhookReceiver { #[diesel(embed)] pub identity: WebhookReceiverIdentity, - pub probes_enabled: bool, pub endpoint: String, + pub probes_enabled: bool, /// child resource generation number, per RFD 192 pub rcgen: Generation, From 1ea3a4adb6ee8d73382e95e463e2264bc57aaec0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 8 Jan 2025 12:59:32 -0800 Subject: [PATCH 025/168] bleh okay this globbing design is inefficient but does work --- .../db-queries/src/db/datastore/webhook_rx.rs | 379 ++++++++++++++++-- nexus/types/src/external_api/params.rs | 8 +- 2 files changed, 350 insertions(+), 37 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 3880d3261f6..c24f472c2da 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -9,11 +9,14 @@ use crate::authz; use crate::context::OpContext; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; +use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::retryable; use crate::db::error::ErrorHandler; +use crate::db::model::Generation; use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; +use crate::db::model::WebhookReceiverIdentity; use crate::db::model::WebhookRxSubscription; use crate::db::pool::DbConnection; use crate::db::TransactionError; @@ -21,56 +24,73 @@ use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use diesel::result::OptionalExtension; +use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; +use crate::db::schema::webhook_rx::dsl as rx_dsl; +use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; + +use std::str::FromStr; + impl DataStore { pub async fn webhook_rx_create( &self, opctx: &OpContext, - receiver: &WebhookReceiver, - subscriptions: &[WebhookGlob], + params: params::WebhookCreate, ) -> CreateResult { - use crate::db::schema::webhook_rx::dsl; // TODO(eliza): someday we gotta allow creating webhooks with more // restrictive permissions... opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; let conn = self.pool_connection_authorized(opctx).await?; - let rx_id = WebhookReceiverUuid::from_untyped_uuid(receiver.id()); - self.transaction_retry_wrapper("webhook_rx_create") + let params::WebhookCreate { + identity, + endpoint, + secrets, + events, + disable_probes, + } = params; + + let globs = events + .iter() + .map(|s| WebhookGlob::from_str(s)) + .collect::, _>>()?; + + let rx = self + .transaction_retry_wrapper("webhook_rx_create") .transaction(&conn, |conn| { - let receiver = receiver.clone(); + // make a fresh UUID for each transaction, in case the + // transaction fails because of a UUID collision. + // + // this probably won't happen, but, ya know... + let id = WebhookReceiverUuid::new_v4(); + let receiver = WebhookReceiver { + identity: WebhookReceiverIdentity::new( + id.into_untyped_uuid(), + identity.clone(), + ), + endpoint: endpoint.to_string(), + probes_enabled: !disable_probes, + rcgen: Generation::new(), + }; + let globs = globs.clone(); async move { - diesel::insert_into(dsl::webhook_rx) + let rx = diesel::insert_into(rx_dsl::webhook_rx) .values(receiver) - // .on_conflict(dsl::id) - // .do_update() - // .set(dsl::time_modified.eq(dsl::time_modified)) - // .returning(WebhookReceiver::as_returning()) - .execute_async(&conn) + .returning(WebhookReceiver::as_returning()) + .get_result_async(&conn) .await?; - // .map_err(|e| { - // if retryable(&e) { - // return TransactionError::Database(e); - // }; - // TransactionError::CustomError(public_error_from_diesel( - // e, - // ErrorHandler::Conflict( - // ResourceType::WebhookReceiver, - // receiver.identity.name.as_str(), - // ), - // )) - // })?; - for glob in subscriptions { + for glob in globs { match self .webhook_add_subscription_on_conn( - WebhookRxSubscription::new(rx_id, glob.clone()), + WebhookRxSubscription::new(id, glob), &conn, ) .await @@ -83,7 +103,7 @@ impl DataStore { } } // TODO(eliza): secrets go here... - Ok(()) + Ok(rx) } }) .await @@ -92,11 +112,11 @@ impl DataStore { e, ErrorHandler::Conflict( ResourceType::WebhookReceiver, - receiver.name().as_str(), + identity.name.as_str(), ), ) })?; - Ok(receiver.clone()) + Ok(rx) } async fn webhook_add_subscription_on_conn( @@ -104,19 +124,312 @@ impl DataStore { subscription: WebhookRxSubscription, conn: &async_bb8_diesel::Connection, ) -> Result { - use crate::db::schema::webhook_rx_subscription::dsl; let rx_id = subscription.rx_id.into_untyped_uuid(); let subscription: WebhookRxSubscription = WebhookReceiver::insert_resource( rx_id, - diesel::insert_into(dsl::webhook_rx_subscription) + diesel::insert_into(subscription_dsl::webhook_rx_subscription) .values(subscription) - .on_conflict((dsl::rx_id, dsl::event_class)) + .on_conflict(( + subscription_dsl::rx_id, + subscription_dsl::event_class, + )) .do_update() - .set(dsl::time_created.eq(diesel::dsl::now)), + .set(subscription_dsl::time_created.eq(diesel::dsl::now)), ) .insert_and_get_result_async(conn) .await?; Ok(subscription) } + + /// List all webhook receivers whose event class subscription globs match + /// the provided `event_class`. + // TODO(eliza): probably paginate this... + pub async fn webhook_rx_list_subscribed_to_event( + &self, + opctx: &OpContext, + event_class: impl ToString, + ) -> ListResultVec<(WebhookReceiver, WebhookRxSubscription)> { + use async_bb8_diesel::AsyncSimpleConnection; + let conn = self.pool_connection_authorized(opctx).await?; + let class = event_class.to_string(); + conn.transaction_async(|conn| async move { + // TODO(eliza): this sucks, don't evaluate the globs every time we + // dispatch an event; do them at webhook-creation time instead, OR + // when updating to a sw version with new event classes... + // This currently requires a full table scan over + // `webhook_rx_subscription`, as we cannot create indices to + // accelerate text search in the present CRDB version... :( + conn.batch_execute_async( + crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL, + ) + .await + .unwrap(); + Self::rx_list_subscribed_query(class) + .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&conn) + .await + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + fn rx_list_subscribed_query( + event_class: String, + ) -> impl RunnableQuery<(WebhookReceiver, WebhookRxSubscription)> { + subscription_dsl::webhook_rx_subscription + .filter( + event_class + .to_string() + .into_sql::() + .similar_to(subscription_dsl::similar_to), + ) + .order_by(subscription_dsl::rx_id.asc()) + .inner_join( + rx_dsl::webhook_rx.on(subscription_dsl::rx_id.eq(rx_dsl::id)), + ) + .filter(rx_dsl::time_deleted.is_null()) + .select(( + WebhookReceiver::as_select(), + WebhookRxSubscription::as_select(), + )) + } + // pub async fn webhook_rx_list( + // &self, + // opctx: &OpContext, + // ) -> ListResultVec { + // let conn = self.pool_connection_authorized(opctx).await?; + // rx_dsl::webhook_rx + // .filter(rx_dsl::time_deleted.is_null()) + // .select(WebhookReceiver::as_select()) + // .load_async::(&*conn) + // .await + // .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + // } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::db::explain::ExplainableAsync; + use crate::db::pub_test_utils::TestDatabase; + use omicron_common::api::external::IdentityMetadataCreateParams; + use omicron_test_utils::dev::{self, LogContext}; + + #[tokio::test] + async fn test_event_class_globs() { + // Test setup + let logctx = dev::test_setup_log("test_event_class_globs"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let foo_star = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "foo-star".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://foo.star".parse().unwrap(), + secrets: Vec::new(), + events: vec!["foo.*".to_string()], + disable_probes: false, + }, + ) + .await + .unwrap(); + + let foo_starstar = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "foo-starstar".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://foo.starstar".parse().unwrap(), + secrets: Vec::new(), + events: vec!["foo.**".to_string()], + disable_probes: false, + }, + ) + .await + .unwrap(); + + let foo_bar = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "foo-bar".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://foo.bar".parse().unwrap(), + secrets: Vec::new(), + events: vec!["foo.bar".to_string()], + disable_probes: false, + }, + ) + .await + .unwrap(); + + let foo_starstar_bar = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "foo-starstar-bar".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://foo.starstar.bar".parse().unwrap(), + secrets: Vec::new(), + events: vec!["foo.**.bar".parse().unwrap()], + disable_probes: false, + }, + ) + .await + .unwrap(); + + let starstar_bar = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "starstar-bar".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://starstar.bar".parse().unwrap(), + secrets: Vec::new(), + events: vec!["**.bar".parse().unwrap()], + disable_probes: false, + }, + ) + .await + .unwrap(); + + async fn check_event( + datastore: &DataStore, + opctx: &OpContext, + // logctx: &LogContext, + event_class: &str, + matches: &[&WebhookReceiver], + not_matches: &[&WebhookReceiver], + ) { + let subscribed = datastore + .webhook_rx_list_subscribed_to_event(opctx, event_class) + .await + .unwrap() + .into_iter() + .map(|(rx, subscription)| { + eprintln!("receiver is subscribed to event {event_class:?}:\n\trx: {rx:?}\n\tsubscription: {subscription:?}"); + rx.identity + }) + .collect::>(); + + for rx in matches { + assert!( + subscribed.contains(&rx.identity), + "expected {rx:?} to be subscribed to {event_class:?}" + ); + } + + for rx in not_matches { + assert!( + !subscribed.contains(&rx.identity), + "expected {rx:?} to not be subscribed to {event_class:?}" + ); + } + } + + check_event( + datastore, + opctx, + "notfoo", + &[], + &[ + &foo_star, + &foo_starstar, + &foo_bar, + &foo_starstar_bar, + &starstar_bar, + ], + ) + .await; + + check_event( + datastore, + opctx, + "foo.bar", + &[&foo_star, &foo_starstar, &foo_bar, &starstar_bar], + &[&foo_starstar_bar], + ) + .await; + + check_event( + datastore, + opctx, + "foo.baz", + &[&foo_star, &foo_starstar], + &[&foo_bar, &foo_starstar_bar, &starstar_bar], + ) + .await; + + check_event( + datastore, + opctx, + "foo.bar.baz", + &[&foo_starstar], + &[&foo_bar, &foo_star, &foo_starstar_bar, &starstar_bar], + ) + .await; + + check_event( + datastore, + opctx, + "foo.baz.bar", + &[&foo_starstar, &foo_starstar_bar, &starstar_bar], + &[&foo_bar, &foo_star], + ) + .await; + + check_event( + datastore, + opctx, + "foo.baz.quux.bar", + &[&foo_starstar, &foo_starstar_bar, &starstar_bar], + &[&foo_bar, &foo_star], + ) + .await; + + check_event( + datastore, + opctx, + "baz.quux.bar", + &[&starstar_bar], + &[&foo_bar, &foo_star, &foo_starstar, &foo_starstar_bar], + ) + .await; + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn explain_event_class_glob() { + let logctx = dev::test_setup_log("explain_event_class_glob"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let query = DataStore::rx_list_subscribed_query("foo.bar".to_string()); + let explanation = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + println!("{explanation}"); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index f167bdc522c..d8e8c1dc852 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2326,8 +2326,8 @@ pub struct EventClassSelector { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookCreate { - /// An identifier for this webhook receiver, which must be unique. - pub name: String, + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, /// The URL that webhook notification requests should be sent to pub endpoint: Url, @@ -2350,8 +2350,8 @@ pub struct WebhookCreate { /// Parameters to update a webhook configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookUpdate { - /// An identifier for this webhook receiver, which must be unique. - pub name: String, + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, /// The URL that webhook notification requests should be sent to pub endpoint: Url, From eca3b71d1b299390aa742ec28f5c8909d66ca6dc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 9 Jan 2025 12:19:08 -0800 Subject: [PATCH 026/168] evaluate subscription globs on creation this way, rather than transforming the globs into patterns that we can match in the database every time an event is dispatched, we instead use globs to generate "exact" subscriptions to specific named event classes. this avoids doing a full scan over the subscription table every time an event is dispatched, since we can look up subscribed receivers by event class name instead. we'll have to do this processing on receiver creation or when a subscription is added, and when new event classes are added (e.g. on software updates). thanks @andrewjstone for pointing me in this direction! --- nexus/db-model/src/schema.rs | 14 +- nexus/db-model/src/webhook_rx.rs | 156 ++++++++--- nexus/db-queries/Cargo.toml | 1 + .../db-queries/src/db/datastore/webhook_rx.rs | 253 ++++++++++++------ schema/crdb/dbinit.sql | 59 +++- 5 files changed, 340 insertions(+), 143 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 285fdf57154..b03c944d80b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2161,7 +2161,16 @@ table! { webhook_rx_subscription (rx_id, event_class) { rx_id -> Uuid, event_class -> Text, - similar_to -> Text, + glob -> Nullable, + time_created -> Timestamptz, + } +} + +table! { + webhook_rx_event_glob (rx_id, glob) { + rx_id -> Uuid, + glob -> Text, + regex -> Text, time_created -> Timestamptz, } } @@ -2169,6 +2178,9 @@ table! { allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_subscription); joinable!(webhook_rx_subscription -> webhook_rx (rx_id)); +allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_event_glob); +joinable!(webhook_rx_event_glob -> webhook_rx (rx_id)); + table! { webhook_event (id) { id -> Uuid, diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index dddb99cd669..af152b7fdff 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -3,7 +3,10 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::collection::DatastoreCollectionConfig; -use crate::schema::{webhook_rx, webhook_rx_secret, webhook_rx_subscription}; +use crate::schema::{ + webhook_rx, webhook_rx_event_glob, webhook_rx_secret, + webhook_rx_subscription, +}; use crate::typed_uuid::DbTypedUuid; use crate::Generation; use chrono::{DateTime, Utc}; @@ -50,6 +53,13 @@ impl DatastoreCollectionConfig for WebhookReceiver { type CollectionIdColumn = webhook_rx_subscription::dsl::rx_id; } +impl DatastoreCollectionConfig for WebhookReceiver { + type CollectionId = Uuid; + type GenerationNumberColumn = webhook_rx::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type CollectionIdColumn = webhook_rx_event_glob::dsl::rx_id; +} + // TODO(eliza): should deliveries/delivery attempts also be treated as children // of a webhook receiver? @@ -70,57 +80,102 @@ pub struct WebhookRxSecret { )] #[diesel(table_name = webhook_rx_subscription)] pub struct WebhookRxSubscription { + pub rx_id: DbTypedUuid, + pub event_class: String, + pub glob: Option, + pub time_created: DateTime, +} + +#[derive( + Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, +)] +#[diesel(table_name = webhook_rx_event_glob)] +pub struct WebhookRxEventGlob { pub rx_id: DbTypedUuid, #[diesel(embed)] pub glob: WebhookGlob, pub time_created: DateTime, } +impl WebhookRxEventGlob { + pub fn new(rx_id: WebhookReceiverUuid, glob: WebhookGlob) -> Self { + Self { rx_id: DbTypedUuid(rx_id), glob, time_created: Utc::now() } + } +} +#[derive(Clone, Debug)] +pub enum WebhookSubscriptionKind { + Glob(WebhookGlob), + Exact(String), +} + +impl WebhookSubscriptionKind { + pub fn new(value: String) -> Result { + if value.is_empty() { + return Err(Error::invalid_value( + "event_class", + "must not be empty", + )); + } + if value.contains('*') { + let regex = WebhookGlob::regex_from_glob(&value)?; + Ok(Self::Glob(WebhookGlob { regex, glob: value })) + } else { + Ok(Self::Exact(value)) + } + } +} + #[derive( Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, )] -#[diesel(table_name = webhook_rx_subscription)] +#[diesel(table_name = webhook_rx_event_glob)] pub struct WebhookGlob { - pub event_class: String, - pub similar_to: String, + pub glob: String, + pub regex: String, } impl FromStr for WebhookGlob { type Err = Error; - fn from_str(event_class: &str) -> Result { - fn seg2regex(segment: &str, similar_to: &mut String) { + fn from_str(glob: &str) -> Result { + let regex = Self::regex_from_glob(glob)?; + Ok(Self { glob: glob.to_string(), regex }) + } +} + +impl WebhookGlob { + fn regex_from_glob(glob: &str) -> Result { + let seg2regex = |segment: &str, + regex: &mut String| + -> Result<(), Error> { match segment { // Match one segment (i.e. any number of segment characters) - "*" => similar_to.push_str("[a-zA-Z0-9\\_\\-]+"), + "*" => regex.push_str("[^\\.]+"), // Match any number of segments - "**" => similar_to.push('%'), - // Match the literal segment. - // Because `_` his a metacharacter in Postgres' SIMILAR TO - // regexes, we've gotta go through and escape them. - s => { - // TODO(eliza): validate what characters are in the segment... - for s in s.split_inclusive('_') { - // Handle the fact that there might not be a `_` in the - // string at all - if let Some(s) = s.strip_suffix('_') { - similar_to.push_str(s); - similar_to.push_str("\\_"); - } else { - similar_to.push_str(s); - } - } + "**" => regex.push_str(".+"), + s if s.contains('*') => { + return Err(Error::invalid_value( + "event_class", + "invalid event class {glob:?}: all segments must be \ + either '*', '**', or any sequence of non-'*' characters", + )) } + // Match the literal segment. + s => regex.push_str(s), } - } + Ok(()) + }; + + // The subscription's regex will always be at least as long as the event + // class glob, plus start and end anchors. + let mut regex = String::with_capacity(glob.len()); - // The subscription's regex will always be at least as long as the event class. - let mut similar_to = String::with_capacity(event_class.len()); - let mut segments = event_class.split('.'); + regex.push('^'); // Start anchor + let mut segments = glob.split('.'); if let Some(segment) = segments.next() { - seg2regex(segment, &mut similar_to); + seg2regex(segment, &mut regex)?; for segment in segments { - similar_to.push('.'); // segment separator - seg2regex(segment, &mut similar_to); + regex.push_str("\\."); // segment separator + seg2regex(segment, &mut regex)?; } } else { return Err(Error::invalid_value( @@ -128,14 +183,29 @@ impl FromStr for WebhookGlob { "must not be empty", )); }; + regex.push('$'); // End anchor - Ok(Self { event_class: event_class.to_string(), similar_to }) + Ok(regex) } } impl WebhookRxSubscription { - pub fn new(rx_id: WebhookReceiverUuid, glob: WebhookGlob) -> Self { - Self { rx_id: DbTypedUuid(rx_id), glob, time_created: Utc::now() } + pub fn exact(rx_id: WebhookReceiverUuid, event_class: String) -> Self { + Self { + rx_id: DbTypedUuid(rx_id), + event_class, + glob: None, + time_created: Utc::now(), + } + } + + pub fn for_glob(glob: &WebhookRxEventGlob, event_class: String) -> Self { + Self { + rx_id: glob.rx_id, + glob: Some(glob.glob.glob.clone()), + event_class, + time_created: Utc::now(), + } } } @@ -146,26 +216,26 @@ mod test { #[test] fn test_event_class_glob_to_regex() { const CASES: &[(&str, &str)] = &[ - ("foo.bar", "foo.bar"), - ("foo.*.bar", "foo.[a-zA-Z0-9\\_\\-]+.bar"), - ("foo.*", "foo.[a-zA-Z0-9\\_\\-]+"), - ("*.foo", "[a-zA-Z0-9\\_\\-]+.foo"), - ("foo.**.bar", "foo.%.bar"), - ("foo.**", "foo.%"), - ("foo_bar.baz", "foo\\_bar.baz"), - ("foo_bar.*.baz", "foo\\_bar.[a-zA-Z0-9\\_\\-]+.baz"), + ("foo.bar", "^foo.bar$"), + ("foo.*.bar", "^foo\\.[^\\.]*\\.bar$"), + ("foo.*", "^foo\\.[^\\.]*$"), + ("*.foo", "^[^\\.]*\\.foo$"), + ("foo.**.bar", "^foo\\..+\\.bar$"), + ("foo.**", "^foo\\..+$"), + ("foo_bar.baz", "^foo_bar\\.baz$"), + ("foo_bar.*.baz", "^foo_bar\\.[^\\.]+\\.baz$"), ]; for (class, regex) in CASES { let glob = match WebhookGlob::from_str(dbg!(class)) { Ok(glob) => glob, Err(error) => panic!( - "event class glob {class:?} should produce the regex + "event class glob {class:?} should produce the regex {regex:?}, but instead failed to parse: {error}" ), }; assert_eq!( dbg!(regex), - dbg!(&glob.similar_to), + dbg!(&glob.regex), "event class {class:?} should produce the regex {regex:?}" ); } diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index db2b70488d7..c306452a5af 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -34,6 +34,7 @@ pq-sys = "*" qorb = { workspace = true, features = [ "qtop" ] } rand.workspace = true ref-cast.workspace = true +regex.workspace = true schemars.workspace = true semver.workspace = true serde.workspace = true diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index c24f472c2da..dec8b6ab0a8 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -11,39 +11,31 @@ use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; -use crate::db::error::retryable; use crate::db::error::ErrorHandler; use crate::db::model::Generation; -use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; use crate::db::model::WebhookReceiverIdentity; +use crate::db::model::WebhookRxEventGlob; use crate::db::model::WebhookRxSubscription; +use crate::db::model::WebhookSubscriptionKind; use crate::db::pool::DbConnection; -use crate::db::TransactionError; -use async_bb8_diesel::AsyncConnection; +use crate::db::schema::webhook_rx::dsl as rx_dsl; +use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; +use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; -use diesel::result::OptionalExtension; use nexus_types::external_api::params; -use nexus_types::identity::Resource; use omicron_common::api::external::CreateResult; -use omicron_common::api::external::DeleteResult; -use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; -use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; -use crate::db::schema::webhook_rx::dsl as rx_dsl; -use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; - -use std::str::FromStr; - impl DataStore { pub async fn webhook_rx_create( &self, opctx: &OpContext, params: params::WebhookCreate, + event_classes: &[&str], ) -> CreateResult { // TODO(eliza): someday we gotta allow creating webhooks with more // restrictive permissions... @@ -58,39 +50,55 @@ impl DataStore { disable_probes, } = params; - let globs = events - .iter() - .map(|s| WebhookGlob::from_str(s)) + let subscriptions = events + .into_iter() + .map(WebhookSubscriptionKind::new) .collect::, _>>()?; - let rx = self - .transaction_retry_wrapper("webhook_rx_create") - .transaction(&conn, |conn| { - // make a fresh UUID for each transaction, in case the - // transaction fails because of a UUID collision. - // - // this probably won't happen, but, ya know... - let id = WebhookReceiverUuid::new_v4(); - let receiver = WebhookReceiver { - identity: WebhookReceiverIdentity::new( - id.into_untyped_uuid(), - identity.clone(), - ), - endpoint: endpoint.to_string(), - probes_enabled: !disable_probes, - rcgen: Generation::new(), - }; - let globs = globs.clone(); - async move { - let rx = diesel::insert_into(rx_dsl::webhook_rx) - .values(receiver) - .returning(WebhookReceiver::as_returning()) - .get_result_async(&conn) - .await?; - for glob in globs { - match self + let rx = + self.transaction_retry_wrapper("webhook_rx_create") + .transaction(&conn, |conn| { + // make a fresh UUID for each transaction, in case the + // transaction fails because of a UUID collision. + // + // this probably won't happen, but, ya know... + let id = WebhookReceiverUuid::new_v4(); + let receiver = WebhookReceiver { + identity: WebhookReceiverIdentity::new( + id.into_untyped_uuid(), + identity.clone(), + ), + endpoint: endpoint.to_string(), + probes_enabled: !disable_probes, + rcgen: Generation::new(), + }; + let subscriptions = subscriptions.clone(); + async move { + let rx = diesel::insert_into(rx_dsl::webhook_rx) + .values(receiver) + .returning(WebhookReceiver::as_returning()) + .get_result_async(&conn) + .await?; + for subscription in subscriptions { + match subscription { + WebhookSubscriptionKind::Glob(glob) => { + match self.webhook_add_glob_on_conn( + opctx, + WebhookRxEventGlob::new(id, glob), + event_classes, + &conn, + ) + .await { + Ok(_) => {} + Err(AsyncInsertError::CollectionNotFound) => {} // we just created it? + Err(AsyncInsertError::DatabaseError(e)) => { + return Err(e); + } + } + }, + WebhookSubscriptionKind::Exact(value) => match self .webhook_add_subscription_on_conn( - WebhookRxSubscription::new(id, glob), + WebhookRxSubscription::exact(id, value), &conn, ) .await @@ -102,20 +110,22 @@ impl DataStore { } } } - // TODO(eliza): secrets go here... - Ok(rx) - } - }) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::WebhookReceiver, - identity.name.as_str(), - ), - ) - })?; + } + + // TODO(eliza): secrets go here... + Ok(rx) + } + }) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::WebhookReceiver, + identity.name.as_str(), + ), + ) + })?; Ok(rx) } @@ -142,6 +152,79 @@ impl DataStore { Ok(subscription) } + async fn webhook_add_glob_on_conn( + &self, + opctx: &OpContext, + glob: WebhookRxEventGlob, + event_classes: &[&str], + conn: &async_bb8_diesel::Connection, + ) -> Result { + let rx_id = glob.rx_id.into_untyped_uuid(); + let glob: WebhookRxEventGlob = WebhookReceiver::insert_resource( + rx_id, + diesel::insert_into(glob_dsl::webhook_rx_event_glob) + .values(glob) + .on_conflict((glob_dsl::rx_id, glob_dsl::glob)) + .do_update() + .set(glob_dsl::time_created.eq(diesel::dsl::now)), + ) + .insert_and_get_result_async(conn) + .await?; + self.webhook_rx_process_glob_on_conn(opctx, &glob, event_classes, conn) + .await?; + Ok(glob) + } + + async fn webhook_rx_process_glob_on_conn( + &self, + opctx: &OpContext, + glob: &WebhookRxEventGlob, + event_classes: &[&str], + conn: &async_bb8_diesel::Connection, + ) -> Result { + let regex = regex::Regex::new(&glob.glob.regex) + .expect("TODO(eliza): handle this more gracefully..."); + let mut created = 0; + for class in event_classes { + if !regex.is_match(class) { + slog::debug!( + &opctx.log, + "webhook glob does not matche event class"; + "webhook_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "regex" => ?regex, + "event_class" => ?class, + ); + continue; + } + + slog::debug!( + &opctx.log, + "webhook glob matches event class"; + "webhook_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "regex" => ?regex, + "event_class" => ?class, + ); + self.webhook_add_subscription_on_conn( + WebhookRxSubscription::for_glob(&glob, class.to_string()), + conn, + ) + .await?; + created += 1; + } + + slog::info!( + &opctx.log, + "created {created} webhook subscriptions for glob"; + "webhook_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "regex" => ?regex, + ); + + Ok(created) + } + /// List all webhook receivers whose event class subscription globs match /// the provided `event_class`. // TODO(eliza): probably paginate this... @@ -150,39 +233,20 @@ impl DataStore { opctx: &OpContext, event_class: impl ToString, ) -> ListResultVec<(WebhookReceiver, WebhookRxSubscription)> { - use async_bb8_diesel::AsyncSimpleConnection; let conn = self.pool_connection_authorized(opctx).await?; let class = event_class.to_string(); - conn.transaction_async(|conn| async move { - // TODO(eliza): this sucks, don't evaluate the globs every time we - // dispatch an event; do them at webhook-creation time instead, OR - // when updating to a sw version with new event classes... - // This currently requires a full table scan over - // `webhook_rx_subscription`, as we cannot create indices to - // accelerate text search in the present CRDB version... :( - conn.batch_execute_async( - crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL, - ) + + Self::rx_list_subscribed_query(class) + .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&*conn) .await - .unwrap(); - Self::rx_list_subscribed_query(class) - .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&conn) - .await - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } fn rx_list_subscribed_query( event_class: String, ) -> impl RunnableQuery<(WebhookReceiver, WebhookRxSubscription)> { subscription_dsl::webhook_rx_subscription - .filter( - event_class - .to_string() - .into_sql::() - .similar_to(subscription_dsl::similar_to), - ) + .filter(subscription_dsl::event_class.eq(event_class)) .order_by(subscription_dsl::rx_id.asc()) .inner_join( rx_dsl::webhook_rx.on(subscription_dsl::rx_id.eq(rx_dsl::id)), @@ -214,7 +278,7 @@ mod test { use crate::db::explain::ExplainableAsync; use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::external::IdentityMetadataCreateParams; - use omicron_test_utils::dev::{self, LogContext}; + use omicron_test_utils::dev; #[tokio::test] async fn test_event_class_globs() { @@ -223,6 +287,16 @@ mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); + let event_classes: &[&str] = &[ + "notfoo", + "foo.bar", + "foo.baz", + "foo.bar.baz", + "foo.baz.bar", + "foo.baz.quux.bar", + "baz.quux.bar", + ]; + let foo_star = datastore .webhook_rx_create( opctx, @@ -236,6 +310,7 @@ mod test { events: vec!["foo.*".to_string()], disable_probes: false, }, + &event_classes, ) .await .unwrap(); @@ -253,6 +328,7 @@ mod test { events: vec!["foo.**".to_string()], disable_probes: false, }, + &event_classes, ) .await .unwrap(); @@ -270,6 +346,7 @@ mod test { events: vec!["foo.bar".to_string()], disable_probes: false, }, + &event_classes, ) .await .unwrap(); @@ -287,6 +364,7 @@ mod test { events: vec!["foo.**.bar".parse().unwrap()], disable_probes: false, }, + &event_classes, ) .await .unwrap(); @@ -304,6 +382,7 @@ mod test { events: vec!["**.bar".parse().unwrap()], disable_probes: false, }, + &event_classes, ) .await .unwrap(); @@ -322,8 +401,12 @@ mod test { .unwrap() .into_iter() .map(|(rx, subscription)| { - eprintln!("receiver is subscribed to event {event_class:?}:\n\trx: {rx:?}\n\tsubscription: {subscription:?}"); - rx.identity + eprintln!( + "receiver is subscribed to event {event_class:?}:\n\t\ + rx: {} ({})\n\tsubscription: {subscription:#?}", + rx.identity.name, rx.identity.id, + ); + rx.identity }) .collect::>(); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index ae64daf97b0..3d32b6c1d91 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4844,31 +4844,51 @@ ON omicron.public.webhook_rx_secret ( ) WHERE time_deleted IS NULL; +-- The set of event class filters (either event class names or event class glob +-- patterns) associated with a webhook receiver. +-- +-- This is used when creating entries in the webhook_rx_subscription table to +-- indicate that a webhook receiver is interested in a given event class. +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class glob to which this receiver is subscribed. + glob STRING(512) NOT NULL, + -- Regex used when evaluating this filter against concrete event classes. + regex STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, glob) +); + +-- Look up all event class globs for a webhook receiver. +CREATE INDEX IF NOT EXISTS lookup_event_globs_for_webhook_rx +ON omicron.public.webhook_rx_event_glob ( + rx_id +); + CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- An event class (or event class glob) to which this receiver is subscribed. + -- An event class to which the receiver is subscribed. event_class STRING(512) NOT NULL, - -- The event class or event classs glob transformed into a patteern for use - -- in SQL `SIMILAR TO` clauses. + -- If this subscription is a concrete instantiation of a glob pattern, the + -- value of the glob that created it (and, a foreign key into + -- `webhook_rx_event_glob`). If the receiver is subscribed to this exact + -- event class, then this is NULL. -- - -- This is a bit interesting: users specify event class globs as sequences - -- of dot-separated segments which may be `*` to match any one segment or - -- `**` to match any number of segments. In order to match webhook events to - -- subscriptions within the database, we transform these into patterns that - -- can be used with a `SIMILAR TO` clause. - similar_to STRING(512) NOT NULL, + -- This is used when deleting a glob subscription, as it is necessary to + -- delete any concrete subscriptions to individual event classes matching + -- that glob. + glob STRING(512), + time_created TIMESTAMPTZ NOT NULL, PRIMARY KEY (rx_id, event_class) ); -CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx -ON omicron.public.webhook_rx_subscription ( - rx_id -); - -- Look up all webhook receivers subscribed to an event class. This is used by -- the dispatcher to determine who is interested in a particular event. CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event @@ -4876,6 +4896,17 @@ ON omicron.public.webhook_rx_subscription ( event_class ); +-- Look up all exact event class subscriptions for a receiver. +-- +-- This is used when generating a view of all user-provided original +-- subscriptions provided for a receiver. That list is generated by looking up +-- all exact event class subscriptions for the receiver ID in this table, +-- combined with the list of all globs in the `webhook_rx_event_glob` table. +CREATE INDEX IF NOT EXISTS lookup_exact_subscriptions_for_webhook_rx +on omicron.public.webhook_rx_subscription ( + rx_id +) WHERE glob IS NULL; + /* * Webhook event message queue. */ From d67913923ca74b8b10b564ac81d6f18788968161 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 10 Jan 2025 10:59:42 -0800 Subject: [PATCH 027/168] rework subscription creation queries a bit --- .../db-queries/src/db/datastore/webhook_rx.rs | 265 +++++++++++------- 1 file changed, 166 insertions(+), 99 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index dec8b6ab0a8..c3d327130b3 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -22,13 +22,17 @@ use crate::db::pool::DbConnection; use crate::db::schema::webhook_rx::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; +use crate::db::TransactionError; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; +use uuid::Uuid; impl DataStore { pub async fn webhook_rx_create( @@ -55,85 +59,145 @@ impl DataStore { .map(WebhookSubscriptionKind::new) .collect::, _>>()?; - let rx = - self.transaction_retry_wrapper("webhook_rx_create") - .transaction(&conn, |conn| { - // make a fresh UUID for each transaction, in case the - // transaction fails because of a UUID collision. - // - // this probably won't happen, but, ya know... - let id = WebhookReceiverUuid::new_v4(); - let receiver = WebhookReceiver { - identity: WebhookReceiverIdentity::new( - id.into_untyped_uuid(), - identity.clone(), - ), - endpoint: endpoint.to_string(), - probes_enabled: !disable_probes, - rcgen: Generation::new(), - }; - let subscriptions = subscriptions.clone(); - async move { - let rx = diesel::insert_into(rx_dsl::webhook_rx) - .values(receiver) - .returning(WebhookReceiver::as_returning()) - .get_result_async(&conn) - .await?; - for subscription in subscriptions { - match subscription { - WebhookSubscriptionKind::Glob(glob) => { - match self.webhook_add_glob_on_conn( - opctx, - WebhookRxEventGlob::new(id, glob), - event_classes, - &conn, - ) - .await { - Ok(_) => {} - Err(AsyncInsertError::CollectionNotFound) => {} // we just created it? - Err(AsyncInsertError::DatabaseError(e)) => { - return Err(e); - } - } - }, - WebhookSubscriptionKind::Exact(value) => match self - .webhook_add_subscription_on_conn( - WebhookRxSubscription::exact(id, value), - &conn, - ) - .await - { - Ok(_) => {} - Err(AsyncInsertError::CollectionNotFound) => {} // we just created it? - Err(AsyncInsertError::DatabaseError(e)) => { - return Err(e); - } - } + let err = OptionalError::new(); + let rx = self + .transaction_retry_wrapper("webhook_rx_create") + .transaction(&conn, |conn| { + // make a fresh UUID for each transaction, in case the + // transaction fails because of a UUID collision. + // + // this probably won't happen, but, ya know... + let id = WebhookReceiverUuid::new_v4(); + let receiver = WebhookReceiver { + identity: WebhookReceiverIdentity::new( + id.into_untyped_uuid(), + identity.clone(), + ), + endpoint: endpoint.to_string(), + probes_enabled: !disable_probes, + rcgen: Generation::new(), + }; + let subscriptions = subscriptions.clone(); + let err = err.clone(); + async move { + let rx = diesel::insert_into(rx_dsl::webhook_rx) + .values(receiver) + .returning(WebhookReceiver::as_returning()) + .get_result_async(&conn) + .await?; + for subscription in subscriptions { + self.add_subscription_on_conn( + opctx, + WebhookReceiverUuid::from_untyped_uuid( + rx.identity.id, + ), + subscription, + event_classes, + &conn, + ) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => err.bail(e), + TransactionError::Database(e) => e, + })?; } - } + // TODO(eliza): secrets go here... + Ok(rx) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::WebhookReceiver, + identity.name.as_str(), + ), + ) + })?; + Ok(rx) + } - // TODO(eliza): secrets go here... - Ok(rx) - } - }) + pub async fn webhook_rx_add_subscription( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + subscription: WebhookSubscriptionKind, + event_classes: &[&str], + ) -> Result<(), Error> { + opctx.authorize(authz::Action::CreateChild, authz_rx).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let num_created = self + .add_subscription_on_conn( + opctx, + authz_rx.id(), + subscription, + event_classes, + &conn, + ) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => e, + TransactionError::Database(e) => public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ), + })?; + + slog::debug!( + &opctx.log, + "added {num_created} webhook subscriptions"; + "webhook_id" => %authz_rx.id(), + ); + + Ok(()) + } + + async fn add_subscription_on_conn( + &self, + opctx: &OpContext, + rx_id: WebhookReceiverUuid, + subscription: WebhookSubscriptionKind, + event_classes: &[&str], + conn: &async_bb8_diesel::Connection, + ) -> Result> { + match subscription { + WebhookSubscriptionKind::Exact(event_class) => self + .add_exact_sub_on_conn( + WebhookRxSubscription::exact(rx_id, event_class), + &conn, + ) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::WebhookReceiver, - identity.name.as_str(), - ), + .map(|_| 1), + WebhookSubscriptionKind::Glob(glob) => { + let glob = WebhookRxEventGlob::new(rx_id, glob); + let rx_id = rx_id.into_untyped_uuid(); + let glob: WebhookRxEventGlob = + WebhookReceiver::insert_resource( + rx_id, + diesel::insert_into(glob_dsl::webhook_rx_event_glob) + .values(glob) + .on_conflict((glob_dsl::rx_id, glob_dsl::glob)) + .do_update() + .set(glob_dsl::time_created.eq(diesel::dsl::now)), ) - })?; - Ok(rx) + .insert_and_get_result_async(conn) + .await + .map_err(async_insert_error_to_txn(rx_id))?; + self.glob_generate_exact_subs(opctx, &glob, event_classes, conn) + .await + } + } } - async fn webhook_add_subscription_on_conn( + async fn add_exact_sub_on_conn( &self, subscription: WebhookRxSubscription, conn: &async_bb8_diesel::Connection, - ) -> Result { + ) -> Result> { let rx_id = subscription.rx_id.into_untyped_uuid(); let subscription: WebhookRxSubscription = WebhookReceiver::insert_resource( @@ -148,42 +212,35 @@ impl DataStore { .set(subscription_dsl::time_created.eq(diesel::dsl::now)), ) .insert_and_get_result_async(conn) - .await?; + .await + .map_err(async_insert_error_to_txn(rx_id))?; Ok(subscription) } - async fn webhook_add_glob_on_conn( - &self, - opctx: &OpContext, - glob: WebhookRxEventGlob, - event_classes: &[&str], - conn: &async_bb8_diesel::Connection, - ) -> Result { - let rx_id = glob.rx_id.into_untyped_uuid(); - let glob: WebhookRxEventGlob = WebhookReceiver::insert_resource( - rx_id, - diesel::insert_into(glob_dsl::webhook_rx_event_glob) - .values(glob) - .on_conflict((glob_dsl::rx_id, glob_dsl::glob)) - .do_update() - .set(glob_dsl::time_created.eq(diesel::dsl::now)), - ) - .insert_and_get_result_async(conn) - .await?; - self.webhook_rx_process_glob_on_conn(opctx, &glob, event_classes, conn) - .await?; - Ok(glob) - } - - async fn webhook_rx_process_glob_on_conn( + async fn glob_generate_exact_subs( &self, opctx: &OpContext, glob: &WebhookRxEventGlob, event_classes: &[&str], conn: &async_bb8_diesel::Connection, - ) -> Result { - let regex = regex::Regex::new(&glob.glob.regex) - .expect("TODO(eliza): handle this more gracefully..."); + ) -> Result> { + let regex = match regex::Regex::new(&glob.glob.regex) { + Ok(r) => r, + Err(error) => { + const MSG: &str = + "webhook glob subscription regex was not a valid regex"; + slog::error!( + &opctx.log, + "{MSG}"; + "glob" => ?glob.glob.glob, + "regex" => ?glob.glob.regex, + "error" => %error, + ); + return Err(TransactionError::CustomError( + Error::internal_error(MSG), + )); + } + }; let mut created = 0; for class in event_classes { if !regex.is_match(class) { @@ -206,7 +263,7 @@ impl DataStore { "regex" => ?regex, "event_class" => ?class, ); - self.webhook_add_subscription_on_conn( + self.add_exact_sub_on_conn( WebhookRxSubscription::for_glob(&glob, class.to_string()), conn, ) @@ -271,6 +328,16 @@ impl DataStore { // } } +fn async_insert_error_to_txn( + rx_id: Uuid, +) -> impl FnOnce(AsyncInsertError) -> TransactionError { + move |e| match e { + AsyncInsertError::CollectionNotFound => TransactionError::CustomError( + Error::not_found_by_id(ResourceType::WebhookReceiver, &rx_id), + ), + AsyncInsertError::DatabaseError(e) => TransactionError::Database(e), + } +} #[cfg(test)] mod test { use super::*; From e894112d8698a94b5c6ed52abbb4de8d8a237b79 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 10 Jan 2025 13:39:33 -0800 Subject: [PATCH 028/168] webhook dispatcher background task --- nexus-config/src/nexus_config.rs | 15 ++ nexus/db-model/src/webhook_delivery.rs | 21 +- nexus/db-model/src/webhook_rx.rs | 11 +- .../src/db/datastore/webhook_event.rs | 204 ++++++++++++++---- .../db-queries/src/db/datastore/webhook_rx.rs | 56 +++-- nexus/examples/config-second.toml | 3 + nexus/examples/config.toml | 3 + nexus/src/app/background/init.rs | 13 +- nexus/src/app/background/tasks/mod.rs | 1 + .../background/tasks/webhook_dispatcher.rs | 72 +++++++ nexus/tests/config.test.toml | 3 + nexus/types/src/internal_api/background.rs | 18 ++ smf/nexus/multi-sled/config-partial.toml | 3 + smf/nexus/single-sled/config-partial.toml | 3 + 14 files changed, 353 insertions(+), 73 deletions(-) create mode 100644 nexus/src/app/background/tasks/webhook_dispatcher.rs diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 10f7c0e3106..2c96bb7f4ab 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -419,6 +419,8 @@ pub struct BackgroundTaskConfig { RegionSnapshotReplacementFinishConfig, /// configuration for TUF artifact replication task pub tuf_artifact_replication: TufArtifactReplicationConfig, + /// configuration for webhook dispatcher task + pub webhook_dispatcher: WebhookDispatcherConfig, } #[serde_as] @@ -735,6 +737,14 @@ pub struct TufArtifactReplicationConfig { pub min_sled_replication: usize, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct WebhookDispatcherConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + /// Configuration for a nexus server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct PackageConfig { @@ -993,6 +1003,7 @@ mod test { region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 3 + webhook_dispatcher.period_secs = 42 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1194,6 +1205,9 @@ mod test { period_secs: Duration::from_secs(300), min_sled_replication: 3, }, + webhook_dispatcher: WebhookDispatcherConfig { + period_secs: Duration::from_secs(42), + } }, default_region_allocation_strategy: crate::nexus_config::RegionAllocationStrategy::Random { @@ -1279,6 +1293,7 @@ mod test { region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 3 + webhook_dispatcher.period_secs = 42 [default_region_allocation_strategy] type = "random" "##, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 0999c17cc1c..591eaaf67b3 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -7,9 +7,11 @@ use crate::schema::{webhook_delivery, webhook_delivery_attempt}; use crate::serde_time_delta::optional_time_delta; use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; +use crate::WebhookEvent; use chrono::{DateTime, TimeDelta, Utc}; use omicron_uuid_kinds::{ - WebhookDeliveryKind, WebhookEventKind, WebhookReceiverKind, + WebhookDeliveryKind, WebhookDeliveryUuid, WebhookEventKind, + WebhookReceiverKind, WebhookReceiverUuid, }; use serde::Deserialize; use serde::Serialize; @@ -58,7 +60,7 @@ pub struct WebhookDelivery { /// `webhook_event`). pub event_id: DbTypedUuid, - /// ID of the receiver to whcih this event is dispatched (foreign key into + /// ID of the receiver to which this event is dispatched (foreign key into /// `webhook_rx`). pub rx_id: DbTypedUuid, @@ -76,6 +78,21 @@ pub struct WebhookDelivery { pub time_completed: Option>, } +impl WebhookDelivery { + pub fn new(event: &WebhookEvent, rx_id: &WebhookReceiverUuid) -> Self { + Self { + // N.B.: perhaps we ought to use timestamp-based UUIDs for these? + id: WebhookDeliveryUuid::new_v4().into(), + event_id: event.id, + rx_id: (*rx_id).into(), + payload: event.event.clone(), + attempts: SqlU8::new(0), + time_created: Utc::now(), + time_completed: None, + } + } +} + /// An individual delivery attempt for a [`WebhookDelivery`]. #[derive( Clone, diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index af152b7fdff..aa97b59c647 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -4,11 +4,12 @@ use crate::collection::DatastoreCollectionConfig; use crate::schema::{ - webhook_rx, webhook_rx_event_glob, webhook_rx_secret, + webhook_delivery, webhook_rx, webhook_rx_event_glob, webhook_rx_secret, webhook_rx_subscription, }; use crate::typed_uuid::DbTypedUuid; use crate::Generation; +use crate::WebhookDelivery; use chrono::{DateTime, Utc}; use db_macros::Resource; use omicron_common::api::external::Error; @@ -28,6 +29,7 @@ use uuid::Uuid; Serialize, Deserialize, )] +#[resource(uuid_kind = WebhookReceiverKind)] #[diesel(table_name = webhook_rx)] pub struct WebhookReceiver { #[diesel(embed)] @@ -60,6 +62,13 @@ impl DatastoreCollectionConfig for WebhookReceiver { type CollectionIdColumn = webhook_rx_event_glob::dsl::rx_id; } +impl DatastoreCollectionConfig for WebhookReceiver { + type CollectionId = Uuid; + type GenerationNumberColumn = webhook_rx::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type CollectionIdColumn = webhook_delivery::dsl::rx_id; +} + // TODO(eliza): should deliveries/delivery attempts also be treated as children // of a webhook receiver? diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index e29469d327f..dbee24eaf02 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -5,55 +5,183 @@ //! [`DataStore`] methods for webhook events and event delivery dispatching. use super::DataStore; +use crate::context::OpContext; +use crate::db::collection_insert::AsyncInsertError; +use crate::db::collection_insert::DatastoreCollection; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::model::WebhookDelivery; +use crate::db::model::WebhookEvent; +use crate::db::model::WebhookReceiver; use crate::db::pool::DbConnection; +use crate::db::schema::webhook_delivery::dsl as delivery_dsl; +use crate::db::schema::webhook_event::dsl as event_dsl; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use diesel::result::OptionalExtension; -use omicron_uuid_kinds::{GenericUuid, WebhookEventUuid}; - -use crate::db::model::WebhookEvent; -use crate::db::schema::webhook_event::dsl as event_dsl; +use nexus_types::identity::Resource; +use nexus_types::internal_api::background::WebhookDispatched; +use omicron_common::api::external::Error; +use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; impl DataStore { - /// Select the next webhook event in need of dispatching. - /// - /// This performs a `SELECT ... FOR UPDATE SKIP LOCKED` on the - /// `webhook_event` table, returning the oldest webhook event which has not - /// yet been dispatched to receivers and which is not actively being - /// dispatched in another transaction. - // NOTE: it would be kinda nice if this query could also select the - // webhook receivers subscribed to this event, but I am not totally sure - // what the CRDB semantics of joining on another table in a `SELECT ... FOR - // UPDATE SKIP LOCKED` query are. We don't want to inadvertantly also lock - // the webhook receivers... - pub async fn webhook_event_select_for_dispatch( + pub async fn webhook_event_dispatch_next( &self, - conn: &async_bb8_diesel::Connection, - ) -> Result, diesel::result::Error> { - event_dsl::webhook_event - .filter(event_dsl::time_dispatched.is_null()) - .order_by(event_dsl::time_created.asc()) - .limit(1) - .for_update() - .skip_locked() - .select(WebhookEvent::as_select()) - .get_result_async(conn) + opctx: &OpContext, + ) -> Result, Error> { + let conn = self.pool_connection_authorized(&opctx).await?; + self.transaction_retry_wrapper("webhook_event_dispatch_next") + .transaction(&conn, |conn| async move { + // Select the next webhook event in need of dispatching. + // + // This performs a `SELECT ... FOR UPDATE SKIP LOCKED` on the + // `webhook_event` table, returning the oldest webhook event which has not + // yet been dispatched to receivers and which is not actively being + // dispatched in another transaction. + // NOTE: it would be kinda nice if this query could also select the + // webhook receivers subscribed to this event, but I am not totally sure + // what the CRDB semantics of joining on another table in a `SELECT ... FOR + // UPDATE SKIP LOCKED` query are. We don't want to inadvertantly also lock + // the webhook receivers... + let Some(event) = event_dsl::webhook_event + .filter(event_dsl::time_dispatched.is_null()) + .order_by(event_dsl::time_created.asc()) + .limit(1) + .for_update() + .skip_locked() + .select(WebhookEvent::as_select()) + .get_result_async(&conn) + .await + .optional()? + else { + slog::debug!( + opctx.log, + "no unlocked webhook events in need of dispatching", + ); + return Ok(None); + }; + + let mut result = WebhookDispatched { + event_id: event.id.into(), + dispatched: 0, + receivers_gone: 0, + }; + + // Find receivers subscribed to this event's class. + let rxs = self + .webhook_rx_list_subscribed_to_event_on_conn( + &event.event_class, + &conn, + ) + .await?; + + slog::debug!( + &opctx.log, + "found {} receivers subscribed to webhook event", rxs.len(); + "event_id" => ?event.id, + "event_class" => &event.event_class, + "receivers" => ?rxs.len(), + ); + + // Create dispatch entries for each receiver subscribed to this + // event class. + for (rx, sub) in rxs { + let rx_id = rx.id(); + slog::trace!( + &opctx.log, + "found receiver subscribed to event"; + "event_id" => ?event.id, + "event_class" => &event.event_class, + "receiver" => ?rx.name(), + "receiver_id" => ?rx_id, + "glob" => ?sub.glob, + ); + match self + .webhook_event_insert_delivery_on_conn( + &opctx.log, &event, &rx_id, &conn, + ) + .await + { + Ok(_) => result.dispatched += 1, + Err(AsyncInsertError::CollectionNotFound) => { + // The receiver has been deleted while we were + // trying to dispatch an event to it. That's fine; + // rather than aborting the transaction and having + // to do all this stuff over, let's just keep going. + slog::debug!( + &opctx.log, + "cannot dispatch event to a receiver that has been deleted"; + "event_id" => ?event.id, + "event_class" => &event.event_class, + "receiver" => ?rx.name(), + "receiver_id" => ?rx_id, + ); + result.receivers_gone += 1; + }, + Err(AsyncInsertError::DatabaseError(e)) => return Err(e), + } + } + + // Finally, set the dispatched timestamp for the event so it + // won't be dispatched by someone else. + diesel::update(event_dsl::webhook_event) + .filter(event_dsl::id.eq(event.id)) + .filter(event_dsl::time_dispatched.is_null()) + .set(event_dsl::time_dispatched.eq(diesel::dsl::now)) + .execute_async(&conn) + .await?; + + Ok(Some(result)) + }) .await - .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - /// Mark the webhook event with the provided UUID as dispatched. - pub async fn webhook_event_set_dispatched( + async fn webhook_event_insert_delivery_on_conn( &self, - event_id: &WebhookEventUuid, + log: &slog::Logger, + event: &WebhookEvent, + rx_id: &WebhookReceiverUuid, conn: &async_bb8_diesel::Connection, - ) -> Result<(), diesel::result::Error> { - diesel::update(event_dsl::webhook_event) - .filter(event_dsl::id.eq(event_id.into_untyped_uuid())) - .filter(event_dsl::time_dispatched.is_null()) - .set(event_dsl::time_dispatched.eq(diesel::dsl::now)) - .execute_async(conn) - .await - .map(|_| ()) // this should always be 1... + ) -> Result { + loop { + let delivery: Option = + WebhookReceiver::insert_resource( + rx_id.into_untyped_uuid(), + diesel::insert_into(delivery_dsl::webhook_delivery) + .values(WebhookDelivery::new(&event, rx_id)) + .on_conflict(delivery_dsl::id) + .do_nothing(), + ) + .insert_and_get_optional_result_async(conn) + .await?; + match delivery { + Some(delivery) => { + // XXX(eliza): is `Debug` too noisy for this? + slog::debug!( + log, + "dispatched webhook event to receiver"; + "event_id" => ?event.id, + "event_class" => &event.event_class, + "receiver_id" => ?rx_id, + ); + return Ok(delivery); + } + // The `ON CONFLICT (id) DO NOTHING` clause triggers if there's + // already a delivery entry with this UUID --- indicating a UUID + // collision. With 128 bits of random UUID, the chances of this + // happening are incredibly unlikely, but let's handle it + // gracefully nonetheless by trying again with a new UUID... + None => { + slog::warn!( + &log, + "webhook delivery UUID collision, retrying..."; + "event_id" => ?event.id, + "event_class" => &event.event_class, + "receiver_id" => ?rx_id, + ); + } + } + } } } diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index c3d327130b3..4c2e74802d8 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -29,10 +29,8 @@ use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; -use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; -use uuid::Uuid; impl DataStore { pub async fn webhook_rx_create( @@ -70,7 +68,7 @@ impl DataStore { let id = WebhookReceiverUuid::new_v4(); let receiver = WebhookReceiver { identity: WebhookReceiverIdentity::new( - id.into_untyped_uuid(), + id, identity.clone(), ), endpoint: endpoint.to_string(), @@ -88,9 +86,7 @@ impl DataStore { for subscription in subscriptions { self.add_subscription_on_conn( opctx, - WebhookReceiverUuid::from_untyped_uuid( - rx.identity.id, - ), + rx.identity.id.into(), subscription, event_classes, &conn, @@ -174,10 +170,9 @@ impl DataStore { .map(|_| 1), WebhookSubscriptionKind::Glob(glob) => { let glob = WebhookRxEventGlob::new(rx_id, glob); - let rx_id = rx_id.into_untyped_uuid(); let glob: WebhookRxEventGlob = WebhookReceiver::insert_resource( - rx_id, + rx_id.into_untyped_uuid(), diesel::insert_into(glob_dsl::webhook_rx_event_glob) .values(glob) .on_conflict((glob_dsl::rx_id, glob_dsl::glob)) @@ -198,10 +193,10 @@ impl DataStore { subscription: WebhookRxSubscription, conn: &async_bb8_diesel::Connection, ) -> Result> { - let rx_id = subscription.rx_id.into_untyped_uuid(); + let rx_id = WebhookReceiverUuid::from(subscription.rx_id); let subscription: WebhookRxSubscription = WebhookReceiver::insert_resource( - rx_id, + rx_id.into_untyped_uuid(), diesel::insert_into(subscription_dsl::webhook_rx_subscription) .values(subscription) .on_conflict(( @@ -284,19 +279,19 @@ impl DataStore { /// List all webhook receivers whose event class subscription globs match /// the provided `event_class`. - // TODO(eliza): probably paginate this... - pub async fn webhook_rx_list_subscribed_to_event( + pub(crate) async fn webhook_rx_list_subscribed_to_event_on_conn( &self, - opctx: &OpContext, event_class: impl ToString, - ) -> ListResultVec<(WebhookReceiver, WebhookRxSubscription)> { - let conn = self.pool_connection_authorized(opctx).await?; + conn: &async_bb8_diesel::Connection, + ) -> Result< + Vec<(WebhookReceiver, WebhookRxSubscription)>, + diesel::result::Error, + > { let class = event_class.to_string(); Self::rx_list_subscribed_query(class) .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&*conn) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } fn rx_list_subscribed_query( @@ -329,12 +324,15 @@ impl DataStore { } fn async_insert_error_to_txn( - rx_id: Uuid, + rx_id: WebhookReceiverUuid, ) -> impl FnOnce(AsyncInsertError) -> TransactionError { move |e| match e { - AsyncInsertError::CollectionNotFound => TransactionError::CustomError( - Error::not_found_by_id(ResourceType::WebhookReceiver, &rx_id), - ), + AsyncInsertError::CollectionNotFound => { + TransactionError::CustomError(Error::not_found_by_id( + ResourceType::WebhookReceiver, + &rx_id.into_untyped_uuid(), + )) + } AsyncInsertError::DatabaseError(e) => TransactionError::Database(e), } } @@ -456,21 +454,26 @@ mod test { async fn check_event( datastore: &DataStore, - opctx: &OpContext, // logctx: &LogContext, event_class: &str, matches: &[&WebhookReceiver], not_matches: &[&WebhookReceiver], ) { let subscribed = datastore - .webhook_rx_list_subscribed_to_event(opctx, event_class) + .webhook_rx_list_subscribed_to_event_on_conn( + event_class, + &datastore + .pool_connection_for_tests() + .await + .expect("can't get ye pool connection for tests!"), + ) .await .unwrap() .into_iter() .map(|(rx, subscription)| { eprintln!( "receiver is subscribed to event {event_class:?}:\n\t\ - rx: {} ({})\n\tsubscription: {subscription:#?}", + rx: {} ({})\n\tsubscription: {subscription:?}", rx.identity.name, rx.identity.id, ); rx.identity @@ -494,7 +497,6 @@ mod test { check_event( datastore, - opctx, "notfoo", &[], &[ @@ -509,7 +511,6 @@ mod test { check_event( datastore, - opctx, "foo.bar", &[&foo_star, &foo_starstar, &foo_bar, &starstar_bar], &[&foo_starstar_bar], @@ -518,7 +519,6 @@ mod test { check_event( datastore, - opctx, "foo.baz", &[&foo_star, &foo_starstar], &[&foo_bar, &foo_starstar_bar, &starstar_bar], @@ -527,7 +527,6 @@ mod test { check_event( datastore, - opctx, "foo.bar.baz", &[&foo_starstar], &[&foo_bar, &foo_star, &foo_starstar_bar, &starstar_bar], @@ -536,7 +535,6 @@ mod test { check_event( datastore, - opctx, "foo.baz.bar", &[&foo_starstar, &foo_starstar_bar, &starstar_bar], &[&foo_bar, &foo_star], @@ -545,7 +543,6 @@ mod test { check_event( datastore, - opctx, "foo.baz.quux.bar", &[&foo_starstar, &foo_starstar_bar, &starstar_bar], &[&foo_bar, &foo_star], @@ -554,7 +551,6 @@ mod test { check_event( datastore, - opctx, "baz.quux.bar", &[&starstar_bar], &[&foo_bar, &foo_star, &foo_starstar, &foo_starstar_bar], diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index c21e470a6d5..eac22843056 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -143,6 +143,9 @@ region_snapshot_replacement_step.period_secs = 30 region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 1 +# In general, the webhook dispatcher will be activated when events are queued, +# so we don't need to periodically activate it *that* frequently. +webhook_dispatcher.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 78a72487c8f..5828527e46b 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -129,6 +129,9 @@ region_snapshot_replacement_step.period_secs = 30 region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 1 +# In general, the webhook dispatcher will be activated when events are queued, +# so we don't need to periodically activate it *that* frequently. +webhook_dispatcher.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index eed15a1e5a5..65887514a7c 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -122,6 +122,7 @@ use super::tasks::sync_switch_configuration::SwitchPortSettingsManager; use super::tasks::tuf_artifact_replication; use super::tasks::v2p_mappings::V2PManager; use super::tasks::vpc_routes; +use super::tasks::webhook_dispatcher::WebhookDispatcher; use super::Activator; use super::Driver; use crate::app::oximeter::PRODUCER_LEASE_DURATION; @@ -180,6 +181,7 @@ pub struct BackgroundTasks { pub task_region_snapshot_replacement_step: Activator, pub task_region_snapshot_replacement_finish: Activator, pub task_tuf_artifact_replication: Activator, + pub task_webhook_dispatcher: Activator, // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as @@ -273,6 +275,7 @@ impl BackgroundTasksInitializer { task_internal_dns_propagation: Activator::new(), task_external_dns_propagation: Activator::new(), + task_webhook_dispatcher: Activator::new(), external_endpoints: external_endpoints_rx, }; @@ -339,6 +342,7 @@ impl BackgroundTasksInitializer { task_region_snapshot_replacement_step, task_region_snapshot_replacement_finish, task_tuf_artifact_replication, + task_webhook_dispatcher, // Add new background tasks here. Be sure to use this binding in a // call to `Driver::register()` below. That's what actually wires // up the Activator to the corresponding background task. @@ -885,7 +889,6 @@ impl BackgroundTasksInitializer { driver.register(TaskDefinition { name: "tuf_artifact_replication", description: "replicate update repo artifacts across sleds", - period: config.tuf_artifact_replication.period_secs, task_impl: Box::new( tuf_artifact_replication::ArtifactReplication::new( datastore.clone(), @@ -898,7 +901,13 @@ impl BackgroundTasksInitializer { activator: task_tuf_artifact_replication, }); - driver + driver.register(TaskDefinition { + description: "dispatches queued webhook events to receivers", + task_impl: Box::new(WebhookDispatcher::new(datastore)), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_webhook_dispatcher, + }); } } diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs index add3e47241a..f1dfa2cdc06 100644 --- a/nexus/src/app/background/tasks/mod.rs +++ b/nexus/src/app/background/tasks/mod.rs @@ -39,3 +39,4 @@ pub mod sync_switch_configuration; pub mod tuf_artifact_replication; pub mod v2p_mappings; pub mod vpc_routes; +pub mod webhook_dispatcher; diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs new file mode 100644 index 00000000000..59c1e849461 --- /dev/null +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task that dispatches queued webhook events to receivers. + +use crate::app::background::BackgroundTask; +use futures::future::BoxFuture; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::internal_api::background::{ + WebhookDispatched, WebhookDispatcherStatus, +}; +use omicron_common::api::external::Error; +use std::sync::Arc; + +pub struct WebhookDispatcher { + datastore: Arc, +} + +impl BackgroundTask for WebhookDispatcher { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + Box::pin(async move { + let mut dispatched = Vec::new(); + let error = + match self.actually_activate(&opctx, &mut dispatched).await { + Ok(_) => { + slog::info!( + &opctx.log, + "webhook dispatching completed successfully"; + "events_dispatched" => dispatched.len(), + ); + None + } + Err(error) => { + slog::error!( + &opctx.log, + "webhook dispatching failed"; + "events_dispatched" => dispatched.len(), + "error" => &error, + ); + Some(error.to_string()) + } + }; + // TODO(eliza): if anything was dispatched successfully, we'll want + // to activate the delivery task, once that exists! + serde_json::json!(WebhookDispatcherStatus { dispatched, error }) + }) + } +} + +impl WebhookDispatcher { + pub fn new(datastore: Arc) -> Self { + Self { datastore } + } + + async fn actually_activate( + &mut self, + opctx: &OpContext, + dispatched: &mut Vec, + ) -> Result<(), Error> { + while let Some(event) = + self.datastore.webhook_event_dispatch_next(opctx).await? + { + dispatched.push(event); + } + Ok(()) + } +} diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 634a98c8937..e9fb9174ae7 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -161,6 +161,9 @@ region_snapshot_replacement_finish.period_secs = 60 tuf_artifact_replication.period_secs = 3600 # Update integration tests are started with 4 sled agents. tuf_artifact_replication.min_sled_replication = 3 +# In general, the webhook dispatcher will be activated when events are queued, +# so we don't need to periodically activate it *that* frequently... +webhook_dispatcher.period_secs = 60 [default_region_allocation_strategy] # we only have one sled in the test environment, so we need to use the diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 80b42bcd412..1ce617cc844 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -9,6 +9,7 @@ use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SupportBundleUuid; +use omicron_uuid_kinds::WebhookEventUuid; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -449,3 +450,20 @@ impl slog::KV for DebugDatasetsRendezvousStats { Ok(()) } } + +/// The status of a `webhook_dispatcher` background task activation. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct WebhookDispatcherStatus { + /// The webhook events dispatched on this activation. + pub dispatched: Vec, + + /// Any error that occurred during activation. + pub error: Option, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct WebhookDispatched { + pub event_id: WebhookEventUuid, + pub dispatched: usize, + pub receivers_gone: usize, +} diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index f9b24c663f1..f097a70151e 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -75,6 +75,9 @@ region_snapshot_replacement_step.period_secs = 30 region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 3 +# In general, the webhook dispatcher will be activated when events are queued, +# so we don't need to periodically activate it *that* frequently. +webhook_dispatcher.period_secs = 60 [default_region_allocation_strategy] # by default, allocate across 3 distinct sleds diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index ba951ba4e07..e2e7b0f033f 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -75,6 +75,9 @@ region_snapshot_replacement_step.period_secs = 30 region_snapshot_replacement_finish.period_secs = 30 tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 1 +# In general, the webhook dispatcher will be activated when events are queued, +# so we don't need to periodically activate it *that* frequently. +webhook_dispatcher.period_secs = 60 [default_region_allocation_strategy] # by default, allocate without requirement for distinct sleds. From 7deece36e86789b849a315ed283a0bbe4985559b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 10 Jan 2025 13:40:40 -0800 Subject: [PATCH 029/168] warnings --- nexus/db-queries/src/db/datastore/webhook_rx.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 4c2e74802d8..3d55f6d8849 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -290,7 +290,7 @@ impl DataStore { let class = event_class.to_string(); Self::rx_list_subscribed_query(class) - .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&*conn) + .load_async::<(WebhookReceiver, WebhookRxSubscription)>(conn) .await } From a35d51a91364caca471a2d5008766086580f485a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 10 Jan 2025 15:46:38 -0800 Subject: [PATCH 030/168] add empty delivery task and activate as needed --- nexus-config/src/nexus_config.rs | 10 +++ nexus/src/app/background/init.rs | 24 +++++- nexus/src/app/background/tasks/mod.rs | 1 + .../background/tasks/webhook_deliverator.rs | 76 +++++++++++++++++++ .../background/tasks/webhook_dispatcher.rs | 35 +++++++-- 5 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 nexus/src/app/background/tasks/webhook_deliverator.rs diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 2c96bb7f4ab..5688ecb1d60 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -421,6 +421,8 @@ pub struct BackgroundTaskConfig { pub tuf_artifact_replication: TufArtifactReplicationConfig, /// configuration for webhook dispatcher task pub webhook_dispatcher: WebhookDispatcherConfig, + /// configuration for webhook deliverator task + pub webhook_deliverator: WebhookDeliveratorConfig, } #[serde_as] @@ -745,6 +747,14 @@ pub struct WebhookDispatcherConfig { pub period_secs: Duration, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct WebhookDeliveratorConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + /// Configuration for a nexus server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct PackageConfig { diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 65887514a7c..d18b0ceceb0 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -122,6 +122,7 @@ use super::tasks::sync_switch_configuration::SwitchPortSettingsManager; use super::tasks::tuf_artifact_replication; use super::tasks::v2p_mappings::V2PManager; use super::tasks::vpc_routes; +use super::tasks::webhook_deliverator::WebhookDeliverator; use super::tasks::webhook_dispatcher::WebhookDispatcher; use super::Activator; use super::Driver; @@ -182,6 +183,7 @@ pub struct BackgroundTasks { pub task_region_snapshot_replacement_finish: Activator, pub task_tuf_artifact_replication: Activator, pub task_webhook_dispatcher: Activator, + pub task_webhook_deliverator: Activator, // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as @@ -272,10 +274,11 @@ impl BackgroundTasksInitializer { task_region_snapshot_replacement_step: Activator::new(), task_region_snapshot_replacement_finish: Activator::new(), task_tuf_artifact_replication: Activator::new(), + task_webhook_dispatcher: Activator::new(), + task_webhook_deliverator: Activator::new(), task_internal_dns_propagation: Activator::new(), task_external_dns_propagation: Activator::new(), - task_webhook_dispatcher: Activator::new(), external_endpoints: external_endpoints_rx, }; @@ -343,6 +346,7 @@ impl BackgroundTasksInitializer { task_region_snapshot_replacement_finish, task_tuf_artifact_replication, task_webhook_dispatcher, + task_webhook_deliverator, // Add new background tasks here. Be sure to use this binding in a // call to `Driver::register()` below. That's what actually wires // up the Activator to the corresponding background task. @@ -903,11 +907,27 @@ impl BackgroundTasksInitializer { driver.register(TaskDefinition { description: "dispatches queued webhook events to receivers", - task_impl: Box::new(WebhookDispatcher::new(datastore)), + period: config.webhook_dispatcher.period_secs, + task_impl: Box::new(WebhookDispatcher::new( + datastore.clone(), + task_webhook_deliverator.clone(), + )), opctx: opctx.child(BTreeMap::new()), watchers: vec![], activator: task_webhook_dispatcher, }); + + driver.register(TaskDefinition { + name: "webhook_deliverator", + description: "sends webhook delivery requests", + period: config.webhook_deliverator.period_secs, + task_impl: Box::new(WebhookDeliverator::new(datastore)), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_webhook_deliverator, + }); + + driver } } diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs index f1dfa2cdc06..05a2e441002 100644 --- a/nexus/src/app/background/tasks/mod.rs +++ b/nexus/src/app/background/tasks/mod.rs @@ -39,4 +39,5 @@ pub mod sync_switch_configuration; pub mod tuf_artifact_replication; pub mod v2p_mappings; pub mod vpc_routes; +pub mod webhook_deliverator; pub mod webhook_dispatcher; diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs new file mode 100644 index 00000000000..b3cfb601e13 --- /dev/null +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::app::background::BackgroundTask; +use futures::future::BoxFuture; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use std::sync::Arc; + +// The Deliverator belongs to an elite order, a hallowed sub-category. He's got +// esprit up to here. Right now he is preparing to carry out his third mission +// of the night. His uniform is black as activated charcoal, filtering the very +// light out of the air. A bullet will bounce off its arachno-fiber weave like a +// wren hitting a patio door, but excess perspiration wafts through it like a +// breeze through a freshly napalmed forest. Where his body has bony +// extremities, the suit has sintered armorgel: feels like gritty jello, +// protects like a stack of telephone books. +// +// When they gave him the job, they gave him a gun. The Deliverator never deals +// in cash, but someone might come after him anyway–might want his car, or his +// cargo. The gun is tiny, aero-styled, lightweight, the kind of a gun a +// fashion designer would carry; it fires teensy darts that fly at five times +// the velocity of an SR-71 spy plane, and when you get done using it, you have +// to plug it in to the cigarette lighter, because it runs on electricity. +// +// The Deliverator never pulled that gun in anger, or in fear. He pulled it once +// in Gila Highlands. Some punks in Gila Highlands, a fancy Burbclave, wanted +// themselves a delivery, and they didn't want to pay for it. Thought they would +// impress the Deliverator with a baseball bat. The Deliverator took out his +// gun, centered its laser doo-hickey on that poised Louisville Slugger, fired +// it. The recoil was immense, as though the weapon had blown up in his hand. +// The middle third of the baseball bat turned into a column of burning sawdust +// accelerating in all directions like a bursting star. Punk ended up holding +// this bat handle with milky smoke pouring out the end. Stupid look on his +// face. Didn't get nothing but trouble from the Deliverator. +// +// Since then the Deliverator has kept the gun in the glove compartment and +// relied, instead, on a matched set of samurai swords, which have always been +// his weapon of choice anyhow. The punks in Gila Highlands weren't afraid of +// the gun, so the Deliverator was forced to use it. But swords need no +// demonstration. +// +// The Deliverator's car has enough potential energy packed into its batteries +// to fire a pound of bacon into the Asteroid Belt. Unlike a bimbo box or a Burb +// beater, the Deliverator's car unloads that power through gaping, gleaming, +// polished sphincters. When the Deliverator puts the hammer down, shit happens. +// You want to talk contact patches? Your car's tires have tiny contact patches, +// talk to the asphalt in four places the size of your tongue. The Deliverator's +// car has big sticky tires with contact patches the size of a fat lady's +// thighs. The Deliverator is in touch with the road, starts like a bad day, +// stops on a peseta. +// +// Why is the Deliverator so equipped? Because people rely on him. He is a role +// model. +// +// --- Neal Stephenson, _Snow Crash_ +pub struct WebhookDeliverator { + datastore: Arc, +} + +impl BackgroundTask for WebhookDeliverator { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + Box::pin( + async move { todo!("eliza: draw the rest of the deliverator") }, + ) + } +} + +impl WebhookDeliverator { + pub fn new(datastore: Arc) -> Self { + Self { datastore } + } +} diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 59c1e849461..8f1bbf544ff 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -4,6 +4,7 @@ //! Background task that dispatches queued webhook events to receivers. +use crate::app::background::Activator; use crate::app::background::BackgroundTask; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; @@ -16,6 +17,7 @@ use std::sync::Arc; pub struct WebhookDispatcher { datastore: Arc, + deliverator: Activator, } impl BackgroundTask for WebhookDispatcher { @@ -28,11 +30,24 @@ impl BackgroundTask for WebhookDispatcher { let error = match self.actually_activate(&opctx, &mut dispatched).await { Ok(_) => { - slog::info!( - &opctx.log, + const MSG: &str = "webhook dispatching completed successfully"; - "events_dispatched" => dispatched.len(), - ); + if !dispatched.is_empty() { + slog::info!( + &opctx.log, + "{MSG}"; + "events_dispatched" => dispatched.len(), + ); + } else { + // no sense cluttering up the logs if we didn't do + // anyuthing interesting today`s` + slog::trace!( + &opctx.log, + "{MSG}"; + "events_dispatched" => dispatched.len(), + ); + }; + None } Err(error) => { @@ -45,16 +60,20 @@ impl BackgroundTask for WebhookDispatcher { Some(error.to_string()) } }; - // TODO(eliza): if anything was dispatched successfully, we'll want - // to activate the delivery task, once that exists! + + // If any new deliveries were dispatched, call the deliverator! + if !dispatched.is_empty() { + self.deliverator.activate(); + } + serde_json::json!(WebhookDispatcherStatus { dispatched, error }) }) } } impl WebhookDispatcher { - pub fn new(datastore: Arc) -> Self { - Self { datastore } + pub fn new(datastore: Arc, deliverator: Activator) -> Self { + Self { datastore, deliverator } } async fn actually_activate( From f5b7087cf4d27ce8551507cc33668150e5033eac Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 10 Jan 2025 15:46:38 -0800 Subject: [PATCH 031/168] add empty delivery task and activate as needed --- nexus-config/src/nexus_config.rs | 5 +++++ nexus/examples/config-second.toml | 1 + nexus/examples/config.toml | 1 + nexus/tests/config.test.toml | 3 ++- smf/nexus/multi-sled/config-partial.toml | 1 + smf/nexus/single-sled/config-partial.toml | 1 + 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 5688ecb1d60..1f6c0d532aa 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -1014,6 +1014,7 @@ mod test { tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 3 webhook_dispatcher.period_secs = 42 + webhook_deliverator.period_secs = 43 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1218,6 +1219,9 @@ mod test { webhook_dispatcher: WebhookDispatcherConfig { period_secs: Duration::from_secs(42), } + webhook_deliverator: WebhookDeliveratorConfig { + period_secs: Duration::from_secs(43), + } }, default_region_allocation_strategy: crate::nexus_config::RegionAllocationStrategy::Random { @@ -1304,6 +1308,7 @@ mod test { tuf_artifact_replication.period_secs = 300 tuf_artifact_replication.min_sled_replication = 3 webhook_dispatcher.period_secs = 42 + webhook_deliverator.period_secs = 43 [default_region_allocation_strategy] type = "random" "##, diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index eac22843056..abdb5699b27 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -146,6 +146,7 @@ tuf_artifact_replication.min_sled_replication = 1 # In general, the webhook dispatcher will be activated when events are queued, # so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 5828527e46b..34f022efd14 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -132,6 +132,7 @@ tuf_artifact_replication.min_sled_replication = 1 # In general, the webhook dispatcher will be activated when events are queued, # so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index e9fb9174ae7..88d59533e19 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -162,8 +162,9 @@ tuf_artifact_replication.period_secs = 3600 # Update integration tests are started with 4 sled agents. tuf_artifact_replication.min_sled_replication = 3 # In general, the webhook dispatcher will be activated when events are queued, -# so we don't need to periodically activate it *that* frequently... +# so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 [default_region_allocation_strategy] # we only have one sled in the test environment, so we need to use the diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index f097a70151e..fe65fe582aa 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -78,6 +78,7 @@ tuf_artifact_replication.min_sled_replication = 3 # In general, the webhook dispatcher will be activated when events are queued, # so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 [default_region_allocation_strategy] # by default, allocate across 3 distinct sleds diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index e2e7b0f033f..5b184d8a932 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -78,6 +78,7 @@ tuf_artifact_replication.min_sled_replication = 1 # In general, the webhook dispatcher will be activated when events are queued, # so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 [default_region_allocation_strategy] # by default, allocate without requirement for distinct sleds. From e6544e7d0b37d339e3fe1ba10b7076e4a6a32e7a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 14 Jan 2025 12:45:00 -0800 Subject: [PATCH 032/168] implement a big chunk of webhook delivery --- Cargo.lock | 1 + nexus-config/src/nexus_config.rs | 16 + nexus/Cargo.toml | 1 + nexus/db-model/src/schema.rs | 2 + nexus/db-model/src/webhook_delivery.rs | 20 + nexus/db-queries/src/db/datastore/mod.rs | 1 + .../src/db/datastore/webhook_delivery.rs | 123 ++++++ .../src/db/datastore/webhook_event.rs | 8 +- .../db-queries/src/db/datastore/webhook_rx.rs | 80 +++- nexus/src/app/background/init.rs | 30 +- .../background/tasks/webhook_deliverator.rs | 369 +++++++++++++++++- nexus/types/src/internal_api/background.rs | 21 + schema/crdb/dbinit.sql | 11 +- 13 files changed, 651 insertions(+), 32 deletions(-) create mode 100644 nexus/db-queries/src/db/datastore/webhook_delivery.rs diff --git a/Cargo.lock b/Cargo.lock index d1d81133a41..8b88dae91fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7039,6 +7039,7 @@ dependencies = [ "itertools 0.13.0", "lldpd-client", "macaddr", + "maplit", "mg-admin-client", "nexus-auth", "nexus-client", diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 1f6c0d532aa..3765824e8d6 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -753,6 +753,14 @@ pub struct WebhookDeliveratorConfig { /// period (in seconds) for periodic activations of this background task #[serde_as(as = "DurationSeconds")] pub period_secs: Duration, + + /// duration after which another Nexus' lease on a delivery attempt is + /// considered expired. + /// + /// this is tuneable to allow testing lease expiration without having to + /// wait a long time. + #[serde(default = "WebhookDeliveratorConfig::default_lease_timeout_secs")] + pub lease_timeout_secs: u64, } /// Configuration for a nexus server @@ -826,6 +834,12 @@ impl std::fmt::Display for SchemeName { } } +impl WebhookDeliveratorConfig { + const fn default_lease_timeout_secs() -> u64 { + 60 // one minute + } +} + #[cfg(test)] mod test { use super::*; @@ -1015,6 +1029,7 @@ mod test { tuf_artifact_replication.min_sled_replication = 3 webhook_dispatcher.period_secs = 42 webhook_deliverator.period_secs = 43 + webhook_deliverator.lease_timeout_secs = 44 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1221,6 +1236,7 @@ mod test { } webhook_deliverator: WebhookDeliveratorConfig { period_secs: Duration::from_secs(43), + lease_timeout_secs: TimeDelta::from_secs(44) } }, default_region_allocation_strategy: diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 8afc36bc03b..39d94a66fb1 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -50,6 +50,7 @@ ipnetwork.workspace = true itertools.workspace = true lldpd-client.workspace = true macaddr.workspace = true +maplit.workspace = true # Not under "dev-dependencies"; these also need to be implemented for # integration tests. nexus-config.workspace = true diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index b03c944d80b..94c59c894fd 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2200,6 +2200,8 @@ table! { attempts -> Int2, time_created -> Timestamptz, time_completed -> Nullable, + deliverator_id -> Nullable, + time_delivery_started -> Nullable } } diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 591eaaf67b3..8892709f230 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -9,6 +9,7 @@ use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; use crate::WebhookEvent; use chrono::{DateTime, TimeDelta, Utc}; +use nexus_types::external_api::views; use omicron_uuid_kinds::{ WebhookDeliveryKind, WebhookDeliveryUuid, WebhookEventKind, WebhookReceiverKind, WebhookReceiverUuid, @@ -121,3 +122,22 @@ pub struct WebhookDeliveryAttempt { pub time_created: DateTime, } + +impl From for views::WebhookDeliveryState { + fn from(result: WebhookDeliveryResult) -> Self { + match result { + WebhookDeliveryResult::FailedHttpError => { + views::WebhookDeliveryState::FailedHttpError + } + WebhookDeliveryResult::FailedTimeoutError => { + views::WebhookDeliveryState::FailedTimeout + } + WebhookDeliveryResult::FailedUnreachable => { + views::WebhookDeliveryState::FailedUnreachable + } + WebhookDeliveryResult::Succeeded => { + views::WebhookDeliveryState::Delivered + } + } + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index aad45576e01..7db3cce5861 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -109,6 +109,7 @@ mod vmm; mod volume; mod volume_repair; mod vpc; +pub mod webhook_delivery; mod webhook_event; mod webhook_rx; mod zpool; diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs new file mode 100644 index 00000000000..73121c745a9 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -0,0 +1,123 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods for webhook event deliveries + +use super::DataStore; +use crate::context::OpContext; +use crate::db::model::WebhookDelivery; +use crate::db::model::WebhookDeliveryAttempt; +use crate::db::schema::webhook_delivery::dsl; +use chrono::TimeDelta; +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use omicron_common::api::external::Error; +use omicron_uuid_kinds::{ + GenericUuid, WebhookDeliveryUuid, WebhookReceiverUuid, +}; +use uuid::Uuid; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum DeliveryAttemptState { + Started, + AlreadyCompleted(DateTime), + InProgress { nexus_id: Uuid, started: DateTime }, +} + +impl DataStore { + pub async fn webhook_rx_delivery_list_ready( + &self, + opctx: &OpContext, + rx_id: &WebhookReceiverUuid, + lease_timeout: TimeDelta, + ) -> ListResultVec { + let conn = self.pool_connection_authorized(opctx).await?; + let now = + diesel::dsl::now.into_sql::(); + dsl::webhook_delivery + .filter(dsl::time_completed.is_null()) + .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) + .filter( + (dsl::time_delivery_started + .is_null() + .and(dsl::deliverator_id.is_null())) + .or(dsl::time_delivery_started.is_not_null().and( + dsl::time_delivery_started + .le(now.nullable() - lease_timeout), + )), + ) + .order_by(dsl::time_created.asc()) + .load_async(&conn) + .await + } + + pub async fn webhook_delivery_start_attempt( + &self, + opctx: &OpContext, + delivery: &WebhookDelivery, + nexus_id: &Uuid, + lease_timeout: TimeDelta, + ) -> Result { + let conn = self.pool_connection_authorized(opctx).await?; + let now = + diesel::dsl::now.into_sql::(); + let id = delivery.id.into_untyped_uuid(); + let updated = diesel::update(dsl::webhook_delivery) + .filter(dsl::time_completed.is_null()) + .filter(dsl::id.eq(id)) + .filter( + (dsl::time_delivery_started + .is_null() + .and(dsl::deliverator_id.is_null())) + .or(dsl::time_delivery_started.is_not_null().and( + dsl::time_delivery_started + .le(now.nullable() - lease_timeout), + )), + ) + .set(( + dsl::time_delivery_started.eq(now.nullable()), + dsl::deliverator_id.eq(nexus_id), + )) + .check_if_exists::(id) + .execute_and_check(&conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + match updated.status { + UpdateStatus::Updated => Ok(true), + UpdateStatus::NotUpdatedButExists { found, .. } => { + if let Some(completed) = found.time_completed { + return Ok(DeliveryAttemptState::AlreadyCompleted( + completed, + )); + } + + if let Some(started) = found.time_delivery_started { + let nexus_id = found.deliverator_id.ok_or_else( + Error::internal_error( + "if a delivery attempt has a last started \ + timestamp, the database should ensure that \ + it also has a Nexus ID", + ), + )?; + return Ok(DeliveryAttemptState::InProgress { + nexus_id, + started, + }); + } + + Err(Error::internal_error("couldn't start delivery attempt for some secret third reason???")) + } + } + } + + pub async fn webhook_delivery_finish_attempt( + &self, + opctx: &OpContext, + delivery: &WebhookDelivery, + nexus_id: &Uuid, + result: &WebhookDeliveryAttempt, + ) -> Result<(), Error> { + Err(Error::internal_error("TODO ELIZA DO THIS PART")) + } +} diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index dbee24eaf02..758065adcd0 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -39,10 +39,10 @@ impl DataStore { // yet been dispatched to receivers and which is not actively being // dispatched in another transaction. // NOTE: it would be kinda nice if this query could also select the - // webhook receivers subscribed to this event, but I am not totally sure - // what the CRDB semantics of joining on another table in a `SELECT ... FOR - // UPDATE SKIP LOCKED` query are. We don't want to inadvertantly also lock - // the webhook receivers... + // webhook receivers subscribed to this event, but this requires + // a `FOR UPDATE OF webhook_event` clause to indicate that we only wish + // to lock the `webhook_event` row and not the receiver. + // Unfortunately, I don't believe Diesel supports this at present. let Some(event) = event_dsl::webhook_event .filter(event_dsl::time_dispatched.is_null()) .order_by(event_dsl::time_created.asc()) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 3d55f6d8849..f06cd4aef24 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -16,11 +16,13 @@ use crate::db::model::Generation; use crate::db::model::WebhookReceiver; use crate::db::model::WebhookReceiverIdentity; use crate::db::model::WebhookRxEventGlob; +use crate::db::model::WebhookRxSecret; use crate::db::model::WebhookRxSubscription; use crate::db::model::WebhookSubscriptionKind; use crate::db::pool::DbConnection; use crate::db::schema::webhook_rx::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; +use crate::db::schema::webhook_rx_secret::dsl as secret_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; @@ -29,6 +31,7 @@ use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; @@ -97,7 +100,7 @@ impl DataStore { TransactionError::Database(e) => e, })?; } - // TODO(eliza): secrets go here... + // TODO(eliza): secrets? Ok(rx) } }) @@ -117,7 +120,13 @@ impl DataStore { Ok(rx) } - pub async fn webhook_rx_add_subscription( + // pub async fn webhook_rx_fetch_all(&self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver) -> Fet + + // + // Subscriptions + // + + pub async fn webhook_rx_subscription_add( &self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver, @@ -309,18 +318,61 @@ impl DataStore { WebhookRxSubscription::as_select(), )) } - // pub async fn webhook_rx_list( - // &self, - // opctx: &OpContext, - // ) -> ListResultVec { - // let conn = self.pool_connection_authorized(opctx).await?; - // rx_dsl::webhook_rx - // .filter(rx_dsl::time_deleted.is_null()) - // .select(WebhookReceiver::as_select()) - // .load_async::(&*conn) - // .await - // .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) - // } + + // + // Secrets + // + + pub async fn webhook_rx_secret_list( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, authz_rx).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + secret_dsl::webhook_rx_secret + .filter(secret_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid())) + .filter(secret_dsl::time_deleted.is_null()) + .select(WebhookRxSecret::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ) + }) + } + + async fn add_secret_on_conn( + &self, + secret: WebhookRxSecret, + conn: &async_bb8_diesel::Connection, + ) -> Result> { + let rx_id = secret.rx_id; + let secret: WebhookRxSecret = WebhookReceiver::insert_resource( + rx_id.into_untyped_uuid(), + diesel::insert_into(secret_dsl::webhook_rx_secret).values(secret), + ) + .insert_and_get_result_async(conn) + .await + .map_err(async_insert_error_to_txn(rx_id.into()))?; + Ok(secret) + } + + pub async fn webhook_rx_list( + &self, + opctx: &OpContext, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(opctx).await?; + rx_dsl::webhook_rx + .filter(rx_dsl::time_deleted.is_null()) + .select(WebhookReceiver::as_select()) + .load_async::(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } } fn async_insert_error_to_txn( diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d18b0ceceb0..b36cdb1b3ef 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -917,14 +917,28 @@ impl BackgroundTasksInitializer { activator: task_webhook_dispatcher, }); - driver.register(TaskDefinition { - name: "webhook_deliverator", - description: "sends webhook delivery requests", - period: config.webhook_deliverator.period_secs, - task_impl: Box::new(WebhookDeliverator::new(datastore)), - opctx: opctx.child(BTreeMap::new()), - watchers: vec![], - activator: task_webhook_deliverator, + driver.register({ + let lease_timeout_secs = + config.webhook_deliverator.lease_timeout_secs; + let lease_timeout = chrono::TimeDelta::seconds( + i64::try_from(lease_timeout_secs).expect( + "webhook_deliverator.lease_timeout_secs must be less \ + than i64::MAX", + ), + ); + TaskDefinition { + name: "webhook_deliverator", + description: "sends webhook delivery requests", + period: config.webhook_deliverator.period_secs, + task_impl: Box::new(WebhookDeliverator::new( + datastore, + lease_timeout, + nexus_id, + )), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_webhook_deliverator, + } }); driver diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index b3cfb601e13..1789beae0ce 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -3,9 +3,24 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::app::background::BackgroundTask; use futures::future::BoxFuture; +use http::HeaderName; +use http::HeaderValue; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::model::{ + WebhookDelivery, WebhookDeliveryAttempt, WebhookDeliveryResult, + WebhookReceiver, +}; +use nexus_db_queries::db::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::DataStore; +use nexus_db_queries::db::DbConnection; +use nexus_types::internal_api::background::{ + WebhookDeliveratorStatus, WebhookRxDeliveryStatus, +}; +use omicron_uuid_kinds::OmicronZoneUuid; +use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; +use tokio::task::JoinSet; // The Deliverator belongs to an elite order, a hallowed sub-category. He's got // esprit up to here. Right now he is preparing to carry out his third mission @@ -54,8 +69,12 @@ use std::sync::Arc; // model. // // --- Neal Stephenson, _Snow Crash_ +#[derive(Clone, Debug)] pub struct WebhookDeliverator { datastore: Arc, + nexus_id: OmicronZoneUuid, + lease_timeout: TimeDelta, + client: reqwest::Client, } impl BackgroundTask for WebhookDeliverator { @@ -63,14 +82,354 @@ impl BackgroundTask for WebhookDeliverator { &'a mut self, opctx: &'a OpContext, ) -> BoxFuture<'a, serde_json::Value> { - Box::pin( - async move { todo!("eliza: draw the rest of the deliverator") }, - ) + Box::pin(async move { + let mut status = WebhookDeliveratorStatus { + by_rx: Default::default(), + error: None, + }; + if let Err(e) = self.actually_activate(opctx, &mut status).await { + slog::error!(&opctx.log, "webhook delivery failed"; "error" => %e); + status.error = Some(e.to_string()); + } + + serde_json::json!(status) + }) } } impl WebhookDeliverator { - pub fn new(datastore: Arc) -> Self { - Self { datastore } + pub fn new( + datastore: Arc, + lease_timeout: TimeDelta, + nexus_id: OmicronZoneUuid, + ) -> Self { + let client = reqwest::Client::builder() + // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow + // redirects. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#_success + .redirect(reqwest::redirect::Policy::none()) + // Per [RFD 538 § 4.3.2][1], the client must be able to connect to a + // webhook receiver endpoint within 10 seconds, or the delivery is + // considered failed. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .connect_timeout(CONNECT_TIMEOUT) + .build() + .expect("failed to configure webhook deliverator client!"); + Self { datastore, nexus_id, lease_timeout, client } + } + + async fn actually_activate( + &mut self, + opctx: &OpContext, + status: &mut WebhookDeliveratorStatus, + ) -> Result<(), Error> { + let rxs = self.datastore.webhook_rx_list(&opctx).await?; + let mut tasks = JoinSet::new(); + for rx in rxs { + let datastore = self.datastore.clone(); + let opctx = opctx.child(maplit::btreemap! { + "receiver_id".to_string() => rx.id().to_string(), + "receiver_name".to_string() => rx.name().to_string(), + }); + let deliverator = self.clone(); + tasks.spawn(async move { + let rx_id = rx.id(); + let status = deliverator.rx_deliver(&opctx, rx).await; + (rx_id, status) + }); + } + + while let Some(result) = tasks.join_next().await { + let (rx_id, rx_status) = result.expect( + "delivery tasks should not be canceled, and nexus is compiled \ + with `panic=\"abort\"`, so they will not have panicked", + ); + status.by_rx.insert(rx_id, rx_status); + } + + Ok(()) + } + + async fn rx_deliver( + &self, + opctx: &OpContext, + rx: WebhookReceiver, + ) -> WebhookRxDeliveryStatus { + let deliveries = match self + .datastore + .webhook_delivery_list_ready(&opctx, rx.id(), self.lease_timeout) + .await + { + Err(e) => { + const MSG: &str = "failed to list ready deliveries"; + slog::error!( + &opctx.log, + "{MSG}"; + "error" => %e, + ); + return WebhookRxDeliveryStatus { + error: Some(format!("{MSG}: {e}")), + ..Default::default() + }; + } + Ok(deliveries) => deliveries, + }; + let mut delivery_status = WebhooKRxDeliveryStatus { + ready: delivery.len(), + ..Default::default() + }; + let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) + .expect("UUIDs should always be a valid header value"); + for delivery in deliveries { + let attempt = delivery.attempts + 1; + match self + .datastore + .webhook_delivery_start_attempt( + opctx, + &delivery, + self.nexus_id, + self.lease_timeout, + ) + .await + { + Ok(DeliveryAttemptState::Started) => { + slog::trace!(&opctx.log, + "webhook event delivery attempt started"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "attempt" => attempt, + ); + } + Ok(DeliveryAttemptState::AlreadyCompleted(time)) => { + slog::debug!( + &opctx.log, + "delivery of this webhookevent was already completed at {time:?}"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "time_completed" => ?time, + ); + delivery_status.already_delivered += 1; + continue; + } + Ok(DeliveryAttemptState::InProgress { + nexus_id, + time_started, + }) => { + slog::debug!( + &opctx.log, + "delivery of this webhook event is in progress by another Nexus"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "nexus_id" => nexus_id, + "time_started" => ?time_started, + ); + delivery_status.in_progress += 1; + continue; + } + Err(error) => { + slog::error!( + &opctx.log, + "unexpected database error error starting webhook delivery attempt"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "error" => %error, + ); + delivery_status + .delivery_errors + .insert(delivery_id, error.to_string()); + continue; + } + } + + // okay, actually do the thing... + let time_attempted = Utc::now(); + let sent_at = time_attempted.to_rfc3339(); + let payload = Payload { + event_class: delivery.event_class, + event_id: delivery.event_id.into(), + data: &delivery.payload, + delivery: DeliveryMetadata { + id: delivery.id.into(), + webhook_id: rx.id(), + sent_at: &sent_at, + }, + }; + let body = match serde_json::to_vec(&payload) { + Ok(body) => body, + Err(e) => { + const MSG: &str = + "failed to serialize webhook event payload"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "error" => %error, + "payload" => ?payload, + ); + delivery_status + .delivery_errors + .insert(delivery.id, format!("{MSG}: {error}")); + continue; + } + }; + // TODO(eliza): signatures! + let request = self + .client + .post(&rx.endpoint) + .header(HDR_RX_ID, hdr_rx_id) + .header(HDR_DELIVERY_ID, delivery_id.to_string()) + .header(HDR_EVENT_ID, delivery.event_id.to_string()) + .header(HDR_EVENT_CLASS, delivery.event_class) + .body(body) + // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to + // each webhook delivery request. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .timeout(Duration::from_secs(30)); + let t0 = Instant::now(); + let result = request.send().await; + let duration = t0.elapsed(); + let (delivery_result, status) = match result { + // Builder errors are our fault, that's weird! + Err(e) if e.is_builder() => { + const MSG: &str = + "internal error constructing webhook delivery request"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "error" => %error, + ); + delivery_status + .delivery_errors + .insert(delivery.id, format!("{MSG}: {error}")); + continue; + } + Err(e) => { + if let Some(status) = e.status() { + slog::warn!( + &opctx.log, + "webhook receiver endpoint returned an HTTP error"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::FailedHttpError, status) + } else { + let result = if e.is_connect() { + WebhookDeliveryResult::FailedUnreachable + } else if e.is_timeout() { + WebhookDeliveryResult::FailedTimeout + } else if e.is_redirect() { + WebhookDeliveryResult::FailedHttpError + } else { + WebhookDeliveryResult::FailedUnreachable + }; + slog::warn!( + &opctx.log, + "webhook delivery request failed"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "error" => %error, + ); + } + } + Ok(rsp) => { + let status = rsp.status(); + slog::debug!( + &opctx.log, + "webhook event delivered successfully"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::Succeeded, Some(status)) + } + }; + // only include a response duration if we actually got a response back + let response_duration = status.map(|_| { + TimeDelta::from_std(duration).expect( + "because we set a 30-second response timeout, there is no \ + way a response duration could ever exceed the max \ + representable TimeDelta of `i64::MAX` milliseconds", + ) + }); + let delivery_attempt = WebhookDeliveryAttempt { + delivery_id: delivery_id.into(), + attempt, + result, + response_status: status.map(|s| s.as_u16() as i16), + response_duration, + time_created: chrono::Utc::now(), + }; + + match self + .datastore + .webhook_delivery_finish_attempt( + opctx, + &delivery, + self.nexus_id, + &delivery_attempt, + ) + .await + { + Err(e) => { + const MSG: &str = + "failed to mark webhook delivery as finished"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => delivery.event_id, + "event_class" => delivery.event_class, + "delivery_id" => delivery_id, + "error" => %e, + ); + delivery_status + .delivery_errors + .insert(delivery_id, format!("{MSG}: {e}")); + } + Ok(_) => { + delivery_status.delivered_ok += 1; + } + } + } } } + +const HDR_DELIVERY_ID: HeaderName = + HeaderName::from_static("x-oxide-delivery-id"); +const HDR_RX_ID: HeaderName = HeaderName::from_static("x-oxide-webhook-id"); +const HDR_EVENT_ID: HeaderName = HeaderName::from_static("x-oxide-event-id"); +const HDR_EVENT_CLASS: HeaderName = + HeaderName::from_static("x-oxide-event-class"); +const HDR_SIG: HeaderName = HeaderName::from_static("x-oxide-signature"); + +#[derive(Serialize, Debug)] +struct Payload<'a> { + event_class: &'a str, + event_id: WebhookEventUuid, + data: &'a serde_json::Value, + delivery: DeliveryMetadata, +} + +#[derive(Serialize, Debug)] +struct DeliveryMetadata { + id: WebhookDeliveryUuid, + webhook_id: WebhookReceiverUuid, + sent_at: &'a str, +} diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 1ce617cc844..ef7bdb2ee09 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::external_api::views::WebhookDelivery; use chrono::DateTime; use chrono::Utc; use omicron_common::update::ArtifactHash; @@ -10,6 +11,7 @@ use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::WebhookEventUuid; +use omicron_uuid_kinds::{WebhookEventUuid, WebhookReceiverUuid}; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -467,3 +469,22 @@ pub struct WebhookDispatched { pub dispatched: usize, pub receivers_gone: usize, } + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct WebhookDeliveratorStatus { + pub by_rx: BTreeMap, + pub error: Option, +} + +#[derive( + Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Default, +)] +pub struct WebhookRxDeliveryStatus { + pub ready: usize, + pub delivered_ok: usize, + pub already_delivered: usize, + pub in_progress: usize, + pub failed_deliveries: Vec, + pub delivery_errors: BTreeMap, + pub error: Option, +} diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 3d32b6c1d91..6ddf0c35efa 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4953,7 +4953,16 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- successfully, or is considered permanently failed. time_completed TIMESTAMPTZ, - CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0) + -- Deliverator coordination bits + deliverator_id UUID, + time_delivery_started TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), + CONSTRAINT only_active_deliveries_have_started_timestamps CHECK ( + ((deliverator_id IS NULL) AND (time_delivery_started IS NULL)) OR ( + (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) + ) + ) ); -- Index for looking up all webhook messages dispatched to a receiver ID From 71cebff467871379632d2dc03de1b94c34f0ad8e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 14 Jan 2025 12:53:08 -0800 Subject: [PATCH 033/168] ag --- nexus-config/src/nexus_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 3765824e8d6..2d54826bddc 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -1233,7 +1233,7 @@ mod test { }, webhook_dispatcher: WebhookDispatcherConfig { period_secs: Duration::from_secs(42), - } + }, webhook_deliverator: WebhookDeliveratorConfig { period_secs: Duration::from_secs(43), lease_timeout_secs: TimeDelta::from_secs(44) From bd8356b8e8534960f5208794d28bbfa83710bf88 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 14 Jan 2025 15:35:41 -0800 Subject: [PATCH 034/168] blarg --- nexus/types/src/internal_api/background.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index ef7bdb2ee09..780f0ac7c67 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -10,8 +10,9 @@ use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SupportBundleUuid; +use omicron_uuid_kinds::WebhookDeliveryUuid; use omicron_uuid_kinds::WebhookEventUuid; -use omicron_uuid_kinds::{WebhookEventUuid, WebhookReceiverUuid}; +use omicron_uuid_kinds::WebhookReceiverUuid; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -470,15 +471,13 @@ pub struct WebhookDispatched { pub receivers_gone: usize, } -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct WebhookDeliveratorStatus { pub by_rx: BTreeMap, pub error: Option, } -#[derive( - Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Default, -)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct WebhookRxDeliveryStatus { pub ready: usize, pub delivered_ok: usize, From 729c245445fb13cdf98ca2cd8d285f5fd77bb30e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 14 Jan 2025 16:12:04 -0800 Subject: [PATCH 035/168] asdf --- nexus/db-model/src/webhook_delivery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 8892709f230..a58530db71f 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -129,7 +129,7 @@ impl From for views::WebhookDeliveryState { WebhookDeliveryResult::FailedHttpError => { views::WebhookDeliveryState::FailedHttpError } - WebhookDeliveryResult::FailedTimeoutError => { + WebhookDeliveryResult::FailedTimeout => { views::WebhookDeliveryState::FailedTimeout } WebhookDeliveryResult::FailedUnreachable => { From 0f57b3317fdde379297771467029e33018f21d60 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 15 Jan 2025 10:28:47 -0800 Subject: [PATCH 036/168] okay there we go --- nexus-config/src/nexus_config.rs | 2 +- nexus/db-model/src/schema.rs | 2 + nexus/db-model/src/webhook_delivery.rs | 9 +- .../src/db/datastore/webhook_delivery.rs | 53 ++++--- .../background/tasks/webhook_deliverator.rs | 149 ++++++++++-------- 5 files changed, 122 insertions(+), 93 deletions(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 2d54826bddc..9a2a1d1a09b 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -1236,7 +1236,7 @@ mod test { }, webhook_deliverator: WebhookDeliveratorConfig { period_secs: Duration::from_secs(43), - lease_timeout_secs: TimeDelta::from_secs(44) + lease_timeout_secs: 44, } }, default_region_allocation_strategy: diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 94c59c894fd..be7cf67366c 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2207,6 +2207,8 @@ table! { allow_tables_to_appear_in_same_query!(webhook_rx, webhook_delivery); joinable!(webhook_delivery -> webhook_rx (rx_id)); +allow_tables_to_appear_in_same_query!(webhook_delivery, webhook_event); +joinable!(webhook_delivery -> webhook_event (event_id)); table! { webhook_delivery_attempt (delivery_id, attempt) { diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index a58530db71f..7a37b776f6e 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -11,8 +11,8 @@ use crate::WebhookEvent; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; use omicron_uuid_kinds::{ - WebhookDeliveryKind, WebhookDeliveryUuid, WebhookEventKind, - WebhookReceiverKind, WebhookReceiverUuid, + OmicronZoneKind, WebhookDeliveryKind, WebhookDeliveryUuid, + WebhookEventKind, WebhookReceiverKind, WebhookReceiverUuid, }; use serde::Deserialize; use serde::Serialize; @@ -77,6 +77,9 @@ pub struct WebhookDelivery { /// The time at which the webhook message was either delivered successfully /// or permanently failed. pub time_completed: Option>, + + pub deliverator_id: Option>, + pub time_delivery_started: Option>, } impl WebhookDelivery { @@ -90,6 +93,8 @@ impl WebhookDelivery { attempts: SqlU8::new(0), time_created: Utc::now(), time_completed: None, + deliverator_id: None, + time_delivery_started: None, } } } diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 73121c745a9..f07c3b01230 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -6,23 +6,28 @@ use super::DataStore; use crate::context::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; +use crate::db::model::WebhookEvent; use crate::db::schema::webhook_delivery::dsl; +use crate::db::schema::webhook_event::dsl as event_dsl; +use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateStatus; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; use chrono::{DateTime, Utc}; use diesel::prelude::*; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{ - GenericUuid, WebhookDeliveryUuid, WebhookReceiverUuid, -}; -use uuid::Uuid; +use omicron_common::api::external::ListResultVec; +use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookReceiverUuid}; #[derive(Debug, Clone, Eq, PartialEq)] pub enum DeliveryAttemptState { Started, AlreadyCompleted(DateTime), - InProgress { nexus_id: Uuid, started: DateTime }, + InProgress { nexus_id: OmicronZoneUuid, started: DateTime }, } impl DataStore { @@ -31,7 +36,7 @@ impl DataStore { opctx: &OpContext, rx_id: &WebhookReceiverUuid, lease_timeout: TimeDelta, - ) -> ListResultVec { + ) -> ListResultVec<(WebhookDelivery, WebhookEvent)> { let conn = self.pool_connection_authorized(opctx).await?; let now = diesel::dsl::now.into_sql::(); @@ -48,15 +53,22 @@ impl DataStore { )), ) .order_by(dsl::time_created.asc()) - .load_async(&conn) + // Join with the `webhook_event` table to get the event class, which + // is necessary to construct delivery requests. + .inner_join( + event_dsl::webhook_event.on(event_dsl::id.eq(dsl::event_id)), + ) + .select((WebhookDelivery::as_select(), WebhookEvent::as_select())) + .load_async(&*conn) .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn webhook_delivery_start_attempt( &self, opctx: &OpContext, delivery: &WebhookDelivery, - nexus_id: &Uuid, + nexus_id: &OmicronZoneUuid, lease_timeout: TimeDelta, ) -> Result { let conn = self.pool_connection_authorized(opctx).await?; @@ -77,31 +89,32 @@ impl DataStore { ) .set(( dsl::time_delivery_started.eq(now.nullable()), - dsl::deliverator_id.eq(nexus_id), + dsl::deliverator_id.eq(nexus_id.into_untyped_uuid()), )) .check_if_exists::(id) .execute_and_check(&conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; match updated.status { - UpdateStatus::Updated => Ok(true), - UpdateStatus::NotUpdatedButExists { found, .. } => { - if let Some(completed) = found.time_completed { + UpdateStatus::Updated => Ok(DeliveryAttemptState::Started), + UpdateStatus::NotUpdatedButExists => { + if let Some(completed) = updated.found.time_completed { return Ok(DeliveryAttemptState::AlreadyCompleted( completed, )); } - if let Some(started) = found.time_delivery_started { - let nexus_id = found.deliverator_id.ok_or_else( - Error::internal_error( - "if a delivery attempt has a last started \ + if let Some(started) = updated.found.time_delivery_started { + let nexus_id = + updated.found.deliverator_id.ok_or_else(|| { + Error::internal_error( + "if a delivery attempt has a last started \ timestamp, the database should ensure that \ it also has a Nexus ID", - ), - )?; + ) + })?; return Ok(DeliveryAttemptState::InProgress { - nexus_id, + nexus_id: nexus_id.into(), started, }); } @@ -115,7 +128,7 @@ impl DataStore { &self, opctx: &OpContext, delivery: &WebhookDelivery, - nexus_id: &Uuid, + nexus_id: &OmicronZoneUuid, result: &WebhookDeliveryAttempt, ) -> Result<(), Error> { Err(Error::internal_error("TODO ELIZA DO THIS PART")) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 1789beae0ce..879a25c8e7f 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -2,24 +2,26 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::app::background::BackgroundTask; +use chrono::{TimeDelta, Utc}; use futures::future::BoxFuture; use http::HeaderName; use http::HeaderValue; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::model::{ - WebhookDelivery, WebhookDeliveryAttempt, WebhookDeliveryResult, - WebhookReceiver, + SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookReceiver, }; -use nexus_db_queries::db::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::DataStore; -use nexus_db_queries::db::DbConnection; +use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ WebhookDeliveratorStatus, WebhookRxDeliveryStatus, }; -use omicron_uuid_kinds::OmicronZoneUuid; -use std::collections::HashMap; +use omicron_common::api::external::Error; +use omicron_uuid_kinds::{ + OmicronZoneUuid, WebhookDeliveryUuid, WebhookEventUuid, WebhookReceiverUuid, +}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::task::JoinSet; // The Deliverator belongs to an elite order, a hallowed sub-category. He's got @@ -69,7 +71,7 @@ use tokio::task::JoinSet; // model. // // --- Neal Stephenson, _Snow Crash_ -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct WebhookDeliverator { datastore: Arc, nexus_id: OmicronZoneUuid, @@ -114,7 +116,7 @@ impl WebhookDeliverator { // considered failed. // // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .connect_timeout(CONNECT_TIMEOUT) + .connect_timeout(Duration::from_secs(10)) .build() .expect("failed to configure webhook deliverator client!"); Self { datastore, nexus_id, lease_timeout, client } @@ -128,7 +130,6 @@ impl WebhookDeliverator { let rxs = self.datastore.webhook_rx_list(&opctx).await?; let mut tasks = JoinSet::new(); for rx in rxs { - let datastore = self.datastore.clone(); let opctx = opctx.child(maplit::btreemap! { "receiver_id".to_string() => rx.id().to_string(), "receiver_name".to_string() => rx.name().to_string(), @@ -159,7 +160,11 @@ impl WebhookDeliverator { ) -> WebhookRxDeliveryStatus { let deliveries = match self .datastore - .webhook_delivery_list_ready(&opctx, rx.id(), self.lease_timeout) + .webhook_rx_delivery_list_ready( + &opctx, + &rx.id(), + self.lease_timeout, + ) .await { Err(e) => { @@ -176,20 +181,22 @@ impl WebhookDeliverator { } Ok(deliveries) => deliveries, }; - let mut delivery_status = WebhooKRxDeliveryStatus { - ready: delivery.len(), + let mut delivery_status = WebhookRxDeliveryStatus { + ready: deliveries.len(), ..Default::default() }; let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) .expect("UUIDs should always be a valid header value"); - for delivery in deliveries { - let attempt = delivery.attempts + 1; + for (delivery, event) in deliveries { + let attempt = (*delivery.attempts) + 1; + let delivery_id = WebhookDeliveryUuid::from(delivery.id); + let event_class = &event.event_class; match self .datastore .webhook_delivery_start_attempt( opctx, &delivery, - self.nexus_id, + &self.nexus_id, self.lease_timeout, ) .await @@ -197,36 +204,33 @@ impl WebhookDeliverator { Ok(DeliveryAttemptState::Started) => { slog::trace!(&opctx.log, "webhook event delivery attempt started"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "attempt" => attempt, ); } Ok(DeliveryAttemptState::AlreadyCompleted(time)) => { slog::debug!( &opctx.log, - "delivery of this webhookevent was already completed at {time:?}"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "delivery of this webhook event was already completed at {time:?}"; + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "time_completed" => ?time, ); delivery_status.already_delivered += 1; continue; } - Ok(DeliveryAttemptState::InProgress { - nexus_id, - time_started, - }) => { + Ok(DeliveryAttemptState::InProgress { nexus_id, started }) => { slog::debug!( &opctx.log, "delivery of this webhook event is in progress by another Nexus"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, - "nexus_id" => nexus_id, - "time_started" => ?time_started, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, + "nexus_id" => %nexus_id, + "time_started" => ?started, ); delivery_status.in_progress += 1; continue; @@ -235,9 +239,9 @@ impl WebhookDeliverator { slog::error!( &opctx.log, "unexpected database error error starting webhook delivery attempt"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "error" => %error, ); delivery_status @@ -251,7 +255,7 @@ impl WebhookDeliverator { let time_attempted = Utc::now(); let sent_at = time_attempted.to_rfc3339(); let payload = Payload { - event_class: delivery.event_class, + event_class: event_class.as_ref(), event_id: delivery.event_id.into(), data: &delivery.payload, delivery: DeliveryMetadata { @@ -268,15 +272,15 @@ impl WebhookDeliverator { slog::error!( &opctx.log, "{MSG}"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, - "error" => %error, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, + "error" => %e, "payload" => ?payload, ); delivery_status .delivery_errors - .insert(delivery.id, format!("{MSG}: {error}")); + .insert(delivery_id, format!("{MSG}: {e}")); continue; } }; @@ -284,10 +288,10 @@ impl WebhookDeliverator { let request = self .client .post(&rx.endpoint) - .header(HDR_RX_ID, hdr_rx_id) + .header(HDR_RX_ID, hdr_rx_id.clone()) .header(HDR_DELIVERY_ID, delivery_id.to_string()) .header(HDR_EVENT_ID, delivery.event_id.to_string()) - .header(HDR_EVENT_CLASS, delivery.event_class) + .header(HDR_EVENT_CLASS, event_class) .body(body) // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to // each webhook delivery request. @@ -305,14 +309,14 @@ impl WebhookDeliverator { slog::error!( &opctx.log, "{MSG}"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, - "error" => %error, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, + "error" => %e, ); delivery_status .delivery_errors - .insert(delivery.id, format!("{MSG}: {error}")); + .insert(delivery_id, format!("{MSG}: {e}")); continue; } Err(e) => { @@ -320,13 +324,13 @@ impl WebhookDeliverator { slog::warn!( &opctx.log, "webhook receiver endpoint returned an HTTP error"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "response_status" => ?status, "response_duration" => ?duration, ); - (WebhookDeliveryResult::FailedHttpError, status) + (WebhookDeliveryResult::FailedHttpError, Some(status)) } else { let result = if e.is_connect() { WebhookDeliveryResult::FailedUnreachable @@ -340,11 +344,12 @@ impl WebhookDeliverator { slog::warn!( &opctx.log, "webhook delivery request failed"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, - "error" => %error, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, + "error" => %e, ); + (result, None) } } Ok(rsp) => { @@ -352,9 +357,9 @@ impl WebhookDeliverator { slog::debug!( &opctx.log, "webhook event delivered successfully"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "response_status" => ?status, "response_duration" => ?duration, ); @@ -370,9 +375,9 @@ impl WebhookDeliverator { ) }); let delivery_attempt = WebhookDeliveryAttempt { - delivery_id: delivery_id.into(), - attempt, - result, + delivery_id: delivery.id, + attempt: SqlU8::new(attempt), + result: delivery_result, response_status: status.map(|s| s.as_u16() as i16), response_duration, time_created: chrono::Utc::now(), @@ -383,7 +388,7 @@ impl WebhookDeliverator { .webhook_delivery_finish_attempt( opctx, &delivery, - self.nexus_id, + &self.nexus_id, &delivery_attempt, ) .await @@ -394,9 +399,9 @@ impl WebhookDeliverator { slog::error!( &opctx.log, "{MSG}"; - "event_id" => delivery.event_id, - "event_class" => delivery.event_class, - "delivery_id" => delivery_id, + "event_id" => %delivery.event_id, + "event_class" => event_class, + "delivery_id" => %delivery_id, "error" => %e, ); delivery_status @@ -408,6 +413,10 @@ impl WebhookDeliverator { } } } + + // TODO(eliza): if no events were sent, do a probe... + + delivery_status } } @@ -419,16 +428,16 @@ const HDR_EVENT_CLASS: HeaderName = HeaderName::from_static("x-oxide-event-class"); const HDR_SIG: HeaderName = HeaderName::from_static("x-oxide-signature"); -#[derive(Serialize, Debug)] +#[derive(serde::Serialize, Debug)] struct Payload<'a> { event_class: &'a str, event_id: WebhookEventUuid, data: &'a serde_json::Value, - delivery: DeliveryMetadata, + delivery: DeliveryMetadata<'a>, } -#[derive(Serialize, Debug)] -struct DeliveryMetadata { +#[derive(serde::Serialize, Debug)] +struct DeliveryMetadata<'a> { id: WebhookDeliveryUuid, webhook_id: WebhookReceiverUuid, sent_at: &'a str, From 848154c923cb510b3f07d782760119a93b990ab5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 15 Jan 2025 10:30:40 -0800 Subject: [PATCH 037/168] don't select the whole event row --- nexus/db-queries/src/db/datastore/webhook_delivery.rs | 5 ++--- nexus/src/app/background/tasks/webhook_deliverator.rs | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index f07c3b01230..e95a421035c 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -10,7 +10,6 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; -use crate::db::model::WebhookEvent; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_event::dsl as event_dsl; use crate::db::update_and_check::UpdateAndCheck; @@ -36,7 +35,7 @@ impl DataStore { opctx: &OpContext, rx_id: &WebhookReceiverUuid, lease_timeout: TimeDelta, - ) -> ListResultVec<(WebhookDelivery, WebhookEvent)> { + ) -> ListResultVec<(WebhookDelivery, String)> { let conn = self.pool_connection_authorized(opctx).await?; let now = diesel::dsl::now.into_sql::(); @@ -58,7 +57,7 @@ impl DataStore { .inner_join( event_dsl::webhook_event.on(event_dsl::id.eq(dsl::event_id)), ) - .select((WebhookDelivery::as_select(), WebhookEvent::as_select())) + .select((WebhookDelivery::as_select(), event_dsl::event_class)) .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 879a25c8e7f..bcda64a9d0c 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -187,10 +187,10 @@ impl WebhookDeliverator { }; let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) .expect("UUIDs should always be a valid header value"); - for (delivery, event) in deliveries { + for (delivery, event_class) in deliveries { let attempt = (*delivery.attempts) + 1; let delivery_id = WebhookDeliveryUuid::from(delivery.id); - let event_class = &event.event_class; + let event_class = &event_class; match self .datastore .webhook_delivery_start_attempt( From d3434674473894abdf239deadca0e98b72015f6b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 15 Jan 2025 11:07:45 -0800 Subject: [PATCH 038/168] wip delivery finish attempt query --- .../src/db/datastore/webhook_delivery.rs | 105 +++++++++++++++--- schema/crdb/dbinit.sql | 4 +- 2 files changed, 91 insertions(+), 18 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index e95a421035c..7e12084bb3e 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -8,11 +8,14 @@ use super::DataStore; use crate::context::OpContext; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; +use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; use crate::db::schema::webhook_delivery::dsl; +use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateAndQueryResult; use crate::db::update_and_check::UpdateStatus; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; @@ -43,13 +46,13 @@ impl DataStore { .filter(dsl::time_completed.is_null()) .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) .filter( - (dsl::time_delivery_started - .is_null() - .and(dsl::deliverator_id.is_null())) - .or(dsl::time_delivery_started.is_not_null().and( - dsl::time_delivery_started - .le(now.nullable() - lease_timeout), - )), + (dsl::deliverator_id.is_null()).or(dsl::time_delivery_started + .is_not_null() + .and( + dsl::time_delivery_started + .le(now.nullable() - lease_timeout), + )), + // TODO(eliza): retry backoffs...? ) .order_by(dsl::time_created.asc()) // Join with the `webhook_event` table to get the event class, which @@ -78,13 +81,12 @@ impl DataStore { .filter(dsl::time_completed.is_null()) .filter(dsl::id.eq(id)) .filter( - (dsl::time_delivery_started - .is_null() - .and(dsl::deliverator_id.is_null())) - .or(dsl::time_delivery_started.is_not_null().and( - dsl::time_delivery_started - .le(now.nullable() - lease_timeout), - )), + dsl::deliverator_id.is_null().or(dsl::time_delivery_started + .is_not_null() + .and( + dsl::time_delivery_started + .le(now.nullable() - lease_timeout), + )), ) .set(( dsl::time_delivery_started.eq(now.nullable()), @@ -128,8 +130,79 @@ impl DataStore { opctx: &OpContext, delivery: &WebhookDelivery, nexus_id: &OmicronZoneUuid, - result: &WebhookDeliveryAttempt, + attempt: &WebhookDeliveryAttempt, ) -> Result<(), Error> { - Err(Error::internal_error("TODO ELIZA DO THIS PART")) + const MAX_ATTEMPTS: u8 = 4; + let conn = self.pool_connection_authorized(opctx).await?; + diesel::insert_into(attempt_dsl::webhook_delivery_attempt) + .values(attempt.clone()) + .on_conflict((attempt_dsl::delivery_id, attempt_dsl::attempt)) + .do_nothing() + .returning(WebhookDeliveryAttempt::as_returning()) + .execute_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + // Has the delivery either completed successfully or exhausted all of + // its retry attempts? + let succeeded = + attempt.result == nexus_db_model::WebhookDeliveryResult::Succeeded; + let failed_permanently = *attempt.attempt >= MAX_ATTEMPTS; + let (completed, new_nexus_id) = if succeeded || failed_permanently { + // If the delivery has succeeded or failed permanently, set the + // "time_completed" timestamp to mark it as finished. Also, leave + // the delivering Nexus ID in place to maintain a record of who + // finished the delivery. + (Some(Utc::now()), Some(nexus_id.into_untyped_uuid())) + } else { + // Otherwise, "unlock" the delivery for other nexii. + (None, None) + }; + let prev_attempts = SqlU8::new((*attempt.attempt) - 1); + let UpdateAndQueryResult { status, found } = + diesel::update(dsl::webhook_delivery) + .filter(dsl::id.eq(delivery.id.into_untyped_uuid())) + .filter(dsl::deliverator_id.eq(nexus_id.into_untyped_uuid())) + .filter(dsl::attempts.eq(prev_attempts)) + // Don't mark a delivery as completed if it's already completed! + .filter(dsl::time_completed.is_null()) + .set(( + dsl::time_completed.eq(completed), + // XXX(eliza): hmm this might be racy; we should probably increment this + // in place and use it to determine the attempt number? + dsl::attempts.eq(attempt.attempt), + dsl::deliverator_id.eq(new_nexus_id), + )) + .check_if_exists::(delivery.id) + .execute_and_check(&conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + if status == UpdateStatus::Updated { + return Ok(()); + } + + if let Some(other_nexus_id) = found.deliverator_id { + return Err(Error::conflict(format!( + "cannot mark delivery completed, as {other_nexus_id:?} was \ + attempting to deliver it", + ))); + } + + if found.time_completed.is_some() { + return Err(Error::conflict( + "delivery was already marked as completed", + )); + } + + if found.attempts != prev_attempts { + return Err(Error::conflict("wrong number of delivery attempts")); + } + + Err(Error::internal_error( + "couldn't update delivery for some other reason i didn't think of here..." + )) } } diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 6ddf0c35efa..e110b556b44 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4958,8 +4958,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( time_delivery_started TIMESTAMPTZ, CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), - CONSTRAINT only_active_deliveries_have_started_timestamps CHECK ( - ((deliverator_id IS NULL) AND (time_delivery_started IS NULL)) OR ( + CONSTRAINT active_deliveries_have_started_timestamps CHECK ( + (deliverator_id IS NULL) OR ( (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) ) ) From 4ad4e04bf4bf152c7399c97fc7e25b4186ca51c1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 16 Jan 2025 10:58:49 -0800 Subject: [PATCH 039/168] secrets plumbing --- nexus/db-model/src/schema.rs | 4 +- nexus/db-model/src/webhook_rx.rs | 74 ++++++++++++++++++- .../db-queries/src/db/datastore/webhook_rx.rs | 26 +++++-- nexus/types/src/external_api/views.rs | 4 +- schema/crdb/dbinit.sql | 4 +- uuid-kinds/src/lib.rs | 1 + 6 files changed, 96 insertions(+), 17 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index be7cf67366c..6926ed02055 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2147,8 +2147,8 @@ table! { table! { webhook_rx_secret (rx_id, signature_id) { rx_id -> Uuid, - signature_id -> Text, - secret -> Binary, + signature_id -> Uuid, + secret -> Text, time_created -> Timestamptz, time_deleted -> Nullable, } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index aa97b59c647..a4fa4629c36 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -12,13 +12,60 @@ use crate::Generation; use crate::WebhookDelivery; use chrono::{DateTime, Utc}; use db_macros::Resource; +use nexus_types::external_api::views; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{WebhookReceiverKind, WebhookReceiverUuid}; +use omicron_uuid_kinds::{ + WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretKind, + WebhookSecretUuid, +}; use serde::{Deserialize, Serialize}; use std::str::FromStr; use uuid::Uuid; -/// A webhook receiver configuration. +/// The full configuration of a webhook receiver, including the +/// [`WebhookReceiver`] itself and its subscriptions and secrets. +pub struct WebhookReceiverConfig { + pub rx: WebhookReceiver, + pub secrets: Vec, + pub events: Vec, +} + +impl TryFrom for views::Webhook { + type Error = Error; + fn try_from( + WebhookReceiverConfig { rx, secrets, events }: WebhookReceiverConfig, + ) -> Result { + let secrets = secrets + .iter() + .map(|WebhookRxSecret { signature_id, .. }| { + views::WebhookSecretId { id: signature_id.to_string() } + }) + .collect(); + let events = events + .into_iter() + .map(WebhookSubscriptionKind::into_event_class_string) + .collect(); + let WebhookReceiver { identity, endpoint, probes_enabled, rcgen: _ } = + rx; + let WebhookReceiverIdentity { id, name, description, .. } = identity; + let endpoint = endpoint.parse().map_err(|e| Error::InternalError { + // This is an internal error, as we should not have ever allowed + // an invalid URL to be inserted into the database... + internal_message: format!("invalid webhook URL {endpoint:?}: {e}",), + })?; + Ok(views::Webhook { + id: id.into(), + name: name.to_string(), + description, + endpoint, + secrets, + events, + disable_probes: !probes_enabled, + }) + } +} + +/// A row in the `webhook_rx` table. #[derive( Clone, Debug, @@ -78,12 +125,24 @@ impl DatastoreCollectionConfig for WebhookReceiver { #[diesel(table_name = webhook_rx_secret)] pub struct WebhookRxSecret { pub rx_id: DbTypedUuid, - pub signature_id: String, - pub secret: Vec, + pub signature_id: DbTypedUuid, + pub secret: String, pub time_created: DateTime, pub time_deleted: Option>, } +impl WebhookRxSecret { + pub fn new(rx_id: WebhookReceiverUuid, secret: String) -> Self { + Self { + rx_id: rx_id.into(), + signature_id: WebhookSecretUuid::new_v4().into(), + secret, + time_created: Utc::now(), + time_deleted: None, + } + } +} + #[derive( Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, )] @@ -132,6 +191,13 @@ impl WebhookSubscriptionKind { Ok(Self::Exact(value)) } } + + fn into_event_class_string(self) -> String { + match self { + Self::Exact(class) => class, + Self::Glob(WebhookGlob { glob, .. }) => glob, + } + } } #[derive( diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index f06cd4aef24..75292dd6b23 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -14,6 +14,7 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Generation; use crate::db::model::WebhookReceiver; +use crate::db::model::WebhookReceiverConfig; use crate::db::model::WebhookReceiverIdentity; use crate::db::model::WebhookRxEventGlob; use crate::db::model::WebhookRxSecret; @@ -41,7 +42,7 @@ impl DataStore { opctx: &OpContext, params: params::WebhookCreate, event_classes: &[&str], - ) -> CreateResult { + ) -> CreateResult { // TODO(eliza): someday we gotta allow creating webhooks with more // restrictive permissions... opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; @@ -59,9 +60,8 @@ impl DataStore { .into_iter() .map(WebhookSubscriptionKind::new) .collect::, _>>()?; - let err = OptionalError::new(); - let rx = self + let (rx, secrets) = self .transaction_retry_wrapper("webhook_rx_create") .transaction(&conn, |conn| { // make a fresh UUID for each transaction, in case the @@ -79,6 +79,7 @@ impl DataStore { rcgen: Generation::new(), }; let subscriptions = subscriptions.clone(); + let secret_keys = secrets.clone(); let err = err.clone(); async move { let rx = diesel::insert_into(rx_dsl::webhook_rx) @@ -100,8 +101,21 @@ impl DataStore { TransactionError::Database(e) => e, })?; } - // TODO(eliza): secrets? - Ok(rx) + let mut secrets = Vec::with_capacity(secret_keys.len()); + for secret in secret_keys { + let secret = self + .add_secret_on_conn( + WebhookRxSecret::new(id, secret), + &conn, + ) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => err.bail(e), + TransactionError::Database(e) => e, + })?; + secrets.push(secret); + } + Ok((rx, secrets)) } }) .await @@ -117,7 +131,7 @@ impl DataStore { ), ) })?; - Ok(rx) + Ok(WebhookReceiverConfig { rx, secrets, events: subscriptions }) } // pub async fn webhook_rx_fetch_all(&self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver) -> Fet diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index ef46dc1587a..7727628803a 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1049,11 +1049,9 @@ pub struct Webhook { pub id: WebhookReceiverUuid, /// The identifier assigned to this webhook receiver upon creation. pub name: String, + pub description: String, /// The URL that webhook notification requests are sent to. pub endpoint: Url, - /// The UUID of the user associated with this webhook receiver for - /// role-based ccess control. - pub actor_id: Uuid, // A list containing the IDs of the secret keys used to sign payloads sent // to this receiver. pub secrets: Vec, diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e110b556b44..f4bc563a6a0 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4829,9 +4829,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, -- ID of this secret. - signature_id STRING(63) NOT NULL, + signature_id UUID NOT NULL, -- Secret value. - secret BYTES NOT NULL, + secret STRING(512) NOT NULL, time_created TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index f03a7049da8..1526dea37d5 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -79,5 +79,6 @@ impl_typed_uuid_kind! { WebhookEvent => "webhook_event", WebhookReceiver => "webhook_receiver", WebhookDelivery => "webhook_delivery", + WebhookSecret => "webhook_secret", Zpool => "zpool", } From ac5bf6753c0c647d88c2b4ec6688c1c5f0bede68 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 22 Jan 2025 14:42:25 -0800 Subject: [PATCH 040/168] start api plumbing --- nexus/db-model/src/schema.rs | 23 ++--- nexus/db-model/src/webhook_rx.rs | 24 ++--- .../db-queries/src/db/datastore/webhook_rx.rs | 89 ++++++++++++++++--- nexus/db-queries/src/db/lookup.rs | 22 +++++ nexus/src/app/mod.rs | 1 + nexus/src/app/webhook.rs | 27 ++++++ nexus/src/external_api/http_entrypoints.rs | 16 ++-- schema/crdb/dbinit.sql | 2 +- 8 files changed, 163 insertions(+), 41 deletions(-) create mode 100644 nexus/src/app/webhook.rs diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 6926ed02055..c028558546c 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2131,7 +2131,7 @@ table! { } table! { - webhook_rx (id) { + webhook_receiver (id) { id -> Uuid, name -> Text, description -> Text, @@ -2154,9 +2154,6 @@ table! { } } -allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_secret); -joinable!(webhook_rx_secret -> webhook_rx (rx_id)); - table! { webhook_rx_subscription (rx_id, event_class) { rx_id -> Uuid, @@ -2175,11 +2172,15 @@ table! { } } -allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_subscription); -joinable!(webhook_rx_subscription -> webhook_rx (rx_id)); - -allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_event_glob); -joinable!(webhook_rx_event_glob -> webhook_rx (rx_id)); +allow_tables_to_appear_in_same_query!( + webhook_receiver, + webhook_rx_secret, + webhook_rx_subscription, + webhook_rx_event_glob +); +joinable!(webhook_rx_subscription -> webhook_receiver (rx_id)); +joinable!(webhook_rx_secret -> webhook_receiver (rx_id)); +joinable!(webhook_rx_event_glob -> webhook_receiver (rx_id)); table! { webhook_event (id) { @@ -2205,8 +2206,8 @@ table! { } } -allow_tables_to_appear_in_same_query!(webhook_rx, webhook_delivery); -joinable!(webhook_delivery -> webhook_rx (rx_id)); +allow_tables_to_appear_in_same_query!(webhook_receiver, webhook_delivery); +joinable!(webhook_delivery -> webhook_receiver (rx_id)); allow_tables_to_appear_in_same_query!(webhook_delivery, webhook_event); joinable!(webhook_delivery -> webhook_event (event_id)); diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index a4fa4629c36..5caa5276993 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -4,8 +4,8 @@ use crate::collection::DatastoreCollectionConfig; use crate::schema::{ - webhook_delivery, webhook_rx, webhook_rx_event_glob, webhook_rx_secret, - webhook_rx_subscription, + webhook_delivery, webhook_receiver, webhook_rx_event_glob, + webhook_rx_secret, webhook_rx_subscription, }; use crate::typed_uuid::DbTypedUuid; use crate::Generation; @@ -65,7 +65,7 @@ impl TryFrom for views::Webhook { } } -/// A row in the `webhook_rx` table. +/// A row in the `webhook_receiver` table. #[derive( Clone, Debug, @@ -77,7 +77,7 @@ impl TryFrom for views::Webhook { Deserialize, )] #[resource(uuid_kind = WebhookReceiverKind)] -#[diesel(table_name = webhook_rx)] +#[diesel(table_name = webhook_receiver)] pub struct WebhookReceiver { #[diesel(embed)] pub identity: WebhookReceiverIdentity, @@ -90,29 +90,29 @@ pub struct WebhookReceiver { impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; - type GenerationNumberColumn = webhook_rx::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type GenerationNumberColumn = webhook_receiver::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; type CollectionIdColumn = webhook_rx_secret::dsl::rx_id; } impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; - type GenerationNumberColumn = webhook_rx::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type GenerationNumberColumn = webhook_receiver::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; type CollectionIdColumn = webhook_rx_subscription::dsl::rx_id; } impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; - type GenerationNumberColumn = webhook_rx::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type GenerationNumberColumn = webhook_receiver::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; type CollectionIdColumn = webhook_rx_event_glob::dsl::rx_id; } impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; - type GenerationNumberColumn = webhook_rx::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_rx::dsl::time_deleted; + type GenerationNumberColumn = webhook_receiver::dsl::rcgen; + type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; type CollectionIdColumn = webhook_delivery::dsl::rx_id; } diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 75292dd6b23..07c3a714c45 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -13,6 +13,7 @@ use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Generation; +use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; use crate::db::model::WebhookReceiverConfig; use crate::db::model::WebhookReceiverIdentity; @@ -21,7 +22,7 @@ use crate::db::model::WebhookRxSecret; use crate::db::model::WebhookRxSubscription; use crate::db::model::WebhookSubscriptionKind; use crate::db::pool::DbConnection; -use crate::db::schema::webhook_rx::dsl as rx_dsl; +use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_secret::dsl as secret_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; @@ -82,7 +83,7 @@ impl DataStore { let secret_keys = secrets.clone(); let err = err.clone(); async move { - let rx = diesel::insert_into(rx_dsl::webhook_rx) + let rx = diesel::insert_into(rx_dsl::webhook_receiver) .values(receiver) .returning(WebhookReceiver::as_returning()) .get_result_async(&conn) @@ -134,7 +135,20 @@ impl DataStore { Ok(WebhookReceiverConfig { rx, secrets, events: subscriptions }) } - // pub async fn webhook_rx_fetch_all(&self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver) -> Fet + pub async fn webhook_rx_config_fetch( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + ) -> Result<(Vec, Vec), Error> + { + opctx.authorize(authz::Action::ListChildren, authz_rx).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let subscriptions = + self.webhook_rx_subscription_list_on_conn(authz_rx, &conn).await?; + let secrets = + self.webhook_rx_secret_list_on_conn(authz_rx, &conn).await?; + Ok((subscriptions, secrets)) + } // // Subscriptions @@ -175,6 +189,46 @@ impl DataStore { Ok(()) } + async fn webhook_rx_subscription_list_on_conn( + &self, + authz_rx: &authz::WebhookReceiver, + conn: &async_bb8_diesel::Connection, + ) -> ListResultVec { + let rx_id = authz_rx.id().into_untyped_uuid(); + + // First, get all the exact subscriptions that aren't from globs. + let exact = subscription_dsl::webhook_rx_subscription + .filter(subscription_dsl::rx_id.eq(rx_id)) + .filter(subscription_dsl::glob.is_null()) + .select(subscription_dsl::event_class) + .load_async::(conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ) + })?; + // Then, get the globs + let globs = glob_dsl::webhook_rx_event_glob + .filter(glob_dsl::rx_id.eq(rx_id)) + .select(WebhookGlob::as_select()) + .load_async::(conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ) + })?; + let subscriptions = exact + .into_iter() + .map(WebhookSubscriptionKind::Exact) + .chain(globs.into_iter().map(WebhookSubscriptionKind::Glob)) + .collect::>(); + Ok(subscriptions) + } + async fn add_subscription_on_conn( &self, opctx: &OpContext, @@ -324,7 +378,8 @@ impl DataStore { .filter(subscription_dsl::event_class.eq(event_class)) .order_by(subscription_dsl::rx_id.asc()) .inner_join( - rx_dsl::webhook_rx.on(subscription_dsl::rx_id.eq(rx_dsl::id)), + rx_dsl::webhook_receiver + .on(subscription_dsl::rx_id.eq(rx_dsl::id)), ) .filter(rx_dsl::time_deleted.is_null()) .select(( @@ -344,11 +399,19 @@ impl DataStore { ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, authz_rx).await?; let conn = self.pool_connection_authorized(&opctx).await?; + self.webhook_rx_secret_list_on_conn(authz_rx, &conn).await + } + + async fn webhook_rx_secret_list_on_conn( + &self, + authz_rx: &authz::WebhookReceiver, + conn: &async_bb8_diesel::Connection, + ) -> ListResultVec { secret_dsl::webhook_rx_secret .filter(secret_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid())) .filter(secret_dsl::time_deleted.is_null()) .select(WebhookRxSecret::as_select()) - .load_async(&*conn) + .load_async(conn) .await .map_err(|e| { public_error_from_diesel( @@ -380,7 +443,7 @@ impl DataStore { ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; let conn = self.pool_connection_authorized(opctx).await?; - rx_dsl::webhook_rx + rx_dsl::webhook_receiver .filter(rx_dsl::time_deleted.is_null()) .select(WebhookReceiver::as_select()) .load_async::(&*conn) @@ -522,8 +585,8 @@ mod test { datastore: &DataStore, // logctx: &LogContext, event_class: &str, - matches: &[&WebhookReceiver], - not_matches: &[&WebhookReceiver], + matches: &[&WebhookReceiverConfig], + not_matches: &[&WebhookReceiverConfig], ) { let subscribed = datastore .webhook_rx_list_subscribed_to_event_on_conn( @@ -546,17 +609,19 @@ mod test { }) .collect::>(); - for rx in matches { + for WebhookReceiverConfig { rx, events, .. } in matches { assert!( subscribed.contains(&rx.identity), - "expected {rx:?} to be subscribed to {event_class:?}" + "expected {rx:?} to be subscribed to {event_class:?}\n\ + subscriptions: {events:?}" ); } - for rx in not_matches { + for WebhookReceiverConfig { rx, events, .. } in not_matches { assert!( !subscribed.contains(&rx.identity), - "expected {rx:?} to not be subscribed to {event_class:?}" + "expected {rx:?} to not be subscribed to {event_class:?}\n\ + subscriptions: {events:?}" ); } } diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index c629dbfc425..518150458d6 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -26,6 +26,7 @@ use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::TufArtifactKind; use omicron_uuid_kinds::TufRepoKind; use omicron_uuid_kinds::TypedUuid; +use omicron_uuid_kinds::WebhookReceiverUuid; use uuid::Uuid; /// Look up an API resource in the database @@ -543,6 +544,16 @@ impl<'a> LookupPath<'a> { { SamlIdentityProvider::PrimaryKey(Root { lookup_root: self }, id) } + + pub fn webhook_receiver_id<'b>( + self, + id: WebhookReceiverUuid, + ) -> WebhookReceiver<'b> + where + 'a: 'b, + { + WebhookReceiver::PrimaryKey(Root { lookup_root: self }, id) + } } /// Represents the head of the selection path for a resource @@ -947,6 +958,17 @@ lookup_resource! { ] } +lookup_resource! { + name = "WebhookReceiver", + ancestors = [], + children = [], + lookup_by_name = false, + soft_deletes = true, + primary_key_columns = [ + { column_name = "id", uuid_kind = WebhookReceiverKind } + ] +} + // Helpers for unifying the interfaces around images pub enum ImageLookup<'a> { diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index db87d7db7d7..271b5ccebb4 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -91,6 +91,7 @@ mod volume; mod vpc; mod vpc_router; mod vpc_subnet; +mod webhook; // Sagas are not part of the "Nexus" implementation, but they are // application logic. diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs new file mode 100644 index 00000000000..b9922ed45d8 --- /dev/null +++ b/nexus/src/app/webhook.rs @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Webhooks + +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::model::WebhookReceiverConfig; +use omicron_common::api::external::LookupResult; +use omicron_uuid_kinds::WebhookReceiverUuid; + +impl super::Nexus { + pub async fn webhook_receiver_config_fetch<'a>( + &'a self, + opctx: &'a OpContext, + id: WebhookReceiverUuid, + ) -> LookupResult { + let (authz_rx, rx) = LookupPath::new(opctx, &self.datastore()) + .webhook_receiver_id(id) + .fetch() + .await?; + let (events, secrets) = + self.datastore().webhook_rx_config_fetch(opctx, &authz_rx).await?; + Ok(WebhookReceiverConfig { rx, secrets, events }) + } +} diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e538615cfe3..c5ff8ca6d44 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6977,19 +6977,25 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_view( rqctx: RequestContext, - _path_params: Path, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); + let params::WebhookPath { webhook_id } = path_params.into_inner(); let handler = async { let nexus = &apictx.context.nexus; let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook = nexus + .webhook_receiver_config_fetch( + &opctx, + omicron_uuid_kinds::WebhookReceiverUuid::from_untyped_uuid( + webhook_id, + ), + ) + .await?; + Ok(HttpResponseOk(views::Webhook::try_from(webhook)?)) }; apictx .context diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f4bc563a6a0..5d9441569ef 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4808,7 +4808,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS one_record_per_volume_resource_usage on omicro * Webhook receivers, receiver secrets, and receiver subscriptions. */ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( /* Identity metadata (resource) */ id UUID PRIMARY KEY, name STRING(63) NOT NULL, From 80f21f2df944f25bd5a5b96f334095d0dac0da55 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 23 Jan 2025 09:48:01 -0800 Subject: [PATCH 041/168] webhook create --- nexus/db-model/src/webhook_rx.rs | 1 + nexus/src/app/webhook.rs | 20 +++++++++++++++++--- nexus/src/external_api/http_entrypoints.rs | 11 +++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 5caa5276993..32dfd0f529e 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -12,6 +12,7 @@ use crate::Generation; use crate::WebhookDelivery; use chrono::{DateTime, Utc}; use db_macros::Resource; +use nexus_types::external_api::params; use nexus_types::external_api::views; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index b9922ed45d8..6437975859e 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -7,13 +7,17 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::WebhookReceiverConfig; +use nexus_types::external_api::params; +use omicron_common::api::external::CreateResult; use omicron_common::api::external::LookupResult; use omicron_uuid_kinds::WebhookReceiverUuid; +pub const EVENT_CLASSES: &[&str] = &["test"]; + impl super::Nexus { - pub async fn webhook_receiver_config_fetch<'a>( - &'a self, - opctx: &'a OpContext, + pub async fn webhook_receiver_config_fetch( + &self, + opctx: &OpContext, id: WebhookReceiverUuid, ) -> LookupResult { let (authz_rx, rx) = LookupPath::new(opctx, &self.datastore()) @@ -24,4 +28,14 @@ impl super::Nexus { self.datastore().webhook_rx_config_fetch(opctx, &authz_rx).await?; Ok(WebhookReceiverConfig { rx, secrets, events }) } + + pub async fn webhook_receiver_create( + &self, + opctx: &OpContext, + params: params::WebhookCreate, + ) -> CreateResult { + // TODO(eliza): validate endpoint URI; reject underlay network IPs for + // SSRF prevention... + self.datastore().webhook_rx_create(&opctx, params, event_classes).await + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index c5ff8ca6d44..c2c26399237 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7006,19 +7006,18 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_create( rqctx: RequestContext, - _params: TypedBody, + params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; + let params = params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let receiver = + nexus.webhook_receiver_create(&opctx, params).await?; + Ok(HttpResponseCreated(views::Webhook::try_from(webhook)?)) }; apictx .context From 7643238f914d80d98ee13a5335a971d056ab7e28 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 23 Jan 2025 12:29:31 -0800 Subject: [PATCH 042/168] okay wow let's have a rudimentary test --- .vscode/settings.json | 5 + Cargo.lock | 1 + nexus/Cargo.toml | 1 + nexus/db-model/src/webhook_rx.rs | 1 - .../src/db/datastore/webhook_event.rs | 29 +++++- .../db-queries/src/db/datastore/webhook_rx.rs | 6 +- .../background/tasks/webhook_deliverator.rs | 52 +++++++---- nexus/src/app/webhook.rs | 39 +++++++- nexus/src/external_api/http_entrypoints.rs | 2 +- nexus/tests/integration_tests/mod.rs | 1 + nexus/tests/integration_tests/webhooks.rs | 91 +++++++++++++++++++ schema/crdb/dbinit.sql | 5 + 12 files changed, 210 insertions(+), 23 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 nexus/tests/integration_tests/webhooks.rs diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000000..fa39d2c768d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "rust-analyzer.check.features": [], + "rust-analyzer.check.command": "check", + "rust-analyzer.completion.autoimport.enable": false +} diff --git a/Cargo.lock b/Cargo.lock index 8b88dae91fc..6b41eec8aa1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7027,6 +7027,7 @@ dependencies = [ "hickory-resolver", "http", "http-body-util", + "httpmock", "httptest", "hubtools", "hyper", diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 39d94a66fb1..c3da4d1aae2 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -163,6 +163,7 @@ hickory-resolver.workspace = true tufaceous.workspace = true tufaceous-lib.workspace = true httptest.workspace = true +httpmock.workspace = true strum.workspace = true [[bench]] diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 32dfd0f529e..5caa5276993 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -12,7 +12,6 @@ use crate::Generation; use crate::WebhookDelivery; use chrono::{DateTime, Utc}; use db_macros::Resource; -use nexus_types::external_api::params; use nexus_types::external_api::views; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 758065adcd0..fe1f68cefa1 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -21,10 +21,34 @@ use diesel::prelude::*; use diesel::result::OptionalExtension; use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookDispatched; +use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; +use omicron_uuid_kinds::{GenericUuid, WebhookEventUuid, WebhookReceiverUuid}; impl DataStore { + pub async fn webhook_event_create( + &self, + opctx: &OpContext, + id: WebhookEventUuid, + event_class: String, + event: serde_json::Value, + ) -> CreateResult { + let conn = self.pool_connection_authorized(&opctx).await?; + let now = + diesel::dsl::now.into_sql::(); + diesel::insert_into(event_dsl::webhook_event) + .values(( + event_dsl::event_class.eq(event_class), + event_dsl::id.eq(id.into_untyped_uuid()), + event_dsl::time_created.eq(now), + event_dsl::event.eq(event), + )) + .returning(WebhookEvent::as_returning()) + .get_result_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn webhook_event_dispatch_next( &self, opctx: &OpContext, @@ -48,7 +72,8 @@ impl DataStore { .order_by(event_dsl::time_created.asc()) .limit(1) .for_update() - .skip_locked() + // TODO(eliza): AGH SKIP LOCKED IS NOT IMPLEMENTED IN CRDB... + // .skip_locked() .select(WebhookEvent::as_select()) .get_result_async(&conn) .await diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 07c3a714c45..4775480f6ea 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -21,6 +21,7 @@ use crate::db::model::WebhookRxEventGlob; use crate::db::model::WebhookRxSecret; use crate::db::model::WebhookRxSubscription; use crate::db::model::WebhookSubscriptionKind; +use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; @@ -32,10 +33,12 @@ use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; +use uuid::Uuid; impl DataStore { pub async fn webhook_rx_create( @@ -440,10 +443,11 @@ impl DataStore { pub async fn webhook_rx_list( &self, opctx: &OpContext, + pagparams: &DataPageParams<'_, Uuid>, ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; let conn = self.pool_connection_authorized(opctx).await?; - rx_dsl::webhook_receiver + paginated(rx_dsl::webhook_receiver, rx_dsl::id, pagparams) .filter(rx_dsl::time_deleted.is_null()) .select(WebhookReceiver::as_select()) .load_async::(&*conn) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index bcda64a9d0c..987b22b407f 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -11,6 +11,7 @@ use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::model::{ SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookReceiver, }; +use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ @@ -18,8 +19,10 @@ use nexus_types::internal_api::background::{ }; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ - OmicronZoneUuid, WebhookDeliveryUuid, WebhookEventUuid, WebhookReceiverUuid, + GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid, WebhookEventUuid, + WebhookReceiverUuid, }; +use std::num::NonZeroU32; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::task::JoinSet; @@ -122,32 +125,47 @@ impl WebhookDeliverator { Self { datastore, nexus_id, lease_timeout, client } } + const MAX_CONCURRENT_RXS: NonZeroU32 = { + match NonZeroU32::new(8) { + Some(nz) => nz, + None => unreachable!(), + } + }; + async fn actually_activate( &mut self, opctx: &OpContext, status: &mut WebhookDeliveratorStatus, ) -> Result<(), Error> { - let rxs = self.datastore.webhook_rx_list(&opctx).await?; let mut tasks = JoinSet::new(); - for rx in rxs { - let opctx = opctx.child(maplit::btreemap! { - "receiver_id".to_string() => rx.id().to_string(), - "receiver_name".to_string() => rx.name().to_string(), - }); - let deliverator = self.clone(); - tasks.spawn(async move { - let rx_id = rx.id(); - let status = deliverator.rx_deliver(&opctx, rx).await; - (rx_id, status) - }); - } + let mut paginator = Paginator::new(Self::MAX_CONCURRENT_RXS); + while let Some(p) = paginator.next() { + let rxs = self + .datastore + .webhook_rx_list(&opctx, &p.current_pagparams()) + .await?; + paginator = p.found_batch(&rxs, &|rx| rx.id().into_untyped_uuid()); + + for rx in rxs { + let opctx = opctx.child(maplit::btreemap! { + "receiver_id".to_string() => rx.id().to_string(), + "receiver_name".to_string() => rx.name().to_string(), + }); + let deliverator = self.clone(); + tasks.spawn(async move { + let rx_id = rx.id(); + let status = deliverator.rx_deliver(&opctx, rx).await; + (rx_id, status) + }); + } - while let Some(result) = tasks.join_next().await { - let (rx_id, rx_status) = result.expect( + while let Some(result) = tasks.join_next().await { + let (rx_id, rx_status) = result.expect( "delivery tasks should not be canceled, and nexus is compiled \ with `panic=\"abort\"`, so they will not have panicked", ); - status.by_rx.insert(rx_id, rx_status); + status.by_rx.insert(rx_id, rx_status); + } } Ok(()) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 6437975859e..7782f2d1305 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -6,10 +6,13 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::Error; use omicron_common::api::external::LookupResult; +use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; pub const EVENT_CLASSES: &[&str] = &["test"]; @@ -36,6 +39,40 @@ impl super::Nexus { ) -> CreateResult { // TODO(eliza): validate endpoint URI; reject underlay network IPs for // SSRF prevention... - self.datastore().webhook_rx_create(&opctx, params, event_classes).await + self.datastore().webhook_rx_create(&opctx, params, EVENT_CLASSES).await + } + + pub async fn webhook_event_publish( + &self, + opctx: &OpContext, + event_class: String, + event: serde_json::Value, + ) -> Result { + if !EVENT_CLASSES.contains(&event_class.as_str()) { + return Err(Error::InternalError { + internal_message: format!( + "unknown webhook event class {event_class:?}" + ), + }); + } + + let id = WebhookEventUuid::new_v4(); + let event = self + .datastore() + .webhook_event_create(opctx, id, event_class, event) + .await?; + slog::debug!( + &opctx.log, + "published webhook event"; + "event_id" => ?id, + "event_class" => ?event.event_class, + "time_created" => ?event.time_created, + ); + + // Once the event has been isnerted, activate the dispatcher task to + // ensure its propagated to receivers. + self.background_tasks.task_webhook_dispatcher.activate(); + + Ok(event) } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index c2c26399237..1a380da2cde 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7017,7 +7017,7 @@ impl NexusExternalApi for NexusExternalApiImpl { crate::context::op_context_for_external_api(&rqctx).await?; let receiver = nexus.webhook_receiver_create(&opctx, params).await?; - Ok(HttpResponseCreated(views::Webhook::try_from(webhook)?)) + Ok(HttpResponseCreated(views::Webhook::try_from(receiver)?)) }; apictx .context diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index dc404736cd1..e288f579fca 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -55,6 +55,7 @@ mod vpc_firewall; mod vpc_routers; mod vpc_subnets; mod vpcs; +mod webhooks; // This module is used only for shared data, not test cases. mod endpoints; diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs new file mode 100644 index 00000000000..889aabcd643 --- /dev/null +++ b/nexus/tests/integration_tests/webhooks.rs @@ -0,0 +1,91 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Webhooks + +use httpmock::prelude::*; +use nexus_db_queries::context::OpContext; +use nexus_test_utils::background::activate_background_task; +use nexus_test_utils::resource_helpers; +use nexus_test_utils_macros::nexus_test; +use nexus_types::external_api::{params, views}; +use omicron_common::api::external::IdentityMetadataCreateParams; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +async fn webhook_create( + ctx: &ControlPlaneTestContext, + params: ¶ms::WebhookCreate, +) -> views::Webhook { + resource_helpers::object_create::( + &ctx.external_client, + "/experimental/v1/webhooks", + params, + ) + .await +} + +#[nexus_test] +async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let server = httpmock::MockServer::start_async().await; + + let mock = server + .mock_async(|when, then| { + let body = serde_json::json!({ + "event_class": "test", + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST).path("/webhooks").json_body_includes(body); + then.status(200); + }) + .await; + let endpoint = + server.url("/webhooks").parse().expect("this should be a valid URL"); + + // Create a webhook receiver. + let webhook = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "my-great-webhook".parse().unwrap(), + description: String::from("my great webhook"), + }, + endpoint, + secrets: vec!["my cool secret".to_string()], + events: vec!["test".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(webhook); + + // Publish an event + let event = nexus + .webhook_event_publish( + &opctx, + "test".to_string(), + serde_json::json!({"hello_world": true}), + ) + .await + .expect("event should be published successfully"); + dbg!(event); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + mock.assert_async().await; +} diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 5d9441569ef..31174a88f99 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4824,6 +4824,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( probes_enabled BOOL NOT NULL ); +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rxs_by_id +ON omicron.public.webhook_receiver (id) +WHERE + time_deleted IS NULL; + CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) From ac71f4f83bf49f6c2704d7ba8f6b473f99192d56 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 23 Jan 2025 13:09:06 -0800 Subject: [PATCH 043/168] caller provides uuid --- nexus/src/app/webhook.rs | 4 ++-- nexus/tests/integration_tests/webhooks.rs | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 7782f2d1305..057c599476c 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -45,6 +45,7 @@ impl super::Nexus { pub async fn webhook_event_publish( &self, opctx: &OpContext, + id: WebhookEventUuid, event_class: String, event: serde_json::Value, ) -> Result { @@ -56,14 +57,13 @@ impl super::Nexus { }); } - let id = WebhookEventUuid::new_v4(); let event = self .datastore() .webhook_event_create(opctx, id, event_class, event) .await?; slog::debug!( &opctx.log, - "published webhook event"; + "enqueued webhook event"; "event_id" => ?id, "event_class" => ?event.event_class, "time_created" => ?event.time_created, diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 889aabcd643..e23d00c8dc9 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -11,6 +11,7 @@ use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::{params, views}; use omicron_common::api::external::IdentityMetadataCreateParams; +use omicron_uuid_kinds::WebhookEventUuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -38,10 +39,12 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let server = httpmock::MockServer::start_async().await; + let id = WebhookEventUuid::new_v4(); let mock = server .mock_async(|when, then| { let body = serde_json::json!({ "event_class": "test", + "event_id": id, "data": { "hello_world": true, } @@ -75,6 +78,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let event = nexus .webhook_event_publish( &opctx, + id, "test".to_string(), serde_json::json!({"hello_world": true}), ) From 92e4fc36a61d52bff87d148a7bd51e8581a597d5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 23 Jan 2025 13:26:19 -0800 Subject: [PATCH 044/168] assert there's some headers --- nexus/tests/integration_tests/webhooks.rs | 40 ++++++++++++++--------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index e23d00c8dc9..e459be50cf6 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -40,20 +40,6 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let server = httpmock::MockServer::start_async().await; let id = WebhookEventUuid::new_v4(); - let mock = server - .mock_async(|when, then| { - let body = serde_json::json!({ - "event_class": "test", - "event_id": id, - "data": { - "hello_world": true, - } - }) - .to_string(); - when.method(POST).path("/webhooks").json_body_includes(body); - then.status(200); - }) - .await; let endpoint = server.url("/webhooks").parse().expect("this should be a valid URL"); @@ -72,7 +58,31 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { }, ) .await; - dbg!(webhook); + dbg!(&webhook); + + let mock = server + .mock_async(|when, then| { + let body = serde_json::json!({ + "event_class": "test", + "event_id": id, + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST) + .path("/webhooks") + .json_body_includes(body) + .header("x-oxide-event-class", "test") + .header("x-oxide-event-id", id.to_string()) + .header("x-oxide-webhook-id", webhook.id.to_string()) + .header("content-type", "application/json") + // This should be present, but we don't know what its' value is + // going to be, so just assert that it's there. + .header_exists("x-oxide-delivery-id"); + then.status(200); + }) + .await; // Publish an event let event = nexus From 8e24395583e7156867557442f111c4323e74060b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 23 Jan 2025 13:34:43 -0800 Subject: [PATCH 045/168] fix missing content-type, separate request build errors --- .../background/tasks/webhook_deliverator.rs | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 987b22b407f..e9d8928ce8a 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -282,11 +282,26 @@ impl WebhookDeliverator { sent_at: &sent_at, }, }; - let body = match serde_json::to_vec(&payload) { - Ok(body) => body, + // TODO(eliza): signatures! + let request = self + .client + .post(&rx.endpoint) + .header(HDR_RX_ID, hdr_rx_id.clone()) + .header(HDR_DELIVERY_ID, delivery_id.to_string()) + .header(HDR_EVENT_ID, delivery.event_id.to_string()) + .header(HDR_EVENT_CLASS, event_class) + .json(&payload) + // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to + // each webhook delivery request. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .timeout(Duration::from_secs(30)) + .build(); + let request = match request { + // We couldn't construct a request for some reason! This one's + // our fault, so don't penalize the receiver for it. Err(e) => { - const MSG: &str = - "failed to serialize webhook event payload"; + const MSG: &str = "failed to construct webhook request"; slog::error!( &opctx.log, "{MSG}"; @@ -301,23 +316,10 @@ impl WebhookDeliverator { .insert(delivery_id, format!("{MSG}: {e}")); continue; } + Ok(r) => r, }; - // TODO(eliza): signatures! - let request = self - .client - .post(&rx.endpoint) - .header(HDR_RX_ID, hdr_rx_id.clone()) - .header(HDR_DELIVERY_ID, delivery_id.to_string()) - .header(HDR_EVENT_ID, delivery.event_id.to_string()) - .header(HDR_EVENT_CLASS, event_class) - .body(body) - // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to - // each webhook delivery request. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .timeout(Duration::from_secs(30)); let t0 = Instant::now(); - let result = request.send().await; + let result = self.client.execute(request).await; let duration = t0.elapsed(); let (delivery_result, status) = match result { // Builder errors are our fault, that's weird! From 564c8b93b32ae0eef029c5593d4d0fb27010fae1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 27 Jan 2025 10:48:08 -0800 Subject: [PATCH 046/168] capture schema versions when creating globs --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/schema_versions.rs | 52 +++++++++++++++++++++++++++ nexus/db-model/src/webhook_rx.rs | 9 ++++- schema/crdb/dbinit.sql | 8 +++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index c028558546c..e08084fd313 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2168,6 +2168,7 @@ table! { rx_id -> Uuid, glob -> Text, regex -> Text, + schema_version -> Text, time_created -> Timestamptz, } } diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index a360eba7eb7..100c83cc301 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -8,6 +8,10 @@ use anyhow::{bail, ensure, Context}; use camino::Utf8Path; +use diesel::expression::AsExpression; +use diesel::pg::{Pg, PgValue}; +use diesel::sql_types::Text; +use diesel::{deserialize, serialize}; use omicron_common::api::external::SemverVersion; use once_cell::sync::Lazy; use std::collections::BTreeMap; @@ -477,6 +481,54 @@ impl SchemaUpgradeStep { } } +/// A newtype around [`SemverVersion`] that implements the [`ToSql`] and +/// [`FromSql`] traits, allowing it to be used as a field in a type representing +/// a DB model. +#[derive( + Clone, + Debug, + Eq, + PartialEq, + PartialOrd, + serde::Serialize, + serde::Deserialize, + deserialize::FromSqlRow, + AsExpression, +)] +#[diesel(sql_type = Text)] +pub struct DbSemverVersion(pub SemverVersion); + +impl deserialize::FromSql for DbSemverVersion { + fn from_sql(value: PgValue<'_>) -> deserialize::Result { + let version = + std::str::from_utf8(value.as_bytes())?.parse::()?; + Ok(Self(version)) + } +} + +impl serialize::ToSql for DbSemverVersion { + fn to_sql<'b>( + &'b self, + out: &mut serialize::Output<'b, '_, Pg>, + ) -> serialize::Result { + use std::io::Write; + out.write_fmt(format_args!("{}", self.0))?; + Ok(serialize::IsNull::No) + } +} + +impl From for DbSemverVersion { + fn from(version: SemverVersion) -> Self { + Self(version) + } +} + +impl From for SemverVersion { + fn from(DbSemverVersion(version): DbSemverVersion) -> Self { + version + } +} + #[cfg(test)] mod test { use super::*; diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 5caa5276993..18d48269884 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -7,6 +7,7 @@ use crate::schema::{ webhook_delivery, webhook_receiver, webhook_rx_event_glob, webhook_rx_secret, webhook_rx_subscription, }; +use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; use crate::Generation; use crate::WebhookDelivery; @@ -163,11 +164,17 @@ pub struct WebhookRxEventGlob { #[diesel(embed)] pub glob: WebhookGlob, pub time_created: DateTime, + pub schema_version: schema_versions::DbSemverVersion, } impl WebhookRxEventGlob { pub fn new(rx_id: WebhookReceiverUuid, glob: WebhookGlob) -> Self { - Self { rx_id: DbTypedUuid(rx_id), glob, time_created: Utc::now() } + Self { + rx_id: DbTypedUuid(rx_id), + glob, + time_created: Utc::now(), + schema_version: schema_versions::SCHEMA_VERSION.into(), + } } } #[derive(Clone, Debug)] diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 31174a88f99..40dba41e48d 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4863,6 +4863,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( -- Regex used when evaluating this filter against concrete event classes. regex STRING(512) NOT NULL, time_created TIMESTAMPTZ NOT NULL, + -- The database schema version at which this glob was last expanded. + -- + -- This is used to detect when a glob must be re-processed to generate exact + -- subscriptions on schema changes. + schema_version STRING(64) NOT NULL, PRIMARY KEY (rx_id, glob) ); @@ -4873,6 +4878,9 @@ ON omicron.public.webhook_rx_event_glob ( rx_id ); +CREATE INDEX IF NOT EXISTS lookup_webhook_globs_by_schema_version +ON omicron.public.webhook_rx_event_glob (schema_version); + CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) From ec661c7b124328e6d492fe88ffe4aed8a4d14b3b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 29 Jan 2025 12:16:49 -0800 Subject: [PATCH 047/168] wip --- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 4 +- nexus/db-model/src/webhook_event.rs | 3 +- nexus/db-model/src/webhook_event_class.rs | 104 ++++++++++++++++++ nexus/db-model/src/webhook_rx.rs | 19 +++- .../src/db/datastore/webhook_delivery.rs | 3 +- .../src/db/datastore/webhook_event.rs | 13 ++- schema/crdb/dbinit.sql | 19 +++- 8 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 nexus/db-model/src/webhook_event_class.rs diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bfa01596c95..c6d4cedd56a 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -67,6 +67,7 @@ mod v2p_mapping; mod vmm_state; mod webhook_delivery; mod webhook_event; +mod webhook_event_class; mod webhook_rx; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables @@ -236,6 +237,7 @@ pub use vpc_router::*; pub use vpc_subnet::*; pub use webhook_delivery::*; pub use webhook_event::*; +pub use webhook_event_class::*; pub use webhook_rx::*; pub use zpool::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index e08084fd313..cb9eb4ca38e 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2157,7 +2157,7 @@ table! { table! { webhook_rx_subscription (rx_id, event_class) { rx_id -> Uuid, - event_class -> Text, + event_class -> crate::WebhookEventClassEnum, glob -> Nullable, time_created -> Timestamptz, } @@ -2188,7 +2188,7 @@ table! { id -> Uuid, time_created -> Timestamptz, time_dispatched -> Nullable, - event_class -> Text, + event_class -> crate::WebhookEventClassEnum, event -> Jsonb, } } diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs index d0f469400ef..03349cf4a33 100644 --- a/nexus/db-model/src/webhook_event.rs +++ b/nexus/db-model/src/webhook_event.rs @@ -4,6 +4,7 @@ use crate::schema::webhook_event; use crate::typed_uuid::DbTypedUuid; +use crate::WebhookEventClass; use chrono::{DateTime, Utc}; use omicron_uuid_kinds::WebhookEventKind; use serde::{Deserialize, Serialize}; @@ -34,7 +35,7 @@ pub struct WebhookEvent { pub time_dispatched: Option>, /// The class of this event. - pub event_class: String, + pub event_class: WebhookEventClass, /// The event's data payload. pub event: serde_json::Value, diff --git a/nexus/db-model/src/webhook_event_class.rs b/nexus/db-model/src/webhook_event_class.rs new file mode 100644 index 00000000000..ed12df8203a --- /dev/null +++ b/nexus/db-model/src/webhook_event_class.rs @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + +impl_enum_type!( + #[derive(SqlType, Debug, Clone)] + #[diesel(postgres_type(name = "webhook_event_class", schema = "public"))] + pub struct WebhookEventClassEnum; + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, + strum::VariantArray, + )] + #[diesel(sql_type = WebhookEventClassEnum)] + pub enum WebhookEventClass; + + TestFoo => b"test.foo" + TestFooBar => b"test.foo.bar" + TestBazBar => b"test.baz.bar" +); + +impl WebhookEventClass { + pub fn as_str(&self) -> &'static str { + // TODO(eliza): it would be really nice if these strings were all + // declared a single time, rather than twice (in both `impl_enum_type!` + // and here)... + match self { + Self::TestFoo => "test.foo", + Self::TestFooBar => "test.foo.bar", + Self::TestBazBar => "test.baz.bar", + } + } + + /// All webhook event classes. + pub const ALL_CLASSES: &'static [Self] = + ::VARIANTS; +} + +impl fmt::Display for WebhookEventClass { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl diesel::query_builder::QueryId for WebhookEventClassEnum { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl std::str::FromStr for WebhookEventClass { + type Err = EventClassParseError; + fn from_str(s: &str) -> Result { + for &class in Self::ALL_CLASSES { + if s == class.as_str() { + return Ok(class); + } + } + + Err(EventClassParseError(())) + } +} + +#[derive(Debug, Eq, PartialEq)] +pub struct EventClassParseError(()); + +impl fmt::Display for EventClassParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "expected one of [")?; + let mut variants = WebhookEventClass::ALL_CLASSES.iter(); + if let Some(v) = variants.next() { + write!(f, "{v}")?; + for v in variants { + write!(f, ", {v}")?; + } + } + f.write_str("]") + } +} + +impl std::error::Error for EventClassParseError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str_roundtrips() { + for &variant in WebhookEventClass::ALL_CLASSES { + assert_eq!(Ok(dbg!(variant)), dbg!(variant.to_string().parse())); + } + } +} diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 18d48269884..8fad69dbc10 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -11,6 +11,7 @@ use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; use crate::Generation; use crate::WebhookDelivery; +use crate::WebhookEventClass; use chrono::{DateTime, Utc}; use db_macros::Resource; use nexus_types::external_api::views; @@ -150,7 +151,7 @@ impl WebhookRxSecret { #[diesel(table_name = webhook_rx_subscription)] pub struct WebhookRxSubscription { pub rx_id: DbTypedUuid, - pub event_class: String, + pub event_class: WebhookEventClass, pub glob: Option, pub time_created: DateTime, } @@ -180,7 +181,7 @@ impl WebhookRxEventGlob { #[derive(Clone, Debug)] pub enum WebhookSubscriptionKind { Glob(WebhookGlob), - Exact(String), + Exact(WebhookEventClass), } impl WebhookSubscriptionKind { @@ -195,13 +196,13 @@ impl WebhookSubscriptionKind { let regex = WebhookGlob::regex_from_glob(&value)?; Ok(Self::Glob(WebhookGlob { regex, glob: value })) } else { - Ok(Self::Exact(value)) + Ok(Self::Exact(value.parse()?)) } } fn into_event_class_string(self) -> String { match self { - Self::Exact(class) => class, + Self::Exact(class) => class.to_string(), Self::Glob(WebhookGlob { glob, .. }) => glob, } } @@ -272,7 +273,10 @@ impl WebhookGlob { } impl WebhookRxSubscription { - pub fn exact(rx_id: WebhookReceiverUuid, event_class: String) -> Self { + pub fn exact( + rx_id: WebhookReceiverUuid, + event_class: WebhookEventClass, + ) -> Self { Self { rx_id: DbTypedUuid(rx_id), event_class, @@ -281,7 +285,10 @@ impl WebhookRxSubscription { } } - pub fn for_glob(glob: &WebhookRxEventGlob, event_class: String) -> Self { + pub fn for_glob( + glob: &WebhookRxEventGlob, + event_class: WebhookEventClass, + ) -> Self { Self { rx_id: glob.rx_id, glob: Some(glob.glob.glob.clone()), diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 7e12084bb3e..e18860c4ba1 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -11,6 +11,7 @@ use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; +use crate::db::model::WebhookEventClass; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; @@ -38,7 +39,7 @@ impl DataStore { opctx: &OpContext, rx_id: &WebhookReceiverUuid, lease_timeout: TimeDelta, - ) -> ListResultVec<(WebhookDelivery, String)> { + ) -> ListResultVec<(WebhookDelivery, WebhookEventClass)> { let conn = self.pool_connection_authorized(opctx).await?; let now = diesel::dsl::now.into_sql::(); diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index fe1f68cefa1..6d6ca113213 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -12,6 +12,7 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookEvent; +use crate::db::model::WebhookEventClass; use crate::db::model::WebhookReceiver; use crate::db::pool::DbConnection; use crate::db::schema::webhook_delivery::dsl as delivery_dsl; @@ -30,7 +31,7 @@ impl DataStore { &self, opctx: &OpContext, id: WebhookEventUuid, - event_class: String, + event_class: WebhookEventClass, event: serde_json::Value, ) -> CreateResult { let conn = self.pool_connection_authorized(&opctx).await?; @@ -104,7 +105,7 @@ impl DataStore { &opctx.log, "found {} receivers subscribed to webhook event", rxs.len(); "event_id" => ?event.id, - "event_class" => &event.event_class, + "event_class" => %event.event_class, "receivers" => ?rxs.len(), ); @@ -116,7 +117,7 @@ impl DataStore { &opctx.log, "found receiver subscribed to event"; "event_id" => ?event.id, - "event_class" => &event.event_class, + "event_class" => %event.event_class, "receiver" => ?rx.name(), "receiver_id" => ?rx_id, "glob" => ?sub.glob, @@ -137,7 +138,7 @@ impl DataStore { &opctx.log, "cannot dispatch event to a receiver that has been deleted"; "event_id" => ?event.id, - "event_class" => &event.event_class, + "event_class" => %event.event_class, "receiver" => ?rx.name(), "receiver_id" => ?rx_id, ); @@ -187,7 +188,7 @@ impl DataStore { log, "dispatched webhook event to receiver"; "event_id" => ?event.id, - "event_class" => &event.event_class, + "event_class" => %event.event_class, "receiver_id" => ?rx_id, ); return Ok(delivery); @@ -202,7 +203,7 @@ impl DataStore { &log, "webhook delivery UUID collision, retrying..."; "event_id" => ?event.id, - "event_class" => &event.event_class, + "event_class" => %event.event_class, "receiver_id" => ?rx_id, ); } diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 40dba41e48d..012a59a3dab 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3447,7 +3447,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_nvme_disk_firmware ( -- the firmware version string for each NVMe slot (0 indexed), a NULL means the -- slot exists but is empty slot_firmware_versions STRING(8)[] CHECK (array_length(slot_firmware_versions, 1) BETWEEN 1 AND 7), - + -- PK consisting of: -- - Which collection this was -- - The sled reporting the disk @@ -4849,6 +4849,19 @@ ON omicron.public.webhook_rx_secret ( ) WHERE time_deleted IS NULL; +-- Webhook event classes. +-- +-- When creating new event classes, be sure to add them here! +CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( + -- Test classes used to test globbing. + -- + -- These are not publicly exposed. + 'test.foo', + 'test.foo.bar', + 'test.baz.bar' + -- Add new event classes here! +); + -- The set of event class filters (either event class names or event class glob -- patterns) associated with a webhook receiver. -- @@ -4886,7 +4899,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, -- An event class to which the receiver is subscribed. - event_class STRING(512) NOT NULL, + event_class omicron.public.webhook_event_class NOT NULL, -- If this subscription is a concrete instantiation of a glob pattern, the -- value of the glob that created it (and, a foreign key into -- `webhook_rx_event_glob`). If the receiver is subscribed to this exact @@ -4930,7 +4943,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( -- Set when dispatch entries have been created for this event. time_dispatched TIMESTAMPTZ, -- The class of event that this is. - event_class STRING(512) NOT NULL, + event_class omicron.public.webhook_event_class NOT NULL, -- Actual event data. The structure of this depends on the event class. event JSONB NOT NULL ); From 2275170280e13bf0ee2087995ae84ae7f633250a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 31 Jan 2025 10:38:12 -0800 Subject: [PATCH 048/168] fromstr --- nexus/db-model/src/webhook_rx.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 8fad69dbc10..8b7d83f9a98 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -9,6 +9,7 @@ use crate::schema::{ }; use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; +use crate::EventClassParseError; use crate::Generation; use crate::WebhookDelivery; use crate::WebhookEventClass; @@ -192,12 +193,16 @@ impl WebhookSubscriptionKind { "must not be empty", )); } + if value.contains('*') { let regex = WebhookGlob::regex_from_glob(&value)?; - Ok(Self::Glob(WebhookGlob { regex, glob: value })) - } else { - Ok(Self::Exact(value.parse()?)) + return Ok(Self::Glob(WebhookGlob { regex, glob: value })); } + + let class = value.parse().map_err(|e: EventClassParseError| { + Error::invalid_value("event_class", e.to_string()) + })?; + Ok(Self::Exact(class)) } fn into_event_class_string(self) -> String { From 3127d46bffc7a7e2c1da09d8057e139c8c519c39 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 31 Jan 2025 13:35:38 -0800 Subject: [PATCH 049/168] actually finish turning event classes into enums --- nexus/db-model/src/webhook_event_class.rs | 34 +- nexus/db-model/src/webhook_rx.rs | 9 +- .../src/db/datastore/webhook_event.rs | 2 +- .../db-queries/src/db/datastore/webhook_rx.rs | 295 ++++++++---------- .../background/tasks/webhook_deliverator.rs | 30 +- nexus/src/app/webhook.rs | 17 +- nexus/tests/integration_tests/webhooks.rs | 9 +- schema/crdb/dbinit.sql | 4 +- 8 files changed, 185 insertions(+), 215 deletions(-) diff --git a/nexus/db-model/src/webhook_event_class.rs b/nexus/db-model/src/webhook_event_class.rs index ed12df8203a..09876707866 100644 --- a/nexus/db-model/src/webhook_event_class.rs +++ b/nexus/db-model/src/webhook_event_class.rs @@ -3,8 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; -use serde::Deserialize; -use serde::Serialize; +use serde::de::{self, Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; use std::fmt; impl_enum_type!( @@ -19,8 +19,6 @@ impl_enum_type!( PartialEq, AsExpression, FromSqlRow, - Serialize, - Deserialize, strum::VariantArray, )] #[diesel(sql_type = WebhookEventClassEnum)] @@ -28,7 +26,9 @@ impl_enum_type!( TestFoo => b"test.foo" TestFooBar => b"test.foo.bar" - TestBazBar => b"test.baz.bar" + TestFooBaz => b"test.foo.baz" + TestQuuxBar => b"test.quux.bar" + TestQuuxBarBaz => b"test.quux.bar.baz" ); impl WebhookEventClass { @@ -39,7 +39,9 @@ impl WebhookEventClass { match self { Self::TestFoo => "test.foo", Self::TestFooBar => "test.foo.bar", - Self::TestBazBar => "test.baz.bar", + Self::TestFooBaz => "test.foo.baz", + Self::TestQuuxBar => "test.quux.bar", + Self::TestQuuxBarBaz => "test.quux.bar.baz", } } @@ -54,6 +56,26 @@ impl fmt::Display for WebhookEventClass { } } +impl Serialize for WebhookEventClass { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for WebhookEventClass { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + <&'de str>::deserialize(deserializer)? + .parse::() + .map_err(de::Error::custom) + } +} + impl diesel::query_builder::QueryId for WebhookEventClassEnum { type QueryId = (); const HAS_STATIC_QUERY_ID: bool = false; diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 8b7d83f9a98..cbe4f53b98d 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -27,6 +27,7 @@ use uuid::Uuid; /// The full configuration of a webhook receiver, including the /// [`WebhookReceiver`] itself and its subscriptions and secrets. +#[derive(Clone, Debug)] pub struct WebhookReceiverConfig { pub rx: WebhookReceiver, pub secrets: Vec, @@ -310,10 +311,10 @@ mod test { #[test] fn test_event_class_glob_to_regex() { const CASES: &[(&str, &str)] = &[ - ("foo.bar", "^foo.bar$"), - ("foo.*.bar", "^foo\\.[^\\.]*\\.bar$"), - ("foo.*", "^foo\\.[^\\.]*$"), - ("*.foo", "^[^\\.]*\\.foo$"), + ("foo.bar", "^foo\\.bar$"), + ("foo.*.bar", "^foo\\.[^\\.]+\\.bar$"), + ("foo.*", "^foo\\.[^\\.]+$"), + ("*.foo", "^[^\\.]+\\.foo$"), ("foo.**.bar", "^foo\\..+\\.bar$"), ("foo.**", "^foo\\..+$"), ("foo_bar.baz", "^foo_bar\\.baz$"), diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 6d6ca113213..280502a98c8 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -96,7 +96,7 @@ impl DataStore { // Find receivers subscribed to this event's class. let rxs = self .webhook_rx_list_subscribed_to_event_on_conn( - &event.event_class, + event.event_class, &conn, ) .await?; diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 4775480f6ea..ad83de42e1a 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -13,6 +13,7 @@ use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Generation; +use crate::db::model::WebhookEventClass; use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; use crate::db::model::WebhookReceiverConfig; @@ -45,7 +46,6 @@ impl DataStore { &self, opctx: &OpContext, params: params::WebhookCreate, - event_classes: &[&str], ) -> CreateResult { // TODO(eliza): someday we gotta allow creating webhooks with more // restrictive permissions... @@ -96,7 +96,6 @@ impl DataStore { opctx, rx.identity.id.into(), subscription, - event_classes, &conn, ) .await @@ -162,18 +161,11 @@ impl DataStore { opctx: &OpContext, authz_rx: &authz::WebhookReceiver, subscription: WebhookSubscriptionKind, - event_classes: &[&str], ) -> Result<(), Error> { opctx.authorize(authz::Action::CreateChild, authz_rx).await?; let conn = self.pool_connection_authorized(opctx).await?; let num_created = self - .add_subscription_on_conn( - opctx, - authz_rx.id(), - subscription, - event_classes, - &conn, - ) + .add_subscription_on_conn(opctx, authz_rx.id(), subscription, &conn) .await .map_err(|e| match e { TransactionError::CustomError(e) => e, @@ -204,7 +196,7 @@ impl DataStore { .filter(subscription_dsl::rx_id.eq(rx_id)) .filter(subscription_dsl::glob.is_null()) .select(subscription_dsl::event_class) - .load_async::(conn) + .load_async::(conn) .await .map_err(|e| { public_error_from_diesel( @@ -237,7 +229,6 @@ impl DataStore { opctx: &OpContext, rx_id: WebhookReceiverUuid, subscription: WebhookSubscriptionKind, - event_classes: &[&str], conn: &async_bb8_diesel::Connection, ) -> Result> { match subscription { @@ -262,8 +253,7 @@ impl DataStore { .insert_and_get_result_async(conn) .await .map_err(async_insert_error_to_txn(rx_id))?; - self.glob_generate_exact_subs(opctx, &glob, event_classes, conn) - .await + self.glob_generate_exact_subs(opctx, &glob, conn).await } } } @@ -296,7 +286,6 @@ impl DataStore { &self, opctx: &OpContext, glob: &WebhookRxEventGlob, - event_classes: &[&str], conn: &async_bb8_diesel::Connection, ) -> Result> { let regex = match regex::Regex::new(&glob.glob.regex) { @@ -317,15 +306,15 @@ impl DataStore { } }; let mut created = 0; - for class in event_classes { - if !regex.is_match(class) { + for &class in WebhookEventClass::ALL_CLASSES { + if !regex.is_match(class.as_str()) { slog::debug!( &opctx.log, "webhook glob does not matche event class"; "webhook_id" => ?glob.rx_id, "glob" => ?glob.glob.glob, "regex" => ?regex, - "event_class" => ?class, + "event_class" => %class, ); continue; } @@ -336,10 +325,10 @@ impl DataStore { "webhook_id" => ?glob.rx_id, "glob" => ?glob.glob.glob, "regex" => ?regex, - "event_class" => ?class, + "event_class" => %class, ); self.add_exact_sub_on_conn( - WebhookRxSubscription::for_glob(&glob, class.to_string()), + WebhookRxSubscription::for_glob(&glob, class), conn, ) .await?; @@ -361,21 +350,19 @@ impl DataStore { /// the provided `event_class`. pub(crate) async fn webhook_rx_list_subscribed_to_event_on_conn( &self, - event_class: impl ToString, + event_class: WebhookEventClass, conn: &async_bb8_diesel::Connection, ) -> Result< Vec<(WebhookReceiver, WebhookRxSubscription)>, diesel::result::Error, > { - let class = event_class.to_string(); - - Self::rx_list_subscribed_query(class) + Self::rx_list_subscribed_query(event_class) .load_async::<(WebhookReceiver, WebhookRxSubscription)>(conn) .await } fn rx_list_subscribed_query( - event_class: String, + event_class: WebhookEventClass, ) -> impl RunnableQuery<(WebhookReceiver, WebhookRxSubscription)> { subscription_dsl::webhook_rx_subscription .filter(subscription_dsl::event_class.eq(event_class)) @@ -484,113 +471,91 @@ mod test { let logctx = dev::test_setup_log("test_event_class_globs"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - - let event_classes: &[&str] = &[ - "notfoo", - "foo.bar", - "foo.baz", - "foo.bar.baz", - "foo.baz.bar", - "foo.baz.quux.bar", - "baz.quux.bar", - ]; - - let foo_star = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "foo-star".parse().unwrap(), - description: String::new(), - }, - endpoint: "http://foo.star".parse().unwrap(), - secrets: Vec::new(), - events: vec!["foo.*".to_string()], - disable_probes: false, - }, - &event_classes, - ) - .await - .unwrap(); - - let foo_starstar = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "foo-starstar".parse().unwrap(), - description: String::new(), - }, - endpoint: "http://foo.starstar".parse().unwrap(), - secrets: Vec::new(), - events: vec!["foo.**".to_string()], - disable_probes: false, - }, - &event_classes, - ) - .await - .unwrap(); - - let foo_bar = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "foo-bar".parse().unwrap(), - description: String::new(), - }, - endpoint: "http://foo.bar".parse().unwrap(), - secrets: Vec::new(), - events: vec!["foo.bar".to_string()], - disable_probes: false, - }, - &event_classes, - ) - .await - .unwrap(); - - let foo_starstar_bar = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "foo-starstar-bar".parse().unwrap(), - description: String::new(), - }, - endpoint: "http://foo.starstar.bar".parse().unwrap(), - secrets: Vec::new(), - events: vec!["foo.**.bar".parse().unwrap()], - disable_probes: false, - }, - &event_classes, - ) - .await - .unwrap(); - - let starstar_bar = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "starstar-bar".parse().unwrap(), - description: String::new(), + let mut all_rxs = Vec::new(); + async fn create_rx( + datastore: &DataStore, + opctx: &OpContext, + all_rxs: &mut Vec, + name: &str, + subscription: &str, + ) -> WebhookReceiverConfig { + let rx = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: String::new(), + }, + endpoint: format!("http://{name}").parse().unwrap(), + secrets: vec![name.to_string()], + events: vec![subscription.to_string()], + disable_probes: false, }, - endpoint: "http://starstar.bar".parse().unwrap(), - secrets: Vec::new(), - events: vec!["**.bar".parse().unwrap()], - disable_probes: false, - }, - &event_classes, - ) - .await - .unwrap(); + ) + .await + .unwrap(); + all_rxs.push(rx.clone()); + rx + } + + let test_star = + create_rx(&datastore, &opctx, &mut all_rxs, "test-star", "test.*") + .await; + let test_starstar = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-starstar", + "test.**", + ) + .await; + let test_foo_star = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-foo-star", + "test.foo.*", + ) + .await; + let test_star_baz = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-star-baz", + "test.*.baz", + ) + .await; + let test_starstar_baz = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-starstar-baz", + "test.**.baz", + ) + .await; + let test_quux_star = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-quux-star", + "test.quux.*", + ) + .await; + let test_quux_starstar = create_rx( + &datastore, + &opctx, + &mut all_rxs, + "test-quux-starstar", + "test.quux.**", + ) + .await; async fn check_event( datastore: &DataStore, - // logctx: &LogContext, - event_class: &str, + all_rxs: &Vec, + event_class: WebhookEventClass, matches: &[&WebhookReceiverConfig], - not_matches: &[&WebhookReceiverConfig], ) { let subscribed = datastore .webhook_rx_list_subscribed_to_event_on_conn( @@ -605,7 +570,7 @@ mod test { .into_iter() .map(|(rx, subscription)| { eprintln!( - "receiver is subscribed to event {event_class:?}:\n\t\ + "receiver is subscribed to event {event_class}:\n\t\ rx: {} ({})\n\tsubscription: {subscription:?}", rx.identity.name, rx.identity.id, ); @@ -616,15 +581,22 @@ mod test { for WebhookReceiverConfig { rx, events, .. } in matches { assert!( subscribed.contains(&rx.identity), - "expected {rx:?} to be subscribed to {event_class:?}\n\ + "expected {rx:?} to be subscribed to {event_class}\n\ subscriptions: {events:?}" ); } + let not_matches = all_rxs.iter().filter( + |WebhookReceiverConfig { rx, .. }| { + matches + .iter() + .all(|match_rx| rx.identity != match_rx.rx.identity) + }, + ); for WebhookReceiverConfig { rx, events, .. } in not_matches { assert!( !subscribed.contains(&rx.identity), - "expected {rx:?} to not be subscribed to {event_class:?}\n\ + "expected {rx:?} to not be subscribed to {event_class}\n\ subscriptions: {events:?}" ); } @@ -632,65 +604,45 @@ mod test { check_event( datastore, - "notfoo", - &[], - &[ - &foo_star, - &foo_starstar, - &foo_bar, - &foo_starstar_bar, - &starstar_bar, - ], + &all_rxs, + WebhookEventClass::TestFoo, + &[&test_star, &test_starstar], ) .await; - check_event( datastore, - "foo.bar", - &[&foo_star, &foo_starstar, &foo_bar, &starstar_bar], - &[&foo_starstar_bar], + &all_rxs, + WebhookEventClass::TestFooBar, + &[&test_starstar, &test_foo_star], ) .await; - check_event( datastore, - "foo.baz", - &[&foo_star, &foo_starstar], - &[&foo_bar, &foo_starstar_bar, &starstar_bar], - ) - .await; - - check_event( - datastore, - "foo.bar.baz", - &[&foo_starstar], - &[&foo_bar, &foo_star, &foo_starstar_bar, &starstar_bar], + &all_rxs, + WebhookEventClass::TestFooBaz, + &[ + &test_starstar, + &test_foo_star, + &test_star_baz, + &test_starstar_baz, + ], ) .await; - check_event( datastore, - "foo.baz.bar", - &[&foo_starstar, &foo_starstar_bar, &starstar_bar], - &[&foo_bar, &foo_star], + &all_rxs, + WebhookEventClass::TestQuuxBar, + &[&test_starstar, &test_quux_star, &test_quux_starstar], ) .await; - check_event( datastore, - "foo.baz.quux.bar", - &[&foo_starstar, &foo_starstar_bar, &starstar_bar], - &[&foo_bar, &foo_star], + &all_rxs, + WebhookEventClass::TestQuuxBarBaz, + &[&test_starstar, &test_quux_starstar, &test_starstar_baz], ) .await; - check_event( - datastore, - "baz.quux.bar", - &[&starstar_bar], - &[&foo_bar, &foo_star, &foo_starstar, &foo_starstar_bar], - ) - .await; // Clean up. db.terminate().await; logctx.cleanup_successful(); @@ -703,7 +655,8 @@ mod test { let pool = db.pool(); let conn = pool.claim().await.unwrap(); - let query = DataStore::rx_list_subscribed_query("foo.bar".to_string()); + let query = + DataStore::rx_list_subscribed_query(WebhookEventClass::TestFooBar); let explanation = query .explain_async(&conn) .await diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index e9d8928ce8a..db7df122f6c 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -9,7 +9,8 @@ use http::HeaderValue; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::model::{ - SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookReceiver, + SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookEventClass, + WebhookReceiver, }; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; @@ -208,7 +209,6 @@ impl WebhookDeliverator { for (delivery, event_class) in deliveries { let attempt = (*delivery.attempts) + 1; let delivery_id = WebhookDeliveryUuid::from(delivery.id); - let event_class = &event_class; match self .datastore .webhook_delivery_start_attempt( @@ -223,7 +223,7 @@ impl WebhookDeliverator { slog::trace!(&opctx.log, "webhook event delivery attempt started"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "attempt" => attempt, ); @@ -233,7 +233,7 @@ impl WebhookDeliverator { &opctx.log, "delivery of this webhook event was already completed at {time:?}"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "time_completed" => ?time, ); @@ -245,7 +245,7 @@ impl WebhookDeliverator { &opctx.log, "delivery of this webhook event is in progress by another Nexus"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "nexus_id" => %nexus_id, "time_started" => ?started, @@ -258,7 +258,7 @@ impl WebhookDeliverator { &opctx.log, "unexpected database error error starting webhook delivery attempt"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "error" => %error, ); @@ -273,7 +273,7 @@ impl WebhookDeliverator { let time_attempted = Utc::now(); let sent_at = time_attempted.to_rfc3339(); let payload = Payload { - event_class: event_class.as_ref(), + event_class, event_id: delivery.event_id.into(), data: &delivery.payload, delivery: DeliveryMetadata { @@ -289,7 +289,7 @@ impl WebhookDeliverator { .header(HDR_RX_ID, hdr_rx_id.clone()) .header(HDR_DELIVERY_ID, delivery_id.to_string()) .header(HDR_EVENT_ID, delivery.event_id.to_string()) - .header(HDR_EVENT_CLASS, event_class) + .header(HDR_EVENT_CLASS, event_class.to_string()) .json(&payload) // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to // each webhook delivery request. @@ -306,7 +306,7 @@ impl WebhookDeliverator { &opctx.log, "{MSG}"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "error" => %e, "payload" => ?payload, @@ -330,7 +330,7 @@ impl WebhookDeliverator { &opctx.log, "{MSG}"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "error" => %e, ); @@ -345,7 +345,7 @@ impl WebhookDeliverator { &opctx.log, "webhook receiver endpoint returned an HTTP error"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "response_status" => ?status, "response_duration" => ?duration, @@ -365,7 +365,7 @@ impl WebhookDeliverator { &opctx.log, "webhook delivery request failed"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "error" => %e, ); @@ -378,7 +378,7 @@ impl WebhookDeliverator { &opctx.log, "webhook event delivered successfully"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "response_status" => ?status, "response_duration" => ?duration, @@ -420,7 +420,7 @@ impl WebhookDeliverator { &opctx.log, "{MSG}"; "event_id" => %delivery.event_id, - "event_class" => event_class, + "event_class" => %event_class, "delivery_id" => %delivery_id, "error" => %e, ); @@ -450,7 +450,7 @@ const HDR_SIG: HeaderName = HeaderName::from_static("x-oxide-signature"); #[derive(serde::Serialize, Debug)] struct Payload<'a> { - event_class: &'a str, + event_class: WebhookEventClass, event_id: WebhookEventUuid, data: &'a serde_json::Value, delivery: DeliveryMetadata<'a>, diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 057c599476c..67a72e8e629 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -7,6 +7,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::WebhookEvent; +use nexus_db_queries::db::model::WebhookEventClass; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_types::external_api::params; use omicron_common::api::external::CreateResult; @@ -15,8 +16,6 @@ use omicron_common::api::external::LookupResult; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; -pub const EVENT_CLASSES: &[&str] = &["test"]; - impl super::Nexus { pub async fn webhook_receiver_config_fetch( &self, @@ -39,24 +38,16 @@ impl super::Nexus { ) -> CreateResult { // TODO(eliza): validate endpoint URI; reject underlay network IPs for // SSRF prevention... - self.datastore().webhook_rx_create(&opctx, params, EVENT_CLASSES).await + self.datastore().webhook_rx_create(&opctx, params).await } pub async fn webhook_event_publish( &self, opctx: &OpContext, id: WebhookEventUuid, - event_class: String, + event_class: WebhookEventClass, event: serde_json::Value, ) -> Result { - if !EVENT_CLASSES.contains(&event_class.as_str()) { - return Err(Error::InternalError { - internal_message: format!( - "unknown webhook event class {event_class:?}" - ), - }); - } - let event = self .datastore() .webhook_event_create(opctx, id, event_class, event) @@ -65,7 +56,7 @@ impl super::Nexus { &opctx.log, "enqueued webhook event"; "event_id" => ?id, - "event_class" => ?event.event_class, + "event_class" => %event.event_class, "time_created" => ?event.time_created, ); diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index e459be50cf6..289c8dbe355 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -5,6 +5,7 @@ //! Webhooks use httpmock::prelude::*; +use nexus_db_model::WebhookEventClass; use nexus_db_queries::context::OpContext; use nexus_test_utils::background::activate_background_task; use nexus_test_utils::resource_helpers; @@ -53,7 +54,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { }, endpoint, secrets: vec!["my cool secret".to_string()], - events: vec!["test".to_string()], + events: vec!["test.foo".to_string()], disable_probes: false, }, ) @@ -63,7 +64,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let mock = server .mock_async(|when, then| { let body = serde_json::json!({ - "event_class": "test", + "event_class": "test.foo", "event_id": id, "data": { "hello_world": true, @@ -73,7 +74,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { when.method(POST) .path("/webhooks") .json_body_includes(body) - .header("x-oxide-event-class", "test") + .header("x-oxide-event-class", "test.foo") .header("x-oxide-event-id", id.to_string()) .header("x-oxide-webhook-id", webhook.id.to_string()) .header("content-type", "application/json") @@ -89,7 +90,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { .webhook_event_publish( &opctx, id, - "test".to_string(), + WebhookEventClass::TestFoo, serde_json::json!({"hello_world": true}), ) .await diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 012a59a3dab..a002d90cac7 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4858,7 +4858,9 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( -- These are not publicly exposed. 'test.foo', 'test.foo.bar', - 'test.baz.bar' + 'test.foo.baz', + 'test.quux.bar', + 'test.quux.bar.baz' -- Add new event classes here! ); From d963b3dd5edb706259bca167bff8e384e9c1d32a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 3 Feb 2025 12:49:38 -0800 Subject: [PATCH 050/168] post rebase fixup --- nexus/src/app/background/init.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index b36cdb1b3ef..8fba4c28674 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -893,6 +893,7 @@ impl BackgroundTasksInitializer { driver.register(TaskDefinition { name: "tuf_artifact_replication", description: "replicate update repo artifacts across sleds", + period: config.tuf_artifact_replication.period_secs, task_impl: Box::new( tuf_artifact_replication::ArtifactReplication::new( datastore.clone(), @@ -906,6 +907,7 @@ impl BackgroundTasksInitializer { }); driver.register(TaskDefinition { + name: "webhook_dispatcher", description: "dispatches queued webhook events to receivers", period: config.webhook_dispatcher.period_secs, task_impl: Box::new(WebhookDispatcher::new( From e6dc20b5d558e3fd3a1f509edd35800a306a43f0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Feb 2025 12:25:54 -0800 Subject: [PATCH 051/168] wip attempt dispatch query --- nexus/db-queries/src/db/queries/mod.rs | 1 + .../src/db/queries/webhook_delivery.rs | 124 ++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 nexus/db-queries/src/db/queries/webhook_delivery.rs diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index 5f34c7cfb3d..755bc5acd9d 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -16,6 +16,7 @@ pub mod region_allocation; pub mod virtual_provisioning_collection_update; pub mod vpc; pub mod vpc_subnet; +pub mod webhook_delivery; /// SQL used to enable full table scans for the duration of the current /// transaction. diff --git a/nexus/db-queries/src/db/queries/webhook_delivery.rs b/nexus/db-queries/src/db/queries/webhook_delivery.rs new file mode 100644 index 00000000000..db9ede1188d --- /dev/null +++ b/nexus/db-queries/src/db/queries/webhook_delivery.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of queries for webhook deliveries + +use crate::db::model::schema::webhook_delivery::{self, dsl}; +use crate::db::model::WebhookDelivery; +use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; +use diesel::pg::Pg; +use diesel::prelude::QueryResult; +use diesel::query_builder::{AstPass, Query, QueryFragment, QueryId}; +use diesel::sql_types; +use diesel::Column; +use diesel::QuerySource; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::WebhookEventUuid; +use omicron_uuid_kinds::WebhookReceiverUuid; + +pub struct WebhookDeliveryDispatch { + rx_id: WebhookReceiverUuid, + event_id: WebhookEventUuid, + insert: Box + Send>, +} + +impl WebhookDeliveryDispatch { + pub fn new(delivery: WebhookDelivery) -> Self { + let rx_id = delivery.rx_id.into(); + let event_id = delivery.event_id.into(); + let insert = Box::new( + diesel::insert_into(dsl::webhook_delivery) + .values(delivery) + .on_conflict(dsl::id) + .do_nothing(), // .returning(WebhookDelivery::as_returning()) + ); + Self { rx_id, event_id, insert } + } +} + +impl QueryFragment for WebhookDeliveryDispatch { + fn walk_ast(&self, mut out: AstPass) -> QueryResult<()> { + self.insert.walk_ast(out.reborrow())?; + // WHERE NOT EXISTS ( + // SELECT 1 FROM omicron.public.webhook_delivery + // WHERE rx_id = $1 AND event_id = $2 + // ) + out.push_sql(" WHERE NOT EXISTS ( SELECT 1 "); + dsl::webhook_delivery.from_clause().walk_ast(out.reborrow())?; + out.push_sql(" WHERE "); + out.push_identifier(dsl::rx_id::NAME); + out.push_sql(" = "); + out.push_bind_param::( + self.rx_id.as_untyped_uuid(), + )?; + out.push_sql(" AND "); + out.push_identifier(dsl::event_id::NAME); + out.push_sql(" = "); + out.push_bind_param::( + self.event_id.as_untyped_uuid(), + )?; + out.push_sql(" )"); + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::db::explain::ExplainableAsync; + use crate::db::model; + use crate::db::model::WebhookEvent; + use crate::db::model::WebhookEventClass; + use crate::db::pub_test_utils::TestDatabase; + use crate::db::raw_query_builder::expectorate_query_contents; + + use anyhow::Context; + use async_bb8_diesel::AsyncRunQueryDsl; + use nexus_types::external_api::params; + use nexus_types::identity::Resource; + use omicron_common::api::external; + use omicron_test_utils::dev; + use uuid::Uuid; + + fn test_dispatch_query() -> WebhookDeliveryDispatch { + let event = WebhookEvent { + id: WebhookEventUuid::nil().into(), + time_created: Utc::now(), + time_dispatched: None, + event_class: WebhookEventClass::Test, + event: serde_json::json!({ "test": "data" }), + }; + let delivery = WebhookDelivery::new(&event, WebhookReceiverId::nil()); + WebhookDeliveryDispatch::new(delivery) + } + + #[tokio::test] + async fn expectorate_delivery_dispatch_query() { + expectorate_query_contents( + &test_dispatch_query(), + "tests/output/webhook_delivery_dispatch_query.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_delivery_dispatch_query() { + let logctx = + dev::test_setup_log("explain_webhook_delivery_dispatch_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let query = test_dispatch_query(); + let explanation = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + eprintln!("{explanation}"); + + db.terminate().await; + logctx.cleanup_successful(); + } +} From d88cde70f436cbe8c8f4469ffaad049cb7e865c0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Feb 2025 14:27:01 -0800 Subject: [PATCH 052/168] start redoing dispatch sstuff thanks @smklein --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_delivery.rs | 11 +- .../src/db/datastore/webhook_event.rs | 110 ++++++++-------- .../db-queries/src/db/datastore/webhook_rx.rs | 1 + nexus/db-queries/src/db/queries/mod.rs | 1 - .../src/db/queries/webhook_delivery.rs | 124 ------------------ schema/crdb/dbinit.sql | 15 +++ 7 files changed, 82 insertions(+), 181 deletions(-) delete mode 100644 nexus/db-queries/src/db/queries/webhook_delivery.rs diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index cb9eb4ca38e..aed8cf253ae 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2198,6 +2198,7 @@ table! { id -> Uuid, event_id -> Uuid, rx_id -> Uuid, + is_redelivery -> Bool, payload -> Jsonb, attempts -> Int2, time_created -> Timestamptz, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 7a37b776f6e..7bcf3937a9a 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -65,6 +65,10 @@ pub struct WebhookDelivery { /// `webhook_rx`). pub rx_id: DbTypedUuid, + /// True if this is an explicitly triggered re-delivery attempt, false if + /// this is an initial dispatch of the event. + pub is_redelivery: bool, + /// The data payload as sent to this receiver. pub payload: serde_json::Value, @@ -83,12 +87,17 @@ pub struct WebhookDelivery { } impl WebhookDelivery { - pub fn new(event: &WebhookEvent, rx_id: &WebhookReceiverUuid) -> Self { + pub fn new( + event: &WebhookEvent, + rx_id: &WebhookReceiverUuid, + is_redelivery: bool, + ) -> Self { Self { // N.B.: perhaps we ought to use timestamp-based UUIDs for these? id: WebhookDeliveryUuid::new_v4().into(), event_id: event.id, rx_id: (*rx_id).into(), + is_redelivery, payload: event.event.clone(), attempts: SqlU8::new(0), time_created: Utc::now(), diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 280502a98c8..115f06c85db 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -57,24 +57,10 @@ impl DataStore { let conn = self.pool_connection_authorized(&opctx).await?; self.transaction_retry_wrapper("webhook_event_dispatch_next") .transaction(&conn, |conn| async move { - // Select the next webhook event in need of dispatching. - // - // This performs a `SELECT ... FOR UPDATE SKIP LOCKED` on the - // `webhook_event` table, returning the oldest webhook event which has not - // yet been dispatched to receivers and which is not actively being - // dispatched in another transaction. - // NOTE: it would be kinda nice if this query could also select the - // webhook receivers subscribed to this event, but this requires - // a `FOR UPDATE OF webhook_event` clause to indicate that we only wish - // to lock the `webhook_event` row and not the receiver. - // Unfortunately, I don't believe Diesel supports this at present. let Some(event) = event_dsl::webhook_event .filter(event_dsl::time_dispatched.is_null()) .order_by(event_dsl::time_created.asc()) .limit(1) - .for_update() - // TODO(eliza): AGH SKIP LOCKED IS NOT IMPLEMENTED IN CRDB... - // .skip_locked() .select(WebhookEvent::as_select()) .get_result_async(&conn) .await @@ -124,7 +110,7 @@ impl DataStore { ); match self .webhook_event_insert_delivery_on_conn( - &opctx.log, &event, &rx_id, &conn, + &event, &rx_id, false, &conn ) .await { @@ -165,49 +151,63 @@ impl DataStore { async fn webhook_event_insert_delivery_on_conn( &self, - log: &slog::Logger, event: &WebhookEvent, rx_id: &WebhookReceiverUuid, + is_redelivery: bool, conn: &async_bb8_diesel::Connection, ) -> Result { - loop { - let delivery: Option = - WebhookReceiver::insert_resource( - rx_id.into_untyped_uuid(), - diesel::insert_into(delivery_dsl::webhook_delivery) - .values(WebhookDelivery::new(&event, rx_id)) - .on_conflict(delivery_dsl::id) - .do_nothing(), - ) - .insert_and_get_optional_result_async(conn) - .await?; - match delivery { - Some(delivery) => { - // XXX(eliza): is `Debug` too noisy for this? - slog::debug!( - log, - "dispatched webhook event to receiver"; - "event_id" => ?event.id, - "event_class" => %event.event_class, - "receiver_id" => ?rx_id, - ); - return Ok(delivery); - } - // The `ON CONFLICT (id) DO NOTHING` clause triggers if there's - // already a delivery entry with this UUID --- indicating a UUID - // collision. With 128 bits of random UUID, the chances of this - // happening are incredibly unlikely, but let's handle it - // gracefully nonetheless by trying again with a new UUID... - None => { - slog::warn!( - &log, - "webhook delivery UUID collision, retrying..."; - "event_id" => ?event.id, - "event_class" => %event.event_class, - "receiver_id" => ?rx_id, - ); - } - } - } + let delivery: WebhookDelivery = WebhookReceiver::insert_resource( + rx_id.into_untyped_uuid(), + diesel::insert_into(delivery_dsl::webhook_delivery) + .values(WebhookDelivery::new(&event, rx_id, is_redelivery)), + ) + .insert_and_get_result_async(conn) + .await?; + Ok(delivery) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::db::pub_test_utils::TestDatabase; + use omicron_test_utils::dev; + + #[tokio::test] + async fn test_dispatched_deliveries_are_unique_per_rx() { + // Test setup + let logctx = + dev::test_setup_log("test_dispatched_deliveries_are_unique_per_rx"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let conn = db.pool_connection_for_tests().await; + + let rx_id = WebhookReceiverUuid::new_v4(); + let event_id = WebhookEventUuid::new_v4(); + let event = datastore + .webhook_event_create( + &opctx, + event_id, + WebhookEventClass::TestFoo, + serde_json::json!({ + "answer": 42, + }), + ) + .await + .expect("can't create ye event"); + + let delivery1 = datastore + .webhook_event_insert_delivery_on_conn(&event, &rx_id, false, conn) + .await + .expect("delivery 1 should insert"); + + let delivery2 = datastore + .webhook_event_insert_delivery_on_conn(&event, &rx_id, false, conn) + .await; + dbg!(delivery2).expect_err("unique constraint should be violated"); + + db.terminate().await; + logctx.cleanup_successful(); } } diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index ad83de42e1a..755cfdc3a21 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -456,6 +456,7 @@ fn async_insert_error_to_txn( AsyncInsertError::DatabaseError(e) => TransactionError::Database(e), } } + #[cfg(test)] mod test { use super::*; diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index 755bc5acd9d..5f34c7cfb3d 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -16,7 +16,6 @@ pub mod region_allocation; pub mod virtual_provisioning_collection_update; pub mod vpc; pub mod vpc_subnet; -pub mod webhook_delivery; /// SQL used to enable full table scans for the duration of the current /// transaction. diff --git a/nexus/db-queries/src/db/queries/webhook_delivery.rs b/nexus/db-queries/src/db/queries/webhook_delivery.rs deleted file mode 100644 index db9ede1188d..00000000000 --- a/nexus/db-queries/src/db/queries/webhook_delivery.rs +++ /dev/null @@ -1,124 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Implementation of queries for webhook deliveries - -use crate::db::model::schema::webhook_delivery::{self, dsl}; -use crate::db::model::WebhookDelivery; -use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; -use diesel::pg::Pg; -use diesel::prelude::QueryResult; -use diesel::query_builder::{AstPass, Query, QueryFragment, QueryId}; -use diesel::sql_types; -use diesel::Column; -use diesel::QuerySource; -use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::WebhookEventUuid; -use omicron_uuid_kinds::WebhookReceiverUuid; - -pub struct WebhookDeliveryDispatch { - rx_id: WebhookReceiverUuid, - event_id: WebhookEventUuid, - insert: Box + Send>, -} - -impl WebhookDeliveryDispatch { - pub fn new(delivery: WebhookDelivery) -> Self { - let rx_id = delivery.rx_id.into(); - let event_id = delivery.event_id.into(); - let insert = Box::new( - diesel::insert_into(dsl::webhook_delivery) - .values(delivery) - .on_conflict(dsl::id) - .do_nothing(), // .returning(WebhookDelivery::as_returning()) - ); - Self { rx_id, event_id, insert } - } -} - -impl QueryFragment for WebhookDeliveryDispatch { - fn walk_ast(&self, mut out: AstPass) -> QueryResult<()> { - self.insert.walk_ast(out.reborrow())?; - // WHERE NOT EXISTS ( - // SELECT 1 FROM omicron.public.webhook_delivery - // WHERE rx_id = $1 AND event_id = $2 - // ) - out.push_sql(" WHERE NOT EXISTS ( SELECT 1 "); - dsl::webhook_delivery.from_clause().walk_ast(out.reborrow())?; - out.push_sql(" WHERE "); - out.push_identifier(dsl::rx_id::NAME); - out.push_sql(" = "); - out.push_bind_param::( - self.rx_id.as_untyped_uuid(), - )?; - out.push_sql(" AND "); - out.push_identifier(dsl::event_id::NAME); - out.push_sql(" = "); - out.push_bind_param::( - self.event_id.as_untyped_uuid(), - )?; - out.push_sql(" )"); - Ok(()) - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::db::explain::ExplainableAsync; - use crate::db::model; - use crate::db::model::WebhookEvent; - use crate::db::model::WebhookEventClass; - use crate::db::pub_test_utils::TestDatabase; - use crate::db::raw_query_builder::expectorate_query_contents; - - use anyhow::Context; - use async_bb8_diesel::AsyncRunQueryDsl; - use nexus_types::external_api::params; - use nexus_types::identity::Resource; - use omicron_common::api::external; - use omicron_test_utils::dev; - use uuid::Uuid; - - fn test_dispatch_query() -> WebhookDeliveryDispatch { - let event = WebhookEvent { - id: WebhookEventUuid::nil().into(), - time_created: Utc::now(), - time_dispatched: None, - event_class: WebhookEventClass::Test, - event: serde_json::json!({ "test": "data" }), - }; - let delivery = WebhookDelivery::new(&event, WebhookReceiverId::nil()); - WebhookDeliveryDispatch::new(delivery) - } - - #[tokio::test] - async fn expectorate_delivery_dispatch_query() { - expectorate_query_contents( - &test_dispatch_query(), - "tests/output/webhook_delivery_dispatch_query.sql", - ) - .await; - } - - #[tokio::test] - async fn explain_delivery_dispatch_query() { - let logctx = - dev::test_setup_log("explain_webhook_delivery_dispatch_query"); - let db = TestDatabase::new_with_pool(&logctx.log).await; - let pool = db.pool(); - let conn = pool.claim().await.unwrap(); - - let query = test_dispatch_query(); - let explanation = query - .explain_async(&conn) - .await - .expect("Failed to explain query - is it valid SQL?"); - - eprintln!("{explanation}"); - - db.terminate().await; - logctx.cleanup_successful(); - } -} diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a002d90cac7..eb904b82401 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4971,6 +4971,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, + -- true if this delivery attempt was triggered by a call to the resend API, + -- false if this is the initial delivery attempt. + is_redelivery BOOL NOT NULL, + payload JSONB NOT NULL, --- Delivery attempt count. Starts at 0. @@ -4993,6 +4997,17 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( ) ); +-- Ensure that initial delivery attempts (nexus-dispatched) are unique to avoid +-- duplicate work when an event is dispatched. For deliveries created by calls +-- to the webhook event resend API, we don't enforce this constraint, to allow +-- re-delivery to be triggered multiple times. +CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx +ON omicron.public.webhook_delivery ( + event_id, rx_id +) +WHERE + is_redelivery = FALSE; + -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx ON omicron.public.webhook_delivery ( From 6bf2578a3279c7c138f553457d2e798674228c91 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 5 Feb 2025 10:44:13 -0800 Subject: [PATCH 053/168] okay cool --- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 2 +- nexus/db-model/src/webhook_delivery.rs | 10 +- .../db-model/src/webhook_delivery_trigger.rs | 49 +++++++ .../src/db/datastore/webhook_event.rs | 137 +++++++++++++++--- schema/crdb/dbinit.sql | 18 ++- 6 files changed, 186 insertions(+), 32 deletions(-) create mode 100644 nexus/db-model/src/webhook_delivery_trigger.rs diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index c6d4cedd56a..a4b5bff2ee8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -66,6 +66,7 @@ mod switch_port; mod v2p_mapping; mod vmm_state; mod webhook_delivery; +mod webhook_delivery_trigger; mod webhook_event; mod webhook_event_class; mod webhook_rx; @@ -236,6 +237,7 @@ pub use vpc_route::*; pub use vpc_router::*; pub use vpc_subnet::*; pub use webhook_delivery::*; +pub use webhook_delivery_trigger::*; pub use webhook_event::*; pub use webhook_event_class::*; pub use webhook_rx::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index aed8cf253ae..b052ce49660 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2198,7 +2198,7 @@ table! { id -> Uuid, event_id -> Uuid, rx_id -> Uuid, - is_redelivery -> Bool, + trigger -> crate::WebhookDeliveryTriggerEnum, payload -> Jsonb, attempts -> Int2, time_created -> Timestamptz, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 7bcf3937a9a..990ad8359ec 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -7,6 +7,7 @@ use crate::schema::{webhook_delivery, webhook_delivery_attempt}; use crate::serde_time_delta::optional_time_delta; use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; +use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; @@ -65,9 +66,8 @@ pub struct WebhookDelivery { /// `webhook_rx`). pub rx_id: DbTypedUuid, - /// True if this is an explicitly triggered re-delivery attempt, false if - /// this is an initial dispatch of the event. - pub is_redelivery: bool, + /// Describes why this delivery was triggered. + pub trigger: WebhookDeliveryTrigger, /// The data payload as sent to this receiver. pub payload: serde_json::Value, @@ -90,14 +90,14 @@ impl WebhookDelivery { pub fn new( event: &WebhookEvent, rx_id: &WebhookReceiverUuid, - is_redelivery: bool, + trigger: WebhookDeliveryTrigger, ) -> Self { Self { // N.B.: perhaps we ought to use timestamp-based UUIDs for these? id: WebhookDeliveryUuid::new_v4().into(), event_id: event.id, rx_id: (*rx_id).into(), - is_redelivery, + trigger, payload: event.event.clone(), attempts: SqlU8::new(0), time_created: Utc::now(), diff --git a/nexus/db-model/src/webhook_delivery_trigger.rs b/nexus/db-model/src/webhook_delivery_trigger.rs new file mode 100644 index 00000000000..0cb94065605 --- /dev/null +++ b/nexus/db-model/src/webhook_delivery_trigger.rs @@ -0,0 +1,49 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + +impl_enum_type!( + #[derive(SqlType, Debug, Clone)] + #[diesel(postgres_type(name = "webhook_delivery_trigger", schema = "public"))] + pub struct WebhookDeliveryTriggerEnum; + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + Serialize, + Deserialize, + AsExpression, + FromSqlRow, + )] + #[diesel(sql_type = WebhookDeliveryTriggerEnum)] + #[serde(rename_all = "snake_case")] + pub enum WebhookDeliveryTrigger; + + Dispatch => b"dispatch" + Resend => b"resend" +); + +impl WebhookDeliveryTrigger { + pub fn as_str(&self) -> &'static str { + // TODO(eliza): it would be really nice if these strings were all + // declared a single time, rather than twice (in both `impl_enum_type!` + // and here)... + match self { + Self::Dispatch => "dispatch", + Self::Resend => "resend", + } + } +} + +impl fmt::Display for WebhookDeliveryTrigger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.as_str()) + } +} diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 115f06c85db..71dfa10481c 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -11,6 +11,7 @@ use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::WebhookDelivery; +use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; use crate::db::model::WebhookReceiver; @@ -19,6 +20,8 @@ use crate::db::schema::webhook_delivery::dsl as delivery_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; +use diesel::result::DatabaseErrorKind; +use diesel::result::Error as DieselError; use diesel::result::OptionalExtension; use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookDispatched; @@ -110,7 +113,7 @@ impl DataStore { ); match self .webhook_event_insert_delivery_on_conn( - &event, &rx_id, false, &conn + &event, &rx_id, WebhookDeliveryTrigger::Dispatch, &conn ) .await { @@ -153,25 +156,52 @@ impl DataStore { &self, event: &WebhookEvent, rx_id: &WebhookReceiverUuid, - is_redelivery: bool, + trigger: WebhookDeliveryTrigger, conn: &async_bb8_diesel::Connection, - ) -> Result { - let delivery: WebhookDelivery = WebhookReceiver::insert_resource( - rx_id.into_untyped_uuid(), - diesel::insert_into(delivery_dsl::webhook_delivery) - .values(WebhookDelivery::new(&event, rx_id, is_redelivery)), - ) - .insert_and_get_result_async(conn) - .await?; - Ok(delivery) + ) -> Result, AsyncInsertError> { + let result: Result = + WebhookReceiver::insert_resource( + rx_id.into_untyped_uuid(), + diesel::insert_into(delivery_dsl::webhook_delivery) + .values(WebhookDelivery::new(&event, rx_id, trigger)), + ) + .insert_and_get_result_async(conn) + .await; + match result { + Ok(delivery) => Ok(Some(delivery)), + + // If the UNIQUE constraint on the index + // "one_webhook_event_dispatch_per_rx" is violated, then that just + // means that a delivery has already been automatically dispatched + // to this receiver. That's fine --- the goal is to ensure that one + // exists, and if it already does, we don't need to do anything. + // + // Note this applies only to dispatched deliveries, not to + // explicitly resent deliveries, as multiple resends for an event + // may be created. + Err(AsyncInsertError::DatabaseError( + DieselError::DatabaseError( + DatabaseErrorKind::UniqueViolation, + info, + ), + )) if info.constraint_name() + == Some("one_webhook_event_dispatch_per_rx") => + { + debug_assert_eq!(trigger, WebhookDeliveryTrigger::Dispatch); + Ok(None) + } + + Err(e) => Err(e), + } } } #[cfg(test)] mod test { use super::*; - use crate::db::pub_test_utils::TestDatabase; + use nexus_types::external_api::params; + use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; #[tokio::test] @@ -181,9 +211,28 @@ mod test { dev::test_setup_log("test_dispatched_deliveries_are_unique_per_rx"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let conn = db.pool_connection_for_tests().await; + // As webhook receivers are a collection that owns the delivery + // resource, we must create a "real" receiver before assigning + // deliveries to it. + let rx = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "test-webhook".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://webhooks.example.com".parse().unwrap(), + secrets: vec!["my cool secret".to_string()], + events: vec!["test.*".to_string()], + disable_probes: false, + }, + ) + .await + .unwrap(); + let rx_id = rx.rx.identity.id.into(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); - let rx_id = WebhookReceiverUuid::new_v4(); let event_id = WebhookEventUuid::new_v4(); let event = datastore .webhook_event_create( @@ -197,15 +246,61 @@ mod test { .await .expect("can't create ye event"); - let delivery1 = datastore - .webhook_event_insert_delivery_on_conn(&event, &rx_id, false, conn) + let dispatch1 = datastore + .webhook_event_insert_delivery_on_conn( + &event, + &rx_id, + WebhookDeliveryTrigger::Dispatch, + &*conn, + ) .await - .expect("delivery 1 should insert"); + .expect("dispatch 1 should insert"); + assert!( + dispatch1.is_some(), + "the first dispatched delivery of an event should be created" + ); - let delivery2 = datastore - .webhook_event_insert_delivery_on_conn(&event, &rx_id, false, conn) - .await; - dbg!(delivery2).expect_err("unique constraint should be violated"); + let dispatch2 = datastore + .webhook_event_insert_delivery_on_conn( + &event, + &rx_id, + WebhookDeliveryTrigger::Dispatch, + &*conn, + ) + .await + .expect("dispatch 2 insert should not fail"); + assert_eq!( + dispatch2, None, + "dispatching an event a second time should do nothing" + ); + + let resend1 = datastore + .webhook_event_insert_delivery_on_conn( + &event, + &rx_id, + WebhookDeliveryTrigger::Resend, + &*conn, + ) + .await + .expect("resend 1 insert should not fail"); + assert!( + resend1.is_some(), + "resending an event should create a new delivery" + ); + + let resend2 = datastore + .webhook_event_insert_delivery_on_conn( + &event, + &rx_id, + WebhookDeliveryTrigger::Resend, + &*conn, + ) + .await + .expect("resend insert should not fail"); + assert!( + resend2.is_some(), + "resending an event a second time should create a new delivery" + ); db.terminate().await; logctx.cleanup_successful(); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index eb904b82401..4effe1c16df 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4963,17 +4963,25 @@ ON omicron.public.webhook_event ( * Webhook message dispatching and delivery attempts. */ +-- Describes why a webhook delivery was triggered +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( + -- This delivery was triggered by the event being dispatched. + 'dispatch', + -- This delivery was triggered by an explicit call to the webhook event + -- resend API. + 'resend' +); + CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( - -- UUID of this dispatch. + -- UUID of this delivery. id UUID PRIMARY KEY, --- UUID of the event (foreign key into `omicron.public.webhook_event`). event_id UUID NOT NULL, -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- true if this delivery attempt was triggered by a call to the resend API, - -- false if this is the initial delivery attempt. - is_redelivery BOOL NOT NULL, + + trigger omicron.public.webhook_delivery_trigger NOT NULL, payload JSONB NOT NULL, @@ -5006,7 +5014,7 @@ ON omicron.public.webhook_delivery ( event_id, rx_id ) WHERE - is_redelivery = FALSE; + trigger = 'dispatch'; -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx From 76a484c02500d077c7f17374d0a8be40a177fe7c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Feb 2025 13:12:18 -0800 Subject: [PATCH 054/168] finish rewriting dispatcher to use unique index to dedup --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_event.rs | 2 + nexus/db-model/src/webhook_rx.rs | 10 - .../src/db/datastore/webhook_delivery.rs | 171 +++++++++++ .../src/db/datastore/webhook_event.rs | 283 ++---------------- .../db-queries/src/db/datastore/webhook_rx.rs | 27 +- .../background/tasks/webhook_dispatcher.rs | 203 ++++++++++--- nexus/types/src/internal_api/background.rs | 9 +- schema/crdb/dbinit.sql | 17 +- 9 files changed, 403 insertions(+), 320 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index b052ce49660..091f8b42090 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2190,6 +2190,7 @@ table! { time_dispatched -> Nullable, event_class -> crate::WebhookEventClassEnum, event -> Jsonb, + num_dispatched -> Int8, } } diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs index 03349cf4a33..5ac53bc9721 100644 --- a/nexus/db-model/src/webhook_event.rs +++ b/nexus/db-model/src/webhook_event.rs @@ -39,4 +39,6 @@ pub struct WebhookEvent { /// The event's data payload. pub event: serde_json::Value, + + pub num_dispatched: i64, } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index cbe4f53b98d..57f27f24210 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -113,16 +113,6 @@ impl DatastoreCollectionConfig for WebhookReceiver { type CollectionIdColumn = webhook_rx_event_glob::dsl::rx_id; } -impl DatastoreCollectionConfig for WebhookReceiver { - type CollectionId = Uuid; - type GenerationNumberColumn = webhook_receiver::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; - type CollectionIdColumn = webhook_delivery::dsl::rx_id; -} - -// TODO(eliza): should deliveries/delivery attempts also be treated as children -// of a webhook receiver? - #[derive( Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, )] diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index e18860c4ba1..2881ddca677 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -12,6 +12,7 @@ use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; use crate::db::model::WebhookEventClass; +use crate::db::pagination::paginated; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; @@ -22,9 +23,12 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; use chrono::{DateTime, Utc}; use diesel::prelude::*; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookReceiverUuid}; +use uuid::Uuid; #[derive(Debug, Clone, Eq, PartialEq)] pub enum DeliveryAttemptState { @@ -34,6 +38,41 @@ pub enum DeliveryAttemptState { } impl DataStore { + pub async fn webhook_delivery_create_batch( + &self, + opctx: &OpContext, + deliveries: Vec, + ) -> CreateResult { + let conn = self.pool_connection_authorized(opctx).await?; + diesel::insert_into(dsl::webhook_delivery) + .values(deliveries) + // N.B. that this is intended to ignore conflicts on the + // "one_webhook_event_dispatch_per_rx" index, but ON CONFLICT ... DO + // NOTHING can't be used with the names of indices, only actual + // UNIQUE CONSTRAINTs. So we just do a blanket ON CONFLICT DO + // NOTHING, which is fine, becausse the only other uniqueness + // constraint is the UUID primary key, and we kind of assume UUID + // collisions don't happen. Oh well. + .on_conflict_do_nothing() + .execute_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn webhook_rx_delivery_list( + &self, + opctx: &OpContext, + rx_id: &WebhookReceiverUuid, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { + let conn = self.pool_connection_authorized(opctx).await?; + paginated(dsl::webhook_delivery, dsl::id, pagparams) + .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn webhook_rx_delivery_list_ready( &self, opctx: &OpContext, @@ -207,3 +246,135 @@ impl DataStore { )) } } + +#[cfg(test)] +mod test { + use super::*; + use crate::db::model::WebhookDeliveryTrigger; + use crate::db::pagination::Paginator; + use crate::db::pub_test_utils::TestDatabase; + use nexus_types::external_api::params; + use omicron_common::api::external::IdentityMetadataCreateParams; + use omicron_test_utils::dev; + use omicron_uuid_kinds::WebhookEventUuid; + + #[tokio::test] + async fn test_dispatched_deliveries_are_unique_per_rx() { + // Test setup + let logctx = + dev::test_setup_log("test_dispatched_deliveries_are_unique_per_rx"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + // As webhook receivers are a collection that owns the delivery + // resource, we must create a "real" receiver before assigning + // deliveries to it. + let rx = datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "test-webhook".parse().unwrap(), + description: String::new(), + }, + endpoint: "http://webhooks.example.com".parse().unwrap(), + secrets: vec!["my cool secret".to_string()], + events: vec!["test.*".to_string()], + disable_probes: false, + }, + ) + .await + .unwrap(); + let rx_id = rx.rx.identity.id.into(); + let event_id = WebhookEventUuid::new_v4(); + let event = datastore + .webhook_event_create( + &opctx, + event_id, + WebhookEventClass::TestFoo, + serde_json::json!({ + "answer": 42, + }), + ) + .await + .expect("can't create ye event"); + + let dispatch1 = WebhookDelivery::new( + &event, + &rx_id, + WebhookDeliveryTrigger::Dispatch, + ); + let inserted = datastore + .webhook_delivery_create_batch(&opctx, vec![dispatch1.clone()]) + .await + .expect("dispatch 1 should insert"); + assert_eq!(inserted, 1, "first dispatched delivery should be created"); + + let dispatch2 = WebhookDelivery::new( + &event, + &rx_id, + WebhookDeliveryTrigger::Dispatch, + ); + let inserted = datastore + .webhook_delivery_create_batch(opctx, vec![dispatch2.clone()]) + .await + .expect("dispatch 2 insert should not fail"); + assert_eq!( + inserted, 0, + "dispatching an event a second time should do nothing" + ); + + let resend1 = WebhookDelivery::new( + &event, + &rx_id, + WebhookDeliveryTrigger::Resend, + ); + let inserted = datastore + .webhook_delivery_create_batch(opctx, vec![resend1.clone()]) + .await + .expect("resend 1 insert should not fail"); + assert_eq!( + inserted, 1, + "resending an event should create a new delivery" + ); + + let resend2 = WebhookDelivery::new( + &event, + &rx_id, + WebhookDeliveryTrigger::Resend, + ); + let inserted = datastore + .webhook_delivery_create_batch(opctx, vec![resend2.clone()]) + .await + .expect("resend 2 insert should not fail"); + assert_eq!( + inserted, 1, + "resending an event a second time should create a new delivery" + ); + + let mut all_deliveries = std::collections::HashSet::new(); + let mut paginator = + Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let deliveries = datastore + .webhook_rx_delivery_list( + &opctx, + &rx_id, + &p.current_pagparams(), + ) + .await + .unwrap(); + paginator = p.found_batch(&deliveries, &|d: &WebhookDelivery| { + d.id.into_untyped_uuid() + }); + all_deliveries.extend(deliveries.into_iter().map(|d| d.id)); + } + + assert!(all_deliveries.contains(&dispatch1.id)); + assert!(!all_deliveries.contains(&dispatch2.id)); + assert!(all_deliveries.contains(&resend1.id)); + assert!(all_deliveries.contains(&resend2.id)); + + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 71dfa10481c..ebb0b7d4727 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -6,28 +6,18 @@ use super::DataStore; use crate::context::OpContext; -use crate::db::collection_insert::AsyncInsertError; -use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::model::WebhookDelivery; -use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; -use crate::db::model::WebhookReceiver; -use crate::db::pool::DbConnection; -use crate::db::schema::webhook_delivery::dsl as delivery_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; -use diesel::result::DatabaseErrorKind; -use diesel::result::Error as DieselError; use diesel::result::OptionalExtension; -use nexus_types::identity::Resource; -use nexus_types::internal_api::background::WebhookDispatched; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{GenericUuid, WebhookEventUuid, WebhookReceiverUuid}; +use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::{GenericUuid, WebhookEventUuid}; impl DataStore { pub async fn webhook_event_create( @@ -46,6 +36,7 @@ impl DataStore { event_dsl::id.eq(id.into_untyped_uuid()), event_dsl::time_created.eq(now), event_dsl::event.eq(event), + event_dsl::num_dispatched.eq(0), )) .returning(WebhookEvent::as_returning()) .get_result_async(&*conn) @@ -53,256 +44,42 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - pub async fn webhook_event_dispatch_next( + pub async fn webhook_event_select_next_for_dispatch( &self, opctx: &OpContext, - ) -> Result, Error> { + ) -> Result, Error> { let conn = self.pool_connection_authorized(&opctx).await?; - self.transaction_retry_wrapper("webhook_event_dispatch_next") - .transaction(&conn, |conn| async move { - let Some(event) = event_dsl::webhook_event - .filter(event_dsl::time_dispatched.is_null()) - .order_by(event_dsl::time_created.asc()) - .limit(1) - .select(WebhookEvent::as_select()) - .get_result_async(&conn) - .await - .optional()? - else { - slog::debug!( - opctx.log, - "no unlocked webhook events in need of dispatching", - ); - return Ok(None); - }; - - let mut result = WebhookDispatched { - event_id: event.id.into(), - dispatched: 0, - receivers_gone: 0, - }; - - // Find receivers subscribed to this event's class. - let rxs = self - .webhook_rx_list_subscribed_to_event_on_conn( - event.event_class, - &conn, - ) - .await?; - - slog::debug!( - &opctx.log, - "found {} receivers subscribed to webhook event", rxs.len(); - "event_id" => ?event.id, - "event_class" => %event.event_class, - "receivers" => ?rxs.len(), - ); - - // Create dispatch entries for each receiver subscribed to this - // event class. - for (rx, sub) in rxs { - let rx_id = rx.id(); - slog::trace!( - &opctx.log, - "found receiver subscribed to event"; - "event_id" => ?event.id, - "event_class" => %event.event_class, - "receiver" => ?rx.name(), - "receiver_id" => ?rx_id, - "glob" => ?sub.glob, - ); - match self - .webhook_event_insert_delivery_on_conn( - &event, &rx_id, WebhookDeliveryTrigger::Dispatch, &conn - ) - .await - { - Ok(_) => result.dispatched += 1, - Err(AsyncInsertError::CollectionNotFound) => { - // The receiver has been deleted while we were - // trying to dispatch an event to it. That's fine; - // rather than aborting the transaction and having - // to do all this stuff over, let's just keep going. - slog::debug!( - &opctx.log, - "cannot dispatch event to a receiver that has been deleted"; - "event_id" => ?event.id, - "event_class" => %event.event_class, - "receiver" => ?rx.name(), - "receiver_id" => ?rx_id, - ); - result.receivers_gone += 1; - }, - Err(AsyncInsertError::DatabaseError(e)) => return Err(e), - } - } - - // Finally, set the dispatched timestamp for the event so it - // won't be dispatched by someone else. - diesel::update(event_dsl::webhook_event) - .filter(event_dsl::id.eq(event.id)) - .filter(event_dsl::time_dispatched.is_null()) - .set(event_dsl::time_dispatched.eq(diesel::dsl::now)) - .execute_async(&conn) - .await?; - - Ok(Some(result)) - }) + event_dsl::webhook_event + .filter(event_dsl::time_dispatched.is_null()) + .order_by(event_dsl::time_created.asc()) + .select(WebhookEvent::as_select()) + .first_async(&*conn) .await + .optional() .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - async fn webhook_event_insert_delivery_on_conn( + pub async fn webhook_event_mark_dispatched( &self, - event: &WebhookEvent, - rx_id: &WebhookReceiverUuid, - trigger: WebhookDeliveryTrigger, - conn: &async_bb8_diesel::Connection, - ) -> Result, AsyncInsertError> { - let result: Result = - WebhookReceiver::insert_resource( - rx_id.into_untyped_uuid(), - diesel::insert_into(delivery_dsl::webhook_delivery) - .values(WebhookDelivery::new(&event, rx_id, trigger)), - ) - .insert_and_get_result_async(conn) - .await; - match result { - Ok(delivery) => Ok(Some(delivery)), - - // If the UNIQUE constraint on the index - // "one_webhook_event_dispatch_per_rx" is violated, then that just - // means that a delivery has already been automatically dispatched - // to this receiver. That's fine --- the goal is to ensure that one - // exists, and if it already does, we don't need to do anything. - // - // Note this applies only to dispatched deliveries, not to - // explicitly resent deliveries, as multiple resends for an event - // may be created. - Err(AsyncInsertError::DatabaseError( - DieselError::DatabaseError( - DatabaseErrorKind::UniqueViolation, - info, - ), - )) if info.constraint_name() - == Some("one_webhook_event_dispatch_per_rx") => - { - debug_assert_eq!(trigger, WebhookDeliveryTrigger::Dispatch); - Ok(None) - } - - Err(e) => Err(e), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::db::pub_test_utils::TestDatabase; - use nexus_types::external_api::params; - use omicron_common::api::external::IdentityMetadataCreateParams; - use omicron_test_utils::dev; - - #[tokio::test] - async fn test_dispatched_deliveries_are_unique_per_rx() { - // Test setup - let logctx = - dev::test_setup_log("test_dispatched_deliveries_are_unique_per_rx"); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - // As webhook receivers are a collection that owns the delivery - // resource, we must create a "real" receiver before assigning - // deliveries to it. - let rx = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "test-webhook".parse().unwrap(), - description: String::new(), - }, - endpoint: "http://webhooks.example.com".parse().unwrap(), - secrets: vec!["my cool secret".to_string()], - events: vec!["test.*".to_string()], - disable_probes: false, - }, - ) - .await - .unwrap(); - let rx_id = rx.rx.identity.id.into(); - let conn = datastore.pool_connection_for_tests().await.unwrap(); - - let event_id = WebhookEventUuid::new_v4(); - let event = datastore - .webhook_event_create( - &opctx, - event_id, - WebhookEventClass::TestFoo, - serde_json::json!({ - "answer": 42, - }), - ) - .await - .expect("can't create ye event"); - - let dispatch1 = datastore - .webhook_event_insert_delivery_on_conn( - &event, - &rx_id, - WebhookDeliveryTrigger::Dispatch, - &*conn, - ) - .await - .expect("dispatch 1 should insert"); - assert!( - dispatch1.is_some(), - "the first dispatched delivery of an event should be created" - ); - - let dispatch2 = datastore - .webhook_event_insert_delivery_on_conn( - &event, - &rx_id, - WebhookDeliveryTrigger::Dispatch, - &*conn, - ) - .await - .expect("dispatch 2 insert should not fail"); - assert_eq!( - dispatch2, None, - "dispatching an event a second time should do nothing" - ); - - let resend1 = datastore - .webhook_event_insert_delivery_on_conn( - &event, - &rx_id, - WebhookDeliveryTrigger::Resend, - &*conn, - ) - .await - .expect("resend 1 insert should not fail"); - assert!( - resend1.is_some(), - "resending an event should create a new delivery" - ); - - let resend2 = datastore - .webhook_event_insert_delivery_on_conn( - &event, - &rx_id, - WebhookDeliveryTrigger::Resend, - &*conn, + opctx: &OpContext, + event_id: &WebhookEventUuid, + subscribed: usize, + ) -> UpdateResult { + let subscribed = i64::try_from(subscribed).map_err(|_| { + // that is way too many webhook receivers! + Error::internal_error( + "webhook event subscribed count exceeds i64::MAX", ) + })?; + let conn = self.pool_connection_authorized(&opctx).await?; + diesel::update(event_dsl::webhook_event) + .filter(event_dsl::id.eq(event_id.into_untyped_uuid())) + .set(( + event_dsl::time_dispatched.eq(diesel::dsl::now), + event_dsl::num_dispatched.eq(subscribed), + )) + .execute_async(&*conn) .await - .expect("resend insert should not fail"); - assert!( - resend2.is_some(), - "resending an event a second time should create a new delivery" - ); - - db.terminate().await; - logctx.cleanup_successful(); + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 755cfdc3a21..c47daae9780 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -348,17 +348,16 @@ impl DataStore { /// List all webhook receivers whose event class subscription globs match /// the provided `event_class`. - pub(crate) async fn webhook_rx_list_subscribed_to_event_on_conn( + pub async fn webhook_rx_list_subscribed_to_event( &self, + opctx: &OpContext, event_class: WebhookEventClass, - conn: &async_bb8_diesel::Connection, - ) -> Result< - Vec<(WebhookReceiver, WebhookRxSubscription)>, - diesel::result::Error, - > { + ) -> Result, Error> { + let conn = self.pool_connection_authorized(opctx).await?; Self::rx_list_subscribed_query(event_class) - .load_async::<(WebhookReceiver, WebhookRxSubscription)>(conn) + .load_async::<(WebhookReceiver, WebhookRxSubscription)>(&*conn) .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } fn rx_list_subscribed_query( @@ -554,18 +553,13 @@ mod test { async fn check_event( datastore: &DataStore, + opctx: &OpContext, all_rxs: &Vec, event_class: WebhookEventClass, matches: &[&WebhookReceiverConfig], ) { let subscribed = datastore - .webhook_rx_list_subscribed_to_event_on_conn( - event_class, - &datastore - .pool_connection_for_tests() - .await - .expect("can't get ye pool connection for tests!"), - ) + .webhook_rx_list_subscribed_to_event(opctx, event_class) .await .unwrap() .into_iter() @@ -605,6 +599,7 @@ mod test { check_event( datastore, + opctx, &all_rxs, WebhookEventClass::TestFoo, &[&test_star, &test_starstar], @@ -612,6 +607,7 @@ mod test { .await; check_event( datastore, + opctx, &all_rxs, WebhookEventClass::TestFooBar, &[&test_starstar, &test_foo_star], @@ -619,6 +615,7 @@ mod test { .await; check_event( datastore, + opctx, &all_rxs, WebhookEventClass::TestFooBaz, &[ @@ -631,6 +628,7 @@ mod test { .await; check_event( datastore, + opctx, &all_rxs, WebhookEventClass::TestQuuxBar, &[&test_starstar, &test_quux_star, &test_quux_starstar], @@ -638,6 +636,7 @@ mod test { .await; check_event( datastore, + opctx, &all_rxs, WebhookEventClass::TestQuuxBarBaz, &[&test_starstar, &test_quux_starstar, &test_starstar_baz], diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 8f1bbf544ff..f21776f8f0b 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -7,8 +7,11 @@ use crate::app::background::Activator; use crate::app::background::BackgroundTask; use futures::future::BoxFuture; +use nexus_db_model::WebhookDelivery; +use nexus_db_model::WebhookDeliveryTrigger; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; +use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ WebhookDispatched, WebhookDispatcherStatus, }; @@ -26,47 +29,61 @@ impl BackgroundTask for WebhookDispatcher { opctx: &'a OpContext, ) -> BoxFuture<'a, serde_json::Value> { Box::pin(async move { - let mut dispatched = Vec::new(); - let error = - match self.actually_activate(&opctx, &mut dispatched).await { - Ok(_) => { - const MSG: &str = - "webhook dispatching completed successfully"; - if !dispatched.is_empty() { - slog::info!( - &opctx.log, - "{MSG}"; - "events_dispatched" => dispatched.len(), - ); - } else { - // no sense cluttering up the logs if we didn't do - // anyuthing interesting today`s` - slog::trace!( - &opctx.log, - "{MSG}"; - "events_dispatched" => dispatched.len(), - ); - }; - - None - } - Err(error) => { - slog::error!( + let mut status = WebhookDispatcherStatus { + dispatched: Vec::new(), + errors: Vec::new(), + no_receivers: Vec::new(), + }; + match self.actually_activate(&opctx, &mut status).await { + Ok(_) if status.errors.is_empty() => { + const MSG: &str = + "webhook dispatching completed successfully"; + if !status.dispatched.is_empty() { + slog::info!( &opctx.log, - "webhook dispatching failed"; - "events_dispatched" => dispatched.len(), - "error" => &error, + "{MSG}"; + "events_dispatched" => status.dispatched.len(), + "events_without_receivers" => status.no_receivers.len(), ); - Some(error.to_string()) - } - }; + } else { + // no sense cluttering up the logs if we didn't do + // anyuthing interesting today + slog::trace!( + &opctx.log, + "{MSG}"; + "events_dispatched" => status.dispatched.len(), + "events_without_receivers" => status.no_receivers.len(), + ); + }; + } + Ok(_) => { + slog::warn!( + &opctx.log, + "webhook dispatching completed with errors"; + "events_dispatched" => status.dispatched.len(), + "events_without_receivers" => status.no_receivers.len(), + "events_failed" => status.errors.len(), + ); + } + Err(error) => { + slog::error!( + &opctx.log, + "webhook dispatching failed"; + "events_dispatched" => status.dispatched.len(), + "events_without_receivers" => status.no_receivers.len(), + "events_failed" => status.errors.len(), + "error" => &error, + ); + status.errors.push(error.to_string()); + } + }; // If any new deliveries were dispatched, call the deliverator! - if !dispatched.is_empty() { + if !status.dispatched.is_empty() { self.deliverator.activate(); } - serde_json::json!(WebhookDispatcherStatus { dispatched, error }) + serde_json::json!(status) }) } } @@ -79,12 +96,124 @@ impl WebhookDispatcher { async fn actually_activate( &mut self, opctx: &OpContext, - dispatched: &mut Vec, + status: &mut WebhookDispatcherStatus, ) -> Result<(), Error> { + // Select the next event that has yet to be dispatched in order of + // creation, until there are none left in need of dispatching. while let Some(event) = - self.datastore.webhook_event_dispatch_next(opctx).await? + self.datastore.webhook_event_select_next_for_dispatch(opctx).await? { - dispatched.push(event); + slog::trace!( + &opctx.log, + "dispatching webhook event..."; + "event_id" => ?event.id, + "event_class" => %event.event_class, + ); + + // Okay, we found an event that needs to be dispatched. Next, get + // list the webhook receivers subscribed to this event class and + // create delivery records for them. + let rxs = match self + .datastore + .webhook_rx_list_subscribed_to_event(&opctx, event.event_class) + .await + { + Ok(rxs) => rxs, + Err(error) => { + const MSG: &str = + "failed to list webhook receivers subscribed to event"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => ?event.id, + "event_class" => %event.event_class, + "error" => &error, + ); + status.errors.push(format!( + "{MSG} {} ({}): {error}", + event.id, event.event_class + )); + // We weren't able to find receivers for this event, so + // *don't* mark it as dispatched --- it's someone else's + // problem now. + continue; + } + }; + + let deliveries: Vec = rxs.into_iter().map(|(rx, sub)| { + slog::trace!(&opctx.log, "webhook receiver is subscribed to event"; + "rx_name" => %rx.name(), + "rx_id" => ?rx.id(), + "event_id" => ?event.id, + "event_class" => %event.event_class, + "glob" => ?sub.glob, + ); + WebhookDelivery::new(&event, &rx.id(), WebhookDeliveryTrigger::Dispatch) + }).collect(); + + let subscribed = if !deliveries.is_empty() { + let subscribed = deliveries.len(); + let dispatched = match self + .datastore + .webhook_delivery_create_batch(&opctx, deliveries) + .await + { + Ok(created) => created, + Err(error) => { + slog::error!(&opctx.log, "failed to insert webhook deliveries"; + "event_id" => ?event.id, + "event_class" => %event.event_class, + "error" => %error, + "num_subscribed" => ?subscribed, + ); + status.errors.push(format!("failed to insert {subscribed} webhook deliveries for event {} ({}): {error}", event.id, event.event_class)); + // We weren't able to create deliveries for this event, so + // *don't* mark it as dispatched. + continue; + } + }; + status.dispatched.push(WebhookDispatched { + event_id: event.id.into(), + subscribed, + dispatched, + }); + slog::debug!( + &opctx.log, + "dispatched webhook event"; + "event_id" => ?event.id, + "event_class" => %event.event_class, + "num_subscribed" => subscribed, + "num_dispatched" => dispatched, + ); + subscribed + } else { + slog::debug!( + &opctx.log, + "no webhook receivers subscribed to event"; + "event_id" => ?event.id, + "event_class" => %event.event_class, + ); + status.no_receivers.push(event.id.into()); + 0 + }; + + if let Err(error) = self + .datastore + .webhook_event_mark_dispatched( + &opctx, + &event.id.into(), + subscribed, + ) + .await + { + slog::error!(&opctx.log, "failed to mark webhook event as dispatched"; + "event_id" => ?event.id, + "event_class" => %event.event_class, + "error" => %error, + "num_subscribed" => subscribed, + ); + status.errors.push(format!("failed to mark webhook event {} ({}) as dispatched: {error}", event.id, event.event_class)); + } } Ok(()) } diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 780f0ac7c67..de88f84a49e 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -460,15 +460,18 @@ pub struct WebhookDispatcherStatus { /// The webhook events dispatched on this activation. pub dispatched: Vec, - /// Any error that occurred during activation. - pub error: Option, + /// Webhook events which did not have receivers. + pub no_receivers: Vec, + + /// Any errors that occurred during activation. + pub errors: Vec, } #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct WebhookDispatched { pub event_id: WebhookEventUuid, + pub subscribed: usize, pub dispatched: usize, - pub receivers_gone: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4effe1c16df..f70aeb62313 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4942,12 +4942,23 @@ on omicron.public.webhook_rx_subscription ( CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( id UUID PRIMARY KEY, time_created TIMESTAMPTZ NOT NULL, - -- Set when dispatch entries have been created for this event. - time_dispatched TIMESTAMPTZ, -- The class of event that this is. event_class omicron.public.webhook_event_class NOT NULL, -- Actual event data. The structure of this depends on the event class. - event JSONB NOT NULL + event JSONB NOT NULL, + + -- Set when dispatch entries have been created for this event. + time_dispatched TIMESTAMPTZ, + -- The number of receivers that this event was dispatched to. + num_dispatched INT8 NOT NULL, + + CONSTRAINT time_dispatched_set_if_dispatched CHECK ( + (num_dispatched = 0) OR (time_dispatched IS NOT NULL) + ), + + CONSTRAINT num_dispatched_is_positive CHECK ( + (num_dispatched >= 0) + ) ); -- Look up webhook events in need of dispatching. From 619edcf30319bdfb75d0ffa6d0f475f56eda963e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Feb 2025 15:26:33 -0800 Subject: [PATCH 055/168] unstub secret_create api --- .../db-queries/src/db/datastore/webhook_rx.rs | 20 +++++++++++++ nexus/src/app/webhook.rs | 26 +++++++++++++++++ nexus/src/external_api/http_entrypoints.rs | 28 +++++++++++-------- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index c47daae9780..d5569ab94db 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -410,6 +410,26 @@ impl DataStore { }) } + pub async fn webhook_rx_secret_create( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + secret: WebhookRxSecret, + ) -> CreateResult { + opctx.authorize(authz::Action::CreateChild, authz_rx).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + let secret = self.add_secret_on_conn(secret, &conn).await.map_err( + |e| match e { + TransactionError::CustomError(e) => e, + TransactionError::Database(e) => public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ), + }, + )?; + Ok(secret) + } + async fn add_secret_on_conn( &self, secret: WebhookRxSecret, diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 67a72e8e629..7ab434ea631 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -9,7 +9,9 @@ use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; use nexus_db_queries::db::model::WebhookReceiverConfig; +use nexus_db_queries::db::model::WebhookRxSecret; use nexus_types::external_api::params; +use nexus_types::external_api::views; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; use omicron_common::api::external::LookupResult; @@ -66,4 +68,28 @@ impl super::Nexus { Ok(event) } + + pub async fn webhook_receiver_secret_add( + &self, + opctx: &OpContext, + id: WebhookReceiverUuid, + secret: String, + ) -> Result { + let (authz_rx, _) = LookupPath::new(opctx, &self.datastore()) + .webhook_receiver_id(id) + .fetch() + .await?; + let secret = WebhookRxSecret::new(authz_rx.id(), secret); + let WebhookRxSecret { signature_id, .. } = self + .datastore() + .webhook_rx_secret_create(opctx, &authz_rx, secret) + .await?; + slog::info!( + &opctx.log, + "added secret to webhook receiver"; + "rx_id" => ?authz_rx.id(), + "secret_id" => ?signature_id, + ); + Ok(views::WebhookSecretId { id: signature_id.to_string() }) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 1a380da2cde..b1775c6446d 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7099,21 +7099,25 @@ impl NexusExternalApi for NexusExternalApiImpl { /// Add a secret to a webhook. async fn webhook_secrets_add( rqctx: RequestContext, - _path_params: Path, - _params: TypedBody, + path_params: Path, + params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + let handler = + async { + let nexus = &apictx.context.nexus; - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) - }; + let params::WebhookPath { webhook_id } = + path_params.into_inner(); + let params::WebhookSecret { secret } = params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let secret = nexus.webhook_receiver_secret_add(&opctx, + omicron_uuid_kinds::WebhookReceiverUuid::from_untyped_uuid( + webhook_id, + ),secret).await?; + Ok(HttpResponseCreated(secret)) + }; apictx .context .external_latencies From a2106fa2ac5da8a93a2d970851fcdbc39b6e506b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 14:39:27 -0800 Subject: [PATCH 056/168] [db-macros] support for children without names in `lookup_resource!` Presently, the `lookup_resource!` macro generates code that will not compile when a child resource has `lookup_by_name = false` in its `lookup_resource!` macro invocation. This is because the code for selecting a child resource by name given the parent is generated in the `lookup_resource!` invocation for the *parent* resource, which doesn't *know* whether the child has `lookup_by_name = true` or `lookup_by_name = false`. This means that this code is unconditionally generated, even when there is no name column for the child (i.e., it is an `Asset` rather than a `Resource`). While we currently have plenty of `lookup_resource!` invocations for `Asset`s, none of them are currently the child of another resource. While working on the webhook implementation, I wanted the `WebhookSecret` resource to be a child of the `WebhookReceiver` resource for authz purposes, but secrets do not have names. This resulted in the `lookup_resource!` macro generating code that wouldn't compile. This commit fixes that issue by changing the child selector functions from being generated by the parent's `lookup_resource!` invocation to instead being generated by the child's `lookup_resource!` invocation. This way, we can decide whether or not to generate the child-by-name selectors based on whether or not the child has `lookup_by_name = true` or not. Since all the `lookup_resource!` invocations are in the same file, it's alright for the child's `lookup_resource!` invocation to generate `impl` blocks for the parent as well. --- nexus/db-macros/src/lookup.rs | 96 +++++++++++++++++------------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index 3d3e93a8636..b6d599c542e 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -127,11 +127,6 @@ pub struct Config { /// [`authz_silo`, `authz_project`]) path_authz_names: Vec, - // Child resources - /// list of names of child resources, in the same form and with the same - /// assumptions as [`Input::name`] (i.e., typically PascalCase) - child_resources: Vec, - // Parent resource, if any /// Information about the parent resource, if any parent: Option, @@ -160,7 +155,6 @@ impl Config { .collect(); path_authz_names.push(resource.authz_name.clone()); - let child_resources = input.children; let parent = input.ancestors.last().map(|s| Resource::for_name(s)); let silo_restricted = !input.visible_outside_silo && input.ancestors.iter().any(|s| s == "Silo"); @@ -177,7 +171,6 @@ impl Config { path_types, path_authz_names, parent, - child_resources, lookup_by_name: input.lookup_by_name, primary_key_columns, soft_deletes: input.soft_deletes, @@ -221,7 +214,7 @@ pub fn lookup_resource( let resource_name = &config.resource.name; let the_basics = generate_struct(&config); let misc_helpers = generate_misc_helpers(&config); - let child_selectors = generate_child_selectors(&config); + let child_selector = generate_child_selector(&config); let lookup_methods = generate_lookup_methods(&config); let database_functions = generate_database_functions(&config); @@ -229,14 +222,14 @@ pub fn lookup_resource( #the_basics impl<'a> #resource_name<'a> { - #child_selectors - #lookup_methods #misc_helpers #database_functions } + + #child_selector }) } @@ -287,63 +280,70 @@ fn generate_struct(config: &Config) -> TokenStream { } } -/// Generates the child selectors for this resource +/// Generates the child selector for this resource's parent /// -/// For example, for the "Project" resource with child resources "Instance" and -/// "Disk", this will generate the `Project::instance_name()` and -/// `Project::disk_name()` functions. -fn generate_child_selectors(config: &Config) -> TokenStream { - let child_resource_types: Vec<_> = - config.child_resources.iter().map(|c| format_ident!("{}", c)).collect(); - let child_selector_fn_names: Vec<_> = config - .child_resources - .iter() - .map(|c| format_ident!("{}_name", heck::AsSnakeCase(c).to_string())) - .collect(); - let child_selector_fn_names_owned: Vec<_> = config - .child_resources - .iter() - .map(|c| { - format_ident!("{}_name_owned", heck::AsSnakeCase(c).to_string()) - }) - .collect(); - let child_selector_fn_docs: Vec<_> = config - .child_resources - .iter() - .map(|child| { - format!( - "Select a resource of type {} within this {}, \ - identified by its name", - child, config.resource.name, - ) - }) - .collect(); +/// For example, for the "Instance" resource with parent resource "Project", +/// this will generate the `Project::instance_name()` and +/// `Project::instance_name_owned()` functions. +/// +/// This is generated by the child resource codegen, rather than by the parent +/// resource codegen, since such functions are only generated for resources with +/// `lookup_by_name = true`. Whether or not this is enabled for the child +/// resource is not known when generating the parent resource code, so it's +/// performed by the child instead. +fn generate_child_selector(config: &Config) -> TokenStream { + // If this resource can only be looked up by ID, we don't need to generate + // child selectors on the parent resource. + if !config.lookup_by_name { + return quote! {}; + } + + // The child selector is generated for the parent resource type. If there + // isn't one, nothing to do here. + let Some(ref parent) = config.parent else { return quote! {} }; + + let parent_resource_type = &parent.name; + let child_selector_fn_name = format_ident!( + "{}_name", + heck::AsSnakeCase(config.resource.name.to_string()).to_string() + ); + + let child_selector_fn_name_owned = format_ident!( + "{}_name_owned", + heck::AsSnakeCase(config.resource.name.to_string()).to_string() + ); + let child_resource_type = &config.resource.name; + + let child_selector_fn_docs = format!( + "Select a resource of type {child_resource_type} within this \ + {parent_resource_type}, identified by its name", + ); quote! { - #( + impl<'a> #parent_resource_type<'a> { #[doc = #child_selector_fn_docs] - pub fn #child_selector_fn_names<'b, 'c>( + pub fn #child_selector_fn_name<'b, 'c>( self, name: &'b Name - ) -> #child_resource_types<'c> + ) -> #child_resource_type<'c> where 'a: 'c, 'b: 'c, { - #child_resource_types::Name(self, name) + #child_resource_type::Name(self, name) } #[doc = #child_selector_fn_docs] - pub fn #child_selector_fn_names_owned<'c>( + pub fn #child_selector_fn_name_owned<'c>( self, name: Name, - ) -> #child_resource_types<'c> + ) -> #child_resource_type<'c> where 'a: 'c, { - #child_resource_types::OwnedName(self, name) + #child_resource_type::OwnedName(self, name) } - )* + } } } From ce34509dad0fcbb386842ea0083592b70af73f8a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 14:39:44 -0800 Subject: [PATCH 057/168] wip --- common/src/api/external/mod.rs | 1 + nexus/auth/src/authz/api_resources.rs | 8 ++++ nexus/db-model/src/schema.rs | 11 +++-- nexus/db-model/src/webhook_rx.rs | 47 +++++++++++-------- .../db-queries/src/db/datastore/webhook_rx.rs | 29 ++++++------ nexus/db-queries/src/db/lookup.rs | 14 +++++- nexus/external-api/src/lib.rs | 13 ++++- nexus/src/app/webhook.rs | 35 ++++++++------ nexus/src/external_api/http_entrypoints.rs | 26 +++++----- nexus/types/src/external_api/params.rs | 10 +++- nexus/types/src/external_api/views.rs | 2 +- schema/crdb/add-webhooks/README.adoc | 2 +- schema/crdb/add-webhooks/up02.sql | 2 +- schema/crdb/add-webhooks/up03.sql | 2 +- schema/crdb/dbinit.sql | 17 +++---- 15 files changed, 134 insertions(+), 85 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index b3236060c8d..c1b85f5e7aa 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1046,6 +1046,7 @@ pub enum ResourceType { ProbeNetworkInterface, LldpLinkConfig, WebhookReceiver, + WebhookSecret, } // IDENTITY METADATA diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index b27ef3fa791..60117799331 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -1050,3 +1050,11 @@ authz_resource! { polar_snippet = FleetChild, } + +authz_resource! { + name = "WebhookSecret", + parent = "WebhookReceiver", + primary_key = { uuid_kind = WebhookSecretKind }, + roles_allowed = false, + polar_snippet = FleetChild, +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 091f8b42090..15349216ae7 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2145,11 +2145,12 @@ table! { } table! { - webhook_rx_secret (rx_id, signature_id) { + webhook_secret (id) { + id -> Uuid, + time_created -> Timestamptz, + time_modified -> Timestamptz, rx_id -> Uuid, - signature_id -> Uuid, secret -> Text, - time_created -> Timestamptz, time_deleted -> Nullable, } } @@ -2175,12 +2176,12 @@ table! { allow_tables_to_appear_in_same_query!( webhook_receiver, - webhook_rx_secret, + webhook_secret, webhook_rx_subscription, webhook_rx_event_glob ); joinable!(webhook_rx_subscription -> webhook_receiver (rx_id)); -joinable!(webhook_rx_secret -> webhook_receiver (rx_id)); +joinable!(webhook_secret -> webhook_receiver (rx_id)); joinable!(webhook_rx_event_glob -> webhook_receiver (rx_id)); table! { diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 57f27f24210..c8bc24cb9be 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -4,21 +4,20 @@ use crate::collection::DatastoreCollectionConfig; use crate::schema::{ - webhook_delivery, webhook_receiver, webhook_rx_event_glob, - webhook_rx_secret, webhook_rx_subscription, + webhook_receiver, webhook_rx_event_glob, webhook_rx_subscription, + webhook_secret, }; use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; use crate::EventClassParseError; use crate::Generation; -use crate::WebhookDelivery; use crate::WebhookEventClass; use chrono::{DateTime, Utc}; -use db_macros::Resource; +use db_macros::{Asset, Resource}; use nexus_types::external_api::views; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ - WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretKind, + GenericUuid, WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretKind, WebhookSecretUuid, }; use serde::{Deserialize, Serialize}; @@ -30,7 +29,7 @@ use uuid::Uuid; #[derive(Clone, Debug)] pub struct WebhookReceiverConfig { pub rx: WebhookReceiver, - pub secrets: Vec, + pub secrets: Vec, pub events: Vec, } @@ -41,8 +40,8 @@ impl TryFrom for views::Webhook { ) -> Result { let secrets = secrets .iter() - .map(|WebhookRxSecret { signature_id, .. }| { - views::WebhookSecretId { id: signature_id.to_string() } + .map(|WebhookSecret { identity, .. }| views::WebhookSecretId { + id: identity.id.into_untyped_uuid(), }) .collect(); let events = events @@ -92,11 +91,11 @@ pub struct WebhookReceiver { pub rcgen: Generation, } -impl DatastoreCollectionConfig for WebhookReceiver { +impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; type GenerationNumberColumn = webhook_receiver::dsl::rcgen; type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; - type CollectionIdColumn = webhook_rx_secret::dsl::rx_id; + type CollectionIdColumn = webhook_secret::dsl::rx_id; } impl DatastoreCollectionConfig for WebhookReceiver { @@ -114,24 +113,32 @@ impl DatastoreCollectionConfig for WebhookReceiver { } #[derive( - Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, + Clone, + Debug, + Queryable, + Selectable, + Insertable, + Serialize, + Deserialize, + Asset, )] -#[diesel(table_name = webhook_rx_secret)] -pub struct WebhookRxSecret { - pub rx_id: DbTypedUuid, - pub signature_id: DbTypedUuid, +#[asset(uuid_kind = WebhookSecretKind)] +#[diesel(table_name = webhook_secret)] +pub struct WebhookSecret { + #[diesel(embed)] + pub identity: WebhookSecretIdentity, + #[diesel(column_name = rx_id)] + pub webhook_receiver_id: DbTypedUuid, pub secret: String, - pub time_created: DateTime, pub time_deleted: Option>, } -impl WebhookRxSecret { +impl WebhookSecret { pub fn new(rx_id: WebhookReceiverUuid, secret: String) -> Self { Self { - rx_id: rx_id.into(), - signature_id: WebhookSecretUuid::new_v4().into(), + identity: WebhookSecretIdentity::new(WebhookSecretUuid::new_v4()), + webhook_receiver_id: rx_id.into(), secret, - time_created: Utc::now(), time_deleted: None, } } diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index d5569ab94db..4b3bb0b9a0d 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -19,15 +19,15 @@ use crate::db::model::WebhookReceiver; use crate::db::model::WebhookReceiverConfig; use crate::db::model::WebhookReceiverIdentity; use crate::db::model::WebhookRxEventGlob; -use crate::db::model::WebhookRxSecret; use crate::db::model::WebhookRxSubscription; +use crate::db::model::WebhookSecret; use crate::db::model::WebhookSubscriptionKind; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; -use crate::db::schema::webhook_rx_secret::dsl as secret_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; +use crate::db::schema::webhook_secret::dsl as secret_dsl; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; @@ -108,7 +108,7 @@ impl DataStore { for secret in secret_keys { let secret = self .add_secret_on_conn( - WebhookRxSecret::new(id, secret), + WebhookSecret::new(id, secret), &conn, ) .await @@ -141,8 +141,7 @@ impl DataStore { &self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver, - ) -> Result<(Vec, Vec), Error> - { + ) -> Result<(Vec, Vec), Error> { opctx.authorize(authz::Action::ListChildren, authz_rx).await?; let conn = self.pool_connection_authorized(opctx).await?; let subscriptions = @@ -385,7 +384,7 @@ impl DataStore { &self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver, - ) -> ListResultVec { + ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, authz_rx).await?; let conn = self.pool_connection_authorized(&opctx).await?; self.webhook_rx_secret_list_on_conn(authz_rx, &conn).await @@ -395,11 +394,11 @@ impl DataStore { &self, authz_rx: &authz::WebhookReceiver, conn: &async_bb8_diesel::Connection, - ) -> ListResultVec { - secret_dsl::webhook_rx_secret + ) -> ListResultVec { + secret_dsl::webhook_secret .filter(secret_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid())) .filter(secret_dsl::time_deleted.is_null()) - .select(WebhookRxSecret::as_select()) + .select(WebhookSecret::as_select()) .load_async(conn) .await .map_err(|e| { @@ -414,8 +413,8 @@ impl DataStore { &self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver, - secret: WebhookRxSecret, - ) -> CreateResult { + secret: WebhookSecret, + ) -> CreateResult { opctx.authorize(authz::Action::CreateChild, authz_rx).await?; let conn = self.pool_connection_authorized(&opctx).await?; let secret = self.add_secret_on_conn(secret, &conn).await.map_err( @@ -432,13 +431,13 @@ impl DataStore { async fn add_secret_on_conn( &self, - secret: WebhookRxSecret, + secret: WebhookSecret, conn: &async_bb8_diesel::Connection, - ) -> Result> { + ) -> Result> { let rx_id = secret.rx_id; - let secret: WebhookRxSecret = WebhookReceiver::insert_resource( + let secret: WebhookSecret = WebhookReceiver::insert_resource( rx_id.into_untyped_uuid(), - diesel::insert_into(secret_dsl::webhook_rx_secret).values(secret), + diesel::insert_into(secret_dsl::webhook_secret).values(secret), ) .insert_and_get_result_async(conn) .await diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 518150458d6..25a262414bb 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -27,6 +27,7 @@ use omicron_uuid_kinds::TufArtifactKind; use omicron_uuid_kinds::TufRepoKind; use omicron_uuid_kinds::TypedUuid; use omicron_uuid_kinds::WebhookReceiverUuid; +use omicron_uuid_kinds::WebhookSecretUuid; use uuid::Uuid; /// Look up an API resource in the database @@ -961,7 +962,7 @@ lookup_resource! { lookup_resource! { name = "WebhookReceiver", ancestors = [], - children = [], + children = ["WebhookSecret"], lookup_by_name = false, soft_deletes = true, primary_key_columns = [ @@ -969,6 +970,17 @@ lookup_resource! { ] } +lookup_resource! { + name = "WebhookSecret", + ancestors = ["WebhookReceiver"], + children = [], + lookup_by_name = false, + soft_deletes = true, + primary_key_columns = [ + { column_name = "id", uuid_kind = WebhookSecretKind } + ] +} + // Helpers for unifying the interfaces around images pub enum ImageLookup<'a> { diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 1f4aef61a63..60d077277f5 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3340,7 +3340,18 @@ pub trait NexusExternalApi { async fn webhook_secrets_add( rqctx: RequestContext, path_params: Path, - params: TypedBody, + params: TypedBody, + ) -> Result, HttpError>; + + /// Delete a secret associated with a webhook receiver by ID. + #[endpoint { + method = DELETE, + path = "/experimental/v1/webhooks/{webhook_id}/secrets/{secret_id}", + tags = ["system/webhooks"], + }] + async fn webhook_secrets_delete( + rqctx: RequestContext, + path_params: Path, ) -> Result, HttpError>; /// List delivery attempts to a webhook receiver. diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 7ab434ea631..3ed5e8208dc 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -4,12 +4,13 @@ //! Webhooks +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; use nexus_db_queries::db::model::WebhookReceiverConfig; -use nexus_db_queries::db::model::WebhookRxSecret; +use nexus_db_queries::db::model::WebhookSecret; use nexus_types::external_api::params; use nexus_types::external_api::views; use omicron_common::api::external::CreateResult; @@ -19,15 +20,22 @@ use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; impl super::Nexus { + pub fn webhook_receiver_lookup<'a>( + &'a self, + opctx: &'a OpContext, + webhook_selector: params::WebhookPath, + ) -> LookupResult> { + Ok(LookupPath::new(opctx, self.datastore()).webhook_receiver_id( + WebhookReceiverUuid::from_untyped_uuid(webhook_selector.id), + )) + } + pub async fn webhook_receiver_config_fetch( &self, opctx: &OpContext, - id: WebhookReceiverUuid, + rx: lookup::WebhookReceiver<'a>, ) -> LookupResult { - let (authz_rx, rx) = LookupPath::new(opctx, &self.datastore()) - .webhook_receiver_id(id) - .fetch() - .await?; + let (authz_rx, rx) = rx.fetch().await?; let (events, secrets) = self.datastore().webhook_rx_config_fetch(opctx, &authz_rx).await?; Ok(WebhookReceiverConfig { rx, secrets, events }) @@ -72,15 +80,12 @@ impl super::Nexus { pub async fn webhook_receiver_secret_add( &self, opctx: &OpContext, - id: WebhookReceiverUuid, + rx: lookup::WebhookReceiver<'_>, secret: String, ) -> Result { - let (authz_rx, _) = LookupPath::new(opctx, &self.datastore()) - .webhook_receiver_id(id) - .fetch() - .await?; - let secret = WebhookRxSecret::new(authz_rx.id(), secret); - let WebhookRxSecret { signature_id, .. } = self + let authz_rx = rx.lookup_for(authz::Action::CreateChild).await?; + let secret = WebhookSecret::new(authz_rx.id(), secret); + let WebhookSecret { id, .. } = self .datastore() .webhook_rx_secret_create(opctx, &authz_rx, secret) .await?; @@ -88,8 +93,8 @@ impl super::Nexus { &opctx.log, "added secret to webhook receiver"; "rx_id" => ?authz_rx.id(), - "secret_id" => ?signature_id, + "secret_id" => ?id, ); - Ok(views::WebhookSecretId { id: signature_id.to_string() }) + Ok(views::WebhookSecretId { id: id.into_untyped_uuid() }) } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index b1775c6446d..05bc5d54a24 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7103,21 +7103,17 @@ impl NexusExternalApi for NexusExternalApiImpl { params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); - let handler = - async { - let nexus = &apictx.context.nexus; - - let params::WebhookPath { webhook_id } = - path_params.into_inner(); - let params::WebhookSecret { secret } = params.into_inner(); - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - let secret = nexus.webhook_receiver_secret_add(&opctx, - omicron_uuid_kinds::WebhookReceiverUuid::from_untyped_uuid( - webhook_id, - ),secret).await?; - Ok(HttpResponseCreated(secret)) - }; + let handler = async { + let nexus = &apictx.context.nexus; + let params::WebhookSecret { secret } = params.into_inner(); + let webhook_selector = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let secret = + nexus.webhook_receiver_secret_add(&opctx, rx, secret).await?; + Ok(HttpResponseCreated(secret)) + }; apictx .context .external_latencies diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index d8e8c1dc852..7f786e88653 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2366,10 +2366,18 @@ pub struct WebhookUpdate { } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct WebhookSecret { +pub struct WebhookSecretCreate { pub secret: String, } +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookSecretSelector { + /// ID of the webhook receiver. + pub webhook_id: Uuid, + /// ID of the secret. + pub secret_id: Uuid, + +} #[derive(Deserialize, JsonSchema)] pub struct WebhookDeliveryPath { pub webhook_id: Uuid, diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 7727628803a..450c70ceb81 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1070,7 +1070,7 @@ pub struct WebhookSecrets { /// The public ID of a secret key assigned to a webhook. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookSecretId { - pub id: String, + pub id: Uuid, } /// A delivery attempt for a webhook event. diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc index ffcb1538a24..40e5f261bad 100644 --- a/schema/crdb/add-webhooks/README.adoc +++ b/schema/crdb/add-webhooks/README.adoc @@ -10,7 +10,7 @@ The individual transactions in this upgrade do the following: ** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores the receiver endpoints that receive webhook events. ** *Receiver secrets*: -*** `up02.sql` creates the `omicron.public.webhook_rx_secret` table, which +*** `up02.sql` creates the `omicron.public.webhook_secret` table, which associates webhook receivers with secret keys and their IDs. *** `up03.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, for looking up all secrets associated with a receiver. diff --git a/schema/crdb/add-webhooks/up02.sql b/schema/crdb/add-webhooks/up02.sql index df945f4299e..c1c9e47b800 100644 --- a/schema/crdb/add-webhooks/up02.sql +++ b/schema/crdb/add-webhooks/up02.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, diff --git a/schema/crdb/add-webhooks/up03.sql b/schema/crdb/add-webhooks/up03.sql index 5a799088577..ab263e8705e 100644 --- a/schema/crdb/add-webhooks/up03.sql +++ b/schema/crdb/add-webhooks/up03.sql @@ -1,5 +1,5 @@ CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx -ON omicron.public.webhook_rx_secret ( +ON omicron.public.webhook_secret ( rx_id ) WHERE time_deleted IS NULL; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f70aeb62313..b9b6953018b 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4829,22 +4829,23 @@ ON omicron.public.webhook_receiver (id) WHERE time_deleted IS NULL; -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( + -- ID of this secret. + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- N.B. that this will always be equal to `time_created` for secrets, as + -- they are never modified once created. + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- ID of this secret. - signature_id UUID NOT NULL, -- Secret value. secret STRING(512) NOT NULL, - time_created TIMESTAMPTZ NOT NULL, - time_deleted TIMESTAMPTZ, - - PRIMARY KEY (signature_id, rx_id) ); CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx -ON omicron.public.webhook_rx_secret ( +ON omicron.public.webhook_secret ( rx_id ) WHERE time_deleted IS NULL; From fe518f626454da8f12f1c310e75b9793218d493a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 14:58:41 -0800 Subject: [PATCH 058/168] [db-macros] handle DbTypedUuid in child resource parent lookup --- nexus/db-macros/src/lookup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index b6d599c542e..82e5cfe21cf 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -791,7 +791,7 @@ fn generate_database_functions(config: &Config) -> TokenStream { quote! { let (#(#ancestors_authz_names,)* _) = #parent_resource_name::lookup_by_id_no_authz( - opctx, datastore, &db_row.#parent_id + opctx, datastore, &db_row.#parent_id.into() ).await?; }, quote! { .filter(dsl::#parent_id.eq(#parent_authz_name.id())) }, From 63f142c8945084f6c5cfa7ae399988d55eff6564 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 15:29:33 -0800 Subject: [PATCH 059/168] agh --- nexus/db-queries/src/db/datastore/webhook_rx.rs | 2 +- nexus/src/app/webhook.rs | 6 ++++-- nexus/src/external_api/http_entrypoints.rs | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 4b3bb0b9a0d..7df81825c3e 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -434,7 +434,7 @@ impl DataStore { secret: WebhookSecret, conn: &async_bb8_diesel::Connection, ) -> Result> { - let rx_id = secret.rx_id; + let rx_id = secret.webhook_receiver_id; let secret: WebhookSecret = WebhookReceiver::insert_resource( rx_id.into_untyped_uuid(), diesel::insert_into(secret_dsl::webhook_secret).values(secret), diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 3ed5e8208dc..a6ca3473aff 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -6,6 +6,7 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; @@ -16,6 +17,7 @@ use nexus_types::external_api::views; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; use omicron_common::api::external::LookupResult; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; @@ -26,14 +28,14 @@ impl super::Nexus { webhook_selector: params::WebhookPath, ) -> LookupResult> { Ok(LookupPath::new(opctx, self.datastore()).webhook_receiver_id( - WebhookReceiverUuid::from_untyped_uuid(webhook_selector.id), + WebhookReceiverUuid::from_untyped_uuid(webhook_selector.webhook_id), )) } pub async fn webhook_receiver_config_fetch( &self, opctx: &OpContext, - rx: lookup::WebhookReceiver<'a>, + rx: lookup::WebhookReceiver<'_>, ) -> LookupResult { let (authz_rx, rx) = rx.fetch().await?; let (events, secrets) = diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 05bc5d54a24..06850be141d 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7100,12 +7100,12 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_secrets_add( rqctx: RequestContext, path_params: Path, - params: TypedBody, + params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; - let params::WebhookSecret { secret } = params.into_inner(); + let params::WebhookSecretCreate { secret } = params.into_inner(); let webhook_selector = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; From 05c4d3ad918bc6bbd736ce2f376e68e5c65308b5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 15:33:40 -0800 Subject: [PATCH 060/168] aghh --- nexus/external-api/src/lib.rs | 2 +- nexus/src/app/webhook.rs | 9 ++++---- nexus/src/external_api/http_entrypoints.rs | 27 ++++++++++++++++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 60d077277f5..df946510acb 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3352,7 +3352,7 @@ pub trait NexusExternalApi { async fn webhook_secrets_delete( rqctx: RequestContext, path_params: Path, - ) -> Result, HttpError>; + ) -> Result; /// List delivery attempts to a webhook receiver. #[endpoint { diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index a6ca3473aff..b4491f76b11 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -85,18 +85,19 @@ impl super::Nexus { rx: lookup::WebhookReceiver<'_>, secret: String, ) -> Result { - let authz_rx = rx.lookup_for(authz::Action::CreateChild).await?; + let (authz_rx,) = rx.lookup_for(authz::Action::CreateChild).await?; let secret = WebhookSecret::new(authz_rx.id(), secret); - let WebhookSecret { id, .. } = self + let WebhookSecret { identity, .. } = self .datastore() .webhook_rx_secret_create(opctx, &authz_rx, secret) .await?; + let secret_id = identity.id(); slog::info!( &opctx.log, "added secret to webhook receiver"; "rx_id" => ?authz_rx.id(), - "secret_id" => ?id, + "secret_id" => ?secret_id, ); - Ok(views::WebhookSecretId { id: id.into_untyped_uuid() }) + Ok(views::WebhookSecretId { id: secret_id.into_untyped_uuid() }) } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 06850be141d..e4309df8cf4 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6980,13 +6980,12 @@ impl NexusExternalApi for NexusExternalApiImpl { path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); - let params::WebhookPath { webhook_id } = path_params.into_inner(); let handler = async { let nexus = &apictx.context.nexus; let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - + let webhook = path_params.into_inner(); let webhook = nexus .webhook_receiver_config_fetch( &opctx, @@ -7121,6 +7120,30 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + /// Delete a secret from a webhook receiver. + async fn webhook_secrets_delete( + rqctx: RequestContext, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn webhook_delivery_list( rqctx: RequestContext, _path_params: Path, From a6d81cfc831daafbd5d1bf1eef99e2bd153de73c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 15:36:21 -0800 Subject: [PATCH 061/168] aghhh --- nexus/db-model/src/webhook_rx.rs | 3 +-- nexus/db-queries/src/db/lookup.rs | 1 - nexus/src/app/webhook.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 13 ++++--------- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index c8bc24cb9be..f5f2f650b38 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -17,8 +17,7 @@ use db_macros::{Asset, Resource}; use nexus_types::external_api::views; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ - GenericUuid, WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretKind, - WebhookSecretUuid, + GenericUuid, WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretUuid, }; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 25a262414bb..24feb5ea58e 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -27,7 +27,6 @@ use omicron_uuid_kinds::TufArtifactKind; use omicron_uuid_kinds::TufRepoKind; use omicron_uuid_kinds::TypedUuid; use omicron_uuid_kinds::WebhookReceiverUuid; -use omicron_uuid_kinds::WebhookSecretUuid; use uuid::Uuid; /// Look up an API resource in the database diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index b4491f76b11..924d6dcb50c 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -91,7 +91,7 @@ impl super::Nexus { .datastore() .webhook_rx_secret_create(opctx, &authz_rx, secret) .await?; - let secret_id = identity.id(); + let secret_id = identity.id; slog::info!( &opctx.log, "added secret to webhook receiver"; diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e4309df8cf4..49a9d933f93 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6985,15 +6985,10 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let webhook = path_params.into_inner(); - let webhook = nexus - .webhook_receiver_config_fetch( - &opctx, - omicron_uuid_kinds::WebhookReceiverUuid::from_untyped_uuid( - webhook_id, - ), - ) - .await?; + let webhook_selector = path_params.into_inner(); + let rx = self.webhook_receiver_lookup(&opctx, webhook_selector)?; + let webhook = + nexus.webhook_receiver_config_fetch(&opctx, rx).await?; Ok(HttpResponseOk(views::Webhook::try_from(webhook)?)) }; apictx From f93e8d496ddb8566f8b3447daa3a14b13c635d7b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Feb 2025 15:39:03 -0800 Subject: [PATCH 062/168] aghhhh --- nexus/src/external_api/http_entrypoints.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 49a9d933f93..2428f3ddf9e 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6986,7 +6986,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let webhook_selector = path_params.into_inner(); - let rx = self.webhook_receiver_lookup(&opctx, webhook_selector)?; + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; let webhook = nexus.webhook_receiver_config_fetch(&opctx, rx).await?; Ok(HttpResponseOk(views::Webhook::try_from(webhook)?)) From 7822fb3be41339681d0c2379653835dbc7f24328 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Feb 2025 13:28:16 -0800 Subject: [PATCH 063/168] deliverator: look up RX secrets, error handling tweaks --- .../background/tasks/webhook_deliverator.rs | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index db7df122f6c..6f8eb211afb 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -1,7 +1,9 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::app::authz; use crate::app::background::BackgroundTask; +use crate::app::db::lookup::LookupPath; use chrono::{TimeDelta, Utc}; use futures::future::BoxFuture; use http::HeaderName; @@ -155,7 +157,21 @@ impl WebhookDeliverator { let deliverator = self.clone(); tasks.spawn(async move { let rx_id = rx.id(); - let status = deliverator.rx_deliver(&opctx, rx).await; + let status = match deliverator.rx_deliver(&opctx, rx).await { + Ok(status) => status, + Err(e) => { + slog::error!( + &opctx.log, + "failed to deliver webhook events to a receiver"; + "rx_id" => ?rx_id, + "error" => %e, + ); + WebhookRxDeliveryStatus { + error: Some(e.to_string()), + ..Default::default() + } + } + }; (rx_id, status) }); } @@ -176,8 +192,23 @@ impl WebhookDeliverator { &self, opctx: &OpContext, rx: WebhookReceiver, - ) -> WebhookRxDeliveryStatus { - let deliveries = match self + ) -> Result { + // First, look up the receiver's secrets and any deliveries for that + // receiver. If any of these lookups fail, bail out, as we can't + // meaningfully deliver any events to a receiver if we don't know what + // they are or how to sign them. + let (authz_rx,) = LookupPath::new(opctx, &self.datastore) + .webhook_receiver_id(rx.id()) + .lookup_for(authz::Action::ListChildren) + .await + .map_err(|e| anyhow::anyhow!("could not look up receiver: {e}"))?; + let secrets = self + .datastore + .webhook_rx_secret_list(opctx, &authz_rx) + .await + .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; + anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); + let deliveries = self .datastore .webhook_rx_delivery_list_ready( &opctx, @@ -185,21 +216,12 @@ impl WebhookDeliverator { self.lease_timeout, ) .await - { - Err(e) => { - const MSG: &str = "failed to list ready deliveries"; - slog::error!( - &opctx.log, - "{MSG}"; - "error" => %e, - ); - return WebhookRxDeliveryStatus { - error: Some(format!("{MSG}: {e}")), - ..Default::default() - }; - } - Ok(deliveries) => deliveries, - }; + .map_err(|e| { + anyhow::anyhow!("could not list ready deliveries: {e}") + })?; + + // Okay, we got everything we need in order to deliver events to this + // receiver. Now, let's actually...do that. let mut delivery_status = WebhookRxDeliveryStatus { ready: deliveries.len(), ..Default::default() @@ -225,7 +247,7 @@ impl WebhookDeliverator { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery_id, - "attempt" => attempt, + "attempt" => ?attempt, ); } Ok(DeliveryAttemptState::AlreadyCompleted(time)) => { @@ -436,7 +458,7 @@ impl WebhookDeliverator { // TODO(eliza): if no events were sent, do a probe... - delivery_status + Ok(delivery_status) } } From 98085e95b0cc669607d1bf7baca45c3c270a57a9 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Feb 2025 14:28:20 -0800 Subject: [PATCH 064/168] oops i forgot to register the authz resources --- nexus/auth/src/authz/api_resources.rs | 1 - nexus/auth/src/authz/oso_generic.rs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 60117799331..fa2e2e67706 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -1048,7 +1048,6 @@ authz_resource! { primary_key = { uuid_kind = WebhookReceiverKind }, roles_allowed = false, polar_snippet = FleetChild, - } authz_resource! { diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 321bb98b1c6..6e5e982b06b 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -160,6 +160,8 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { Sled::init(), TufRepo::init(), TufArtifact::init(), + WebhookReceiver::init(), + WebhookSecret::init(), Zpool::init(), Service::init(), UserBuiltin::init(), From a4952d9beaac152a008a144dd9c8f9d870002e13 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Feb 2025 14:28:29 -0800 Subject: [PATCH 065/168] goddamn sql trailing comma nonsense --- schema/crdb/dbinit.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index b9b6953018b..6f8f65c9cea 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4836,12 +4836,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( -- N.B. that this will always be equal to `time_created` for secrets, as -- they are never modified once created. time_modified TIMESTAMPTZ NOT NULL, - time_deleted TIMESTAMPTZ + time_deleted TIMESTAMPTZ, -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, -- Secret value. - secret STRING(512) NOT NULL, + secret STRING(512) NOT NULL ); CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx From 2d13b4e33cef56a79ba4c5cf98fc28716e2db9f5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Feb 2025 14:33:48 -0800 Subject: [PATCH 066/168] sign webhook payloads --- Cargo.lock | 1 + Cargo.toml | 1 + nexus/Cargo.toml | 1 + .../background/tasks/webhook_deliverator.rs | 71 +++++++++++++++++-- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b41eec8aa1..9916a5002dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7025,6 +7025,7 @@ dependencies = [ "headers", "hex", "hickory-resolver", + "hmac", "http", "http-body-util", "httpmock", diff --git a/Cargo.toml b/Cargo.toml index b0505ae695e..6c0bd5a7a64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -421,6 +421,7 @@ hickory-resolver = "0.24.2" hickory-server = "0.24.2" highway = "1.2.0" hkdf = "0.12.4" +hmac = "0.12.1" http = "1.2.0" http-body = "1.0.1" http-body-util = "0.1.2" diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index c3da4d1aae2..74ac7421e4c 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -39,6 +39,7 @@ gateway-client.workspace = true headers.workspace = true hex.workspace = true hickory-resolver.workspace = true +hmac.workspace = true http.workspace = true http-body-util.workspace = true hyper.workspace = true diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 6f8eb211afb..c73c7da03c5 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -6,13 +6,14 @@ use crate::app::background::BackgroundTask; use crate::app::db::lookup::LookupPath; use chrono::{TimeDelta, Utc}; use futures::future::BoxFuture; +use hmac::{Hmac, Mac}; use http::HeaderName; use http::HeaderValue; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; use nexus_db_queries::db::model::{ SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookEventClass, - WebhookReceiver, + WebhookReceiver, WebhookSecret, }; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; @@ -25,11 +26,11 @@ use omicron_uuid_kinds::{ GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid, WebhookEventUuid, WebhookReceiverUuid, }; +use sha2::Sha256; use std::num::NonZeroU32; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::task::JoinSet; - // The Deliverator belongs to an elite order, a hallowed sub-category. He's got // esprit up to here. Right now he is preparing to carry out his third mission // of the night. His uniform is black as activated charcoal, filtering the very @@ -202,11 +203,19 @@ impl WebhookDeliverator { .lookup_for(authz::Action::ListChildren) .await .map_err(|e| anyhow::anyhow!("could not look up receiver: {e}"))?; - let secrets = self + let mut secrets = self .datastore .webhook_rx_secret_list(opctx, &authz_rx) .await - .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; + .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))? + .into_iter() + .map(|WebhookSecret { identity, secret, .. }| { + let mac = Hmac::::new_from_slice(secret.as_bytes()) + .expect("HMAC key can be any size; this should never fail"); + (identity.id, mac) + }) + .collect::>(); + anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); let deliveries = self .datastore @@ -304,21 +313,69 @@ impl WebhookDeliverator { sent_at: &sent_at, }, }; - // TODO(eliza): signatures! - let request = self + // N.B. that we serialize the body "ourselves" rather than just + // passing it to `RequestBuilder::json` because we must access + // the serialized body in order to calculate HMAC signatures. + // This means we have to add the `Content-Type` ourselves below. + let body = match serde_json::to_vec(&payload) { + Ok(body) => body, + Err(e) => { + const MSG: &'static str = + "event payload could not be serialized"; + slog::error!( + &opctx.log, + "webhook {MSG}"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery_id, + "error" => %e, + ); + + // This really shouldn't happen --- we expect the event + // payload will always be valid JSON. We could *probably* + // just panic here unconditionally, but it seems nicer to + // try and do the other events. But, if there's ever a bug + // that breaks serialization for *all* webhook payloads, + // I'd like the tests to fail in a more obvious way than + // eventually timing out waiting for the event to be + // delivered ... + if cfg!(debug_assertions) { + panic!("{MSG}:{e}\npayload: {payload:#?}"); + } + delivery_status + .delivery_errors + .insert(delivery_id, format!("{MSG}: {e}")); + continue; + } + }; + let mut request = self .client .post(&rx.endpoint) .header(HDR_RX_ID, hdr_rx_id.clone()) .header(HDR_DELIVERY_ID, delivery_id.to_string()) .header(HDR_EVENT_ID, delivery.event_id.to_string()) .header(HDR_EVENT_CLASS, event_class.to_string()) - .json(&payload) + .header(http::header::CONTENT_TYPE, "application/json"); + + // For each secret assigned to this webhook, calculate the HMAC and add a signature header. + for (secret_id, mac) in &mut secrets { + mac.update(&body); + let sig_bytes = mac.finalize_reset().into_bytes(); + let sig = hex::encode(&sig_bytes[..]); + request = request.header( + HDR_SIG, + format!("a=sha256&id={secret_id}&s={sig}"), + ); + } + let request = request + .body(body) // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to // each webhook delivery request. // // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure .timeout(Duration::from_secs(30)) .build(); + let request = match request { // We couldn't construct a request for some reason! This one's // our fault, so don't penalize the receiver for it. From e53865bef2bd88b8d5f265778c422304c70a9845 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Feb 2025 10:21:51 -0800 Subject: [PATCH 067/168] verify signatures in webhook integration tests --- nexus/tests/integration_tests/webhooks.rs | 96 +++++++++++++++++------ 1 file changed, 74 insertions(+), 22 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 289c8dbe355..4811aea5ad7 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -4,6 +4,7 @@ //! Webhooks +use hmac::{Hmac, Mac}; use httpmock::prelude::*; use nexus_db_model::WebhookEventClass; use nexus_db_queries::context::OpContext; @@ -13,6 +14,7 @@ use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::{params, views}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_uuid_kinds::WebhookEventUuid; +use sha2::Sha256; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -29,6 +31,54 @@ async fn webhook_create( .await } +fn is_valid_for_webhook( + webhook: &views::Webhook, +) -> impl FnOnce(httpmock::When) -> httpmock::When { + let path = webhook.endpoint.path().to_string(); + let id = webhook.id.to_string(); + move |when| { + when.path(path) + .header("x-oxide-webhook-id", id) + .header_exists("x-oxide-delivery-id") + .header_exists("x-oxide-signature") + .header("content-type", "application/json") + } +} + +fn signature_verifies( + secret_id: String, + secret: Vec, +) -> impl Fn(&HttpMockRequest) -> bool { + move |req| { + let hdrs = req.headers(); + let Some(sig_hdr) = hdrs.get_all("x-oxide-signature").iter().filter_map(|hdr| { + // Signature header format: + // a={algorithm}&id={secret_id}&s={signature} + let hdr = dbg!(hdr.to_str()).expect("all x-oxide-signature headers should be valid utf-8"); + // Strip the expected algorithm part. Note that we only support + // SHA256 for now. Panic if this is invalid. + let hdr = hdr.strip_prefix("a=sha256").expect("all x-oxide-signature headers should have the SHA256 algorithm"); + // Strip the leading `&id=` for the ID part, panicking if this + // is not found. + let hdr = hdr.strip_prefix("&id=").expect("all x-oxide-signature headers should have a secret ID ID part"); + // If the ID isn't the one we're looking for, we want to keep + // going, so just return `None` here + let hdr = hdr.strip_prefix(secret_id.as_str())?; + // Finally, extract the signature part by stripping the &s= + // prefix. + hdr.strip_prefix("&s=") + }).next() else { + panic!("no x-oxide-signature header for secret with ID {secret_id} found"); + }; + let sig_bytes = hex::decode(sig_hdr) + .expect("x-oxide-signature signature value should be a hex string"); + let mut mac = Hmac::::new_from_slice(&secret[..]) + .expect("HMAC secrets can be any length"); + mac.update(req.body().as_ref()); + mac.verify_slice(&sig_bytes).is_ok() + } +} + #[nexus_test] async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let nexus = cptestctx.server.server_context().nexus.clone(); @@ -61,29 +111,31 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { .await; dbg!(&webhook); - let mock = server - .mock_async(|when, then| { - let body = serde_json::json!({ - "event_class": "test.foo", - "event_id": id, - "data": { - "hello_world": true, - } + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + let body = serde_json::json!({ + "event_class": "test.foo", + "event_id": id, + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id.to_string(), + "my cool secret".as_bytes().to_vec(), + )) + .json_body_includes(body) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()); + then.status(200); }) - .to_string(); - when.method(POST) - .path("/webhooks") - .json_body_includes(body) - .header("x-oxide-event-class", "test.foo") - .header("x-oxide-event-id", id.to_string()) - .header("x-oxide-webhook-id", webhook.id.to_string()) - .header("content-type", "application/json") - // This should be present, but we don't know what its' value is - // going to be, so just assert that it's there. - .header_exists("x-oxide-delivery-id"); - then.status(200); - }) - .await; + .await + }; // Publish an event let event = nexus From ddc648cac8d95aa59ad01609b3ad0d111278e0cf Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Feb 2025 11:43:33 -0800 Subject: [PATCH 068/168] test receivers with multiple secrets --- nexus/tests/integration_tests/webhooks.rs | 134 +++++++++++++++++++++- 1 file changed, 129 insertions(+), 5 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 4811aea5ad7..668907b21f6 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -14,7 +14,9 @@ use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::{params, views}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_uuid_kinds::WebhookEventUuid; +use omicron_uuid_kinds::WebhookReceiverUuid; use sha2::Sha256; +use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -31,6 +33,22 @@ async fn webhook_create( .await } +async fn secret_add( + ctx: &ControlPlaneTestContext, + webhook_id: WebhookReceiverUuid, + params: ¶ms::WebhookSecretCreate, +) -> views::WebhookSecretId { + resource_helpers::object_create::< + params::WebhookSecretCreate, + views::WebhookSecretId, + >( + &ctx.external_client, + &format!("/experimental/v1/webhooks/{webhook_id}/secrets"), + params, + ) + .await +} + fn is_valid_for_webhook( webhook: &views::Webhook, ) -> impl FnOnce(httpmock::When) -> httpmock::When { @@ -46,9 +64,10 @@ fn is_valid_for_webhook( } fn signature_verifies( - secret_id: String, + secret_id: Uuid, secret: Vec, ) -> impl Fn(&HttpMockRequest) -> bool { + let secret_id = secret_id.to_string(); move |req| { let hdrs = req.headers(); let Some(sig_hdr) = hdrs.get_all("x-oxide-signature").iter().filter_map(|hdr| { @@ -124,14 +143,14 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { }) .to_string(); when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()) .and(is_valid_for_webhook(&webhook)) .is_true(signature_verifies( - webhook.secrets[0].id.to_string(), + webhook.secrets[0].id, "my cool secret".as_bytes().to_vec(), )) - .json_body_includes(body) - .header("x-oxide-event-class", "test.foo") - .header("x-oxide-event-id", id.to_string()); + .json_body_includes(body); then.status(200); }) .await @@ -156,3 +175,108 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { mock.assert_async().await; } + +#[nexus_test] +async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let server = httpmock::MockServer::start_async().await; + + let id = WebhookEventUuid::new_v4(); + let endpoint = + server.url("/webhooks").parse().expect("this should be a valid URL"); + + const SECRET1: &str = "it's an older code, sir, but it checks out"; + const SECRET2: &str = "Joshua"; + const SECRET3: &str = "Setec Astronomy"; + + // Create a webhook receiver. + let webhook = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "my-great-webhook".parse().unwrap(), + description: String::from("my great webhook"), + }, + endpoint, + secrets: vec![SECRET1.to_string()], + events: vec!["test.foo".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(&webhook); + + let secret1_id = webhook.secrets[0].id; + + // Add a second secret to the webhook receiver. + let secret2_id = dbg!( + secret_add( + &cptestctx, + webhook.id, + ¶ms::WebhookSecretCreate { secret: SECRET2.to_string() }, + ) + .await + ) + .id; + + // And a third one, just for kicks. + let secret3_id = dbg!( + secret_add( + &cptestctx, + webhook.id, + ¶ms::WebhookSecretCreate { secret: SECRET3.to_string() }, + ) + .await + ) + .id; + + let mock = server + .mock_async(|when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()) + .and(is_valid_for_webhook(&webhook)) + // There should be a signature header present for all three + // secrets, and they should all verify the contents of the + // webhook request. + .is_true(signature_verifies( + secret1_id, + SECRET1.as_bytes().to_vec(), + )) + .is_true(signature_verifies( + secret2_id, + SECRET2.as_bytes().to_vec(), + )) + .is_true(signature_verifies( + secret3_id, + SECRET3.as_bytes().to_vec(), + )); + then.status(200); + }) + .await; + + // Publish an event + let event = nexus + .webhook_event_publish( + &opctx, + id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello_world": true}), + ) + .await + .expect("event should be published successfully"); + dbg!(event); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + mock.assert_async().await; +} From e537872c13669ceb320152cb730f111543a5bc48 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Feb 2025 13:38:55 -0800 Subject: [PATCH 069/168] workaround for alexliesenfeld/httpmock#119 --- nexus/tests/integration_tests/webhooks.rs | 54 ++++++++++++++--------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 668907b21f6..b951683f246 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -69,26 +69,40 @@ fn signature_verifies( ) -> impl Fn(&HttpMockRequest) -> bool { let secret_id = secret_id.to_string(); move |req| { - let hdrs = req.headers(); - let Some(sig_hdr) = hdrs.get_all("x-oxide-signature").iter().filter_map(|hdr| { - // Signature header format: - // a={algorithm}&id={secret_id}&s={signature} - let hdr = dbg!(hdr.to_str()).expect("all x-oxide-signature headers should be valid utf-8"); - // Strip the expected algorithm part. Note that we only support - // SHA256 for now. Panic if this is invalid. - let hdr = hdr.strip_prefix("a=sha256").expect("all x-oxide-signature headers should have the SHA256 algorithm"); - // Strip the leading `&id=` for the ID part, panicking if this - // is not found. - let hdr = hdr.strip_prefix("&id=").expect("all x-oxide-signature headers should have a secret ID ID part"); - // If the ID isn't the one we're looking for, we want to keep - // going, so just return `None` here - let hdr = hdr.strip_prefix(secret_id.as_str())?; - // Finally, extract the signature part by stripping the &s= - // prefix. - hdr.strip_prefix("&s=") - }).next() else { - panic!("no x-oxide-signature header for secret with ID {secret_id} found"); - }; + // N.B. that `HttpMockRequest::headers_vec()`, which returns a + // `Vec<(String, String)>` is used here, rather than + // `HttpMockRequest::headers()`, which returns a `HeaderMap`. This is + // currently necessary because of a `httpmock` bug where, when multiple + // values for the same header are present in the request, the map + // returned by `headers()` will only contain one of those values. See: + // https://github.com/alexliesenfeld/httpmock/issues/119 + let hdrs = req.headers_vec(); + let Some(sig_hdr) = hdrs.iter().find_map(|(name, hdr)| { + if name != "x-oxide-signature" { + return None; + } + // Signature header format: + // a={algorithm}&id={secret_id}&s={signature} + + // Strip the expected algorithm part. Note that we only support + // SHA256 for now. Panic if this is invalid. + let hdr = dbg!(hdr) + .strip_prefix("a=sha256") + .expect("all x-oxide-signature headers should be SHA256"); + // Strip the leading `&id=` for the ID part, panicking if this + // is not found. + let hdr = hdr.strip_prefix("&id=").expect( + "all x-oxide-signature headers should have a secret ID part", + ); + // If the ID isn't the one we're looking for, we want to keep + // going, so just return `None` here + let hdr = hdr.strip_prefix(secret_id.as_str())?; + // Finally, extract the signature part by stripping the &s= + // prefix. + hdr.strip_prefix("&s=") + }) else { + panic!("no x-oxide-signature header for secret with ID {secret_id} found"); + }; let sig_bytes = hex::decode(sig_hdr) .expect("x-oxide-signature signature value should be a hex string"); let mut mac = Hmac::::new_from_slice(&secret[..]) From c8f8433dfe89ce7503c09aae9ee88d80e8c4368d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Feb 2025 13:57:58 -0800 Subject: [PATCH 070/168] add tests for dispatching to multiple receivers --- nexus/tests/integration_tests/webhooks.rs | 165 ++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index b951683f246..ca83b61c26a 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -294,3 +294,168 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { mock.assert_async().await; } + +#[nexus_test] +async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let bar_event_id = WebhookEventUuid::new_v4(); + let baz_event_id = WebhookEventUuid::new_v4(); + + // Create three webhook receivers + let srv_bar = httpmock::MockServer::start_async().await; + const BAR_SECRET: &str = "this is bar's secret"; + let rx_bar = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "webhooked-on-phonics".parse().unwrap(), + description: String::from("webhooked on phonics"), + }, + endpoint: srv_bar + .url("/webhooks") + .parse() + .expect("this should be a valid URL"), + secrets: vec![BAR_SECRET.to_string()], + events: vec!["test.foo.bar".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(&rx_bar); + let mock_bar = { + let webhook = rx_bar.clone(); + srv_bar + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo.bar") + .header("x-oxide-event-id", bar_event_id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + BAR_SECRET.as_bytes().to_vec(), + )); + then.status(200); + }) + .await + }; + + let srv_baz = httpmock::MockServer::start_async().await; + const BAZ_SECRET: &str = "this is baz's secret"; + let rx_baz = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "webhook-line-and-sinker".parse().unwrap(), + description: String::from("webhook, line, and sinker"), + }, + endpoint: srv_baz + .url("/webhooks") + .parse() + .expect("this should be a valid URL"), + secrets: vec![BAZ_SECRET.to_string()], + events: vec!["test.foo.baz".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(&rx_baz); + let mock_baz = { + let webhook = rx_baz.clone(); + srv_baz + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo.baz") + .header("x-oxide-event-id", baz_event_id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + BAZ_SECRET.as_bytes().to_vec(), + )); + then.status(200); + }) + .await + }; + + let srv_star = httpmock::MockServer::start_async().await; + const STAR_SECRET: &str = "this is star's secret"; + let rx_star = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "globulated".parse().unwrap(), + description: String::from("this one has globs"), + }, + endpoint: srv_star + .url("/webhooks") + .parse() + .expect("this should be a valid URL"), + secrets: vec![STAR_SECRET.to_string()], + events: vec!["test.foo.*".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(&rx_star); + let mock_star = { + let webhook = rx_star.clone(); + srv_star + .mock_async(move |when, then| { + when.method(POST) + .header_matches( + "x-oxide-event-class", + "test\\.foo\\.ba[rz]", + ) + .header_exists("x-oxide-event-id") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + STAR_SECRET.as_bytes().to_vec(), + )); + then.status(200); + }) + .await + }; + + // Publish a test.foo.bar event + let event = nexus + .webhook_event_publish( + &opctx, + bar_event_id, + WebhookEventClass::TestFooBar, + serde_json::json!({"lol": "webhooked on phonics"}), + ) + .await + .expect("event should be published successfully"); + dbg!(event); + // Publish a test.foo.baz event + let event = nexus + .webhook_event_publish( + &opctx, + baz_event_id, + WebhookEventClass::TestFooBaz, + serde_json::json!({"lol": "webhook, line, and sinker"}), + ) + .await + .expect("event should be published successfully"); + dbg!(event); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + // The `test.foo.bar` receiver should have received 1 event. + mock_bar.assert_calls_async(1).await; + + // The `test.foo.baz` receiver should have received 1 event. + mock_baz.assert_calls_async(1).await; + + // The `test.foo.*` receiver should have received both events. + mock_star.assert_calls_async(2).await; +} From d9b479bfcb2b559fd04a766b1ea42c02197e5d53 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 13 Feb 2025 15:53:52 -0800 Subject: [PATCH 071/168] apparently reqwest will return Ok for 5xxs --- .../background/tasks/webhook_deliverator.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index c73c7da03c5..5988aed96a1 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -453,16 +453,29 @@ impl WebhookDeliverator { } Ok(rsp) => { let status = rsp.status(); - slog::debug!( - &opctx.log, - "webhook event delivered successfully"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "response_status" => ?status, - "response_duration" => ?duration, - ); - (WebhookDeliveryResult::Succeeded, Some(status)) + if status.is_success() { + slog::debug!( + &opctx.log, + "webhook event delivered successfully"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery_id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::Succeeded, Some(status)) + } else { + slog::warn!( + &opctx.log, + "webhook receiver endpoint returned an HTTP error"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery_id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::FailedHttpError, Some(status)) + } } }; // only include a response duration if we actually got a response back From 47234f7b85aa08910dd96c55ee7f7d2f60005561 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 13 Feb 2025 16:04:18 -0800 Subject: [PATCH 072/168] retry backoff! --- nexus-config/src/nexus_config.rs | 28 +++ .../src/db/datastore/webhook_delivery.rs | 44 ++++- nexus/src/app/background/init.rs | 43 +++-- .../background/tasks/webhook_deliverator.rs | 15 +- nexus/tests/config.test.toml | 5 + nexus/tests/integration_tests/webhooks.rs | 169 ++++++++++++++++++ 6 files changed, 276 insertions(+), 28 deletions(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 9a2a1d1a09b..70e92f38bf5 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -761,6 +761,22 @@ pub struct WebhookDeliveratorConfig { /// wait a long time. #[serde(default = "WebhookDeliveratorConfig::default_lease_timeout_secs")] pub lease_timeout_secs: u64, + + /// backoff period for the first retry of a failed delivery attempt. + /// + /// this is tuneable to allow testing delivery retries without having to + /// wait a long time. + #[serde(default = "WebhookDeliveratorConfig::default_first_retry_backoff")] + pub first_retry_backoff_secs: u64, + + /// backoff period for the seecond retry of a failed delivery attempt. + /// + /// this is tuneable to allow testing delivery retries without having to + /// wait a long time. + #[serde( + default = "WebhookDeliveratorConfig::default_second_retry_backoff" + )] + pub second_retry_backoff_secs: u64, } /// Configuration for a nexus server @@ -838,6 +854,14 @@ impl WebhookDeliveratorConfig { const fn default_lease_timeout_secs() -> u64 { 60 // one minute } + + const fn default_first_retry_backoff() -> u64 { + 60 // one minute + } + + const fn default_second_retry_backoff() -> u64 { + 60 * 5 // five minutes + } } #[cfg(test)] @@ -1030,6 +1054,8 @@ mod test { webhook_dispatcher.period_secs = 42 webhook_deliverator.period_secs = 43 webhook_deliverator.lease_timeout_secs = 44 + webhook_deliverator.first_retry_backoff_secs = 45 + webhook_deliverator.second_retry_backoff_secs = 45 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1237,6 +1263,8 @@ mod test { webhook_deliverator: WebhookDeliveratorConfig { period_secs: Duration::from_secs(43), lease_timeout_secs: 44, + first_retry_backoff_secs: 45, + second_retry_backoff_secs: 46, } }, default_region_allocation_strategy: diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 2881ddca677..d8a3ec43ada 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -37,6 +37,23 @@ pub enum DeliveryAttemptState { InProgress { nexus_id: OmicronZoneUuid, started: DateTime }, } +#[derive(Debug, Clone)] +pub struct DeliveryConfig { + pub first_retry_backoff: TimeDelta, + pub second_retry_backoff: TimeDelta, + pub lease_timeout: TimeDelta, +} + +impl Default for DeliveryConfig { + fn default() -> Self { + Self { + lease_timeout: TimeDelta::seconds(60), // 1 minute + first_retry_backoff: TimeDelta::seconds(60), // 1 minute + second_retry_backoff: TimeDelta::seconds(60 * 5), // 5 minutes + } + } +} + impl DataStore { pub async fn webhook_delivery_create_batch( &self, @@ -77,7 +94,7 @@ impl DataStore { &self, opctx: &OpContext, rx_id: &WebhookReceiverUuid, - lease_timeout: TimeDelta, + cfg: &DeliveryConfig, ) -> ListResultVec<(WebhookDelivery, WebhookEventClass)> { let conn = self.pool_connection_authorized(opctx).await?; let now = @@ -90,9 +107,26 @@ impl DataStore { .is_not_null() .and( dsl::time_delivery_started - .le(now.nullable() - lease_timeout), + .le(now.nullable() - cfg.lease_timeout), + )), + ) + .filter( + // Retry backoffs: one of the following must be true: + // - the delivery has not yet been attempted, + dsl::attempts + .eq(0) + // - this is the first retry and the previous attempt was at + // least `first_retry_backoff` ago, or + .or(dsl::attempts.eq(1).and( + dsl::time_delivery_started + .le(now.nullable() - cfg.first_retry_backoff), + )) + // - this is the second retry, and the previous attempt was at + // least `second_retry_backoff` ago. + .or(dsl::attempts.eq(2).and( + dsl::time_delivery_started + .le(now.nullable() - cfg.second_retry_backoff), )), - // TODO(eliza): retry backoffs...? ) .order_by(dsl::time_created.asc()) // Join with the `webhook_event` table to get the event class, which @@ -172,7 +206,7 @@ impl DataStore { nexus_id: &OmicronZoneUuid, attempt: &WebhookDeliveryAttempt, ) -> Result<(), Error> { - const MAX_ATTEMPTS: u8 = 4; + const MAX_ATTEMPTS: u8 = 3; let conn = self.pool_connection_authorized(opctx).await?; diesel::insert_into(attempt_dsl::webhook_delivery_attempt) .values(attempt.clone()) @@ -191,7 +225,7 @@ impl DataStore { let (completed, new_nexus_id) = if succeeded || failed_permanently { // If the delivery has succeeded or failed permanently, set the // "time_completed" timestamp to mark it as finished. Also, leave - // the delivering Nexus ID in place to maintain a record of who + // the delivering Nexus ID in place to maintain a record of who // finished the delivery. (Some(Utc::now()), Some(nexus_id.into_untyped_uuid())) } else { diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 8fba4c28674..a9a0cf01202 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -122,7 +122,7 @@ use super::tasks::sync_switch_configuration::SwitchPortSettingsManager; use super::tasks::tuf_artifact_replication; use super::tasks::v2p_mappings::V2PManager; use super::tasks::vpc_routes; -use super::tasks::webhook_deliverator::WebhookDeliverator; +use super::tasks::webhook_deliverator; use super::tasks::webhook_dispatcher::WebhookDispatcher; use super::Activator; use super::Driver; @@ -920,23 +920,38 @@ impl BackgroundTasksInitializer { }); driver.register({ - let lease_timeout_secs = - config.webhook_deliverator.lease_timeout_secs; - let lease_timeout = chrono::TimeDelta::seconds( - i64::try_from(lease_timeout_secs).expect( - "webhook_deliverator.lease_timeout_secs must be less \ - than i64::MAX", + let nexus_config::WebhookDeliveratorConfig { + lease_timeout_secs, + period_secs, + first_retry_backoff_secs, + second_retry_backoff_secs, + } = config.webhook_deliverator; + let cfg = webhook_deliverator::DeliveryConfig { + lease_timeout: chrono::TimeDelta::seconds( + lease_timeout_secs.try_into().expect( + "invalid webhook_deliverator.lease_timeout_secs", + ), ), - ); + first_retry_backoff: chrono::TimeDelta::seconds( + first_retry_backoff_secs.try_into().expect( + "invalid webhook_deliverator.first_retry_backoff_secs", + ), + ), + second_retry_backoff: chrono::TimeDelta::seconds( + second_retry_backoff_secs.try_into().expect( + "invalid webhook_deliverator.first_retry_backoff_secs", + ), + ), + }; TaskDefinition { name: "webhook_deliverator", description: "sends webhook delivery requests", - period: config.webhook_deliverator.period_secs, - task_impl: Box::new(WebhookDeliverator::new( - datastore, - lease_timeout, - nexus_id, - )), + period: period_secs, + task_impl: Box::new( + webhook_deliverator::WebhookDeliverator::new( + datastore, cfg, nexus_id, + ), + ), opctx: opctx.child(BTreeMap::new()), watchers: vec![], activator: task_webhook_deliverator, diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 5988aed96a1..94a8360431f 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -11,6 +11,7 @@ use http::HeaderName; use http::HeaderValue; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; +pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; use nexus_db_queries::db::model::{ SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookEventClass, WebhookReceiver, WebhookSecret, @@ -82,8 +83,8 @@ use tokio::task::JoinSet; pub struct WebhookDeliverator { datastore: Arc, nexus_id: OmicronZoneUuid, - lease_timeout: TimeDelta, client: reqwest::Client, + cfg: DeliveryConfig, } impl BackgroundTask for WebhookDeliverator { @@ -109,7 +110,7 @@ impl BackgroundTask for WebhookDeliverator { impl WebhookDeliverator { pub fn new( datastore: Arc, - lease_timeout: TimeDelta, + cfg: DeliveryConfig, nexus_id: OmicronZoneUuid, ) -> Self { let client = reqwest::Client::builder() @@ -126,7 +127,7 @@ impl WebhookDeliverator { .connect_timeout(Duration::from_secs(10)) .build() .expect("failed to configure webhook deliverator client!"); - Self { datastore, nexus_id, lease_timeout, client } + Self { datastore, nexus_id, cfg, client } } const MAX_CONCURRENT_RXS: NonZeroU32 = { @@ -219,11 +220,7 @@ impl WebhookDeliverator { anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); let deliveries = self .datastore - .webhook_rx_delivery_list_ready( - &opctx, - &rx.id(), - self.lease_timeout, - ) + .webhook_rx_delivery_list_ready(&opctx, &rx.id(), &self.cfg) .await .map_err(|e| { anyhow::anyhow!("could not list ready deliveries: {e}") @@ -246,7 +243,7 @@ impl WebhookDeliverator { opctx, &delivery, &self.nexus_id, - self.lease_timeout, + self.cfg.lease_timeout, ) .await { diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 88d59533e19..697b0df4c98 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -165,6 +165,11 @@ tuf_artifact_replication.min_sled_replication = 3 # so we don't need to periodically activate it *that* frequently. webhook_dispatcher.period_secs = 60 webhook_deliverator.period_secs = 60 +# In order to test webhook delivery retry behavior without waiting for a long +# time, turn these backoff periods down from multiple minutes to just a couple +# seconds. +webhook_deliverator.first_retry_backoff_secs = 10 +webhook_deliverator.second_retry_backoff_secs = 20 [default_region_allocation_strategy] # we only have one sled in the test environment, so we need to use the diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index ca83b61c26a..8aa03fdaba2 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -459,3 +459,172 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { // The `test.foo.*` receiver should have received both events. mock_star.assert_calls_async(2).await; } + +#[nexus_test] +async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let server = httpmock::MockServer::start_async().await; + + let id = WebhookEventUuid::new_v4(); + let endpoint = + server.url("/webhooks").parse().expect("this should be a valid URL"); + + // Create a webhook receiver. + let webhook = webhook_create( + &cptestctx, + ¶ms::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "my-great-webhook".parse().unwrap(), + description: String::from("my great webhook"), + }, + endpoint, + secrets: vec!["my cool secret".to_string()], + events: vec!["test.foo".to_string()], + disable_probes: false, + }, + ) + .await; + dbg!(&webhook); + + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + let body = serde_json::json!({ + "event_class": "test.foo", + "event_id": id, + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + "my cool secret".as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(500); + }) + .await + }; + + // Publish an event + let event = nexus + .webhook_event_publish( + &opctx, + id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello_world": true}), + ) + .await + .expect("event should be published successfully"); + dbg!(event); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + mock.assert_calls_async(1).await; + + // Okay, we are now in backoff. Activate the deliverator again --- no new + // event should be delivered. + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + // Activating the deliverator whilst in backoff should not send another + // request. + mock.assert_calls_async(1).await; + mock.delete_async().await; + + // Okay, now let's return a different 5xx status. + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + let body = serde_json::json!({ + "event_class": "test.foo", + "event_id": id, + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + "my cool secret".as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(503); + }) + .await + }; + + // Wait out the backoff period for the first request. + tokio::time::sleep(std::time::Duration::from_secs(15)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(1).await; + + // Again, we should be in backoff, so no request will be sent. + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(1).await; + mock.delete_async().await; + + // Finally, allow the request to succeed. + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + let body = serde_json::json!({ + "event_class": "test.foo", + "event_id": id, + "data": { + "hello_world": true, + } + }) + .to_string(); + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + "my cool secret".as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(200); + }) + .await + }; + + // + tokio::time::sleep(std::time::Duration::from_secs(15)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(0).await; + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_async().await; +} From d8540a4600d1c124ab613496a8b77035f8693181 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 24 Feb 2025 14:21:13 -0800 Subject: [PATCH 073/168] lockfile 'n' stuff --- Cargo.lock | 2 +- nexus/external-api/output/nexus_tags.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dfb1172048b..d851782eb50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7544,6 +7544,7 @@ dependencies = [ "indicatif", "inout", "itertools 0.10.5", + "itertools 0.12.1", "lalrpop-util", "lazy_static", "libc", @@ -7553,7 +7554,6 @@ dependencies = [ "memchr", "mio", "newtype-uuid", - "nix 0.29.0", "nom", "num-bigint-dig", "num-integer", diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 948ce1d1207..007ffec4e54 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -273,6 +273,7 @@ webhook_delivery_resend POST /experimental/v1/webhooks/{web webhook_event_class_list GET /experimental/v1/webhook-events/classes webhook_event_class_view GET /experimental/v1/webhook-events/classes/{name} webhook_secrets_add POST /experimental/v1/webhooks/{webhook_id}/secrets +webhook_secrets_delete DELETE /experimental/v1/webhooks/{webhook_id}/secrets/{secret_id} webhook_secrets_list GET /experimental/v1/webhooks/{webhook_id}/secrets webhook_update PUT /experimental/v1/webhooks/{webhook_id} webhook_view GET /experimental/v1/webhooks/{webhook_id} From 31f2fa98a33e4021edd6af0f5c46d68a3a988e1d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 24 Feb 2025 14:31:32 -0800 Subject: [PATCH 074/168] remove children --- nexus/db-queries/src/db/lookup.rs | 2 -- nexus/src/app/background/init.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 4e32f466acd..ab1029ac641 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -921,7 +921,6 @@ lookup_resource! { lookup_resource! { name = "WebhookReceiver", ancestors = [], - children = ["WebhookSecret"], lookup_by_name = false, soft_deletes = true, primary_key_columns = [ @@ -932,7 +931,6 @@ lookup_resource! { lookup_resource! { name = "WebhookSecret", ancestors = ["WebhookReceiver"], - children = [], lookup_by_name = false, soft_deletes = true, primary_key_columns = [ diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index c16a058835d..883f0e4b9d1 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -917,7 +917,7 @@ impl BackgroundTasksInitializer { process", period: config.read_only_region_replacement_start.period_secs, task_impl: Box::new(ReadOnlyRegionReplacementDetector::new( - datastore, + datastore.clone(), )), opctx: opctx.child(BTreeMap::new()), watchers: vec![], From e79bf649ecc431d27e150a4d020e82908ed7af31 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 24 Feb 2025 14:36:37 -0800 Subject: [PATCH 075/168] webhook receivers are lookup by name now --- nexus/db-queries/src/db/lookup.rs | 26 ++++++++- nexus/external-api/output/nexus_tags.txt | 16 +++--- nexus/external-api/src/lib.rs | 28 +++++----- nexus/src/app/webhook.rs | 20 +++++-- nexus/src/external_api/http_entrypoints.rs | 12 ++-- nexus/types/src/external_api/params.rs | 17 ++++-- openapi/nexus.json | 65 ++++++++++------------ 7 files changed, 110 insertions(+), 74 deletions(-) diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index ab1029ac641..fdf793b998d 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -554,6 +554,30 @@ impl<'a> LookupPath<'a> { { WebhookReceiver::PrimaryKey(Root { lookup_root: self }, id) } + + /// Select a resource of type [`WebhookReceiver`], identified by its name + pub fn webhook_receiver_name<'b, 'c>( + self, + name: &'b Name, + ) -> WebhookReceiver<'c> + where + 'a: 'c, + 'b: 'c, + { + WebhookReceiver::Name(Root { lookup_root: self }, name) + } + + /// Select a resource of type [`WebhookReceiver`], identified by its owned name + pub fn webhook_receiver_name_owned<'b, 'c>( + self, + name: Name, + ) -> WebhookReceiver<'c> + where + 'a: 'c, + 'b: 'c, + { + WebhookReceiver::OwnedName(Root { lookup_root: self }, name) + } } /// Represents the head of the selection path for a resource @@ -921,7 +945,7 @@ lookup_resource! { lookup_resource! { name = "WebhookReceiver", ancestors = [], - lookup_by_name = false, + lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", uuid_kind = WebhookReceiverKind } diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 007ffec4e54..add01500209 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -267,16 +267,16 @@ ping GET /v1/ping API operations found with tag "system/webhooks" OPERATION ID METHOD URL PATH webhook_create POST /experimental/v1/webhooks -webhook_delete DELETE /experimental/v1/webhooks/{webhook_id} -webhook_delivery_list GET /experimental/v1/webhooks/{webhook_id}/deliveries -webhook_delivery_resend POST /experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend +webhook_delete DELETE /experimental/v1/webhooks/{webhook} +webhook_delivery_list GET /experimental/v1/webhooks/{webhook}/deliveries +webhook_delivery_resend POST /experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend webhook_event_class_list GET /experimental/v1/webhook-events/classes webhook_event_class_view GET /experimental/v1/webhook-events/classes/{name} -webhook_secrets_add POST /experimental/v1/webhooks/{webhook_id}/secrets -webhook_secrets_delete DELETE /experimental/v1/webhooks/{webhook_id}/secrets/{secret_id} -webhook_secrets_list GET /experimental/v1/webhooks/{webhook_id}/secrets -webhook_update PUT /experimental/v1/webhooks/{webhook_id} -webhook_view GET /experimental/v1/webhooks/{webhook_id} +webhook_secrets_add POST /experimental/v1/webhooks/{webhook}/secrets +webhook_secrets_delete DELETE /experimental/v1/webhooks/{webhook}/secrets/{secret_id} +webhook_secrets_list GET /experimental/v1/webhooks/{webhook}/secrets +webhook_update PUT /experimental/v1/webhooks/{webhook} +webhook_view GET /experimental/v1/webhooks/{webhook} API operations found with tag "vpcs" OPERATION ID METHOD URL PATH diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index fd293e7d6c9..3915b650881 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3503,12 +3503,12 @@ pub trait NexusExternalApi { /// Get the configuration for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook_id}", + path = "/experimental/v1/webhooks/{webhook}", tags = ["system/webhooks"], }] async fn webhook_view( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result, HttpError>; /// Create a new webhook receiver. @@ -3525,53 +3525,53 @@ pub trait NexusExternalApi { /// Update the configuration of an existing webhook receiver. #[endpoint { method = PUT, - path = "/experimental/v1/webhooks/{webhook_id}", + path = "/experimental/v1/webhooks/{webhook}", tags = ["system/webhooks"], }] async fn webhook_update( rqctx: RequestContext, - path_params: Path, + path_params: Path, params: TypedBody, ) -> Result; /// Delete a webhook receiver. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/{webhook_id}", + path = "/experimental/v1/webhooks/{webhook}", tags = ["system/webhooks"], }] async fn webhook_delete( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result; /// List the IDs of secrets for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook_id}/secrets", + path = "/experimental/v1/webhooks/{webhook}/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_list( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result, HttpError>; /// Add a secret to a webhook receiver. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook_id}/secrets", + path = "/experimental/v1/webhooks/{webhook}/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_add( rqctx: RequestContext, - path_params: Path, + path_params: Path, params: TypedBody, ) -> Result, HttpError>; /// Delete a secret associated with a webhook receiver by ID. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/{webhook_id}/secrets/{secret_id}", + path = "/experimental/v1/webhooks/{webhook}/secrets/{secret_id}", tags = ["system/webhooks"], }] async fn webhook_secrets_delete( @@ -3582,19 +3582,19 @@ pub trait NexusExternalApi { /// List delivery attempts to a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook_id}/deliveries", + path = "/experimental/v1/webhooks/{webhook}/deliveries", tags = ["system/webhooks"], }] async fn webhook_delivery_list( rqctx: RequestContext, - path_params: Path, + path_params: Path, query_params: Query, ) -> Result>, HttpError>; /// Request re-delivery of a webhook event. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend", + path = "/experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend", tags = ["system/webhooks"], }] async fn webhook_delivery_resend( diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 924d6dcb50c..f69e00e43f4 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -17,6 +17,7 @@ use nexus_types::external_api::views; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; @@ -25,11 +26,22 @@ impl super::Nexus { pub fn webhook_receiver_lookup<'a>( &'a self, opctx: &'a OpContext, - webhook_selector: params::WebhookPath, + webhook_selector: params::WebhookSelector, ) -> LookupResult> { - Ok(LookupPath::new(opctx, self.datastore()).webhook_receiver_id( - WebhookReceiverUuid::from_untyped_uuid(webhook_selector.webhook_id), - )) + match webhook_selector.webhook { + NameOrId::Id(id) => { + let webhook = LookupPath::new(opctx, &self.db_datastore) + .webhook_receiver_id( + WebhookReceiverUuid::from_untyped_uuid(id), + ); + Ok(webhook) + } + NameOrId::Name(name) => { + let webhook = LookupPath::new(opctx, &self.db_datastore) + .webhook_receiver_name_owned(name.into()); + Ok(webhook) + } + } } pub async fn webhook_receiver_config_fetch( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e208d73e220..261c8451417 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7332,7 +7332,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_view( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7377,7 +7377,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_update( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, _params: TypedBody, ) -> Result { let apictx = rqctx.context(); @@ -7401,7 +7401,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delete( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -7424,7 +7424,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_secrets_list( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7448,7 +7448,7 @@ impl NexusExternalApi for NexusExternalApiImpl { /// Add a secret to a webhook. async fn webhook_secrets_add( rqctx: RequestContext, - path_params: Path, + path_params: Path, params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); @@ -7496,7 +7496,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_list( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, _query_params: Query, ) -> Result>, HttpError> { diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index ad127ed5ea2..4b0188648ec 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -95,7 +95,6 @@ path_param!(CertificatePath, certificate, "certificate"); id_path_param!(SupportBundlePath, support_bundle, "support bundle"); id_path_param!(GroupPath, group_id, "group"); -id_path_param!(WebhookPath, webhook_id, "webhook"); // TODO: The hardware resources should be represented by its UUID or a hardware // ID that can be used to deterministically generate the UUID. @@ -2390,6 +2389,12 @@ pub struct EventClassSelector { pub name: String, } +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookSelector { + /// The name or ID of the webhook receiver. + pub webhook: NameOrId, +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookCreate { #[serde(flatten)] @@ -2438,14 +2443,16 @@ pub struct WebhookSecretCreate { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookSecretSelector { - /// ID of the webhook receiver. - pub webhook_id: Uuid, + /// Selects the webhook receiver + #[serde(flatten)] + pub webhook: WebhookSelector, /// ID of the secret. pub secret_id: Uuid, - } #[derive(Deserialize, JsonSchema)] pub struct WebhookDeliveryPath { - pub webhook_id: Uuid, + /// Selects the webhook receiver + #[serde(flatten)] + pub webhook: WebhookSelector, pub event_id: Uuid, } diff --git a/openapi/nexus.json b/openapi/nexus.json index c0377ec869b..f36fc9d8225 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -760,7 +760,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}": { + "/experimental/v1/webhooks/{webhook}": { "get": { "tags": [ "system/webhooks" @@ -770,12 +770,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -807,12 +806,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -847,12 +845,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -869,7 +866,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}/deliveries": { + "/experimental/v1/webhooks/{webhook}/deliveries": { "get": { "tags": [ "system/webhooks" @@ -879,12 +876,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } }, { @@ -938,7 +934,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend": { + "/experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend": { "post": { "tags": [ "system/webhooks" @@ -957,11 +953,11 @@ }, { "in": "path", - "name": "webhook_id", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -985,7 +981,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}/secrets": { + "/experimental/v1/webhooks/{webhook}/secrets": { "get": { "tags": [ "system/webhooks" @@ -995,12 +991,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -1032,12 +1027,11 @@ "parameters": [ { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -1071,7 +1065,7 @@ } } }, - "/experimental/v1/webhooks/{webhook_id}/secrets/{secret_id}": { + "/experimental/v1/webhooks/{webhook}/secrets/{secret_id}": { "delete": { "tags": [ "system/webhooks" @@ -1091,12 +1085,11 @@ }, { "in": "path", - "name": "webhook_id", - "description": "ID of the webhook receiver.", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], From 366fa2b67534f18a8ed4a8e8f87c63e13d05c582 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 11:16:34 -0800 Subject: [PATCH 076/168] probe api stub --- nexus/external-api/output/nexus_tags.txt | 1 + nexus/external-api/src/lib.rs | 15 +++++++ nexus/src/external_api/http_entrypoints.rs | 24 +++++++++++ nexus/types/src/external_api/params.rs | 9 +++++ openapi/nexus.json | 46 ++++++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index add01500209..41300513437 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -272,6 +272,7 @@ webhook_delivery_list GET /experimental/v1/webhooks/{web webhook_delivery_resend POST /experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend webhook_event_class_list GET /experimental/v1/webhook-events/classes webhook_event_class_view GET /experimental/v1/webhook-events/classes/{name} +webhook_probe POST /experimental/v1/webhooks/{webhook}/probe webhook_secrets_add POST /experimental/v1/webhooks/{webhook}/secrets webhook_secrets_delete DELETE /experimental/v1/webhooks/{webhook}/secrets/{secret_id} webhook_secrets_list GET /experimental/v1/webhooks/{webhook}/secrets diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 3915b650881..57b07022d81 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3545,6 +3545,21 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; + /// Send a liveness probe request to a webhook receiver. + // TODO(eliza): this return type isn't quite right, as the response should + // have a typed body, but any status code, as we return a resposne with the + // status code from the webhook endpoint... + #[endpoint { + method = POST, + path = "/experimental/v1/webhooks/{webhook}/probe", + tags = ["system/webhooks"], + }] + async fn webhook_probe( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + /// List the IDs of secrets for a webhook receiver. #[endpoint { method = GET, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 261c8451417..f053a757186 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7422,6 +7422,30 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn webhook_probe( + rqctx: RequestContext, + _path_params: Path, + _query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus + .unimplemented_todo(&opctx, crate::app::Unimpl::Public) + .await + .into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn webhook_secrets_list( rqctx: RequestContext, _path_params: Path, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 4b0188648ec..deb5b36b026 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2449,6 +2449,7 @@ pub struct WebhookSecretSelector { /// ID of the secret. pub secret_id: Uuid, } + #[derive(Deserialize, JsonSchema)] pub struct WebhookDeliveryPath { /// Selects the webhook receiver @@ -2456,3 +2457,11 @@ pub struct WebhookDeliveryPath { pub webhook: WebhookSelector, pub event_id: Uuid, } + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookProbe { + /// If true, resend all events that have not been delivered successfully if + /// the probe request succeeds. + #[serde(default)] + pub resend: bool, +} diff --git a/openapi/nexus.json b/openapi/nexus.json index f36fc9d8225..25087295b17 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -981,6 +981,52 @@ } } }, + "/experimental/v1/webhooks/{webhook}/probe": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Send a liveness probe request to a webhook receiver.", + "operationId": "webhook_probe", + "parameters": [ + { + "in": "path", + "name": "webhook", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "resend", + "description": "If true, resend all events that have not been delivered successfully if the probe request succeeds.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDelivery" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/experimental/v1/webhooks/{webhook}/secrets": { "get": { "tags": [ From d9b080a569af6e8603b3774d27e75159c2fc9e7a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 11:42:18 -0800 Subject: [PATCH 077/168] share webhook reqwest client with the API --- nexus/src/app/background/init.rs | 10 +++++- .../background/tasks/webhook_deliverator.rs | 26 ++------------- nexus/src/app/mod.rs | 33 +++++++++++++++++++ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 883f0e4b9d1..cde879faa43 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -967,7 +967,10 @@ impl BackgroundTasksInitializer { period: period_secs, task_impl: Box::new( webhook_deliverator::WebhookDeliverator::new( - datastore, cfg, nexus_id, + datastore, + cfg, + nexus_id, + args.webhook_delivery_client, ), ), opctx: opctx.child(BTreeMap::new()), @@ -1002,6 +1005,11 @@ pub struct BackgroundTasksData { pub saga_recovery: saga_recovery::SagaRecoveryHelpers>, /// Channel for TUF repository artifacts to be replicated out to sleds pub tuf_artifact_replication_rx: mpsc::Receiver, + /// `reqwest::Client` for webhook delivery requests. + /// + /// This is shared with the external API as it's also used when sending + /// webhook liveness probe requests from the API. + pub webhook_delivery_client: reqwest::Client, } /// Starts the three DNS-propagation-related background tasks for either diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 94a8360431f..a1682009680 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -30,7 +30,7 @@ use omicron_uuid_kinds::{ use sha2::Sha256; use std::num::NonZeroU32; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::task::JoinSet; // The Deliverator belongs to an elite order, a hallowed sub-category. He's got // esprit up to here. Right now he is preparing to carry out his third mission @@ -112,21 +112,8 @@ impl WebhookDeliverator { datastore: Arc, cfg: DeliveryConfig, nexus_id: OmicronZoneUuid, + client: reqwest::Client, ) -> Self { - let client = reqwest::Client::builder() - // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow - // redirects. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#_success - .redirect(reqwest::redirect::Policy::none()) - // Per [RFD 538 § 4.3.2][1], the client must be able to connect to a - // webhook receiver endpoint within 10 seconds, or the delivery is - // considered failed. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .connect_timeout(Duration::from_secs(10)) - .build() - .expect("failed to configure webhook deliverator client!"); Self { datastore, nexus_id, cfg, client } } @@ -364,14 +351,7 @@ impl WebhookDeliverator { format!("a=sha256&id={secret_id}&s={sig}"), ); } - let request = request - .body(body) - // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to - // each webhook delivery request. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .timeout(Duration::from_secs(30)) - .build(); + let request = request.body(body).build(); let request = match request { // We couldn't construct a request for some reason! This one's diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 271b5ccebb4..3ab9892edc9 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -171,6 +171,13 @@ pub struct Nexus { /// Client to the timeseries database. timeseries_client: oximeter_db::Client, + /// `reqwest` client used for webhook delivery requests. + /// + /// This lives on the Nexus struct as we would like to use the same client + /// pool for the webhook deliverator background task and the webhook probe + /// API. + webhook_delivery_client: reqwest::Client, + /// Contents of the trusted root role for the TUF repository. #[allow(dead_code)] updates_config: Option, @@ -435,6 +442,28 @@ impl Nexus { Some(address) => oximeter_db::Client::new(*address, &log), }; + let webhook_delivery_client = reqwest::ClientBuilder::new() + // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow + // redirects. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#_success + .redirect(reqwest::redirect::Policy::none()) + // Per [RFD 538 § 4.3.2][1], the client must be able to connect to a + // webhook receiver endpoint within 10 seconds, or the delivery is + // considered failed. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .connect_timeout(std::time::Duration::from_secs(10)) + // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to + // each webhook delivery request. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .timeout(std::time::Duration::from_secs(30)) + .build() + .map_err(|e| { + format!("failed to build webhook delivery client: {e}") + })?; + // TODO-cleanup We may want to make the populator a first-class // background task. let populate_ctx = OpContext::for_background( @@ -484,6 +513,7 @@ impl Nexus { populate_status, reqwest_client, timeseries_client, + webhook_delivery_client, updates_config: config.pkg.updates.clone(), tunables: config.pkg.tunables.clone(), opctx_alloc: OpContext::for_background( @@ -564,6 +594,9 @@ impl Nexus { resolver, saga_starter: task_nexus.sagas.clone(), producer_registry: task_registry, + webhook_delivery_client: task_nexus + .webhook_delivery_client + .clone(), saga_recovery: SagaRecoveryHelpers { recovery_opctx: saga_recovery_opctx, From 890588d17ca894987822f1d6860e048724864bf7 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 14:10:35 -0800 Subject: [PATCH 078/168] factor out actually sending requests for reuse in probes --- .../background/tasks/webhook_deliverator.rs | 248 ++-------------- nexus/src/app/webhook.rs | 269 ++++++++++++++++++ 2 files changed, 285 insertions(+), 232 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index a1682009680..d19f61ddcdf 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -4,18 +4,12 @@ use crate::app::authz; use crate::app::background::BackgroundTask; use crate::app::db::lookup::LookupPath; -use chrono::{TimeDelta, Utc}; +use crate::app::webhook::WebhookDeliveryClient; use futures::future::BoxFuture; -use hmac::{Hmac, Mac}; -use http::HeaderName; -use http::HeaderValue; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; -use nexus_db_queries::db::model::{ - SqlU8, WebhookDeliveryAttempt, WebhookDeliveryResult, WebhookEventClass, - WebhookReceiver, WebhookSecret, -}; +use nexus_db_queries::db::model::WebhookReceiver; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; @@ -23,15 +17,11 @@ use nexus_types::internal_api::background::{ WebhookDeliveratorStatus, WebhookRxDeliveryStatus, }; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{ - GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid, WebhookEventUuid, - WebhookReceiverUuid, -}; -use sha2::Sha256; +use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid}; use std::num::NonZeroU32; use std::sync::Arc; -use std::time::Instant; use tokio::task::JoinSet; + // The Deliverator belongs to an elite order, a hallowed sub-category. He's got // esprit up to here. Right now he is preparing to carry out his third mission // of the night. His uniform is black as activated charcoal, filtering the very @@ -191,20 +181,14 @@ impl WebhookDeliverator { .lookup_for(authz::Action::ListChildren) .await .map_err(|e| anyhow::anyhow!("could not look up receiver: {e}"))?; - let mut secrets = self + let secrets = self .datastore .webhook_rx_secret_list(opctx, &authz_rx) .await - .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))? - .into_iter() - .map(|WebhookSecret { identity, secret, .. }| { - let mac = Hmac::::new_from_slice(secret.as_bytes()) - .expect("HMAC key can be any size; this should never fail"); - (identity.id, mac) - }) - .collect::>(); + .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; + let mut client = + WebhookDeliveryClient::new(&self.client, secrets, &rx)?; - anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); let deliveries = self .datastore .webhook_rx_delivery_list_ready(&opctx, &rx.id(), &self.cfg) @@ -219,8 +203,7 @@ impl WebhookDeliverator { ready: deliveries.len(), ..Default::default() }; - let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) - .expect("UUIDs should always be a valid header value"); + for (delivery, event_class) in deliveries { let attempt = (*delivery.attempts) + 1; let delivery_id = WebhookDeliveryUuid::from(delivery.id); @@ -285,191 +268,17 @@ impl WebhookDeliverator { } // okay, actually do the thing... - let time_attempted = Utc::now(); - let sent_at = time_attempted.to_rfc3339(); - let payload = Payload { - event_class, - event_id: delivery.event_id.into(), - data: &delivery.payload, - delivery: DeliveryMetadata { - id: delivery.id.into(), - webhook_id: rx.id(), - sent_at: &sent_at, - }, - }; - // N.B. that we serialize the body "ourselves" rather than just - // passing it to `RequestBuilder::json` because we must access - // the serialized body in order to calculate HMAC signatures. - // This means we have to add the `Content-Type` ourselves below. - let body = match serde_json::to_vec(&payload) { - Ok(body) => body, - Err(e) => { - const MSG: &'static str = - "event payload could not be serialized"; - slog::error!( - &opctx.log, - "webhook {MSG}"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "error" => %e, - ); - - // This really shouldn't happen --- we expect the event - // payload will always be valid JSON. We could *probably* - // just panic here unconditionally, but it seems nicer to - // try and do the other events. But, if there's ever a bug - // that breaks serialization for *all* webhook payloads, - // I'd like the tests to fail in a more obvious way than - // eventually timing out waiting for the event to be - // delivered ... - if cfg!(debug_assertions) { - panic!("{MSG}:{e}\npayload: {payload:#?}"); - } - delivery_status - .delivery_errors - .insert(delivery_id, format!("{MSG}: {e}")); - continue; - } - }; - let mut request = self - .client - .post(&rx.endpoint) - .header(HDR_RX_ID, hdr_rx_id.clone()) - .header(HDR_DELIVERY_ID, delivery_id.to_string()) - .header(HDR_EVENT_ID, delivery.event_id.to_string()) - .header(HDR_EVENT_CLASS, event_class.to_string()) - .header(http::header::CONTENT_TYPE, "application/json"); - - // For each secret assigned to this webhook, calculate the HMAC and add a signature header. - for (secret_id, mac) in &mut secrets { - mac.update(&body); - let sig_bytes = mac.finalize_reset().into_bytes(); - let sig = hex::encode(&sig_bytes[..]); - request = request.header( - HDR_SIG, - format!("a=sha256&id={secret_id}&s={sig}"), - ); - } - let request = request.body(body).build(); - - let request = match request { - // We couldn't construct a request for some reason! This one's - // our fault, so don't penalize the receiver for it. - Err(e) => { - const MSG: &str = "failed to construct webhook request"; - slog::error!( - &opctx.log, - "{MSG}"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "error" => %e, - "payload" => ?payload, - ); - delivery_status - .delivery_errors - .insert(delivery_id, format!("{MSG}: {e}")); - continue; - } - Ok(r) => r, - }; - let t0 = Instant::now(); - let result = self.client.execute(request).await; - let duration = t0.elapsed(); - let (delivery_result, status) = match result { - // Builder errors are our fault, that's weird! - Err(e) if e.is_builder() => { - const MSG: &str = - "internal error constructing webhook delivery request"; - slog::error!( - &opctx.log, - "{MSG}"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "error" => %e, - ); + let delivery_attempt = match client + .send_delivery_request(opctx, &delivery, event_class) + .await + { + Ok(delivery) => delivery, + Err(error) => { delivery_status .delivery_errors - .insert(delivery_id, format!("{MSG}: {e}")); + .insert(delivery_id, format!("{error:?}")); continue; } - Err(e) => { - if let Some(status) = e.status() { - slog::warn!( - &opctx.log, - "webhook receiver endpoint returned an HTTP error"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "response_status" => ?status, - "response_duration" => ?duration, - ); - (WebhookDeliveryResult::FailedHttpError, Some(status)) - } else { - let result = if e.is_connect() { - WebhookDeliveryResult::FailedUnreachable - } else if e.is_timeout() { - WebhookDeliveryResult::FailedTimeout - } else if e.is_redirect() { - WebhookDeliveryResult::FailedHttpError - } else { - WebhookDeliveryResult::FailedUnreachable - }; - slog::warn!( - &opctx.log, - "webhook delivery request failed"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "error" => %e, - ); - (result, None) - } - } - Ok(rsp) => { - let status = rsp.status(); - if status.is_success() { - slog::debug!( - &opctx.log, - "webhook event delivered successfully"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "response_status" => ?status, - "response_duration" => ?duration, - ); - (WebhookDeliveryResult::Succeeded, Some(status)) - } else { - slog::warn!( - &opctx.log, - "webhook receiver endpoint returned an HTTP error"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "response_status" => ?status, - "response_duration" => ?duration, - ); - (WebhookDeliveryResult::FailedHttpError, Some(status)) - } - } - }; - // only include a response duration if we actually got a response back - let response_duration = status.map(|_| { - TimeDelta::from_std(duration).expect( - "because we set a 30-second response timeout, there is no \ - way a response duration could ever exceed the max \ - representable TimeDelta of `i64::MAX` milliseconds", - ) - }); - let delivery_attempt = WebhookDeliveryAttempt { - delivery_id: delivery.id, - attempt: SqlU8::new(attempt), - result: delivery_result, - response_status: status.map(|s| s.as_u16() as i16), - response_duration, - time_created: chrono::Utc::now(), }; match self @@ -503,31 +312,6 @@ impl WebhookDeliverator { } } - // TODO(eliza): if no events were sent, do a probe... - Ok(delivery_status) } } - -const HDR_DELIVERY_ID: HeaderName = - HeaderName::from_static("x-oxide-delivery-id"); -const HDR_RX_ID: HeaderName = HeaderName::from_static("x-oxide-webhook-id"); -const HDR_EVENT_ID: HeaderName = HeaderName::from_static("x-oxide-event-id"); -const HDR_EVENT_CLASS: HeaderName = - HeaderName::from_static("x-oxide-event-class"); -const HDR_SIG: HeaderName = HeaderName::from_static("x-oxide-signature"); - -#[derive(serde::Serialize, Debug)] -struct Payload<'a> { - event_class: WebhookEventClass, - event_id: WebhookEventUuid, - data: &'a serde_json::Value, - delivery: DeliveryMetadata<'a>, -} - -#[derive(serde::Serialize, Debug)] -struct DeliveryMetadata<'a> { - id: WebhookDeliveryUuid, - webhook_id: WebhookReceiverUuid, - sent_at: &'a str, -} diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index f69e00e43f4..df9e5a80878 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -4,23 +4,39 @@ //! Webhooks +use anyhow::Context; +use chrono::TimeDelta; +use chrono::Utc; +use hmac::{Hmac, Mac}; +use http::HeaderName; +use http::HeaderValue; +use nexus_db_model::WebhookReceiver; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::model::SqlU8; +use nexus_db_queries::db::model::WebhookDelivery; +use nexus_db_queries::db::model::WebhookDeliveryAttempt; +use nexus_db_queries::db::model::WebhookDeliveryResult; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_db_queries::db::model::WebhookSecret; use nexus_types::external_api::params; use nexus_types::external_api::views; +use nexus_types::identity::Resource; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::WebhookDeliveryUuid; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; +use omicron_uuid_kinds::WebhookSecretUuid; +use sha2::Sha256; +use std::time::Instant; impl super::Nexus { pub fn webhook_receiver_lookup<'a>( @@ -91,6 +107,14 @@ impl super::Nexus { Ok(event) } + pub fn webhook_receiver_probe( + &self, + _rx: lookup::WebhookReceiver<'_>, + _params: params::WebhookProbe, + ) -> Result { + todo!() + } + pub async fn webhook_receiver_secret_add( &self, opctx: &OpContext, @@ -113,3 +137,248 @@ impl super::Nexus { Ok(views::WebhookSecretId { id: secret_id.into_untyped_uuid() }) } } + +pub(crate) struct WebhookDeliveryClient<'a> { + client: &'a reqwest::Client, + rx: &'a WebhookReceiver, + secrets: Vec<(WebhookSecretUuid, Hmac)>, + hdr_rx_id: http::HeaderValue, +} + +impl<'a> WebhookDeliveryClient<'a> { + pub(crate) fn new( + client: &'a reqwest::Client, + secrets: impl IntoIterator, + rx: &'a WebhookReceiver, + ) -> Result { + let secrets = secrets + .into_iter() + .map(|WebhookSecret { identity, secret, .. }| { + let mac = Hmac::::new_from_slice(secret.as_bytes()) + .expect("HMAC key can be any size; this should never fail"); + (identity.id.into(), mac) + }) + .collect::>(); + + anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); + let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) + .expect("UUIDs should always be a valid header value"); + Ok(Self { client, secrets, hdr_rx_id, rx }) + } + + pub(crate) async fn send_delivery_request( + &mut self, + opctx: &OpContext, + delivery: &WebhookDelivery, + event_class: WebhookEventClass, + ) -> Result { + const HDR_DELIVERY_ID: HeaderName = + HeaderName::from_static("x-oxide-delivery-id"); + const HDR_RX_ID: HeaderName = + HeaderName::from_static("x-oxide-webhook-id"); + const HDR_EVENT_ID: HeaderName = + HeaderName::from_static("x-oxide-event-id"); + const HDR_EVENT_CLASS: HeaderName = + HeaderName::from_static("x-oxide-event-class"); + const HDR_SIG: HeaderName = + HeaderName::from_static("x-oxide-signature"); + + #[derive(serde::Serialize, Debug)] + struct Payload<'a> { + event_class: WebhookEventClass, + event_id: WebhookEventUuid, + data: &'a serde_json::Value, + delivery: DeliveryMetadata<'a>, + } + + #[derive(serde::Serialize, Debug)] + struct DeliveryMetadata<'a> { + id: WebhookDeliveryUuid, + webhook_id: WebhookReceiverUuid, + sent_at: &'a str, + } + + // okay, actually do the thing... + let time_attempted = Utc::now(); + let sent_at = time_attempted.to_rfc3339(); + let payload = Payload { + event_class, + event_id: delivery.event_id.into(), + data: &delivery.payload, + delivery: DeliveryMetadata { + id: delivery.id.into(), + webhook_id: self.rx.id(), + sent_at: &sent_at, + }, + }; + // N.B. that we serialize the body "ourselves" rather than just + // passing it to `RequestBuilder::json` because we must access + // the serialized body in order to calculate HMAC signatures. + // This means we have to add the `Content-Type` ourselves below. + let body = match serde_json::to_vec(&payload) { + Ok(body) => body, + Err(e) => { + const MSG: &'static str = + "event payload could not be serialized"; + slog::error!( + &opctx.log, + "webhook {MSG}"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "error" => %e, + ); + + // This really shouldn't happen --- we expect the event + // payload will always be valid JSON. We could *probably* + // just panic here unconditionally, but it seems nicer to + // try and do the other events. But, if there's ever a bug + // that breaks serialization for *all* webhook payloads, + // I'd like the tests to fail in a more obvious way than + // eventually timing out waiting for the event to be + // delivered ... + if cfg!(debug_assertions) { + panic!("{MSG}: {e}\npayload: {payload:#?}"); + } + return Err(e).context(MSG); + } + }; + let mut request = self + .client + // TODO(eliza): this ain't gonna cut it! Rather than using the `reqwest` + // client, which performs its own DNS resolution, we gotta resolve the + // domain to an IP ourselves so that we can prevent SSRF by + // rejecting underlay network IPs. Which probably means managing our + // own client pool. Ag. + .post(&self.rx.endpoint) + .header(HDR_RX_ID, self.hdr_rx_id.clone()) + .header(HDR_DELIVERY_ID, delivery.id.to_string()) + .header(HDR_EVENT_ID, delivery.event_id.to_string()) + .header(HDR_EVENT_CLASS, event_class.to_string()) + .header(http::header::CONTENT_TYPE, "application/json"); + + // For each secret assigned to this webhook, calculate the HMAC and add a signature header. + for (secret_id, mac) in &mut self.secrets { + mac.update(&body); + let sig_bytes = mac.finalize_reset().into_bytes(); + let sig = hex::encode(&sig_bytes[..]); + request = request + .header(HDR_SIG, format!("a=sha256&id={secret_id}&s={sig}")); + } + let request = request.body(body).build(); + + let request = match request { + // We couldn't construct a request for some reason! This one's + // our fault, so don't penalize the receiver for it. + Err(e) => { + const MSG: &str = "failed to construct webhook request"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "error" => %e, + "payload" => ?payload, + ); + return Err(e).context(MSG); + } + Ok(r) => r, + }; + let t0 = Instant::now(); + let result = self.client.execute(request).await; + let duration = t0.elapsed(); + let (delivery_result, status) = match result { + // Builder errors are our fault, that's weird! + Err(e) if e.is_builder() => { + const MSG: &str = + "internal error constructing webhook delivery request"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "error" => %e, + ); + return Err(e).context(MSG); + } + Err(e) => { + if let Some(status) = e.status() { + slog::warn!( + &opctx.log, + "webhook receiver endpoint returned an HTTP error"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::FailedHttpError, Some(status)) + } else { + let result = if e.is_connect() { + WebhookDeliveryResult::FailedUnreachable + } else if e.is_timeout() { + WebhookDeliveryResult::FailedTimeout + } else if e.is_redirect() { + WebhookDeliveryResult::FailedHttpError + } else { + WebhookDeliveryResult::FailedUnreachable + }; + slog::warn!( + &opctx.log, + "webhook delivery request failed"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "error" => %e, + ); + (result, None) + } + } + Ok(rsp) => { + let status = rsp.status(); + if status.is_success() { + slog::debug!( + &opctx.log, + "webhook event delivered successfully"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::Succeeded, Some(status)) + } else { + slog::warn!( + &opctx.log, + "webhook receiver endpoint returned an HTTP error"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery.id, + "response_status" => ?status, + "response_duration" => ?duration, + ); + (WebhookDeliveryResult::FailedHttpError, Some(status)) + } + } + }; + // only include a response duration if we actually got a response back + let response_duration = status.map(|_| { + TimeDelta::from_std(duration).expect( + "because we set a 30-second response timeout, there is no \ + way a response duration could ever exceed the max \ + representable TimeDelta of `i64::MAX` milliseconds", + ) + }); + + Ok(WebhookDeliveryAttempt { + delivery_id: delivery.id, + attempt: SqlU8::new(delivery.attempts.0 + 1), + result: delivery_result, + response_status: status.map(|s| s.as_u16() as i16), + response_duration, + time_created: chrono::Utc::now(), + }) + } +} From aee6285e01408d4e8e261e37ea8fef05be47b600 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 14:45:46 -0800 Subject: [PATCH 079/168] use external DNS resolver for webhook endpoints --- .../background/tasks/webhook_deliverator.rs | 4 +- nexus/src/app/mod.rs | 27 ++-------- nexus/src/app/webhook.rs | 54 ++++++++++++++++--- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index d19f61ddcdf..e37dc2d25f2 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -4,7 +4,7 @@ use crate::app::authz; use crate::app::background::BackgroundTask; use crate::app::db::lookup::LookupPath; -use crate::app::webhook::WebhookDeliveryClient; +use crate::app::webhook::WebhookReceiverClient; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; @@ -187,7 +187,7 @@ impl WebhookDeliverator { .await .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; let mut client = - WebhookDeliveryClient::new(&self.client, secrets, &rx)?; + WebhookReceiverClient::new(&self.client, secrets, &rx)?; let deliveries = self .datastore diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 3ab9892edc9..0cead8977ac 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -442,28 +442,6 @@ impl Nexus { Some(address) => oximeter_db::Client::new(*address, &log), }; - let webhook_delivery_client = reqwest::ClientBuilder::new() - // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow - // redirects. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#_success - .redirect(reqwest::redirect::Policy::none()) - // Per [RFD 538 § 4.3.2][1], the client must be able to connect to a - // webhook receiver endpoint within 10 seconds, or the delivery is - // considered failed. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .connect_timeout(std::time::Duration::from_secs(10)) - // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to - // each webhook delivery request. - // - // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure - .timeout(std::time::Duration::from_secs(30)) - .build() - .map_err(|e| { - format!("failed to build webhook delivery client: {e}") - })?; - // TODO-cleanup We may want to make the populator a first-class // background task. let populate_ctx = OpContext::for_background( @@ -499,6 +477,11 @@ impl Nexus { )) }; + let webhook_delivery_client = + webhook::delivery_client(&external_resolver).map_err(|e| { + format!("failed to build webhook delivery client: {e}") + })?; + let nexus = Nexus { id: config.deployment.id, rack_id, diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index df9e5a80878..f551a4e467f 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -4,6 +4,7 @@ //! Webhooks +use crate::app::external_dns; use anyhow::Context; use chrono::TimeDelta; use chrono::Utc; @@ -36,6 +37,8 @@ use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use omicron_uuid_kinds::WebhookSecretUuid; use sha2::Sha256; +use std::sync::Arc; +use std::time::Duration; use std::time::Instant; impl super::Nexus { @@ -138,14 +141,56 @@ impl super::Nexus { } } -pub(crate) struct WebhookDeliveryClient<'a> { +/// Construct a [`reqwest::Client`] configured for webhook delivery requests. +pub(super) fn delivery_client( + external_dns: &Arc, +) -> Result { + /// A wrapper around [`external_dns::Resolver`] which rejects IP addresses that + /// are underlay network IPs. + struct WebhookDnsResolver { + external_dns: Arc, + } + + impl reqwest::dns::Resolve for WebhookDnsResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + // TODO(eliza): this is where we have to actually return an error if the + // DNS name resolves to an underlay IP! Figure that out! + self.external_dns.resolve(name) + } + } + + reqwest::Client::builder() + // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow + // redirects. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#_success + .redirect(reqwest::redirect::Policy::none()) + // Per [RFD 538 § 4.3.2][1], the client must be able to connect to a + // webhook receiver endpoint within 10 seconds, or the delivery is + // considered failed. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .connect_timeout(Duration::from_secs(10)) + // Per [RFD 538 § 4.3.2][1], a 30-second timeout is applied to + // each webhook delivery request. + // + // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure + .timeout(Duration::from_secs(30)) + // my god...it's full of Arcs... + .dns_resolver(Arc::new(WebhookDnsResolver { + external_dns: external_dns.clone(), + })) + .build() +} + +pub(crate) struct WebhookReceiverClient<'a> { client: &'a reqwest::Client, rx: &'a WebhookReceiver, secrets: Vec<(WebhookSecretUuid, Hmac)>, hdr_rx_id: http::HeaderValue, } -impl<'a> WebhookDeliveryClient<'a> { +impl<'a> WebhookReceiverClient<'a> { pub(crate) fn new( client: &'a reqwest::Client, secrets: impl IntoIterator, @@ -245,11 +290,6 @@ impl<'a> WebhookDeliveryClient<'a> { }; let mut request = self .client - // TODO(eliza): this ain't gonna cut it! Rather than using the `reqwest` - // client, which performs its own DNS resolution, we gotta resolve the - // domain to an IP ourselves so that we can prevent SSRF by - // rejecting underlay network IPs. Which probably means managing our - // own client pool. Ag. .post(&self.rx.endpoint) .header(HDR_RX_ID, self.hdr_rx_id.clone()) .header(HDR_DELIVERY_ID, delivery.id.to_string()) From 5c979433add67307f06006ef180f72c0ba605922 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 15:11:26 -0800 Subject: [PATCH 080/168] reticulating delivery triggers --- .../db-model/src/webhook_delivery_trigger.rs | 41 +++++++++++++------ .../src/db/datastore/webhook_delivery.rs | 19 ++++----- .../background/tasks/webhook_dispatcher.rs | 2 +- nexus/types/src/external_api/views.rs | 32 ++++++++++++++- schema/crdb/dbinit.sql | 8 ++-- 5 files changed, 75 insertions(+), 27 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery_trigger.rs b/nexus/db-model/src/webhook_delivery_trigger.rs index 0cb94065605..a7a5b3d7b97 100644 --- a/nexus/db-model/src/webhook_delivery_trigger.rs +++ b/nexus/db-model/src/webhook_delivery_trigger.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; +use nexus_types::external_api::views; use serde::Deserialize; use serde::Serialize; use std::fmt; @@ -26,24 +27,40 @@ impl_enum_type!( #[serde(rename_all = "snake_case")] pub enum WebhookDeliveryTrigger; - Dispatch => b"dispatch" + Event => b"event" Resend => b"resend" + Probe => b"probe" + ); -impl WebhookDeliveryTrigger { - pub fn as_str(&self) -> &'static str { - // TODO(eliza): it would be really nice if these strings were all - // declared a single time, rather than twice (in both `impl_enum_type!` - // and here)... - match self { - Self::Dispatch => "dispatch", - Self::Resend => "resend", +impl fmt::Display for WebhookDeliveryTrigger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Forward to the canonical implementation in nexus-types. + views::WebhookDeliveryTrigger::from(*self).fmt(f) + } +} + +impl From for views::WebhookDeliveryTrigger { + fn from(trigger: WebhookDeliveryTrigger) -> Self { + match trigger { + WebhookDeliveryTrigger::Event => Self::Event, + WebhookDeliveryTrigger::Resend => Self::Resend, + WebhookDeliveryTrigger::Probe => Self::Probe, } } } -impl fmt::Display for WebhookDeliveryTrigger { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.as_str()) +impl From for WebhookDeliveryTrigger { + fn from(trigger: views::WebhookDeliveryTrigger) -> Self { + match trigger { + views::WebhookDeliveryTrigger::Event => Self::Event, + views::WebhookDeliveryTrigger::Resend => Self::Resend, + views::WebhookDeliveryTrigger::Probe => Self::Probe, + } } } + +impl diesel::query_builder::QueryId for WebhookDeliveryTriggerEnum { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index d8a3ec43ada..1e7a4304675 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -11,6 +11,7 @@ use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; +use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEventClass; use crate::db::pagination::paginated; use crate::db::schema::webhook_delivery::dsl; @@ -100,6 +101,10 @@ impl DataStore { let now = diesel::dsl::now.into_sql::(); dsl::webhook_delivery + // Filter out deliveries triggered by probe requests, as those are + // executed synchronously by the probe endpoint, rather than by the + // webhook deliverator. + .filter(dsl::trigger.ne(WebhookDeliveryTrigger::Probe)) .filter(dsl::time_completed.is_null()) .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) .filter( @@ -332,22 +337,16 @@ mod test { .await .expect("can't create ye event"); - let dispatch1 = WebhookDelivery::new( - &event, - &rx_id, - WebhookDeliveryTrigger::Dispatch, - ); + let dispatch1 = + WebhookDelivery::new(&event, &rx_id, WebhookDeliveryTrigger::Event); let inserted = datastore .webhook_delivery_create_batch(&opctx, vec![dispatch1.clone()]) .await .expect("dispatch 1 should insert"); assert_eq!(inserted, 1, "first dispatched delivery should be created"); - let dispatch2 = WebhookDelivery::new( - &event, - &rx_id, - WebhookDeliveryTrigger::Dispatch, - ); + let dispatch2 = + WebhookDelivery::new(&event, &rx_id, WebhookDeliveryTrigger::Event); let inserted = datastore .webhook_delivery_create_batch(opctx, vec![dispatch2.clone()]) .await diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index f21776f8f0b..591cb1e3629 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -148,7 +148,7 @@ impl WebhookDispatcher { "event_class" => %event.event_class, "glob" => ?sub.glob, ); - WebhookDelivery::new(&event, &rx.id(), WebhookDeliveryTrigger::Dispatch) + WebhookDelivery::new(&event, &rx.id(), WebhookDeliveryTrigger::Event) }).collect(); let subscribed = if !deliveries.is_empty() { diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 5562e5ccf90..45323940891 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1092,7 +1092,6 @@ pub struct WebhookSecretId { } /// A delivery attempt for a webhook event. - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookDelivery { /// The UUID of this delivery attempt. @@ -1114,6 +1113,9 @@ pub struct WebhookDelivery { /// webhook delivery has not yet been attempted (`state` is "pending"). pub time_sent: Option>, + /// Why this delivery was performed. + pub trigger: WebhookDeliveryTrigger, + /// Describes the response returned by the receiver endpoint. /// /// This is present if the webhook has been delivered successfully, or if the @@ -1147,6 +1149,34 @@ pub enum WebhookDeliveryState { FailedTimeout, } +/// The reason a webhook event was delivered +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WebhookDeliveryTrigger { + /// Delivery was triggered by the event occurring for the first time. + Event, + /// Delivery was triggered by a request to resend the event. + Resend, + /// This delivery is a liveness probe. + Probe, +} + +impl WebhookDeliveryTrigger { + pub fn as_str(&self) -> &'static str { + match self { + Self::Event => "event", + Self::Resend => "resend", + Self::Probe => "probe", + } + } +} + +impl fmt::Display for WebhookDeliveryTrigger { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + /// The response received from a webhook receiver endpoint. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookDeliveryResponse { diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 80c4fd1f974..4dc0960e5b2 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5137,10 +5137,12 @@ ON omicron.public.webhook_event ( -- Describes why a webhook delivery was triggered CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( -- This delivery was triggered by the event being dispatched. - 'dispatch', + 'event', -- This delivery was triggered by an explicit call to the webhook event -- resend API. - 'resend' + 'resend', + --- This delivery is a liveness probe. + 'probe' ); CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( @@ -5185,7 +5187,7 @@ ON omicron.public.webhook_delivery ( event_id, rx_id ) WHERE - trigger = 'dispatch'; + trigger = 'event'; -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx From 0a85b401d7746c36bb7ecc5cb546b0d7dad3a743 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 15:18:55 -0800 Subject: [PATCH 081/168] remove `disable_probes` --- nexus/db-model/src/schema.rs | 1 - nexus/db-model/src/webhook_rx.rs | 5 +- .../src/db/datastore/webhook_delivery.rs | 1 - .../db-queries/src/db/datastore/webhook_rx.rs | 11 +--- nexus/tests/integration_tests/webhooks.rs | 6 --- nexus/types/src/external_api/params.rs | 7 --- nexus/types/src/external_api/views.rs | 2 - openapi/nexus.json | 50 +++++++++++++------ schema/crdb/add-webhooks/up01.sql | 4 +- schema/crdb/dbinit.sql | 4 +- 10 files changed, 40 insertions(+), 51 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 224d8af3cb9..204dc684c53 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2190,7 +2190,6 @@ table! { time_modified -> Timestamptz, time_deleted -> Nullable, endpoint -> Text, - probes_enabled -> Bool, rcgen -> Int8, } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index f5f2f650b38..a7937baadfc 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -47,8 +47,7 @@ impl TryFrom for views::Webhook { .into_iter() .map(WebhookSubscriptionKind::into_event_class_string) .collect(); - let WebhookReceiver { identity, endpoint, probes_enabled, rcgen: _ } = - rx; + let WebhookReceiver { identity, endpoint, rcgen: _ } = rx; let WebhookReceiverIdentity { id, name, description, .. } = identity; let endpoint = endpoint.parse().map_err(|e| Error::InternalError { // This is an internal error, as we should not have ever allowed @@ -62,7 +61,6 @@ impl TryFrom for views::Webhook { endpoint, secrets, events, - disable_probes: !probes_enabled, }) } } @@ -84,7 +82,6 @@ pub struct WebhookReceiver { #[diesel(embed)] pub identity: WebhookReceiverIdentity, pub endpoint: String, - pub probes_enabled: bool, /// child resource generation number, per RFD 192 pub rcgen: Generation, diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 1e7a4304675..cc9bd511665 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -318,7 +318,6 @@ mod test { endpoint: "http://webhooks.example.com".parse().unwrap(), secrets: vec!["my cool secret".to_string()], events: vec!["test.*".to_string()], - disable_probes: false, }, ) .await diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 7df81825c3e..4cf43cdbb77 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -52,13 +52,8 @@ impl DataStore { opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; let conn = self.pool_connection_authorized(opctx).await?; - let params::WebhookCreate { - identity, - endpoint, - secrets, - events, - disable_probes, - } = params; + let params::WebhookCreate { identity, endpoint, secrets, events } = + params; let subscriptions = events .into_iter() @@ -79,7 +74,6 @@ impl DataStore { identity.clone(), ), endpoint: endpoint.to_string(), - probes_enabled: !disable_probes, rcgen: Generation::new(), }; let subscriptions = subscriptions.clone(); @@ -509,7 +503,6 @@ mod test { endpoint: format!("http://{name}").parse().unwrap(), secrets: vec![name.to_string()], events: vec![subscription.to_string()], - disable_probes: false, }, ) .await diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 8aa03fdaba2..9519b558c6d 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -138,7 +138,6 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { endpoint, secrets: vec!["my cool secret".to_string()], events: vec!["test.foo".to_string()], - disable_probes: false, }, ) .await; @@ -220,7 +219,6 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { endpoint, secrets: vec![SECRET1.to_string()], events: vec!["test.foo".to_string()], - disable_probes: false, }, ) .await; @@ -323,7 +321,6 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { .expect("this should be a valid URL"), secrets: vec![BAR_SECRET.to_string()], events: vec!["test.foo.bar".to_string()], - disable_probes: false, }, ) .await; @@ -360,7 +357,6 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { .expect("this should be a valid URL"), secrets: vec![BAZ_SECRET.to_string()], events: vec!["test.foo.baz".to_string()], - disable_probes: false, }, ) .await; @@ -397,7 +393,6 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { .expect("this should be a valid URL"), secrets: vec![STAR_SECRET.to_string()], events: vec!["test.foo.*".to_string()], - disable_probes: false, }, ) .await; @@ -486,7 +481,6 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { endpoint, secrets: vec!["my cool secret".to_string()], events: vec!["test.foo".to_string()], - disable_probes: false, }, ) .await; diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index deb5b36b026..3bfccadae60 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2412,10 +2412,6 @@ pub struct WebhookCreate { /// webhook will not be subscribed to any events. #[serde(default)] pub events: Vec, - - /// If `true`, liveness probe requests will not be sent to this webhook receiver. - #[serde(default)] - pub disable_probes: bool, } /// Parameters to update a webhook configuration. @@ -2431,9 +2427,6 @@ pub struct WebhookUpdate { /// /// If this list is empty, the webhook will not be subscribed to any events. pub events: Vec, - - /// If `true`, liveness probe requests will not be sent to this webhook receiver. - pub disable_probes: bool, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 45323940891..cecd21fa5ce 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1075,8 +1075,6 @@ pub struct Webhook { pub secrets: Vec, /// The list of event classes to which this receiver is subscribed. pub events: Vec, - /// If `true`, liveness probe requests are not sent to this receiver. - pub disable_probes: bool, } /// A list of the IDs of secrets associated with a webhook. diff --git a/openapi/nexus.json b/openapi/nexus.json index 25087295b17..68c1d0e5321 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -25053,10 +25053,6 @@ "description": { "type": "string" }, - "disable_probes": { - "description": "If `true`, liveness probe requests are not sent to this receiver.", - "type": "boolean" - }, "endpoint": { "description": "The URL that webhook notification requests are sent to.", "type": "string", @@ -25090,7 +25086,6 @@ }, "required": [ "description", - "disable_probes", "endpoint", "events", "id", @@ -25105,11 +25100,6 @@ "description": { "type": "string" }, - "disable_probes": { - "description": "If `true`, liveness probe requests will not be sent to this webhook receiver.", - "default": false, - "type": "boolean" - }, "endpoint": { "description": "The URL that webhook notification requests should be sent to", "type": "string", @@ -25191,6 +25181,14 @@ "type": "string", "format": "date-time" }, + "trigger": { + "description": "Why this delivery was performed.", + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDeliveryTrigger" + } + ] + }, "webhook_id": { "description": "The UUID of the webhook receiver that this event was delivered to.", "allOf": [ @@ -25205,6 +25203,7 @@ "event_id", "id", "state", + "trigger", "webhook_id" ] }, @@ -25303,6 +25302,32 @@ } ] }, + "WebhookDeliveryTrigger": { + "description": "The reason a webhook event was delivered", + "oneOf": [ + { + "description": "Delivery was triggered by the event occurring for the first time.", + "type": "string", + "enum": [ + "event" + ] + }, + { + "description": "Delivery was triggered by a request to resend the event.", + "type": "string", + "enum": [ + "resend" + ] + }, + { + "description": "This delivery is a liveness probe.", + "type": "string", + "enum": [ + "probe" + ] + } + ] + }, "WebhookSecretCreate": { "type": "object", "properties": { @@ -25350,10 +25375,6 @@ "nullable": true, "type": "string" }, - "disable_probes": { - "description": "If `true`, liveness probe requests will not be sent to this webhook receiver.", - "type": "boolean" - }, "endpoint": { "description": "The URL that webhook notification requests should be sent to", "type": "string", @@ -25376,7 +25397,6 @@ } }, "required": [ - "disable_probes", "endpoint", "events" ] diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql index 76c519d9727..1f3954b46be 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/add-webhooks/up01.sql @@ -9,7 +9,5 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( -- Child resource generation rcgen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. - endpoint STRING(512) NOT NULL, - -- Whether or not liveness probes are sent to this receiver. - probes_enabled BOOL NOT NULL + endpoint STRING(512) NOT NULL ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4dc0960e5b2..1e8e1f770e1 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4978,9 +4978,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( -- Child resource generation rcgen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. - endpoint STRING(512) NOT NULL, - -- Whether or not liveness probes are sent to this receiver. - probes_enabled BOOL NOT NULL + endpoint STRING(512) NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rxs_by_id From 6024b2db8df7d085c154a7bbea0bfaa62e0a4676 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 16:26:58 -0800 Subject: [PATCH 082/168] reticulating probes --- nexus/db-model/src/webhook_delivery.rs | 28 +++++++++++- .../background/tasks/webhook_deliverator.rs | 5 +-- nexus/src/app/webhook.rs | 45 ++++++++++++++++--- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 990ad8359ec..acce4dec6d4 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -12,7 +12,7 @@ use crate::WebhookEvent; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; use omicron_uuid_kinds::{ - OmicronZoneKind, WebhookDeliveryKind, WebhookDeliveryUuid, + OmicronZoneKind, OmicronZoneUuid, WebhookDeliveryKind, WebhookDeliveryUuid, WebhookEventKind, WebhookReceiverKind, WebhookReceiverUuid, }; use serde::Deserialize; @@ -106,6 +106,32 @@ impl WebhookDelivery { time_delivery_started: None, } } + + pub fn new_probe( + rx_id: &WebhookReceiverUuid, + deliverator_id: &OmicronZoneUuid, + ) -> Self { + Self { + // Just kinda make something up... + id: WebhookDeliveryUuid::new_v4().into(), + + // XXX(eliza): hmm, should we just have one UUID for all probe events + // and treat them as redeliveries of one thing? Why or why not? + // UUIDs are basically free, right? On the other hand, if we care about + // not having the event UUID not point to an entry in the events table + // that doesn't exist, perhaps we'd rather just put one entry in there + // for probes rather than create a new one for each probe... + event_id: WebhookEventUud::new_v4().into(), + rx_id: (*rx_id).into(), + trigger: WebhookDeliveryTrigger::Probe, + payload: serde_json::json!({}), + attempts: SqlU8::new(1), + time_created: Utc::now(), + time_completed: None, + deliverator_id: Some((*deliverator_id).into()), + time_delivery_started: Some(Utc::now()), + } + } } /// An individual delivery attempt for a [`WebhookDelivery`]. diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index e37dc2d25f2..d80aa8d9719 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -4,7 +4,7 @@ use crate::app::authz; use crate::app::background::BackgroundTask; use crate::app::db::lookup::LookupPath; -use crate::app::webhook::WebhookReceiverClient; +use crate::app::webhook::ReceiverClient; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; @@ -186,8 +186,7 @@ impl WebhookDeliverator { .webhook_rx_secret_list(opctx, &authz_rx) .await .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; - let mut client = - WebhookReceiverClient::new(&self.client, secrets, &rx)?; + let mut client = ReceiverClient::new(&self.client, secrets, &rx)?; let deliveries = self .datastore diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index f551a4e467f..ae3944f8264 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -110,11 +110,39 @@ impl super::Nexus { Ok(event) } - pub fn webhook_receiver_probe( + pub async fn webhook_receiver_probe( &self, - _rx: lookup::WebhookReceiver<'_>, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, _params: params::WebhookProbe, ) -> Result { + let (authz_rx, rx) = rx.fetch_for(authz::Action::ListChildren).await?; + let secrets = + self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await?; + let mut client = + ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; + let delivery = WebhookDelivery::new_probe(&authz_rx.id(), &self.id); + + let attempt = match outcome = client + .send_delivery_request(opctx, &delivery, WebhookEventClass::Probe) + .await + { + Ok(attempt) => attempt, + Err(e) => { + slog::error!( + &opctx.log, + "failed to probe webhook receiver"; + "rx_id" => %authz_rx.id(), + "rx_name" => %rx.name(), + "delivery_id" => %delivery.id, + "error" => %e, + ); + return Err(Error::InternalError { + internal_message: e.to_string(), + }); + } + }; + todo!() } @@ -183,19 +211,19 @@ pub(super) fn delivery_client( .build() } -pub(crate) struct WebhookReceiverClient<'a> { +pub(crate) struct ReceiverClient<'a> { client: &'a reqwest::Client, rx: &'a WebhookReceiver, secrets: Vec<(WebhookSecretUuid, Hmac)>, hdr_rx_id: http::HeaderValue, } -impl<'a> WebhookReceiverClient<'a> { +impl<'a> ReceiverClient<'a> { pub(crate) fn new( client: &'a reqwest::Client, secrets: impl IntoIterator, rx: &'a WebhookReceiver, - ) -> Result { + ) -> Result { let secrets = secrets .into_iter() .map(|WebhookSecret { identity, secret, .. }| { @@ -204,8 +232,11 @@ impl<'a> WebhookReceiverClient<'a> { (identity.id.into(), mac) }) .collect::>(); - - anyhow::ensure!(!secrets.is_empty(), "receiver has no secrets"); + if secrets.is_empty() { + return Err(Error::invalid_request( + "receiver has no secrets, so delivery requests cannot be sent", + )); + } let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) .expect("UUIDs should always be a valid header value"); Ok(Self { client, secrets, hdr_rx_id, rx }) From e668a3e9ab6d6ac3e2f2e9430d8feb4fe4e023cc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 25 Feb 2025 16:55:56 -0800 Subject: [PATCH 083/168] reticulating probes --- nexus/src/app/webhook.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index ae3944f8264..bc985fbc9e8 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -123,7 +123,7 @@ impl super::Nexus { ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; let delivery = WebhookDelivery::new_probe(&authz_rx.id(), &self.id); - let attempt = match outcome = client + let attempt = match client .send_delivery_request(opctx, &delivery, WebhookEventClass::Probe) .await { @@ -272,6 +272,7 @@ impl<'a> ReceiverClient<'a> { id: WebhookDeliveryUuid, webhook_id: WebhookReceiverUuid, sent_at: &'a str, + trigger: views::WebhookDeliveryTrigger, } // okay, actually do the thing... @@ -285,6 +286,7 @@ impl<'a> ReceiverClient<'a> { id: delivery.id.into(), webhook_id: self.rx.id(), sent_at: &sent_at, + trigger: delivery.trigger.into(), }, }; // N.B. that we serialize the body "ourselves" rather than just @@ -302,6 +304,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "error" => %e, ); @@ -349,6 +352,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "error" => %e, "payload" => ?payload, ); @@ -370,6 +374,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "error" => %e, ); return Err(e).context(MSG); @@ -382,6 +387,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "response_status" => ?status, "response_duration" => ?duration, ); @@ -402,6 +408,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "error" => %e, ); (result, None) @@ -416,6 +423,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "response_status" => ?status, "response_duration" => ?duration, ); @@ -427,6 +435,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, + "delivery_trigger" => %delivery.trigger, "response_status" => ?status, "response_duration" => ?duration, ); From 70050485828b1ed1c04ce39ba7bae3a2e6024d5a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 26 Feb 2025 14:49:11 -0800 Subject: [PATCH 084/168] reticulating probes --- nexus/db-model/src/webhook_delivery.rs | 12 ++++++++++-- nexus/db-model/src/webhook_event_class.rs | 2 ++ schema/crdb/dbinit.sql | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index acce4dec6d4..5e6512f6807 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -13,7 +13,8 @@ use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; use omicron_uuid_kinds::{ OmicronZoneKind, OmicronZoneUuid, WebhookDeliveryKind, WebhookDeliveryUuid, - WebhookEventKind, WebhookReceiverKind, WebhookReceiverUuid, + WebhookEventKind, WebhookEventUuid, WebhookReceiverKind, + WebhookReceiverUuid, }; use serde::Deserialize; use serde::Serialize; @@ -121,7 +122,7 @@ impl WebhookDelivery { // not having the event UUID not point to an entry in the events table // that doesn't exist, perhaps we'd rather just put one entry in there // for probes rather than create a new one for each probe... - event_id: WebhookEventUud::new_v4().into(), + event_id: WebhookEventUuid::new_v4().into(), rx_id: (*rx_id).into(), trigger: WebhookDeliveryTrigger::Probe, payload: serde_json::json!({}), @@ -132,6 +133,13 @@ impl WebhookDelivery { time_delivery_started: Some(Utc::now()), } } + + pub fn to_api_delivery( + &self, + attempt: &WebhookDeliveryAttempt, + ) -> views::WebhookDelivery { + todo!() + } } /// An individual delivery attempt for a [`WebhookDelivery`]. diff --git a/nexus/db-model/src/webhook_event_class.rs b/nexus/db-model/src/webhook_event_class.rs index 09876707866..d6c9ba43964 100644 --- a/nexus/db-model/src/webhook_event_class.rs +++ b/nexus/db-model/src/webhook_event_class.rs @@ -24,6 +24,7 @@ impl_enum_type!( #[diesel(sql_type = WebhookEventClassEnum)] pub enum WebhookEventClass; + Probe => b"probe" TestFoo => b"test.foo" TestFooBar => b"test.foo.bar" TestFooBaz => b"test.foo.baz" @@ -37,6 +38,7 @@ impl WebhookEventClass { // declared a single time, rather than twice (in both `impl_enum_type!` // and here)... match self { + Self::Probe => "probe", Self::TestFoo => "test.foo", Self::TestFooBar => "test.foo.bar", Self::TestFooBaz => "test.foo.baz", diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1e8e1f770e1..33c75cb8e3a 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5011,6 +5011,8 @@ ON omicron.public.webhook_secret ( -- -- When creating new event classes, be sure to add them here! CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( + -- Liveness probes, which are technically not real events, but, you know... + 'probe', -- Test classes used to test globbing. -- -- These are not publicly exposed. From 8cb055a63eb731b3a5bdd8c84567740de1928744 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 26 Feb 2025 15:49:04 -0800 Subject: [PATCH 085/168] reticulating a lot of probes --- nexus/db-model/src/webhook_delivery.rs | 30 ++++++++++++++- nexus/src/app/webhook.rs | 43 ++++++++++++---------- nexus/src/external_api/http_entrypoints.rs | 15 +++++--- nexus/types/src/external_api/views.rs | 7 ++-- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 5e6512f6807..57d7c8dff1d 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -9,8 +9,10 @@ use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; +use crate::WebhookEventClass; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::{ OmicronZoneKind, OmicronZoneUuid, WebhookDeliveryKind, WebhookDeliveryUuid, WebhookEventKind, WebhookEventUuid, WebhookReceiverKind, @@ -136,9 +138,24 @@ impl WebhookDelivery { pub fn to_api_delivery( &self, - attempt: &WebhookDeliveryAttempt, + event_class: WebhookEventClass, + attempt: Option<&WebhookDeliveryAttempt>, ) -> views::WebhookDelivery { - todo!() + views::WebhookDelivery { + id: self.id.into_untyped_uuid(), + webhook_id: self.rx_id.into(), + event_class: event_class.as_str().to_owned(), + event_id: self.event_id.into(), + state: attempt + .map(|attempt| attempt.result.into()) + .unwrap_or(views::WebhookDeliveryState::Pending), + trigger: self.trigger.into(), + response: attempt.and_then(WebhookDeliveryAttempt::response_view), + time_sent: attempt.map(|attempt| attempt.time_created), + attempt: attempt + .map(|attempt| attempt.attempt.0 as usize) + .unwrap_or(1), + } } } @@ -171,6 +188,15 @@ pub struct WebhookDeliveryAttempt { pub time_created: DateTime, } +impl WebhookDeliveryAttempt { + fn response_view(&self) -> Option { + Some(views::WebhookDeliveryResponse { + status: self.response_status? as u16, // i hate that this has to signed in the database... + duration_ms: self.response_duration?.num_milliseconds() as usize, + }) + } +} + impl From for views::WebhookDeliveryState { fn from(result: WebhookDeliveryResult) -> Self { match result { diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index bc985fbc9e8..c51401712f3 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -123,27 +123,30 @@ impl super::Nexus { ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; let delivery = WebhookDelivery::new_probe(&authz_rx.id(), &self.id); - let attempt = match client - .send_delivery_request(opctx, &delivery, WebhookEventClass::Probe) - .await - { - Ok(attempt) => attempt, - Err(e) => { - slog::error!( - &opctx.log, - "failed to probe webhook receiver"; - "rx_id" => %authz_rx.id(), - "rx_name" => %rx.name(), - "delivery_id" => %delivery.id, - "error" => %e, - ); - return Err(Error::InternalError { - internal_message: e.to_string(), - }); - } - }; + const CLASS: WebhookEventClass = WebhookEventClass::Probe; + + let attempt = + match client.send_delivery_request(opctx, &delivery, CLASS).await { + Ok(attempt) => attempt, + Err(e) => { + slog::error!( + &opctx.log, + "failed to probe webhook receiver"; + "rx_id" => %authz_rx.id(), + "rx_name" => %rx.name(), + "delivery_id" => %delivery.id, + "error" => %e, + ); + return Err(Error::InternalError { + internal_message: e.to_string(), + }); + } + }; + + // TODO(eliza): this is where we would resend all the failed stuff + // if requested... - todo!() + Ok(delivery.to_api_delivery(CLASS, Some(&attempt))) } pub async fn webhook_receiver_secret_add( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index f053a757186..787c1c731df 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7424,8 +7424,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_probe( rqctx: RequestContext, - _path_params: Path, - _query_params: Query, + path_params: Path, + query_params: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7434,10 +7434,13 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook_selector = path_params.into_inner(); + let probe_params = query_params.into_inner(); + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let delivery = + nexus.webhook_receiver_probe(&opctx, rx, probe_params).await?; + // TODO(eliza): send the status code that came back from the probe req... + Ok(HttpResponseOk(delivery)) }; apictx .context diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index cecd21fa5ce..234e88b2b4d 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1123,10 +1123,9 @@ pub struct WebhookDelivery { /// "failed_unreachable"). pub response: Option, - /// The UUID of a previous delivery attempt that this is a repeat of, if - /// this was a resending of a previous delivery. If this is the first time - /// this event has been delivered, this is `null`. - pub resent_for: Option, + /// Attempt number, starting at 1. If this is a retry of a previous failed + /// delivery, this value indicates that. + pub attempt: usize, } /// The state of a webhook delivery attempt. From fd706ea997f94213d745df1e3c95f44cd9843436 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 10:33:03 -0800 Subject: [PATCH 086/168] add test for probes --- nexus/tests/integration_tests/webhooks.rs | 223 ++++++++++++++++++---- nexus/types/src/external_api/views.rs | 4 +- 2 files changed, 189 insertions(+), 38 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 9519b558c6d..13cf9e448b7 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -9,6 +9,9 @@ use httpmock::prelude::*; use nexus_db_model::WebhookEventClass; use nexus_db_queries::context::OpContext; use nexus_test_utils::background::activate_background_task; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::{params, views}; @@ -16,23 +19,45 @@ use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use sha2::Sha256; +use std::time::Duration; use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; +const WEBHOOKS_BASE_PATH: &str = "/experimental/v1/webhooks"; + async fn webhook_create( ctx: &ControlPlaneTestContext, params: ¶ms::WebhookCreate, ) -> views::Webhook { resource_helpers::object_create::( &ctx.external_client, - "/experimental/v1/webhooks", + WEBHOOKS_BASE_PATH, params, ) .await } +fn my_great_webhook_params( + mock: &httpmock::MockServer, +) -> params::WebhookCreate { + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: "my-great-webhook".parse().unwrap(), + description: String::from("my great webhook"), + }, + endpoint: mock + .url("/webhooks") + .parse() + .expect("this should be a valid URL"), + secrets: vec![MY_COOL_SECRET.to_string()], + events: vec!["test.foo".to_string()], + } +} + +const MY_COOL_SECRET: &str = "my cool secret"; + async fn secret_add( ctx: &ControlPlaneTestContext, webhook_id: WebhookReceiverUuid, @@ -43,12 +68,34 @@ async fn secret_add( views::WebhookSecretId, >( &ctx.external_client, - &format!("/experimental/v1/webhooks/{webhook_id}/secrets"), + &format!("{WEBHOOKS_BASE_PATH}/{webhook_id}/secrets"), params, ) .await } +async fn webhook_send_probe( + ctx: &ControlPlaneTestContext, + webhook_id: &WebhookReceiverUuid, + resend: bool, + status: http::StatusCode, +) -> views::WebhookDelivery { + let pathparams = if resend { "?resend=true" } else { "" }; + let path = format!("{WEBHOOKS_BASE_PATH}/{webhook_id}/probe{pathparams}"); + NexusRequest::new( + RequestBuilder::new(&ctx.external_client, http::Method::POST, &path) + .expect_status(Some(status)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap_or_else(|e| { + panic!("failed to make \"POST\" request to {path}: {e}") + }) + .parsed_body() + .unwrap() +} + fn is_valid_for_webhook( webhook: &views::Webhook, ) -> impl FnOnce(httpmock::When) -> httpmock::When { @@ -124,23 +171,10 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let server = httpmock::MockServer::start_async().await; let id = WebhookEventUuid::new_v4(); - let endpoint = - server.url("/webhooks").parse().expect("this should be a valid URL"); // Create a webhook receiver. - let webhook = webhook_create( - &cptestctx, - ¶ms::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "my-great-webhook".parse().unwrap(), - description: String::from("my great webhook"), - }, - endpoint, - secrets: vec!["my cool secret".to_string()], - events: vec!["test.foo".to_string()], - }, - ) - .await; + let webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; dbg!(&webhook); let mock = { @@ -161,7 +195,7 @@ async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { .and(is_valid_for_webhook(&webhook)) .is_true(signature_verifies( webhook.secrets[0].id, - "my cool secret".as_bytes().to_vec(), + MY_COOL_SECRET.as_bytes().to_vec(), )) .json_body_includes(body); then.status(200); @@ -467,23 +501,10 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { let server = httpmock::MockServer::start_async().await; let id = WebhookEventUuid::new_v4(); - let endpoint = - server.url("/webhooks").parse().expect("this should be a valid URL"); // Create a webhook receiver. - let webhook = webhook_create( - &cptestctx, - ¶ms::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: "my-great-webhook".parse().unwrap(), - description: String::from("my great webhook"), - }, - endpoint, - secrets: vec!["my cool secret".to_string()], - events: vec!["test.foo".to_string()], - }, - ) - .await; + let webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; dbg!(&webhook); let mock = { @@ -504,7 +525,7 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { .and(is_valid_for_webhook(&webhook)) .is_true(signature_verifies( webhook.secrets[0].id, - "my cool secret".as_bytes().to_vec(), + MY_COOL_SECRET.as_bytes().to_vec(), )) .json_body_includes(body); then.status(500); @@ -560,7 +581,7 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { .and(is_valid_for_webhook(&webhook)) .is_true(signature_verifies( webhook.secrets[0].id, - "my cool secret".as_bytes().to_vec(), + MY_COOL_SECRET.as_bytes().to_vec(), )) .json_body_includes(body); then.status(503); @@ -601,7 +622,7 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { .and(is_valid_for_webhook(&webhook)) .is_true(signature_verifies( webhook.secrets[0].id, - "my cool secret".as_bytes().to_vec(), + MY_COOL_SECRET.as_bytes().to_vec(), )) .json_body_includes(body); then.status(200); @@ -622,3 +643,133 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { ); mock.assert_async().await; } + +#[nexus_test] +async fn test_probe(cptestctx: &ControlPlaneTestContext) { + let nexus = cptestctx.server.server_context().nexus.clone(); + + let datastore = nexus.datastore(); + let server = httpmock::MockServer::start_async().await; + + // Create a webhook receiver. + let webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&webhook); + + let body = serde_json::json!({ + "event_class": "probe", + "data": {} + }) + .to_string(); + + // First, configure the receiver server to return a successful response but + // only after the delivery timeout has elapsed. + let mock = server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then + // Delivery timeout is 30 seconds. + .delay(Duration::from_secs(35)) + // After the timeout, return something that would be considered + // a success. + .status(200); + }) + .await; + + // Send a probe. The probe should fail due to a timeout. + let probe1 = webhook_send_probe( + &cptestctx, + &webhook.id, + false, + http::StatusCode::OK, + ) + .await; + dbg!(probe1); + + mock.assert_async().await; + + assert_eq!(probe1.attempt, 1); + assert_eq!(probe1.event_class, "probe"); + assert_eq!(probe1.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe1.state, views::WebhookDeliveryState::FailedTimeout); + + // Next, configure the receiver server to return a 5xx error + mock.delete_async().await; + let mock = server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(503); + }) + .await; + + let probe2 = webhook_send_probe( + &cptestctx, + &webhook.id, + false, + http::StatusCode::OK, + ) + .await; + dbg!(probe2); + + mock.assert_async().await; + assert_eq!(probe2.attempt, 1); + assert_eq!(probe2.event_class, "probe"); + assert_eq!(probe2.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe2.state, views::WebhookDeliveryState::FailedHttpError); + assert_ne!( + probe2.id, probe1.id, + "a new delivery ID should be assigned to each probe" + ); + + mock.delete_async().await; + // Finally, configure the receiver server to return a success. + let mock = server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(200); + }) + .await; + + let probe3 = webhook_send_probe( + &cptestctx, + &webhook.id, + false, + http::StatusCode::OK, + ) + .await; + dbg!(probe3); + mock.assert_async().await; + assert_eq!(probe3.attempt, 1); + assert_eq!(probe3.event_class, "probe"); + assert_eq!(probe3.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe3.state, views::WebhookDeliveryState::Delivered); + assert_ne!( + probe3.id, probe1.id, + "a new delivery ID should be assigned to each probe" + ); + assert_ne!( + probe3.id, probe2.id, + "a new delivery ID should be assigned to each probe" + ); +} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 234e88b2b4d..5a7b425e5ba 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1129,7 +1129,7 @@ pub struct WebhookDelivery { } /// The state of a webhook delivery attempt. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum WebhookDeliveryState { /// The webhook event has not yet been delivered. @@ -1147,7 +1147,7 @@ pub enum WebhookDeliveryState { } /// The reason a webhook event was delivered -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum WebhookDeliveryTrigger { /// Delivery was triggered by the event occurring for the first time. From ca1f998ecdec9ce2a8c6039cc4911e27c9f51391 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 10:33:15 -0800 Subject: [PATCH 087/168] fix probes having attempt 2 --- nexus/db-model/src/webhook_delivery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 57d7c8dff1d..a200b472498 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -128,7 +128,7 @@ impl WebhookDelivery { rx_id: (*rx_id).into(), trigger: WebhookDeliveryTrigger::Probe, payload: serde_json::json!({}), - attempts: SqlU8::new(1), + attempts: SqlU8::new(0), time_created: Utc::now(), time_completed: None, deliverator_id: Some((*deliverator_id).into()), From 63ad0c8a2a756d83f91f0ccd6d28593c8f051478 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 10:41:10 -0800 Subject: [PATCH 088/168] fixup probe test --- nexus/tests/integration_tests/webhooks.rs | 113 ++++++++++++---------- 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 13cf9e448b7..03749c02f97 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -646,9 +646,6 @@ async fn test_retry_backoff(cptestctx: &ControlPlaneTestContext) { #[nexus_test] async fn test_probe(cptestctx: &ControlPlaneTestContext) { - let nexus = cptestctx.server.server_context().nexus.clone(); - - let datastore = nexus.datastore(); let server = httpmock::MockServer::start_async().await; // Create a webhook receiver. @@ -664,24 +661,30 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { // First, configure the receiver server to return a successful response but // only after the delivery timeout has elapsed. - let mock = server - .mock_async(move |when, then| { - when.method(POST) - .header("x-oxide-event-class", "probe") - .and(is_valid_for_webhook(&webhook)) - .is_true(signature_verifies( - webhook.secrets[0].id, - MY_COOL_SECRET.as_bytes().to_vec(), - )) - .json_body_includes(body); - then - // Delivery timeout is 30 seconds. - .delay(Duration::from_secs(35)) - // After the timeout, return something that would be considered - // a success. - .status(200); - }) - .await; + let mock = { + let webhook = webhook.clone(); + let body = body.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then + // Delivery timeout is 30 seconds. + // TODO(eliza): it would be really nice if this test didn't + // have to wait 30 seconds... + .delay(Duration::from_secs(35)) + // After the timeout, return something that would be considered + // a success. + .status(200); + }) + .await + }; // Send a probe. The probe should fail due to a timeout. let probe1 = webhook_send_probe( @@ -691,7 +694,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { http::StatusCode::OK, ) .await; - dbg!(probe1); + dbg!(&probe1); mock.assert_async().await; @@ -702,19 +705,23 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { // Next, configure the receiver server to return a 5xx error mock.delete_async().await; - let mock = server - .mock_async(move |when, then| { - when.method(POST) - .header("x-oxide-event-class", "probe") - .and(is_valid_for_webhook(&webhook)) - .is_true(signature_verifies( - webhook.secrets[0].id, - MY_COOL_SECRET.as_bytes().to_vec(), - )) - .json_body_includes(body); - then.status(503); - }) - .await; + let mock = { + let webhook = webhook.clone(); + let body = body.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(503); + }) + .await + }; let probe2 = webhook_send_probe( &cptestctx, @@ -723,7 +730,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { http::StatusCode::OK, ) .await; - dbg!(probe2); + dbg!(&probe2); mock.assert_async().await; assert_eq!(probe2.attempt, 1); @@ -736,20 +743,24 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { ); mock.delete_async().await; - // Finally, configure the receiver server to return a success. - let mock = server - .mock_async(move |when, then| { - when.method(POST) - .header("x-oxide-event-class", "probe") - .and(is_valid_for_webhook(&webhook)) - .is_true(signature_verifies( - webhook.secrets[0].id, - MY_COOL_SECRET.as_bytes().to_vec(), - )) - .json_body_includes(body); - then.status(200); - }) - .await; + // Finally, configure the receiver server to return a success. + let mock = { + let webhook = webhook.clone(); + let body = body.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(200); + }) + .await + }; let probe3 = webhook_send_probe( &cptestctx, @@ -758,7 +769,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { http::StatusCode::OK, ) .await; - dbg!(probe3); + dbg!(&probe3); mock.assert_async().await; assert_eq!(probe3.attempt, 1); assert_eq!(probe3.event_class, "probe"); From 6d07590b8a84773fee7cd5ef1735faaddf0d3ea5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 12:25:12 -0800 Subject: [PATCH 089/168] reticulating delivery list --- nexus/db-fixed-data/src/lib.rs | 4 +- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_delivery.rs | 20 ++++--- .../db-model/src/webhook_delivery_trigger.rs | 5 ++ nexus/db-model/src/webhook_event.rs | 6 ++ .../src/db/datastore/webhook_delivery.rs | 55 ++++++++++++++++--- schema/crdb/dbinit.sql | 20 +++++++ 7 files changed, 93 insertions(+), 18 deletions(-) diff --git a/nexus/db-fixed-data/src/lib.rs b/nexus/db-fixed-data/src/lib.rs index 0bfc0127ff2..f97e6e35bad 100644 --- a/nexus/db-fixed-data/src/lib.rs +++ b/nexus/db-fixed-data/src/lib.rs @@ -21,7 +21,7 @@ // these are valid v4 uuids, and they're as unlikely to collide with a future // uuid as any random uuid is.) // -// The specific kinds of resources to which we've assigned uuids: +// The specific kinds of resources to which we've assigned uuids:| // // UUID PREFIX RESOURCE // 001de000-05e4 built-in users ("05e4" looks a bit like "user") @@ -31,6 +31,8 @@ // 001de000-074c built-in services vpc // 001de000-c470 built-in services vpc subnets // 001de000-all0 singleton ID for source IP allowlist ("all0" is like "allow") +// 001de000-7768 singleton ID for webhook probe event ('wh' for 'webhook' +// is ascii 0x77 0x68). use std::sync::LazyLock; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 204dc684c53..34ee5600141 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2263,6 +2263,7 @@ table! { allow_tables_to_appear_in_same_query!(webhook_receiver, webhook_delivery); joinable!(webhook_delivery -> webhook_receiver (rx_id)); allow_tables_to_appear_in_same_query!(webhook_delivery, webhook_event); +allow_tables_to_appear_in_same_query!(webhook_delivery_attempt, webhook_event); joinable!(webhook_delivery -> webhook_event (event_id)); table! { diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index a200b472498..2a637ca10b3 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -35,6 +35,7 @@ impl_enum_type!( FromSqlRow, Serialize, Deserialize, + strum::VariantArray, )] #[diesel(sql_type = WebhookDeliveryResultEnum)] pub enum WebhookDeliveryResult; @@ -117,14 +118,13 @@ impl WebhookDelivery { Self { // Just kinda make something up... id: WebhookDeliveryUuid::new_v4().into(), - - // XXX(eliza): hmm, should we just have one UUID for all probe events - // and treat them as redeliveries of one thing? Why or why not? - // UUIDs are basically free, right? On the other hand, if we care about - // not having the event UUID not point to an entry in the events table - // that doesn't exist, perhaps we'd rather just put one entry in there - // for probes rather than create a new one for each probe... - event_id: WebhookEventUuid::new_v4().into(), + // There's a singleton entry in the `webhook_event` table for + // probes, so that we can reference a real event ID but need not + // create a bunch of duplicate empty events every time a probe is sent. + event_id: WebhookEventUuid::from_untyped_uuid( + WebhookEvent::PROBE_EVENT_ID, + ) + .into(), rx_id: (*rx_id).into(), trigger: WebhookDeliveryTrigger::Probe, payload: serde_json::json!({}), @@ -215,3 +215,7 @@ impl From for views::WebhookDeliveryState { } } } + +impl WebhookDeliveryResult { + pub const ALL: &'static [Self] = ::VARIANTS; +} diff --git a/nexus/db-model/src/webhook_delivery_trigger.rs b/nexus/db-model/src/webhook_delivery_trigger.rs index a7a5b3d7b97..640372b44e9 100644 --- a/nexus/db-model/src/webhook_delivery_trigger.rs +++ b/nexus/db-model/src/webhook_delivery_trigger.rs @@ -22,6 +22,7 @@ impl_enum_type!( Deserialize, AsExpression, FromSqlRow, + strum::VariantArray, )] #[diesel(sql_type = WebhookDeliveryTriggerEnum)] #[serde(rename_all = "snake_case")] @@ -33,6 +34,10 @@ impl_enum_type!( ); +impl WebhookDeliveryTrigger { + pub const ALL: &'static [Self] = ::VARIANTS; +} + impl fmt::Display for WebhookDeliveryTrigger { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // Forward to the canonical implementation in nexus-types. diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs index 5ac53bc9721..ac9d62045f4 100644 --- a/nexus/db-model/src/webhook_event.rs +++ b/nexus/db-model/src/webhook_event.rs @@ -42,3 +42,9 @@ pub struct WebhookEvent { pub num_dispatched: i64, } + +impl WebhookEvent { + /// UUID of the singleton event entry for webhook liveness probes. + pub const PROBE_EVENT_ID: uuid::Uuid = + uuid::Uuid::from_u128(0x001de000_7768_4000_8000_000000000001); +} diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index cc9bd511665..81815be4e87 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -11,9 +11,10 @@ use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; +use crate::db::model::WebhookDeliveryResult; use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEventClass; -use crate::db::pagination::paginated; +use crate::db::pagination::paginated_multicolumn; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; @@ -77,15 +78,39 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - pub async fn webhook_rx_delivery_list( + pub async fn webhook_rx_delivery_list_attempts( &self, opctx: &OpContext, rx_id: &WebhookReceiverUuid, - pagparams: &DataPageParams<'_, Uuid>, - ) -> ListResultVec { + triggers: &'static [WebhookDeliveryTrigger], + results: &'static [WebhookDeliveryResult], + pagparams: &DataPageParams<'_, (Uuid, SqlU8)>, + ) -> ListResultVec<( + WebhookDelivery, + Option, + WebhookEventClass, + )> { let conn = self.pool_connection_authorized(opctx).await?; - paginated(dsl::webhook_delivery, dsl::id, pagparams) + let query = dsl::webhook_delivery.left_join( + attempt_dsl::webhook_delivery_attempt + .on(dsl::id.eq(attempt_dsl::delivery_id)), + ); + paginated_multicolumn(query, (dsl::id, attempt_dsl::attempt), pagparams) .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) + .filter(dsl::trigger.eq_any(triggers)) + .filter( + attempt_dsl::result + .eq_any(results) + .or(attempt_dsl::result.is_null()), + ) + .inner_join( + event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), + ) + .select(( + WebhookDelivery::as_select(), + Option::::as_select(), + event_dsl::event_class, + )) .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) @@ -388,17 +413,29 @@ mod test { Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); while let Some(p) = paginator.next() { let deliveries = datastore - .webhook_rx_delivery_list( + .webhook_rx_delivery_list_attempts( &opctx, &rx_id, + WebhookDeliveryTrigger::ALL, + WebhookDeliveryResult::ALL, &p.current_pagparams(), ) .await .unwrap(); - paginator = p.found_batch(&deliveries, &|d: &WebhookDelivery| { - d.id.into_untyped_uuid() + paginator = p.found_batch(&deliveries, &|(d, a, _): &( + WebhookDelivery, + Option, + WebhookEventClass, + )| { + ( + *d.id.as_untyped_uuid(), + a.as_ref() + .map(|attempt| attempt.attempt) + .unwrap_or(SqlU8::new(0)), + ) }); - all_deliveries.extend(deliveries.into_iter().map(|d| d.id)); + all_deliveries + .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); } assert!(all_deliveries.contains(&dispatch1.id)); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 33c75cb8e3a..9bcfe5e3042 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5121,6 +5121,26 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( ) ); +-- Singleton probe event +INSERT INTO omicron.public.webhook_event ( + id, + time_created, + event_class, + event, + time_dispatched, + num_dispatched +) VALUES ( + -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. + '001de000-7768-4000-8000-000000000001', + NOW(), + 'probe', + '{}', + -- Pretend to be dispatched so we won't show up in "list events needing + -- dispatch" queries + NOW(), + 0 +) ON CONFLICT DO NOTHING; + -- Look up webhook events in need of dispatching. -- -- This is used by the message dispatcher when looking for events to dispatch. From 70032b0333eede3b3a56ff02634434072d528029 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 13:24:43 -0800 Subject: [PATCH 090/168] reticulating delivery list query --- nexus/db-model/src/webhook_delivery.rs | 33 +++-- .../src/db/datastore/webhook_delivery.rs | 122 +++++++++++++++--- nexus/tests/integration_tests/webhooks.rs | 8 +- nexus/types/src/external_api/shared.rs | 18 +++ nexus/types/src/external_api/views.rs | 20 +-- 5 files changed, 149 insertions(+), 52 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 2a637ca10b3..96d05d81311 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -11,6 +11,7 @@ use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; use crate::WebhookEventClass; use chrono::{DateTime, TimeDelta, Utc}; +use nexus_types::external_api::shared::WebhookDeliveryState; use nexus_types::external_api::views; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::{ @@ -148,7 +149,7 @@ impl WebhookDelivery { event_id: self.event_id.into(), state: attempt .map(|attempt| attempt.result.into()) - .unwrap_or(views::WebhookDeliveryState::Pending), + .unwrap_or(WebhookDeliveryState::Pending), trigger: self.trigger.into(), response: attempt.and_then(WebhookDeliveryAttempt::response_view), time_sent: attempt.map(|attempt| attempt.time_created), @@ -197,25 +198,41 @@ impl WebhookDeliveryAttempt { } } -impl From for views::WebhookDeliveryState { +impl From for WebhookDeliveryState { fn from(result: WebhookDeliveryResult) -> Self { match result { WebhookDeliveryResult::FailedHttpError => { - views::WebhookDeliveryState::FailedHttpError + WebhookDeliveryState::FailedHttpError } WebhookDeliveryResult::FailedTimeout => { - views::WebhookDeliveryState::FailedTimeout + WebhookDeliveryState::FailedTimeout } WebhookDeliveryResult::FailedUnreachable => { - views::WebhookDeliveryState::FailedUnreachable - } - WebhookDeliveryResult::Succeeded => { - views::WebhookDeliveryState::Delivered + WebhookDeliveryState::FailedUnreachable } + WebhookDeliveryResult::Succeeded => WebhookDeliveryState::Delivered, } } } impl WebhookDeliveryResult { pub const ALL: &'static [Self] = ::VARIANTS; + + pub fn from_api_state(state: WebhookDeliveryState) -> Option { + match state { + WebhookDeliveryState::FailedHttpError => { + Some(WebhookDeliveryResult::FailedHttpError) + } + WebhookDeliveryState::FailedTimeout => { + Some(WebhookDeliveryResult::FailedTimeout) + } + WebhookDeliveryState::FailedUnreachable => { + Some(WebhookDeliveryResult::FailedUnreachable) + } + WebhookDeliveryState::Delivered => { + Some(WebhookDeliveryResult::Succeeded) + } + WebhookDeliveryState::Pending => None, + } + } } diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 81815be4e87..80afd823b24 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -25,6 +25,7 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; use chrono::{DateTime, Utc}; use diesel::prelude::*; +use nexus_types::external_api::shared::WebhookDeliveryState; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -83,7 +84,7 @@ impl DataStore { opctx: &OpContext, rx_id: &WebhookReceiverUuid, triggers: &'static [WebhookDeliveryTrigger], - results: &'static [WebhookDeliveryResult], + states: impl IntoIterator, pagparams: &DataPageParams<'_, (Uuid, SqlU8)>, ) -> ListResultVec<( WebhookDelivery, @@ -91,29 +92,108 @@ impl DataStore { WebhookEventClass, )> { let conn = self.pool_connection_authorized(opctx).await?; + // The way we construct this query is a bit complex, so let's break down + // why we do it this way. + // + // We would like to list delivery attempts that are in the provided + // `states`. If a delivery has been attempted at least once, there will + // be a record in the `webhook_delivery_attempts` table including a + // `result` column that contains a `WebhookDeliveryResult`. The + // `WebhookDeliveryResult` SQL enum represents the subset of + // `WebhookDeliveryState`s that are not `Pending` (e.g. the delivery + // attempt has succeeded or failed). On the other hand, if a delivery + // has not yet been attempted, there will be *no* corresponding records + // in `webhook_delivery_attempts`. So, based on whether or not the + // requested list of states includes `Pending`, we either want to select + // all delivery records where there is a corresponding attempt record in + // one of the requested states (if we don't want `Pending` deliveries), + // *OR*, we want to select all deliveries where there is a corresponding + // attempt record in the requested states *and* all delivery records + // where there is no corresponding attempt record (because the delivery + // is pending). Due to diesel being Like That, whether or not we add an + // `OR result IS NULL` clause changes the type of the query being built, + // so we must do that last, and execute the query in one of two `if` + // branches based on whether or not this clause is added. + // + // Additionally, we must paginate this query by both delivery ID and + // attempt number, since we may return multiple attempts of the same + // delivery. Because `paginated_multicolumn` requiers that the + // paginated expression implement `diesel::QuerySource`, we must first + // construct a selection by JOINing the delivery table and the attempt + // table, without applying any filters, pass the joined tables to + // `paginated_multicolumn`, and _then_ filtering the paginated query and + // JOINing again with the `event` table to get the event class as well. + // + // Neither of these weird contortions actually change the resultant SQL + // for the query, but they make the code for cosntructing it a bit + // wacky, so I figured it was worth writing this down for future + // generations. + let mut includes_pending = false; + let states = states + .into_iter() + .filter_map(|state| { + if state == WebhookDeliveryState::Pending { + includes_pending = true; + } + WebhookDeliveryResult::from_api_state(state) + }) + .collect::>(); + + // Join the delivery table with the attempts table. If a delivery has + // not been attempted yet, there will be no attempts, so this is a LEFT + // JOIN. let query = dsl::webhook_delivery.left_join( attempt_dsl::webhook_delivery_attempt .on(dsl::id.eq(attempt_dsl::delivery_id)), ); - paginated_multicolumn(query, (dsl::id, attempt_dsl::attempt), pagparams) - .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) - .filter(dsl::trigger.eq_any(triggers)) - .filter( - attempt_dsl::result - .eq_any(results) - .or(attempt_dsl::result.is_null()), - ) - .inner_join( - event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), - ) - .select(( - WebhookDelivery::as_select(), - Option::::as_select(), - event_dsl::event_class, - )) - .load_async(&*conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + + // Paginate the query, ordering by the delivery ID and then the attempt + // number of the attempt. + let query = paginated_multicolumn( + query, + (dsl::id, attempt_dsl::attempt), + pagparams, + ) + // Select only deliveries that are to the receiver we're interested in, + // and were initiated by the triggers we're interested in. + .filter( + dsl::rx_id + .eq(rx_id.into_untyped_uuid()) + .and(dsl::trigger.eq_any(triggers)), + ) + // Finally, join again with the event table on the delivery's event ID, + // so that we can grab the event class of the event that initiated this delivery. + .inner_join( + event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), + ) + .select(( + WebhookDelivery::as_select(), + Option::::as_select(), + event_dsl::event_class, + )); + + // Before we actually execute the query, add a filter clause to select + // only attempts in the requested states. This branches on + // `includes_pending` as whether or not we care about pending deliveries + // adds an additional clause, changing the type of all the diesel junk + // and preventing us from assigning the query to the same variable in + // both cases, so we just run it immediatel. + if includes_pending { + query + .filter( + attempt_dsl::result + .eq_any(states) + .or(attempt_dsl::result.is_null()), + ) + .load_async(&*conn) + .await + } else { + query + .filter(attempt_dsl::result.eq_any(states)) + .load_async(&*conn) + .await + } + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn webhook_rx_delivery_list_ready( @@ -417,7 +497,7 @@ mod test { &opctx, &rx_id, WebhookDeliveryTrigger::ALL, - WebhookDeliveryResult::ALL, + std::iter::once(WebhookDeliveryState::Pending), &p.current_pagparams(), ) .await diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 03749c02f97..5694baa761d 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -14,7 +14,7 @@ use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; -use nexus_types::external_api::{params, views}; +use nexus_types::external_api::{params, shared, views}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; @@ -701,7 +701,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { assert_eq!(probe1.attempt, 1); assert_eq!(probe1.event_class, "probe"); assert_eq!(probe1.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe1.state, views::WebhookDeliveryState::FailedTimeout); + assert_eq!(probe1.state, shared::WebhookDeliveryState::FailedTimeout); // Next, configure the receiver server to return a 5xx error mock.delete_async().await; @@ -736,7 +736,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { assert_eq!(probe2.attempt, 1); assert_eq!(probe2.event_class, "probe"); assert_eq!(probe2.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe2.state, views::WebhookDeliveryState::FailedHttpError); + assert_eq!(probe2.state, shared::WebhookDeliveryState::FailedHttpError); assert_ne!( probe2.id, probe1.id, "a new delivery ID should be assigned to each probe" @@ -774,7 +774,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { assert_eq!(probe3.attempt, 1); assert_eq!(probe3.event_class, "probe"); assert_eq!(probe3.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe3.state, views::WebhookDeliveryState::Delivered); + assert_eq!(probe3.state, shared::WebhookDeliveryState::Delivered); assert_ne!( probe3.id, probe1.id, "a new delivery ID should be assigned to each probe" diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs index ae8214a637c..344552eeb43 100644 --- a/nexus/types/src/external_api/shared.rs +++ b/nexus/types/src/external_api/shared.rs @@ -510,3 +510,21 @@ impl RelayState { .context("json from relay state string") } } + +/// The state of a webhook delivery attempt. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WebhookDeliveryState { + /// The webhook event has not yet been delivered. + Pending, + /// The webhook event has been delivered successfully. + Delivered, + /// A webhook request was sent to the endpoint, and it + /// returned a HTTP error status code indicating an error. + FailedHttpError, + /// The webhook request could not be sent to the receiver endpoint. + FailedUnreachable, + /// A connection to the receiver endpoint was successfully established, but + /// no response was received within the delivery timeout. + FailedTimeout, +} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 5a7b425e5ba..57560e1ec7d 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1105,7 +1105,7 @@ pub struct WebhookDelivery { pub event_id: WebhookEventUuid, /// The state of the delivery attempt. - pub state: WebhookDeliveryState, + pub state: shared::WebhookDeliveryState, /// The time at which the webhook delivery was attempted, or `null` if /// webhook delivery has not yet been attempted (`state` is "pending"). @@ -1128,24 +1128,6 @@ pub struct WebhookDelivery { pub attempt: usize, } -/// The state of a webhook delivery attempt. -#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub enum WebhookDeliveryState { - /// The webhook event has not yet been delivered. - Pending, - /// The webhook event has been delivered successfully. - Delivered, - /// A webhook request was sent to the endpoint, and it - /// returned a HTTP error status code indicating an error. - FailedHttpError, - /// The webhook request could not be sent to the receiver endpoint. - FailedUnreachable, - /// A connection to the receiver endpoint was successfully established, but - /// no response was received within the delivery timeout. - FailedTimeout, -} - /// The reason a webhook event was delivered #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] From 3eb73689cb7e3591a13f11f1e4e0017030812611 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 27 Feb 2025 15:29:49 -0800 Subject: [PATCH 091/168] query for listing failed resendable deliveries --- nexus/db-model/src/schema.rs | 3 +- nexus/db-model/src/webhook_delivery.rs | 30 +++--- .../src/db/datastore/webhook_delivery.rs | 97 ++++++++++++++++++- .../webhook_rx_list_resendable_events.sql | 24 +++++ schema/crdb/dbinit.sql | 14 ++- 5 files changed, 147 insertions(+), 21 deletions(-) create mode 100644 nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 34ee5600141..e8dbbf9ef42 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2256,7 +2256,8 @@ table! { time_created -> Timestamptz, time_completed -> Nullable, deliverator_id -> Nullable, - time_delivery_started -> Nullable + time_delivery_started -> Nullable, + failed_permanently -> Bool, } } diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 96d05d81311..cf80117305b 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -87,6 +87,8 @@ pub struct WebhookDelivery { /// or permanently failed. pub time_completed: Option>, + pub failed_permanently: bool, + pub deliverator_id: Option>, pub time_delivery_started: Option>, } @@ -109,6 +111,7 @@ impl WebhookDelivery { time_completed: None, deliverator_id: None, time_delivery_started: None, + failed_permanently: false, } } @@ -134,6 +137,7 @@ impl WebhookDelivery { time_completed: None, deliverator_id: Some((*deliverator_id).into()), time_delivery_started: Some(Utc::now()), + failed_permanently: false, } } @@ -201,16 +205,10 @@ impl WebhookDeliveryAttempt { impl From for WebhookDeliveryState { fn from(result: WebhookDeliveryResult) -> Self { match result { - WebhookDeliveryResult::FailedHttpError => { - WebhookDeliveryState::FailedHttpError - } - WebhookDeliveryResult::FailedTimeout => { - WebhookDeliveryState::FailedTimeout - } - WebhookDeliveryResult::FailedUnreachable => { - WebhookDeliveryState::FailedUnreachable - } - WebhookDeliveryResult::Succeeded => WebhookDeliveryState::Delivered, + WebhookDeliveryResult::FailedHttpError => Self::FailedHttpError, + WebhookDeliveryResult::FailedTimeout => Self::FailedTimeout, + WebhookDeliveryResult::FailedUnreachable => Self::FailedUnreachable, + WebhookDeliveryResult::Succeeded => Self::Delivered, } } } @@ -221,17 +219,13 @@ impl WebhookDeliveryResult { pub fn from_api_state(state: WebhookDeliveryState) -> Option { match state { WebhookDeliveryState::FailedHttpError => { - Some(WebhookDeliveryResult::FailedHttpError) - } - WebhookDeliveryState::FailedTimeout => { - Some(WebhookDeliveryResult::FailedTimeout) + Some(Self::FailedHttpError) } + WebhookDeliveryState::FailedTimeout => Some(Self::FailedTimeout), WebhookDeliveryState::FailedUnreachable => { - Some(WebhookDeliveryResult::FailedUnreachable) - } - WebhookDeliveryState::Delivered => { - Some(WebhookDeliveryResult::Succeeded) + Some(Self::FailedUnreachable) } + WebhookDeliveryState::Delivered => Some(Self::Succeeded), WebhookDeliveryState::Pending => None, } } diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 80afd823b24..b6f5a9b3d6f 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -6,6 +6,7 @@ use super::DataStore; use crate::context::OpContext; +use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; @@ -13,8 +14,10 @@ use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; use crate::db::model::WebhookDeliveryResult; use crate::db::model::WebhookDeliveryTrigger; +use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; use crate::db::pagination::paginated_multicolumn; +use crate::db::schema; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; use crate::db::schema::webhook_event::dsl as event_dsl; @@ -79,6 +82,54 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Returns a list of all permanently-failed deliveries which are eligible + /// to resend should a liveness probe with `resend=true` succeed. + pub async fn webhook_rx_list_resendable_events( + &self, + opctx: &OpContext, + rx_id: &WebhookReceiverUuid, + ) -> ListResultVec { + Self::rx_list_resendable_events_query(*rx_id) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + fn rx_list_resendable_events_query( + rx_id: WebhookReceiverUuid, + ) -> impl RunnableQuery { + use diesel::dsl::*; + let (delivery, also_delivery) = diesel::alias!( + schema::webhook_delivery as delivery, + schema::webhook_delivery as also_delivey + ); + event_dsl::webhook_event + .filter(event_dsl::event_class.ne(WebhookEventClass::Probe)) + .inner_join( + delivery.on(delivery.field(dsl::event_id).eq(event_dsl::id)), + ) + .filter(delivery.field(dsl::rx_id).eq(rx_id.into_untyped_uuid())) + .filter(not(exists( + also_delivery + .select(also_delivery.field(dsl::id)) + .filter( + also_delivery.field(dsl::event_id).eq(event_dsl::id), + ) + .filter( + also_delivery.field(dsl::failed_permanently).eq(&false), + ) + .filter( + also_delivery + .field(dsl::trigger) + .ne(WebhookDeliveryTrigger::Probe), + ), + ))) + .select(WebhookEvent::as_select()) + // the inner join means we may return the same event multiple times, + // so only return distinct events. + .distinct() + } + pub async fn webhook_rx_delivery_list_attempts( &self, opctx: &OpContext, @@ -331,7 +382,7 @@ impl DataStore { // its retry attempts? let succeeded = attempt.result == nexus_db_model::WebhookDeliveryResult::Succeeded; - let failed_permanently = *attempt.attempt >= MAX_ATTEMPTS; + let failed_permanently = !succeeded && *attempt.attempt >= MAX_ATTEMPTS; let (completed, new_nexus_id) = if succeeded || failed_permanently { // If the delivery has succeeded or failed permanently, set the // "time_completed" timestamp to mark it as finished. Also, leave @@ -342,6 +393,7 @@ impl DataStore { // Otherwise, "unlock" the delivery for other nexii. (None, None) }; + let prev_attempts = SqlU8::new((*attempt.attempt) - 1); let UpdateAndQueryResult { status, found } = diesel::update(dsl::webhook_delivery) @@ -356,6 +408,7 @@ impl DataStore { // in place and use it to determine the attempt number? dsl::attempts.eq(attempt.attempt), dsl::deliverator_id.eq(new_nexus_id), + dsl::failed_permanently.eq(failed_permanently), )) .check_if_exists::(delivery.id) .execute_and_check(&conn) @@ -394,9 +447,11 @@ impl DataStore { #[cfg(test)] mod test { use super::*; + use crate::db::explain::ExplainableAsync; use crate::db::model::WebhookDeliveryTrigger; use crate::db::pagination::Paginator; use crate::db::pub_test_utils::TestDatabase; + use crate::db::raw_query_builder::expectorate_query_contents; use nexus_types::external_api::params; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; @@ -526,4 +581,44 @@ mod test { db.terminate().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn expectorate_rx_list_resendable() { + let query = DataStore::rx_list_resendable_events_query( + WebhookReceiverUuid::nil(), + ); + + expectorate_query_contents( + &query, + "tests/output/webhook_rx_list_resendable_events.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_rx_list_resendable_events() { + let logctx = dev::test_setup_log("explain_rx_list_resendable_events"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let query = DataStore::rx_list_resendable_events_query( + WebhookReceiverUuid::nil(), + ); + let explanation = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + eprintln!("{explanation}"); + + assert!( + !explanation.contains("FULL SCAN"), + "Found an unexpected FULL SCAN: {}", + explanation + ); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql new file mode 100644 index 00000000000..e1597c83fbd --- /dev/null +++ b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql @@ -0,0 +1,24 @@ +SELECT + DISTINCT + webhook_event.id, + webhook_event.time_created, + webhook_event.time_dispatched, + webhook_event.event_class, + webhook_event.event, + webhook_event.num_dispatched +FROM + webhook_event INNER JOIN webhook_delivery AS delivery ON delivery.event_id = webhook_event.id +WHERE + (webhook_event.event_class != $1 AND delivery.rx_id = $2) + AND NOT + ( + EXISTS( + SELECT + also_delivey.id + FROM + webhook_delivery AS also_delivey + WHERE + (also_delivey.event_id = webhook_event.id AND also_delivey.failed_permanently = $3) + AND also_delivey.trigger != $4 + ) + ) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 9bcfe5e3042..9790818e9a5 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5185,7 +5185,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. time_completed TIMESTAMPTZ, - + -- If true, this webhook delivery has failed permanently and is eligible to + -- be resent. + failed_permanently BOOLEAN NOT NULL, -- Deliverator coordination bits deliverator_id UUID, time_delivery_started TIMESTAMPTZ, @@ -5195,6 +5197,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( (deliverator_id IS NULL) OR ( (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) ) + ), + CONSTRAINT failed_permanently_only_if_completed CHECK ( + (failed_permanently IS false) OR (failed_permanently AND (time_completed IS NOT NULL)) ) ); @@ -5215,6 +5220,13 @@ ON omicron.public.webhook_delivery ( rx_id, event_id ); +-- Index for looking up all delivery attempts for an event +CREATE INDEX IF NOT EXISTS lookup_deliveries_for_event +ON omicron.public.webhook_delivery ( + event_id +); + + -- Index for looking up all currently in-flight webhook messages, and ordering -- them by their creation times. CREATE INDEX IF NOT EXISTS webhook_delivery_in_flight From 233c7d54b5b1f0da6ed8f67fd2e74d97964e102d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 28 Feb 2025 11:55:11 -0800 Subject: [PATCH 092/168] resend failed deliveries on successful probes --- nexus/db-model/src/schema.rs | 2 +- nexus/external-api/src/lib.rs | 2 +- nexus/src/app/webhook.rs | 77 +++++++- nexus/src/external_api/http_entrypoints.rs | 6 +- nexus/tests/integration_tests/webhooks.rs | 194 +++++++++++++++++++-- nexus/types/src/external_api/views.rs | 20 ++- openapi/nexus.json | 39 ++++- 7 files changed, 301 insertions(+), 39 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index e8dbbf9ef42..5ce6ff18bc4 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2255,9 +2255,9 @@ table! { attempts -> Int2, time_created -> Timestamptz, time_completed -> Nullable, + failed_permanently -> Bool, deliverator_id -> Nullable, time_delivery_started -> Nullable, - failed_permanently -> Bool, } } diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 57b07022d81..8be737decc3 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3558,7 +3558,7 @@ pub trait NexusExternalApi { rqctx: RequestContext, path_params: Path, query_params: Query, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// List the IDs of secrets for a webhook receiver. #[endpoint { diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index c51401712f3..68047dc4da7 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -20,6 +20,7 @@ use nexus_db_queries::db::model::SqlU8; use nexus_db_queries::db::model::WebhookDelivery; use nexus_db_queries::db::model::WebhookDeliveryAttempt; use nexus_db_queries::db::model::WebhookDeliveryResult; +use nexus_db_queries::db::model::WebhookDeliveryTrigger; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; use nexus_db_queries::db::model::WebhookReceiverConfig; @@ -114,14 +115,16 @@ impl super::Nexus { &self, opctx: &OpContext, rx: lookup::WebhookReceiver<'_>, - _params: params::WebhookProbe, - ) -> Result { + params: params::WebhookProbe, + ) -> Result { let (authz_rx, rx) = rx.fetch_for(authz::Action::ListChildren).await?; + let rx_id = authz_rx.id(); + let datastore = self.datastore(); let secrets = - self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await?; + datastore.webhook_rx_secret_list(opctx, &authz_rx).await?; let mut client = ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; - let delivery = WebhookDelivery::new_probe(&authz_rx.id(), &self.id); + let delivery = WebhookDelivery::new_probe(&rx_id, &self.id); const CLASS: WebhookEventClass = WebhookEventClass::Probe; @@ -143,10 +146,70 @@ impl super::Nexus { } }; - // TODO(eliza): this is where we would resend all the failed stuff - // if requested... + let resends_started = if params.resend + && attempt.result == WebhookDeliveryResult::Succeeded + { + slog::debug!( + &opctx.log, + "webhook liveness probe succeeded, resending failed deliveries..."; + "rx_id" => %authz_rx.id(), + "rx_name" => %rx.name(), + "delivery_id" => %delivery.id, + ); - Ok(delivery.to_api_delivery(CLASS, Some(&attempt))) + let deliveries = datastore + .webhook_rx_list_resendable_events(opctx, &rx_id) + .await + .map_err(|e| { + e.internal_context("error listing events to resend") + })? + .into_iter() + .map(|event| { + WebhookDelivery::new( + &event, + &rx_id, + WebhookDeliveryTrigger::Resend, + ) + }) + .collect::>(); + slog::trace!( + &opctx.log, + "found {} failed events to resend", deliveries.len(); + "rx_id" => %authz_rx.id(), + "rx_name" => %rx.name(), + "delivery_id" => %delivery.id, + ); + let started = datastore + .webhook_delivery_create_batch(&opctx, deliveries) + .await + .map_err(|e| { + e.internal_context( + "error creating deliveries to resend failed events", + ) + })?; + + if started > 0 { + slog::info!( + &opctx.log, + "webhook liveness probe succeeded, created {started} re-deliveries"; + "rx_id" => %authz_rx.id(), + "rx_name" => %rx.name(), + "delivery_id" => %delivery.id, + ); + // If new deliveries were created, activate the webhook + // deliverator background task to start actually delivering + // them. + self.background_tasks.task_webhook_deliverator.activate(); + } + Some(started) + } else { + None + }; + + Ok(views::WebhookProbeResult { + probe: delivery.to_api_delivery(CLASS, Some(&attempt)), + resends_started, + }) } pub async fn webhook_receiver_secret_add( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 787c1c731df..89c28680106 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7426,7 +7426,7 @@ impl NexusExternalApi for NexusExternalApiImpl { rqctx: RequestContext, path_params: Path, query_params: Query, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7437,10 +7437,10 @@ impl NexusExternalApi for NexusExternalApiImpl { let webhook_selector = path_params.into_inner(); let probe_params = query_params.into_inner(); let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; - let delivery = + let result = nexus.webhook_receiver_probe(&opctx, rx, probe_params).await?; // TODO(eliza): send the status code that came back from the probe req... - Ok(HttpResponseOk(delivery)) + Ok(HttpResponseOk(result)) }; apictx .context diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 5694baa761d..7d122921f45 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -79,7 +79,7 @@ async fn webhook_send_probe( webhook_id: &WebhookReceiverUuid, resend: bool, status: http::StatusCode, -) -> views::WebhookDelivery { +) -> views::WebhookProbeResult { let pathparams = if resend { "?resend=true" } else { "" }; let path = format!("{WEBHOOKS_BASE_PATH}/{webhook_id}/probe{pathparams}"); NexusRequest::new( @@ -133,7 +133,7 @@ fn signature_verifies( // Strip the expected algorithm part. Note that we only support // SHA256 for now. Panic if this is invalid. - let hdr = dbg!(hdr) + let hdr = hdr .strip_prefix("a=sha256") .expect("all x-oxide-signature headers should be SHA256"); // Strip the leading `&id=` for the ID part, panicking if this @@ -698,10 +698,14 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { mock.assert_async().await; - assert_eq!(probe1.attempt, 1); - assert_eq!(probe1.event_class, "probe"); - assert_eq!(probe1.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe1.state, shared::WebhookDeliveryState::FailedTimeout); + assert_eq!(probe1.probe.attempt, 1); + assert_eq!(probe1.probe.event_class, "probe"); + assert_eq!(probe1.probe.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe1.probe.state, shared::WebhookDeliveryState::FailedTimeout); + assert_eq!( + probe1.resends_started, None, + "we did not request events be resent" + ); // Next, configure the receiver server to return a 5xx error mock.delete_async().await; @@ -733,14 +737,21 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { dbg!(&probe2); mock.assert_async().await; - assert_eq!(probe2.attempt, 1); - assert_eq!(probe2.event_class, "probe"); - assert_eq!(probe2.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe2.state, shared::WebhookDeliveryState::FailedHttpError); + assert_eq!(probe2.probe.attempt, 1); + assert_eq!(probe2.probe.event_class, "probe"); + assert_eq!(probe2.probe.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!( + probe2.probe.state, + shared::WebhookDeliveryState::FailedHttpError + ); assert_ne!( - probe2.id, probe1.id, + probe2.probe.id, probe1.probe.id, "a new delivery ID should be assigned to each probe" ); + assert_eq!( + probe2.resends_started, None, + "we did not request events be resent" + ); mock.delete_async().await; // Finally, configure the receiver server to return a success. @@ -771,16 +782,165 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { .await; dbg!(&probe3); mock.assert_async().await; - assert_eq!(probe3.attempt, 1); - assert_eq!(probe3.event_class, "probe"); - assert_eq!(probe3.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe3.state, shared::WebhookDeliveryState::Delivered); + assert_eq!(probe3.probe.attempt, 1); + assert_eq!(probe3.probe.event_class, "probe"); + assert_eq!(probe3.probe.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe3.probe.state, shared::WebhookDeliveryState::Delivered); assert_ne!( - probe3.id, probe1.id, + probe3.probe.id, probe1.probe.id, "a new delivery ID should be assigned to each probe" ); assert_ne!( - probe3.id, probe2.id, + probe3.probe.id, probe2.probe.id, "a new delivery ID should be assigned to each probe" ); + assert_eq!( + probe3.resends_started, None, + "we did not request events be resent" + ); +} + +#[nexus_test] +async fn test_probe_resends_failed_deliveries( + cptestctx: &ControlPlaneTestContext, +) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + let server = httpmock::MockServer::start_async().await; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + // Create a webhook receiver. + let webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&webhook); + + let event1_id = WebhookEventUuid::new_v4(); + let event2_id = WebhookEventUuid::new_v4(); + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo") + // either event + .header_matches( + "x-oxide-event-id", + format!("({event1_id})|({event2_id})"), + ) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )); + then.status(500); + }) + .await + }; + + // Publish both events + dbg!(nexus + .webhook_event_publish( + &opctx, + event1_id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello": "world"}), + ) + .await + .expect("event1 should be published successfully")); + dbg!(nexus + .webhook_event_publish( + &opctx, + event2_id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello": "emeryville"}), + ) + .await + .expect("event2 should be published successfully")); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(2).await; + + // Backoff 1 + tokio::time::sleep(std::time::Duration::from_secs(11)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(4).await; + + // Backoff 2 + tokio::time::sleep(std::time::Duration::from_secs(22)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(6).await; + + mock.delete_async().await; + + // Allow a probe to succeed + let probe_mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + let body = serde_json::json!({ + "event_class": "probe", + "data": { + } + }) + .to_string(); + when.method(POST) + .header("x-oxide-event-class", "probe") + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(200); + }) + .await + }; + + // Allow events to succeed. + let mock = { + let webhook = webhook.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo") + // either event + .header_matches( + "x-oxide-event-id", + format!("({event1_id})|({event2_id})"), + ) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )); + then.status(200); + }) + .await + }; + + // Send a probe with ?resend=true + let probe = + webhook_send_probe(&cptestctx, &webhook.id, true, http::StatusCode::OK) + .await; + dbg!(&probe); + probe_mock.assert_async().await; + probe_mock.delete_async().await; + assert_eq!(probe.probe.state, shared::WebhookDeliveryState::Delivered); + assert_eq!(probe.resends_started, Some(2)); + + // Both events should be resent. + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(2).await; } diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 57560e1ec7d..cb0d44246fa 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1090,7 +1090,7 @@ pub struct WebhookSecretId { } /// A delivery attempt for a webhook event. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] pub struct WebhookDelivery { /// The UUID of this delivery attempt. pub id: Uuid, @@ -1157,7 +1157,7 @@ impl fmt::Display for WebhookDeliveryTrigger { } /// The response received from a webhook receiver endpoint. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct WebhookDeliveryResponse { /// The HTTP status code returned from the webhook endpoint. pub status: u16, @@ -1165,7 +1165,21 @@ pub struct WebhookDeliveryResponse { pub duration_ms: usize, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct WebhookDeliveryId { pub delivery_id: Uuid, } +/// Data describing the result of a webhook liveness probe attempt. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] +pub struct WebhookProbeResult { + /// The outcome of the probe request. + pub probe: WebhookDelivery, + /// If the probe request succeeded, and resending failed deliveries on + /// success was requested, the number of new delivery attempts started. + /// Otherwise, if the probe did not succeed, or resending failed deliveries + /// was not requested, this is null. + /// + /// Note that this may be 0, if there were no events found which had not + /// been delivered successfully to this receiver. + pub resends_started: Option, +} diff --git a/openapi/nexus.json b/openapi/nexus.json index 68c1d0e5321..3d0dd2b2e28 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -1013,7 +1013,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookDelivery" + "$ref": "#/components/schemas/WebhookProbeResult" } } } @@ -25135,6 +25135,12 @@ "description": "A delivery attempt for a webhook event.", "type": "object", "properties": { + "attempt": { + "description": "Attempt number, starting at 1. If this is a retry of a previous failed delivery, this value indicates that.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, "event_class": { "description": "The event class.", "type": "string" @@ -25152,12 +25158,6 @@ "type": "string", "format": "uuid" }, - "resent_for": { - "nullable": true, - "description": "The UUID of a previous delivery attempt that this is a repeat of, if this was a resending of a previous delivery. If this is the first time this event has been delivered, this is `null`.", - "type": "string", - "format": "uuid" - }, "response": { "nullable": true, "description": "Describes the response returned by the receiver endpoint.\n\nThis is present if the webhook has been delivered successfully, or if the endpoint returned an HTTP error (`state` is \"delivered\" or \"failed_http_error\"). This is `null` if the webhook has not yet been delivered, or if the endpoint was unreachable (`state` is \"pending\" or \"failed_unreachable\").", @@ -25199,6 +25199,7 @@ } }, "required": [ + "attempt", "event_class", "event_id", "id", @@ -25328,6 +25329,30 @@ } ] }, + "WebhookProbeResult": { + "description": "Data describing the result of a webhook liveness probe attempt.", + "type": "object", + "properties": { + "probe": { + "description": "The outcome of the probe request.", + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDelivery" + } + ] + }, + "resends_started": { + "nullable": true, + "description": "If the probe request succeeded, and resending failed deliveries on success was requested, the number of new delivery attempts started. Otherwise, if the probe did not succeed, or resending failed deliveries was not requested, this is null.\n\nNote that this may be 0, if there were no events found which had not been delivered successfully to this receiver.", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "probe" + ] + }, "WebhookSecretCreate": { "type": "object", "properties": { From e7ee2f68dddb81d49e767574cf7d6e209410fc8e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 28 Feb 2025 11:55:37 -0800 Subject: [PATCH 093/168] fix deliverator OMDB status counting failures as ok --- .../background/tasks/webhook_deliverator.rs | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index d80aa8d9719..c09750670f6 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -9,6 +9,7 @@ use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; +use nexus_db_queries::db::model::WebhookDeliveryResult; use nexus_db_queries::db::model::WebhookReceiver; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; @@ -280,7 +281,7 @@ impl WebhookDeliverator { } }; - match self + if let Err(e) = self .datastore .webhook_delivery_finish_attempt( opctx, @@ -290,24 +291,27 @@ impl WebhookDeliverator { ) .await { - Err(e) => { - const MSG: &str = - "failed to mark webhook delivery as finished"; - slog::error!( - &opctx.log, - "{MSG}"; - "event_id" => %delivery.event_id, - "event_class" => %event_class, - "delivery_id" => %delivery_id, - "error" => %e, - ); - delivery_status - .delivery_errors - .insert(delivery_id, format!("{MSG}: {e}")); - } - Ok(_) => { - delivery_status.delivered_ok += 1; - } + const MSG: &str = "failed to mark webhook delivery as finished"; + slog::error!( + &opctx.log, + "{MSG}"; + "event_id" => %delivery.event_id, + "event_class" => %event_class, + "delivery_id" => %delivery_id, + "error" => %e, + ); + delivery_status + .delivery_errors + .insert(delivery_id, format!("{MSG}: {e}")); + } + + if delivery_attempt.result == WebhookDeliveryResult::Succeeded { + delivery_status.delivered_ok += 1; + } else { + delivery_status.failed_deliveries.push( + delivery + .to_api_delivery(event_class, Some(&delivery_attempt)), + ); } } From b21c1d2af7d15b0f796df586a5f1c602e7f287a9 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 28 Feb 2025 12:58:22 -0800 Subject: [PATCH 094/168] query to list globs that need reprocessing --- .../db-queries/src/db/datastore/webhook_rx.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 4cf43cdbb77..1cf2e7670a4 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -370,6 +370,50 @@ impl DataStore { )) } + // + // Glob reprocessing + // + + pub async fn webhook_glob_list_outdated( + &self, + opctx: &OpContext, + ) -> ListResultVec { + use crate::db::model::{DbSemverVersion, SCHEMA_VERSION}; + + let (current_version, target_version) = + self.database_schema_version().await.map_err(|e| { + e.internal_context("couldn't load db schema version") + })?; + + if let Some(target) = target_version { + return Err(Error::InternalError { + internal_message: format!( + "webhook glob reprocessing must wait until the migration \ + from {current_version} to {target} has completed", + ), + }); + } + if current_version != SCHEMA_VERSION { + return Err(Error::InternalError { + internal_message: format!( + "cannot reprocess webhook globs, as our schema version \ + ({SCHEMA_VERSION}) doess not match the current version \ + ({current_version})", + ), + }); + } + + glob_dsl::webhook_rx_event_glob + .filter( + glob_dsl::schema_version + .ne(DbSemverVersion::from(SCHEMA_VERSION)), + ) + .select(WebhookRxEventGlob::as_select()) + .load_async(&*self.pool_connection_authorized(&opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + // // Secrets // From fa0d05c06a8f215348d6b6b367aea7ce2b58fc6a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 3 Mar 2025 11:23:34 -0800 Subject: [PATCH 095/168] glob reprocessing --- .../db-queries/src/db/datastore/webhook_rx.rs | 142 ++++++++++++++++-- .../background/tasks/webhook_dispatcher.rs | 74 ++++++++- nexus/types/src/internal_api/background.rs | 13 ++ 3 files changed, 218 insertions(+), 11 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 1cf2e7670a4..d36c3bf6684 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -12,6 +12,7 @@ use crate::db::collection_insert::DatastoreCollection; use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; +use crate::db::model::DbSemverVersion; use crate::db::model::Generation; use crate::db::model::WebhookEventClass; use crate::db::model::WebhookGlob; @@ -22,7 +23,9 @@ use crate::db::model::WebhookRxEventGlob; use crate::db::model::WebhookRxSubscription; use crate::db::model::WebhookSecret; use crate::db::model::WebhookSubscriptionKind; +use crate::db::model::SCHEMA_VERSION; use crate::db::pagination::paginated; +use crate::db::pagination::paginated_multicolumn; use crate::db::pool::DbConnection; use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; @@ -33,6 +36,7 @@ use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; +use nexus_types::internal_api::background::WebhookGlobStatus; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -377,9 +381,8 @@ impl DataStore { pub async fn webhook_glob_list_outdated( &self, opctx: &OpContext, + pagparams: &DataPageParams<'_, (Uuid, String)>, ) -> ListResultVec { - use crate::db::model::{DbSemverVersion, SCHEMA_VERSION}; - let (current_version, target_version) = self.database_schema_version().await.map_err(|e| { e.internal_context("couldn't load db schema version") @@ -403,15 +406,134 @@ impl DataStore { }); } - glob_dsl::webhook_rx_event_glob - .filter( - glob_dsl::schema_version - .ne(DbSemverVersion::from(SCHEMA_VERSION)), - ) - .select(WebhookRxEventGlob::as_select()) - .load_async(&*self.pool_connection_authorized(&opctx).await?) + paginated_multicolumn( + glob_dsl::webhook_rx_event_glob, + (glob_dsl::rx_id, glob_dsl::glob), + pagparams, + ) + .filter( + glob_dsl::schema_version.ne(DbSemverVersion::from(SCHEMA_VERSION)), + ) + .select(WebhookRxEventGlob::as_select()) + .load_async(&*self.pool_connection_authorized(&opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn webhook_glob_reprocess( + &self, + opctx: &OpContext, + glob: &WebhookRxEventGlob, + ) -> Result { + slog::trace!( + opctx.log, + "reprocessing outdated webhook glob"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "prior_version" => %glob.schema_version.0, + "current_version" => %SCHEMA_VERSION, + ); + let conn = self.pool_connection_authorized(opctx).await?; + let err = OptionalError::new(); + let status = self + .transaction_retry_wrapper("webhook_glob_reprocess") + .transaction(&*conn, |conn| { + let glob = glob.clone(); + let err = err.clone(); + async move { + let deleted = diesel::delete( + subscription_dsl::webhook_rx_subscription, + ) + .filter(subscription_dsl::glob.eq(glob.glob.glob.clone())) + .filter(subscription_dsl::rx_id.eq(glob.rx_id)) + .execute_async(&conn) + .await?; + let created = self + .glob_generate_exact_subs(opctx, &glob, &conn) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => { + err.bail(Err(e)) + } + TransactionError::Database(e) => e, + })?; + let did_update = + diesel::update(glob_dsl::webhook_rx_event_glob) + .filter( + glob_dsl::rx_id + .eq(glob.rx_id.into_untyped_uuid()), + ) + .filter(glob_dsl::glob.eq(glob.glob.glob.clone())) + .filter( + glob_dsl::schema_version + .eq(glob.schema_version.clone()), + ) + .set( + glob_dsl::schema_version + .eq(DbSemverVersion::from(SCHEMA_VERSION)), + ) + .execute_async(&conn) + .await; + match did_update { + // Either the glob has been reprocessed by someone else, or + // it has been deleted. + Err(diesel::result::Error::NotFound) | Ok(0) => { + return Err(err.bail(Ok( + WebhookGlobStatus::AlreadyReprocessed, + ))); + } + Err(e) => return Err(e), + Ok(updated) => { + debug_assert_eq!(updated, 1); + } + } + + Ok(WebhookGlobStatus::Reprocessed { + created, + deleted, + prev_version: glob.schema_version.clone().into(), + }) + } + }) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .or_else(|e| { + if let Some(err) = err.take() { + err + } else { + Err(public_error_from_diesel(e, ErrorHandler::Server)) + } + })?; + + match status { + WebhookGlobStatus::Reprocessed { + created, + deleted, + ref prev_version, + } => { + slog::debug!( + opctx.log, + "reprocessed outdated webhook glob"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "prev_version" => %prev_version, + "current_version" => %SCHEMA_VERSION, + "subscriptions_created" => ?created, + "subscriptions_deleted" => ?deleted, + ); + } + WebhookGlobStatus::AlreadyReprocessed => { + slog::trace!( + opctx.log, + "outdated webhook glob was either already reprocessed or deleted"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "prev_version" => %glob.schema_version.0, + "current_version" => %SCHEMA_VERSION, + ); + } + } + + Ok(status) } // diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 591cb1e3629..904000c7613 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -9,13 +9,17 @@ use crate::app::background::BackgroundTask; use futures::future::BoxFuture; use nexus_db_model::WebhookDelivery; use nexus_db_model::WebhookDeliveryTrigger; +use nexus_db_model::SCHEMA_VERSION; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::datastore::SQL_BATCH_SIZE; +use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ - WebhookDispatched, WebhookDispatcherStatus, + WebhookDispatched, WebhookDispatcherStatus, WebhookGlobStatus, }; use omicron_common::api::external::Error; +use omicron_uuid_kinds::GenericUuid; use std::sync::Arc; pub struct WebhookDispatcher { @@ -30,6 +34,8 @@ impl BackgroundTask for WebhookDispatcher { ) -> BoxFuture<'a, serde_json::Value> { Box::pin(async move { let mut status = WebhookDispatcherStatus { + globs_reprocessed: Default::default(), + glob_version: SCHEMA_VERSION, dispatched: Vec::new(), errors: Vec::new(), no_receivers: Vec::new(), @@ -98,6 +104,72 @@ impl WebhookDispatcher { opctx: &OpContext, status: &mut WebhookDispatcherStatus, ) -> Result<(), Error> { + // Before dispatching any events, ensure that all webhook globs are up + // to date with the current schema version. + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + let mut globs_reprocessed = 0; + let mut globs_failed = 0; + let mut globs_already_reprocessed = 0; + while let Some(p) = paginator.next() { + let batch = self + .datastore + .webhook_glob_list_outdated(opctx, &p.current_pagparams()) + .await + .map_err(|e| { + e.internal_context("failed to list outdated webhook globs") + })?; + paginator = p.found_batch(&batch, &|glob| { + (glob.rx_id.into_untyped_uuid(), glob.glob.glob.clone()) + }); + for glob in batch { + let result = self + .datastore + .webhook_glob_reprocess(opctx, &glob) + .await + .map_err(|e| { + globs_failed += 1; + slog::warn!( + &opctx.log, + "failed to reprocess webhook glob"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "glob_version" => %glob.schema_version.0, + "error" => %e, + ); + e.to_string() + }) + .inspect(|status| match status { + WebhookGlobStatus::Reprocessed { .. } => { + globs_reprocessed += 1 + } + WebhookGlobStatus::AlreadyReprocessed => { + globs_already_reprocessed += 1 + } + }); + let rx_statuses = status + .globs_reprocessed + .entry(glob.rx_id.into()) + .or_default(); + rx_statuses.insert(glob.glob.glob, result); + } + } + if globs_failed > 0 { + slog::warn!( + &opctx.log, + "webhook glob reprocessing completed with failures"; + "globs_failed" => ?globs_failed, + "globs_reprocessed" => ?globs_reprocessed, + "globs_already_reprocessed" => ?globs_already_reprocessed, + ); + } else if globs_reprocessed > 0 { + slog::info!( + &opctx.log, + "webhook glob reprocessed"; + "globs_reprocessed" => ?globs_reprocessed, + "globs_already_reprocessed" => ?globs_already_reprocessed, + ); + } + // Select the next event that has yet to be dispatched in order of // creation, until there are none left in need of dispatching. while let Some(event) = diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 0838c5a8beb..13b85ae981a 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -5,6 +5,7 @@ use crate::external_api::views::WebhookDelivery; use chrono::DateTime; use chrono::Utc; +use omicron_common::api::external::SemverVersion; use omicron_common::update::ArtifactHash; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::CollectionUuid; @@ -457,6 +458,10 @@ impl slog::KV for DebugDatasetsRendezvousStats { /// The status of a `webhook_dispatcher` background task activation. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct WebhookDispatcherStatus { + pub globs_reprocessed: BTreeMap, + + pub glob_version: SemverVersion, + /// The webhook events dispatched on this activation. pub dispatched: Vec, @@ -467,6 +472,14 @@ pub struct WebhookDispatcherStatus { pub errors: Vec, } +type ReprocessedGlobs = BTreeMap>; + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub enum WebhookGlobStatus { + AlreadyReprocessed, + Reprocessed { created: usize, deleted: usize, prev_version: SemverVersion }, +} + #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct WebhookDispatched { pub event_id: WebhookEventUuid, From 86b723e51475f090097c0d7e9c5ec390382376c5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 3 Mar 2025 12:08:45 -0800 Subject: [PATCH 096/168] delint --- nexus/db-queries/src/db/datastore/webhook_rx.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index d36c3bf6684..bd6653581a2 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -437,7 +437,7 @@ impl DataStore { let err = OptionalError::new(); let status = self .transaction_retry_wrapper("webhook_glob_reprocess") - .transaction(&*conn, |conn| { + .transaction(&conn, |conn| { let glob = glob.clone(); let err = err.clone(); async move { From c61f2fbffd38b4e4fb5ca89ceedbe9cfa78d7a0e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 3 Mar 2025 13:17:50 -0800 Subject: [PATCH 097/168] redo migration to reflect newest schema --- nexus/db-model/src/schema_versions.rs | 2 +- schema/crdb/add-webhooks/README.adoc | 51 ------------------ schema/crdb/add-webhooks/up04.sql | 19 ------- schema/crdb/add-webhooks/up05.sql | 4 -- schema/crdb/add-webhooks/up07.sql | 10 ---- schema/crdb/add-webhooks/up09.sql | 20 ------- schema/crdb/add-webhooks/up10.sql | 5 -- schema/crdb/add-webhooks/up11.sql | 7 --- schema/crdb/add-webhooks/up14.sql | 4 -- schema/crdb/dbinit.sql | 15 +++--- schema/crdb/webhooks/README.adoc | 54 +++++++++++++++++++ .../crdb/{add-webhooks => webhooks}/up01.sql | 2 +- schema/crdb/webhooks/up02.sql | 4 ++ .../up02.sql => webhooks/up03.sql} | 15 +++--- .../up03.sql => webhooks/up04.sql} | 0 schema/crdb/webhooks/up05.sql | 13 +++++ schema/crdb/webhooks/up06.sql | 17 ++++++ schema/crdb/webhooks/up07.sql | 4 ++ schema/crdb/webhooks/up08.sql | 2 + schema/crdb/webhooks/up09.sql | 20 +++++++ .../up06.sql => webhooks/up10.sql} | 2 +- schema/crdb/webhooks/up11.sql | 4 ++ schema/crdb/webhooks/up12.sql | 21 ++++++++ schema/crdb/webhooks/up13.sql | 19 +++++++ .../up08.sql => webhooks/up14.sql} | 3 -- schema/crdb/webhooks/up15.sql | 9 ++++ schema/crdb/webhooks/up16.sql | 37 +++++++++++++ schema/crdb/webhooks/up17.sql | 6 +++ schema/crdb/webhooks/up18.sql | 4 ++ schema/crdb/webhooks/up19.sql | 4 ++ schema/crdb/webhooks/up20.sql | 5 ++ .../up12.sql => webhooks/up21.sql} | 0 .../up13.sql => webhooks/up22.sql} | 1 + schema/crdb/webhooks/up23.sql | 4 ++ 34 files changed, 246 insertions(+), 141 deletions(-) delete mode 100644 schema/crdb/add-webhooks/README.adoc delete mode 100644 schema/crdb/add-webhooks/up04.sql delete mode 100644 schema/crdb/add-webhooks/up05.sql delete mode 100644 schema/crdb/add-webhooks/up07.sql delete mode 100644 schema/crdb/add-webhooks/up09.sql delete mode 100644 schema/crdb/add-webhooks/up10.sql delete mode 100644 schema/crdb/add-webhooks/up11.sql delete mode 100644 schema/crdb/add-webhooks/up14.sql create mode 100644 schema/crdb/webhooks/README.adoc rename schema/crdb/{add-webhooks => webhooks}/up01.sql (86%) create mode 100644 schema/crdb/webhooks/up02.sql rename schema/crdb/{add-webhooks/up02.sql => webhooks/up03.sql} (57%) rename schema/crdb/{add-webhooks/up03.sql => webhooks/up04.sql} (100%) create mode 100644 schema/crdb/webhooks/up05.sql create mode 100644 schema/crdb/webhooks/up06.sql create mode 100644 schema/crdb/webhooks/up07.sql create mode 100644 schema/crdb/webhooks/up08.sql create mode 100644 schema/crdb/webhooks/up09.sql rename schema/crdb/{add-webhooks/up06.sql => webhooks/up10.sql} (77%) create mode 100644 schema/crdb/webhooks/up11.sql create mode 100644 schema/crdb/webhooks/up12.sql create mode 100644 schema/crdb/webhooks/up13.sql rename schema/crdb/{add-webhooks/up08.sql => webhooks/up14.sql} (52%) create mode 100644 schema/crdb/webhooks/up15.sql create mode 100644 schema/crdb/webhooks/up16.sql create mode 100644 schema/crdb/webhooks/up17.sql create mode 100644 schema/crdb/webhooks/up18.sql create mode 100644 schema/crdb/webhooks/up19.sql create mode 100644 schema/crdb/webhooks/up20.sql rename schema/crdb/{add-webhooks/up12.sql => webhooks/up21.sql} (100%) rename schema/crdb/{add-webhooks/up13.sql => webhooks/up22.sql} (99%) create mode 100644 schema/crdb/webhooks/up23.sql diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 5661630c5c7..b9008193c63 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -32,7 +32,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), - KnownVersion::new(127, "add-webhooks"), + KnownVersion::new(127, "webhooks"), KnownVersion::new(126, "affinity"), KnownVersion::new(125, "blueprint-disposition-expunged-cleanup"), KnownVersion::new(124, "support-read-only-region-replacement"), diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc deleted file mode 100644 index 40e5f261bad..00000000000 --- a/schema/crdb/add-webhooks/README.adoc +++ /dev/null @@ -1,51 +0,0 @@ -# Overview - -This migration adds initial tables required for webhook delivery. - -## Upgrade steps - -The individual transactions in this upgrade do the following: - -* *Webhook receivers*: -** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores -the receiver endpoints that receive webhook events. -** *Receiver secrets*: -*** `up02.sql` creates the `omicron.public.webhook_secret` table, which -associates webhook receivers with secret keys and their IDs. -*** `up03.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, -for looking up all secrets associated with a receiver. -** *Receiver subscriptions*: -*** `up04.sql` creates the `omicron.public.webhook_rx_subscription` table, which -associates a webhook receiver with multiple event classes that the receiver is -subscribed to. -*** `up05.sql` creates an index `lookup_webhook_subscriptions_by_rx` for -looking up all event classes that a receiver ID is subscribed to. -*** `up06.sql` creates an index `lookup_webhook_rxs_for_event` on -`omicron.public.webhook_rx_subscription` for looking up all receivers subscribed -to a particular event class. -* *Webhook message queue*: -** `up07.sql` creates the `omicron.public.webhook_event` table, which contains the -queue of un-dispatched webhook events. The dispatcher operates on entries in -this queue, dispatching the event to receivers and generating the payload for -each receiver. -** `up08.sql` creates the `lookup_undispatched_webhook_events` index on -`omicron.public.webhook_event` for looking up webhook messages which have not yet been -dispatched and ordering by their creation times. -* *Webhook message dispatching and delivery attempts*: -** *Dispatch table*: -*** `up09.sql` creates the table `omicron.public.webhook_delivery`, which -tracks the webhook messages that have been dispatched to receivers. -*** `up10.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up -entries in `omicron.public.webhook_delivery` by receiver ID. -*** `up11.sql` creates an index `webhook_delivery_in_flight` for looking up all currently in-flight webhook -messages (entries in `omicron.public.webhook_delivery` where the -`time_completed` field has not been set). -** *Delivery attempts*: -*** `up12.sql` creates the enum `omicron.public.webhook_delivery_result`, -representing the potential outcomes of a webhook delivery attempt. -*** `up13.sql` creates the table `omicron.public.webhook_delivery_attempt`, -which records each individual delivery attempt for a webhook message in the -`webhook_delivery` table. -*** `up14.sql` creates an index `lookup_webhook_delivery_attempt_for_msg` on -`omicron.public.webhook_delivery_attempt`, for looking up all attempts to -deliver a message with a given dispatch ID. diff --git a/schema/crdb/add-webhooks/up04.sql b/schema/crdb/add-webhooks/up04.sql deleted file mode 100644 index e5c6b433ca8..00000000000 --- a/schema/crdb/add-webhooks/up04.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - -- An event class (or event class glob) to which this receiver is subscribed. - event_class STRING(512) NOT NULL, - -- The event class or event classs glob transformed into a patteern for use - -- in SQL `SIMILAR TO` clauses. - -- - -- This is a bit interesting: users specify event class globs as sequences - -- of dot-separated segments which may be `*` to match any one segment or - -- `**` to match any number of segments. In order to match webhook events to - -- subscriptions within the database, we transform these into patterns that - -- can be used with a `SIMILAR TO` clause. - similar_to STRING(512) NOT NULL, - time_created TIMESTAMPTZ NOT NULL, - - PRIMARY KEY (rx_id, event_class) -); diff --git a/schema/crdb/add-webhooks/up05.sql b/schema/crdb/add-webhooks/up05.sql deleted file mode 100644 index 4ffe7cbce05..00000000000 --- a/schema/crdb/add-webhooks/up05.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx -ON omicron.public.webhook_rx_subscription ( - rx_id -); diff --git a/schema/crdb/add-webhooks/up07.sql b/schema/crdb/add-webhooks/up07.sql deleted file mode 100644 index 99e00dfaebc..00000000000 --- a/schema/crdb/add-webhooks/up07.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( - id UUID PRIMARY KEY, - time_created TIMESTAMPTZ NOT NULL, - -- Set when dispatch entries have been created for this event. - time_dispatched TIMESTAMPTZ, - -- The class of event that this is. - event_class STRING(512) NOT NULL, - -- Actual event data. The structure of this depends on the event class. - event JSONB NOT NULL -); diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql deleted file mode 100644 index 74569aed13c..00000000000 --- a/schema/crdb/add-webhooks/up09.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( - -- UUID of this dispatch. - id UUID PRIMARY KEY, - --- UUID of the event (foreign key into `omicron.public.webhook_event`). - event_id UUID NOT NULL, - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - payload JSONB NOT NULL, - - --- Delivery attempt count. Starts at 0. - attempts INT2 NOT NULL, - - time_created TIMESTAMPTZ NOT NULL, - -- If this is set, then this webhook message has either been delivered - -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, - - CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0) -); diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql deleted file mode 100644 index 0e67ca550f9..00000000000 --- a/schema/crdb/add-webhooks/up10.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Index for looking up all webhook messages dispatched to a receiver ID -CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx -ON omicron.public.webhook_delivery ( - rx_id, event_id -); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql deleted file mode 100644 index b3c36debbbd..00000000000 --- a/schema/crdb/add-webhooks/up11.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Index for looking up all currently in-flight webhook deliveries, and ordering --- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_delivery_in_flight -ON omicron.public.webhook_delivery ( - time_created, id -) WHERE - time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up14.sql b/schema/crdb/add-webhooks/up14.sql deleted file mode 100644 index 06ad7d4f937..00000000000 --- a/schema/crdb/add-webhooks/up14.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempt_for_msg -ON omicron.public.webhook_delivery_attempt ( - delivery_id -); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 9790818e9a5..6ea1bc4856d 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5048,12 +5048,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( ); -- Look up all event class globs for a webhook receiver. -CREATE INDEX IF NOT EXISTS lookup_event_globs_for_webhook_rx +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_for_rx ON omicron.public.webhook_rx_event_glob ( rx_id ); -CREATE INDEX IF NOT EXISTS lookup_webhook_globs_by_schema_version +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_by_schema_version ON omicron.public.webhook_rx_event_glob (schema_version); CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( @@ -5079,7 +5079,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( -- Look up all webhook receivers subscribed to an event class. This is used by -- the dispatcher to determine who is interested in a particular event. -CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event +CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event_class ON omicron.public.webhook_rx_subscription ( event_class ); @@ -5215,21 +5215,20 @@ WHERE trigger = 'event'; -- Index for looking up all webhook messages dispatched to a receiver ID -CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx ON omicron.public.webhook_delivery ( rx_id, event_id ); -- Index for looking up all delivery attempts for an event -CREATE INDEX IF NOT EXISTS lookup_deliveries_for_event +CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event ON omicron.public.webhook_delivery ( event_id ); - -- Index for looking up all currently in-flight webhook messages, and ordering -- them by their creation times. -CREATE INDEX IF NOT EXISTS webhook_delivery_in_flight +CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight ON omicron.public.webhook_delivery ( time_created, id ) WHERE @@ -5294,7 +5293,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( ) ); -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempt_for_msg +CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery ON omicron.public.webhook_delivery_attempt ( delivery_id ); diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc new file mode 100644 index 00000000000..dd32df7877d --- /dev/null +++ b/schema/crdb/webhooks/README.adoc @@ -0,0 +1,54 @@ +# Overview + +This migration adds initial tables required for webhook delivery. + +## Upgrade steps + +The individual transactions in this upgrade do the following: + +* *Webhook receivers*: +** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores +the receiver endpoints that receive webhook events. +** `up02.sql` creates the `lookup_webhook_rxs_by_id` index on that table, for listing non-deleted webhook receivers. +** *Secrets*: +*** `up03.sql` creates the `omicron.public.webhook_secret` table, which +associates webhook receivers with secret keys and their IDs. +*** `up04.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, +for looking up all secrets associated with a receiver. +* *Event classes, subscriptions, and globbing*: +** `up05.sql` creates the `omicron.public.webhook_event_class` enum type +** *Globs*: +*** `up06.sql` creates the `omicron.public.webhook_rx_event_glob` table, which contains any subscriptions created by a receiver that have glob patterns. This table is used when generating exact subscription from globs. +*** `up07.sql` creates the `lookup_webhook_event_globs_for_rx` indes on `webhook_rx_event_glob`, for looking up all globs belonging to a receiver by ID. +*** `up08.sql` creates the `lookup_webhook_event_globs_by_schema_version` index on `webhook_rx_event_glob`, for searching for globs with outdated schema versions. +** *Subscriptions*: +*** `up09.sql` creates the `omicron.public.webhook_rx_subscription` table, which tracks the event classes that a receiver is subscribed to. If a row in this table represents a subscription that was generated by a glob, this table also references the glob record. +*** `up10.sql` creates the `lookup_webhook_rxs_for_event_class` index on `webhook_rx_subscription`, for listing all the receivers subscribed to an event class +*** `up11.sql` creates the `lookup_exact_subscriptions_for_webhook_rx` index, for looking up the exact subscriptions (not globs) for a receiver by ID. This is used along with `lookup_webhook_event_globs_for_rx` index when listing the user-provided event class strings. +* *Webhook events*: +** `up12.sql` creates the `omicron.public.webhook_event` table, which contains the +values of actual webhook events. The dispatcher operates on entries in +this queue, dispatching the event to receivers and generating the payload for +each receiver. +** `up13.sql` inserts the singleton row in `webhook_event` used for liveness probes. This singleton exists so that delivery records for liveness probes can have event UUIDs that point at a real entry in `webhook_event`, without requiring a new event entry to be created for each probe. +** `up14.sql` creates the `lookup_undispatched_webhook_events` index on `webhook_event` for looking up webhook messages which have not yet been dispatched, and ordering by their creation times. +* *Webhook message dispatching and delivery attempts*: +** *Dispatch table*: +*** `up15.sql` creates the `omicron.public.webhook_delivery_trigger` enum, which tracks why a webhook delivery was initiated. +*** `up16.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. +*** `up17.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. ++ +This index functions as a `UNIQUE` constraint on the tuple of `(event_id, rx_id)`, but ONLY for rows with `trigger = 'event'`. This ensures that concurrently-executing webhook dispatchers will not create multiple deliveries when dispatching a new event, but permits multiple re-deliveries of an event to be explicitly triggered. +*** `up18.sql` creates an index `lookup_webhook_delivery_dispatched_to_rx` for looking up +entries in `webhook_delivery` by receiver ID. +*** `up19.sql` creates an index `lookup_webhook_deliveries_for_event` on `webhook_delivery` for looking up deliveries by event UUID. +*** `up20.sql` creates an index `webhook_deliveries_in_flight` for looking up all currently in-flight webhook +deliveries (entries where the `time_completed` field has not been set). +** *Delivery attempts*: +*** `up21.sql` creates the enum `omicron.public.webhook_delivery_result`, +representing the potential outcomes of a webhook delivery attempt. +*** `up22.sql` creates the table `omicron.public.webhook_delivery_attempt`, +which records each individual delivery attempt for a webhook delivery in the +`webhook_delivery` table. +*** `up23.sql` creates an index `lookup_attempts_for_webhook_delivery` on +`omicron.public.webhook_delivery_attempt`, for looking up the attempts for a given delivery ID. diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/webhooks/up01.sql similarity index 86% rename from schema/crdb/add-webhooks/up01.sql rename to schema/crdb/webhooks/up01.sql index 1f3954b46be..318088787c9 100644 --- a/schema/crdb/add-webhooks/up01.sql +++ b/schema/crdb/webhooks/up01.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( +CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( /* Identity metadata (resource) */ id UUID PRIMARY KEY, name STRING(63) NOT NULL, diff --git a/schema/crdb/webhooks/up02.sql b/schema/crdb/webhooks/up02.sql new file mode 100644 index 00000000000..618b6f64677 --- /dev/null +++ b/schema/crdb/webhooks/up02.sql @@ -0,0 +1,4 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rxs_by_id +ON omicron.public.webhook_receiver (id) +WHERE + time_deleted IS NULL; diff --git a/schema/crdb/add-webhooks/up02.sql b/schema/crdb/webhooks/up03.sql similarity index 57% rename from schema/crdb/add-webhooks/up02.sql rename to schema/crdb/webhooks/up03.sql index c1c9e47b800..4f46ece10a2 100644 --- a/schema/crdb/add-webhooks/up02.sql +++ b/schema/crdb/webhooks/up03.sql @@ -1,13 +1,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( + -- ID of this secret. + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- N.B. that this will always be equal to `time_created` for secrets, as + -- they are never modified once created. + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, -- UUID of the webhook receiver (foreign key into -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - -- ID of this secret. - signature_id STRING(63) NOT NULL, -- Secret value. - secret BYTES NOT NULL, - time_created TIMESTAMPTZ NOT NULL, - time_deleted TIMESTAMPTZ, - - PRIMARY KEY (signature_id, rx_id) + secret STRING(512) NOT NULL ); diff --git a/schema/crdb/add-webhooks/up03.sql b/schema/crdb/webhooks/up04.sql similarity index 100% rename from schema/crdb/add-webhooks/up03.sql rename to schema/crdb/webhooks/up04.sql diff --git a/schema/crdb/webhooks/up05.sql b/schema/crdb/webhooks/up05.sql new file mode 100644 index 00000000000..e00847f3cee --- /dev/null +++ b/schema/crdb/webhooks/up05.sql @@ -0,0 +1,13 @@ +CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( + -- Liveness probes, which are technically not real events, but, you know... + 'probe', + -- Test classes used to test globbing. + -- + -- These are not publicly exposed. + 'test.foo', + 'test.foo.bar', + 'test.foo.baz', + 'test.quux.bar', + 'test.quux.bar.baz' + -- Add new event classes here! +); diff --git a/schema/crdb/webhooks/up06.sql b/schema/crdb/webhooks/up06.sql new file mode 100644 index 00000000000..8aa06b10f56 --- /dev/null +++ b/schema/crdb/webhooks/up06.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class glob to which this receiver is subscribed. + glob STRING(512) NOT NULL, + -- Regex used when evaluating this filter against concrete event classes. + regex STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- The database schema version at which this glob was last expanded. + -- + -- This is used to detect when a glob must be re-processed to generate exact + -- subscriptions on schema changes. + schema_version STRING(64) NOT NULL, + + PRIMARY KEY (rx_id, glob) +); diff --git a/schema/crdb/webhooks/up07.sql b/schema/crdb/webhooks/up07.sql new file mode 100644 index 00000000000..c8dfef06f4e --- /dev/null +++ b/schema/crdb/webhooks/up07.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_for_rx +ON omicron.public.webhook_rx_event_glob ( + rx_id +); diff --git a/schema/crdb/webhooks/up08.sql b/schema/crdb/webhooks/up08.sql new file mode 100644 index 00000000000..be6363dae71 --- /dev/null +++ b/schema/crdb/webhooks/up08.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_by_schema_version +ON omicron.public.webhook_rx_event_glob (schema_version); diff --git a/schema/crdb/webhooks/up09.sql b/schema/crdb/webhooks/up09.sql new file mode 100644 index 00000000000..ffd6a0e8278 --- /dev/null +++ b/schema/crdb/webhooks/up09.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which the receiver is subscribed. + event_class omicron.public.webhook_event_class NOT NULL, + -- If this subscription is a concrete instantiation of a glob pattern, the + -- value of the glob that created it (and, a foreign key into + -- `webhook_rx_event_glob`). If the receiver is subscribed to this exact + -- event class, then this is NULL. + -- + -- This is used when deleting a glob subscription, as it is necessary to + -- delete any concrete subscriptions to individual event classes matching + -- that glob. + glob STRING(512), + + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) +); diff --git a/schema/crdb/add-webhooks/up06.sql b/schema/crdb/webhooks/up10.sql similarity index 77% rename from schema/crdb/add-webhooks/up06.sql rename to schema/crdb/webhooks/up10.sql index 407424440c6..8f79b29b605 100644 --- a/schema/crdb/add-webhooks/up06.sql +++ b/schema/crdb/webhooks/up10.sql @@ -1,6 +1,6 @@ -- Look up all webhook receivers subscribed to an event class. This is used by -- the dispatcher to determine who is interested in a particular event. -CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event +CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event_class ON omicron.public.webhook_rx_subscription ( event_class ); diff --git a/schema/crdb/webhooks/up11.sql b/schema/crdb/webhooks/up11.sql new file mode 100644 index 00000000000..0b3b6becd0e --- /dev/null +++ b/schema/crdb/webhooks/up11.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_exact_subscriptions_for_webhook_rx +on omicron.public.webhook_rx_subscription ( + rx_id +) WHERE glob IS NULL; diff --git a/schema/crdb/webhooks/up12.sql b/schema/crdb/webhooks/up12.sql new file mode 100644 index 00000000000..968bca5d40f --- /dev/null +++ b/schema/crdb/webhooks/up12.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- The class of event that this is. + event_class omicron.public.webhook_event_class NOT NULL, + -- Actual event data. The structure of this depends on the event class. + event JSONB NOT NULL, + + -- Set when dispatch entries have been created for this event. + time_dispatched TIMESTAMPTZ, + -- The number of receivers that this event was dispatched to. + num_dispatched INT8 NOT NULL, + + CONSTRAINT time_dispatched_set_if_dispatched CHECK ( + (num_dispatched = 0) OR (time_dispatched IS NOT NULL) + ), + + CONSTRAINT num_dispatched_is_positive CHECK ( + (num_dispatched >= 0) + ) +); diff --git a/schema/crdb/webhooks/up13.sql b/schema/crdb/webhooks/up13.sql new file mode 100644 index 00000000000..5941c172c0d --- /dev/null +++ b/schema/crdb/webhooks/up13.sql @@ -0,0 +1,19 @@ +-- Singleton probe event +INSERT INTO omicron.public.webhook_event ( + id, + time_created, + event_class, + event, + time_dispatched, + num_dispatched +) VALUES ( + -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. + '001de000-7768-4000-8000-000000000001', + NOW(), + 'probe', + '{}', + -- Pretend to be dispatched so we won't show up in "list events needing + -- dispatch" queries + NOW(), + 0 +) ON CONFLICT DO NOTHING; diff --git a/schema/crdb/add-webhooks/up08.sql b/schema/crdb/webhooks/up14.sql similarity index 52% rename from schema/crdb/add-webhooks/up08.sql rename to schema/crdb/webhooks/up14.sql index c64ae2db0df..81542e30acd 100644 --- a/schema/crdb/add-webhooks/up08.sql +++ b/schema/crdb/webhooks/up14.sql @@ -1,6 +1,3 @@ --- Look up webhook messages in need of dispatching. --- --- This is used by the message dispatcher when looking for messages to dispatch. CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_events ON omicron.public.webhook_event ( id, time_created diff --git a/schema/crdb/webhooks/up15.sql b/schema/crdb/webhooks/up15.sql new file mode 100644 index 00000000000..562c6abce9d --- /dev/null +++ b/schema/crdb/webhooks/up15.sql @@ -0,0 +1,9 @@ +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( + -- This delivery was triggered by the event being dispatched. + 'event', + -- This delivery was triggered by an explicit call to the webhook event + -- resend API. + 'resend', + --- This delivery is a liveness probe. + 'probe' +); diff --git a/schema/crdb/webhooks/up16.sql b/schema/crdb/webhooks/up16.sql new file mode 100644 index 00000000000..466ad2ae955 --- /dev/null +++ b/schema/crdb/webhooks/up16.sql @@ -0,0 +1,37 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( + -- UUID of this delivery. + id UUID PRIMARY KEY, + --- UUID of the event (foreign key into `omicron.public.webhook_event`). + event_id UUID NOT NULL, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + + trigger omicron.public.webhook_delivery_trigger NOT NULL, + + payload JSONB NOT NULL, + + --- Delivery attempt count. Starts at 0. + attempts INT2 NOT NULL, + + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, + -- If true, this webhook delivery has failed permanently and is eligible to + -- be resent. + failed_permanently BOOLEAN NOT NULL, + -- Deliverator coordination bits + deliverator_id UUID, + time_delivery_started TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), + CONSTRAINT active_deliveries_have_started_timestamps CHECK ( + (deliverator_id IS NULL) OR ( + (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) + ) + ), + CONSTRAINT failed_permanently_only_if_completed CHECK ( + (failed_permanently IS false) OR (failed_permanently AND (time_completed IS NOT NULL)) + ) +); diff --git a/schema/crdb/webhooks/up17.sql b/schema/crdb/webhooks/up17.sql new file mode 100644 index 00000000000..2c825749aae --- /dev/null +++ b/schema/crdb/webhooks/up17.sql @@ -0,0 +1,6 @@ +CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx +ON omicron.public.webhook_delivery ( + event_id, rx_id +) +WHERE + trigger = 'event'; diff --git a/schema/crdb/webhooks/up18.sql b/schema/crdb/webhooks/up18.sql new file mode 100644 index 00000000000..e3ff154e746 --- /dev/null +++ b/schema/crdb/webhooks/up18.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx +ON omicron.public.webhook_delivery ( + rx_id, event_id +); diff --git a/schema/crdb/webhooks/up19.sql b/schema/crdb/webhooks/up19.sql new file mode 100644 index 00000000000..49709646b92 --- /dev/null +++ b/schema/crdb/webhooks/up19.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event +ON omicron.public.webhook_delivery ( + event_id +); diff --git a/schema/crdb/webhooks/up20.sql b/schema/crdb/webhooks/up20.sql new file mode 100644 index 00000000000..ba3c68c7e44 --- /dev/null +++ b/schema/crdb/webhooks/up20.sql @@ -0,0 +1,5 @@ +CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight +ON omicron.public.webhook_delivery ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up12.sql b/schema/crdb/webhooks/up21.sql similarity index 100% rename from schema/crdb/add-webhooks/up12.sql rename to schema/crdb/webhooks/up21.sql diff --git a/schema/crdb/add-webhooks/up13.sql b/schema/crdb/webhooks/up22.sql similarity index 99% rename from schema/crdb/add-webhooks/up13.sql rename to schema/crdb/webhooks/up22.sql index 04e3acbf9db..0f41d31ae77 100644 --- a/schema/crdb/add-webhooks/up13.sql +++ b/schema/crdb/webhooks/up22.sql @@ -1,3 +1,4 @@ + CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- Foreign key into `omicron.public.webhook_delivery`. delivery_id UUID NOT NULL, diff --git a/schema/crdb/webhooks/up23.sql b/schema/crdb/webhooks/up23.sql new file mode 100644 index 00000000000..cd08b4b316b --- /dev/null +++ b/schema/crdb/webhooks/up23.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery +ON omicron.public.webhook_delivery_attempt ( + delivery_id +); From 16a4c14f9189ca4fc44fe1367e811f1538bc70bb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 3 Mar 2025 15:36:16 -0800 Subject: [PATCH 098/168] start on a test for glob reprocessing --- nexus/db-model/src/webhook_event_class.rs | 2 + .../background/tasks/webhook_dispatcher.rs | 162 ++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/nexus/db-model/src/webhook_event_class.rs b/nexus/db-model/src/webhook_event_class.rs index d6c9ba43964..3654693b079 100644 --- a/nexus/db-model/src/webhook_event_class.rs +++ b/nexus/db-model/src/webhook_event_class.rs @@ -17,6 +17,8 @@ impl_enum_type!( Clone, Debug, PartialEq, + Eq, + Hash, AsExpression, FromSqlRow, strum::VariantArray, diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 904000c7613..0919a733592 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -290,3 +290,165 @@ impl WebhookDispatcher { Ok(()) } } + +#[cfg(test)] +mod test { + use super::*; + use nexus_db_queries::db; + // use nexus_test_utils::resource_helpers; + use async_bb8_diesel::AsyncRunQueryDsl; + use diesel::prelude::*; + use nexus_test_utils_macros::nexus_test; + use omicron_common::api::external::IdentityMetadataCreateParams; + use omicron_common::api::external::SemverVersion; + use omicron_uuid_kinds::WebhookEventUuid; + use omicron_uuid_kinds::WebhookReceiverUuid; + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + // Tests that stale webhook event class globs are reprocessed prior to event + // dispatching. + #[nexus_test(server = crate::Server)] + async fn test_glob_reprocessing(cptestctx: &ControlPlaneTestContext) { + use nexus_db_model::schema::webhook_receiver::dsl as rx_dsl; + use nexus_db_model::schema::webhook_rx_event_glob::dsl as glob_dsl; + use nexus_db_model::schema::webhook_rx_subscription::dsl as subscription_dsl; + + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let rx_id = WebhookReceiverUuid::new_v4(); + let conn = datastore + .pool_connection_for_tests() + .await + .expect("can't get ye pool_connection_for_tests"); + + // Unfortunately, we've gotta hand-create the receiver and its + // subscriptions, so that we can create a set of globs that differs from + // those generated by the currrent schema. + diesel::insert_into(rx_dsl::webhook_receiver) + .values(db::model::WebhookReceiver { + identity: db::model::WebhookReceiverIdentity::new( + rx_id, + IdentityMetadataCreateParams { + name: "my-cool-webhook".parse().unwrap(), + description: "it's my cool webhook".to_string(), + }, + ), + + endpoint: "http://webhooks.elizas.website".parse().unwrap(), + rcgen: db::model::Generation::new(), + }) + .execute_async(&*conn) + .await + .expect("receiver entry should create"); + + const GLOB_PATTERN: &str = "test.*.bar"; + let glob = GLOB_PATTERN + .parse::() + .expect("'test.*.bar should be an acceptable glob"); + let mut glob = db::model::WebhookRxEventGlob::new(rx_id, glob); + // Just make something up that's obviously outdated... + glob.schema_version = SemverVersion::new(100, 0, 0).into(); + diesel::insert_into(glob_dsl::webhook_rx_event_glob) + .values(glob.clone()) + .execute_async(&*conn) + .await + .expect("should insert glob entry"); + diesel::insert_into(subscription_dsl::webhook_rx_subscription) + .values( + // Pretend `test.quux.bar` doesn't exist yet + db::model::WebhookRxSubscription::for_glob( + &glob, + db::model::WebhookEventClass::TestFooBar, + ), + ) + .execute_async(&*conn) + .await + .expect("should insert glob entry"); + // Also give the webhook receiver a secret just so everything + // looks normalish. + let (authz_rx, _) = db::lookup::LookupPath::new(&opctx, datastore) + .webhook_receiver_id(rx_id) + .fetch() + .await + .expect("webhook rx should be there"); + datastore + .webhook_rx_secret_create( + &opctx, + &authz_rx, + db::model::WebhookSecret::new(rx_id, "TRUSTNO1".to_string()), + ) + .await + .expect("cant insert ye secret???"); + + // OKAY GREAT NOW THAT WE DID ALL THAT STUFF let's see if it actually + // works... + + // N.B. that we are using the `DataStore::webhook_event_create` method + // rather than `Nexus::webhook_event_publish` (the expected entrypoint + // to publishing a webhook event) because `webhook_event_publish` also + // activates the dispatcher task, and for this test, we would like to be + // responsible for activating it. + datastore + .webhook_event_create( + &opctx, + WebhookEventUuid::new_v4(), + db::model::WebhookEventClass::TestQuuxBar, + serde_json::json!({"msg": "help im trapped in a webhook event factory"}), + ) + .await + .expect("creating the event should work"); + + // okay now do the thing + let mut status = WebhookDispatcherStatus { + globs_reprocessed: Default::default(), + glob_version: SCHEMA_VERSION, + dispatched: Vec::new(), + errors: Vec::new(), + no_receivers: Vec::new(), + }; + let mut task = WebhookDispatcher::new( + datastore.clone(), + nexus.background_tasks.task_webhook_deliverator.clone(), + ); + task.actually_activate(&opctx, &mut status) + .await + .expect("activation should succeed"); + + // The globs should have been reprocessed, creating a subscription to + // `test.quux.bar`. + let subscriptions = subscription_dsl::webhook_rx_subscription + .filter(subscription_dsl::rx_id.eq(rx_id.into_untyped_uuid())) + .load_async::(&*conn) + .await + .expect("should be able to get subscriptions") + .into_iter() + .map(|sub| { + // throw away the "time_created" fields so that assertions are + // easier... + assert_eq!( + sub.glob.as_deref(), + Some(GLOB_PATTERN), + "found a subscription to {} that was not from our glob: {sub:?}", + sub.event_class, + ); + sub.event_class + }).collect::>(); + assert_eq!(subscriptions.len(), 2); + assert!( + subscriptions.contains(&db::model::WebhookEventClass::TestFooBar), + "subscription to test.foo.bar should exist; subscriptions: \ + {subscriptions:?}", + ); + assert!( + subscriptions.contains(&db::model::WebhookEventClass::TestQuuxBar), + "subscription to test.quux.bar should exist; subscriptions: \ + {subscriptions:?}", + ); + } +} From 6c3e70315b89c93637a0f0a37695292ed6589ebc Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Mar 2025 09:39:47 -0800 Subject: [PATCH 099/168] assert more stuff in glob-reprocessing test --- .../background/tasks/webhook_dispatcher.rs | 58 ++++++++++++++++++- nexus/src/app/webhook.rs | 4 ++ nexus/types/src/external_api/shared.rs | 16 ++++- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 0919a733592..cd122ea0b1b 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -294,11 +294,11 @@ impl WebhookDispatcher { #[cfg(test)] mod test { use super::*; - use nexus_db_queries::db; - // use nexus_test_utils::resource_helpers; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; + use nexus_db_queries::db; use nexus_test_utils_macros::nexus_test; + use nexus_types::external_api::shared::WebhookDeliveryState; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::SemverVersion; use omicron_uuid_kinds::WebhookEventUuid; @@ -394,10 +394,11 @@ mod test { // to publishing a webhook event) because `webhook_event_publish` also // activates the dispatcher task, and for this test, we would like to be // responsible for activating it. + let event_id = WebhookEventUuid::new_v4(); datastore .webhook_event_create( &opctx, - WebhookEventUuid::new_v4(), + event_id, db::model::WebhookEventClass::TestQuuxBar, serde_json::json!({"msg": "help im trapped in a webhook event factory"}), ) @@ -412,6 +413,7 @@ mod test { errors: Vec::new(), no_receivers: Vec::new(), }; + let mut task = WebhookDispatcher::new( datastore.clone(), nexus.background_tasks.task_webhook_deliverator.clone(), @@ -450,5 +452,55 @@ mod test { "subscription to test.quux.bar should exist; subscriptions: \ {subscriptions:?}", ); + let rx_reprocessed_globs = status.globs_reprocessed.get(&rx_id).expect( + "expected there to be an entry in status.globs_reprocessed \ + for our glob", + ); + let reprocessed_entry = dbg!(rx_reprocessed_globs).get(GLOB_PATTERN); + assert!( + matches!( + reprocessed_entry, + Some(Ok(WebhookGlobStatus::Reprocessed { .. })) + ), + "glob status should be 'reprocessed'" + ); + + // There should now be a delivery entry for the event we published. + // + // Use `webhook_rx_delivery_list_attempts` rather than + // `webhook_rx_delivery_list_ready`, even though it's a bit more + // complex due to requiring pagination. This is because the + // webhook_deliverator background task may have activated and might + // attempt to deliver the event, making it no longer show up in the + // "ready" query. + let mut paginator = Paginator::new(db::datastore::SQL_BATCH_SIZE); + let mut deliveries = Vec::new(); + while let Some(p) = paginator.next() { + let batch = datastore + .webhook_rx_delivery_list_attempts( + &opctx, + &rx_id, + &[WebhookDeliveryTrigger::Event], + WebhookDeliveryState::ALL.iter().copied(), + &p.current_pagparams(), + ) + .await + .unwrap(); + paginator = p.found_batch(&batch, &|(delivery, attempt, _)| { + let id = delivery.id.into_untyped_uuid(); + let attempt = attempt + .as_ref() + .map(|attempt| attempt.attempt) + .unwrap_or_else(|| 0.into()); + (id, attempt) + }); + deliveries.extend(batch); + } + let event = + deliveries.iter().find(|(d, _, _)| d.event_id == event_id.into()); + assert!( + dbg!(event).is_some(), + "delivery entry for dispatched event must exist" + ); } } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 68047dc4da7..85232549232 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -3,6 +3,10 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Webhooks +//! +//! # Webhooks: Theory and Practice +//! +//! For a discussion of use crate::app::external_dns; use anyhow::Context; diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs index 344552eeb43..b1dad30e51f 100644 --- a/nexus/types/src/external_api/shared.rs +++ b/nexus/types/src/external_api/shared.rs @@ -512,7 +512,17 @@ impl RelayState { } /// The state of a webhook delivery attempt. -#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] +#[derive( + Copy, + Clone, + Debug, + Eq, + PartialEq, + Deserialize, + Serialize, + JsonSchema, + strum::VariantArray, +)] #[serde(rename_all = "snake_case")] pub enum WebhookDeliveryState { /// The webhook event has not yet been delivered. @@ -528,3 +538,7 @@ pub enum WebhookDeliveryState { /// no response was received within the delivery timeout. FailedTimeout, } + +impl WebhookDeliveryState { + pub const ALL: &[Self] = ::VARIANTS; +} From 8c93e91f0337f1e6a1a3e834720e0ccbda9d2e8a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Mar 2025 10:25:45 -0800 Subject: [PATCH 100/168] use proper `IdentityMetadata` in views --- nexus/db-model/src/webhook_rx.rs | 21 ++++++------ nexus/tests/integration_tests/webhooks.rs | 41 +++++++++-------------- nexus/types/src/external_api/views.rs | 14 ++++---- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index a7937baadfc..ddef7e83bb1 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -15,6 +15,7 @@ use crate::WebhookEventClass; use chrono::{DateTime, Utc}; use db_macros::{Asset, Resource}; use nexus_types::external_api::views; +use nexus_types::identity::Resource; use omicron_common::api::external::Error; use omicron_uuid_kinds::{ GenericUuid, WebhookReceiverKind, WebhookReceiverUuid, WebhookSecretUuid, @@ -47,17 +48,17 @@ impl TryFrom for views::Webhook { .into_iter() .map(WebhookSubscriptionKind::into_event_class_string) .collect(); - let WebhookReceiver { identity, endpoint, rcgen: _ } = rx; - let WebhookReceiverIdentity { id, name, description, .. } = identity; - let endpoint = endpoint.parse().map_err(|e| Error::InternalError { - // This is an internal error, as we should not have ever allowed - // an invalid URL to be inserted into the database... - internal_message: format!("invalid webhook URL {endpoint:?}: {e}",), - })?; + let endpoint = + rx.endpoint.parse().map_err(|e| Error::InternalError { + // This is an internal error, as we should not have ever allowed + // an invalid URL to be inserted into the database... + internal_message: format!( + "invalid webhook URL {:?}: {e}", + rx.endpoint, + ), + })?; Ok(views::Webhook { - id: id.into(), - name: name.to_string(), - description, + identity: rx.identity(), endpoint, secrets, events, diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 7d122921f45..f0271daf587 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -100,7 +100,7 @@ fn is_valid_for_webhook( webhook: &views::Webhook, ) -> impl FnOnce(httpmock::When) -> httpmock::When { let path = webhook.endpoint.path().to_string(); - let id = webhook.id.to_string(); + let id = webhook.identity.id.to_string(); move |when| { when.path(path) .header("x-oxide-webhook-id", id) @@ -257,6 +257,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { ) .await; dbg!(&webhook); + let rx_id = WebhookReceiverUuid::from_untyped_uuid(webhook.identity.id); let secret1_id = webhook.secrets[0].id; @@ -264,7 +265,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { let secret2_id = dbg!( secret_add( &cptestctx, - webhook.id, + rx_id, ¶ms::WebhookSecretCreate { secret: SECRET2.to_string() }, ) .await @@ -275,7 +276,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { let secret3_id = dbg!( secret_add( &cptestctx, - webhook.id, + rx_id, ¶ms::WebhookSecretCreate { secret: SECRET3.to_string() }, ) .await @@ -652,6 +653,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { let webhook = webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; dbg!(&webhook); + let rx_id = WebhookReceiverUuid::from_untyped_uuid(webhook.identity.id); let body = serde_json::json!({ "event_class": "probe", @@ -687,13 +689,9 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { }; // Send a probe. The probe should fail due to a timeout. - let probe1 = webhook_send_probe( - &cptestctx, - &webhook.id, - false, - http::StatusCode::OK, - ) - .await; + let probe1 = + webhook_send_probe(&cptestctx, &rx_id, false, http::StatusCode::OK) + .await; dbg!(&probe1); mock.assert_async().await; @@ -727,13 +725,9 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { .await }; - let probe2 = webhook_send_probe( - &cptestctx, - &webhook.id, - false, - http::StatusCode::OK, - ) - .await; + let probe2 = + webhook_send_probe(&cptestctx, &rx_id, false, http::StatusCode::OK) + .await; dbg!(&probe2); mock.assert_async().await; @@ -773,13 +767,9 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { .await }; - let probe3 = webhook_send_probe( - &cptestctx, - &webhook.id, - false, - http::StatusCode::OK, - ) - .await; + let probe3 = + webhook_send_probe(&cptestctx, &rx_id, false, http::StatusCode::OK) + .await; dbg!(&probe3); mock.assert_async().await; assert_eq!(probe3.probe.attempt, 1); @@ -929,8 +919,9 @@ async fn test_probe_resends_failed_deliveries( }; // Send a probe with ?resend=true + let rx_id = WebhookReceiverUuid::from_untyped_uuid(webhook.identity.id); let probe = - webhook_send_probe(&cptestctx, &webhook.id, true, http::StatusCode::OK) + webhook_send_probe(&cptestctx, &rx_id, true, http::StatusCode::OK) .await; dbg!(&probe); probe_mock.assert_async().await; diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index cb0d44246fa..3f050197466 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1061,13 +1061,13 @@ pub struct EventClass { } /// The configuration for a webhook. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive( + ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, +)] pub struct Webhook { - /// The UUID of this webhook receiver. - pub id: WebhookReceiverUuid, - /// The identifier assigned to this webhook receiver upon creation. - pub name: String, - pub description: String, + #[serde(flatten)] + pub identity: IdentityMetadata, + /// The URL that webhook notification requests are sent to. pub endpoint: Url, // A list containing the IDs of the secret keys used to sign payloads sent @@ -1084,7 +1084,7 @@ pub struct WebhookSecrets { } /// The public ID of a secret key assigned to a webhook. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct WebhookSecretId { pub id: Uuid, } From ad4ba5bc97f41e4b93dd269ad6bfc7400666c5f0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Mar 2025 10:55:01 -0800 Subject: [PATCH 101/168] quick test for webhook lookups --- nexus/tests/integration_tests/webhooks.rs | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index f0271daf587..45631b61974 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -4,6 +4,7 @@ //! Webhooks +use dropshot::test_util::ClientTestContext; use hmac::{Hmac, Mac}; use httpmock::prelude::*; use nexus_db_model::WebhookEventClass; @@ -16,6 +17,8 @@ use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::{params, shared, views}; use omicron_common::api::external::IdentityMetadataCreateParams; +use omicron_common::api::external::NameOrId; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use sha2::Sha256; @@ -39,6 +42,32 @@ async fn webhook_create( .await } +fn get_webhooks_url(name_or_id: impl Into) -> String { + let name_or_id = name_or_id.into(); + format!("{WEBHOOKS_BASE_PATH}/{name_or_id}") +} + +async fn webhook_get( + client: &ClientTestContext, + webhook_url: &str, +) -> views::Webhook { + webhook_get_as(client, webhook_url, AuthnMode::PrivilegedUser).await +} + +async fn webhook_get_as( + client: &ClientTestContext, + webhook_url: &str, + authn_as: AuthnMode, +) -> views::Webhook { + NexusRequest::object_get(client, &webhook_url) + .authn_as(authn_as) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + fn my_great_webhook_params( mock: &httpmock::MockServer, ) -> params::WebhookCreate { @@ -159,6 +188,28 @@ fn signature_verifies( } } +#[nexus_test] +async fn test_webhook_get(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + let server = httpmock::MockServer::start_async().await; + + // Create a webhook receiver. + let created_webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&created_webhook); + + // Fetch the receiver by name. + let by_id_url = get_webhooks_url(created_webhook.identity.id); + let webhook_view = webhook_get(client, &by_id_url).await; + assert_eq!(created_webhook, webhook_view); + + // Fetch the receiver by name. + let by_name_url = get_webhooks_url(created_webhook.identity.name.clone()); + let webhook_view = webhook_get(client, &by_name_url).await; + assert_eq!(created_webhook, webhook_view); +} + #[nexus_test] async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let nexus = cptestctx.server.server_context().nexus.clone(); From 54236f772a4cb8f1270db41eb2d34e3e607f1925 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 4 Mar 2025 13:02:44 -0800 Subject: [PATCH 102/168] ssrf prevention will be done via IP_BOUND_IF --- nexus/src/app/webhook.rs | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 85232549232..11a49da6ef5 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -243,20 +243,6 @@ impl super::Nexus { pub(super) fn delivery_client( external_dns: &Arc, ) -> Result { - /// A wrapper around [`external_dns::Resolver`] which rejects IP addresses that - /// are underlay network IPs. - struct WebhookDnsResolver { - external_dns: Arc, - } - - impl reqwest::dns::Resolve for WebhookDnsResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - // TODO(eliza): this is where we have to actually return an error if the - // DNS name resolves to an underlay IP! Figure that out! - self.external_dns.resolve(name) - } - } - reqwest::Client::builder() // Per [RFD 538 § 4.3.1][1], webhook delivery does *not* follow // redirects. @@ -274,10 +260,7 @@ pub(super) fn delivery_client( // // [1]: https://rfd.shared.oxide.computer/rfd/538#delivery-failure .timeout(Duration::from_secs(30)) - // my god...it's full of Arcs... - .dns_resolver(Arc::new(WebhookDnsResolver { - external_dns: external_dns.clone(), - })) + .dns_resolver(external_dns.clone()) .build() } From 440ab83bd4e53603938463631814c4a77073e5c2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 5 Mar 2025 11:00:30 -0800 Subject: [PATCH 103/168] implement webhook rx delete --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_delivery.rs | 4 + .../db-queries/src/db/datastore/webhook_rx.rs | 93 +++++++++++++++++++ nexus/src/app/webhook.rs | 1 + nexus/src/external_api/http_entrypoints.rs | 12 ++- schema/crdb/dbinit.sql | 10 ++ schema/crdb/webhooks/README.adoc | 3 +- schema/crdb/webhooks/up22.sql | 5 + schema/crdb/webhooks/up24.sql | 0 9 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 schema/crdb/webhooks/up24.sql diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 5ce6ff18bc4..32518ba138d 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2271,6 +2271,7 @@ table! { webhook_delivery_attempt (delivery_id, attempt) { delivery_id -> Uuid, attempt -> Int2, + rx_id -> Uuid, result -> crate::WebhookDeliveryResultEnum, response_status -> Nullable, response_duration -> Nullable, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index cf80117305b..4e186aac0ac 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -183,6 +183,10 @@ pub struct WebhookDeliveryAttempt { /// Attempt number (retry count). pub attempt: SqlU8, + /// ID of the receiver to which this event is dispatched (foreign key into + /// `webhook_rx`). + pub rx_id: DbTypedUuid, + pub result: WebhookDeliveryResult, pub response_status: Option, diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index bd6653581a2..ff102a9eb39 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -27,6 +27,8 @@ use crate::db::model::SCHEMA_VERSION; use crate::db::pagination::paginated; use crate::db::pagination::paginated_multicolumn; use crate::db::pool::DbConnection; +use crate::db::schema::webhook_delivery::dsl as delivery_dsl; +use crate::db::schema::webhook_delivery_attempt::dsl as delivery_attempt_dsl; use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; @@ -39,6 +41,7 @@ use nexus_types::external_api::params; use nexus_types::internal_api::background::WebhookGlobStatus; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; @@ -149,6 +152,96 @@ impl DataStore { Ok((subscriptions, secrets)) } + pub async fn webhook_rx_delete( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_rx).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let rx_id = authz_rx.id().into_untyped_uuid(); + // First, mark the webhook receiver record as deleted. + diesel::update(rx_dsl::webhook_receiver) + .filter(rx_dsl::id.eq(rx_id)) + .filter(rx_dsl::time_deleted.is_null()) + .set(( + rx_dsl::time_deleted.eq(chrono::Utc::now()), + rx_dsl::rcgen.eq(rx_dsl::rcgen + 1), + )) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to mark receiver as deleted") + })?; + + // Now, delete the webhook's secrets. + let secrets_deleted = diesel::delete(secret_dsl::webhook_secret) + .filter(secret_dsl::rx_id.eq(rx_id)) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to delete secrets") + })?; + + // Delete subscriptions and globs. + let exact_subscriptions_deleted = + diesel::delete(subscription_dsl::webhook_rx_subscription) + .filter(subscription_dsl::rx_id.eq(rx_id)) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to delete exact subscriptions", + ) + })?; + + let globs_deleted = diesel::delete(glob_dsl::webhook_rx_event_glob) + .filter(glob_dsl::rx_id.eq(rx_id)) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to delete globs") + })?; + + let deliveries_deleted = diesel::delete(delivery_dsl::webhook_delivery) + .filter(delivery_dsl::rx_id.eq(rx_id)) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to delete delivery records") + })?; + + let delivery_attempts_deleted = + diesel::delete(delivery_attempt_dsl::webhook_delivery_attempt) + .filter(delivery_attempt_dsl::rx_id.eq(rx_id)) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to delete delivery attempt records", + ) + })?; + + slog::debug!( + &opctx.log, + "deleted webhook receiver"; + "rx_id" => %rx_id, + "secrets_deleted" => ?secrets_deleted, + "exact_subscriptions_deleted" => ?exact_subscriptions_deleted, + "globs_deleted" => ?globs_deleted, + "deliveries_deleted" => ?deliveries_deleted, + "delivery_attempts_deleted" => ?delivery_attempts_deleted, + ); + + Ok(()) + } + // // Subscriptions // diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 11a49da6ef5..c8fb78eeee7 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -507,6 +507,7 @@ impl<'a> ReceiverClient<'a> { Ok(WebhookDeliveryAttempt { delivery_id: delivery.id, + rx_id: delivery.rx_id, attempt: SqlU8::new(delivery.attempts.0 + 1), result: delivery_result, response_status: status.map(|s| s.as_u16() as i16), diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 89c28680106..3be42dc603b 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7401,7 +7401,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delete( rqctx: RequestContext, - _path_params: Path, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -7410,10 +7410,12 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook_selector = path_params.into_inner(); + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let (authz_rx,) = rx.lookup_for(authz::Action::Delete).await?; + nexus.datastore().webhook_rx_delete(&opctx, &authz_rx).await?; + + Ok(HttpResponseDeleted()) }; apictx .context diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 6ea1bc4856d..ee7de41ee22 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5252,6 +5252,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( delivery_id UUID NOT NULL, -- attempt number. attempt INT2 NOT NULL, + + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + result omicron.public.webhook_delivery_result NOT NULL, -- A status code > 599 would be Very Surprising, so rather than using an -- INT4 to store a full unsigned 16-bit number in the database, we'll use a @@ -5298,6 +5303,11 @@ ON omicron.public.webhook_delivery_attempt ( delivery_id ); +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_to_rx +ON omicron.public.webhook_delivery_attempt ( + rx_id +); + /* * Keep this at the end of file so that the database does not contain a version * until it is fully populated. diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index dd32df7877d..7155fd742c2 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -51,4 +51,5 @@ representing the potential outcomes of a webhook delivery attempt. which records each individual delivery attempt for a webhook delivery in the `webhook_delivery` table. *** `up23.sql` creates an index `lookup_attempts_for_webhook_delivery` on -`omicron.public.webhook_delivery_attempt`, for looking up the attempts for a given delivery ID. +`webhook_delivery_attempt`, for looking up the attempts for a given delivery ID. +*** `up24.sql` creates an index `lookup_webhook_delivery_attempts_to_rx` on the `webhook_delivery_attempt` table, for looking up delivery attempts to a given receiver ID. This is primarily used for deleting delivery attempts when a receiver is deleted. diff --git a/schema/crdb/webhooks/up22.sql b/schema/crdb/webhooks/up22.sql index 0f41d31ae77..d3ebfd096b7 100644 --- a/schema/crdb/webhooks/up22.sql +++ b/schema/crdb/webhooks/up22.sql @@ -4,6 +4,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( delivery_id UUID NOT NULL, -- attempt number. attempt INT2 NOT NULL, + + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + result omicron.public.webhook_delivery_result NOT NULL, -- A status code > 599 would be Very Surprising, so rather than using an -- INT4 to store a full unsigned 16-bit number in the database, we'll use a diff --git a/schema/crdb/webhooks/up24.sql b/schema/crdb/webhooks/up24.sql new file mode 100644 index 00000000000..e69de29bb2d From 3c7f65c71fa8b24ce193bfb9b2d196772173b763 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 5 Mar 2025 13:55:05 -0800 Subject: [PATCH 104/168] flatten API routes (per @david-crespo) also, implement the secret-delete route while i'm here --- .../db-queries/src/db/datastore/webhook_rx.rs | 24 +- nexus/db-queries/src/db/lookup.rs | 14 +- nexus/external-api/output/nexus_tags.txt | 24 +- nexus/external-api/src/lib.rs | 33 +- nexus/src/app/webhook.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 36 ++- nexus/types/src/external_api/params.rs | 8 +- openapi/nexus.json | 297 +++++++++--------- 8 files changed, 244 insertions(+), 194 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index ff102a9eb39..e21fade3640 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -45,7 +45,8 @@ use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; -use omicron_uuid_kinds::{GenericUuid, WebhookReceiverUuid}; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::WebhookReceiverUuid; use uuid::Uuid; impl DataStore { @@ -682,6 +683,27 @@ impl DataStore { Ok(secret) } + pub async fn webhook_rx_secret_delete( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + authz_secret: &authz::WebhookSecret, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_secret).await?; + diesel::delete(secret_dsl::webhook_secret) + .filter(secret_dsl::id.eq(authz_secret.id().into_untyped_uuid())) + .filter(secret_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(&opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_secret), + ) + })?; + Ok(()) + } + async fn add_secret_on_conn( &self, secret: WebhookSecret, diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index fdf793b998d..e835ce16415 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -27,6 +27,7 @@ use omicron_uuid_kinds::TufArtifactKind; use omicron_uuid_kinds::TufRepoKind; use omicron_uuid_kinds::TypedUuid; use omicron_uuid_kinds::WebhookReceiverUuid; +use omicron_uuid_kinds::WebhookSecretUuid; use uuid::Uuid; /// Look up an API resource in the database @@ -578,6 +579,17 @@ impl<'a> LookupPath<'a> { { WebhookReceiver::OwnedName(Root { lookup_root: self }, name) } + + /// Select a resource of type [`WebhookSecret`], identified by its UUID. + pub fn webhook_secret_id<'b>( + self, + id: WebhookSecretUuid, + ) -> WebhookSecret<'b> + where + 'a: 'b, + { + WebhookSecret::PrimaryKey(Root { lookup_root: self }, id) + } } /// Represents the head of the selection path for a resource @@ -956,7 +968,7 @@ lookup_resource! { name = "WebhookSecret", ancestors = ["WebhookReceiver"], lookup_by_name = false, - soft_deletes = true, + soft_deletes = false, primary_key_columns = [ { column_name = "id", uuid_kind = WebhookSecretKind } ] diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 41300513437..b9a9dea1bcb 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -266,18 +266,18 @@ ping GET /v1/ping API operations found with tag "system/webhooks" OPERATION ID METHOD URL PATH -webhook_create POST /experimental/v1/webhooks -webhook_delete DELETE /experimental/v1/webhooks/{webhook} -webhook_delivery_list GET /experimental/v1/webhooks/{webhook}/deliveries -webhook_delivery_resend POST /experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend -webhook_event_class_list GET /experimental/v1/webhook-events/classes -webhook_event_class_view GET /experimental/v1/webhook-events/classes/{name} -webhook_probe POST /experimental/v1/webhooks/{webhook}/probe -webhook_secrets_add POST /experimental/v1/webhooks/{webhook}/secrets -webhook_secrets_delete DELETE /experimental/v1/webhooks/{webhook}/secrets/{secret_id} -webhook_secrets_list GET /experimental/v1/webhooks/{webhook}/secrets -webhook_update PUT /experimental/v1/webhooks/{webhook} -webhook_view GET /experimental/v1/webhooks/{webhook} +webhook_create POST /experimental/v1/webhooks/receivers +webhook_delete DELETE /experimental/v1/webhooks/receivers/{receiver} +webhook_delivery_list GET /experimental/v1/webhooks/deliveries +webhook_delivery_resend POST /experimental/v1/webhooks/deliveries/{event_id}/resend +webhook_event_class_list GET /experimental/v1/webhooks/event-classes +webhook_event_class_view GET /experimental/v1/webhooks/event-classes/{name} +webhook_probe POST /experimental/v1/webhooks/receivers/{receiver}/probe +webhook_secrets_add POST /experimental/v1/webhooks/secrets +webhook_secrets_delete DELETE /experimental/v1/webhooks/secrets/{secret_id} +webhook_secrets_list GET /experimental/v1/webhooks/secrets +webhook_update PUT /experimental/v1/webhooks/receivers/{receiver} +webhook_view GET /experimental/v1/webhooks/receivers/{receiver} API operations found with tag "vpcs" OPERATION ID METHOD URL PATH diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 8be737decc3..05fff550248 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3479,7 +3479,7 @@ pub trait NexusExternalApi { /// List webhook event classes #[endpoint { method = GET, - path = "/experimental/v1/webhook-events/classes", + path = "/experimental/v1/webhooks/event-classes", tags = ["system/webhooks"], }] async fn webhook_event_class_list( @@ -3492,7 +3492,7 @@ pub trait NexusExternalApi { /// Fetch details on an event class by name. #[endpoint { method = GET, - path ="/experimental/v1/webhook-events/classes/{name}", + path ="/experimental/v1/webhooks/event-classes/{name}", tags = ["system/webhooks"], }] async fn webhook_event_class_view( @@ -3503,7 +3503,7 @@ pub trait NexusExternalApi { /// Get the configuration for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook}", + path = "/experimental/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_view( @@ -3514,7 +3514,7 @@ pub trait NexusExternalApi { /// Create a new webhook receiver. #[endpoint { method = POST, - path = "/experimental/v1/webhooks", + path = "/experimental/v1/webhooks/receivers", tags = ["system/webhooks"], }] async fn webhook_create( @@ -3525,7 +3525,7 @@ pub trait NexusExternalApi { /// Update the configuration of an existing webhook receiver. #[endpoint { method = PUT, - path = "/experimental/v1/webhooks/{webhook}", + path = "/experimental/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_update( @@ -3537,7 +3537,7 @@ pub trait NexusExternalApi { /// Delete a webhook receiver. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/{webhook}", + path = "/experimental/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_delete( @@ -3551,7 +3551,7 @@ pub trait NexusExternalApi { // status code from the webhook endpoint... #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook}/probe", + path = "/experimental/v1/webhooks/receivers/{receiver}/probe", tags = ["system/webhooks"], }] async fn webhook_probe( @@ -3563,30 +3563,30 @@ pub trait NexusExternalApi { /// List the IDs of secrets for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook}/secrets", + path = "/experimental/v1/webhooks/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_list( rqctx: RequestContext, - path_params: Path, + query_params: Query, ) -> Result, HttpError>; /// Add a secret to a webhook receiver. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook}/secrets", + path = "/experimental/v1/webhooks/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_add( rqctx: RequestContext, - path_params: Path, + query_params: Query, params: TypedBody, ) -> Result, HttpError>; /// Delete a secret associated with a webhook receiver by ID. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/{webhook}/secrets/{secret_id}", + path = "/experimental/v1/webhooks/secrets/{secret_id}", tags = ["system/webhooks"], }] async fn webhook_secrets_delete( @@ -3597,24 +3597,25 @@ pub trait NexusExternalApi { /// List delivery attempts to a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/{webhook}/deliveries", + path = "/experimental/v1/webhooks/deliveries", tags = ["system/webhooks"], }] async fn webhook_delivery_list( rqctx: RequestContext, - path_params: Path, - query_params: Query, + receiver: Query, + pagination: Query, ) -> Result>, HttpError>; /// Request re-delivery of a webhook event. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend", + path = "/experimental/v1/webhooks/deliveries/{event_id}/resend", tags = ["system/webhooks"], }] async fn webhook_delivery_resend( rqctx: RequestContext, path_params: Path, + receiver: Query, ) -> Result, HttpError>; } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index c8fb78eeee7..7ff0f763c77 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -52,7 +52,7 @@ impl super::Nexus { opctx: &'a OpContext, webhook_selector: params::WebhookSelector, ) -> LookupResult> { - match webhook_selector.webhook { + match webhook_selector.receiver { NameOrId::Id(id) => { let webhook = LookupPath::new(opctx, &self.db_datastore) .webhook_receiver_id( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 3be42dc603b..998bb2a8621 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7453,7 +7453,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_secrets_list( rqctx: RequestContext, - _path_params: Path, + _query_params: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7477,16 +7477,17 @@ impl NexusExternalApi for NexusExternalApiImpl { /// Add a secret to a webhook. async fn webhook_secrets_add( rqctx: RequestContext, - path_params: Path, + query_params: Query, params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; - let params::WebhookSecretCreate { secret } = params.into_inner(); - let webhook_selector = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + + let params::WebhookSecretCreate { secret } = params.into_inner(); + let webhook_selector = query_params.into_inner(); let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; let secret = nexus.webhook_receiver_secret_add(&opctx, rx, secret).await?; @@ -7502,8 +7503,11 @@ impl NexusExternalApi for NexusExternalApiImpl { /// Delete a secret from a webhook receiver. async fn webhook_secrets_delete( rqctx: RequestContext, - _path_params: Path, + path_params: Path, ) -> Result { + use nexus_db_queries::db::lookup::LookupPath; + use omicron_uuid_kinds::WebhookSecretUuid; + let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7511,10 +7515,21 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let params::WebhookSecretSelector { secret_id } = + path_params.into_inner(); + + let (authz_rx, authz_secret) = + LookupPath::new(&opctx, nexus.datastore()) + .webhook_secret_id(WebhookSecretUuid::from_untyped_uuid( + secret_id, + )) + .lookup_for(authz::Action::Delete) + .await?; + nexus + .datastore() + .webhook_rx_secret_delete(&opctx, &authz_rx, &authz_secret) + .await?; + Ok(HttpResponseDeleted()) }; apictx .context @@ -7525,7 +7540,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_list( rqctx: RequestContext, - _path_params: Path, + _receiver: Query, _query_params: Query, ) -> Result>, HttpError> { @@ -7551,6 +7566,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_resend( rqctx: RequestContext, _path_params: Path, + _receiver: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 3bfccadae60..4a8d8948be9 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2392,7 +2392,7 @@ pub struct EventClassSelector { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookSelector { /// The name or ID of the webhook receiver. - pub webhook: NameOrId, + pub receiver: NameOrId, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -2436,18 +2436,12 @@ pub struct WebhookSecretCreate { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookSecretSelector { - /// Selects the webhook receiver - #[serde(flatten)] - pub webhook: WebhookSelector, /// ID of the secret. pub secret_id: Uuid, } #[derive(Deserialize, JsonSchema)] pub struct WebhookDeliveryPath { - /// Selects the webhook receiver - #[serde(flatten)] - pub webhook: WebhookSelector, pub event_id: Uuid, } diff --git a/openapi/nexus.json b/openapi/nexus.json index 3d0dd2b2e28..6ed8268facd 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -624,7 +624,122 @@ } } }, - "/experimental/v1/webhook-events/classes": { + "/experimental/v1/webhooks/deliveries": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List delivery attempts to a webhook receiver.", + "operationId": "webhook_delivery_list", + "parameters": [ + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/experimental/v1/webhooks/deliveries/{event_id}/resend": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Request re-delivery of a webhook event.", + "operationId": "webhook_delivery_resend", + "parameters": [ + { + "in": "path", + "name": "event_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryId" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/experimental/v1/webhooks/event-classes": { "get": { "tags": [ "system/webhooks" @@ -685,7 +800,7 @@ } } }, - "/experimental/v1/webhook-events/classes/{name}": { + "/experimental/v1/webhooks/event-classes/{name}": { "get": { "tags": [ "system/webhooks" @@ -723,7 +838,7 @@ } } }, - "/experimental/v1/webhooks": { + "/experimental/v1/webhooks/receivers": { "post": { "tags": [ "system/webhooks" @@ -760,7 +875,7 @@ } } }, - "/experimental/v1/webhooks/{webhook}": { + "/experimental/v1/webhooks/receivers/{receiver}": { "get": { "tags": [ "system/webhooks" @@ -770,7 +885,7 @@ "parameters": [ { "in": "path", - "name": "webhook", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -806,7 +921,7 @@ "parameters": [ { "in": "path", - "name": "webhook", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -845,7 +960,7 @@ "parameters": [ { "in": "path", - "name": "webhook", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -866,122 +981,7 @@ } } }, - "/experimental/v1/webhooks/{webhook}/deliveries": { - "get": { - "tags": [ - "system/webhooks" - ], - "summary": "List delivery attempts to a webhook receiver.", - "operationId": "webhook_delivery_list", - "parameters": [ - { - "in": "path", - "name": "webhook", - "description": "The name or ID of the webhook receiver.", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/IdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/WebhookDeliveryResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [] - } - } - }, - "/experimental/v1/webhooks/{webhook}/deliveries/{event_id}/resend": { - "post": { - "tags": [ - "system/webhooks" - ], - "summary": "Request re-delivery of a webhook event.", - "operationId": "webhook_delivery_resend", - "parameters": [ - { - "in": "path", - "name": "event_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "path", - "name": "webhook", - "description": "The name or ID of the webhook receiver.", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/WebhookDeliveryId" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/experimental/v1/webhooks/{webhook}/probe": { + "/experimental/v1/webhooks/receivers/{receiver}/probe": { "post": { "tags": [ "system/webhooks" @@ -991,7 +991,7 @@ "parameters": [ { "in": "path", - "name": "webhook", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -1027,7 +1027,7 @@ } } }, - "/experimental/v1/webhooks/{webhook}/secrets": { + "/experimental/v1/webhooks/secrets": { "get": { "tags": [ "system/webhooks" @@ -1036,8 +1036,8 @@ "operationId": "webhook_secrets_list", "parameters": [ { - "in": "path", - "name": "webhook", + "in": "query", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -1072,8 +1072,8 @@ "operationId": "webhook_secrets_add", "parameters": [ { - "in": "path", - "name": "webhook", + "in": "query", + "name": "receiver", "description": "The name or ID of the webhook receiver.", "required": true, "schema": { @@ -1111,7 +1111,7 @@ } } }, - "/experimental/v1/webhooks/{webhook}/secrets/{secret_id}": { + "/experimental/v1/webhooks/secrets/{secret_id}": { "delete": { "tags": [ "system/webhooks" @@ -1128,15 +1128,6 @@ "type": "string", "format": "uuid" } - }, - { - "in": "path", - "name": "webhook", - "description": "The name or ID of the webhook receiver.", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } } ], "responses": { @@ -25051,6 +25042,7 @@ "type": "object", "properties": { "description": { + "description": "human-readable free-form text about a resource", "type": "string" }, "endpoint": { @@ -25066,22 +25058,33 @@ } }, "id": { - "description": "The UUID of this webhook receiver.", + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", "allOf": [ { - "$ref": "#/components/schemas/TypedUuidForWebhookReceiverKind" + "$ref": "#/components/schemas/Name" } ] }, - "name": { - "description": "The identifier assigned to this webhook receiver upon creation.", - "type": "string" - }, "secrets": { "type": "array", "items": { "$ref": "#/components/schemas/WebhookSecretId" } + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" } }, "required": [ @@ -25090,7 +25093,9 @@ "events", "id", "name", - "secrets" + "secrets", + "time_created", + "time_modified" ] }, "WebhookCreate": { From 3730f161bd41a72e93f4844e64b113439abb9748 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 5 Mar 2025 15:24:26 -0800 Subject: [PATCH 105/168] properly fix tests --- nexus/tests/integration_tests/webhooks.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 45631b61974..766f19774cc 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -28,7 +28,8 @@ use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; -const WEBHOOKS_BASE_PATH: &str = "/experimental/v1/webhooks"; +const RECEIVERS_BASE_PATH: &str = "/experimental/v1/webhooks/receivers"; +const SECRETS_BASE_PATH: &str = "/experimental/v1/webhooks/secrets"; async fn webhook_create( ctx: &ControlPlaneTestContext, @@ -36,7 +37,7 @@ async fn webhook_create( ) -> views::Webhook { resource_helpers::object_create::( &ctx.external_client, - WEBHOOKS_BASE_PATH, + RECEIVERS_BASE_PATH, params, ) .await @@ -44,7 +45,7 @@ async fn webhook_create( fn get_webhooks_url(name_or_id: impl Into) -> String { let name_or_id = name_or_id.into(); - format!("{WEBHOOKS_BASE_PATH}/{name_or_id}") + format!("{RECEIVERS_BASE_PATH}/{name_or_id}") } async fn webhook_get( @@ -97,7 +98,7 @@ async fn secret_add( views::WebhookSecretId, >( &ctx.external_client, - &format!("{WEBHOOKS_BASE_PATH}/{webhook_id}/secrets"), + &format!("{SECRETS_BASE_PATH}/?receiver={webhook_id}"), params, ) .await @@ -110,7 +111,7 @@ async fn webhook_send_probe( status: http::StatusCode, ) -> views::WebhookProbeResult { let pathparams = if resend { "?resend=true" } else { "" }; - let path = format!("{WEBHOOKS_BASE_PATH}/{webhook_id}/probe{pathparams}"); + let path = format!("{RECEIVERS_BASE_PATH}/{webhook_id}/probe{pathparams}"); NexusRequest::new( RequestBuilder::new(&ctx.external_client, http::Method::POST, &path) .expect_status(Some(status)), From 46a73f24827181963bcb255d74a6b33ec3d4ff91 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 5 Mar 2025 15:48:37 -0800 Subject: [PATCH 106/168] rm experimental --- nexus/external-api/output/nexus_tags.txt | 24 +- nexus/external-api/src/lib.rs | 26 +- nexus/tests/integration_tests/webhooks.rs | 4 +- openapi/nexus.json | 3296 ++++++++++----------- 4 files changed, 1675 insertions(+), 1675 deletions(-) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index b9a9dea1bcb..8d829f9c373 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -266,18 +266,18 @@ ping GET /v1/ping API operations found with tag "system/webhooks" OPERATION ID METHOD URL PATH -webhook_create POST /experimental/v1/webhooks/receivers -webhook_delete DELETE /experimental/v1/webhooks/receivers/{receiver} -webhook_delivery_list GET /experimental/v1/webhooks/deliveries -webhook_delivery_resend POST /experimental/v1/webhooks/deliveries/{event_id}/resend -webhook_event_class_list GET /experimental/v1/webhooks/event-classes -webhook_event_class_view GET /experimental/v1/webhooks/event-classes/{name} -webhook_probe POST /experimental/v1/webhooks/receivers/{receiver}/probe -webhook_secrets_add POST /experimental/v1/webhooks/secrets -webhook_secrets_delete DELETE /experimental/v1/webhooks/secrets/{secret_id} -webhook_secrets_list GET /experimental/v1/webhooks/secrets -webhook_update PUT /experimental/v1/webhooks/receivers/{receiver} -webhook_view GET /experimental/v1/webhooks/receivers/{receiver} +webhook_create POST /v1/webhooks/receivers +webhook_delete DELETE /v1/webhooks/receivers/{receiver} +webhook_delivery_list GET /v1/webhooks/deliveries +webhook_delivery_resend POST /v1/webhooks/deliveries/{event_id}/resend +webhook_event_class_list GET /v1/webhooks/event-classes +webhook_event_class_view GET /v1/webhooks/event-classes/{name} +webhook_probe POST /v1/webhooks/receivers/{receiver}/probe +webhook_secrets_add POST /v1/webhooks/secrets +webhook_secrets_delete DELETE /v1/webhooks/secrets/{secret_id} +webhook_secrets_list GET /v1/webhooks/secrets +webhook_update PUT /v1/webhooks/receivers/{receiver} +webhook_view GET /v1/webhooks/receivers/{receiver} API operations found with tag "vpcs" OPERATION ID METHOD URL PATH diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 05fff550248..69e217b7426 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3474,12 +3474,12 @@ pub trait NexusExternalApi { params: TypedBody, ) -> Result, HttpError>; - // Webhooks (experimental) + // Webhooks /// List webhook event classes #[endpoint { method = GET, - path = "/experimental/v1/webhooks/event-classes", + path = "/v1/webhooks/event-classes", tags = ["system/webhooks"], }] async fn webhook_event_class_list( @@ -3492,7 +3492,7 @@ pub trait NexusExternalApi { /// Fetch details on an event class by name. #[endpoint { method = GET, - path ="/experimental/v1/webhooks/event-classes/{name}", + path ="/v1/webhooks/event-classes/{name}", tags = ["system/webhooks"], }] async fn webhook_event_class_view( @@ -3503,7 +3503,7 @@ pub trait NexusExternalApi { /// Get the configuration for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/receivers/{receiver}", + path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_view( @@ -3514,7 +3514,7 @@ pub trait NexusExternalApi { /// Create a new webhook receiver. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/receivers", + path = "/v1/webhooks/receivers", tags = ["system/webhooks"], }] async fn webhook_create( @@ -3525,7 +3525,7 @@ pub trait NexusExternalApi { /// Update the configuration of an existing webhook receiver. #[endpoint { method = PUT, - path = "/experimental/v1/webhooks/receivers/{receiver}", + path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_update( @@ -3537,7 +3537,7 @@ pub trait NexusExternalApi { /// Delete a webhook receiver. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/receivers/{receiver}", + path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] async fn webhook_delete( @@ -3551,7 +3551,7 @@ pub trait NexusExternalApi { // status code from the webhook endpoint... #[endpoint { method = POST, - path = "/experimental/v1/webhooks/receivers/{receiver}/probe", + path = "/v1/webhooks/receivers/{receiver}/probe", tags = ["system/webhooks"], }] async fn webhook_probe( @@ -3563,7 +3563,7 @@ pub trait NexusExternalApi { /// List the IDs of secrets for a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/secrets", + path = "/v1/webhooks/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_list( @@ -3574,7 +3574,7 @@ pub trait NexusExternalApi { /// Add a secret to a webhook receiver. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/secrets", + path = "/v1/webhooks/secrets", tags = ["system/webhooks"], }] async fn webhook_secrets_add( @@ -3586,7 +3586,7 @@ pub trait NexusExternalApi { /// Delete a secret associated with a webhook receiver by ID. #[endpoint { method = DELETE, - path = "/experimental/v1/webhooks/secrets/{secret_id}", + path = "/v1/webhooks/secrets/{secret_id}", tags = ["system/webhooks"], }] async fn webhook_secrets_delete( @@ -3597,7 +3597,7 @@ pub trait NexusExternalApi { /// List delivery attempts to a webhook receiver. #[endpoint { method = GET, - path = "/experimental/v1/webhooks/deliveries", + path = "/v1/webhooks/deliveries", tags = ["system/webhooks"], }] async fn webhook_delivery_list( @@ -3609,7 +3609,7 @@ pub trait NexusExternalApi { /// Request re-delivery of a webhook event. #[endpoint { method = POST, - path = "/experimental/v1/webhooks/deliveries/{event_id}/resend", + path = "/v1/webhooks/deliveries/{event_id}/resend", tags = ["system/webhooks"], }] async fn webhook_delivery_resend( diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 766f19774cc..df21c596498 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -28,8 +28,8 @@ use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; -const RECEIVERS_BASE_PATH: &str = "/experimental/v1/webhooks/receivers"; -const SECRETS_BASE_PATH: &str = "/experimental/v1/webhooks/secrets"; +const RECEIVERS_BASE_PATH: &str = "/v1/webhooks/receivers"; +const SECRETS_BASE_PATH: &str = "/v1/webhooks/secrets"; async fn webhook_create( ctx: &ControlPlaneTestContext, diff --git a/openapi/nexus.json b/openapi/nexus.json index 6ed8268facd..d0a6c8784bb 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -624,108 +624,52 @@ } } }, - "/experimental/v1/webhooks/deliveries": { - "get": { + "/login/{silo_name}/saml/{provider_name}": { + "post": { "tags": [ - "system/webhooks" + "login" ], - "summary": "List delivery attempts to a webhook receiver.", - "operationId": "webhook_delivery_list", + "summary": "Authenticate a user via SAML", + "operationId": "login_saml", "parameters": [ { - "in": "query", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", + "in": "path", + "name": "provider_name", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/Name" } }, { - "in": "query", - "name": "sort_by", + "in": "path", + "name": "silo_name", + "required": true, "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/Name" } } ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/WebhookDeliveryResultsPage" - } + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" } } }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } + "required": true }, - "x-dropshot-pagination": { - "required": [] - } - } - }, - "/experimental/v1/webhooks/deliveries/{event_id}/resend": { - "post": { - "tags": [ - "system/webhooks" - ], - "summary": "Request re-delivery of a webhook event.", - "operationId": "webhook_delivery_resend", - "parameters": [ - { - "in": "path", - "name": "event_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "query", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { + "303": { + "description": "redirect (see other)", + "headers": { + "location": { + "description": "HTTP \"Location\" header", + "style": "simple", + "required": true, "schema": { - "$ref": "#/components/schemas/WebhookDeliveryId" + "type": "string" } } } @@ -739,23 +683,14 @@ } } }, - "/experimental/v1/webhooks/event-classes": { + "/v1/affinity-groups": { "get": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "List webhook event classes", - "operationId": "webhook_event_class_list", + "summary": "List affinity groups", + "operationId": "affinity_group_list", "parameters": [ - { - "in": "query", - "name": "filter", - "description": "An optional glob pattern for filtering event class names.", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "limit", @@ -775,6 +710,21 @@ "nullable": true, "type": "string" } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } } ], "responses": { @@ -783,7 +733,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EventClassResultsPage" + "$ref": "#/components/schemas/AffinityGroupResultsPage" } } } @@ -796,60 +746,33 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "project" + ] } - } - }, - "/experimental/v1/webhooks/event-classes/{name}": { - "get": { + }, + "post": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Fetch details on an event class by name.", - "operationId": "webhook_event_class_view", + "summary": "Create an affinity group", + "operationId": "affinity_group_create", "parameters": [ { - "in": "path", - "name": "name", - "description": "The name of the event class.", + "in": "query", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EventClass" - } - } + "$ref": "#/components/schemas/NameOrId" } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" } - } - } - }, - "/experimental/v1/webhooks/receivers": { - "post": { - "tags": [ - "system/webhooks" ], - "summary": "Create a new webhook receiver.", - "operationId": "webhook_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookCreate" + "$ref": "#/components/schemas/AffinityGroupCreate" } } }, @@ -861,7 +784,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Webhook" + "$ref": "#/components/schemas/AffinityGroup" } } } @@ -875,18 +798,26 @@ } } }, - "/experimental/v1/webhooks/receivers/{receiver}": { + "/v1/affinity-groups/{affinity_group}": { "get": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Get the configuration for a webhook receiver.", - "operationId": "webhook_view", + "summary": "Fetch an affinity group", + "operationId": "affinity_group_view", "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "path", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", + "name": "affinity_group", + "description": "Name or ID of the affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -899,7 +830,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Webhook" + "$ref": "#/components/schemas/AffinityGroup" } } } @@ -914,15 +845,23 @@ }, "put": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Update the configuration of an existing webhook receiver.", - "operationId": "webhook_update", + "summary": "Update an affinity group", + "operationId": "affinity_group_update", "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "path", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", + "name": "affinity_group", + "description": "Name or ID of the affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -933,15 +872,22 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookUpdate" + "$ref": "#/components/schemas/AffinityGroupUpdate" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -953,16 +899,24 @@ }, "delete": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Delete a webhook receiver.", - "operationId": "webhook_delete", + "summary": "Delete an affinity group", + "operationId": "affinity_group_delete", "parameters": [ { - "in": "path", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -981,29 +935,56 @@ } } }, - "/experimental/v1/webhooks/receivers/{receiver}/probe": { - "post": { + "/v1/affinity-groups/{affinity_group}/members": { + "get": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Send a liveness probe request to a webhook receiver.", - "operationId": "webhook_probe", + "summary": "List members of an affinity group", + "operationId": "affinity_group_member_list", "parameters": [ { - "in": "path", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "resend", - "description": "If true, resend all events that have not been delivered successfully if the probe request succeeds.", + "name": "sort_by", "schema": { - "type": "boolean" + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" } } ], @@ -1013,7 +994,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookProbeResult" + "$ref": "#/components/schemas/AffinityGroupMemberResultsPage" } } } @@ -1024,21 +1005,39 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/experimental/v1/webhooks/secrets": { + "/v1/affinity-groups/{affinity_group}/members/instance/{instance}": { "get": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "List the IDs of secrets for a webhook receiver.", - "operationId": "webhook_secrets_list", + "summary": "Fetch an affinity group member", + "operationId": "affinity_group_member_instance_view", "parameters": [ { "in": "query", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1051,7 +1050,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookSecrets" + "$ref": "#/components/schemas/AffinityGroupMember" } } } @@ -1066,38 +1065,43 @@ }, "post": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Add a secret to a webhook receiver.", - "operationId": "webhook_secrets_add", + "summary": "Add a member to an affinity group", + "operationId": "affinity_group_member_instance_add", "parameters": [ { "in": "query", - "name": "receiver", - "description": "The name or ID of the webhook receiver.", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/WebhookSecretCreate" - } - } - }, - "required": true - }, "responses": { "201": { "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookSecretId" + "$ref": "#/components/schemas/AffinityGroupMember" } } } @@ -1109,89 +1113,42 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/experimental/v1/webhooks/secrets/{secret_id}": { + }, "delete": { "tags": [ - "system/webhooks" + "affinity" ], - "summary": "Delete a secret associated with a webhook receiver by ID.", - "operationId": "webhook_secrets_delete", + "summary": "Remove a member from an affinity group", + "operationId": "affinity_group_member_instance_delete", "parameters": [ { - "in": "path", - "name": "secret_id", - "description": "ID of the secret.", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } - } - ], - "responses": { - "204": { - "description": "successful deletion" }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/login/{silo_name}/saml/{provider_name}": { - "post": { - "tags": [ - "login" - ], - "summary": "Authenticate a user via SAML", - "operationId": "login_saml", - "parameters": [ { "in": "path", - "name": "provider_name", + "name": "affinity_group", "required": true, "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "path", - "name": "silo_name", + "name": "instance", "required": true, "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/octet-stream": { - "schema": { - "type": "string", - "format": "binary" - } - } - }, - "required": true - }, "responses": { - "303": { - "description": "redirect (see other)", - "headers": { - "location": { - "description": "HTTP \"Location\" header", - "style": "simple", - "required": true, - "schema": { - "type": "string" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -1202,13 +1159,13 @@ } } }, - "/v1/affinity-groups": { + "/v1/anti-affinity-groups": { "get": { "tags": [ "affinity" ], - "summary": "List affinity groups", - "operationId": "affinity_group_list", + "summary": "List anti-affinity groups", + "operationId": "anti_affinity_group_list", "parameters": [ { "in": "query", @@ -1252,7 +1209,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupResultsPage" + "$ref": "#/components/schemas/AntiAffinityGroupResultsPage" } } } @@ -1274,8 +1231,8 @@ "tags": [ "affinity" ], - "summary": "Create an affinity group", - "operationId": "affinity_group_create", + "summary": "Create an anti-affinity group", + "operationId": "anti_affinity_group_create", "parameters": [ { "in": "query", @@ -1291,7 +1248,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupCreate" + "$ref": "#/components/schemas/AntiAffinityGroupCreate" } } }, @@ -1303,7 +1260,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroup" + "$ref": "#/components/schemas/AntiAffinityGroup" } } } @@ -1317,13 +1274,13 @@ } } }, - "/v1/affinity-groups/{affinity_group}": { + "/v1/anti-affinity-groups/{anti_affinity_group}": { "get": { "tags": [ "affinity" ], - "summary": "Fetch an affinity group", - "operationId": "affinity_group_view", + "summary": "Fetch an anti-affinity group", + "operationId": "anti_affinity_group_view", "parameters": [ { "in": "query", @@ -1335,8 +1292,8 @@ }, { "in": "path", - "name": "affinity_group", - "description": "Name or ID of the affinity group", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1349,7 +1306,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroup" + "$ref": "#/components/schemas/AntiAffinityGroup" } } } @@ -1366,8 +1323,8 @@ "tags": [ "affinity" ], - "summary": "Update an affinity group", - "operationId": "affinity_group_update", + "summary": "Update an anti-affinity group", + "operationId": "anti_affinity_group_update", "parameters": [ { "in": "query", @@ -1379,8 +1336,8 @@ }, { "in": "path", - "name": "affinity_group", - "description": "Name or ID of the affinity group", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1391,7 +1348,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupUpdate" + "$ref": "#/components/schemas/AntiAffinityGroupUpdate" } } }, @@ -1403,7 +1360,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroup" + "$ref": "#/components/schemas/AntiAffinityGroup" } } } @@ -1420,8 +1377,8 @@ "tags": [ "affinity" ], - "summary": "Delete an affinity group", - "operationId": "affinity_group_delete", + "summary": "Delete an anti-affinity group", + "operationId": "anti_affinity_group_delete", "parameters": [ { "in": "query", @@ -1433,8 +1390,8 @@ }, { "in": "path", - "name": "affinity_group", - "description": "Name or ID of the affinity group", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1454,13 +1411,13 @@ } } }, - "/v1/affinity-groups/{affinity_group}/members": { + "/v1/anti-affinity-groups/{anti_affinity_group}/members": { "get": { "tags": [ "affinity" ], - "summary": "List members of an affinity group", - "operationId": "affinity_group_member_list", + "summary": "List members of an anti-affinity group", + "operationId": "anti_affinity_group_member_list", "parameters": [ { "in": "query", @@ -1499,8 +1456,8 @@ }, { "in": "path", - "name": "affinity_group", - "description": "Name or ID of the affinity group", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1513,7 +1470,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupMemberResultsPage" + "$ref": "#/components/schemas/AntiAffinityGroupMemberResultsPage" } } } @@ -1530,13 +1487,13 @@ } } }, - "/v1/affinity-groups/{affinity_group}/members/instance/{instance}": { + "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { "get": { "tags": [ "affinity" ], - "summary": "Fetch an affinity group member", - "operationId": "affinity_group_member_instance_view", + "summary": "Fetch an anti-affinity group member", + "operationId": "anti_affinity_group_member_instance_view", "parameters": [ { "in": "query", @@ -1548,7 +1505,7 @@ }, { "in": "path", - "name": "affinity_group", + "name": "anti_affinity_group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1569,7 +1526,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupMember" + "$ref": "#/components/schemas/AntiAffinityGroupMember" } } } @@ -1586,8 +1543,8 @@ "tags": [ "affinity" ], - "summary": "Add a member to an affinity group", - "operationId": "affinity_group_member_instance_add", + "summary": "Add a member to an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_add", "parameters": [ { "in": "query", @@ -1599,7 +1556,7 @@ }, { "in": "path", - "name": "affinity_group", + "name": "anti_affinity_group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1620,7 +1577,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AffinityGroupMember" + "$ref": "#/components/schemas/AntiAffinityGroupMember" } } } @@ -1637,8 +1594,8 @@ "tags": [ "affinity" ], - "summary": "Remove a member from an affinity group", - "operationId": "affinity_group_member_instance_delete", + "summary": "Remove a member from an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_delete", "parameters": [ { "in": "query", @@ -1650,7 +1607,7 @@ }, { "in": "path", - "name": "affinity_group", + "name": "anti_affinity_group", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1678,13 +1635,14 @@ } } }, - "/v1/anti-affinity-groups": { + "/v1/certificates": { "get": { "tags": [ - "affinity" + "silos" ], - "summary": "List anti-affinity groups", - "operationId": "anti_affinity_group_list", + "summary": "List certificates for external endpoints", + "description": "Returns a list of TLS certificates used for the external API (for the current Silo). These are sorted by creation date, with the most recent certificates appearing first.", + "operationId": "certificate_list", "parameters": [ { "in": "query", @@ -1706,14 +1664,6 @@ "type": "string" } }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", @@ -1728,7 +1678,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupResultsPage" + "$ref": "#/components/schemas/CertificateResultsPage" } } } @@ -1741,33 +1691,21 @@ } }, "x-dropshot-pagination": { - "required": [ - "project" - ] + "required": [] } }, "post": { "tags": [ - "affinity" - ], - "summary": "Create an anti-affinity group", - "operationId": "anti_affinity_group_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } + "silos" ], + "summary": "Create new system-wide x.509 certificate", + "description": "This certificate is automatically used by the Oxide Control plane to serve external connections.", + "operationId": "certificate_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupCreate" + "$ref": "#/components/schemas/CertificateCreate" } } }, @@ -1779,7 +1717,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroup" + "$ref": "#/components/schemas/Certificate" } } } @@ -1793,93 +1731,32 @@ } } }, - "/v1/anti-affinity-groups/{anti_affinity_group}": { + "/v1/certificates/{certificate}": { "get": { "tags": [ - "affinity" - ], - "summary": "Fetch an anti-affinity group", - "operationId": "anti_affinity_group_view", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "description": "Name or ID of the anti affinity group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AntiAffinityGroup" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "affinity" + "silos" ], - "summary": "Update an anti-affinity group", - "operationId": "anti_affinity_group_update", + "summary": "Fetch certificate", + "description": "Returns the details of a specific certificate", + "operationId": "certificate_view", "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "path", - "name": "anti_affinity_group", - "description": "Name or ID of the anti affinity group", + "name": "certificate", + "description": "Name or ID of the certificate", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupUpdate" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroup" + "$ref": "#/components/schemas/Certificate" } } } @@ -1894,23 +1771,16 @@ }, "delete": { "tags": [ - "affinity" + "silos" ], - "summary": "Delete an anti-affinity group", - "operationId": "anti_affinity_group_delete", + "summary": "Delete certificate", + "description": "Permanently delete a certificate. This operation cannot be undone.", + "operationId": "certificate_delete", "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "path", - "name": "anti_affinity_group", - "description": "Name or ID of the anti affinity group", + "name": "certificate", + "description": "Name or ID of the certificate", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -1930,13 +1800,13 @@ } } }, - "/v1/anti-affinity-groups/{anti_affinity_group}/members": { + "/v1/disks": { "get": { "tags": [ - "affinity" + "disks" ], - "summary": "List members of an anti-affinity group", - "operationId": "anti_affinity_group_member_list", + "summary": "List disks", + "operationId": "disk_list", "parameters": [ { "in": "query", @@ -1970,16 +1840,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "description": "Name or ID of the anti affinity group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -1989,7 +1850,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupMemberResultsPage" + "$ref": "#/components/schemas/DiskResultsPage" } } } @@ -2002,50 +1863,45 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "project" + ] } - } - }, - "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { - "get": { + }, + "post": { "tags": [ - "affinity" + "disks" ], - "summary": "Fetch an anti-affinity group member", - "operationId": "anti_affinity_group_member_instance_view", + "summary": "Create a disk", + "operationId": "disk_create", "parameters": [ { "in": "query", "name": "project", "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskCreate" + } + } + }, + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupMember" + "$ref": "#/components/schemas/Disk" } } } @@ -2057,46 +1913,41 @@ "$ref": "#/components/responses/Error" } } - }, - "post": { + } + }, + "/v1/disks/{disk}": { + "get": { "tags": [ - "affinity" + "disks" ], - "summary": "Add a member to an anti-affinity group", - "operationId": "anti_affinity_group_member_instance_add", + "summary": "Fetch disk", + "operationId": "disk_view", "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "path", - "name": "anti_affinity_group", + "name": "disk", + "description": "Name or ID of the disk", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "path", - "name": "instance", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupMember" + "$ref": "#/components/schemas/Disk" } } } @@ -2111,31 +1962,24 @@ }, "delete": { "tags": [ - "affinity" + "disks" ], - "summary": "Remove a member from an anti-affinity group", - "operationId": "anti_affinity_group_member_instance_delete", + "summary": "Delete disk", + "operationId": "disk_delete", "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "path", - "name": "anti_affinity_group", + "name": "disk", + "description": "Name or ID of the disk", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "path", - "name": "instance", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -2154,92 +1998,45 @@ } } }, - "/v1/certificates": { - "get": { + "/v1/disks/{disk}/bulk-write": { + "post": { "tags": [ - "silos" + "disks" ], - "summary": "List certificates for external endpoints", - "description": "Returns a list of TLS certificates used for the external API (for the current Silo). These are sorted by creation date, with the most recent certificates appearing first.", - "operationId": "certificate_list", + "summary": "Import blocks into disk", + "operationId": "disk_bulk_write_import", "parameters": [ { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "in": "path", + "name": "disk", + "description": "Name or ID of the disk", + "required": true, "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", + "name": "project", + "description": "Name or ID of the project", "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CertificateResultsPage" - } - } + "$ref": "#/components/schemas/NameOrId" } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] - } - }, - "post": { - "tags": [ - "silos" ], - "summary": "Create new system-wide x.509 certificate", - "description": "This certificate is automatically used by the Oxide Control plane to serve external connections.", - "operationId": "certificate_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CertificateCreate" + "$ref": "#/components/schemas/ImportBlocksBulkWrite" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Certificate" - } - } - } + "204": { + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2250,57 +2047,28 @@ } } }, - "/v1/certificates/{certificate}": { - "get": { + "/v1/disks/{disk}/bulk-write-start": { + "post": { "tags": [ - "silos" + "disks" ], - "summary": "Fetch certificate", - "description": "Returns the details of a specific certificate", - "operationId": "certificate_view", + "summary": "Start importing blocks into disk", + "description": "Start the process of importing blocks into a disk", + "operationId": "disk_bulk_write_import_start", "parameters": [ { "in": "path", - "name": "certificate", - "description": "Name or ID of the certificate", + "name": "disk", + "description": "Name or ID of the disk", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Certificate" - } - } - } }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "silos" - ], - "summary": "Delete certificate", - "description": "Permanently delete a certificate. This operation cannot be undone.", - "operationId": "certificate_delete", - "parameters": [ { - "in": "path", - "name": "certificate", - "description": "Name or ID of the certificate", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -2308,7 +2076,7 @@ ], "responses": { "204": { - "description": "successful deletion" + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2319,32 +2087,22 @@ } } }, - "/v1/disks": { - "get": { + "/v1/disks/{disk}/bulk-write-stop": { + "post": { "tags": [ "disks" ], - "summary": "List disks", - "operationId": "disk_list", + "summary": "Stop importing blocks into disk", + "description": "Stop the process of importing blocks into a disk", + "operationId": "disk_bulk_write_import_stop", "parameters": [ { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "in": "path", + "name": "disk", + "description": "Name or ID of the disk", + "required": true, "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } }, { @@ -2354,25 +2112,11 @@ "schema": { "$ref": "#/components/schemas/NameOrId" } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DiskResultsPage" - } - } - } + "204": { + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2380,25 +2124,30 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] } - }, + } + }, + "/v1/disks/{disk}/finalize": { "post": { "tags": [ "disks" ], - "summary": "Create a disk", - "operationId": "disk_create", + "summary": "Confirm disk block import completion", + "operationId": "disk_finalize_import", "parameters": [ + { + "in": "path", + "name": "disk", + "description": "Name or ID of the disk", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "project", "description": "Name or ID of the project", - "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -2408,22 +2157,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DiskCreate" + "$ref": "#/components/schemas/FinalizeDisk" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Disk" - } - } - } + "204": { + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2434,23 +2176,76 @@ } } }, - "/v1/disks/{disk}": { + "/v1/disks/{disk}/metrics/{metric}": { "get": { "tags": [ "disks" ], - "summary": "Fetch disk", - "operationId": "disk_view", + "summary": "Fetch disk metrics", + "operationId": "disk_metrics_list", "parameters": [ { "in": "path", "name": "disk", - "description": "Name or ID of the disk", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "path", + "name": "metric", + "required": true, + "schema": { + "$ref": "#/components/schemas/DiskMetricName" + } + }, + { + "in": "query", + "name": "end_time", + "description": "An exclusive end time of metrics.", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "order", + "description": "Query result order", + "schema": { + "$ref": "#/components/schemas/PaginationOrder" + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "start_time", + "description": "An inclusive start time of metrics.", + "schema": { + "type": "string", + "format": "date-time" + } + }, { "in": "query", "name": "project", @@ -2466,7 +2261,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Disk" + "$ref": "#/components/schemas/MeasurementResultsPage" } } } @@ -2477,22 +2272,41 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [ + "end_time", + "start_time" + ] } - }, - "delete": { + } + }, + "/v1/floating-ips": { + "get": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Delete disk", - "operationId": "disk_delete", + "summary": "List floating IPs", + "operationId": "floating_ip_list", "parameters": [ { - "in": "path", - "name": "disk", - "description": "Name or ID of the disk", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" } }, { @@ -2502,11 +2316,25 @@ "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } } ], "responses": { - "204": { - "description": "successful deletion" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIpResultsPage" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -2514,30 +2342,25 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] } - } - }, - "/v1/disks/{disk}/bulk-write": { + }, "post": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Import blocks into disk", - "operationId": "disk_bulk_write_import", + "summary": "Create floating IP", + "operationId": "floating_ip_create", "parameters": [ - { - "in": "path", - "name": "disk", - "description": "Name or ID of the disk", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "project", "description": "Name or ID of the project", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -2547,15 +2370,22 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImportBlocksBulkWrite" + "$ref": "#/components/schemas/FloatingIpCreate" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -2566,19 +2396,18 @@ } } }, - "/v1/disks/{disk}/bulk-write-start": { - "post": { + "/v1/floating-ips/{floating_ip}": { + "get": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Start importing blocks into disk", - "description": "Start the process of importing blocks into a disk", - "operationId": "disk_bulk_write_import_start", + "summary": "Fetch floating IP", + "operationId": "floating_ip_view", "parameters": [ { "in": "path", - "name": "disk", - "description": "Name or ID of the disk", + "name": "floating_ip", + "description": "Name or ID of the floating IP", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -2594,8 +2423,15 @@ } ], "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -2604,21 +2440,18 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/disks/{disk}/bulk-write-stop": { - "post": { + }, + "put": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Stop importing blocks into disk", - "description": "Stop the process of importing blocks into a disk", - "operationId": "disk_bulk_write_import_stop", + "summary": "Update floating IP", + "operationId": "floating_ip_update", "parameters": [ { "in": "path", - "name": "disk", - "description": "Name or ID of the disk", + "name": "floating_ip", + "description": "Name or ID of the floating IP", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -2633,9 +2466,26 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIpUpdate" + } + } + }, + "required": true + }, "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -2644,20 +2494,18 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/disks/{disk}/finalize": { - "post": { + }, + "delete": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Confirm disk block import completion", - "operationId": "disk_finalize_import", + "summary": "Delete floating IP", + "operationId": "floating_ip_delete", "parameters": [ { "in": "path", - "name": "disk", - "description": "Name or ID of the disk", + "name": "floating_ip", + "description": "Name or ID of the floating IP", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -2672,19 +2520,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FinalizeDisk" - } - } - }, - "required": true - }, "responses": { "204": { - "description": "resource updated" + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2695,39 +2533,117 @@ } } }, - "/v1/disks/{disk}/metrics/{metric}": { - "get": { + "/v1/floating-ips/{floating_ip}/attach": { + "post": { "tags": [ - "disks" + "floating-ips" ], - "summary": "Fetch disk metrics", - "operationId": "disk_metrics_list", + "summary": "Attach floating IP", + "description": "Attach floating IP to an instance or other resource.", + "operationId": "floating_ip_attach", "parameters": [ { "in": "path", - "name": "disk", + "name": "floating_ip", + "description": "Name or ID of the floating IP", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIpAttach" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "successfully enqueued operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/floating-ips/{floating_ip}/detach": { + "post": { + "tags": [ + "floating-ips" + ], + "summary": "Detach floating IP", + "operationId": "floating_ip_detach", + "parameters": [ { "in": "path", - "name": "metric", + "name": "floating_ip", + "description": "Name or ID of the floating IP", "required": true, "schema": { - "$ref": "#/components/schemas/DiskMetricName" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "end_time", - "description": "An exclusive end time of metrics.", + "name": "project", + "description": "Name or ID of the project", "schema": { - "type": "string", - "format": "date-time" + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "202": { + "description": "successfully enqueued operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } } }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/groups": { + "get": { + "tags": [ + "silos" + ], + "summary": "List groups", + "operationId": "group_list", + "parameters": [ { "in": "query", "name": "limit", @@ -2739,14 +2655,6 @@ "minimum": 1 } }, - { - "in": "query", - "name": "order", - "description": "Query result order", - "schema": { - "$ref": "#/components/schemas/PaginationOrder" - } - }, { "in": "query", "name": "page_token", @@ -2758,19 +2666,51 @@ }, { "in": "query", - "name": "start_time", - "description": "An inclusive start time of metrics.", + "name": "sort_by", "schema": { - "type": "string", - "format": "date-time" + "$ref": "#/components/schemas/IdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupResultsPage" + } + } } }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/groups/{group_id}": { + "get": { + "tags": [ + "silos" + ], + "summary": "Fetch group", + "operationId": "group_view", + "parameters": [ { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "in": "path", + "name": "group_id", + "description": "ID of the group", + "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "uuid" } } ], @@ -2780,7 +2720,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/MeasurementResultsPage" + "$ref": "#/components/schemas/Group" } } } @@ -2791,22 +2731,17 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "end_time", - "start_time" - ] } } }, - "/v1/floating-ips": { + "/v1/images": { "get": { "tags": [ - "floating-ips" + "images" ], - "summary": "List floating IPs", - "operationId": "floating_ip_list", + "summary": "List images", + "description": "List images which are global or scoped to the specified project. The images are returned sorted by creation date, with the most recent images appearing first.", + "operationId": "image_list", "parameters": [ { "in": "query", @@ -2850,7 +2785,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIpResultsPage" + "$ref": "#/components/schemas/ImageResultsPage" } } } @@ -2863,23 +2798,21 @@ } }, "x-dropshot-pagination": { - "required": [ - "project" - ] + "required": [] } }, "post": { "tags": [ - "floating-ips" + "images" ], - "summary": "Create floating IP", - "operationId": "floating_ip_create", + "summary": "Create image", + "description": "Create a new image in a project.", + "operationId": "image_create", "parameters": [ { "in": "query", "name": "project", "description": "Name or ID of the project", - "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -2889,7 +2822,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIpCreate" + "$ref": "#/components/schemas/ImageCreate" } } }, @@ -2901,7 +2834,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIp" + "$ref": "#/components/schemas/Image" } } } @@ -2915,18 +2848,19 @@ } } }, - "/v1/floating-ips/{floating_ip}": { + "/v1/images/{image}": { "get": { "tags": [ - "floating-ips" + "images" ], - "summary": "Fetch floating IP", - "operationId": "floating_ip_view", + "summary": "Fetch image", + "description": "Fetch the details for a specific image in a project.", + "operationId": "image_view", "parameters": [ { "in": "path", - "name": "floating_ip", - "description": "Name or ID of the floating IP", + "name": "image", + "description": "Name or ID of the image", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -2947,7 +2881,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIp" + "$ref": "#/components/schemas/Image" } } } @@ -2960,17 +2894,18 @@ } } }, - "put": { + "delete": { "tags": [ - "floating-ips" + "images" ], - "summary": "Update floating IP", - "operationId": "floating_ip_update", + "summary": "Delete image", + "description": "Permanently delete an image from a project. This operation cannot be undone. Any instances in the project using the image will continue to run, however new instances can not be created with this image.", + "operationId": "image_delete", "parameters": [ { "in": "path", - "name": "floating_ip", - "description": "Name or ID of the floating IP", + "name": "image", + "description": "Name or ID of the image", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -2985,26 +2920,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FloatingIpUpdate" - } - } - }, - "required": true - }, "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FloatingIp" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -3013,18 +2931,21 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/images/{image}/demote": { + "post": { "tags": [ - "floating-ips" + "images" ], - "summary": "Delete floating IP", - "operationId": "floating_ip_delete", + "summary": "Demote silo image", + "description": "Demote silo image to be visible only to a specified project", + "operationId": "image_demote", "parameters": [ { "in": "path", - "name": "floating_ip", - "description": "Name or ID of the floating IP", + "name": "image", + "description": "Name or ID of the image", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3034,68 +2955,19 @@ "in": "query", "name": "project", "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/floating-ips/{floating_ip}/attach": { - "post": { - "tags": [ - "floating-ips" - ], - "summary": "Attach floating IP", - "description": "Attach floating IP to an instance or other resource.", - "operationId": "floating_ip_attach", - "parameters": [ - { - "in": "path", - "name": "floating_ip", - "description": "Name or ID of the floating IP", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FloatingIpAttach" - } - } - }, - "required": true - }, "responses": { "202": { "description": "successfully enqueued operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIp" + "$ref": "#/components/schemas/Image" } } } @@ -3109,18 +2981,19 @@ } } }, - "/v1/floating-ips/{floating_ip}/detach": { + "/v1/images/{image}/promote": { "post": { "tags": [ - "floating-ips" + "images" ], - "summary": "Detach floating IP", - "operationId": "floating_ip_detach", + "summary": "Promote project image", + "description": "Promote project image to be visible to all projects in the silo", + "operationId": "image_promote", "parameters": [ { "in": "path", - "name": "floating_ip", - "description": "Name or ID of the floating IP", + "name": "image", + "description": "Name or ID of the image", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3141,7 +3014,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FloatingIp" + "$ref": "#/components/schemas/Image" } } } @@ -3155,13 +3028,13 @@ } } }, - "/v1/groups": { + "/v1/instances": { "get": { "tags": [ - "silos" + "instances" ], - "summary": "List groups", - "operationId": "group_list", + "summary": "List instances", + "operationId": "instance_list", "parameters": [ { "in": "query", @@ -3183,11 +3056,19 @@ "type": "string" } }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -3197,7 +3078,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GroupResultsPage" + "$ref": "#/components/schemas/InstanceResultsPage" } } } @@ -3210,36 +3091,45 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "project" + ] } - } - }, - "/v1/groups/{group_id}": { - "get": { + }, + "post": { "tags": [ - "silos" + "instances" ], - "summary": "Fetch group", - "operationId": "group_view", + "summary": "Create instance", + "operationId": "instance_create", "parameters": [ { - "in": "path", - "name": "group_id", - "description": "ID of the group", + "in": "query", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceCreate" + } + } + }, + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Group" + "$ref": "#/components/schemas/Instance" } } } @@ -3253,35 +3143,14 @@ } } }, - "/v1/images": { + "/v1/instances/{instance}": { "get": { "tags": [ - "images" + "instances" ], - "summary": "List images", - "description": "List images which are global or scoped to the specified project. The images are returned sorted by creation date, with the most recent images appearing first.", - "operationId": "image_list", + "summary": "Fetch instance", + "operationId": "instance_view", "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "project", @@ -3291,10 +3160,12 @@ } }, { - "in": "query", - "name": "sort_by", + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -3304,7 +3175,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImageResultsPage" + "$ref": "#/components/schemas/Instance" } } } @@ -3315,18 +3186,14 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } }, - "post": { + "put": { "tags": [ - "images" + "instances" ], - "summary": "Create image", - "description": "Create a new image in a project.", - "operationId": "image_create", + "summary": "Update instance", + "operationId": "instance_update", "parameters": [ { "in": "query", @@ -3335,25 +3202,34 @@ "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImageCreate" + "$ref": "#/components/schemas/InstanceUpdate" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Image" + "$ref": "#/components/schemas/Instance" } } } @@ -3365,45 +3241,35 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/images/{image}": { - "get": { + }, + "delete": { "tags": [ - "images" + "instances" ], - "summary": "Fetch image", - "description": "Fetch the details for a specific image in a project.", - "operationId": "image_view", + "summary": "Delete instance", + "operationId": "instance_delete", "parameters": [ { - "in": "path", - "name": "image", - "description": "Name or ID of the image", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Image" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -3412,22 +3278,34 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/instances/{instance}/disks": { + "get": { "tags": [ - "images" + "instances" ], - "summary": "Delete image", - "description": "Permanently delete an image from a project. This operation cannot be undone. Any instances in the project using the image will continue to run, however new instances can not be created with this image.", - "operationId": "image_delete", + "summary": "List disks for instance", + "operationId": "instance_disk_list", "parameters": [ { - "in": "path", - "name": "image", - "description": "Name or ID of the image", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" } }, { @@ -3437,11 +3315,34 @@ "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "responses": { - "204": { - "description": "successful deletion" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskResultsPage" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -3449,22 +3350,24 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/images/{image}/demote": { + "/v1/instances/{instance}/disks/attach": { "post": { "tags": [ - "images" + "instances" ], - "summary": "Demote silo image", - "description": "Demote silo image to be visible only to a specified project", - "operationId": "image_demote", + "summary": "Attach disk to instance", + "operationId": "instance_disk_attach", "parameters": [ { "in": "path", - "name": "image", - "description": "Name or ID of the image", + "name": "instance", + "description": "Name or ID of the instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3474,19 +3377,28 @@ "in": "query", "name": "project", "description": "Name or ID of the project", - "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskPath" + } + } + }, + "required": true + }, "responses": { "202": { "description": "successfully enqueued operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Image" + "$ref": "#/components/schemas/Disk" } } } @@ -3500,19 +3412,18 @@ } } }, - "/v1/images/{image}/promote": { + "/v1/instances/{instance}/disks/detach": { "post": { "tags": [ - "images" + "instances" ], - "summary": "Promote project image", - "description": "Promote project image to be visible to all projects in the silo", - "operationId": "image_promote", + "summary": "Detach disk from instance", + "operationId": "instance_disk_detach", "parameters": [ { "in": "path", - "name": "image", - "description": "Name or ID of the image", + "name": "instance", + "description": "Name or ID of the instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3527,13 +3438,23 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskPath" + } + } + }, + "required": true + }, "responses": { "202": { "description": "successfully enqueued operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Image" + "$ref": "#/components/schemas/Disk" } } } @@ -3547,34 +3468,14 @@ } } }, - "/v1/instances": { + "/v1/instances/{instance}/external-ips": { "get": { "tags": [ "instances" ], - "summary": "List instances", - "operationId": "instance_list", + "summary": "List external IP addresses", + "operationId": "instance_external_ip_list", "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "project", @@ -3584,10 +3485,12 @@ } }, { - "in": "query", - "name": "sort_by", + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -3597,7 +3500,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceResultsPage" + "$ref": "#/components/schemas/ExternalIpResultsPage" } } } @@ -3608,25 +3511,30 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] } - }, + } + }, + "/v1/instances/{instance}/external-ips/ephemeral": { "post": { "tags": [ "instances" ], - "summary": "Create instance", - "operationId": "instance_create", + "summary": "Allocate and attach ephemeral IP to instance", + "operationId": "instance_ephemeral_ip_attach", "parameters": [ + { + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "project", "description": "Name or ID of the project", - "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3636,19 +3544,19 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceCreate" + "$ref": "#/components/schemas/EphemeralIpCreate" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", + "202": { + "description": "successfully enqueued operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Instance" + "$ref": "#/components/schemas/ExternalIp" } } } @@ -3660,44 +3568,35 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/instances/{instance}": { - "get": { + }, + "delete": { "tags": [ "instances" ], - "summary": "Fetch instance", - "operationId": "instance_view", + "summary": "Detach and deallocate ephemeral IP from instance", + "operationId": "instance_ephemeral_ip_detach", "parameters": [ { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Instance" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -3706,13 +3605,15 @@ "$ref": "#/components/responses/Error" } } - }, - "put": { + } + }, + "/v1/instances/{instance}/reboot": { + "post": { "tags": [ "instances" ], - "summary": "Update instance", - "operationId": "instance_update", + "summary": "Reboot an instance", + "operationId": "instance_reboot", "parameters": [ { "in": "query", @@ -3732,19 +3633,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceUpdate" - } - } - }, - "required": true - }, "responses": { - "200": { - "description": "successful operation", + "202": { + "description": "successfully enqueued operation", "content": { "application/json": { "schema": { @@ -3760,22 +3651,16 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/instances/{instance}/serial-console": { + "get": { "tags": [ "instances" ], - "summary": "Delete instance", - "operationId": "instance_delete", + "summary": "Fetch instance serial console", + "operationId": "instance_serial_console", "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "path", "name": "instance", @@ -3784,69 +3669,44 @@ "schema": { "$ref": "#/components/schemas/NameOrId" } - } - ], - "responses": { - "204": { - "description": "successful deletion" }, - "4XX": { - "$ref": "#/components/responses/Error" + { + "in": "query", + "name": "from_start", + "description": "Character index in the serial buffer from which to read, counting the bytes output since instance start. If this is not provided, `most_recent` must be provided, and if this *is* provided, `most_recent` must *not* be provided.", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + } }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/instances/{instance}/disks": { - "get": { - "tags": [ - "instances" - ], - "summary": "List disks for instance", - "operationId": "instance_disk_list", - "parameters": [ { "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "name": "max_bytes", + "description": "Maximum number of bytes of buffered serial console contents to return. If the requested range runs to the end of the available buffer, the data returned will be shorter than `max_bytes`.", "schema": { "nullable": true, "type": "integer", - "format": "uint32", - "minimum": 1 + "format": "uint64", + "minimum": 0 } }, { "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "name": "most_recent", + "description": "Character index in the serial buffer from which to read, counting *backward* from the most recently buffered data retrieved from the instance. (See note on `from_start` about mutual exclusivity)", "schema": { "nullable": true, - "type": "string" + "type": "integer", + "format": "uint64", + "minimum": 0 } }, { "in": "query", "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3858,7 +3718,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DiskResultsPage" + "$ref": "#/components/schemas/InstanceSerialConsoleData" } } } @@ -3869,19 +3729,16 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/instances/{instance}/disks/attach": { - "post": { + "/v1/instances/{instance}/serial-console/stream": { + "get": { "tags": [ "instances" ], - "summary": "Attach disk to instance", - "operationId": "instance_disk_attach", + "summary": "Stream instance serial console", + "operationId": "instance_serial_console_stream", "parameters": [ { "in": "path", @@ -3892,52 +3749,47 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "most_recent", + "description": "Character index in the serial buffer from which to read, counting *backward* from the most recently buffered data retrieved from the instance.", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DiskPath" - } - } - }, - "required": true - }, "responses": { - "202": { - "description": "successfully enqueued operation", + "default": { + "description": "", "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Disk" - } + "*/*": { + "schema": {} } } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" } - } + }, + "x-dropshot-websocket": {} } }, - "/v1/instances/{instance}/disks/detach": { - "post": { + "/v1/instances/{instance}/ssh-public-keys": { + "get": { "tags": [ "instances" ], - "summary": "Detach disk from instance", - "operationId": "instance_disk_detach", + "summary": "List SSH public keys for instance", + "description": "List SSH public keys injected via cloud-init during instance creation. Note that this list is a snapshot in time and will not reflect updates made after the instance is created.", + "operationId": "instance_ssh_public_key_list", "parameters": [ { "in": "path", @@ -3950,51 +3802,24 @@ }, { "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DiskPath" - } + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, - "required": true - }, - "responses": { - "202": { - "description": "successfully enqueued operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Disk" - } - } + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" } }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/instances/{instance}/external-ips": { - "get": { - "tags": [ - "instances" - ], - "summary": "List external IP addresses", - "operationId": "instance_external_ip_list", - "parameters": [ { "in": "query", "name": "project", @@ -4004,12 +3829,10 @@ } }, { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "sort_by", "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -4019,7 +3842,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ExternalIpResultsPage" + "$ref": "#/components/schemas/SshKeyResultsPage" } } } @@ -4030,52 +3853,45 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/instances/{instance}/external-ips/ephemeral": { + "/v1/instances/{instance}/start": { "post": { "tags": [ "instances" ], - "summary": "Allocate and attach ephemeral IP to instance", - "operationId": "instance_ephemeral_ip_attach", + "summary": "Boot instance", + "operationId": "instance_start", "parameters": [ { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "project", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "in": "path", + "name": "instance", + "description": "Name or ID of the instance", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EphemeralIpCreate" - } - } - }, - "required": true - }, "responses": { "202": { "description": "successfully enqueued operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ExternalIp" + "$ref": "#/components/schemas/Instance" } } } @@ -4087,52 +3903,15 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { - "tags": [ - "instances" - ], - "summary": "Detach and deallocate ephemeral IP from instance", - "operationId": "instance_ephemeral_ip_detach", - "parameters": [ - { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } } }, - "/v1/instances/{instance}/reboot": { + "/v1/instances/{instance}/stop": { "post": { "tags": [ "instances" ], - "summary": "Reboot an instance", - "operationId": "instance_reboot", + "summary": "Stop instance", + "operationId": "instance_stop", "parameters": [ { "in": "query", @@ -4172,60 +3951,61 @@ } } }, - "/v1/instances/{instance}/serial-console": { + "/v1/internet-gateway-ip-addresses": { "get": { "tags": [ - "instances" + "vpcs" ], - "summary": "Fetch instance serial console", - "operationId": "instance_serial_console", + "summary": "List IP addresses attached to internet gateway", + "operationId": "internet_gateway_ip_address_list", "parameters": [ { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "from_start", - "description": "Character index in the serial buffer from which to read, counting the bytes output since instance start. If this is not provided, `most_recent` must be provided, and if this *is* provided, `most_recent` must *not* be provided.", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { "nullable": true, "type": "integer", - "format": "uint64", - "minimum": 0 + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "max_bytes", - "description": "Maximum number of bytes of buffered serial console contents to return. If the requested range runs to the end of the available buffer, the data returned will be shorter than `max_bytes`.", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { "nullable": true, - "type": "integer", - "format": "uint64", - "minimum": 0 + "type": "string" } }, { "in": "query", - "name": "most_recent", - "description": "Character index in the serial buffer from which to read, counting *backward* from the most recently buffered data retrieved from the instance. (See note on `from_start` about mutual exclusivity)", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { - "nullable": true, - "type": "integer", - "format": "uint64", - "minimum": 0 + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -4237,7 +4017,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceSerialConsoleData" + "$ref": "#/components/schemas/InternetGatewayIpAddressResultsPage" } } } @@ -4248,21 +4028,24 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [ + "gateway" + ] } - } - }, - "/v1/instances/{instance}/serial-console/stream": { - "get": { + }, + "post": { "tags": [ - "instances" + "vpcs" ], - "summary": "Stream instance serial console", - "operationId": "instance_serial_console_stream", + "summary": "Attach IP address to internet gateway", + "operationId": "internet_gateway_ip_address_create", "parameters": [ { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -4270,98 +4053,38 @@ }, { "in": "query", - "name": "most_recent", - "description": "Character index in the serial buffer from which to read, counting *backward* from the most recently buffered data retrieved from the instance.", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { - "nullable": true, - "type": "integer", - "format": "uint64", - "minimum": 0 + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "responses": { - "default": { - "description": "", - "content": { - "*/*": { - "schema": {} + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InternetGatewayIpAddressCreate" } } - } - }, - "x-dropshot-websocket": {} - } - }, - "/v1/instances/{instance}/ssh-public-keys": { - "get": { - "tags": [ - "instances" - ], - "summary": "List SSH public keys for instance", - "description": "List SSH public keys injected via cloud-init during instance creation. Note that this list is a snapshot in time and will not reflect updates made after the instance is created.", - "operationId": "instance_ssh_public_key_list", - "parameters": [ - { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SshKeyResultsPage" + "$ref": "#/components/schemas/InternetGatewayIpAddress" } } } @@ -4372,94 +4095,62 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/instances/{instance}/start": { - "post": { + "/v1/internet-gateway-ip-addresses/{address}": { + "delete": { "tags": [ - "instances" + "vpcs" ], - "summary": "Boot instance", - "operationId": "instance_start", + "summary": "Detach IP address from internet gateway", + "operationId": "internet_gateway_ip_address_delete", "parameters": [ { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "in": "path", + "name": "address", + "description": "Name or ID of the IP address", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "cascade", + "description": "Also delete routes targeting this gateway element.", "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "202": { - "description": "successfully enqueued operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Instance" - } - } + "type": "boolean" } }, - "4XX": { - "$ref": "#/components/responses/Error" + { + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/instances/{instance}/stop": { - "post": { - "tags": [ - "instances" - ], - "summary": "Stop instance", - "operationId": "instance_stop", - "parameters": [ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { - "in": "path", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "202": { - "description": "successfully enqueued operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Instance" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -4470,13 +4161,13 @@ } } }, - "/v1/internet-gateway-ip-addresses": { + "/v1/internet-gateway-ip-pools": { "get": { "tags": [ "vpcs" ], - "summary": "List IP addresses attached to internet gateway", - "operationId": "internet_gateway_ip_address_list", + "summary": "List IP pools attached to internet gateway", + "operationId": "internet_gateway_ip_pool_list", "parameters": [ { "in": "query", @@ -4536,217 +4227,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InternetGatewayIpAddressResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "gateway" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Attach IP address to internet gateway", - "operationId": "internet_gateway_ip_address_create", - "parameters": [ - { - "in": "query", - "name": "gateway", - "description": "Name or ID of the internet gateway", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InternetGatewayIpAddressCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InternetGatewayIpAddress" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/internet-gateway-ip-addresses/{address}": { - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Detach IP address from internet gateway", - "operationId": "internet_gateway_ip_address_delete", - "parameters": [ - { - "in": "path", - "name": "address", - "description": "Name or ID of the IP address", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "cascade", - "description": "Also delete routes targeting this gateway element.", - "schema": { - "type": "boolean" - } - }, - { - "in": "query", - "name": "gateway", - "description": "Name or ID of the internet gateway", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/internet-gateway-ip-pools": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List IP pools attached to internet gateway", - "operationId": "internet_gateway_ip_pool_list", - "parameters": [ - { - "in": "query", - "name": "gateway", - "description": "Name or ID of the internet gateway", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InternetGatewayIpPoolResultsPage" + "$ref": "#/components/schemas/InternetGatewayIpPoolResultsPage" } } } @@ -12384,6 +11865,525 @@ } } } + }, + "/v1/webhooks/deliveries": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List delivery attempts to a webhook receiver.", + "operationId": "webhook_delivery_list", + "parameters": [ + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/webhooks/deliveries/{event_id}/resend": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Request re-delivery of a webhook event.", + "operationId": "webhook_delivery_resend", + "parameters": [ + { + "in": "path", + "name": "event_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookDeliveryId" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/event-classes": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List webhook event classes", + "operationId": "webhook_event_class_list", + "parameters": [ + { + "in": "query", + "name": "filter", + "description": "An optional glob pattern for filtering event class names.", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EventClassResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/webhooks/event-classes/{name}": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "Fetch details on an event class by name.", + "operationId": "webhook_event_class_view", + "parameters": [ + { + "in": "path", + "name": "name", + "description": "The name of the event class.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EventClass" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/receivers": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Create a new webhook receiver.", + "operationId": "webhook_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/receivers/{receiver}": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "Get the configuration for a webhook receiver.", + "operationId": "webhook_view", + "parameters": [ + { + "in": "path", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "system/webhooks" + ], + "summary": "Update the configuration of an existing webhook receiver.", + "operationId": "webhook_update", + "parameters": [ + { + "in": "path", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/webhooks" + ], + "summary": "Delete a webhook receiver.", + "operationId": "webhook_delete", + "parameters": [ + { + "in": "path", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/receivers/{receiver}/probe": { + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Send a liveness probe request to a webhook receiver.", + "operationId": "webhook_probe", + "parameters": [ + { + "in": "path", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "resend", + "description": "If true, resend all events that have not been delivered successfully if the probe request succeeds.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookProbeResult" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/secrets": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List the IDs of secrets for a webhook receiver.", + "operationId": "webhook_secrets_list", + "parameters": [ + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecrets" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "system/webhooks" + ], + "summary": "Add a secret to a webhook receiver.", + "operationId": "webhook_secrets_add", + "parameters": [ + { + "in": "query", + "name": "receiver", + "description": "The name or ID of the webhook receiver.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecretCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookSecretId" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/webhooks/secrets/{secret_id}": { + "delete": { + "tags": [ + "system/webhooks" + ], + "summary": "Delete a secret associated with a webhook receiver by ID.", + "operationId": "webhook_secrets_delete", + "parameters": [ + { + "in": "path", + "name": "secret_id", + "description": "ID of the secret.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } } }, "components": { From 2aaca1ed992025c9adb16d6667f588675aa10767 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Mar 2025 11:47:11 -0800 Subject: [PATCH 107/168] implement most of the object-list APIs --- nexus/db-model/src/webhook_rx.rs | 6 + .../db-queries/src/db/datastore/webhook_rx.rs | 125 ++++++++++++------ nexus/external-api/src/lib.rs | 11 ++ .../background/tasks/webhook_deliverator.rs | 42 +++--- nexus/src/app/webhook.rs | 20 +++ nexus/src/external_api/http_entrypoints.rs | 53 +++++++- nexus/tests/integration_tests/webhooks.rs | 59 +++++++++ 7 files changed, 245 insertions(+), 71 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index ddef7e83bb1..29240632484 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -141,6 +141,12 @@ impl WebhookSecret { } } +impl From for views::WebhookSecretId { + fn from(secret: WebhookSecret) -> Self { + Self { id: secret.identity.id.into_untyped_uuid() } + } +} + #[derive( Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, )] diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index e21fade3640..56b0c3f0324 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -14,6 +14,7 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::DbSemverVersion; use crate::db::model::Generation; +use crate::db::model::Name; use crate::db::model::WebhookEventClass; use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; @@ -38,7 +39,9 @@ use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; +use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookGlobStatus; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; @@ -47,6 +50,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookReceiverUuid; +use ref_cast::RefCast; use uuid::Uuid; impl DataStore { @@ -145,11 +149,21 @@ impl DataStore { authz_rx: &authz::WebhookReceiver, ) -> Result<(Vec, Vec), Error> { opctx.authorize(authz::Action::ListChildren, authz_rx).await?; - let conn = self.pool_connection_authorized(opctx).await?; + self.rx_config_fetch_on_conn( + authz_rx.id(), + &*self.pool_connection_authorized(opctx).await?, + ) + .await + } + + async fn rx_config_fetch_on_conn( + &self, + rx_id: WebhookReceiverUuid, + conn: &async_bb8_diesel::Connection, + ) -> Result<(Vec, Vec), Error> { let subscriptions = - self.webhook_rx_subscription_list_on_conn(authz_rx, &conn).await?; - let secrets = - self.webhook_rx_secret_list_on_conn(authz_rx, &conn).await?; + self.rx_subscription_list_on_conn(rx_id, &conn).await?; + let secrets = self.rx_secret_list_on_conn(rx_id, &conn).await?; Ok((subscriptions, secrets)) } @@ -243,6 +257,55 @@ impl DataStore { Ok(()) } + pub async fn webhook_rx_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let conn = self.pool_connection_authorized(opctx).await?; + + // As we would like to return a list of `WebhookReceiverConfig` structs, + // which own `Vec`s of the receiver's secrets and event class + // subscriptions, we'll do this by first querying the database to load + // all the receivers, and then querying for their individual lists of + // secrets and event class subscriptions. + // + // This is a bit unfortunate, and it would be nicer to do this with + // JOINs, but it's a bit hairy as the subscriptions come from both the + // `webhook_rx_subscription` and `webhook_rx_glob` tables... + + let receivers = match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(rx_dsl::webhook_receiver, rx_dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + rx_dsl::webhook_receiver, + rx_dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(rx_dsl::time_deleted.is_null()) + .select(WebhookReceiver::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to list receivers") + })?; + + // Now that we've got the current page of receivers, go and get their + // event subscriptions and secrets. + let mut result = Vec::with_capacity(receivers.len()); + for rx in receivers { + let secrets = self.rx_secret_list_on_conn(rx.id(), &*conn).await?; + let events = + self.rx_subscription_list_on_conn(rx.id(), &*conn).await?; + result.push(WebhookReceiverConfig { rx, secrets, events }); + } + + Ok(result) + } + // // Subscriptions // @@ -275,37 +338,36 @@ impl DataStore { Ok(()) } - async fn webhook_rx_subscription_list_on_conn( + async fn rx_subscription_list_on_conn( &self, - authz_rx: &authz::WebhookReceiver, + rx_id: WebhookReceiverUuid, conn: &async_bb8_diesel::Connection, ) -> ListResultVec { - let rx_id = authz_rx.id().into_untyped_uuid(); + // TODO(eliza): rather than performing two separate queries, this could + // perhaps be expressed using a SQL `union`, with an added "label" + // column to distinguish between globs and exact subscriptions, but this + // is a bit more complex, and would require raw SQL... // First, get all the exact subscriptions that aren't from globs. let exact = subscription_dsl::webhook_rx_subscription - .filter(subscription_dsl::rx_id.eq(rx_id)) + .filter(subscription_dsl::rx_id.eq(rx_id.into_untyped_uuid())) .filter(subscription_dsl::glob.is_null()) .select(subscription_dsl::event_class) .load_async::(conn) .await .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_rx), - ) + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to list exact subscriptions") })?; // Then, get the globs let globs = glob_dsl::webhook_rx_event_glob - .filter(glob_dsl::rx_id.eq(rx_id)) + .filter(glob_dsl::rx_id.eq(rx_id.into_untyped_uuid())) .select(WebhookGlob::as_select()) .load_async::(conn) .await .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_rx), - ) + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to list glob subscriptions") })?; let subscriptions = exact .into_iter() @@ -641,25 +703,23 @@ impl DataStore { ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, authz_rx).await?; let conn = self.pool_connection_authorized(&opctx).await?; - self.webhook_rx_secret_list_on_conn(authz_rx, &conn).await + self.rx_secret_list_on_conn(authz_rx.id(), &conn).await } - async fn webhook_rx_secret_list_on_conn( + async fn rx_secret_list_on_conn( &self, - authz_rx: &authz::WebhookReceiver, + rx_id: WebhookReceiverUuid, conn: &async_bb8_diesel::Connection, ) -> ListResultVec { secret_dsl::webhook_secret - .filter(secret_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid())) + .filter(secret_dsl::rx_id.eq(rx_id.into_untyped_uuid())) .filter(secret_dsl::time_deleted.is_null()) .select(WebhookSecret::as_select()) .load_async(conn) .await .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_rx), - ) + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to list webhook receiver secrets") }) } @@ -719,21 +779,6 @@ impl DataStore { .map_err(async_insert_error_to_txn(rx_id.into()))?; Ok(secret) } - - pub async fn webhook_rx_list( - &self, - opctx: &OpContext, - pagparams: &DataPageParams<'_, Uuid>, - ) -> ListResultVec { - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; - let conn = self.pool_connection_authorized(opctx).await?; - paginated(rx_dsl::webhook_receiver, rx_dsl::id, pagparams) - .filter(rx_dsl::time_deleted.is_null()) - .select(WebhookReceiver::as_select()) - .load_async::(&*conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) - } } fn async_insert_error_to_txn( diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 69e217b7426..0395a2bc27a 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3500,6 +3500,17 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; + /// List webhook receivers. + #[endpoint { + method = GET, + path = "/v1/webhooks/receivers/", + tags = ["system/webhooks"], + }] + async fn webhook_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + /// Get the configuration for a webhook receiver. #[endpoint { method = GET, diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index c09750670f6..3c5cea3bedc 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -1,22 +1,20 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::app::authz; use crate::app::background::BackgroundTask; -use crate::app::db::lookup::LookupPath; use crate::app::webhook::ReceiverClient; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; use nexus_db_queries::db::model::WebhookDeliveryResult; -use nexus_db_queries::db::model::WebhookReceiver; +use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; -use nexus_types::internal_api::background::{ - WebhookDeliveratorStatus, WebhookRxDeliveryStatus, -}; +use nexus_types::internal_api::background::WebhookDeliveratorStatus; +use nexus_types::internal_api::background::WebhookRxDeliveryStatus; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::Error; use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid}; use std::num::NonZeroU32; @@ -125,18 +123,24 @@ impl WebhookDeliverator { while let Some(p) = paginator.next() { let rxs = self .datastore - .webhook_rx_list(&opctx, &p.current_pagparams()) + .webhook_rx_list( + &opctx, + &PaginatedBy::Id(p.current_pagparams()), + ) .await?; - paginator = p.found_batch(&rxs, &|rx| rx.id().into_untyped_uuid()); + paginator = p + .found_batch(&rxs, &|WebhookReceiverConfig { rx, .. }| { + rx.id().into_untyped_uuid() + }); for rx in rxs { + let rx_id = rx.rx.id(); let opctx = opctx.child(maplit::btreemap! { - "receiver_id".to_string() => rx.id().to_string(), - "receiver_name".to_string() => rx.name().to_string(), + "receiver_id".to_string() => rx_id.to_string(), + "receiver_name".to_string() => rx.rx.name().to_string(), }); let deliverator = self.clone(); tasks.spawn(async move { - let rx_id = rx.id(); let status = match deliverator.rx_deliver(&opctx, rx).await { Ok(status) => status, Err(e) => { @@ -171,22 +175,8 @@ impl WebhookDeliverator { async fn rx_deliver( &self, opctx: &OpContext, - rx: WebhookReceiver, + WebhookReceiverConfig { rx, secrets, .. }: WebhookReceiverConfig, ) -> Result { - // First, look up the receiver's secrets and any deliveries for that - // receiver. If any of these lookups fail, bail out, as we can't - // meaningfully deliver any events to a receiver if we don't know what - // they are or how to sign them. - let (authz_rx,) = LookupPath::new(opctx, &self.datastore) - .webhook_receiver_id(rx.id()) - .lookup_for(authz::Action::ListChildren) - .await - .map_err(|e| anyhow::anyhow!("could not look up receiver: {e}"))?; - let secrets = self - .datastore - .webhook_rx_secret_list(opctx, &authz_rx) - .await - .map_err(|e| anyhow::anyhow!("could not list secrets: {e}"))?; let mut client = ReceiverClient::new(&self.client, secrets, &rx)?; let deliveries = self diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 7ff0f763c77..0aca03e23b7 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -32,8 +32,10 @@ use nexus_db_queries::db::model::WebhookSecret; use nexus_types::external_api::params; use nexus_types::external_api::views; use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_uuid_kinds::GenericUuid; @@ -68,6 +70,15 @@ impl super::Nexus { } } + pub async fn webhook_receiver_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + self.datastore().webhook_rx_list(opctx, pagparams).await + } + pub async fn webhook_receiver_config_fetch( &self, opctx: &OpContext, @@ -79,6 +90,15 @@ impl super::Nexus { Ok(WebhookReceiverConfig { rx, secrets, events }) } + pub async fn webhook_receiver_secrets_list( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + ) -> ListResultVec { + let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; + self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await + } + pub async fn webhook_receiver_create( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 998bb2a8621..c6de2a308b1 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7330,6 +7330,43 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn webhook_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pagparams, scan_params)?; + + let rxs = nexus + .webhook_receiver_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(views::Webhook::try_from) + .collect::, _>>()?; + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + rxs, + &marker_for_name_or_id, + )?)) + }; + + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn webhook_view( rqctx: RequestContext, path_params: Path, @@ -7453,7 +7490,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_secrets_list( rqctx: RequestContext, - _query_params: Query, + query_params: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7462,10 +7499,16 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook_selector = query_params.into_inner(); + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let secrets = nexus + .webhook_receiver_secrets_list(&opctx, rx) + .await? + .into_iter() + .map(Into::into) + .collect(); + + Ok(HttpResponseOk(views::WebhookSecrets { secrets })) }; apictx .context diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index df21c596498..398928b06b6 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -69,6 +69,32 @@ async fn webhook_get_as( .unwrap() } +async fn webhook_rx_list(client: &ClientTestContext) -> Vec { + resource_helpers::objects_list_page_authz::( + client, + RECEIVERS_BASE_PATH, + ) + .await + .items +} + +async fn webhook_secrets_get( + client: &ClientTestContext, + webhook_name_or_id: impl Into, +) -> views::WebhookSecrets { + let name_or_id = webhook_name_or_id.into(); + NexusRequest::object_get( + client, + &format!("{SECRETS_BASE_PATH}/?receiver={name_or_id}"), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + fn my_great_webhook_params( mock: &httpmock::MockServer, ) -> params::WebhookCreate { @@ -313,6 +339,21 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { let secret1_id = webhook.secrets[0].id; + let client = &cptestctx.external_client; + let assert_secrets_get = |mut expected: Vec| async move { + let mut actual = webhook_secrets_get(client, rx_id.into_untyped_uuid()) + .await + .secrets + .into_iter() + .map(|secret| secret.id) + .collect::>(); + actual.sort(); + expected.sort(); + assert_eq!(expected, actual); + }; + + assert_secrets_get(vec![secret1_id]).await; + // Add a second secret to the webhook receiver. let secret2_id = dbg!( secret_add( @@ -323,6 +364,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { .await ) .id; + assert_secrets_get(vec![secret1_id, secret2_id]).await; // And a third one, just for kicks. let secret3_id = dbg!( @@ -334,6 +376,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { .await ) .id; + assert_secrets_get(vec![secret1_id, secret2_id, secret3_id]).await; let mock = server .mock_async(|when, then| { @@ -384,6 +427,7 @@ async fn test_multiple_secrets(cptestctx: &ControlPlaneTestContext) { async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { let nexus = cptestctx.server.server_context().nexus.clone(); let internal_client = &cptestctx.internal_client; + let client = &cptestctx.external_client; let datastore = nexus.datastore(); let opctx = @@ -392,6 +436,13 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { let bar_event_id = WebhookEventUuid::new_v4(); let baz_event_id = WebhookEventUuid::new_v4(); + let assert_webhook_rx_list_matches = |mut expected: Vec| async move { + let mut actual = webhook_rx_list(client).await; + actual.sort_by_key(|rx| rx.identity.id); + expected.sort_by_key(|rx| rx.identity.id); + assert_eq!(expected, actual); + }; + // Create three webhook receivers let srv_bar = httpmock::MockServer::start_async().await; const BAR_SECRET: &str = "this is bar's secret"; @@ -412,6 +463,7 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { ) .await; dbg!(&rx_bar); + assert_webhook_rx_list_matches(vec![rx_bar.clone()]).await; let mock_bar = { let webhook = rx_bar.clone(); srv_bar @@ -448,6 +500,7 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { ) .await; dbg!(&rx_baz); + assert_webhook_rx_list_matches(vec![rx_bar.clone(), rx_baz.clone()]).await; let mock_baz = { let webhook = rx_baz.clone(); srv_baz @@ -484,6 +537,12 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { ) .await; dbg!(&rx_star); + assert_webhook_rx_list_matches(vec![ + rx_bar.clone(), + rx_baz.clone(), + rx_star.clone(), + ]) + .await; let mock_star = { let webhook = rx_star.clone(); srv_star From cdb84edaee9b8e61fc6757064f98ef67211e34bb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Mar 2025 12:18:59 -0800 Subject: [PATCH 108/168] clearer API naming (less metonymy) --- nexus/db-model/src/webhook_rx.rs | 6 +- nexus/external-api/output/nexus_tags.txt | 9 +- nexus/external-api/src/lib.rs | 32 ++-- nexus/src/app/webhook.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 39 ++-- nexus/tests/integration_tests/webhooks.rs | 36 ++-- nexus/types/src/external_api/params.rs | 2 +- nexus/types/src/external_api/views.rs | 2 +- openapi/nexus.json | 212 ++++++++++++++------- 9 files changed, 211 insertions(+), 129 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 29240632484..88b5fe022fd 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -33,11 +33,11 @@ pub struct WebhookReceiverConfig { pub events: Vec, } -impl TryFrom for views::Webhook { +impl TryFrom for views::WebhookReceiver { type Error = Error; fn try_from( WebhookReceiverConfig { rx, secrets, events }: WebhookReceiverConfig, - ) -> Result { + ) -> Result { let secrets = secrets .iter() .map(|WebhookSecret { identity, .. }| views::WebhookSecretId { @@ -57,7 +57,7 @@ impl TryFrom for views::Webhook { rx.endpoint, ), })?; - Ok(views::Webhook { + Ok(views::WebhookReceiver { identity: rx.identity(), endpoint, secrets, diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 8d829f9c373..34a163f1c7a 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -266,18 +266,19 @@ ping GET /v1/ping API operations found with tag "system/webhooks" OPERATION ID METHOD URL PATH -webhook_create POST /v1/webhooks/receivers -webhook_delete DELETE /v1/webhooks/receivers/{receiver} webhook_delivery_list GET /v1/webhooks/deliveries webhook_delivery_resend POST /v1/webhooks/deliveries/{event_id}/resend webhook_event_class_list GET /v1/webhooks/event-classes webhook_event_class_view GET /v1/webhooks/event-classes/{name} -webhook_probe POST /v1/webhooks/receivers/{receiver}/probe +webhook_receiver_create POST /v1/webhooks/receivers +webhook_receiver_delete DELETE /v1/webhooks/receivers/{receiver} +webhook_receiver_list GET /v1/webhooks/receivers +webhook_receiver_probe POST /v1/webhooks/receivers/{receiver}/probe +webhook_receiver_view GET /v1/webhooks/receivers/{receiver} webhook_secrets_add POST /v1/webhooks/secrets webhook_secrets_delete DELETE /v1/webhooks/secrets/{secret_id} webhook_secrets_list GET /v1/webhooks/secrets webhook_update PUT /v1/webhooks/receivers/{receiver} -webhook_view GET /v1/webhooks/receivers/{receiver} API operations found with tag "vpcs" OPERATION ID METHOD URL PATH diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 0395a2bc27a..69725c27223 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3506,10 +3506,10 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers/", tags = ["system/webhooks"], }] - async fn webhook_list( + async fn webhook_receiver_list( rqctx: RequestContext, query_params: Query, - ) -> Result>, HttpError>; + ) -> Result>, HttpError>; /// Get the configuration for a webhook receiver. #[endpoint { @@ -3517,10 +3517,10 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] - async fn webhook_view( + async fn webhook_receiver_view( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError>; + path_params: Path, + ) -> Result, HttpError>; /// Create a new webhook receiver. #[endpoint { @@ -3528,10 +3528,10 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers", tags = ["system/webhooks"], }] - async fn webhook_create( + async fn webhook_receiver_create( rqctx: RequestContext, params: TypedBody, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Update the configuration of an existing webhook receiver. #[endpoint { @@ -3541,7 +3541,7 @@ pub trait NexusExternalApi { }] async fn webhook_update( rqctx: RequestContext, - path_params: Path, + path_params: Path, params: TypedBody, ) -> Result; @@ -3551,9 +3551,9 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] - async fn webhook_delete( + async fn webhook_receiver_delete( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result; /// Send a liveness probe request to a webhook receiver. @@ -3565,9 +3565,9 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers/{receiver}/probe", tags = ["system/webhooks"], }] - async fn webhook_probe( + async fn webhook_receiver_probe( rqctx: RequestContext, - path_params: Path, + path_params: Path, query_params: Query, ) -> Result, HttpError>; @@ -3579,7 +3579,7 @@ pub trait NexusExternalApi { }] async fn webhook_secrets_list( rqctx: RequestContext, - query_params: Query, + query_params: Query, ) -> Result, HttpError>; /// Add a secret to a webhook receiver. @@ -3590,7 +3590,7 @@ pub trait NexusExternalApi { }] async fn webhook_secrets_add( rqctx: RequestContext, - query_params: Query, + query_params: Query, params: TypedBody, ) -> Result, HttpError>; @@ -3613,7 +3613,7 @@ pub trait NexusExternalApi { }] async fn webhook_delivery_list( rqctx: RequestContext, - receiver: Query, + receiver: Query, pagination: Query, ) -> Result>, HttpError>; @@ -3626,7 +3626,7 @@ pub trait NexusExternalApi { async fn webhook_delivery_resend( rqctx: RequestContext, path_params: Path, - receiver: Query, + receiver: Query, ) -> Result, HttpError>; } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 0aca03e23b7..7a987e6244f 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -52,7 +52,7 @@ impl super::Nexus { pub fn webhook_receiver_lookup<'a>( &'a self, opctx: &'a OpContext, - webhook_selector: params::WebhookSelector, + webhook_selector: params::WebhookReceiverSelector, ) -> LookupResult> { match webhook_selector.receiver { NameOrId::Id(id) => { diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index c6de2a308b1..e25bae6b505 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7330,10 +7330,11 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_list( + async fn webhook_receiver_list( rqctx: RequestContext, query_params: Query, - ) -> Result>, HttpError> { + ) -> Result>, HttpError> + { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7350,7 +7351,7 @@ impl NexusExternalApi for NexusExternalApiImpl { .webhook_receiver_list(&opctx, &paginated_by) .await? .into_iter() - .map(views::Webhook::try_from) + .map(views::WebhookReceiver::try_from) .collect::, _>>()?; Ok(HttpResponseOk(ScanByNameOrId::results_page( @@ -7367,10 +7368,10 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_view( + async fn webhook_receiver_view( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { + path_params: Path, + ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7381,7 +7382,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; let webhook = nexus.webhook_receiver_config_fetch(&opctx, rx).await?; - Ok(HttpResponseOk(views::Webhook::try_from(webhook)?)) + Ok(HttpResponseOk(views::WebhookReceiver::try_from(webhook)?)) }; apictx .context @@ -7390,10 +7391,10 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_create( + async fn webhook_receiver_create( rqctx: RequestContext, params: TypedBody, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7403,7 +7404,7 @@ impl NexusExternalApi for NexusExternalApiImpl { crate::context::op_context_for_external_api(&rqctx).await?; let receiver = nexus.webhook_receiver_create(&opctx, params).await?; - Ok(HttpResponseCreated(views::Webhook::try_from(receiver)?)) + Ok(HttpResponseCreated(views::WebhookReceiver::try_from(receiver)?)) }; apictx .context @@ -7414,7 +7415,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_update( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, _params: TypedBody, ) -> Result { let apictx = rqctx.context(); @@ -7436,9 +7437,9 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_delete( + async fn webhook_receiver_delete( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -7461,9 +7462,9 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_probe( + async fn webhook_receiver_probe( rqctx: RequestContext, - path_params: Path, + path_params: Path, query_params: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); @@ -7490,7 +7491,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_secrets_list( rqctx: RequestContext, - query_params: Query, + query_params: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7520,7 +7521,7 @@ impl NexusExternalApi for NexusExternalApiImpl { /// Add a secret to a webhook. async fn webhook_secrets_add( rqctx: RequestContext, - query_params: Query, + query_params: Query, params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); @@ -7583,7 +7584,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_list( rqctx: RequestContext, - _receiver: Query, + _receiver: Query, _query_params: Query, ) -> Result>, HttpError> { @@ -7609,7 +7610,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_resend( rqctx: RequestContext, _path_params: Path, - _receiver: Query, + _receiver: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 398928b06b6..3d794ce0d10 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -34,12 +34,11 @@ const SECRETS_BASE_PATH: &str = "/v1/webhooks/secrets"; async fn webhook_create( ctx: &ControlPlaneTestContext, params: ¶ms::WebhookCreate, -) -> views::Webhook { - resource_helpers::object_create::( - &ctx.external_client, - RECEIVERS_BASE_PATH, - params, - ) +) -> views::WebhookReceiver { + resource_helpers::object_create::< + params::WebhookCreate, + views::WebhookReceiver, + >(&ctx.external_client, RECEIVERS_BASE_PATH, params) .await } @@ -51,7 +50,7 @@ fn get_webhooks_url(name_or_id: impl Into) -> String { async fn webhook_get( client: &ClientTestContext, webhook_url: &str, -) -> views::Webhook { +) -> views::WebhookReceiver { webhook_get_as(client, webhook_url, AuthnMode::PrivilegedUser).await } @@ -59,7 +58,7 @@ async fn webhook_get_as( client: &ClientTestContext, webhook_url: &str, authn_as: AuthnMode, -) -> views::Webhook { +) -> views::WebhookReceiver { NexusRequest::object_get(client, &webhook_url) .authn_as(authn_as) .execute() @@ -69,8 +68,10 @@ async fn webhook_get_as( .unwrap() } -async fn webhook_rx_list(client: &ClientTestContext) -> Vec { - resource_helpers::objects_list_page_authz::( +async fn webhook_rx_list( + client: &ClientTestContext, +) -> Vec { + resource_helpers::objects_list_page_authz::( client, RECEIVERS_BASE_PATH, ) @@ -153,7 +154,7 @@ async fn webhook_send_probe( } fn is_valid_for_webhook( - webhook: &views::Webhook, + webhook: &views::WebhookReceiver, ) -> impl FnOnce(httpmock::When) -> httpmock::When { let path = webhook.endpoint.path().to_string(); let id = webhook.identity.id.to_string(); @@ -436,12 +437,13 @@ async fn test_multiple_receivers(cptestctx: &ControlPlaneTestContext) { let bar_event_id = WebhookEventUuid::new_v4(); let baz_event_id = WebhookEventUuid::new_v4(); - let assert_webhook_rx_list_matches = |mut expected: Vec| async move { - let mut actual = webhook_rx_list(client).await; - actual.sort_by_key(|rx| rx.identity.id); - expected.sort_by_key(|rx| rx.identity.id); - assert_eq!(expected, actual); - }; + let assert_webhook_rx_list_matches = + |mut expected: Vec| async move { + let mut actual = webhook_rx_list(client).await; + actual.sort_by_key(|rx| rx.identity.id); + expected.sort_by_key(|rx| rx.identity.id); + assert_eq!(expected, actual); + }; // Create three webhook receivers let srv_bar = httpmock::MockServer::start_async().await; diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 4a8d8948be9..66fdf3310a0 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2390,7 +2390,7 @@ pub struct EventClassSelector { } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct WebhookSelector { +pub struct WebhookReceiverSelector { /// The name or ID of the webhook receiver. pub receiver: NameOrId, } diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 3f050197466..63d4495fdf7 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1064,7 +1064,7 @@ pub struct EventClass { #[derive( ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, )] -pub struct Webhook { +pub struct WebhookReceiver { #[serde(flatten)] pub identity: IdentityMetadata, diff --git a/openapi/nexus.json b/openapi/nexus.json index d0a6c8784bb..7afa9df59a7 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12081,12 +12081,69 @@ } }, "/v1/webhooks/receivers": { + "get": { + "tags": [ + "system/webhooks" + ], + "summary": "List webhook receivers.", + "operationId": "webhook_receiver_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookReceiverResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, "post": { "tags": [ "system/webhooks" ], "summary": "Create a new webhook receiver.", - "operationId": "webhook_create", + "operationId": "webhook_receiver_create", "requestBody": { "content": { "application/json": { @@ -12103,7 +12160,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Webhook" + "$ref": "#/components/schemas/WebhookReceiver" } } } @@ -12123,7 +12180,7 @@ "system/webhooks" ], "summary": "Get the configuration for a webhook receiver.", - "operationId": "webhook_view", + "operationId": "webhook_receiver_view", "parameters": [ { "in": "path", @@ -12141,7 +12198,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Webhook" + "$ref": "#/components/schemas/WebhookReceiver" } } } @@ -12198,7 +12255,7 @@ "system/webhooks" ], "summary": "Delete a webhook receiver.", - "operationId": "webhook_delete", + "operationId": "webhook_receiver_delete", "parameters": [ { "in": "path", @@ -12229,7 +12286,7 @@ "system/webhooks" ], "summary": "Send a liveness probe request to a webhook receiver.", - "operationId": "webhook_probe", + "operationId": "webhook_receiver_probe", "parameters": [ { "in": "path", @@ -25037,67 +25094,6 @@ } } }, - "Webhook": { - "description": "The configuration for a webhook.", - "type": "object", - "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" - }, - "endpoint": { - "description": "The URL that webhook notification requests are sent to.", - "type": "string", - "format": "uri" - }, - "events": { - "description": "The list of event classes to which this receiver is subscribed.", - "type": "array", - "items": { - "type": "string" - } - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - }, - "secrets": { - "type": "array", - "items": { - "$ref": "#/components/schemas/WebhookSecretId" - } - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "description", - "endpoint", - "events", - "id", - "name", - "secrets", - "time_created", - "time_modified" - ] - }, "WebhookCreate": { "description": "Create-time identity-related parameters", "type": "object", @@ -25358,6 +25354,88 @@ "probe" ] }, + "WebhookReceiver": { + "description": "The configuration for a webhook.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "endpoint": { + "description": "The URL that webhook notification requests are sent to.", + "type": "string", + "format": "uri" + }, + "events": { + "description": "The list of event classes to which this receiver is subscribed.", + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookSecretId" + } + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "endpoint", + "events", + "id", + "name", + "secrets", + "time_created", + "time_modified" + ] + }, + "WebhookReceiverResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookReceiver" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, "WebhookSecretCreate": { "type": "object", "properties": { From 6b0ae4ea142983d365b47fbd0f4446a2860c500c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Mar 2025 12:19:08 -0800 Subject: [PATCH 109/168] clippy placation --- nexus/db-queries/src/db/datastore/webhook_rx.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 56b0c3f0324..78f835801b4 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -297,9 +297,9 @@ impl DataStore { // event subscriptions and secrets. let mut result = Vec::with_capacity(receivers.len()); for rx in receivers { - let secrets = self.rx_secret_list_on_conn(rx.id(), &*conn).await?; + let secrets = self.rx_secret_list_on_conn(rx.id(), &conn).await?; let events = - self.rx_subscription_list_on_conn(rx.id(), &*conn).await?; + self.rx_subscription_list_on_conn(rx.id(), &conn).await?; result.push(WebhookReceiverConfig { rx, secrets, events }); } From bcc4215742d77bd415c0e676608edbc215924199 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Mar 2025 15:25:50 -0800 Subject: [PATCH 110/168] reticulating deliveries --- nexus/db-model/src/webhook_delivery.rs | 63 +++-- .../src/db/datastore/webhook_delivery.rs | 243 +++++++----------- nexus/external-api/src/lib.rs | 1 + .../background/tasks/webhook_deliverator.rs | 13 +- .../background/tasks/webhook_dispatcher.rs | 59 ++--- nexus/src/app/webhook.rs | 14 +- nexus/src/external_api/http_entrypoints.rs | 5 +- nexus/types/src/external_api/params.rs | 30 +++ nexus/types/src/external_api/shared.rs | 32 --- nexus/types/src/external_api/views.rs | 129 ++++++++-- nexus/types/src/internal_api/background.rs | 14 +- 11 files changed, 337 insertions(+), 266 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 4e186aac0ac..359f9c31b30 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -11,7 +11,6 @@ use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; use crate::WebhookEventClass; use chrono::{DateTime, TimeDelta, Utc}; -use nexus_types::external_api::shared::WebhookDeliveryState; use nexus_types::external_api::views; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::{ @@ -90,6 +89,7 @@ pub struct WebhookDelivery { pub failed_permanently: bool, pub deliverator_id: Option>, + pub time_delivery_started: Option>, } @@ -144,23 +144,28 @@ impl WebhookDelivery { pub fn to_api_delivery( &self, event_class: WebhookEventClass, - attempt: Option<&WebhookDeliveryAttempt>, + attempts: &[WebhookDeliveryAttempt], ) -> views::WebhookDelivery { - views::WebhookDelivery { + let mut view = views::WebhookDelivery { id: self.id.into_untyped_uuid(), webhook_id: self.rx_id.into(), event_class: event_class.as_str().to_owned(), event_id: self.event_id.into(), - state: attempt - .map(|attempt| attempt.result.into()) - .unwrap_or(WebhookDeliveryState::Pending), + state: if self.failed_permanently { + views::WebhookDeliveryState::Failed + } else if self.time_completed.is_some() { + views::WebhookDeliveryState::Delivered + } else { + views::WebhookDeliveryState::Pending + }, trigger: self.trigger.into(), - response: attempt.and_then(WebhookDeliveryAttempt::response_view), - time_sent: attempt.map(|attempt| attempt.time_created), - attempt: attempt - .map(|attempt| attempt.attempt.0 as usize) - .unwrap_or(1), - } + attempts: attempts + .iter() + .map(views::WebhookDeliveryAttempt::from) + .collect(), + }; + view.attempts.sort_by_key(|a| a.attempt); + view } } @@ -206,31 +211,25 @@ impl WebhookDeliveryAttempt { } } -impl From for WebhookDeliveryState { +impl From<&'_ WebhookDeliveryAttempt> for views::WebhookDeliveryAttempt { + fn from(attempt: &WebhookDeliveryAttempt) -> Self { + let response = attempt.response_view(); + Self { + attempt: attempt.attempt.0 as usize, + result: attempt.result.into(), + time_sent: attempt.time_created, + response, + } + } +} + +impl From for views::WebhookDeliveryAttemptResult { fn from(result: WebhookDeliveryResult) -> Self { match result { WebhookDeliveryResult::FailedHttpError => Self::FailedHttpError, WebhookDeliveryResult::FailedTimeout => Self::FailedTimeout, WebhookDeliveryResult::FailedUnreachable => Self::FailedUnreachable, - WebhookDeliveryResult::Succeeded => Self::Delivered, - } - } -} - -impl WebhookDeliveryResult { - pub const ALL: &'static [Self] = ::VARIANTS; - - pub fn from_api_state(state: WebhookDeliveryState) -> Option { - match state { - WebhookDeliveryState::FailedHttpError => { - Some(Self::FailedHttpError) - } - WebhookDeliveryState::FailedTimeout => Some(Self::FailedTimeout), - WebhookDeliveryState::FailedUnreachable => { - Some(Self::FailedUnreachable) - } - WebhookDeliveryState::Delivered => Some(Self::Succeeded), - WebhookDeliveryState::Pending => None, + WebhookDeliveryResult::Succeeded => Self::Succeeded, } } } diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index b6f5a9b3d6f..9f0a13d96c0 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -12,11 +12,11 @@ use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; -use crate::db::model::WebhookDeliveryResult; use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; -use crate::db::pagination::paginated_multicolumn; +use crate::db::pagination::paginated; +use crate::db::pool::DbConnection; use crate::db::schema; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; @@ -28,12 +28,15 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; use chrono::{DateTime, Utc}; use diesel::prelude::*; -use nexus_types::external_api::shared::WebhookDeliveryState; +use nexus_types::external_api::params::WebhookDeliveryStateFilter; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; -use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookReceiverUuid}; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::WebhookDeliveryUuid; +use omicron_uuid_kinds::WebhookReceiverUuid; use uuid::Uuid; #[derive(Debug, Clone, Eq, PartialEq)] @@ -130,121 +133,61 @@ impl DataStore { .distinct() } - pub async fn webhook_rx_delivery_list_attempts( + pub async fn webhook_rx_delivery_list_on_conn( &self, - opctx: &OpContext, rx_id: &WebhookReceiverUuid, triggers: &'static [WebhookDeliveryTrigger], - states: impl IntoIterator, - pagparams: &DataPageParams<'_, (Uuid, SqlU8)>, - ) -> ListResultVec<( - WebhookDelivery, - Option, - WebhookEventClass, - )> { - let conn = self.pool_connection_authorized(opctx).await?; - // The way we construct this query is a bit complex, so let's break down - // why we do it this way. - // - // We would like to list delivery attempts that are in the provided - // `states`. If a delivery has been attempted at least once, there will - // be a record in the `webhook_delivery_attempts` table including a - // `result` column that contains a `WebhookDeliveryResult`. The - // `WebhookDeliveryResult` SQL enum represents the subset of - // `WebhookDeliveryState`s that are not `Pending` (e.g. the delivery - // attempt has succeeded or failed). On the other hand, if a delivery - // has not yet been attempted, there will be *no* corresponding records - // in `webhook_delivery_attempts`. So, based on whether or not the - // requested list of states includes `Pending`, we either want to select - // all delivery records where there is a corresponding attempt record in - // one of the requested states (if we don't want `Pending` deliveries), - // *OR*, we want to select all deliveries where there is a corresponding - // attempt record in the requested states *and* all delivery records - // where there is no corresponding attempt record (because the delivery - // is pending). Due to diesel being Like That, whether or not we add an - // `OR result IS NULL` clause changes the type of the query being built, - // so we must do that last, and execute the query in one of two `if` - // branches based on whether or not this clause is added. - // - // Additionally, we must paginate this query by both delivery ID and - // attempt number, since we may return multiple attempts of the same - // delivery. Because `paginated_multicolumn` requiers that the - // paginated expression implement `diesel::QuerySource`, we must first - // construct a selection by JOINing the delivery table and the attempt - // table, without applying any filters, pass the joined tables to - // `paginated_multicolumn`, and _then_ filtering the paginated query and - // JOINing again with the `event` table to get the event class as well. - // - // Neither of these weird contortions actually change the resultant SQL - // for the query, but they make the code for cosntructing it a bit - // wacky, so I figured it was worth writing this down for future - // generations. - let mut includes_pending = false; - let states = states - .into_iter() - .filter_map(|state| { - if state == WebhookDeliveryState::Pending { - includes_pending = true; - } - WebhookDeliveryResult::from_api_state(state) - }) - .collect::>(); - - // Join the delivery table with the attempts table. If a delivery has - // not been attempted yet, there will be no attempts, so this is a LEFT - // JOIN. - let query = dsl::webhook_delivery.left_join( - attempt_dsl::webhook_delivery_attempt - .on(dsl::id.eq(attempt_dsl::delivery_id)), - ); - - // Paginate the query, ordering by the delivery ID and then the attempt - // number of the attempt. - let query = paginated_multicolumn( - query, - (dsl::id, attempt_dsl::attempt), - pagparams, - ) - // Select only deliveries that are to the receiver we're interested in, - // and were initiated by the triggers we're interested in. - .filter( - dsl::rx_id - .eq(rx_id.into_untyped_uuid()) - .and(dsl::trigger.eq_any(triggers)), - ) - // Finally, join again with the event table on the delivery's event ID, - // so that we can grab the event class of the event that initiated this delivery. - .inner_join( - event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), - ) - .select(( - WebhookDelivery::as_select(), - Option::::as_select(), - event_dsl::event_class, - )); - - // Before we actually execute the query, add a filter clause to select - // only attempts in the requested states. This branches on - // `includes_pending` as whether or not we care about pending deliveries - // adds an additional clause, changing the type of all the diesel junk - // and preventing us from assigning the query to the same variable in - // both cases, so we just run it immediatel. - if includes_pending { - query - .filter( - attempt_dsl::result - .eq_any(states) - .or(attempt_dsl::result.is_null()), - ) - .load_async(&*conn) - .await - } else { - query - .filter(attempt_dsl::result.eq_any(states)) - .load_async(&*conn) - .await + state_filter: WebhookDeliveryStateFilter, + pagparams: &DataPageParams<'_, Uuid>, + conn: &async_bb8_diesel::Connection, + ) -> ListResultVec<(WebhookDelivery, WebhookEventClass)> { + // Paginate the query, ordered by delivery UUID. + let mut query = paginated(dsl::webhook_delivery, dsl::id, pagparams) + // Select only deliveries that are to the receiver we're interested in, + // and were initiated by the triggers we're interested in. + .filter( + dsl::rx_id + .eq(rx_id.into_untyped_uuid()) + .and(dsl::trigger.eq_any(triggers)), + ) + // Join with the event table on the delivery's event ID, + // so that we can grab the event class of the event that initiated + // this delivery. + .inner_join( + event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), + ); + if !state_filter.include_failed() { + query = query.filter(dsl::failed_permanently.eq(false)); } - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + if !state_filter.include_pending() { + query = query.filter(dsl::time_completed.is_not_null()); + } + if !state_filter.include_succeeded() { + // If successful deliveries are excluded + query = + query.filter(dsl::time_completed.is_null().or( + dsl::failed_permanently.eq(state_filter.include_failed()), + )); + } + query + .select((WebhookDelivery::as_select(), event_dsl::event_class)) + .load_async(conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn webhook_delivery_attempt_list_on_conn( + &self, + delivery_id: &WebhookDeliveryUuid, + conn: &async_bb8_diesel::Connection, + ) -> ListResultVec { + attempt_dsl::webhook_delivery_attempt + .filter( + attempt_dsl::delivery_id.eq(delivery_id.into_untyped_uuid()), + ) + .load_async(conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn webhook_rx_delivery_list_ready( @@ -452,6 +395,7 @@ mod test { use crate::db::pagination::Paginator; use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::expectorate_query_contents; + use dropshot::PaginationOrder; use nexus_types::external_api::params; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; @@ -543,40 +487,41 @@ mod test { "resending an event a second time should create a new delivery" ); - let mut all_deliveries = std::collections::HashSet::new(); - let mut paginator = - Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); - while let Some(p) = paginator.next() { - let deliveries = datastore - .webhook_rx_delivery_list_attempts( - &opctx, - &rx_id, - WebhookDeliveryTrigger::ALL, - std::iter::once(WebhookDeliveryState::Pending), - &p.current_pagparams(), - ) - .await - .unwrap(); - paginator = p.found_batch(&deliveries, &|(d, a, _): &( - WebhookDelivery, - Option, - WebhookEventClass, - )| { - ( - *d.id.as_untyped_uuid(), - a.as_ref() - .map(|attempt| attempt.attempt) - .unwrap_or(SqlU8::new(0)), - ) - }); - all_deliveries - .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); - } - - assert!(all_deliveries.contains(&dispatch1.id)); - assert!(!all_deliveries.contains(&dispatch2.id)); - assert!(all_deliveries.contains(&resend1.id)); - assert!(all_deliveries.contains(&resend2.id)); + todo!("ELIZA PUT THIS PART BACK"); + // let mut all_deliveries = std::collections::HashSet::new(); + // let mut paginator = + // Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); + // while let Some(p) = paginator.next() { + // let deliveries = datastore + // .webhook_rx_delivery_list_attempts( + // &opctx, + // &rx_id, + // WebhookDeliveryTrigger::ALL, + // std::iter::once(WebhookDeliveryState::Pending), + // &p.current_pagparams(), + // ) + // .await + // .unwrap(); + // paginator = p.found_batch(&deliveries, &|(d, a, _): &( + // WebhookDelivery, + // Option, + // WebhookEventClass, + // )| { + // ( + // *d.id.as_untyped_uuid(), + // a.as_ref() + // .map(|attempt| attempt.attempt) + // .unwrap_or(SqlU8::new(0)), + // ) + // }); + // all_deliveries + // .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); + // } + + // assert!(all_deliveries.contains(&dispatch1.id)); + // assert!(!all_deliveries.contains(&dispatch2.id)); + // assert!(all_deliveries.contains(&resend1.id)); + // assert!(all_deliveries.contains(&resend2.id)); db.terminate().await; logctx.cleanup_successful(); diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 69725c27223..093055befd2 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3614,6 +3614,7 @@ pub trait NexusExternalApi { async fn webhook_delivery_list( rqctx: RequestContext, receiver: Query, + state_filter: Query, pagination: Query, ) -> Result>, HttpError>; diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 3c5cea3bedc..99a767fed18 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -13,6 +13,7 @@ use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookDeliveratorStatus; +use nexus_types::internal_api::background::WebhookDeliveryFailure; use nexus_types::internal_api::background::WebhookRxDeliveryStatus; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::Error; @@ -299,8 +300,16 @@ impl WebhookDeliverator { delivery_status.delivered_ok += 1; } else { delivery_status.failed_deliveries.push( - delivery - .to_api_delivery(event_class, Some(&delivery_attempt)), + WebhookDeliveryFailure { + delivery_id, + event_id: delivery.event_id.into(), + attempt: delivery_attempt.attempt.0 as usize, + result: delivery_attempt.result.into(), + response_status: delivery_attempt + .response_status + .map(|status| status as u16), + response_duration: delivery_attempt.response_duration, + }, ); } } diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index cd122ea0b1b..373adbc6909 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -473,34 +473,35 @@ mod test { // webhook_deliverator background task may have activated and might // attempt to deliver the event, making it no longer show up in the // "ready" query. - let mut paginator = Paginator::new(db::datastore::SQL_BATCH_SIZE); - let mut deliveries = Vec::new(); - while let Some(p) = paginator.next() { - let batch = datastore - .webhook_rx_delivery_list_attempts( - &opctx, - &rx_id, - &[WebhookDeliveryTrigger::Event], - WebhookDeliveryState::ALL.iter().copied(), - &p.current_pagparams(), - ) - .await - .unwrap(); - paginator = p.found_batch(&batch, &|(delivery, attempt, _)| { - let id = delivery.id.into_untyped_uuid(); - let attempt = attempt - .as_ref() - .map(|attempt| attempt.attempt) - .unwrap_or_else(|| 0.into()); - (id, attempt) - }); - deliveries.extend(batch); - } - let event = - deliveries.iter().find(|(d, _, _)| d.event_id == event_id.into()); - assert!( - dbg!(event).is_some(), - "delivery entry for dispatched event must exist" - ); + todo!("ELIZA PUT THIS PART BACK"); + // let mut paginator = Paginator::new(db::datastore::SQL_BATCH_SIZE); + // let mut deliveries = Vec::new(); + // while let Some(p) = paginator.next() { + // let batch = datastore + // .webhook_rx_delivery_list_attempts( + // &opctx, + // &rx_id, + // &[WebhookDeliveryTrigger::Event], + // WebhookDeliveryState::ALL.iter().copied(), + // &p.current_pagparams(), + // ) + // .await + // .unwrap(); + // paginator = p.found_batch(&batch, &|(delivery, attempt, _)| { + // let id = delivery.id.into_untyped_uuid(); + // let attempt = attempt + // .as_ref() + // .map(|attempt| attempt.attempt) + // .unwrap_or_else(|| 0.into()); + // (id, attempt) + // }); + // deliveries.extend(batch); + // } + // let event = + // deliveries.iter().find(|(d, _, _)| d.event_id == event_id.into()); + // assert!( + // dbg!(event).is_some(), + // "delivery entry for dispatched event must exist" + // ); } } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 7a987e6244f..f2886d1fee5 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -34,6 +34,7 @@ use nexus_types::external_api::views; use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; @@ -47,6 +48,7 @@ use sha2::Sha256; use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use uuid::Uuid; impl super::Nexus { pub fn webhook_receiver_lookup<'a>( @@ -135,6 +137,16 @@ impl super::Nexus { Ok(event) } + pub async fn webhook_receiver_delivery_list( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + filter: params::WebhookDeliveryStateFilter, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { + todo!() + } + pub async fn webhook_receiver_probe( &self, opctx: &OpContext, @@ -231,7 +243,7 @@ impl super::Nexus { }; Ok(views::WebhookProbeResult { - probe: delivery.to_api_delivery(CLASS, Some(&attempt)), + probe: delivery.to_api_delivery(CLASS, &[attempt]), resends_started, }) } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e25bae6b505..d77b0ddbcde 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7584,8 +7584,9 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_list( rqctx: RequestContext, - _receiver: Query, - _query_params: Query, + receiver: Query, + filter: Query, + pagparams: Query, ) -> Result>, HttpError> { let apictx = rqctx.context(); diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 66fdf3310a0..f27b2fcccef 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2445,6 +2445,36 @@ pub struct WebhookDeliveryPath { pub event_id: Uuid, } +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct WebhookDeliveryStateFilter { + pub pending: Option, + pub failed: Option, + pub succeeded: Option, +} + +impl WebhookDeliveryStateFilter { + pub const ALL: Self = + Self { pending: Some(true), failed: Some(true), succeeded: Some(true) }; + + pub fn include_pending(&self) -> bool { + self.is_all_none() || self.pending == Some(true) + } + + pub fn include_failed(&self) -> bool { + self.is_all_none() || self.failed == Some(true) + } + + pub fn include_succeeded(&self) -> bool { + self.is_all_none() || self.succeeded == Some(true) + } + + fn is_all_none(&self) -> bool { + self.pending.is_none() + && self.failed.is_none() + && self.succeeded.is_none() + } +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookProbe { /// If true, resend all events that have not been delivered successfully if diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs index b1dad30e51f..ae8214a637c 100644 --- a/nexus/types/src/external_api/shared.rs +++ b/nexus/types/src/external_api/shared.rs @@ -510,35 +510,3 @@ impl RelayState { .context("json from relay state string") } } - -/// The state of a webhook delivery attempt. -#[derive( - Copy, - Clone, - Debug, - Eq, - PartialEq, - Deserialize, - Serialize, - JsonSchema, - strum::VariantArray, -)] -#[serde(rename_all = "snake_case")] -pub enum WebhookDeliveryState { - /// The webhook event has not yet been delivered. - Pending, - /// The webhook event has been delivered successfully. - Delivered, - /// A webhook request was sent to the endpoint, and it - /// returned a HTTP error status code indicating an error. - FailedHttpError, - /// The webhook request could not be sent to the receiver endpoint. - FailedUnreachable, - /// A connection to the receiver endpoint was successfully established, but - /// no response was received within the delivery timeout. - FailedTimeout, -} - -impl WebhookDeliveryState { - pub const ALL: &[Self] = ::VARIANTS; -} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 63d4495fdf7..fe1715252a0 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1089,7 +1089,7 @@ pub struct WebhookSecretId { pub id: Uuid, } -/// A delivery attempt for a webhook event. +/// A delivery of a webhook event. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] pub struct WebhookDelivery { /// The UUID of this delivery attempt. @@ -1104,28 +1104,58 @@ pub struct WebhookDelivery { /// The UUID of the event. pub event_id: WebhookEventUuid, - /// The state of the delivery attempt. - pub state: shared::WebhookDeliveryState, - - /// The time at which the webhook delivery was attempted, or `null` if - /// webhook delivery has not yet been attempted (`state` is "pending"). - pub time_sent: Option>, + /// The state of this delivery. + pub state: WebhookDeliveryState, /// Why this delivery was performed. pub trigger: WebhookDeliveryTrigger, - /// Describes the response returned by the receiver endpoint. + /// Individual attempts to deliver this webhook event, and their outcomes. + pub attempts: Vec, +} + +/// The state of a webhook delivery attempt. +#[derive( + Copy, + Clone, + Debug, + Eq, + PartialEq, + Deserialize, + Serialize, + JsonSchema, + strum::VariantArray, +)] +#[serde(rename_all = "snake_case")] +pub enum WebhookDeliveryState { + /// The webhook event has not yet been delivered successfully. /// - /// This is present if the webhook has been delivered successfully, or if the - /// endpoint returned an HTTP error (`state` is "delivered" or - /// "failed_http_error"). This is `null` if the webhook has not yet been - /// delivered, or if the endpoint was unreachable (`state` is "pending" or - /// "failed_unreachable"). - pub response: Option, + /// Either no delivery attempts have yet been performed, or the delivery has + /// failed at least once but has retries remaining. + Pending, + /// The webhook event has been delivered successfully. + Delivered, + /// The webhook delivery attempt has failed permanently and will not be + /// retried again. + Failed, +} - /// Attempt number, starting at 1. If this is a retry of a previous failed - /// delivery, this value indicates that. - pub attempt: usize, +impl WebhookDeliveryState { + pub const ALL: &[Self] = ::VARIANTS; + + pub fn as_str(&self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Delivered => "delivered", + Self::Failed => "failed", + } + } +} + +impl fmt::Display for WebhookDeliveryState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } } /// The reason a webhook event was delivered @@ -1156,6 +1186,70 @@ impl fmt::Display for WebhookDeliveryTrigger { } } +/// An individual delivery attempt for a webhook event. +/// +/// This represents a single HTTP request that was sent to the receiver, and its +/// outcome. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +pub struct WebhookDeliveryAttempt { + /// The time at which the webhook delivery was attempted. + pub time_sent: DateTime, + + /// The attempt number. + pub attempt: usize, + + /// The outcome of this delivery attempt: either the event was delivered + /// successfully, or the request failed for one of several reasons. + pub result: WebhookDeliveryAttemptResult, + + pub response: Option, +} + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Deserialize, + Serialize, + JsonSchema, + strum::VariantArray, +)] +#[serde(rename_all = "snake_case")] +pub enum WebhookDeliveryAttemptResult { + /// The webhook event has been delivered successfully. + Succeeded, + /// A webhook request was sent to the endpoint, and it + /// returned a HTTP error status code indicating an error. + FailedHttpError, + /// The webhook request could not be sent to the receiver endpoint. + FailedUnreachable, + /// A connection to the receiver endpoint was successfully established, but + /// no response was received within the delivery timeout. + FailedTimeout, +} + +impl WebhookDeliveryAttemptResult { + pub const ALL: &[Self] = ::VARIANTS; + pub const ALL_FAILED: &[Self] = + &[Self::FailedHttpError, Self::FailedUnreachable, Self::FailedTimeout]; + + pub fn as_str(&self) -> &'static str { + match self { + Self::Succeeded => "succeeded", + Self::FailedHttpError => "failed_http_error", + Self::FailedTimeout => "failed_timeout", + Self::FailedUnreachable => "failed_unreachable", + } + } +} + +impl fmt::Display for WebhookDeliveryAttemptResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + /// The response received from a webhook receiver endpoint. #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct WebhookDeliveryResponse { @@ -1169,6 +1263,7 @@ pub struct WebhookDeliveryResponse { pub struct WebhookDeliveryId { pub delivery_id: Uuid, } + /// Data describing the result of a webhook liveness probe attempt. #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct WebhookProbeResult { diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 13b85ae981a..e1973e8e5f0 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::external_api::views::WebhookDelivery; +use crate::external_api::views; use chrono::DateTime; use chrono::Utc; use omicron_common::api::external::SemverVersion; @@ -499,11 +499,21 @@ pub struct WebhookRxDeliveryStatus { pub delivered_ok: usize, pub already_delivered: usize, pub in_progress: usize, - pub failed_deliveries: Vec, + pub failed_deliveries: Vec, pub delivery_errors: BTreeMap, pub error: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookDeliveryFailure { + pub delivery_id: WebhookDeliveryUuid, + pub event_id: WebhookEventUuid, + pub attempt: usize, + pub result: views::WebhookDeliveryAttemptResult, + pub response_status: Option, + pub response_duration: Option, +} + /// The status of a `read_only_region_replacement_start` background task /// activation #[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq)] From 6422da02a710ef3dc3df43ce70030740f1a9efe3 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 6 Mar 2025 15:55:34 -0800 Subject: [PATCH 111/168] reticulating deliveries --- .../src/db/datastore/webhook_delivery.rs | 52 +++++++++++-------- .../background/tasks/webhook_dispatcher.rs | 1 - nexus/src/app/webhook.rs | 21 +++++++- nexus/tests/integration_tests/webhooks.rs | 26 ++++++---- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 9f0a13d96c0..dd64662e2bb 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -16,7 +16,6 @@ use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; use crate::db::pagination::paginated; -use crate::db::pool::DbConnection; use crate::db::schema; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; @@ -35,7 +34,6 @@ use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; -use omicron_uuid_kinds::WebhookDeliveryUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use uuid::Uuid; @@ -133,14 +131,19 @@ impl DataStore { .distinct() } - pub async fn webhook_rx_delivery_list_on_conn( + pub async fn webhook_rx_delivery_list( &self, + opctx: &OpContext, rx_id: &WebhookReceiverUuid, triggers: &'static [WebhookDeliveryTrigger], state_filter: WebhookDeliveryStateFilter, pagparams: &DataPageParams<'_, Uuid>, - conn: &async_bb8_diesel::Connection, - ) -> ListResultVec<(WebhookDelivery, WebhookEventClass)> { + ) -> ListResultVec<( + WebhookDelivery, + WebhookEventClass, + Vec, + )> { + let conn = self.pool_connection_authorized(opctx).await?; // Paginate the query, ordered by delivery UUID. let mut query = paginated(dsl::webhook_delivery, dsl::id, pagparams) // Select only deliveries that are to the receiver we're interested in, @@ -169,25 +172,32 @@ impl DataStore { dsl::failed_permanently.eq(state_filter.include_failed()), )); } - query + + let deliveries = query .select((WebhookDelivery::as_select(), event_dsl::event_class)) - .load_async(conn) + .load_async::<(WebhookDelivery, WebhookEventClass)>(&*conn) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) - } + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - pub async fn webhook_delivery_attempt_list_on_conn( - &self, - delivery_id: &WebhookDeliveryUuid, - conn: &async_bb8_diesel::Connection, - ) -> ListResultVec { - attempt_dsl::webhook_delivery_attempt - .filter( - attempt_dsl::delivery_id.eq(delivery_id.into_untyped_uuid()), - ) - .load_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let mut result = Vec::with_capacity(deliveries.len()); + for (delivery, class) in deliveries { + let attempts = attempt_dsl::webhook_delivery_attempt + .filter( + attempt_dsl::delivery_id + .eq(delivery.id.into_untyped_uuid()), + ) + .select(WebhookDeliveryAttempt::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to list attempts for a delivery", + ) + })?; + result.push((delivery, class, attempts)); + } + Ok(result) } pub async fn webhook_rx_delivery_list_ready( diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 373adbc6909..35236c84a67 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -298,7 +298,6 @@ mod test { use diesel::prelude::*; use nexus_db_queries::db; use nexus_test_utils_macros::nexus_test; - use nexus_types::external_api::shared::WebhookDeliveryState; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::SemverVersion; use omicron_uuid_kinds::WebhookEventUuid; diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index f2886d1fee5..aec37b84afd 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -144,7 +144,26 @@ impl super::Nexus { filter: params::WebhookDeliveryStateFilter, pagparams: &DataPageParams<'_, Uuid>, ) -> ListResultVec { - todo!() + let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; + let deliveries = self + .datastore() + .webhook_rx_delivery_list( + opctx, + &authz_rx.id(), + &[ + WebhookDeliveryTrigger::Event, + WebhookDeliveryTrigger::Resend, + ], + filter, + pagparams, + ) + .await? + .into_iter() + .map(|(delivery, class, attempts)| { + delivery.to_api_delivery(class, &attempts) + }) + .collect(); + Ok(deliveries) } pub async fn webhook_receiver_probe( diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 3d794ce0d10..6179762f81c 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -809,10 +809,13 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { mock.assert_async().await; - assert_eq!(probe1.probe.attempt, 1); + assert_eq!( + probe1.probe.attempts[0].result, + views::WebhookDeliveryAttemptResult::FailedTimeout + ); assert_eq!(probe1.probe.event_class, "probe"); assert_eq!(probe1.probe.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe1.probe.state, shared::WebhookDeliveryState::FailedTimeout); + assert_eq!(probe1.probe.state, views::WebhookDeliveryState::Failed); assert_eq!( probe1.resends_started, None, "we did not request events be resent" @@ -844,13 +847,13 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { dbg!(&probe2); mock.assert_async().await; - assert_eq!(probe2.probe.attempt, 1); - assert_eq!(probe2.probe.event_class, "probe"); - assert_eq!(probe2.probe.trigger, views::WebhookDeliveryTrigger::Probe); assert_eq!( - probe2.probe.state, - shared::WebhookDeliveryState::FailedHttpError + probe2.probe.attempts[0].result, + views::WebhookDeliveryAttemptResult::FailedHttpError ); + assert_eq!(probe2.probe.event_class, "probe"); + assert_eq!(probe2.probe.trigger, views::WebhookDeliveryTrigger::Probe); + assert_eq!(probe2.probe.state, views::WebhookDeliveryState::Failed); assert_ne!( probe2.probe.id, probe1.probe.id, "a new delivery ID should be assigned to each probe" @@ -885,10 +888,13 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { .await; dbg!(&probe3); mock.assert_async().await; - assert_eq!(probe3.probe.attempt, 1); + assert_eq!( + probe3.probe.attempts[0].state, + views::WebhookDeliveryAttemptResult::Succeeded + ); assert_eq!(probe3.probe.event_class, "probe"); assert_eq!(probe3.probe.trigger, views::WebhookDeliveryTrigger::Probe); - assert_eq!(probe3.probe.state, shared::WebhookDeliveryState::Delivered); + assert_eq!(probe3.probe.state, views::WebhookDeliveryState::Delivered); assert_ne!( probe3.probe.id, probe1.probe.id, "a new delivery ID should be assigned to each probe" @@ -1039,7 +1045,7 @@ async fn test_probe_resends_failed_deliveries( dbg!(&probe); probe_mock.assert_async().await; probe_mock.delete_async().await; - assert_eq!(probe.probe.state, shared::WebhookDeliveryState::Delivered); + assert_eq!(probe.probe.state, views::WebhookDeliveryState::Delivered); assert_eq!(probe.resends_started, Some(2)); // Both events should be resent. From b72f03c266ce7a281c12e33a949222fb818a8a16 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Mar 2025 13:37:07 -0800 Subject: [PATCH 112/168] bite the bullet and give deliveries a state enum --- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 4 +- nexus/db-model/src/webhook_delivery.rs | 53 ++++--- nexus/db-model/src/webhook_delivery_state.rs | 67 ++++++++ .../src/db/datastore/webhook_delivery.rs | 145 +++++++++-------- .../background/tasks/webhook_deliverator.rs | 6 +- .../background/tasks/webhook_dispatcher.rs | 55 +++---- nexus/src/app/webhook.rs | 53 +++++-- nexus/src/external_api/http_entrypoints.rs | 20 ++- nexus/tests/integration_tests/webhooks.rs | 4 +- nexus/types/src/external_api/params.rs | 27 +++- nexus/types/src/external_api/views.rs | 5 + openapi/nexus.json | 150 +++++++++++++----- schema/crdb/dbinit.sql | 27 +++- schema/crdb/webhooks/README.adoc | 20 +-- schema/crdb/webhooks/up16.sql | 44 +---- schema/crdb/webhooks/up17.sql | 44 ++++- schema/crdb/webhooks/up18.sql | 8 +- schema/crdb/webhooks/up19.sql | 4 +- schema/crdb/webhooks/up20.sql | 7 +- schema/crdb/webhooks/up21.sql | 17 +- schema/crdb/webhooks/up22.sql | 61 ++----- schema/crdb/webhooks/up23.sql | 53 ++++++- schema/crdb/webhooks/up24.sql | 4 + schema/crdb/webhooks/up25.sql | 0 25 files changed, 555 insertions(+), 325 deletions(-) create mode 100644 nexus/db-model/src/webhook_delivery_state.rs create mode 100644 schema/crdb/webhooks/up25.sql diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 14e5afaaf9d..1a1a185c7a0 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -67,6 +67,7 @@ mod switch_port; mod v2p_mapping; mod vmm_state; mod webhook_delivery; +mod webhook_delivery_state; mod webhook_delivery_trigger; mod webhook_event; mod webhook_event_class; @@ -239,6 +240,7 @@ pub use vpc_route::*; pub use vpc_router::*; pub use vpc_subnet::*; pub use webhook_delivery::*; +pub use webhook_delivery_state::*; pub use webhook_delivery_trigger::*; pub use webhook_event::*; pub use webhook_event_class::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 32518ba138d..45cc27fa08c 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2255,7 +2255,7 @@ table! { attempts -> Int2, time_created -> Timestamptz, time_completed -> Nullable, - failed_permanently -> Bool, + state -> crate::WebhookDeliveryStateEnum, deliverator_id -> Nullable, time_delivery_started -> Nullable, } @@ -2272,7 +2272,7 @@ table! { delivery_id -> Uuid, attempt -> Int2, rx_id -> Uuid, - result -> crate::WebhookDeliveryResultEnum, + result -> crate::WebhookDeliveryAttemptResultEnum, response_status -> Nullable, response_duration -> Nullable, time_created -> Timestamptz, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 359f9c31b30..97f3561e23c 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -7,6 +7,7 @@ use crate::schema::{webhook_delivery, webhook_delivery_attempt}; use crate::serde_time_delta::optional_time_delta; use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; +use crate::WebhookDeliveryState; use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; use crate::WebhookEventClass; @@ -23,8 +24,8 @@ use serde::Serialize; impl_enum_type!( #[derive(SqlType, Debug, Clone)] - #[diesel(postgres_type(name = "webhook_delivery_result", schema = "public"))] - pub struct WebhookDeliveryResultEnum; + #[diesel(postgres_type(name = "webhook_delivery_attempt_result", schema = "public"))] + pub struct WebhookDeliveryAttemptResultEnum; #[derive( Copy, @@ -37,8 +38,8 @@ impl_enum_type!( Deserialize, strum::VariantArray, )] - #[diesel(sql_type = WebhookDeliveryResultEnum)] - pub enum WebhookDeliveryResult; + #[diesel(sql_type = WebhookDeliveryAttemptResultEnum)] + pub enum WebhookDeliveryAttemptResult; FailedHttpError => b"failed_http_error" FailedUnreachable => b"failed_unreachable" @@ -86,7 +87,7 @@ pub struct WebhookDelivery { /// or permanently failed. pub time_completed: Option>, - pub failed_permanently: bool, + pub state: WebhookDeliveryState, pub deliverator_id: Option>, @@ -111,7 +112,7 @@ impl WebhookDelivery { time_completed: None, deliverator_id: None, time_delivery_started: None, - failed_permanently: false, + state: WebhookDeliveryState::Pending, } } @@ -131,13 +132,13 @@ impl WebhookDelivery { .into(), rx_id: (*rx_id).into(), trigger: WebhookDeliveryTrigger::Probe, + state: WebhookDeliveryState::Pending, payload: serde_json::json!({}), attempts: SqlU8::new(0), time_created: Utc::now(), time_completed: None, deliverator_id: Some((*deliverator_id).into()), time_delivery_started: Some(Utc::now()), - failed_permanently: false, } } @@ -151,19 +152,17 @@ impl WebhookDelivery { webhook_id: self.rx_id.into(), event_class: event_class.as_str().to_owned(), event_id: self.event_id.into(), - state: if self.failed_permanently { - views::WebhookDeliveryState::Failed - } else if self.time_completed.is_some() { - views::WebhookDeliveryState::Delivered - } else { - views::WebhookDeliveryState::Pending - }, + state: self.state.into(), trigger: self.trigger.into(), attempts: attempts .iter() .map(views::WebhookDeliveryAttempt::from) .collect(), }; + // Make sure attempts are in order; each attempt entry also includes an + // attempt number, which should be used authoritatively to determine the + // ordering of attempts, but it seems nice to also sort the list, + // because we can... view.attempts.sort_by_key(|a| a.attempt); view } @@ -192,7 +191,7 @@ pub struct WebhookDeliveryAttempt { /// `webhook_rx`). pub rx_id: DbTypedUuid, - pub result: WebhookDeliveryResult, + pub result: WebhookDeliveryAttemptResult, pub response_status: Option, @@ -223,13 +222,25 @@ impl From<&'_ WebhookDeliveryAttempt> for views::WebhookDeliveryAttempt { } } -impl From for views::WebhookDeliveryAttemptResult { - fn from(result: WebhookDeliveryResult) -> Self { +impl WebhookDeliveryAttemptResult { + pub fn is_failed(&self) -> bool { + views::WebhookDeliveryAttemptResult::from(*self).is_failed() + } +} + +impl From + for views::WebhookDeliveryAttemptResult +{ + fn from(result: WebhookDeliveryAttemptResult) -> Self { match result { - WebhookDeliveryResult::FailedHttpError => Self::FailedHttpError, - WebhookDeliveryResult::FailedTimeout => Self::FailedTimeout, - WebhookDeliveryResult::FailedUnreachable => Self::FailedUnreachable, - WebhookDeliveryResult::Succeeded => Self::Succeeded, + WebhookDeliveryAttemptResult::FailedHttpError => { + Self::FailedHttpError + } + WebhookDeliveryAttemptResult::FailedTimeout => Self::FailedTimeout, + WebhookDeliveryAttemptResult::FailedUnreachable => { + Self::FailedUnreachable + } + WebhookDeliveryAttemptResult::Succeeded => Self::Succeeded, } } } diff --git a/nexus/db-model/src/webhook_delivery_state.rs b/nexus/db-model/src/webhook_delivery_state.rs new file mode 100644 index 00000000000..c381b4323cf --- /dev/null +++ b/nexus/db-model/src/webhook_delivery_state.rs @@ -0,0 +1,67 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use nexus_types::external_api::views; +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + +impl_enum_type!( + #[derive(SqlType, Debug, Clone)] + #[diesel(postgres_type(name = "webhook_delivery_state", schema = "public"))] + pub struct WebhookDeliveryStateEnum; + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + Serialize, + Deserialize, + AsExpression, + FromSqlRow, + strum::VariantArray, + )] + #[diesel(sql_type = WebhookDeliveryStateEnum)] + #[serde(rename_all = "snake_case")] + pub enum WebhookDeliveryState; + + Pending => b"pending" + Failed => b"failed" + Delivered => b"delivered" + +); + +impl fmt::Display for WebhookDeliveryState { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Forward to the canonical implementation in nexus-types. + views::WebhookDeliveryState::from(*self).fmt(f) + } +} + +impl From for views::WebhookDeliveryState { + fn from(trigger: WebhookDeliveryState) -> Self { + match trigger { + WebhookDeliveryState::Pending => Self::Pending, + WebhookDeliveryState::Failed => Self::Failed, + WebhookDeliveryState::Delivered => Self::Delivered, + } + } +} + +impl From for WebhookDeliveryState { + fn from(trigger: views::WebhookDeliveryState) -> Self { + match trigger { + views::WebhookDeliveryState::Pending => Self::Pending, + views::WebhookDeliveryState::Failed => Self::Failed, + views::WebhookDeliveryState::Delivered => Self::Delivered, + } + } +} + +impl diesel::query_builder::QueryId for WebhookDeliveryStateEnum { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index dd64662e2bb..c540d007e08 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -12,6 +12,8 @@ use crate::db::error::ErrorHandler; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; +use crate::db::model::WebhookDeliveryAttemptResult; +use crate::db::model::WebhookDeliveryState; use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; @@ -27,7 +29,6 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::TimeDelta; use chrono::{DateTime, Utc}; use diesel::prelude::*; -use nexus_types::external_api::params::WebhookDeliveryStateFilter; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -117,7 +118,9 @@ impl DataStore { also_delivery.field(dsl::event_id).eq(event_dsl::id), ) .filter( - also_delivery.field(dsl::failed_permanently).eq(&false), + also_delivery + .field(dsl::state) + .ne(WebhookDeliveryState::Failed), ) .filter( also_delivery @@ -136,7 +139,7 @@ impl DataStore { opctx: &OpContext, rx_id: &WebhookReceiverUuid, triggers: &'static [WebhookDeliveryTrigger], - state_filter: WebhookDeliveryStateFilter, + only_states: Vec, pagparams: &DataPageParams<'_, Uuid>, ) -> ListResultVec<( WebhookDelivery, @@ -159,18 +162,8 @@ impl DataStore { .inner_join( event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), ); - if !state_filter.include_failed() { - query = query.filter(dsl::failed_permanently.eq(false)); - } - if !state_filter.include_pending() { - query = query.filter(dsl::time_completed.is_not_null()); - } - if !state_filter.include_succeeded() { - // If successful deliveries are excluded - query = - query.filter(dsl::time_completed.is_null().or( - dsl::failed_permanently.eq(state_filter.include_failed()), - )); + if !only_states.is_empty() { + query = query.filter(dsl::state.eq_any(only_states)); } let deliveries = query @@ -214,7 +207,12 @@ impl DataStore { // executed synchronously by the probe endpoint, rather than by the // webhook deliverator. .filter(dsl::trigger.ne(WebhookDeliveryTrigger::Probe)) - .filter(dsl::time_completed.is_null()) + // Only select deliveries that are still in progress. + .filter( + dsl::time_completed + .is_null() + .and(dsl::state.eq(WebhookDeliveryState::Pending)), + ) .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) .filter( (dsl::deliverator_id.is_null()).or(dsl::time_delivery_started @@ -266,7 +264,11 @@ impl DataStore { diesel::dsl::now.into_sql::(); let id = delivery.id.into_untyped_uuid(); let updated = diesel::update(dsl::webhook_delivery) - .filter(dsl::time_completed.is_null()) + .filter( + dsl::time_completed + .is_null() + .and(dsl::state.eq(WebhookDeliveryState::Pending)), + ) .filter(dsl::id.eq(id)) .filter( dsl::deliverator_id.is_null().or(dsl::time_delivery_started @@ -333,19 +335,30 @@ impl DataStore { // Has the delivery either completed successfully or exhausted all of // its retry attempts? - let succeeded = - attempt.result == nexus_db_model::WebhookDeliveryResult::Succeeded; - let failed_permanently = !succeeded && *attempt.attempt >= MAX_ATTEMPTS; - let (completed, new_nexus_id) = if succeeded || failed_permanently { - // If the delivery has succeeded or failed permanently, set the - // "time_completed" timestamp to mark it as finished. Also, leave - // the delivering Nexus ID in place to maintain a record of who - // finished the delivery. - (Some(Utc::now()), Some(nexus_id.into_untyped_uuid())) - } else { - // Otherwise, "unlock" the delivery for other nexii. - (None, None) - }; + let new_state = + if attempt.result == WebhookDeliveryAttemptResult::Succeeded { + // The delivery has completed successfully. + WebhookDeliveryState::Delivered + } else if *attempt.attempt >= MAX_ATTEMPTS { + // The delivery attempt failed, and we are out of retries. This + // delivery has failed permanently. + WebhookDeliveryState::Failed + } else { + // This delivery attempt failed, but we still have retries + // remaining, so the delivery remains pending. + WebhookDeliveryState::Pending + }; + let (completed, new_nexus_id) = + if new_state != WebhookDeliveryState::Pending { + // If the delivery has succeeded or failed permanently, set the + // "time_completed" timestamp to mark it as finished. Also, leave + // the delivering Nexus ID in place to maintain a record of who + // finished the delivery. + (Some(Utc::now()), Some(nexus_id.into_untyped_uuid())) + } else { + // Otherwise, "unlock" the delivery for other nexii. + (None, None) + }; let prev_attempts = SqlU8::new((*attempt.attempt) - 1); let UpdateAndQueryResult { status, found } = @@ -354,14 +367,18 @@ impl DataStore { .filter(dsl::deliverator_id.eq(nexus_id.into_untyped_uuid())) .filter(dsl::attempts.eq(prev_attempts)) // Don't mark a delivery as completed if it's already completed! - .filter(dsl::time_completed.is_null()) + .filter( + dsl::time_completed + .is_null() + .and(dsl::state.eq(WebhookDeliveryState::Pending)), + ) .set(( + dsl::state.eq(new_state), dsl::time_completed.eq(completed), // XXX(eliza): hmm this might be racy; we should probably increment this // in place and use it to determine the attempt number? dsl::attempts.eq(attempt.attempt), dsl::deliverator_id.eq(new_nexus_id), - dsl::failed_permanently.eq(failed_permanently), )) .check_if_exists::(delivery.id) .execute_and_check(&conn) @@ -381,7 +398,9 @@ impl DataStore { ))); } - if found.time_completed.is_some() { + if found.time_completed.is_some() + || found.state != WebhookDeliveryState::Pending + { return Err(Error::conflict( "delivery was already marked as completed", )); @@ -405,7 +424,6 @@ mod test { use crate::db::pagination::Paginator; use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::expectorate_query_contents; - use dropshot::PaginationOrder; use nexus_types::external_api::params; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; @@ -497,41 +515,30 @@ mod test { "resending an event a second time should create a new delivery" ); - todo!("ELIZA PUT THIS PART BACK"); - // let mut all_deliveries = std::collections::HashSet::new(); - // let mut paginator = - // Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); - // while let Some(p) = paginator.next() { - // let deliveries = datastore - // .webhook_rx_delivery_list_attempts( - // &opctx, - // &rx_id, - // WebhookDeliveryTrigger::ALL, - // std::iter::once(WebhookDeliveryState::Pending), - // &p.current_pagparams(), - // ) - // .await - // .unwrap(); - // paginator = p.found_batch(&deliveries, &|(d, a, _): &( - // WebhookDelivery, - // Option, - // WebhookEventClass, - // )| { - // ( - // *d.id.as_untyped_uuid(), - // a.as_ref() - // .map(|attempt| attempt.attempt) - // .unwrap_or(SqlU8::new(0)), - // ) - // }); - // all_deliveries - // .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); - // } - - // assert!(all_deliveries.contains(&dispatch1.id)); - // assert!(!all_deliveries.contains(&dispatch2.id)); - // assert!(all_deliveries.contains(&resend1.id)); - // assert!(all_deliveries.contains(&resend2.id)); + let mut all_deliveries = std::collections::HashSet::new(); + let mut paginator = + Paginator::new(crate::db::datastore::SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let deliveries = datastore + .webhook_rx_delivery_list( + &opctx, + &rx_id, + WebhookDeliveryTrigger::ALL, + Vec::new(), + &p.current_pagparams(), + ) + .await + .unwrap(); + paginator = p + .found_batch(&deliveries, &|(d, _, _)| *d.id.as_untyped_uuid()); + all_deliveries + .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); + } + + assert!(all_deliveries.contains(&dispatch1.id)); + assert!(!all_deliveries.contains(&dispatch2.id)); + assert!(all_deliveries.contains(&resend1.id)); + assert!(all_deliveries.contains(&resend2.id)); db.terminate().await; logctx.cleanup_successful(); diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 99a767fed18..aff95e7b4ad 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -7,7 +7,7 @@ use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; -use nexus_db_queries::db::model::WebhookDeliveryResult; +use nexus_db_queries::db::model::WebhookDeliveryAttemptResult; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; @@ -296,7 +296,9 @@ impl WebhookDeliverator { .insert(delivery_id, format!("{MSG}: {e}")); } - if delivery_attempt.result == WebhookDeliveryResult::Succeeded { + if delivery_attempt.result + == WebhookDeliveryAttemptResult::Succeeded + { delivery_status.delivered_ok += 1; } else { delivery_status.failed_deliveries.push( diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 35236c84a67..39a0c9f8cef 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -466,41 +466,34 @@ mod test { // There should now be a delivery entry for the event we published. // - // Use `webhook_rx_delivery_list_attempts` rather than + // Use `webhook_rx_delivery_list` rather than // `webhook_rx_delivery_list_ready`, even though it's a bit more // complex due to requiring pagination. This is because the // webhook_deliverator background task may have activated and might // attempt to deliver the event, making it no longer show up in the // "ready" query. - todo!("ELIZA PUT THIS PART BACK"); - // let mut paginator = Paginator::new(db::datastore::SQL_BATCH_SIZE); - // let mut deliveries = Vec::new(); - // while let Some(p) = paginator.next() { - // let batch = datastore - // .webhook_rx_delivery_list_attempts( - // &opctx, - // &rx_id, - // &[WebhookDeliveryTrigger::Event], - // WebhookDeliveryState::ALL.iter().copied(), - // &p.current_pagparams(), - // ) - // .await - // .unwrap(); - // paginator = p.found_batch(&batch, &|(delivery, attempt, _)| { - // let id = delivery.id.into_untyped_uuid(); - // let attempt = attempt - // .as_ref() - // .map(|attempt| attempt.attempt) - // .unwrap_or_else(|| 0.into()); - // (id, attempt) - // }); - // deliveries.extend(batch); - // } - // let event = - // deliveries.iter().find(|(d, _, _)| d.event_id == event_id.into()); - // assert!( - // dbg!(event).is_some(), - // "delivery entry for dispatched event must exist" - // ); + let mut paginator = Paginator::new(db::datastore::SQL_BATCH_SIZE); + let mut deliveries = Vec::new(); + while let Some(p) = paginator.next() { + let batch = datastore + .webhook_rx_delivery_list( + &opctx, + &rx_id, + &[WebhookDeliveryTrigger::Event], + Vec::new(), + &p.current_pagparams(), + ) + .await + .unwrap(); + paginator = + p.found_batch(&batch, &|(d, _, _)| d.id.into_untyped_uuid()); + deliveries.extend(batch); + } + let event = + deliveries.iter().find(|(d, _, _)| d.event_id == event_id.into()); + assert!( + dbg!(event).is_some(), + "delivery entry for dispatched event must exist" + ); } } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index aec37b84afd..880f2586dbc 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -23,7 +23,8 @@ use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::SqlU8; use nexus_db_queries::db::model::WebhookDelivery; use nexus_db_queries::db::model::WebhookDeliveryAttempt; -use nexus_db_queries::db::model::WebhookDeliveryResult; +use nexus_db_queries::db::model::WebhookDeliveryAttemptResult; +use nexus_db_queries::db::model::WebhookDeliveryState; use nexus_db_queries::db::model::WebhookDeliveryTrigger; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; @@ -145,16 +146,32 @@ impl super::Nexus { pagparams: &DataPageParams<'_, Uuid>, ) -> ListResultVec { let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; + let only_states = if filter.include_all() { + Vec::new() + } else { + let mut states = Vec::with_capacity(3); + if filter.include_failed() { + states.push(WebhookDeliveryState::Failed); + } + if filter.include_pending() { + states.push(WebhookDeliveryState::Pending); + } + if filter.include_delivered() { + states.push(WebhookDeliveryState::Delivered); + } + states + }; let deliveries = self .datastore() .webhook_rx_delivery_list( opctx, &authz_rx.id(), + // No probes; they could have their own list endpoint later... &[ WebhookDeliveryTrigger::Event, WebhookDeliveryTrigger::Resend, ], - filter, + only_states, pagparams, ) .await? @@ -179,7 +196,7 @@ impl super::Nexus { datastore.webhook_rx_secret_list(opctx, &authz_rx).await?; let mut client = ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; - let delivery = WebhookDelivery::new_probe(&rx_id, &self.id); + let mut delivery = WebhookDelivery::new_probe(&rx_id, &self.id); const CLASS: WebhookEventClass = WebhookEventClass::Probe; @@ -201,8 +218,16 @@ impl super::Nexus { } }; + // Update the delivery state based on the result of the probe attempt. + // Otherwise, it will still appear "pending", which is obviously wrong. + delivery.state = if attempt.result.is_failed() { + WebhookDeliveryState::Failed + } else { + WebhookDeliveryState::Delivered + }; + let resends_started = if params.resend - && attempt.result == WebhookDeliveryResult::Succeeded + && attempt.result == WebhookDeliveryAttemptResult::Succeeded { slog::debug!( &opctx.log, @@ -495,16 +520,19 @@ impl<'a> ReceiverClient<'a> { "response_status" => ?status, "response_duration" => ?duration, ); - (WebhookDeliveryResult::FailedHttpError, Some(status)) + ( + WebhookDeliveryAttemptResult::FailedHttpError, + Some(status), + ) } else { let result = if e.is_connect() { - WebhookDeliveryResult::FailedUnreachable + WebhookDeliveryAttemptResult::FailedUnreachable } else if e.is_timeout() { - WebhookDeliveryResult::FailedTimeout + WebhookDeliveryAttemptResult::FailedTimeout } else if e.is_redirect() { - WebhookDeliveryResult::FailedHttpError + WebhookDeliveryAttemptResult::FailedHttpError } else { - WebhookDeliveryResult::FailedUnreachable + WebhookDeliveryAttemptResult::FailedUnreachable }; slog::warn!( &opctx.log, @@ -531,7 +559,7 @@ impl<'a> ReceiverClient<'a> { "response_status" => ?status, "response_duration" => ?duration, ); - (WebhookDeliveryResult::Succeeded, Some(status)) + (WebhookDeliveryAttemptResult::Succeeded, Some(status)) } else { slog::warn!( &opctx.log, @@ -543,7 +571,10 @@ impl<'a> ReceiverClient<'a> { "response_status" => ?status, "response_duration" => ?duration, ); - (WebhookDeliveryResult::FailedHttpError, Some(status)) + ( + WebhookDeliveryAttemptResult::FailedHttpError, + Some(status), + ) } } }; diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index d77b0ddbcde..248413f3933 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7586,7 +7586,7 @@ impl NexusExternalApi for NexusExternalApiImpl { rqctx: RequestContext, receiver: Query, filter: Query, - pagparams: Query, + query: Query, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -7596,10 +7596,20 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook_selector = receiver.into_inner(); + let filter = filter.into_inner(); + let query = query.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let deliveries = nexus + .webhook_receiver_delivery_list(&opctx, rx, filter, &pagparams) + .await?; + + Ok(HttpResponseOk(ScanById::results_page( + &query, + deliveries, + &|_, d| d.id, + )?)) }; apictx .context diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 6179762f81c..65b96cf7a75 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -15,7 +15,7 @@ use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; -use nexus_types::external_api::{params, shared, views}; +use nexus_types::external_api::{params, views}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::NameOrId; use omicron_uuid_kinds::GenericUuid; @@ -889,7 +889,7 @@ async fn test_probe(cptestctx: &ControlPlaneTestContext) { dbg!(&probe3); mock.assert_async().await; assert_eq!( - probe3.probe.attempts[0].state, + probe3.probe.attempts[0].result, views::WebhookDeliveryAttemptResult::Succeeded ); assert_eq!(probe3.probe.event_class, "probe"); diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index f27b2fcccef..0e3e78168c8 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2449,29 +2449,42 @@ pub struct WebhookDeliveryPath { pub struct WebhookDeliveryStateFilter { pub pending: Option, pub failed: Option, - pub succeeded: Option, + pub delivered: Option, +} + +impl Default for WebhookDeliveryStateFilter { + fn default() -> Self { + Self::ALL + } } impl WebhookDeliveryStateFilter { pub const ALL: Self = - Self { pending: Some(true), failed: Some(true), succeeded: Some(true) }; + Self { pending: Some(true), failed: Some(true), delivered: Some(true) }; pub fn include_pending(&self) -> bool { - self.is_all_none() || self.pending == Some(true) + self.pending == Some(true) || self.is_all_none() } pub fn include_failed(&self) -> bool { - self.is_all_none() || self.failed == Some(true) + self.failed == Some(true) || self.is_all_none() + } + + pub fn include_delivered(&self) -> bool { + self.delivered == Some(true) || self.is_all_none() } - pub fn include_succeeded(&self) -> bool { - self.is_all_none() || self.succeeded == Some(true) + pub fn include_all(&self) -> bool { + self.is_all_none() + || (self.pending == Some(true) + && self.failed == Some(true) + && self.delivered == Some(true)) } fn is_all_none(&self) -> bool { self.pending.is_none() && self.failed.is_none() - && self.succeeded.is_none() + && self.delivered.is_none() } } diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index fe1715252a0..ff269170b59 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1242,6 +1242,11 @@ impl WebhookDeliveryAttemptResult { Self::FailedUnreachable => "failed_unreachable", } } + + /// Returns `true` if this `WebhookDeliveryAttemptResult` represents a failure + pub fn is_failed(&self) -> bool { + *self != Self::Succeeded + } } impl fmt::Display for WebhookDeliveryAttemptResult { diff --git a/openapi/nexus.json b/openapi/nexus.json index 7afa9df59a7..719567e7852 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -11883,6 +11883,30 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "delivered", + "schema": { + "nullable": true, + "type": "boolean" + } + }, + { + "in": "query", + "name": "failed", + "schema": { + "nullable": true, + "type": "boolean" + } + }, + { + "in": "query", + "name": "pending", + "schema": { + "nullable": true, + "type": "boolean" + } + }, { "in": "query", "name": "limit", @@ -11917,7 +11941,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookDeliveryResultsPage" + "$ref": "#/components/schemas/WebhookDeliveryAttemptResultsPage" } } } @@ -25133,14 +25157,15 @@ ] }, "WebhookDelivery": { - "description": "A delivery attempt for a webhook event.", + "description": "A delivery of a webhook event.", "type": "object", "properties": { - "attempt": { - "description": "Attempt number, starting at 1. If this is a retry of a previous failed delivery, this value indicates that.", - "type": "integer", - "format": "uint", - "minimum": 0 + "attempts": { + "description": "Individual attempts to deliver this webhook event, and their outcomes.", + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookDeliveryAttempt" + } }, "event_class": { "description": "The event class.", @@ -25159,29 +25184,14 @@ "type": "string", "format": "uuid" }, - "response": { - "nullable": true, - "description": "Describes the response returned by the receiver endpoint.\n\nThis is present if the webhook has been delivered successfully, or if the endpoint returned an HTTP error (`state` is \"delivered\" or \"failed_http_error\"). This is `null` if the webhook has not yet been delivered, or if the endpoint was unreachable (`state` is \"pending\" or \"failed_unreachable\").", - "allOf": [ - { - "$ref": "#/components/schemas/WebhookDeliveryResponse" - } - ] - }, "state": { - "description": "The state of the delivery attempt.", + "description": "The state of this delivery.", "allOf": [ { "$ref": "#/components/schemas/WebhookDeliveryState" } ] }, - "time_sent": { - "nullable": true, - "description": "The time at which the webhook delivery was attempted, or `null` if webhook delivery has not yet been attempted (`state` is \"pending\").", - "type": "string", - "format": "date-time" - }, "trigger": { "description": "Why this delivery was performed.", "allOf": [ @@ -25200,7 +25210,7 @@ } }, "required": [ - "attempt", + "attempts", "event_class", "event_id", "id", @@ -25209,6 +25219,76 @@ "webhook_id" ] }, + "WebhookDeliveryAttempt": { + "description": "An individual delivery attempt for a webhook event.\n\nThis represents a single HTTP request that was sent to the receiver, and its outcome.", + "type": "object", + "properties": { + "attempt": { + "description": "The attempt number.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "response": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDeliveryResponse" + } + ] + }, + "result": { + "description": "The outcome of this delivery attempt: either the event was delivered successfully, or the request failed for one of several reasons.", + "allOf": [ + { + "$ref": "#/components/schemas/WebhookDeliveryAttemptResult" + } + ] + }, + "time_sent": { + "description": "The time at which the webhook delivery was attempted.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "attempt", + "result", + "time_sent" + ] + }, + "WebhookDeliveryAttemptResult": { + "oneOf": [ + { + "description": "The webhook event has been delivered successfully.", + "type": "string", + "enum": [ + "succeeded" + ] + }, + { + "description": "A webhook request was sent to the endpoint, and it returned a HTTP error status code indicating an error.", + "type": "string", + "enum": [ + "failed_http_error" + ] + }, + { + "description": "The webhook request could not be sent to the receiver endpoint.", + "type": "string", + "enum": [ + "failed_unreachable" + ] + }, + { + "description": "A connection to the receiver endpoint was successfully established, but no response was received within the delivery timeout.", + "type": "string", + "enum": [ + "failed_timeout" + ] + } + ] + }, "WebhookDeliveryId": { "type": "object", "properties": { @@ -25243,7 +25323,7 @@ "status" ] }, - "WebhookDeliveryResultsPage": { + "WebhookDeliveryAttemptResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -25268,7 +25348,7 @@ "description": "The state of a webhook delivery attempt.", "oneOf": [ { - "description": "The webhook event has not yet been delivered.", + "description": "The webhook event has not yet been delivered successfully.\n\nEither no delivery attempts have yet been performed, or the delivery has failed at least once but has retries remaining.", "type": "string", "enum": [ "pending" @@ -25282,24 +25362,10 @@ ] }, { - "description": "A webhook request was sent to the endpoint, and it returned a HTTP error status code indicating an error.", + "description": "The webhook delivery attempt has failed permanently and will not be retried again.", "type": "string", "enum": [ - "failed_http_error" - ] - }, - { - "description": "The webhook request could not be sent to the receiver endpoint.", - "type": "string", - "enum": [ - "failed_unreachable" - ] - }, - { - "description": "A connection to the receiver endpoint was successfully established, but no response was received within the delivery timeout.", - "type": "string", - "enum": [ - "failed_timeout" + "failed" ] } ] diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index ee7de41ee22..960d9b8b461 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5165,6 +5165,16 @@ CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( 'probe' ); +-- Describes the state of a webhook delivery +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_state AS ENUM ( + -- This delivery has not yet completed. + 'pending', + -- This delivery has failed. + 'failed', + --- This delivery has completed successfully. + 'delivered' +); + CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- UUID of this delivery. id UUID PRIMARY KEY, @@ -5185,9 +5195,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- If this is set, then this webhook message has either been delivered -- successfully, or is considered permanently failed. time_completed TIMESTAMPTZ, - -- If true, this webhook delivery has failed permanently and is eligible to - -- be resent. - failed_permanently BOOLEAN NOT NULL, + + state omicron.public.webhook_delivery_state NOT NULL, + -- Deliverator coordination bits deliverator_id UUID, time_delivery_started TIMESTAMPTZ, @@ -5195,11 +5205,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), CONSTRAINT active_deliveries_have_started_timestamps CHECK ( (deliverator_id IS NULL) OR ( - (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) + deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL ) ), - CONSTRAINT failed_permanently_only_if_completed CHECK ( - (failed_permanently IS false) OR (failed_permanently AND (time_completed IS NOT NULL)) + CONSTRAINT time_completed_iff_not_pending CHECK ( + (state = 'pending' AND time_completed IS NULL) OR + (state != 'pending' AND time_completed IS NOT NULL) ) ); @@ -5234,7 +5245,7 @@ ON omicron.public.webhook_delivery ( ) WHERE time_completed IS NULL; -CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_attempt_result as ENUM ( -- The delivery attempt failed with an HTTP error. 'failed_http_error', -- The delivery attempt failed because the receiver endpoint was @@ -5257,7 +5268,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - result omicron.public.webhook_delivery_result NOT NULL, + result omicron.public.webhook_delivery_attempt_result NOT NULL, -- A status code > 599 would be Very Surprising, so rather than using an -- INT4 to store a full unsigned 16-bit number in the database, we'll use a -- signed 16-bit integer with a check constraint that it's unsigned. diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index 7155fd742c2..ba7704559ab 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -35,21 +35,23 @@ each receiver. * *Webhook message dispatching and delivery attempts*: ** *Dispatch table*: *** `up15.sql` creates the `omicron.public.webhook_delivery_trigger` enum, which tracks why a webhook delivery was initiated. -*** `up16.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. -*** `up17.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. + +*** `up16.sql` creates the `omicron.public.webhook_delivery_state` enum, representing the current state of a webhook delivery. +*** `up17.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. +*** `up18.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. + This index functions as a `UNIQUE` constraint on the tuple of `(event_id, rx_id)`, but ONLY for rows with `trigger = 'event'`. This ensures that concurrently-executing webhook dispatchers will not create multiple deliveries when dispatching a new event, but permits multiple re-deliveries of an event to be explicitly triggered. -*** `up18.sql` creates an index `lookup_webhook_delivery_dispatched_to_rx` for looking up +*** `up19.sql` creates an index `lookup_webhook_delivery_dispatched_to_rx` for looking up entries in `webhook_delivery` by receiver ID. -*** `up19.sql` creates an index `lookup_webhook_deliveries_for_event` on `webhook_delivery` for looking up deliveries by event UUID. -*** `up20.sql` creates an index `webhook_deliveries_in_flight` for looking up all currently in-flight webhook +*** `up20.sql` creates an index `lookup_webhook_deliveries_for_event` on `webhook_delivery` for looking up deliveries by event UUID. +*** `up21.sql` creates an index `webhook_deliveries_in_flight` for looking up all currently in-flight webhook deliveries (entries where the `time_completed` field has not been set). ** *Delivery attempts*: -*** `up21.sql` creates the enum `omicron.public.webhook_delivery_result`, +*** `up22.sql` creates the enum `omicron.public.webhook_delivery_attempt_result`, representing the potential outcomes of a webhook delivery attempt. -*** `up22.sql` creates the table `omicron.public.webhook_delivery_attempt`, +*** `up23.sql` creates the table `omicron.public.webhook_delivery_attempt`, which records each individual delivery attempt for a webhook delivery in the `webhook_delivery` table. -*** `up23.sql` creates an index `lookup_attempts_for_webhook_delivery` on +*** `up24.sql` creates an index `lookup_attempts_for_webhook_delivery` on `webhook_delivery_attempt`, for looking up the attempts for a given delivery ID. -*** `up24.sql` creates an index `lookup_webhook_delivery_attempts_to_rx` on the `webhook_delivery_attempt` table, for looking up delivery attempts to a given receiver ID. This is primarily used for deleting delivery attempts when a receiver is deleted. +*** `up25.sql` creates an index `lookup_webhook_delivery_attempts_to_rx` on the `webhook_delivery_attempt` table, for looking up delivery attempts to a given receiver ID. This is primarily used for deleting delivery attempts when a receiver is deleted. diff --git a/schema/crdb/webhooks/up16.sql b/schema/crdb/webhooks/up16.sql index 466ad2ae955..9dbcedca319 100644 --- a/schema/crdb/webhooks/up16.sql +++ b/schema/crdb/webhooks/up16.sql @@ -1,37 +1,9 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( - -- UUID of this delivery. - id UUID PRIMARY KEY, - --- UUID of the event (foreign key into `omicron.public.webhook_event`). - event_id UUID NOT NULL, - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - - trigger omicron.public.webhook_delivery_trigger NOT NULL, - - payload JSONB NOT NULL, - - --- Delivery attempt count. Starts at 0. - attempts INT2 NOT NULL, - - time_created TIMESTAMPTZ NOT NULL, - -- If this is set, then this webhook message has either been delivered - -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, - -- If true, this webhook delivery has failed permanently and is eligible to - -- be resent. - failed_permanently BOOLEAN NOT NULL, - -- Deliverator coordination bits - deliverator_id UUID, - time_delivery_started TIMESTAMPTZ, - - CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), - CONSTRAINT active_deliveries_have_started_timestamps CHECK ( - (deliverator_id IS NULL) OR ( - (deliverator_id IS NOT NULL) AND (time_delivery_started IS NOT NULL) - ) - ), - CONSTRAINT failed_permanently_only_if_completed CHECK ( - (failed_permanently IS false) OR (failed_permanently AND (time_completed IS NOT NULL)) - ) +-- Describes the state of a webhook delivery +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_state AS ENUM ( + -- This delivery has not yet completed. + 'pending', + -- This delivery has failed. + 'failed', + --- This delivery has completed successfully. + 'delivered' ); diff --git a/schema/crdb/webhooks/up17.sql b/schema/crdb/webhooks/up17.sql index 2c825749aae..b65d5af7618 100644 --- a/schema/crdb/webhooks/up17.sql +++ b/schema/crdb/webhooks/up17.sql @@ -1,6 +1,38 @@ -CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx -ON omicron.public.webhook_delivery ( - event_id, rx_id -) -WHERE - trigger = 'event'; +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( + -- UUID of this delivery. + id UUID PRIMARY KEY, + --- UUID of the event (foreign key into `omicron.public.webhook_event`). + event_id UUID NOT NULL, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + + trigger omicron.public.webhook_delivery_trigger NOT NULL, + + payload JSONB NOT NULL, + + --- Delivery attempt count. Starts at 0. + attempts INT2 NOT NULL, + + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, + + state omicron.public.webhook_delivery_state NOT NULL, + + -- Deliverator coordination bits + deliverator_id UUID, + time_delivery_started TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), + CONSTRAINT active_deliveries_have_started_timestamps CHECK ( + (deliverator_id IS NULL) OR ( + deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL + ) + ), + CONSTRAINT time_completed_iff_not_pending CHECK ( + (state = 'pending' AND time_completed IS NULL) OR + (state != 'pending' AND time_completed IS NOT NULL) + ) +); diff --git a/schema/crdb/webhooks/up18.sql b/schema/crdb/webhooks/up18.sql index e3ff154e746..2c825749aae 100644 --- a/schema/crdb/webhooks/up18.sql +++ b/schema/crdb/webhooks/up18.sql @@ -1,4 +1,6 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx +CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx ON omicron.public.webhook_delivery ( - rx_id, event_id -); + event_id, rx_id +) +WHERE + trigger = 'event'; diff --git a/schema/crdb/webhooks/up19.sql b/schema/crdb/webhooks/up19.sql index 49709646b92..e3ff154e746 100644 --- a/schema/crdb/webhooks/up19.sql +++ b/schema/crdb/webhooks/up19.sql @@ -1,4 +1,4 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx ON omicron.public.webhook_delivery ( - event_id + rx_id, event_id ); diff --git a/schema/crdb/webhooks/up20.sql b/schema/crdb/webhooks/up20.sql index ba3c68c7e44..49709646b92 100644 --- a/schema/crdb/webhooks/up20.sql +++ b/schema/crdb/webhooks/up20.sql @@ -1,5 +1,4 @@ -CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight +CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event ON omicron.public.webhook_delivery ( - time_created, id -) WHERE - time_completed IS NULL; + event_id +); diff --git a/schema/crdb/webhooks/up21.sql b/schema/crdb/webhooks/up21.sql index 06ce3110625..ba3c68c7e44 100644 --- a/schema/crdb/webhooks/up21.sql +++ b/schema/crdb/webhooks/up21.sql @@ -1,12 +1,5 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_result as ENUM ( - -- The delivery attempt failed with an HTTP error. - 'failed_http_error', - -- The delivery attempt failed because the receiver endpoint was - -- unreachable. - 'failed_unreachable', - --- The delivery attempt connected successfully but no response was received - -- within the timeout. - 'failed_timeout', - -- The delivery attempt succeeded. - 'succeeded' -); +CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight +ON omicron.public.webhook_delivery ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/webhooks/up22.sql b/schema/crdb/webhooks/up22.sql index d3ebfd096b7..41e16a4da73 100644 --- a/schema/crdb/webhooks/up22.sql +++ b/schema/crdb/webhooks/up22.sql @@ -1,51 +1,12 @@ - -CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( - -- Foreign key into `omicron.public.webhook_delivery`. - delivery_id UUID NOT NULL, - -- attempt number. - attempt INT2 NOT NULL, - - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - - result omicron.public.webhook_delivery_result NOT NULL, - -- A status code > 599 would be Very Surprising, so rather than using an - -- INT4 to store a full unsigned 16-bit number in the database, we'll use a - -- signed 16-bit integer with a check constraint that it's unsigned. - response_status INT2, - response_duration INTERVAL, - time_created TIMESTAMPTZ NOT NULL, - - PRIMARY KEY (delivery_id, attempt), - - -- Attempt numbers start at 1 - CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), - - -- Ensure response status codes are not negative. - -- We could be more prescriptive here, and also check that they're >= 100 - -- and <= 599, but some servers may return weird stuff, and we'd like to be - -- able to record that they did that. - CONSTRAINT response_status_is_unsigned CHECK ( - (response_status IS NOT NULL AND response_status >= 0) OR - (response_status IS NULL) - ), - - CONSTRAINT response_iff_not_unreachable CHECK ( - ( - -- If the result is 'succeedeed' or 'failed_http_error', response - -- data must be present. - (result = 'succeeded' OR result = 'failed_http_error') AND ( - response_status IS NOT NULL AND - response_duration IS NOT NULL - ) - ) OR ( - -- If the result is 'failed_unreachable' or 'failed_timeout', no - -- response data is present. - (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( - response_status IS NULL AND - response_duration IS NULL - ) - ) - ) +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_attempt_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + --- The delivery attempt connected successfully but no response was received + -- within the timeout. + 'failed_timeout', + -- The delivery attempt succeeded. + 'succeeded' ); diff --git a/schema/crdb/webhooks/up23.sql b/schema/crdb/webhooks/up23.sql index cd08b4b316b..0a7c228563f 100644 --- a/schema/crdb/webhooks/up23.sql +++ b/schema/crdb/webhooks/up23.sql @@ -1,4 +1,51 @@ -CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery -ON omicron.public.webhook_delivery_attempt ( - delivery_id + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( + -- Foreign key into `omicron.public.webhook_delivery`. + delivery_id UUID NOT NULL, + -- attempt number. + attempt INT2 NOT NULL, + + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + + result omicron.public.webhook_delivery_attempt_result NOT NULL, + -- A status code > 599 would be Very Surprising, so rather than using an + -- INT4 to store a full unsigned 16-bit number in the database, we'll use a + -- signed 16-bit integer with a check constraint that it's unsigned. + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (delivery_id, attempt), + + -- Attempt numbers start at 1 + CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), + + -- Ensure response status codes are not negative. + -- We could be more prescriptive here, and also check that they're >= 100 + -- and <= 599, but some servers may return weird stuff, and we'd like to be + -- able to record that they did that. + CONSTRAINT response_status_is_unsigned CHECK ( + (response_status IS NOT NULL AND response_status >= 0) OR + (response_status IS NULL) + ), + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable' or 'failed_timeout', no + -- response data is present. + (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) ); diff --git a/schema/crdb/webhooks/up24.sql b/schema/crdb/webhooks/up24.sql index e69de29bb2d..cd08b4b316b 100644 --- a/schema/crdb/webhooks/up24.sql +++ b/schema/crdb/webhooks/up24.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery +ON omicron.public.webhook_delivery_attempt ( + delivery_id +); diff --git a/schema/crdb/webhooks/up25.sql b/schema/crdb/webhooks/up25.sql new file mode 100644 index 00000000000..e69de29bb2d From 0e435ed974211aad30b027414feed0870e3f982e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Mar 2025 13:45:45 -0800 Subject: [PATCH 113/168] enums in their own file --- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/webhook_delivery.rs | 50 +------------- .../src/webhook_delivery_attempt_result.rs | 65 +++++++++++++++++++ 3 files changed, 68 insertions(+), 49 deletions(-) create mode 100644 nexus/db-model/src/webhook_delivery_attempt_result.rs diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 1a1a185c7a0..49a6f5b5aa1 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -67,6 +67,7 @@ mod switch_port; mod v2p_mapping; mod vmm_state; mod webhook_delivery; +mod webhook_delivery_attempt_result; mod webhook_delivery_state; mod webhook_delivery_trigger; mod webhook_event; @@ -240,6 +241,7 @@ pub use vpc_route::*; pub use vpc_router::*; pub use vpc_subnet::*; pub use webhook_delivery::*; +pub use webhook_delivery_attempt_result::*; pub use webhook_delivery_state::*; pub use webhook_delivery_trigger::*; pub use webhook_event::*; diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 97f3561e23c..b8566063830 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -2,11 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::impl_enum_type; use crate::schema::{webhook_delivery, webhook_delivery_attempt}; use crate::serde_time_delta::optional_time_delta; use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; +use crate::WebhookDeliveryAttemptResult; use crate::WebhookDeliveryState; use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; @@ -22,31 +22,6 @@ use omicron_uuid_kinds::{ use serde::Deserialize; use serde::Serialize; -impl_enum_type!( - #[derive(SqlType, Debug, Clone)] - #[diesel(postgres_type(name = "webhook_delivery_attempt_result", schema = "public"))] - pub struct WebhookDeliveryAttemptResultEnum; - - #[derive( - Copy, - Clone, - Debug, - PartialEq, - AsExpression, - FromSqlRow, - Serialize, - Deserialize, - strum::VariantArray, - )] - #[diesel(sql_type = WebhookDeliveryAttemptResultEnum)] - pub enum WebhookDeliveryAttemptResult; - - FailedHttpError => b"failed_http_error" - FailedUnreachable => b"failed_unreachable" - FailedTimeout => b"failed_timeout" - Succeeded => b"succeeded" -); - /// A webhook delivery dispatch entry. #[derive( Clone, @@ -221,26 +196,3 @@ impl From<&'_ WebhookDeliveryAttempt> for views::WebhookDeliveryAttempt { } } } - -impl WebhookDeliveryAttemptResult { - pub fn is_failed(&self) -> bool { - views::WebhookDeliveryAttemptResult::from(*self).is_failed() - } -} - -impl From - for views::WebhookDeliveryAttemptResult -{ - fn from(result: WebhookDeliveryAttemptResult) -> Self { - match result { - WebhookDeliveryAttemptResult::FailedHttpError => { - Self::FailedHttpError - } - WebhookDeliveryAttemptResult::FailedTimeout => Self::FailedTimeout, - WebhookDeliveryAttemptResult::FailedUnreachable => { - Self::FailedUnreachable - } - WebhookDeliveryAttemptResult::Succeeded => Self::Succeeded, - } - } -} diff --git a/nexus/db-model/src/webhook_delivery_attempt_result.rs b/nexus/db-model/src/webhook_delivery_attempt_result.rs new file mode 100644 index 00000000000..b75bef35147 --- /dev/null +++ b/nexus/db-model/src/webhook_delivery_attempt_result.rs @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use nexus_types::external_api::views; +use serde::Deserialize; +use serde::Serialize; +use std::fmt; + +impl_enum_type!( + #[derive(SqlType, Debug, Clone)] + #[diesel(postgres_type(name = "webhook_delivery_attempt_result", schema = "public"))] + pub struct WebhookDeliveryAttemptResultEnum; + + #[derive( + Copy, + Clone, + Debug, + PartialEq, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, + strum::VariantArray, + )] + #[diesel(sql_type = WebhookDeliveryAttemptResultEnum)] + pub enum WebhookDeliveryAttemptResult; + + FailedHttpError => b"failed_http_error" + FailedUnreachable => b"failed_unreachable" + FailedTimeout => b"failed_timeout" + Succeeded => b"succeeded" +); + +impl WebhookDeliveryAttemptResult { + pub fn is_failed(&self) -> bool { + // Use canonical implementation from the API type. + views::WebhookDeliveryAttemptResult::from(*self).is_failed() + } +} + +impl fmt::Display for WebhookDeliveryAttemptResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Use canonical format from the API type. + views::WebhookDeliveryAttemptResult::from(*self).fmt(f) + } +} + +impl From + for views::WebhookDeliveryAttemptResult +{ + fn from(result: WebhookDeliveryAttemptResult) -> Self { + match result { + WebhookDeliveryAttemptResult::FailedHttpError => { + Self::FailedHttpError + } + WebhookDeliveryAttemptResult::FailedTimeout => Self::FailedTimeout, + WebhookDeliveryAttemptResult::FailedUnreachable => { + Self::FailedUnreachable + } + WebhookDeliveryAttemptResult::Succeeded => Self::Succeeded, + } + } +} From 7adca879fb2456023e07de1b4e3517ecc8d487c2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Mar 2025 13:59:01 -0800 Subject: [PATCH 114/168] stick nexus IDs in attempt records mainly just for OMDB use, figured it was worth writing this down --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_delivery.rs | 2 ++ .../app/background/tasks/webhook_deliverator.rs | 3 ++- nexus/src/app/webhook.rs | 14 +++++++++++--- schema/crdb/dbinit.sql | 2 ++ schema/crdb/webhooks/up23.sql | 2 ++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 45cc27fa08c..398c2e3e650 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2276,6 +2276,7 @@ table! { response_status -> Nullable, response_duration -> Nullable, time_created -> Timestamptz, + deliverator_id -> Uuid, } } diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index b8566063830..cde4d118df4 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -174,6 +174,8 @@ pub struct WebhookDeliveryAttempt { pub response_duration: Option, pub time_created: DateTime, + + pub deliverator_id: DbTypedUuid, } impl WebhookDeliveryAttempt { diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index aff95e7b4ad..3cd3146090c 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -178,7 +178,8 @@ impl WebhookDeliverator { opctx: &OpContext, WebhookReceiverConfig { rx, secrets, .. }: WebhookReceiverConfig, ) -> Result { - let mut client = ReceiverClient::new(&self.client, secrets, &rx)?; + let mut client = + ReceiverClient::new(&self.client, secrets, &rx, self.nexus_id)?; let deliveries = self .datastore diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 880f2586dbc..b5735084240 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -41,6 +41,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::WebhookDeliveryUuid; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; @@ -194,8 +195,12 @@ impl super::Nexus { let datastore = self.datastore(); let secrets = datastore.webhook_rx_secret_list(opctx, &authz_rx).await?; - let mut client = - ReceiverClient::new(&self.webhook_delivery_client, secrets, &rx)?; + let mut client = ReceiverClient::new( + &self.webhook_delivery_client, + secrets, + &rx, + self.id, + )?; let mut delivery = WebhookDelivery::new_probe(&rx_id, &self.id); const CLASS: WebhookEventClass = WebhookEventClass::Probe; @@ -345,6 +350,7 @@ pub(crate) struct ReceiverClient<'a> { rx: &'a WebhookReceiver, secrets: Vec<(WebhookSecretUuid, Hmac)>, hdr_rx_id: http::HeaderValue, + nexus_id: OmicronZoneUuid, } impl<'a> ReceiverClient<'a> { @@ -352,6 +358,7 @@ impl<'a> ReceiverClient<'a> { client: &'a reqwest::Client, secrets: impl IntoIterator, rx: &'a WebhookReceiver, + nexus_id: OmicronZoneUuid, ) -> Result { let secrets = secrets .into_iter() @@ -368,7 +375,7 @@ impl<'a> ReceiverClient<'a> { } let hdr_rx_id = HeaderValue::try_from(rx.id().to_string()) .expect("UUIDs should always be a valid header value"); - Ok(Self { client, secrets, hdr_rx_id, rx }) + Ok(Self { client, secrets, hdr_rx_id, rx, nexus_id }) } pub(crate) async fn send_delivery_request( @@ -595,6 +602,7 @@ impl<'a> ReceiverClient<'a> { response_status: status.map(|s| s.as_u16() as i16), response_duration, time_created: chrono::Utc::now(), + deliverator_id: self.nexus_id.into(), }) } } diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 960d9b8b461..4cbcc0eb831 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5275,6 +5275,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, + -- UUID of the Nexus who did this delivery attempt. + deliverator_id UUID NOT NULL, PRIMARY KEY (delivery_id, attempt), diff --git a/schema/crdb/webhooks/up23.sql b/schema/crdb/webhooks/up23.sql index 0a7c228563f..4cc19f7db16 100644 --- a/schema/crdb/webhooks/up23.sql +++ b/schema/crdb/webhooks/up23.sql @@ -16,6 +16,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( response_status INT2, response_duration INTERVAL, time_created TIMESTAMPTZ NOT NULL, + -- UUID of the Nexus who did this delivery attempt. + deliverator_id UUID NOT NULL, PRIMARY KEY (delivery_id, attempt), From 7813be8cc3954ee55e6bce93114a1d2e970ae7de Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Mar 2025 14:26:52 -0800 Subject: [PATCH 115/168] make events lookupable --- common/src/api/external/mod.rs | 1 + nexus/auth/src/authz/api_resources.rs | 8 ++++++ nexus/auth/src/authz/oso_generic.rs | 1 + nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/webhook_delivery.rs | 3 +- nexus/db-model/src/webhook_event.rs | 12 ++++---- .../src/db/datastore/webhook_event.rs | 17 ++++++----- nexus/db-queries/src/db/lookup.rs | 19 +++++++++++++ .../background/tasks/webhook_dispatcher.rs | 28 ++++++++++--------- nexus/src/app/webhook.rs | 2 +- schema/crdb/dbinit.sql | 3 ++ schema/crdb/webhooks/up12.sql | 1 + schema/crdb/webhooks/up13.sql | 2 ++ 13 files changed, 67 insertions(+), 31 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 2d6f9178814..be80598e022 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1063,6 +1063,7 @@ pub enum ResourceType { Probe, ProbeNetworkInterface, LldpLinkConfig, + WebhookEvent, WebhookReceiver, WebhookSecret, } diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 2c579320b72..9d3a8bf3f5f 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -1043,6 +1043,14 @@ authz_resource! { polar_snippet = FleetChild, } +authz_resource! { + name = "WebhookEvent", + parent = "Fleet", + primary_key = { uuid_kind = WebhookEventKind }, + roles_allowed = false, + polar_snippet = FleetChild, +} + authz_resource! { name = "WebhookReceiver", parent = "Fleet", diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 6e5e982b06b..1622d3cfa0b 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -160,6 +160,7 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { Sled::init(), TufRepo::init(), TufArtifact::init(), + WebhookEvent::init(), WebhookReceiver::init(), WebhookSecret::init(), Zpool::init(), diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 398c2e3e650..1398dbc71ba 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2238,6 +2238,7 @@ table! { webhook_event (id) { id -> Uuid, time_created -> Timestamptz, + time_modified -> Timestamptz, time_dispatched -> Nullable, event_class -> crate::WebhookEventClassEnum, event -> Jsonb, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index cde4d118df4..923a5fd9cab 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -13,6 +13,7 @@ use crate::WebhookEvent; use crate::WebhookEventClass; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; +use nexus_types::identity::Asset; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::{ OmicronZoneKind, OmicronZoneUuid, WebhookDeliveryKind, WebhookDeliveryUuid, @@ -78,7 +79,7 @@ impl WebhookDelivery { Self { // N.B.: perhaps we ought to use timestamp-based UUIDs for these? id: WebhookDeliveryUuid::new_v4().into(), - event_id: event.id, + event_id: event.id().into(), rx_id: (*rx_id).into(), trigger, payload: event.event.clone(), diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs index ac9d62045f4..f5d5531b72e 100644 --- a/nexus/db-model/src/webhook_event.rs +++ b/nexus/db-model/src/webhook_event.rs @@ -3,10 +3,9 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::schema::webhook_event; -use crate::typed_uuid::DbTypedUuid; use crate::WebhookEventClass; use chrono::{DateTime, Utc}; -use omicron_uuid_kinds::WebhookEventKind; +use db_macros::Asset; use serde::{Deserialize, Serialize}; /// A webhook event. @@ -19,14 +18,13 @@ use serde::{Deserialize, Serialize}; Deserialize, Insertable, PartialEq, + Asset, )] #[diesel(table_name = webhook_event)] +#[asset(uuid_kind = WebhookEventKind)] pub struct WebhookEvent { - /// ID of the event. - pub id: DbTypedUuid, - - /// The time this event was created. - pub time_created: DateTime, + #[diesel(embed)] + pub identity: WebhookEventIdentity, /// The time at which this event was dispatched by creating entries in the /// `webhook_delivery` table. diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index ebb0b7d4727..855f5e1b4df 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -10,6 +10,7 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; +use crate::db::model::WebhookEventIdentity; use crate::db::schema::webhook_event::dsl as event_dsl; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; @@ -28,16 +29,14 @@ impl DataStore { event: serde_json::Value, ) -> CreateResult { let conn = self.pool_connection_authorized(&opctx).await?; - let now = - diesel::dsl::now.into_sql::(); diesel::insert_into(event_dsl::webhook_event) - .values(( - event_dsl::event_class.eq(event_class), - event_dsl::id.eq(id.into_untyped_uuid()), - event_dsl::time_created.eq(now), - event_dsl::event.eq(event), - event_dsl::num_dispatched.eq(0), - )) + .values(WebhookEvent { + identity: WebhookEventIdentity::new(id), + time_dispatched: None, + event_class, + event, + num_dispatched: 0, + }) .returning(WebhookEvent::as_returning()) .get_result_async(&*conn) .await diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index e835ce16415..435e7b06c1e 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -26,6 +26,7 @@ use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::TufArtifactKind; use omicron_uuid_kinds::TufRepoKind; use omicron_uuid_kinds::TypedUuid; +use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use omicron_uuid_kinds::WebhookSecretUuid; use uuid::Uuid; @@ -590,6 +591,14 @@ impl<'a> LookupPath<'a> { { WebhookSecret::PrimaryKey(Root { lookup_root: self }, id) } + + /// Select a resource of type [`WebhookEvent`], identified by its UUID. + pub fn webhook_event_id<'b>(self, id: WebhookEventUuid) -> WebhookEvent<'b> + where + 'a: 'b, + { + WebhookEvent::PrimaryKey(Root { lookup_root: self }, id) + } } /// Represents the head of the selection path for a resource @@ -974,6 +983,16 @@ lookup_resource! { ] } +lookup_resource! { + name = "WebhookEvent", + ancestors = [], + lookup_by_name = false, + soft_deletes = false, + primary_key_columns = [ + { column_name = "id", uuid_kind = WebhookEventKind } + ] +} + // Helpers for unifying the interfaces around images pub enum ImageLookup<'a> { diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 39a0c9f8cef..9454c1d7fc1 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -14,6 +14,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::SQL_BATCH_SIZE; use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; +use nexus_types::identity::Asset; use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ WebhookDispatched, WebhookDispatcherStatus, WebhookGlobStatus, @@ -178,7 +179,7 @@ impl WebhookDispatcher { slog::trace!( &opctx.log, "dispatching webhook event..."; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, ); @@ -197,13 +198,14 @@ impl WebhookDispatcher { slog::error!( &opctx.log, "{MSG}"; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, "error" => &error, ); status.errors.push(format!( "{MSG} {} ({}): {error}", - event.id, event.event_class + event.id(), + event.event_class )); // We weren't able to find receivers for this event, so // *don't* mark it as dispatched --- it's someone else's @@ -216,7 +218,7 @@ impl WebhookDispatcher { slog::trace!(&opctx.log, "webhook receiver is subscribed to event"; "rx_name" => %rx.name(), "rx_id" => ?rx.id(), - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, "glob" => ?sub.glob, ); @@ -233,26 +235,26 @@ impl WebhookDispatcher { Ok(created) => created, Err(error) => { slog::error!(&opctx.log, "failed to insert webhook deliveries"; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, "error" => %error, "num_subscribed" => ?subscribed, ); - status.errors.push(format!("failed to insert {subscribed} webhook deliveries for event {} ({}): {error}", event.id, event.event_class)); + status.errors.push(format!("failed to insert {subscribed} webhook deliveries for event {} ({}): {error}", event.id(), event.event_class)); // We weren't able to create deliveries for this event, so // *don't* mark it as dispatched. continue; } }; status.dispatched.push(WebhookDispatched { - event_id: event.id.into(), + event_id: event.id().into(), subscribed, dispatched, }); slog::debug!( &opctx.log, "dispatched webhook event"; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, "num_subscribed" => subscribed, "num_dispatched" => dispatched, @@ -262,10 +264,10 @@ impl WebhookDispatcher { slog::debug!( &opctx.log, "no webhook receivers subscribed to event"; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, ); - status.no_receivers.push(event.id.into()); + status.no_receivers.push(event.id().into()); 0 }; @@ -273,18 +275,18 @@ impl WebhookDispatcher { .datastore .webhook_event_mark_dispatched( &opctx, - &event.id.into(), + &event.id().into(), subscribed, ) .await { slog::error!(&opctx.log, "failed to mark webhook event as dispatched"; - "event_id" => ?event.id, + "event_id" => ?event.id(), "event_class" => %event.event_class, "error" => %error, "num_subscribed" => subscribed, ); - status.errors.push(format!("failed to mark webhook event {} ({}) as dispatched: {error}", event.id, event.event_class)); + status.errors.push(format!("failed to mark webhook event {} ({}) as dispatched: {error}", event.id(), event.event_class)); } } Ok(()) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index b5735084240..8de7271ace8 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -129,7 +129,7 @@ impl super::Nexus { "enqueued webhook event"; "event_id" => ?id, "event_class" => %event.event_class, - "time_created" => ?event.time_created, + "time_created" => ?event.identity.time_created, ); // Once the event has been isnerted, activate the dispatcher task to diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4cbcc0eb831..60c8bb0c2f5 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5102,6 +5102,7 @@ on omicron.public.webhook_rx_subscription ( CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( id UUID PRIMARY KEY, time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, -- The class of event that this is. event_class omicron.public.webhook_event_class NOT NULL, -- Actual event data. The structure of this depends on the event class. @@ -5125,6 +5126,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( INSERT INTO omicron.public.webhook_event ( id, time_created, + time_modified, event_class, event, time_dispatched, @@ -5133,6 +5135,7 @@ INSERT INTO omicron.public.webhook_event ( -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. '001de000-7768-4000-8000-000000000001', NOW(), + NOW(), 'probe', '{}', -- Pretend to be dispatched so we won't show up in "list events needing diff --git a/schema/crdb/webhooks/up12.sql b/schema/crdb/webhooks/up12.sql index 968bca5d40f..3b2f2450d1f 100644 --- a/schema/crdb/webhooks/up12.sql +++ b/schema/crdb/webhooks/up12.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( id UUID PRIMARY KEY, time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, -- The class of event that this is. event_class omicron.public.webhook_event_class NOT NULL, -- Actual event data. The structure of this depends on the event class. diff --git a/schema/crdb/webhooks/up13.sql b/schema/crdb/webhooks/up13.sql index 5941c172c0d..8184c755660 100644 --- a/schema/crdb/webhooks/up13.sql +++ b/schema/crdb/webhooks/up13.sql @@ -2,6 +2,7 @@ INSERT INTO omicron.public.webhook_event ( id, time_created, + time_modified, event_class, event, time_dispatched, @@ -10,6 +11,7 @@ INSERT INTO omicron.public.webhook_event ( -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. '001de000-7768-4000-8000-000000000001', NOW(), + NOW(), 'probe', '{}', -- Pretend to be dispatched so we won't show up in "list events needing From 8e7c039c739ab7e0ba43da1b28abef70075c9ef5 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 7 Mar 2025 16:38:01 -0800 Subject: [PATCH 116/168] add thingy for checking if an rx is subscribed --- nexus/db-model/src/schema.rs | 3 +- .../db-queries/src/db/datastore/webhook_rx.rs | 146 ++++++++++++++++-- nexus/external-api/src/lib.rs | 2 +- nexus/src/app/webhook.rs | 10 ++ nexus/src/external_api/http_entrypoints.rs | 2 +- nexus/types/src/external_api/params.rs | 2 +- 6 files changed, 144 insertions(+), 21 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 1398dbc71ba..cc7c65ab14a 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2228,7 +2228,8 @@ allow_tables_to_appear_in_same_query!( webhook_receiver, webhook_secret, webhook_rx_subscription, - webhook_rx_event_glob + webhook_rx_event_glob, + webhook_event, ); joinable!(webhook_rx_subscription -> webhook_receiver (rx_id)); joinable!(webhook_secret -> webhook_receiver (rx_id)); diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 78f835801b4..b633e02b91c 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -30,6 +30,7 @@ use crate::db::pagination::paginated_multicolumn; use crate::db::pool::DbConnection; use crate::db::schema::webhook_delivery::dsl as delivery_dsl; use crate::db::schema::webhook_delivery_attempt::dsl as delivery_attempt_dsl; +use crate::db::schema::webhook_event::dsl as event_dsl; use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; @@ -310,6 +311,30 @@ impl DataStore { // Subscriptions // + pub async fn webhook_rx_is_subscribed_to_event( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + authz_event: &authz::WebhookEvent, + ) -> Result { + let conn = self.pool_connection_authorized(opctx).await?; + let event_class = event_dsl::webhook_event + .filter(event_dsl::id.eq(authz_event.id().into_untyped_uuid())) + .select(event_dsl::event_class) + .single_value(); + subscription_dsl::webhook_rx_subscription + .filter( + subscription_dsl::rx_id.eq(authz_rx.id().into_untyped_uuid()), + ) + .filter(subscription_dsl::event_class.nullable().eq(event_class)) + .select(subscription_dsl::rx_id) + .first_async::(&*conn) + .await + .optional() + .map(|x| x.is_some()) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn webhook_rx_subscription_add( &self, opctx: &OpContext, @@ -798,11 +823,55 @@ fn async_insert_error_to_txn( #[cfg(test)] mod test { use super::*; - + use crate::authz; use crate::db::explain::ExplainableAsync; + use crate::db::lookup::LookupPath; use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; + use omicron_uuid_kinds::WebhookEventUuid; + + async fn create_receiver( + datastore: &DataStore, + opctx: &OpContext, + name: &str, + events: Vec, + ) -> WebhookReceiverConfig { + datastore + .webhook_rx_create( + opctx, + params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "it'sa webhook".to_string(), + }, + endpoint: format!("http://{name}").parse().unwrap(), + secrets: vec![name.to_string()], + events, + }, + ) + .await + .expect("cant create ye webhook receiver!!!!") + } + + async fn create_event( + datastore: &DataStore, + opctx: &OpContext, + event_class: WebhookEventClass, + ) -> (authz::WebhookEvent, crate::db::model::WebhookEvent) { + let id = WebhookEventUuid::new_v4(); + datastore + .webhook_event_create(opctx, id, event_class, serde_json::json!({})) + .await + .expect("cant create ye event"); + LookupPath::new(opctx, datastore) + .webhook_event_id(id) + .fetch() + .await + .expect( + "cant get ye event (i just created it, so this is extra weird?)", + ) + } #[tokio::test] async fn test_event_class_globs() { @@ -810,7 +879,7 @@ mod test { let logctx = dev::test_setup_log("test_event_class_globs"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let mut all_rxs = Vec::new(); + let mut all_rxs: Vec = Vec::new(); async fn create_rx( datastore: &DataStore, opctx: &OpContext, @@ -818,21 +887,13 @@ mod test { name: &str, subscription: &str, ) -> WebhookReceiverConfig { - let rx = datastore - .webhook_rx_create( - opctx, - params::WebhookCreate { - identity: IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: String::new(), - }, - endpoint: format!("http://{name}").parse().unwrap(), - secrets: vec![name.to_string()], - events: vec![subscription.to_string()], - }, - ) - .await - .unwrap(); + let rx = create_receiver( + datastore, + opctx, + name, + vec![subscription.to_string()], + ) + .await; all_rxs.push(rx.clone()); rx } @@ -1004,4 +1065,55 @@ mod test { db.terminate().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn test_rx_is_subscribed_to_event() { + // Test setup + let logctx = dev::test_setup_log("test_rx_is_subscribed_to_event"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let rx = create_receiver( + datastore, + opctx, + "webhooked-on-phonics", + vec!["test.*.bar".to_string()], + ) + .await; + + let (authz_rx, _) = LookupPath::new(opctx, datastore) + .webhook_receiver_id(rx.rx.id()) + .fetch() + .await + .expect("cant get ye receiver"); + + let (authz_foo, _) = + create_event(datastore, opctx, WebhookEventClass::TestFoo).await; + let (authz_foo_bar, _) = + create_event(datastore, opctx, WebhookEventClass::TestFooBar).await; + let (authz_quux_bar, _) = + create_event(datastore, opctx, WebhookEventClass::TestQuuxBar) + .await; + + let is_subscribed_foo = datastore + .webhook_rx_is_subscribed_to_event(opctx, &authz_rx, &authz_foo) + .await; + assert_eq!(is_subscribed_foo, Ok(false)); + + let is_subscribed_foo_bar = datastore + .webhook_rx_is_subscribed_to_event(opctx, &authz_rx, &authz_foo_bar) + .await; + assert_eq!(is_subscribed_foo_bar, Ok(true)); + + let is_subscribed_quux_bar = datastore + .webhook_rx_is_subscribed_to_event( + opctx, + &authz_rx, + &authz_quux_bar, + ) + .await; + assert_eq!(is_subscribed_quux_bar, Ok(true)); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 093055befd2..e9de94daacf 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3626,7 +3626,7 @@ pub trait NexusExternalApi { }] async fn webhook_delivery_resend( rqctx: RequestContext, - path_params: Path, + path_params: Path, receiver: Query, ) -> Result, HttpError>; } diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 8de7271ace8..dd5f51082e3 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -74,6 +74,16 @@ impl super::Nexus { } } + pub fn webhook_event_lookup<'a>( + &'a self, + opctx: &'a OpContext, + params::WebhookEventSelector { event_id }: params::WebhookEventSelector, + ) -> LookupResult> { + let event = LookupPath::new(opctx, &self.db_datastore) + .webhook_event_id(WebhookEventUuid::from_untyped_uuid(event_id)); + Ok(event) + } + pub async fn webhook_receiver_list( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 248413f3933..5e2ad9b361b 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7620,7 +7620,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_resend( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, _receiver: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 0e3e78168c8..a400ddf8f0c 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2441,7 +2441,7 @@ pub struct WebhookSecretSelector { } #[derive(Deserialize, JsonSchema)] -pub struct WebhookDeliveryPath { +pub struct WebhookEventSelector { pub event_id: Uuid, } From e147a82d2a5df04241dd116cf14110499b7c1849 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Sat, 8 Mar 2025 16:42:58 -0800 Subject: [PATCH 117/168] implement resend api --- nexus/src/app/webhook.rs | 56 +++++++ nexus/src/external_api/http_entrypoints.rs | 18 ++- nexus/tests/integration_tests/webhooks.rs | 180 +++++++++++++++++++++ 3 files changed, 248 insertions(+), 6 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index dd5f51082e3..6614f7161b9 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -123,6 +123,62 @@ impl super::Nexus { self.datastore().webhook_rx_create(&opctx, params).await } + pub async fn webhook_receiver_event_resend( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + event: lookup::WebhookEvent<'_>, + ) -> CreateResult { + let (authz_rx,) = rx.lookup_for(authz::Action::CreateChild).await?; + let (authz_event, event) = event.fetch().await?; + let datastore = self.datastore(); + + let is_subscribed = datastore + .webhook_rx_is_subscribed_to_event(opctx, &authz_rx, &authz_event) + .await?; + if !is_subscribed { + return Err(Error::invalid_request(format!( + "cannot resend event: receiver is not subscribed to the '{}' \ + event class", + event.event_class, + ))); + } + + let delivery = WebhookDelivery::new( + &event, + &authz_rx.id(), + WebhookDeliveryTrigger::Resend, + ); + let delivery_id = delivery.id.into(); + + if let Err(e) = + datastore.webhook_delivery_create_batch(opctx, vec![delivery]).await + { + slog::error!( + &opctx.log, + "failed to create new delivery to resend webhook event"; + "rx_id" => ?authz_rx.id(), + "event_id" => ?authz_event.id(), + "event_class" => %event.event_class, + "delivery_id" => ?delivery_id, + "error" => %e, + ); + return Err(e); + } + + slog::info!( + &opctx.log, + "resending webhook event"; + "rx_id" => ?authz_rx.id(), + "event_id" => ?authz_event.id(), + "event_class" => %event.event_class, + "delivery_id" => ?delivery_id, + ); + + self.background_tasks.task_webhook_deliverator.activate(); + Ok(delivery_id) + } + pub async fn webhook_event_publish( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 5e2ad9b361b..558f8dec7ae 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7620,8 +7620,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_delivery_resend( rqctx: RequestContext, - _path_params: Path, - _receiver: Query, + path_params: Path, + receiver: Query, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { @@ -7630,10 +7630,16 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let event_selector = path_params.into_inner(); + let webhook_selector = receiver.into_inner(); + let event = nexus.webhook_event_lookup(&opctx, event_selector)?; + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + let delivery_id = + nexus.webhook_receiver_event_resend(&opctx, rx, event).await?; + + Ok(HttpResponseCreated(views::WebhookDeliveryId { + delivery_id: delivery_id.into_untyped_uuid(), + })) }; apictx .context diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 65b96cf7a75..746d565f931 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -30,6 +30,7 @@ type ControlPlaneTestContext = const RECEIVERS_BASE_PATH: &str = "/v1/webhooks/receivers"; const SECRETS_BASE_PATH: &str = "/v1/webhooks/secrets"; +const DELIVERIES_BASE_PATH: &str = "/v1/webhooks/deliveries"; async fn webhook_create( ctx: &ControlPlaneTestContext, @@ -96,6 +97,56 @@ async fn webhook_secrets_get( .unwrap() } +fn resend_url( + webhook_name_or_id: impl Into, + event_id: WebhookEventUuid, +) -> String { + let rx = webhook_name_or_id.into(); + format!("{DELIVERIES_BASE_PATH}/{event_id}/resend?receiver={rx}") +} +async fn webhook_delivery_resend( + client: &ClientTestContext, + webhook_name_or_id: impl Into, + event_id: WebhookEventUuid, +) -> views::WebhookDeliveryId { + let req = RequestBuilder::new( + client, + http::Method::POST, + &resend_url(webhook_name_or_id, event_id), + ) + .body::(None) + .expect_status(Some(http::StatusCode::CREATED)); + NexusRequest::new(req) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +async fn webhook_delivery_resend_error( + client: &ClientTestContext, + webhook_name_or_id: impl Into, + event_id: WebhookEventUuid, + status: http::StatusCode, +) -> dropshot::HttpErrorResponseBody { + let req = RequestBuilder::new( + client, + http::Method::POST, + &resend_url(webhook_name_or_id, event_id), + ) + .body::(None) + .expect_status(Some(status)); + NexusRequest::new(req) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + fn my_great_webhook_params( mock: &httpmock::MockServer, ) -> params::WebhookCreate { @@ -1054,3 +1105,132 @@ async fn test_probe_resends_failed_deliveries( ); mock.assert_calls_async(2).await; } + +#[nexus_test] +async fn test_api_resends_failed_deliveries( + cptestctx: &ControlPlaneTestContext, +) { + let nexus = cptestctx.server.server_context().nexus.clone(); + let internal_client = &cptestctx.internal_client; + let client = &cptestctx.external_client; + let server = httpmock::MockServer::start_async().await; + + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + // Create a webhook receiver. + let webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&webhook); + + let event1_id = WebhookEventUuid::new_v4(); + let event2_id = WebhookEventUuid::new_v4(); + let body = serde_json::json!({ + "event_class": "test.foo", + "event_id": event1_id, + "data": { + "hello_world": true, + } + }) + .to_string(); + let mock = { + let webhook = webhook.clone(); + let body = body.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", event1_id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(500); + }) + .await + }; + + // Publish an event + let event1 = nexus + .webhook_event_publish( + &opctx, + event1_id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello_world": true}), + ) + .await + .expect("event should be published successfully"); + dbg!(event1); + + // Publish another event that our receiver is not subscribed to. + let event2 = nexus + .webhook_event_publish( + &opctx, + event2_id, + WebhookEventClass::TestQuuxBar, + serde_json::json!({"hello_world": true}), + ) + .await + .expect("event should be published successfully"); + dbg!(event2); + + dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + tokio::time::sleep(std::time::Duration::from_secs(11)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + tokio::time::sleep(std::time::Duration::from_secs(22)).await; + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + + mock.assert_calls_async(3).await; + mock.delete_async().await; + + let mock = { + let webhook = webhook.clone(); + let body = body.clone(); + server + .mock_async(move |when, then| { + when.method(POST) + .header("x-oxide-event-class", "test.foo") + .header("x-oxide-event-id", event1_id.to_string()) + .and(is_valid_for_webhook(&webhook)) + .is_true(signature_verifies( + webhook.secrets[0].id, + MY_COOL_SECRET.as_bytes().to_vec(), + )) + .json_body_includes(body); + then.status(200); + }) + .await + }; + + // Try to resend event 1. + let delivery = + webhook_delivery_resend(client, webhook.identity.id, event1_id).await; + dbg!(delivery); + + // Try to resend event 2. This should fail, as the receiver is not + // subscribed to this event class. + let error = webhook_delivery_resend_error( + client, + webhook.identity.id, + event2_id, + http::StatusCode::BAD_REQUEST, + ) + .await; + dbg!(error); + + dbg!( + activate_background_task(internal_client, "webhook_deliverator").await + ); + mock.assert_calls_async(1).await; +} From 0bcd73fd98a5dddf6b884241c51fb42afd326afb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 10:02:23 -0700 Subject: [PATCH 118/168] rename receiver update endpoint for consistency --- nexus/external-api/output/nexus_tags.txt | 2 +- nexus/external-api/src/lib.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 2 +- openapi/nexus.json | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 34a163f1c7a..b468f95f423 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -274,11 +274,11 @@ webhook_receiver_create POST /v1/webhooks/receivers webhook_receiver_delete DELETE /v1/webhooks/receivers/{receiver} webhook_receiver_list GET /v1/webhooks/receivers webhook_receiver_probe POST /v1/webhooks/receivers/{receiver}/probe +webhook_receiver_update PUT /v1/webhooks/receivers/{receiver} webhook_receiver_view GET /v1/webhooks/receivers/{receiver} webhook_secrets_add POST /v1/webhooks/secrets webhook_secrets_delete DELETE /v1/webhooks/secrets/{secret_id} webhook_secrets_list GET /v1/webhooks/secrets -webhook_update PUT /v1/webhooks/receivers/{receiver} API operations found with tag "vpcs" OPERATION ID METHOD URL PATH diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index e9de94daacf..8f4f39dfd9b 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3539,7 +3539,7 @@ pub trait NexusExternalApi { path = "/v1/webhooks/receivers/{receiver}", tags = ["system/webhooks"], }] - async fn webhook_update( + async fn webhook_receiver_update( rqctx: RequestContext, path_params: Path, params: TypedBody, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 558f8dec7ae..339fce1d175 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7413,7 +7413,7 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_update( + async fn webhook_receiver_update( rqctx: RequestContext, _path_params: Path, _params: TypedBody, diff --git a/openapi/nexus.json b/openapi/nexus.json index 719567e7852..b58de47ef94 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -11941,7 +11941,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookDeliveryAttemptResultsPage" + "$ref": "#/components/schemas/WebhookDeliveryResultsPage" } } } @@ -12240,7 +12240,7 @@ "system/webhooks" ], "summary": "Update the configuration of an existing webhook receiver.", - "operationId": "webhook_update", + "operationId": "webhook_receiver_update", "parameters": [ { "in": "path", @@ -25323,7 +25323,7 @@ "status" ] }, - "WebhookDeliveryAttemptResultsPage": { + "WebhookDeliveryResultsPage": { "description": "A single page of results", "type": "object", "properties": { From 4891e3f0facb0c29b16eafbc3c87ef0998f593b3 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 10:37:51 -0700 Subject: [PATCH 119/168] make webhook receiver names unique --- .../db-queries/src/db/datastore/webhook_rx.rs | 14 ++++- nexus/tests/integration_tests/webhooks.rs | 28 ++++++++- schema/crdb/dbinit.sql | 8 ++- schema/crdb/webhooks/README.adoc | 49 ++++++++------- schema/crdb/webhooks/up02.sql | 2 +- schema/crdb/webhooks/up03.sql | 20 ++---- schema/crdb/webhooks/up04.sql | 19 ++++-- schema/crdb/webhooks/up05.sql | 18 ++---- schema/crdb/webhooks/up06.sql | 26 ++++---- schema/crdb/webhooks/up07.sql | 19 +++++- schema/crdb/webhooks/up08.sql | 6 +- schema/crdb/webhooks/up09.sql | 22 +------ schema/crdb/webhooks/up10.sql | 24 +++++-- schema/crdb/webhooks/up11.sql | 10 +-- schema/crdb/webhooks/up12.sql | 26 ++------ schema/crdb/webhooks/up13.sql | 43 ++++++------- schema/crdb/webhooks/up14.sql | 25 ++++++-- schema/crdb/webhooks/up15.sql | 13 ++-- schema/crdb/webhooks/up16.sql | 16 ++--- schema/crdb/webhooks/up17.sql | 45 +++---------- schema/crdb/webhooks/up18.sql | 44 +++++++++++-- schema/crdb/webhooks/up19.sql | 8 ++- schema/crdb/webhooks/up20.sql | 4 +- schema/crdb/webhooks/up21.sql | 7 +-- schema/crdb/webhooks/up22.sql | 17 ++--- schema/crdb/webhooks/up23.sql | 63 ++++--------------- schema/crdb/webhooks/up24.sql | 55 +++++++++++++++- schema/crdb/webhooks/up25.sql | 4 ++ schema/crdb/webhooks/up26.sql | 0 29 files changed, 343 insertions(+), 292 deletions(-) create mode 100644 schema/crdb/webhooks/up26.sql diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index b633e02b91c..a7458710cce 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -92,12 +92,24 @@ impl DataStore { let subscriptions = subscriptions.clone(); let secret_keys = secrets.clone(); let err = err.clone(); + let name = identity.name.clone(); async move { let rx = diesel::insert_into(rx_dsl::webhook_receiver) .values(receiver) .returning(WebhookReceiver::as_returning()) .get_result_async(&conn) - .await?; + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::WebhookReceiver, + name.as_str(), + ), + ) + }) + })?; for subscription in subscriptions { self.add_subscription_on_conn( opctx, diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 746d565f931..a06f20741d0 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -268,7 +268,7 @@ fn signature_verifies( } #[nexus_test] -async fn test_webhook_get(cptestctx: &ControlPlaneTestContext) { +async fn test_webhook_receiver_get(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; let server = httpmock::MockServer::start_async().await; @@ -289,6 +289,32 @@ async fn test_webhook_get(cptestctx: &ControlPlaneTestContext) { assert_eq!(created_webhook, webhook_view); } +#[nexus_test] +async fn test_webhook_receiver_names_are_unique( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + let server = httpmock::MockServer::start_async().await; + + // Create a webhook receiver. + let created_webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&created_webhook); + + let error = resource_helpers::object_create_error( + &client, + RECEIVERS_BASE_PATH, + &my_great_webhook_params(&server), + http::StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + dbg!(&error).message, + "already exists: webhook-receiver \"my-great-webhook\"" + ); +} + #[nexus_test] async fn test_event_delivery(cptestctx: &ControlPlaneTestContext) { let nexus = cptestctx.server.server_context().nexus.clone(); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 60c8bb0c2f5..fa2d1417bac 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4981,11 +4981,17 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( endpoint STRING(512) NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rxs_by_id +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_receiver_by_id ON omicron.public.webhook_receiver (id) WHERE time_deleted IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_receiver_by_name +ON omicron.public.webhook_receiver ( + name +) WHERE + time_deleted IS NULL; + CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( -- ID of this secret. id UUID PRIMARY KEY, diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index ba7704559ab..5d98d8689c1 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -9,49 +9,50 @@ The individual transactions in this upgrade do the following: * *Webhook receivers*: ** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores the receiver endpoints that receive webhook events. -** `up02.sql` creates the `lookup_webhook_rxs_by_id` index on that table, for listing non-deleted webhook receivers. +** `up02.sql` creates the `lookup_webhook_rx_by_id` index on that table, for listing non-deleted webhook receivers. +** `up03.sql` creates the `lookup_webhook_rx_by_name` index on that table, for looking up receivers by name (and ensuring names are unique across all non-deleted receivers). ** *Secrets*: -*** `up03.sql` creates the `omicron.public.webhook_secret` table, which +*** `up04.sql` creates the `omicron.public.webhook_secret` table, which associates webhook receivers with secret keys and their IDs. -*** `up04.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, +*** `up05.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, for looking up all secrets associated with a receiver. * *Event classes, subscriptions, and globbing*: -** `up05.sql` creates the `omicron.public.webhook_event_class` enum type +** `up06.sql` creates the `omicron.public.webhook_event_class` enum type ** *Globs*: -*** `up06.sql` creates the `omicron.public.webhook_rx_event_glob` table, which contains any subscriptions created by a receiver that have glob patterns. This table is used when generating exact subscription from globs. -*** `up07.sql` creates the `lookup_webhook_event_globs_for_rx` indes on `webhook_rx_event_glob`, for looking up all globs belonging to a receiver by ID. -*** `up08.sql` creates the `lookup_webhook_event_globs_by_schema_version` index on `webhook_rx_event_glob`, for searching for globs with outdated schema versions. +*** `up07.sql` creates the `omicron.public.webhook_rx_event_glob` table, which contains any subscriptions created by a receiver that have glob patterns. This table is used when generating exact subscription from globs. +*** `up08.sql` creates the `lookup_webhook_event_globs_for_rx` indes on `webhook_rx_event_glob`, for looking up all globs belonging to a receiver by ID. +*** `up09.sql` creates the `lookup_webhook_event_globs_by_schema_version` index on `webhook_rx_event_glob`, for searching for globs with outdated schema versions. ** *Subscriptions*: -*** `up09.sql` creates the `omicron.public.webhook_rx_subscription` table, which tracks the event classes that a receiver is subscribed to. If a row in this table represents a subscription that was generated by a glob, this table also references the glob record. -*** `up10.sql` creates the `lookup_webhook_rxs_for_event_class` index on `webhook_rx_subscription`, for listing all the receivers subscribed to an event class -*** `up11.sql` creates the `lookup_exact_subscriptions_for_webhook_rx` index, for looking up the exact subscriptions (not globs) for a receiver by ID. This is used along with `lookup_webhook_event_globs_for_rx` index when listing the user-provided event class strings. +*** `up10.sql` creates the `omicron.public.webhook_rx_subscription` table, which tracks the event classes that a receiver is subscribed to. If a row in this table represents a subscription that was generated by a glob, this table also references the glob record. +*** `up11.sql` creates the `lookup_webhook_rxs_for_event_class` index on `webhook_rx_subscription`, for listing all the receivers subscribed to an event class +*** `up12.sql` creates the `lookup_exact_subscriptions_for_webhook_rx` index, for looking up the exact subscriptions (not globs) for a receiver by ID. This is used along with `lookup_webhook_event_globs_for_rx` index when listing the user-provided event class strings. * *Webhook events*: -** `up12.sql` creates the `omicron.public.webhook_event` table, which contains the +** `up13.sql` creates the `omicron.public.webhook_event` table, which contains the values of actual webhook events. The dispatcher operates on entries in this queue, dispatching the event to receivers and generating the payload for each receiver. -** `up13.sql` inserts the singleton row in `webhook_event` used for liveness probes. This singleton exists so that delivery records for liveness probes can have event UUIDs that point at a real entry in `webhook_event`, without requiring a new event entry to be created for each probe. -** `up14.sql` creates the `lookup_undispatched_webhook_events` index on `webhook_event` for looking up webhook messages which have not yet been dispatched, and ordering by their creation times. +** `up14.sql` inserts the singleton row in `webhook_event` used for liveness probes. This singleton exists so that delivery records for liveness probes can have event UUIDs that point at a real entry in `webhook_event`, without requiring a new event entry to be created for each probe. +** `up15.sql` creates the `lookup_undispatched_webhook_events` index on `webhook_event` for looking up webhook messages which have not yet been dispatched, and ordering by their creation times. * *Webhook message dispatching and delivery attempts*: ** *Dispatch table*: -*** `up15.sql` creates the `omicron.public.webhook_delivery_trigger` enum, which tracks why a webhook delivery was initiated. +*** `up16.sql` creates the `omicron.public.webhook_delivery_trigger` enum, which tracks why a webhook delivery was initiated. -*** `up16.sql` creates the `omicron.public.webhook_delivery_state` enum, representing the current state of a webhook delivery. -*** `up17.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. -*** `up18.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. +*** `up1s7ql` creates the `omicron.public.webhook_delivery_state` enum, representing the current state of a webhook delivery. +*** `up18.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. +*** `up19.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. + This index functions as a `UNIQUE` constraint on the tuple of `(event_id, rx_id)`, but ONLY for rows with `trigger = 'event'`. This ensures that concurrently-executing webhook dispatchers will not create multiple deliveries when dispatching a new event, but permits multiple re-deliveries of an event to be explicitly triggered. -*** `up19.sql` creates an index `lookup_webhook_delivery_dispatched_to_rx` for looking up +*** `up20.sql` creates an index `lookup_webhook_delivery_dispatched_to_rx` for looking up entries in `webhook_delivery` by receiver ID. -*** `up20.sql` creates an index `lookup_webhook_deliveries_for_event` on `webhook_delivery` for looking up deliveries by event UUID. -*** `up21.sql` creates an index `webhook_deliveries_in_flight` for looking up all currently in-flight webhook +*** `up21.sql` creates an index `lookup_webhook_deliveries_for_event` on `webhook_delivery` for looking up deliveries by event UUID. +*** `up22.sql` creates an index `webhook_deliveries_in_flight` for looking up all currently in-flight webhook deliveries (entries where the `time_completed` field has not been set). ** *Delivery attempts*: -*** `up22.sql` creates the enum `omicron.public.webhook_delivery_attempt_result`, +*** `up23.sql` creates the enum `omicron.public.webhook_delivery_attempt_result`, representing the potential outcomes of a webhook delivery attempt. -*** `up23.sql` creates the table `omicron.public.webhook_delivery_attempt`, +*** `up24.sql` creates the table `omicron.public.webhook_delivery_attempt`, which records each individual delivery attempt for a webhook delivery in the `webhook_delivery` table. -*** `up24.sql` creates an index `lookup_attempts_for_webhook_delivery` on +*** `up25.sql` creates an index `lookup_attempts_for_webhook_delivery` on `webhook_delivery_attempt`, for looking up the attempts for a given delivery ID. -*** `up25.sql` creates an index `lookup_webhook_delivery_attempts_to_rx` on the `webhook_delivery_attempt` table, for looking up delivery attempts to a given receiver ID. This is primarily used for deleting delivery attempts when a receiver is deleted. +*** `up26.sql` creates an index `lookup_webhook_delivery_attempts_to_rx` on the `webhook_delivery_attempt` table, for looking up delivery attempts to a given receiver ID. This is primarily used for deleting delivery attempts when a receiver is deleted. diff --git a/schema/crdb/webhooks/up02.sql b/schema/crdb/webhooks/up02.sql index 618b6f64677..f2f069346c2 100644 --- a/schema/crdb/webhooks/up02.sql +++ b/schema/crdb/webhooks/up02.sql @@ -1,4 +1,4 @@ -CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rxs_by_id +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rx_by_id ON omicron.public.webhook_receiver (id) WHERE time_deleted IS NULL; diff --git a/schema/crdb/webhooks/up03.sql b/schema/crdb/webhooks/up03.sql index 4f46ece10a2..863bf0eaf6b 100644 --- a/schema/crdb/webhooks/up03.sql +++ b/schema/crdb/webhooks/up03.sql @@ -1,14 +1,6 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( - -- ID of this secret. - id UUID PRIMARY KEY, - time_created TIMESTAMPTZ NOT NULL, - -- N.B. that this will always be equal to `time_created` for secrets, as - -- they are never modified once created. - time_modified TIMESTAMPTZ NOT NULL, - time_deleted TIMESTAMPTZ, - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - -- Secret value. - secret STRING(512) NOT NULL -); + +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rx_by_name +ON omicron.public.webhook_receiver ( + name +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/webhooks/up04.sql b/schema/crdb/webhooks/up04.sql index ab263e8705e..4f46ece10a2 100644 --- a/schema/crdb/webhooks/up04.sql +++ b/schema/crdb/webhooks/up04.sql @@ -1,5 +1,14 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx -ON omicron.public.webhook_secret ( - rx_id -) WHERE - time_deleted IS NULL; +CREATE TABLE IF NOT EXISTS omicron.public.webhook_secret ( + -- ID of this secret. + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + -- N.B. that this will always be equal to `time_created` for secrets, as + -- they are never modified once created. + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- Secret value. + secret STRING(512) NOT NULL +); diff --git a/schema/crdb/webhooks/up05.sql b/schema/crdb/webhooks/up05.sql index e00847f3cee..ab263e8705e 100644 --- a/schema/crdb/webhooks/up05.sql +++ b/schema/crdb/webhooks/up05.sql @@ -1,13 +1,5 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( - -- Liveness probes, which are technically not real events, but, you know... - 'probe', - -- Test classes used to test globbing. - -- - -- These are not publicly exposed. - 'test.foo', - 'test.foo.bar', - 'test.foo.baz', - 'test.quux.bar', - 'test.quux.bar.baz' - -- Add new event classes here! -); +CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx +ON omicron.public.webhook_secret ( + rx_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/webhooks/up06.sql b/schema/crdb/webhooks/up06.sql index 8aa06b10f56..e00847f3cee 100644 --- a/schema/crdb/webhooks/up06.sql +++ b/schema/crdb/webhooks/up06.sql @@ -1,17 +1,13 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - -- An event class glob to which this receiver is subscribed. - glob STRING(512) NOT NULL, - -- Regex used when evaluating this filter against concrete event classes. - regex STRING(512) NOT NULL, - time_created TIMESTAMPTZ NOT NULL, - -- The database schema version at which this glob was last expanded. +CREATE TYPE IF NOT EXISTS omicron.public.webhook_event_class AS ENUM ( + -- Liveness probes, which are technically not real events, but, you know... + 'probe', + -- Test classes used to test globbing. -- - -- This is used to detect when a glob must be re-processed to generate exact - -- subscriptions on schema changes. - schema_version STRING(64) NOT NULL, - - PRIMARY KEY (rx_id, glob) + -- These are not publicly exposed. + 'test.foo', + 'test.foo.bar', + 'test.foo.baz', + 'test.quux.bar', + 'test.quux.bar.baz' + -- Add new event classes here! ); diff --git a/schema/crdb/webhooks/up07.sql b/schema/crdb/webhooks/up07.sql index c8dfef06f4e..8aa06b10f56 100644 --- a/schema/crdb/webhooks/up07.sql +++ b/schema/crdb/webhooks/up07.sql @@ -1,4 +1,17 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_for_rx -ON omicron.public.webhook_rx_event_glob ( - rx_id +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_event_glob ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class glob to which this receiver is subscribed. + glob STRING(512) NOT NULL, + -- Regex used when evaluating this filter against concrete event classes. + regex STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- The database schema version at which this glob was last expanded. + -- + -- This is used to detect when a glob must be re-processed to generate exact + -- subscriptions on schema changes. + schema_version STRING(64) NOT NULL, + + PRIMARY KEY (rx_id, glob) ); diff --git a/schema/crdb/webhooks/up08.sql b/schema/crdb/webhooks/up08.sql index be6363dae71..c8dfef06f4e 100644 --- a/schema/crdb/webhooks/up08.sql +++ b/schema/crdb/webhooks/up08.sql @@ -1,2 +1,4 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_by_schema_version -ON omicron.public.webhook_rx_event_glob (schema_version); +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_for_rx +ON omicron.public.webhook_rx_event_glob ( + rx_id +); diff --git a/schema/crdb/webhooks/up09.sql b/schema/crdb/webhooks/up09.sql index ffd6a0e8278..be6363dae71 100644 --- a/schema/crdb/webhooks/up09.sql +++ b/schema/crdb/webhooks/up09.sql @@ -1,20 +1,2 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - -- An event class to which the receiver is subscribed. - event_class omicron.public.webhook_event_class NOT NULL, - -- If this subscription is a concrete instantiation of a glob pattern, the - -- value of the glob that created it (and, a foreign key into - -- `webhook_rx_event_glob`). If the receiver is subscribed to this exact - -- event class, then this is NULL. - -- - -- This is used when deleting a glob subscription, as it is necessary to - -- delete any concrete subscriptions to individual event classes matching - -- that glob. - glob STRING(512), - - time_created TIMESTAMPTZ NOT NULL, - - PRIMARY KEY (rx_id, event_class) -); +CREATE INDEX IF NOT EXISTS lookup_webhook_event_globs_by_schema_version +ON omicron.public.webhook_rx_event_glob (schema_version); diff --git a/schema/crdb/webhooks/up10.sql b/schema/crdb/webhooks/up10.sql index 8f79b29b605..ffd6a0e8278 100644 --- a/schema/crdb/webhooks/up10.sql +++ b/schema/crdb/webhooks/up10.sql @@ -1,6 +1,20 @@ --- Look up all webhook receivers subscribed to an event class. This is used by --- the dispatcher to determine who is interested in a particular event. -CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event_class -ON omicron.public.webhook_rx_subscription ( - event_class +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which the receiver is subscribed. + event_class omicron.public.webhook_event_class NOT NULL, + -- If this subscription is a concrete instantiation of a glob pattern, the + -- value of the glob that created it (and, a foreign key into + -- `webhook_rx_event_glob`). If the receiver is subscribed to this exact + -- event class, then this is NULL. + -- + -- This is used when deleting a glob subscription, as it is necessary to + -- delete any concrete subscriptions to individual event classes matching + -- that glob. + glob STRING(512), + + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) ); diff --git a/schema/crdb/webhooks/up11.sql b/schema/crdb/webhooks/up11.sql index 0b3b6becd0e..8f79b29b605 100644 --- a/schema/crdb/webhooks/up11.sql +++ b/schema/crdb/webhooks/up11.sql @@ -1,4 +1,6 @@ -CREATE INDEX IF NOT EXISTS lookup_exact_subscriptions_for_webhook_rx -on omicron.public.webhook_rx_subscription ( - rx_id -) WHERE glob IS NULL; +-- Look up all webhook receivers subscribed to an event class. This is used by +-- the dispatcher to determine who is interested in a particular event. +CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event_class +ON omicron.public.webhook_rx_subscription ( + event_class +); diff --git a/schema/crdb/webhooks/up12.sql b/schema/crdb/webhooks/up12.sql index 3b2f2450d1f..0b3b6becd0e 100644 --- a/schema/crdb/webhooks/up12.sql +++ b/schema/crdb/webhooks/up12.sql @@ -1,22 +1,4 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( - id UUID PRIMARY KEY, - time_created TIMESTAMPTZ NOT NULL, - time_modified TIMESTAMPTZ NOT NULL, - -- The class of event that this is. - event_class omicron.public.webhook_event_class NOT NULL, - -- Actual event data. The structure of this depends on the event class. - event JSONB NOT NULL, - - -- Set when dispatch entries have been created for this event. - time_dispatched TIMESTAMPTZ, - -- The number of receivers that this event was dispatched to. - num_dispatched INT8 NOT NULL, - - CONSTRAINT time_dispatched_set_if_dispatched CHECK ( - (num_dispatched = 0) OR (time_dispatched IS NOT NULL) - ), - - CONSTRAINT num_dispatched_is_positive CHECK ( - (num_dispatched >= 0) - ) -); +CREATE INDEX IF NOT EXISTS lookup_exact_subscriptions_for_webhook_rx +on omicron.public.webhook_rx_subscription ( + rx_id +) WHERE glob IS NULL; diff --git a/schema/crdb/webhooks/up13.sql b/schema/crdb/webhooks/up13.sql index 8184c755660..3b2f2450d1f 100644 --- a/schema/crdb/webhooks/up13.sql +++ b/schema/crdb/webhooks/up13.sql @@ -1,21 +1,22 @@ --- Singleton probe event -INSERT INTO omicron.public.webhook_event ( - id, - time_created, - time_modified, - event_class, - event, - time_dispatched, - num_dispatched -) VALUES ( - -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. - '001de000-7768-4000-8000-000000000001', - NOW(), - NOW(), - 'probe', - '{}', - -- Pretend to be dispatched so we won't show up in "list events needing - -- dispatch" queries - NOW(), - 0 -) ON CONFLICT DO NOTHING; +CREATE TABLE IF NOT EXISTS omicron.public.webhook_event ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + -- The class of event that this is. + event_class omicron.public.webhook_event_class NOT NULL, + -- Actual event data. The structure of this depends on the event class. + event JSONB NOT NULL, + + -- Set when dispatch entries have been created for this event. + time_dispatched TIMESTAMPTZ, + -- The number of receivers that this event was dispatched to. + num_dispatched INT8 NOT NULL, + + CONSTRAINT time_dispatched_set_if_dispatched CHECK ( + (num_dispatched = 0) OR (time_dispatched IS NOT NULL) + ), + + CONSTRAINT num_dispatched_is_positive CHECK ( + (num_dispatched >= 0) + ) +); diff --git a/schema/crdb/webhooks/up14.sql b/schema/crdb/webhooks/up14.sql index 81542e30acd..8184c755660 100644 --- a/schema/crdb/webhooks/up14.sql +++ b/schema/crdb/webhooks/up14.sql @@ -1,4 +1,21 @@ -CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_events -ON omicron.public.webhook_event ( - id, time_created -) WHERE time_dispatched IS NULL; +-- Singleton probe event +INSERT INTO omicron.public.webhook_event ( + id, + time_created, + time_modified, + event_class, + event, + time_dispatched, + num_dispatched +) VALUES ( + -- NOTE: this UUID is duplicated in nexus_db_model::webhook_event. + '001de000-7768-4000-8000-000000000001', + NOW(), + NOW(), + 'probe', + '{}', + -- Pretend to be dispatched so we won't show up in "list events needing + -- dispatch" queries + NOW(), + 0 +) ON CONFLICT DO NOTHING; diff --git a/schema/crdb/webhooks/up15.sql b/schema/crdb/webhooks/up15.sql index 562c6abce9d..81542e30acd 100644 --- a/schema/crdb/webhooks/up15.sql +++ b/schema/crdb/webhooks/up15.sql @@ -1,9 +1,4 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( - -- This delivery was triggered by the event being dispatched. - 'event', - -- This delivery was triggered by an explicit call to the webhook event - -- resend API. - 'resend', - --- This delivery is a liveness probe. - 'probe' -); +CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_events +ON omicron.public.webhook_event ( + id, time_created +) WHERE time_dispatched IS NULL; diff --git a/schema/crdb/webhooks/up16.sql b/schema/crdb/webhooks/up16.sql index 9dbcedca319..562c6abce9d 100644 --- a/schema/crdb/webhooks/up16.sql +++ b/schema/crdb/webhooks/up16.sql @@ -1,9 +1,9 @@ --- Describes the state of a webhook delivery -CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_state AS ENUM ( - -- This delivery has not yet completed. - 'pending', - -- This delivery has failed. - 'failed', - --- This delivery has completed successfully. - 'delivered' +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_trigger AS ENUM ( + -- This delivery was triggered by the event being dispatched. + 'event', + -- This delivery was triggered by an explicit call to the webhook event + -- resend API. + 'resend', + --- This delivery is a liveness probe. + 'probe' ); diff --git a/schema/crdb/webhooks/up17.sql b/schema/crdb/webhooks/up17.sql index b65d5af7618..9dbcedca319 100644 --- a/schema/crdb/webhooks/up17.sql +++ b/schema/crdb/webhooks/up17.sql @@ -1,38 +1,9 @@ -CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( - -- UUID of this delivery. - id UUID PRIMARY KEY, - --- UUID of the event (foreign key into `omicron.public.webhook_event`). - event_id UUID NOT NULL, - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - - trigger omicron.public.webhook_delivery_trigger NOT NULL, - - payload JSONB NOT NULL, - - --- Delivery attempt count. Starts at 0. - attempts INT2 NOT NULL, - - time_created TIMESTAMPTZ NOT NULL, - -- If this is set, then this webhook message has either been delivered - -- successfully, or is considered permanently failed. - time_completed TIMESTAMPTZ, - - state omicron.public.webhook_delivery_state NOT NULL, - - -- Deliverator coordination bits - deliverator_id UUID, - time_delivery_started TIMESTAMPTZ, - - CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), - CONSTRAINT active_deliveries_have_started_timestamps CHECK ( - (deliverator_id IS NULL) OR ( - deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL - ) - ), - CONSTRAINT time_completed_iff_not_pending CHECK ( - (state = 'pending' AND time_completed IS NULL) OR - (state != 'pending' AND time_completed IS NOT NULL) - ) +-- Describes the state of a webhook delivery +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_state AS ENUM ( + -- This delivery has not yet completed. + 'pending', + -- This delivery has failed. + 'failed', + --- This delivery has completed successfully. + 'delivered' ); diff --git a/schema/crdb/webhooks/up18.sql b/schema/crdb/webhooks/up18.sql index 2c825749aae..b65d5af7618 100644 --- a/schema/crdb/webhooks/up18.sql +++ b/schema/crdb/webhooks/up18.sql @@ -1,6 +1,38 @@ -CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx -ON omicron.public.webhook_delivery ( - event_id, rx_id -) -WHERE - trigger = 'event'; +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( + -- UUID of this delivery. + id UUID PRIMARY KEY, + --- UUID of the event (foreign key into `omicron.public.webhook_event`). + event_id UUID NOT NULL, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + + trigger omicron.public.webhook_delivery_trigger NOT NULL, + + payload JSONB NOT NULL, + + --- Delivery attempt count. Starts at 0. + attempts INT2 NOT NULL, + + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, + + state omicron.public.webhook_delivery_state NOT NULL, + + -- Deliverator coordination bits + deliverator_id UUID, + time_delivery_started TIMESTAMPTZ, + + CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), + CONSTRAINT active_deliveries_have_started_timestamps CHECK ( + (deliverator_id IS NULL) OR ( + deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL + ) + ), + CONSTRAINT time_completed_iff_not_pending CHECK ( + (state = 'pending' AND time_completed IS NULL) OR + (state != 'pending' AND time_completed IS NOT NULL) + ) +); diff --git a/schema/crdb/webhooks/up19.sql b/schema/crdb/webhooks/up19.sql index e3ff154e746..2c825749aae 100644 --- a/schema/crdb/webhooks/up19.sql +++ b/schema/crdb/webhooks/up19.sql @@ -1,4 +1,6 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx +CREATE UNIQUE INDEX IF NOT EXISTS one_webhook_event_dispatch_per_rx ON omicron.public.webhook_delivery ( - rx_id, event_id -); + event_id, rx_id +) +WHERE + trigger = 'event'; diff --git a/schema/crdb/webhooks/up20.sql b/schema/crdb/webhooks/up20.sql index 49709646b92..e3ff154e746 100644 --- a/schema/crdb/webhooks/up20.sql +++ b/schema/crdb/webhooks/up20.sql @@ -1,4 +1,4 @@ -CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx ON omicron.public.webhook_delivery ( - event_id + rx_id, event_id ); diff --git a/schema/crdb/webhooks/up21.sql b/schema/crdb/webhooks/up21.sql index ba3c68c7e44..49709646b92 100644 --- a/schema/crdb/webhooks/up21.sql +++ b/schema/crdb/webhooks/up21.sql @@ -1,5 +1,4 @@ -CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight +CREATE INDEX IF NOT EXISTS lookup_webhook_deliveries_for_event ON omicron.public.webhook_delivery ( - time_created, id -) WHERE - time_completed IS NULL; + event_id +); diff --git a/schema/crdb/webhooks/up22.sql b/schema/crdb/webhooks/up22.sql index 41e16a4da73..ba3c68c7e44 100644 --- a/schema/crdb/webhooks/up22.sql +++ b/schema/crdb/webhooks/up22.sql @@ -1,12 +1,5 @@ -CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_attempt_result as ENUM ( - -- The delivery attempt failed with an HTTP error. - 'failed_http_error', - -- The delivery attempt failed because the receiver endpoint was - -- unreachable. - 'failed_unreachable', - --- The delivery attempt connected successfully but no response was received - -- within the timeout. - 'failed_timeout', - -- The delivery attempt succeeded. - 'succeeded' -); +CREATE INDEX IF NOT EXISTS webhook_deliveries_in_flight +ON omicron.public.webhook_delivery ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/webhooks/up23.sql b/schema/crdb/webhooks/up23.sql index 4cc19f7db16..41e16a4da73 100644 --- a/schema/crdb/webhooks/up23.sql +++ b/schema/crdb/webhooks/up23.sql @@ -1,53 +1,12 @@ - -CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( - -- Foreign key into `omicron.public.webhook_delivery`. - delivery_id UUID NOT NULL, - -- attempt number. - attempt INT2 NOT NULL, - - -- UUID of the webhook receiver (foreign key into - -- `omicron.public.webhook_rx`) - rx_id UUID NOT NULL, - - result omicron.public.webhook_delivery_attempt_result NOT NULL, - -- A status code > 599 would be Very Surprising, so rather than using an - -- INT4 to store a full unsigned 16-bit number in the database, we'll use a - -- signed 16-bit integer with a check constraint that it's unsigned. - response_status INT2, - response_duration INTERVAL, - time_created TIMESTAMPTZ NOT NULL, - -- UUID of the Nexus who did this delivery attempt. - deliverator_id UUID NOT NULL, - - PRIMARY KEY (delivery_id, attempt), - - -- Attempt numbers start at 1 - CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), - - -- Ensure response status codes are not negative. - -- We could be more prescriptive here, and also check that they're >= 100 - -- and <= 599, but some servers may return weird stuff, and we'd like to be - -- able to record that they did that. - CONSTRAINT response_status_is_unsigned CHECK ( - (response_status IS NOT NULL AND response_status >= 0) OR - (response_status IS NULL) - ), - - CONSTRAINT response_iff_not_unreachable CHECK ( - ( - -- If the result is 'succeedeed' or 'failed_http_error', response - -- data must be present. - (result = 'succeeded' OR result = 'failed_http_error') AND ( - response_status IS NOT NULL AND - response_duration IS NOT NULL - ) - ) OR ( - -- If the result is 'failed_unreachable' or 'failed_timeout', no - -- response data is present. - (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( - response_status IS NULL AND - response_duration IS NULL - ) - ) - ) +CREATE TYPE IF NOT EXISTS omicron.public.webhook_delivery_attempt_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + --- The delivery attempt connected successfully but no response was received + -- within the timeout. + 'failed_timeout', + -- The delivery attempt succeeded. + 'succeeded' ); diff --git a/schema/crdb/webhooks/up24.sql b/schema/crdb/webhooks/up24.sql index cd08b4b316b..4cc19f7db16 100644 --- a/schema/crdb/webhooks/up24.sql +++ b/schema/crdb/webhooks/up24.sql @@ -1,4 +1,53 @@ -CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery -ON omicron.public.webhook_delivery_attempt ( - delivery_id + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery_attempt ( + -- Foreign key into `omicron.public.webhook_delivery`. + delivery_id UUID NOT NULL, + -- attempt number. + attempt INT2 NOT NULL, + + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + + result omicron.public.webhook_delivery_attempt_result NOT NULL, + -- A status code > 599 would be Very Surprising, so rather than using an + -- INT4 to store a full unsigned 16-bit number in the database, we'll use a + -- signed 16-bit integer with a check constraint that it's unsigned. + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + -- UUID of the Nexus who did this delivery attempt. + deliverator_id UUID NOT NULL, + + PRIMARY KEY (delivery_id, attempt), + + -- Attempt numbers start at 1 + CONSTRAINT attempts_start_at_1 CHECK (attempt >= 1), + + -- Ensure response status codes are not negative. + -- We could be more prescriptive here, and also check that they're >= 100 + -- and <= 599, but some servers may return weird stuff, and we'd like to be + -- able to record that they did that. + CONSTRAINT response_status_is_unsigned CHECK ( + (response_status IS NOT NULL AND response_status >= 0) OR + (response_status IS NULL) + ), + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable' or 'failed_timeout', no + -- response data is present. + (result = 'failed_unreachable' OR result = 'failed_timeout') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) ); diff --git a/schema/crdb/webhooks/up25.sql b/schema/crdb/webhooks/up25.sql index e69de29bb2d..cd08b4b316b 100644 --- a/schema/crdb/webhooks/up25.sql +++ b/schema/crdb/webhooks/up25.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_attempts_for_webhook_delivery +ON omicron.public.webhook_delivery_attempt ( + delivery_id +); diff --git a/schema/crdb/webhooks/up26.sql b/schema/crdb/webhooks/up26.sql new file mode 100644 index 00000000000..e69de29bb2d From e0ffe665220ab9b6fedfd65146a34ea9bcd11ffa Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 11:14:41 -0700 Subject: [PATCH 120/168] fix webhook-receiver-delete concurrency control --- .../db-queries/src/db/datastore/webhook_rx.rs | 200 +++++++++++------- nexus/src/app/webhook.rs | 10 + nexus/src/external_api/http_entrypoints.rs | 3 +- nexus/tests/integration_tests/webhooks.rs | 29 ++- 4 files changed, 165 insertions(+), 77 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index a7458710cce..393fa356a19 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -184,90 +184,142 @@ impl DataStore { &self, opctx: &OpContext, authz_rx: &authz::WebhookReceiver, + db_rx: &WebhookReceiver, ) -> DeleteResult { opctx.authorize(authz::Action::Delete, authz_rx).await?; - let conn = self.pool_connection_authorized(opctx).await?; let rx_id = authz_rx.id().into_untyped_uuid(); - // First, mark the webhook receiver record as deleted. - diesel::update(rx_dsl::webhook_receiver) - .filter(rx_dsl::id.eq(rx_id)) - .filter(rx_dsl::time_deleted.is_null()) - .set(( - rx_dsl::time_deleted.eq(chrono::Utc::now()), - rx_dsl::rcgen.eq(rx_dsl::rcgen + 1), - )) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context("failed to mark receiver as deleted") - })?; - // Now, delete the webhook's secrets. - let secrets_deleted = diesel::delete(secret_dsl::webhook_secret) - .filter(secret_dsl::rx_id.eq(rx_id)) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context("failed to delete secrets") - })?; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("webhook_rx_delete").transaction( + &conn, + |conn| { + let err = err.clone(); + async move { + let now = chrono::Utc::now(); + // Delete the webhook's secrets. + let secrets_deleted = + diesel::delete(secret_dsl::webhook_secret) + .filter(secret_dsl::rx_id.eq(rx_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + .internal_context( + "failed to delete secrets", + ) + }) + })?; - // Delete subscriptions and globs. - let exact_subscriptions_deleted = - diesel::delete(subscription_dsl::webhook_rx_subscription) - .filter(subscription_dsl::rx_id.eq(rx_id)) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context( - "failed to delete exact subscriptions", - ) - })?; + // Delete subscriptions and globs. + let exact_subscriptions_deleted = diesel::delete( + subscription_dsl::webhook_rx_subscription, + ) + .filter(subscription_dsl::rx_id.eq(rx_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to delete exact subscriptions", + ) + }) + })?; - let globs_deleted = diesel::delete(glob_dsl::webhook_rx_event_glob) - .filter(glob_dsl::rx_id.eq(rx_id)) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context("failed to delete globs") - })?; + let globs_deleted = + diesel::delete(glob_dsl::webhook_rx_event_glob) + .filter(glob_dsl::rx_id.eq(rx_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + .internal_context("failed to delete globs") + }) + })?; - let deliveries_deleted = diesel::delete(delivery_dsl::webhook_delivery) - .filter(delivery_dsl::rx_id.eq(rx_id)) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context("failed to delete delivery records") - })?; + let deliveries_deleted = + diesel::delete(delivery_dsl::webhook_delivery) + .filter(delivery_dsl::rx_id.eq(rx_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + .internal_context( + "failed to delete delivery records", + ) + }) + })?; - let delivery_attempts_deleted = - diesel::delete(delivery_attempt_dsl::webhook_delivery_attempt) - .filter(delivery_attempt_dsl::rx_id.eq(rx_id)) - .execute_async(&*conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - .internal_context( - "failed to delete delivery attempt records", - ) - })?; + let delivery_attempts_deleted = diesel::delete( + delivery_attempt_dsl::webhook_delivery_attempt, + ) + .filter(delivery_attempt_dsl::rx_id.eq(rx_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to delete delivery attempt records", + ) + }) + })?; + // Finally, mark the webhook receiver record as deleted, + // provided that none of its children were modified in the interim. + let deleted = diesel::update(rx_dsl::webhook_receiver) + .filter(rx_dsl::id.eq(rx_id)) + .filter(rx_dsl::time_deleted.is_null()) + .filter(rx_dsl::rcgen.eq(db_rx.rcgen)) + .set(rx_dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await + .map_err(|e| err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to mark receiver as deleted", + ) + }))?; + if deleted == 0 { + return Err(err.bail(Error::conflict( + "deletion failed due to concurrent modification", + ))); + } - slog::debug!( - &opctx.log, - "deleted webhook receiver"; - "rx_id" => %rx_id, - "secrets_deleted" => ?secrets_deleted, - "exact_subscriptions_deleted" => ?exact_subscriptions_deleted, - "globs_deleted" => ?globs_deleted, - "deliveries_deleted" => ?deliveries_deleted, - "delivery_attempts_deleted" => ?delivery_attempts_deleted, - ); + slog::info!( + &opctx.log, + "deleted webhook receiver"; + "rx_id" => %rx_id, + "rx_name" => %db_rx.identity.name, + "secrets_deleted" => ?secrets_deleted, + "exact_subscriptions_deleted" => ?exact_subscriptions_deleted, + "globs_deleted" => ?globs_deleted, + "deliveries_deleted" => ?deliveries_deleted, + "delivery_attempts_deleted" => ?delivery_attempts_deleted, + ); - Ok(()) + Ok(()) + } + }, + ).await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + }) } pub async fn webhook_rx_list( diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 6614f7161b9..835b82008a6 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -36,6 +36,7 @@ use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; @@ -123,6 +124,15 @@ impl super::Nexus { self.datastore().webhook_rx_create(&opctx, params).await } + pub async fn webhook_receiver_delete( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + ) -> DeleteResult { + let (authz_rx, db_rx) = rx.fetch_for(authz::Action::Delete).await?; + self.datastore().webhook_rx_delete(&opctx, &authz_rx, &db_rx).await + } + pub async fn webhook_receiver_event_resend( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 339fce1d175..4ab475ba75f 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7450,8 +7450,7 @@ impl NexusExternalApi for NexusExternalApiImpl { let webhook_selector = path_params.into_inner(); let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; - let (authz_rx,) = rx.lookup_for(authz::Action::Delete).await?; - nexus.datastore().webhook_rx_delete(&opctx, &authz_rx).await?; + nexus.webhook_receiver_delete(&opctx, rx).await?; Ok(HttpResponseDeleted()) }; diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index a06f20741d0..0a9e34956a7 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -278,7 +278,7 @@ async fn test_webhook_receiver_get(cptestctx: &ControlPlaneTestContext) { webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; dbg!(&created_webhook); - // Fetch the receiver by name. + // Fetch the receiver by ID. let by_id_url = get_webhooks_url(created_webhook.identity.id); let webhook_view = webhook_get(client, &by_id_url).await; assert_eq!(created_webhook, webhook_view); @@ -289,6 +289,33 @@ async fn test_webhook_receiver_get(cptestctx: &ControlPlaneTestContext) { assert_eq!(created_webhook, webhook_view); } +#[nexus_test] +async fn test_webhook_receiver_create_delete( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + let server = httpmock::MockServer::start_async().await; + + // Create a webhook receiver. + let created_webhook = + webhook_create(&cptestctx, &my_great_webhook_params(&server)).await; + dbg!(&created_webhook); + + resource_helpers::object_delete( + client, + &format!("{RECEIVERS_BASE_PATH}/{}", created_webhook.identity.name), + ) + .await; + + // It should be gone now. + resource_helpers::object_delete_error( + client, + &format!("{RECEIVERS_BASE_PATH}/{}", created_webhook.identity.name), + http::StatusCode::NOT_FOUND, + ) + .await; +} #[nexus_test] async fn test_webhook_receiver_names_are_unique( cptestctx: &ControlPlaneTestContext, From 7e6643e3e2f2a9ccb25b4b832fa5b059e01920c2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 15:44:26 -0700 Subject: [PATCH 121/168] redo receiver CRUD --- nexus/db-model/src/schema.rs | 3 +- nexus/db-model/src/webhook_rx.rs | 47 ++- .../db-queries/src/db/datastore/webhook_rx.rs | 333 ++++++++++++------ nexus/external-api/src/lib.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 2 +- nexus/types/src/external_api/params.rs | 4 +- schema/crdb/dbinit.sql | 9 +- schema/crdb/webhooks/up01.sql | 9 +- 8 files changed, 269 insertions(+), 140 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index cc7c65ab14a..2982d2bb726 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2190,7 +2190,8 @@ table! { time_modified -> Timestamptz, time_deleted -> Nullable, endpoint -> Text, - rcgen -> Int8, + secret_gen -> Int8, + subscription_gen -> Int8, } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 88b5fe022fd..8c1e113927d 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -11,6 +11,7 @@ use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; use crate::EventClassParseError; use crate::Generation; +use crate::Name; use crate::WebhookEventClass; use chrono::{DateTime, Utc}; use db_macros::{Asset, Resource}; @@ -84,29 +85,32 @@ pub struct WebhookReceiver { pub identity: WebhookReceiverIdentity, pub endpoint: String, - /// child resource generation number, per RFD 192 - pub rcgen: Generation, + /// child resource generation number for secrets, per RFD 192 + pub secret_gen: Generation, + /// child resource generation number for event subscriptions, per RFD 192 + pub subscription_gen: Generation, } +// Note that while we have both a `secret_gen` and a `subscription_gen`, we only +// implement `DatastoreCollection` for secrets, not subscriptions. This is +// because subscriptions are updated in a batch, using a transaction, rather +// than via add and delete operations for individual IDs, like secrets. impl DatastoreCollectionConfig for WebhookReceiver { type CollectionId = Uuid; - type GenerationNumberColumn = webhook_receiver::dsl::rcgen; + type GenerationNumberColumn = webhook_receiver::dsl::secret_gen; type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; type CollectionIdColumn = webhook_secret::dsl::rx_id; } -impl DatastoreCollectionConfig for WebhookReceiver { - type CollectionId = Uuid; - type GenerationNumberColumn = webhook_receiver::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; - type CollectionIdColumn = webhook_rx_subscription::dsl::rx_id; -} - -impl DatastoreCollectionConfig for WebhookReceiver { - type CollectionId = Uuid; - type GenerationNumberColumn = webhook_receiver::dsl::rcgen; - type CollectionTimeDeletedColumn = webhook_receiver::dsl::time_deleted; - type CollectionIdColumn = webhook_rx_event_glob::dsl::rx_id; +/// Describes a set of updates for the [`WebhookReceiver`] model. +#[derive(Clone, AsChangeset)] +#[diesel(table_name = webhook_receiver)] +pub struct WebhookReceiverUpdate { + pub name: Option, + pub description: Option, + pub endpoint: Option, + pub time_modified: DateTime, + pub subscription_gen: Option, } #[derive( @@ -180,7 +184,7 @@ impl WebhookRxEventGlob { } } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq, Hash)] pub enum WebhookSubscriptionKind { Glob(WebhookGlob), Exact(WebhookEventClass), @@ -215,7 +219,16 @@ impl WebhookSubscriptionKind { } #[derive( - Clone, Debug, Queryable, Selectable, Insertable, Serialize, Deserialize, + Clone, + Debug, + Eq, + PartialEq, + Hash, + Queryable, + Selectable, + Insertable, + Serialize, + Deserialize, )] #[diesel(table_name = webhook_rx_event_glob)] pub struct WebhookGlob { diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 393fa356a19..e64e8890fb4 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -7,6 +7,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; +use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::datastore::RunnableQuery; @@ -49,6 +50,7 @@ use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use ref_cast::RefCast; @@ -87,7 +89,8 @@ impl DataStore { identity.clone(), ), endpoint: endpoint.to_string(), - rcgen: Generation::new(), + secret_gen: Generation::new(), + subscription_gen: Generation::new(), }; let subscriptions = subscriptions.clone(); let secret_keys = secrets.clone(); @@ -110,19 +113,19 @@ impl DataStore { ) }) })?; - for subscription in subscriptions { - self.add_subscription_on_conn( - opctx, - rx.identity.id.into(), - subscription, - &conn, - ) - .await - .map_err(|e| match e { - TransactionError::CustomError(e) => err.bail(e), - TransactionError::Database(e) => e, - })?; - } + + self.rx_add_subscriptions_on_conn( + opctx, + rx.identity.id.into(), + &subscriptions, + &conn, + ) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => err.bail(e), + TransactionError::Database(e) => e, + })?; + let mut secrets = Vec::with_capacity(secret_keys.len()); for secret in secret_keys { let secret = self @@ -282,7 +285,8 @@ impl DataStore { let deleted = diesel::update(rx_dsl::webhook_receiver) .filter(rx_dsl::id.eq(rx_id)) .filter(rx_dsl::time_deleted.is_null()) - .filter(rx_dsl::rcgen.eq(db_rx.rcgen)) + .filter(rx_dsl::subscription_gen.eq(db_rx.subscription_gen)) + .filter(rx_dsl::secret_gen.eq(db_rx.secret_gen)) .set(rx_dsl::time_deleted.eq(now)) .execute_async(&conn) .await @@ -322,6 +326,80 @@ impl DataStore { }) } + pub async fn webhook_rx_update( + &self, + opctx: &OpContext, + authz_rx: &authz::WebhookReceiver, + db_rx: &WebhookReceiver, + update: params::WebhookReceiverUpdate, + ) -> UpdateResult { + use std::collections::HashSet; + + opctx.authorize(authz::Action::Modify, authz_rx).await?; + let rx_id = authz_rx.id(); + + let conn = self.pool_connection_authorized(opctx).await?; + + let new_subscriptions = update + .events + .into_iter() + .map(WebhookSubscriptionKind::new) + .collect::, _>>()?; + let curr_subscriptions = self + .rx_subscription_list_on_conn(rx_id, &conn) + .await? + .into_iter() + .collect::>(); + let update = db::model::WebhookReceiverUpdate { + subscription_gen: None, + name: update.identity.name.map(db::model::Name), + description: update.identity.description, + endpoint: update.endpoint.as_ref().map(ToString::to_string), + time_modified: chrono::Utc::now(), + }; + + let err = OptionalError::new(); + let rx = self.transaction_retry_wrapper("webhook_rx_update") + .transaction(&conn, |conn| { + let mut update = update.clone(); + let new_subscriptions = new_subscriptions.clone(); + let curr_subscriptions = curr_subscriptions.clone(); + let db_rx = db_rx.clone(); + let err = err.clone(); + async move { + let subs_added = self.rx_add_subscriptions_on_conn(opctx, rx_id, new_subscriptions.difference(&curr_subscriptions), &conn).await + .map_err(|e| match e { + TransactionError::CustomError(e) => err.bail(e), + TransactionError::Database(e) => e, + })?; + let subs_deleted = self.rx_delete_subscriptions_on_conn(opctx, rx_id, curr_subscriptions.difference(&new_subscriptions).cloned().collect::>(), &conn).await?; + if subs_added + subs_deleted > 0 { + update.subscription_gen = Some(db_rx.subscription_gen.next().into()); + } + diesel::update(rx_dsl::webhook_receiver) + .filter(rx_dsl::id.eq(rx_id.into_untyped_uuid())) + .filter(rx_dsl::time_deleted.is_null()) + .filter(rx_dsl::subscription_gen.eq(db_rx.subscription_gen)) + .set(update).returning(WebhookReceiver::as_returning()) + .get_result_async(&conn) + .await + .map_err(|e| err.bail_retryable_or_else(e, |_| Error::conflict("cannot update receiver configuration as its state changed concurrently"))) + } + + }) + .await .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(WebhookReceiverConfig { + rx, + events: new_subscriptions.into_iter().collect::>(), + secrets: self.rx_secret_list_on_conn(rx_id, &*conn).await?, + }) + } + pub async fn webhook_rx_list( &self, opctx: &OpContext, @@ -399,34 +477,6 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - pub async fn webhook_rx_subscription_add( - &self, - opctx: &OpContext, - authz_rx: &authz::WebhookReceiver, - subscription: WebhookSubscriptionKind, - ) -> Result<(), Error> { - opctx.authorize(authz::Action::CreateChild, authz_rx).await?; - let conn = self.pool_connection_authorized(opctx).await?; - let num_created = self - .add_subscription_on_conn(opctx, authz_rx.id(), subscription, &conn) - .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_rx), - ), - })?; - - slog::debug!( - &opctx.log, - "added {num_created} webhook subscriptions"; - "webhook_id" => %authz_rx.id(), - ); - - Ok(()) - } - async fn rx_subscription_list_on_conn( &self, rx_id: WebhookReceiverUuid, @@ -466,62 +516,117 @@ impl DataStore { Ok(subscriptions) } - async fn add_subscription_on_conn( + async fn rx_add_subscriptions_on_conn( &self, opctx: &OpContext, rx_id: WebhookReceiverUuid, - subscription: WebhookSubscriptionKind, + subscriptions: impl IntoIterator, conn: &async_bb8_diesel::Connection, ) -> Result> { - match subscription { - WebhookSubscriptionKind::Exact(event_class) => self - .add_exact_sub_on_conn( - WebhookRxSubscription::exact(rx_id, event_class), - &conn, - ) - .await - .map(|_| 1), - WebhookSubscriptionKind::Glob(glob) => { - let glob = WebhookRxEventGlob::new(rx_id, glob); - let glob: WebhookRxEventGlob = - WebhookReceiver::insert_resource( - rx_id.into_untyped_uuid(), + let now = chrono::Utc::now(); + let mut exact = Vec::new(); + let mut n_globs = 0; + let mut n_glob_subscriptions = 0; + for subscription in subscriptions { + match subscription { + WebhookSubscriptionKind::Glob(glob) => { + let glob = WebhookRxEventGlob::new(rx_id, glob.clone()); + n_glob_subscriptions += self + .glob_generate_exact_subs(opctx, &glob, conn) + .await?; + + let created = diesel::insert_into(glob_dsl::webhook_rx_event_glob) .values(glob) - .on_conflict((glob_dsl::rx_id, glob_dsl::glob)) - .do_update() - .set(glob_dsl::time_created.eq(diesel::dsl::now)), + .on_conflict_do_nothing() + .execute_async(conn) + .await?; + n_globs += created; + } + WebhookSubscriptionKind::Exact(event_class) => { + exact.push(WebhookRxSubscription { + rx_id: rx_id.into(), + event_class: *event_class, + glob: None, + time_created: now, + }); + } + } + } + + let n_exact = + self.add_exact_subscription_batch_on_conn(exact, conn).await?; + slog::info!( + opctx.log, + "inserted new subscriptions for webhook receiver"; + "rx_id" => ?rx_id, + "globs" => ?n_globs, + "glob_subscriptions" => ?n_glob_subscriptions, + "exact_subscriptions" => ?n_exact, + ); + Ok(n_exact + n_globs) + } + + async fn rx_delete_subscriptions_on_conn( + &self, + opctx: &OpContext, + rx_id: WebhookReceiverUuid, + subscriptions: impl IntoIterator, + conn: &async_bb8_diesel::Connection, + ) -> Result { + let mut n_exact = 0; + let mut n_glob_subscriptions = 0; + let mut n_globs = 0; + let rx_id = rx_id.into_untyped_uuid(); + for subscription in subscriptions { + match subscription { + WebhookSubscriptionKind::Glob(glob) => { + n_glob_subscriptions += diesel::delete( + subscription_dsl::webhook_rx_subscription, ) - .insert_and_get_result_async(conn) - .await - .map_err(async_insert_error_to_txn(rx_id))?; - self.glob_generate_exact_subs(opctx, &glob, conn).await + .filter(subscription_dsl::rx_id.eq(rx_id)) + .filter(subscription_dsl::glob.eq(glob.glob.clone())) + .execute_async(conn) + .await?; + n_globs += diesel::delete(glob_dsl::webhook_rx_event_glob) + .filter(glob_dsl::rx_id.eq(rx_id)) + .filter(glob_dsl::glob.eq(glob.glob)) + .execute_async(conn) + .await?; + } + WebhookSubscriptionKind::Exact(event_class) => { + n_exact += diesel::delete( + subscription_dsl::webhook_rx_subscription, + ) + .filter(subscription_dsl::rx_id.eq(rx_id)) + .filter(subscription_dsl::event_class.eq(event_class)) + .execute_async(conn) + .await?; + } } } + + slog::info!( + opctx.log, + "deleted subscriptions for webhook receiver"; + "rx_id" => ?rx_id, + "globs" => ?n_globs, + "glob_subscriptions" => ?n_glob_subscriptions, + "exact_subscriptions" => ?n_exact, + ); + Ok(n_exact + n_globs) } - async fn add_exact_sub_on_conn( + async fn add_exact_subscription_batch_on_conn( &self, - subscription: WebhookRxSubscription, + subscriptions: Vec, conn: &async_bb8_diesel::Connection, - ) -> Result> { - let rx_id = WebhookReceiverUuid::from(subscription.rx_id); - let subscription: WebhookRxSubscription = - WebhookReceiver::insert_resource( - rx_id.into_untyped_uuid(), - diesel::insert_into(subscription_dsl::webhook_rx_subscription) - .values(subscription) - .on_conflict(( - subscription_dsl::rx_id, - subscription_dsl::event_class, - )) - .do_update() - .set(subscription_dsl::time_created.eq(diesel::dsl::now)), - ) - .insert_and_get_result_async(conn) + ) -> Result { + diesel::insert_into(subscription_dsl::webhook_rx_subscription) + .values(subscriptions) + .on_conflict_do_nothing() + .execute_async(conn) .await - .map_err(async_insert_error_to_txn(rx_id))?; - Ok(subscription) } async fn glob_generate_exact_subs( @@ -547,36 +652,36 @@ impl DataStore { )); } }; - let mut created = 0; - for &class in WebhookEventClass::ALL_CLASSES { - if !regex.is_match(class.as_str()) { - slog::debug!( - &opctx.log, - "webhook glob does not matche event class"; - "webhook_id" => ?glob.rx_id, - "glob" => ?glob.glob.glob, - "regex" => ?regex, - "event_class" => %class, - ); - continue; - } - - slog::debug!( - &opctx.log, - "webhook glob matches event class"; - "webhook_id" => ?glob.rx_id, - "glob" => ?glob.glob.glob, - "regex" => ?regex, - "event_class" => %class, - ); - self.add_exact_sub_on_conn( - WebhookRxSubscription::for_glob(&glob, class), - conn, - ) - .await?; - created += 1; - } - + let subscriptions = WebhookEventClass::ALL_CLASSES + .iter() + .filter_map(|class| { + if regex.is_match(class.as_str()) { + slog::debug!( + &opctx.log, + "webhook glob matches event class"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "regex" => ?regex, + "event_class" => %class, + ); + Some(WebhookRxSubscription::for_glob(&glob, *class)) + } else { + slog::trace!( + &opctx.log, + "webhook glob does not match event class"; + "rx_id" => ?glob.rx_id, + "glob" => ?glob.glob.glob, + "regex" => ?regex, + "event_class" => %class, + ); + None + } + }) + .collect::>(); + let created = self + .add_exact_subscription_batch_on_conn(subscriptions, conn) + .await + .map_err(TransactionError::Database)?; slog::info!( &opctx.log, "created {created} webhook subscriptions for glob"; diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 8f4f39dfd9b..c7c74af7746 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3542,7 +3542,7 @@ pub trait NexusExternalApi { async fn webhook_receiver_update( rqctx: RequestContext, path_params: Path, - params: TypedBody, + params: TypedBody, ) -> Result; /// Delete a webhook receiver. diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 4ab475ba75f..5d2479d7f46 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7416,7 +7416,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_receiver_update( rqctx: RequestContext, _path_params: Path, - _params: TypedBody, + _params: TypedBody, ) -> Result { let apictx = rqctx.context(); let handler = async { diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index a400ddf8f0c..bb43604ab57 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2416,12 +2416,12 @@ pub struct WebhookCreate { /// Parameters to update a webhook configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct WebhookUpdate { +pub struct WebhookReceiverUpdate { #[serde(flatten)] pub identity: IdentityMetadataUpdateParams, /// The URL that webhook notification requests should be sent to - pub endpoint: Url, + pub endpoint: Option, /// A list of webhook event classes to subscribe to. /// diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index fa2d1417bac..36ca3ed02e4 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4975,8 +4975,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( time_created TIMESTAMPTZ NOT NULL, time_modified TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, - -- Child resource generation - rcgen INT NOT NULL, + -- Child resource generation for secrets. + secret_gen INT NOT NULL, + + -- Child resource generation for subscriptions. This is separate from + -- `secret_gen`, as updating secrets and updating subscriptions are separate + -- operations which don't conflict with each other. + subscription_gen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL ); diff --git a/schema/crdb/webhooks/up01.sql b/schema/crdb/webhooks/up01.sql index 318088787c9..d4afbb5ea93 100644 --- a/schema/crdb/webhooks/up01.sql +++ b/schema/crdb/webhooks/up01.sql @@ -6,8 +6,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( time_created TIMESTAMPTZ NOT NULL, time_modified TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, - -- Child resource generation - rcgen INT NOT NULL, + -- Child resource generation for secrets. + secret_gen INT NOT NULL, + + -- Child resource generation for subscriptions. This is separate from + -- `secret_gen`, as updating secrets and updating subscriptions are separate + -- operations which don't conflict with each other. + subscription_gen INT NOT NULL, -- URL of the endpoint webhooks are delivered to. endpoint STRING(512) NOT NULL ); From ff9c213f453f91ea02d4ea2108b617533e96b185 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 17:35:01 -0700 Subject: [PATCH 122/168] oops forgot to update this --- nexus/src/app/background/tasks/webhook_dispatcher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 9454c1d7fc1..cdebc84593d 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -342,7 +342,8 @@ mod test { ), endpoint: "http://webhooks.elizas.website".parse().unwrap(), - rcgen: db::model::Generation::new(), + secret_gen: db::model::Generation::new(), + subscription_gen: db::model::Generation::new(), }) .execute_async(&*conn) .await From 4c7210942009b062d3108177821ca38e07ea8422 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 10 Mar 2025 17:35:24 -0700 Subject: [PATCH 123/168] ensure index is used when deleting secrets --- nexus/db-queries/src/db/datastore/webhook_rx.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index e64e8890fb4..564b6261b42 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -204,6 +204,7 @@ impl DataStore { let secrets_deleted = diesel::delete(secret_dsl::webhook_secret) .filter(secret_dsl::rx_id.eq(rx_id)) + .filter(secret_dsl::time_deleted.is_null()) .execute_async(&conn) .await .map_err(|e| { From 9b262a04bc6fccd7139ed3e97f4c42d30b04925a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 11:01:09 -0700 Subject: [PATCH 124/168] fix merge mistakes --- nexus/db-model/src/schema_versions.rs | 54 +++------------------------ nexus/src/app/background/init.rs | 2 - 2 files changed, 5 insertions(+), 51 deletions(-) diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 25c120d0773..3207d8357e0 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,11 +16,15 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(129, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(130, 0, 0); /// List of all past database schema versions, in *reverse* order /// +/// If you want to change the Omicron database schema, you must update this. +static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { + vec![ // +- The next version goes here! Duplicate this line, uncomment + // | the *second* copy, then update that copy for your version, // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), @@ -478,54 +482,6 @@ impl SchemaUpgradeStep { } } -/// A newtype around [`SemverVersion`] that implements the [`ToSql`] and -/// [`FromSql`] traits, allowing it to be used as a field in a type representing -/// a DB model. -#[derive( - Clone, - Debug, - Eq, - PartialEq, - PartialOrd, - serde::Serialize, - serde::Deserialize, - deserialize::FromSqlRow, - AsExpression, -)] -#[diesel(sql_type = Text)] -pub struct DbSemverVersion(pub SemverVersion); - -impl deserialize::FromSql for DbSemverVersion { - fn from_sql(value: PgValue<'_>) -> deserialize::Result { - let version = - std::str::from_utf8(value.as_bytes())?.parse::()?; - Ok(Self(version)) - } -} - -impl serialize::ToSql for DbSemverVersion { - fn to_sql<'b>( - &'b self, - out: &mut serialize::Output<'b, '_, Pg>, - ) -> serialize::Result { - use std::io::Write; - out.write_fmt(format_args!("{}", self.0))?; - Ok(serialize::IsNull::No) - } -} - -impl From for DbSemverVersion { - fn from(version: SemverVersion) -> Self { - Self(version) - } -} - -impl From for SemverVersion { - fn from(DbSemverVersion(version): DbSemverVersion) -> Self { - version - } -} - #[cfg(test)] mod test { use super::*; diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index a1e4eccbf24..d99b4811d7f 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -88,8 +88,6 @@ //! mistakes. use super::Activator; -use super::Activator; -use super::Driver; use super::Driver; use super::driver::TaskDefinition; use super::tasks::abandoned_vmm_reaper; From f9c238bda057f5f5c5671ecf42fd515608dff9af Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 11:22:11 -0700 Subject: [PATCH 125/168] post merge fixup (mostly for #7570) --- nexus/db-model/src/webhook_rx.rs | 15 ++++++++------- nexus/db-queries/src/db/datastore/webhook_rx.rs | 14 +++++++------- nexus/types/src/internal_api/background.rs | 9 ++++++--- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index 8c1e113927d..b91c35c7bc0 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -2,6 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::EventClassParseError; +use crate::Generation; +use crate::Name; +use crate::SemverVersion; +use crate::WebhookEventClass; use crate::collection::DatastoreCollectionConfig; use crate::schema::{ webhook_receiver, webhook_rx_event_glob, webhook_rx_subscription, @@ -9,10 +14,6 @@ use crate::schema::{ }; use crate::schema_versions; use crate::typed_uuid::DbTypedUuid; -use crate::EventClassParseError; -use crate::Generation; -use crate::Name; -use crate::WebhookEventClass; use chrono::{DateTime, Utc}; use db_macros::{Asset, Resource}; use nexus_types::external_api::views; @@ -171,7 +172,7 @@ pub struct WebhookRxEventGlob { #[diesel(embed)] pub glob: WebhookGlob, pub time_created: DateTime, - pub schema_version: schema_versions::DbSemverVersion, + pub schema_version: SemverVersion, } impl WebhookRxEventGlob { @@ -259,10 +260,10 @@ impl WebhookGlob { "event_class", "invalid event class {glob:?}: all segments must be \ either '*', '**', or any sequence of non-'*' characters", - )) + )); } // Match the literal segment. - s => regex.push_str(s), + s => regex.push_str(s), } Ok(()) }; diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 564b6261b42..9836135c364 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -8,14 +8,16 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; +use crate::db::TransactionError; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::datastore::RunnableQuery; -use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::model::DbSemverVersion; +use crate::db::error::public_error_from_diesel; use crate::db::model::Generation; use crate::db::model::Name; +use crate::db::model::SCHEMA_VERSION; +use crate::db::model::SemverVersion; use crate::db::model::WebhookEventClass; use crate::db::model::WebhookGlob; use crate::db::model::WebhookReceiver; @@ -25,7 +27,6 @@ use crate::db::model::WebhookRxEventGlob; use crate::db::model::WebhookRxSubscription; use crate::db::model::WebhookSecret; use crate::db::model::WebhookSubscriptionKind; -use crate::db::model::SCHEMA_VERSION; use crate::db::pagination::paginated; use crate::db::pagination::paginated_multicolumn; use crate::db::pool::DbConnection; @@ -36,14 +37,12 @@ use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; use crate::db::schema::webhook_secret::dsl as secret_dsl; -use crate::db::TransactionError; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookGlobStatus; -use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; @@ -51,6 +50,7 @@ use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::WebhookReceiverUuid; use ref_cast::RefCast; @@ -763,7 +763,7 @@ impl DataStore { pagparams, ) .filter( - glob_dsl::schema_version.ne(DbSemverVersion::from(SCHEMA_VERSION)), + glob_dsl::schema_version.ne(SemverVersion::from(SCHEMA_VERSION)), ) .select(WebhookRxEventGlob::as_select()) .load_async(&*self.pool_connection_authorized(&opctx).await?) @@ -821,7 +821,7 @@ impl DataStore { ) .set( glob_dsl::schema_version - .eq(DbSemverVersion::from(SCHEMA_VERSION)), + .eq(SemverVersion::from(SCHEMA_VERSION)), ) .execute_async(&conn) .await; diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index e1973e8e5f0..99a21876d10 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -5,7 +5,6 @@ use crate::external_api::views; use chrono::DateTime; use chrono::Utc; -use omicron_common::api::external::SemverVersion; use omicron_common::update::ArtifactHash; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::CollectionUuid; @@ -460,7 +459,7 @@ impl slog::KV for DebugDatasetsRendezvousStats { pub struct WebhookDispatcherStatus { pub globs_reprocessed: BTreeMap, - pub glob_version: SemverVersion, + pub glob_version: semver::Version, /// The webhook events dispatched on this activation. pub dispatched: Vec, @@ -477,7 +476,11 @@ type ReprocessedGlobs = BTreeMap>; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum WebhookGlobStatus { AlreadyReprocessed, - Reprocessed { created: usize, deleted: usize, prev_version: SemverVersion }, + Reprocessed { + created: usize, + deleted: usize, + prev_version: semver::Version, + }, } #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] From 1768140ff175364c9e53391f8ba3f369afaa99eb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 12:25:33 -0700 Subject: [PATCH 126/168] make subscription changes optional in rx updates --- .../db-queries/src/db/datastore/webhook_rx.rs | 177 +++++++++++++----- nexus/types/src/external_api/params.rs | 2 +- 2 files changed, 127 insertions(+), 52 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index 9836135c364..db6def69a45 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -37,6 +37,8 @@ use crate::db::schema::webhook_receiver::dsl as rx_dsl; use crate::db::schema::webhook_rx_event_glob::dsl as glob_dsl; use crate::db::schema::webhook_rx_subscription::dsl as subscription_dsl; use crate::db::schema::webhook_secret::dsl as secret_dsl; +use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateStatus; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; @@ -332,73 +334,146 @@ impl DataStore { opctx: &OpContext, authz_rx: &authz::WebhookReceiver, db_rx: &WebhookReceiver, - update: params::WebhookReceiverUpdate, + params: params::WebhookReceiverUpdate, ) -> UpdateResult { use std::collections::HashSet; opctx.authorize(authz::Action::Modify, authz_rx).await?; - let rx_id = authz_rx.id(); - let conn = self.pool_connection_authorized(opctx).await?; - let new_subscriptions = update - .events - .into_iter() - .map(WebhookSubscriptionKind::new) - .collect::, _>>()?; - let curr_subscriptions = self - .rx_subscription_list_on_conn(rx_id, &conn) - .await? - .into_iter() - .collect::>(); + let rx_id = authz_rx.id(); let update = db::model::WebhookReceiverUpdate { subscription_gen: None, - name: update.identity.name.map(db::model::Name), - description: update.identity.description, - endpoint: update.endpoint.as_ref().map(ToString::to_string), + name: params.identity.name.map(db::model::Name), + description: params.identity.description, + endpoint: params.endpoint.as_ref().map(ToString::to_string), time_modified: chrono::Utc::now(), }; - let err = OptionalError::new(); - let rx = self.transaction_retry_wrapper("webhook_rx_update") - .transaction(&conn, |conn| { - let mut update = update.clone(); - let new_subscriptions = new_subscriptions.clone(); - let curr_subscriptions = curr_subscriptions.clone(); - let db_rx = db_rx.clone(); - let err = err.clone(); - async move { - let subs_added = self.rx_add_subscriptions_on_conn(opctx, rx_id, new_subscriptions.difference(&curr_subscriptions), &conn).await - .map_err(|e| match e { + // If the update changes event class subscriptions, query to get the + // current subscriptions so we can determine the difference in order to + // apply the update. + // + // If we are changing subscriptions, we must perform the changes to the + // subscription table in a transaction with the changes to the receiver + // table, so that we can undo those changes should the receiver update fail. + let rx = if let Some(new_subscriptions) = params.events { + let new_subscriptions = new_subscriptions + .into_iter() + .map(WebhookSubscriptionKind::new) + .collect::, _>>()?; + let curr_subscriptions = self + .rx_subscription_list_on_conn(rx_id, &conn) + .await? + .into_iter() + .collect::>(); + let err = OptionalError::new(); + self.transaction_retry_wrapper("webhook_rx_update") + .transaction(&conn, |conn| { + let mut update = update.clone(); + let new_subscriptions = new_subscriptions.clone(); + let curr_subscriptions = curr_subscriptions.clone(); + let db_rx = db_rx.clone(); + let err = err.clone(); + async move { + let subs_added = self + .rx_add_subscriptions_on_conn( + opctx, + rx_id, + new_subscriptions + .difference(&curr_subscriptions), + &conn, + ) + .await + .map_err(|e| match e { TransactionError::CustomError(e) => err.bail(e), TransactionError::Database(e) => e, })?; - let subs_deleted = self.rx_delete_subscriptions_on_conn(opctx, rx_id, curr_subscriptions.difference(&new_subscriptions).cloned().collect::>(), &conn).await?; - if subs_added + subs_deleted > 0 { - update.subscription_gen = Some(db_rx.subscription_gen.next().into()); + let subs_deleted = self + .rx_delete_subscriptions_on_conn( + opctx, + rx_id, + curr_subscriptions + .difference(&new_subscriptions) + .cloned() + .collect::>(), + &conn, + ) + .await?; + if subs_added + subs_deleted > 0 { + update.subscription_gen = + Some(db_rx.subscription_gen.next().into()); + } + self.rx_record_update_on_conn(&db_rx, update, &conn) + .await + .map_err(|e| match e { + TransactionError::CustomError(e) => err.bail(e), + TransactionError::Database(e) => e, + }) } - diesel::update(rx_dsl::webhook_receiver) - .filter(rx_dsl::id.eq(rx_id.into_untyped_uuid())) - .filter(rx_dsl::time_deleted.is_null()) - .filter(rx_dsl::subscription_gen.eq(db_rx.subscription_gen)) - .set(update).returning(WebhookReceiver::as_returning()) - .get_result_async(&conn) - .await - .map_err(|e| err.bail_retryable_or_else(e, |_| Error::conflict("cannot update receiver configuration as its state changed concurrently"))) - } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ) + })? + } else { + // If we are *not* changing subscriptions, we can just update the + // receiver record, eliding the transaction. This will still fail if + // the subscription generation has changed since we snapshotted the + // receiver. + self.rx_record_update_on_conn(db_rx, update, &conn).await.map_err( + |e| match e { + TransactionError::CustomError(e) => e, + TransactionError::Database(e) => public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_rx), + ), + }, + )? + }; - }) - .await .map_err(|e| { - if let Some(err) = err.take() { - return err; - } - public_error_from_diesel(e, ErrorHandler::Server) - })?; - Ok(WebhookReceiverConfig { - rx, - events: new_subscriptions.into_iter().collect::>(), - secrets: self.rx_secret_list_on_conn(rx_id, &*conn).await?, - }) + // Query to get the secrets and subscriptions for the returned view. + let (events, secrets) = + self.rx_config_fetch_on_conn(rx_id, &conn).await?; + Ok(WebhookReceiverConfig { rx, events, secrets }) + } + + /// Update the `webhook_receiver` record for the provided webhook receiver + /// and update. + /// + /// This is factored out as it may or may not be run in a transaction, + /// depending on whether or not event subscriptions have changed. + async fn rx_record_update_on_conn( + &self, + curr: &WebhookReceiver, + update: db::model::WebhookReceiverUpdate, + conn: &async_bb8_diesel::Connection, + ) -> Result> { + let rx_id = curr.identity.id.into_untyped_uuid(); + let result = diesel::update(rx_dsl::webhook_receiver) + .filter(rx_dsl::id.eq(rx_id)) + .filter(rx_dsl::time_deleted.is_null()) + .filter(rx_dsl::subscription_gen.eq(curr.subscription_gen)) + .set(update) + .check_if_exists::(rx_id) + .execute_and_check(&conn) + .await + .map_err(TransactionError::Database)?; + + match result.status { + UpdateStatus::Updated => Ok(result.found), + UpdateStatus::NotUpdatedButExists => Err(Error::conflict( + "cannot update receiver configuration, as it has changed \ + concurrently", + ) + .into()), + } } pub async fn webhook_rx_list( diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 13b3427036f..72f5fb4f0ae 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2433,7 +2433,7 @@ pub struct WebhookReceiverUpdate { /// A list of webhook event classes to subscribe to. /// /// If this list is empty, the webhook will not be subscribed to any events. - pub events: Vec, + pub events: Option>, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] From 26a029f4046d48992246b652ac97debe17b69f8d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 13:19:44 -0700 Subject: [PATCH 127/168] blech more test updates --- nexus/src/app/background/tasks/webhook_dispatcher.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index cdebc84593d..49d34a97a48 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -7,13 +7,13 @@ use crate::app::background::Activator; use crate::app::background::BackgroundTask; use futures::future::BoxFuture; +use nexus_db_model::SCHEMA_VERSION; use nexus_db_model::WebhookDelivery; use nexus_db_model::WebhookDeliveryTrigger; -use nexus_db_model::SCHEMA_VERSION; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; use nexus_db_queries::db::datastore::SQL_BATCH_SIZE; use nexus_db_queries::db::pagination::Paginator; -use nexus_db_queries::db::DataStore; use nexus_types::identity::Asset; use nexus_types::identity::Resource; use nexus_types::internal_api::background::{ @@ -301,7 +301,6 @@ mod test { use nexus_db_queries::db; use nexus_test_utils_macros::nexus_test; use omicron_common::api::external::IdentityMetadataCreateParams; - use omicron_common::api::external::SemverVersion; use omicron_uuid_kinds::WebhookEventUuid; use omicron_uuid_kinds::WebhookReceiverUuid; @@ -355,7 +354,7 @@ mod test { .expect("'test.*.bar should be an acceptable glob"); let mut glob = db::model::WebhookRxEventGlob::new(rx_id, glob); // Just make something up that's obviously outdated... - glob.schema_version = SemverVersion::new(100, 0, 0).into(); + glob.schema_version = db::model::SemverVersion::new(100, 0, 0); diesel::insert_into(glob_dsl::webhook_rx_event_glob) .values(glob.clone()) .execute_async(&*conn) From 9f2ae151891e6f7841e5fbb1fcfc441ba3ee7d72 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 14:04:58 -0700 Subject: [PATCH 128/168] you have to actually implement the endpoint I AM VERY SMART --- .../db-queries/src/db/datastore/webhook_rx.rs | 7 +--- nexus/src/app/webhook.rs | 37 +++++++++++++------ nexus/src/external_api/http_entrypoints.rs | 14 ++++--- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_rx.rs b/nexus/db-queries/src/db/datastore/webhook_rx.rs index db6def69a45..a6e95466d62 100644 --- a/nexus/db-queries/src/db/datastore/webhook_rx.rs +++ b/nexus/db-queries/src/db/datastore/webhook_rx.rs @@ -335,7 +335,7 @@ impl DataStore { authz_rx: &authz::WebhookReceiver, db_rx: &WebhookReceiver, params: params::WebhookReceiverUpdate, - ) -> UpdateResult { + ) -> UpdateResult { use std::collections::HashSet; opctx.authorize(authz::Action::Modify, authz_rx).await?; @@ -438,10 +438,7 @@ impl DataStore { )? }; - // Query to get the secrets and subscriptions for the returned view. - let (events, secrets) = - self.rx_config_fetch_on_conn(rx_id, &conn).await?; - Ok(WebhookReceiverConfig { rx, events, secrets }) + Ok(rx) } /// Update the `webhook_receiver` record for the provided webhook receiver diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 835b82008a6..670cdc2bc48 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -6,7 +6,7 @@ //! //! # Webhooks: Theory and Practice //! -//! For a discussion of +//! [RFD 538] describes the user-facing use crate::app::external_dns; use anyhow::Context; @@ -33,7 +33,6 @@ use nexus_db_queries::db::model::WebhookSecret; use nexus_types::external_api::params; use nexus_types::external_api::views; use nexus_types::identity::Resource; -use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; @@ -41,6 +40,8 @@ use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; +use omicron_common::api::external::UpdateResult; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::WebhookDeliveryUuid; @@ -105,15 +106,6 @@ impl super::Nexus { Ok(WebhookReceiverConfig { rx, secrets, events }) } - pub async fn webhook_receiver_secrets_list( - &self, - opctx: &OpContext, - rx: lookup::WebhookReceiver<'_>, - ) -> ListResultVec { - let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; - self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await - } - pub async fn webhook_receiver_create( &self, opctx: &OpContext, @@ -124,6 +116,20 @@ impl super::Nexus { self.datastore().webhook_rx_create(&opctx, params).await } + pub async fn webhook_receiver_update( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + params: params::WebhookReceiverUpdate, + ) -> UpdateResult<()> { + let (authz_rx, rx) = rx.fetch_for(authz::Action::Modify).await?; + let _ = self + .datastore() + .webhook_rx_update(opctx, &authz_rx, &rx, params) + .await?; + Ok(()) + } + pub async fn webhook_receiver_delete( &self, opctx: &OpContext, @@ -133,6 +139,15 @@ impl super::Nexus { self.datastore().webhook_rx_delete(&opctx, &authz_rx, &db_rx).await } + pub async fn webhook_receiver_secrets_list( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + ) -> ListResultVec { + let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; + self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await + } + pub async fn webhook_receiver_event_resend( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 38dfc2a3953..88e8b4aa4b8 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7801,8 +7801,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_receiver_update( rqctx: RequestContext, - _path_params: Path, - _params: TypedBody, + path_params: Path, + params: TypedBody, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -7811,10 +7811,12 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let webhook_selector = path_params.into_inner(); + let params = params.into_inner(); + let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; + nexus.webhook_receiver_update(&opctx, rx, params).await?; + + Ok(HttpResponseUpdatedNoContent()) }; apictx .context From dbfc8a71932bbaf27a7fdda29fea6f6e7830cdcb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 16:53:21 -0700 Subject: [PATCH 129/168] fix authz for webhook secrets --- nexus/auth/src/authz/api_resources.rs | 2 +- nexus/auth/src/authz/omicron.polar | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index b3e44f5a72a..b1e07b5460b 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -1123,5 +1123,5 @@ authz_resource! { parent = "WebhookReceiver", primary_key = { uuid_kind = WebhookSecretKind }, roles_allowed = false, - polar_snippet = FleetChild, + polar_snippet = Custom, } diff --git a/nexus/auth/src/authz/omicron.polar b/nexus/auth/src/authz/omicron.polar index c1e463e9bf3..319e6c4e04f 100644 --- a/nexus/auth/src/authz/omicron.polar +++ b/nexus/auth/src/authz/omicron.polar @@ -593,3 +593,14 @@ has_role(USER_DB_INIT: AuthenticatedActor, "admin", _silo: Silo); # Allow the internal API admin permissions on all silos. has_role(USER_INTERNAL_API: AuthenticatedActor, "admin", _silo: Silo); + +resource WebhookSecret { + permissions = [ "read", "modify" ]; + relations = { parent_webhook_receiver: WebhookReceiver }; + + "read" if "read" on "parent_webhook_receiver"; + "modify" if "modify" on "parent_webhook_receiver"; +} + +has_relation(rx: WebhookReceiver, "parent_webhook_receiver", secret: WebhookSecret) + if secret.webhook_receiver = rx; From 6bb16ac4db2f5319c43e637b9f6740753047af9e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 16:53:53 -0700 Subject: [PATCH 130/168] add authz tests for most of webhooks --- nexus/tests/integration_tests/endpoints.rs | 117 ++++++++++++++++++ nexus/tests/integration_tests/unauthorized.rs | 12 ++ .../output/uncovered-authz-endpoints.txt | 6 +- 3 files changed, 132 insertions(+), 3 deletions(-) diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index a606a0c94c2..f5116f2e9bb 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -1144,6 +1144,65 @@ pub static DEMO_TARGET_RELEASE: LazyLock = system_version: Version::new(0, 0, 0), }); +// Webhooks +pub static WEBHOOK_RECEIVERS_URL: &'static str = "/v1/webhooks/receivers"; + +pub static DEMO_WEBHOOK_RECEIVER_NAME: LazyLock = + LazyLock::new(|| "my-great-webhook".parse().unwrap()); +pub static DEMO_WEBHOOK_RECEIVER_CREATE: LazyLock = + LazyLock::new(|| params::WebhookCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_WEBHOOK_RECEIVER_NAME.clone(), + description: "webhook, line, and sinker".to_string(), + }, + endpoint: "https://example.com/my-great-webhook".parse().unwrap(), + secrets: vec!["my cool secret".to_string()], + events: vec!["test.foo.bar".to_string(), "test.*".to_string()], + }); + +pub static DEMO_WEBHOOK_RECEIVER_UPDATE: LazyLock< + params::WebhookReceiverUpdate, +> = LazyLock::new(|| params::WebhookReceiverUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: Some("webhooked on phonics".to_string()), + }, + endpoint: Some("https://example.com/my-cool-webhook".parse().unwrap()), + events: Some(vec![ + "test.foo.bar".to_string(), + "test.*".to_string(), + "test.**.baz".to_string(), + ]), +}); + +pub static DEMO_WEBHOOK_RECEIVER_URL: LazyLock = LazyLock::new(|| { + format!("{WEBHOOK_RECEIVERS_URL}/{}", *DEMO_WEBHOOK_RECEIVER_NAME) +}); +pub static DEMO_WEBHOOK_RECEIVER_PROBE_URL: LazyLock = + LazyLock::new(|| { + format!("{WEBHOOK_RECEIVERS_URL}/{}/probe", *DEMO_WEBHOOK_RECEIVER_NAME) + }); +pub static DEMO_WEBHOOK_DELIVERY_URL: LazyLock = LazyLock::new(|| { + format!("/v1/webhooks/deliveries?receiver={}", *DEMO_WEBHOOK_RECEIVER_NAME) +}); + +pub static DEMO_WEBHOOK_SECRETS_URL: LazyLock = LazyLock::new(|| { + format!("/v1/webhooks/secrets?receiver={}", *DEMO_WEBHOOK_RECEIVER_NAME) +}); + +pub static DEMO_WEBHOOK_SECRET_DELETE_URL: LazyLock = + LazyLock::new(|| { + format!( + "/v1/webhooks/secrets/{{id}}?receiver={}", + *DEMO_WEBHOOK_RECEIVER_NAME + ) + }); + +pub static DEMO_WEBHOOK_SECRET_CREATE: LazyLock = + LazyLock::new(|| params::WebhookSecretCreate { + secret: "TRUSTNO1".to_string(), + }); + /// Describes an API endpoint to be verified by the "unauthorized" test /// /// These structs are also used to check whether we're covering all endpoints in @@ -2713,5 +2772,63 @@ pub static VERIFY_ENDPOINTS: LazyLock> = ), ], }, + // Webhooks + VerifyEndpoint { + url: &WEBHOOK_RECEIVERS_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Post( + serde_json::to_value(&*DEMO_WEBHOOK_RECEIVER_CREATE) + .unwrap(), + ), + ], + }, + VerifyEndpoint { + url: &DEMO_WEBHOOK_RECEIVER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Put( + serde_json::to_value(&*DEMO_WEBHOOK_RECEIVER_UPDATE) + .unwrap(), + ), + AllowedMethod::Delete, + ], + }, + VerifyEndpoint { + url: &DEMO_WEBHOOK_RECEIVER_PROBE_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Post( + serde_json::to_value(()).unwrap(), + )], + }, + VerifyEndpoint { + url: &DEMO_WEBHOOK_SECRETS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Post( + serde_json::to_value(&*DEMO_WEBHOOK_SECRET_CREATE) + .unwrap(), + ), + ], + }, + VerifyEndpoint { + url: &DEMO_WEBHOOK_SECRET_DELETE_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Delete], + }, + VerifyEndpoint { + url: &DEMO_WEBHOOK_DELIVERY_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Get], + }, ] }); diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 8df0bffdb4e..5c9b534d9c1 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -370,6 +370,18 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { body: serde_json::to_value(()).unwrap(), id_routes: vec!["/experimental/v1/system/support-bundles/{id}"], }, + // Create a webhook receiver + SetupReq::Post { + url: &WEBHOOK_RECEIVERS_URL, + body: serde_json::to_value(&*DEMO_WEBHOOK_RECEIVER_CREATE).unwrap(), + id_routes: vec![], + }, + // Create a secret for that receiver + SetupReq::Post { + url: &DEMO_WEBHOOK_SECRETS_URL, + body: serde_json::to_value(&*DEMO_WEBHOOK_SECRET_CREATE).unwrap(), + id_routes: vec![&*DEMO_WEBHOOK_SECRET_DELETE_URL], + }, ] }); diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 6375d1ff901..78e1dd048f8 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -9,16 +9,16 @@ ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") networking_switch_port_status (get "/v1/system/hardware/switch-port/{port}/status") +webhook_event_class_list (get "/v1/webhooks/event-classes") +webhook_event_class_view (get "/v1/webhooks/event-classes/{name}") support_bundle_head (head "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_head_file (head "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") device_auth_request (post "/device/auth") device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") probe_create (post "/experimental/v1/probes") -webhook_create (post "/experimental/v1/webhooks") -webhook_delivery_resend (post "/experimental/v1/webhooks/{webhook_id}/deliveries/{event_id}/resend") -webhook_secrets_add (post "/experimental/v1/webhooks/{webhook_id}/secrets") login_saml (post "/login/{silo_name}/saml/{provider_name}") login_local (post "/v1/login/{silo_name}/local") logout (post "/v1/logout") networking_switch_port_lldp_config_update (post "/v1/system/hardware/switch-port/{port}/lldp/config") +webhook_delivery_resend (post "/v1/webhooks/deliveries/{event_id}/resend") From bf822deff4468f9dacd1b7c433ea91d59c142a7e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 11 Mar 2025 17:13:38 -0700 Subject: [PATCH 131/168] rustfmt & clippy placation --- nexus/db-model/src/webhook_delivery.rs | 6 +-- nexus/db-model/src/webhook_event.rs | 2 +- .../src/db/datastore/webhook_delivery.rs | 8 ++-- .../src/db/datastore/webhook_event.rs | 2 +- .../background/tasks/webhook_deliverator.rs | 4 +- .../background/tasks/webhook_dispatcher.rs | 10 ++--- nexus/tests/integration_tests/webhooks.rs | 44 +++++++++++-------- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 923a5fd9cab..907fbf69b52 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -2,15 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::schema::{webhook_delivery, webhook_delivery_attempt}; -use crate::serde_time_delta::optional_time_delta; -use crate::typed_uuid::DbTypedUuid; use crate::SqlU8; use crate::WebhookDeliveryAttemptResult; use crate::WebhookDeliveryState; use crate::WebhookDeliveryTrigger; use crate::WebhookEvent; use crate::WebhookEventClass; +use crate::schema::{webhook_delivery, webhook_delivery_attempt}; +use crate::serde_time_delta::optional_time_delta; +use crate::typed_uuid::DbTypedUuid; use chrono::{DateTime, TimeDelta, Utc}; use nexus_types::external_api::views; use nexus_types::identity::Asset; diff --git a/nexus/db-model/src/webhook_event.rs b/nexus/db-model/src/webhook_event.rs index f5d5531b72e..4a56cb2bd12 100644 --- a/nexus/db-model/src/webhook_event.rs +++ b/nexus/db-model/src/webhook_event.rs @@ -2,8 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::schema::webhook_event; use crate::WebhookEventClass; +use crate::schema::webhook_event; use chrono::{DateTime, Utc}; use db_macros::Asset; use serde::{Deserialize, Serialize}; diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index c540d007e08..4fa89cdd37c 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -7,8 +7,8 @@ use super::DataStore; use crate::context::OpContext; use crate::db::datastore::RunnableQuery; -use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; +use crate::db::error::public_error_from_diesel; use crate::db::model::SqlU8; use crate::db::model::WebhookDelivery; use crate::db::model::WebhookDeliveryAttempt; @@ -310,7 +310,9 @@ impl DataStore { }); } - Err(Error::internal_error("couldn't start delivery attempt for some secret third reason???")) + Err(Error::internal_error( + "couldn't start delivery attempt for some secret third reason???", + )) } } } @@ -411,7 +413,7 @@ impl DataStore { } Err(Error::internal_error( - "couldn't update delivery for some other reason i didn't think of here..." + "couldn't update delivery for some other reason i didn't think of here...", )) } } diff --git a/nexus/db-queries/src/db/datastore/webhook_event.rs b/nexus/db-queries/src/db/datastore/webhook_event.rs index 855f5e1b4df..cac1a0b77a8 100644 --- a/nexus/db-queries/src/db/datastore/webhook_event.rs +++ b/nexus/db-queries/src/db/datastore/webhook_event.rs @@ -6,8 +6,8 @@ use super::DataStore; use crate::context::OpContext; -use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; +use crate::db::error::public_error_from_diesel; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; use crate::db::model::WebhookEventIdentity; diff --git a/nexus/src/app/background/tasks/webhook_deliverator.rs b/nexus/src/app/background/tasks/webhook_deliverator.rs index 3cd3146090c..76b3e565417 100644 --- a/nexus/src/app/background/tasks/webhook_deliverator.rs +++ b/nexus/src/app/background/tasks/webhook_deliverator.rs @@ -5,18 +5,18 @@ use crate::app::background::BackgroundTask; use crate::app::webhook::ReceiverClient; use futures::future::BoxFuture; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; use nexus_db_queries::db::datastore::webhook_delivery::DeliveryAttemptState; pub use nexus_db_queries::db::datastore::webhook_delivery::DeliveryConfig; use nexus_db_queries::db::model::WebhookDeliveryAttemptResult; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_db_queries::db::pagination::Paginator; -use nexus_db_queries::db::DataStore; use nexus_types::identity::Resource; use nexus_types::internal_api::background::WebhookDeliveratorStatus; use nexus_types::internal_api::background::WebhookDeliveryFailure; use nexus_types::internal_api::background::WebhookRxDeliveryStatus; -use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::Error; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid, WebhookDeliveryUuid}; use std::num::NonZeroU32; use std::sync::Arc; diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 49d34a97a48..92168977008 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -247,7 +247,7 @@ impl WebhookDispatcher { } }; status.dispatched.push(WebhookDispatched { - event_id: event.id().into(), + event_id: event.id(), subscribed, dispatched, }); @@ -267,17 +267,13 @@ impl WebhookDispatcher { "event_id" => ?event.id(), "event_class" => %event.event_class, ); - status.no_receivers.push(event.id().into()); + status.no_receivers.push(event.id()); 0 }; if let Err(error) = self .datastore - .webhook_event_mark_dispatched( - &opctx, - &event.id().into(), - subscribed, - ) + .webhook_event_mark_dispatched(&opctx, &event.id(), subscribed) .await { slog::error!(&opctx.log, "failed to mark webhook event as dispatched"; diff --git a/nexus/tests/integration_tests/webhooks.rs b/nexus/tests/integration_tests/webhooks.rs index 0a9e34956a7..3fd1b554889 100644 --- a/nexus/tests/integration_tests/webhooks.rs +++ b/nexus/tests/integration_tests/webhooks.rs @@ -256,7 +256,9 @@ fn signature_verifies( // prefix. hdr.strip_prefix("&s=") }) else { - panic!("no x-oxide-signature header for secret with ID {secret_id} found"); + panic!( + "no x-oxide-signature header for secret with ID {secret_id} found" + ); }; let sig_bytes = hex::decode(sig_hdr) .expect("x-oxide-signature signature value should be a hex string"); @@ -1054,24 +1056,28 @@ async fn test_probe_resends_failed_deliveries( }; // Publish both events - dbg!(nexus - .webhook_event_publish( - &opctx, - event1_id, - WebhookEventClass::TestFoo, - serde_json::json!({"hello": "world"}), - ) - .await - .expect("event1 should be published successfully")); - dbg!(nexus - .webhook_event_publish( - &opctx, - event2_id, - WebhookEventClass::TestFoo, - serde_json::json!({"hello": "emeryville"}), - ) - .await - .expect("event2 should be published successfully")); + dbg!( + nexus + .webhook_event_publish( + &opctx, + event1_id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello": "world"}), + ) + .await + .expect("event1 should be published successfully") + ); + dbg!( + nexus + .webhook_event_publish( + &opctx, + event2_id, + WebhookEventClass::TestFoo, + serde_json::json!({"hello": "emeryville"}), + ) + .await + .expect("event2 should be published successfully") + ); dbg!(activate_background_task(internal_client, "webhook_dispatcher").await); dbg!( From 52fa34650167a150667778684b04433191edce2f Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 09:44:03 -0700 Subject: [PATCH 132/168] reorganize methods in webhooks.rs --- nexus/src/app/webhook.rs | 268 +++++++++++++++++++++------------------ 1 file changed, 142 insertions(+), 126 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 670cdc2bc48..e5e9574025e 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -55,6 +55,36 @@ use std::time::Instant; use uuid::Uuid; impl super::Nexus { + pub async fn webhook_event_publish( + &self, + opctx: &OpContext, + id: WebhookEventUuid, + event_class: WebhookEventClass, + event: serde_json::Value, + ) -> Result { + let event = self + .datastore() + .webhook_event_create(opctx, id, event_class, event) + .await?; + slog::debug!( + &opctx.log, + "enqueued webhook event"; + "event_id" => ?id, + "event_class" => %event.event_class, + "time_created" => ?event.identity.time_created, + ); + + // Once the event has been isnerted, activate the dispatcher task to + // ensure its propagated to receivers. + self.background_tasks.task_webhook_dispatcher.activate(); + + Ok(event) + } + + // + // Lookups + // + pub fn webhook_receiver_lookup<'a>( &'a self, opctx: &'a OpContext, @@ -86,6 +116,10 @@ impl super::Nexus { Ok(event) } + // + // Receiver configuration API methods + // + pub async fn webhook_receiver_list( &self, opctx: &OpContext, @@ -139,6 +173,10 @@ impl super::Nexus { self.datastore().webhook_rx_delete(&opctx, &authz_rx, &db_rx).await } + // + // Receiver secret API methods + // + pub async fn webhook_receiver_secrets_list( &self, opctx: &OpContext, @@ -148,132 +186,31 @@ impl super::Nexus { self.datastore().webhook_rx_secret_list(opctx, &authz_rx).await } - pub async fn webhook_receiver_event_resend( + pub async fn webhook_receiver_secret_add( &self, opctx: &OpContext, rx: lookup::WebhookReceiver<'_>, - event: lookup::WebhookEvent<'_>, - ) -> CreateResult { + secret: String, + ) -> Result { let (authz_rx,) = rx.lookup_for(authz::Action::CreateChild).await?; - let (authz_event, event) = event.fetch().await?; - let datastore = self.datastore(); - - let is_subscribed = datastore - .webhook_rx_is_subscribed_to_event(opctx, &authz_rx, &authz_event) + let secret = WebhookSecret::new(authz_rx.id(), secret); + let WebhookSecret { identity, .. } = self + .datastore() + .webhook_rx_secret_create(opctx, &authz_rx, secret) .await?; - if !is_subscribed { - return Err(Error::invalid_request(format!( - "cannot resend event: receiver is not subscribed to the '{}' \ - event class", - event.event_class, - ))); - } - - let delivery = WebhookDelivery::new( - &event, - &authz_rx.id(), - WebhookDeliveryTrigger::Resend, - ); - let delivery_id = delivery.id.into(); - - if let Err(e) = - datastore.webhook_delivery_create_batch(opctx, vec![delivery]).await - { - slog::error!( - &opctx.log, - "failed to create new delivery to resend webhook event"; - "rx_id" => ?authz_rx.id(), - "event_id" => ?authz_event.id(), - "event_class" => %event.event_class, - "delivery_id" => ?delivery_id, - "error" => %e, - ); - return Err(e); - } - + let secret_id = identity.id; slog::info!( &opctx.log, - "resending webhook event"; + "added secret to webhook receiver"; "rx_id" => ?authz_rx.id(), - "event_id" => ?authz_event.id(), - "event_class" => %event.event_class, - "delivery_id" => ?delivery_id, - ); - - self.background_tasks.task_webhook_deliverator.activate(); - Ok(delivery_id) - } - - pub async fn webhook_event_publish( - &self, - opctx: &OpContext, - id: WebhookEventUuid, - event_class: WebhookEventClass, - event: serde_json::Value, - ) -> Result { - let event = self - .datastore() - .webhook_event_create(opctx, id, event_class, event) - .await?; - slog::debug!( - &opctx.log, - "enqueued webhook event"; - "event_id" => ?id, - "event_class" => %event.event_class, - "time_created" => ?event.identity.time_created, + "secret_id" => ?secret_id, ); - - // Once the event has been isnerted, activate the dispatcher task to - // ensure its propagated to receivers. - self.background_tasks.task_webhook_dispatcher.activate(); - - Ok(event) + Ok(views::WebhookSecretId { id: secret_id.into_untyped_uuid() }) } - pub async fn webhook_receiver_delivery_list( - &self, - opctx: &OpContext, - rx: lookup::WebhookReceiver<'_>, - filter: params::WebhookDeliveryStateFilter, - pagparams: &DataPageParams<'_, Uuid>, - ) -> ListResultVec { - let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; - let only_states = if filter.include_all() { - Vec::new() - } else { - let mut states = Vec::with_capacity(3); - if filter.include_failed() { - states.push(WebhookDeliveryState::Failed); - } - if filter.include_pending() { - states.push(WebhookDeliveryState::Pending); - } - if filter.include_delivered() { - states.push(WebhookDeliveryState::Delivered); - } - states - }; - let deliveries = self - .datastore() - .webhook_rx_delivery_list( - opctx, - &authz_rx.id(), - // No probes; they could have their own list endpoint later... - &[ - WebhookDeliveryTrigger::Event, - WebhookDeliveryTrigger::Resend, - ], - only_states, - pagparams, - ) - .await? - .into_iter() - .map(|(delivery, class, attempts)| { - delivery.to_api_delivery(class, &attempts) - }) - .collect(); - Ok(deliveries) - } + // + // Receiver event delivery API methods + // pub async fn webhook_receiver_probe( &self, @@ -388,26 +325,105 @@ impl super::Nexus { }) } - pub async fn webhook_receiver_secret_add( + pub async fn webhook_receiver_event_resend( &self, opctx: &OpContext, rx: lookup::WebhookReceiver<'_>, - secret: String, - ) -> Result { + event: lookup::WebhookEvent<'_>, + ) -> CreateResult { let (authz_rx,) = rx.lookup_for(authz::Action::CreateChild).await?; - let secret = WebhookSecret::new(authz_rx.id(), secret); - let WebhookSecret { identity, .. } = self - .datastore() - .webhook_rx_secret_create(opctx, &authz_rx, secret) + let (authz_event, event) = event.fetch().await?; + let datastore = self.datastore(); + + let is_subscribed = datastore + .webhook_rx_is_subscribed_to_event(opctx, &authz_rx, &authz_event) .await?; - let secret_id = identity.id; + if !is_subscribed { + return Err(Error::invalid_request(format!( + "cannot resend event: receiver is not subscribed to the '{}' \ + event class", + event.event_class, + ))); + } + + let delivery = WebhookDelivery::new( + &event, + &authz_rx.id(), + WebhookDeliveryTrigger::Resend, + ); + let delivery_id = delivery.id.into(); + + if let Err(e) = + datastore.webhook_delivery_create_batch(opctx, vec![delivery]).await + { + slog::error!( + &opctx.log, + "failed to create new delivery to resend webhook event"; + "rx_id" => ?authz_rx.id(), + "event_id" => ?authz_event.id(), + "event_class" => %event.event_class, + "delivery_id" => ?delivery_id, + "error" => %e, + ); + return Err(e); + } + slog::info!( &opctx.log, - "added secret to webhook receiver"; + "resending webhook event"; "rx_id" => ?authz_rx.id(), - "secret_id" => ?secret_id, + "event_id" => ?authz_event.id(), + "event_class" => %event.event_class, + "delivery_id" => ?delivery_id, ); - Ok(views::WebhookSecretId { id: secret_id.into_untyped_uuid() }) + + self.background_tasks.task_webhook_deliverator.activate(); + Ok(delivery_id) + } + + pub async fn webhook_receiver_delivery_list( + &self, + opctx: &OpContext, + rx: lookup::WebhookReceiver<'_>, + filter: params::WebhookDeliveryStateFilter, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { + let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; + let only_states = if filter.include_all() { + Vec::new() + } else { + let mut states = Vec::with_capacity(3); + if filter.include_failed() { + states.push(WebhookDeliveryState::Failed); + } + if filter.include_pending() { + states.push(WebhookDeliveryState::Pending); + } + if filter.include_delivered() { + states.push(WebhookDeliveryState::Delivered); + } + states + }; + let deliveries = self + .datastore() + .webhook_rx_delivery_list( + opctx, + &authz_rx.id(), + // No probes; they could have their own list endpoint later... + &[ + WebhookDeliveryTrigger::Event, + WebhookDeliveryTrigger::Resend, + ], + only_states, + pagparams, + ) + .await? + .into_iter() + .map(|(delivery, class, attempts)| { + delivery.to_api_delivery(class, &attempts) + }) + .collect(); + Ok(deliveries) } } From 5a191d0aee260fa2576788db6bc6afb005846ba3 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 09:54:22 -0700 Subject: [PATCH 133/168] make secret-delete more like other API endpoints --- nexus/src/app/webhook.rs | 31 ++++++++++++++++++++++ nexus/src/external_api/http_entrypoints.rs | 20 +++----------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index e5e9574025e..007c1c778c0 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -106,6 +106,18 @@ impl super::Nexus { } } + pub fn webhook_secret_lookup<'a>( + &'a self, + opctx: &'a OpContext, + secret_selector: params::WebhookSecretSelector, + ) -> LookupResult> { + let lookup = LookupPath::new(&opctx, self.datastore()) + .webhook_secret_id(WebhookSecretUuid::from_untyped_uuid( + secret_selector.secret_id, + )); + Ok(lookup) + } + pub fn webhook_event_lookup<'a>( &'a self, opctx: &'a OpContext, @@ -208,6 +220,25 @@ impl super::Nexus { Ok(views::WebhookSecretId { id: secret_id.into_untyped_uuid() }) } + pub async fn webhook_receiver_secret_delete( + &self, + opctx: &OpContext, + secret: lookup::WebhookSecret<'_>, + ) -> DeleteResult { + let (authz_rx, authz_secret) = + secret.lookup_for(authz::Action::Delete).await?; + self.datastore() + .webhook_rx_secret_delete(&opctx, &authz_rx, &authz_secret) + .await?; + slog::info!( + &opctx.log, + "deleted secret from webhook receiver"; + "rx_id" => ?authz_rx.id(), + "secret_id" => ?authz_secret.id(), + ); + Ok(()) + } + // // Receiver event delivery API methods // diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 88e8b4aa4b8..11a164de93e 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7936,9 +7936,6 @@ impl NexusExternalApi for NexusExternalApiImpl { rqctx: RequestContext, path_params: Path, ) -> Result { - use nexus_db_queries::db::lookup::LookupPath; - use omicron_uuid_kinds::WebhookSecretUuid; - let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; @@ -7946,20 +7943,11 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let params::WebhookSecretSelector { secret_id } = - path_params.into_inner(); + let secret_selector = path_params.into_inner(); + let secret = + nexus.webhook_secret_lookup(&opctx, secret_selector)?; + nexus.webhook_receiver_secret_delete(&opctx, secret).await?; - let (authz_rx, authz_secret) = - LookupPath::new(&opctx, nexus.datastore()) - .webhook_secret_id(WebhookSecretUuid::from_untyped_uuid( - secret_id, - )) - .lookup_for(authz::Action::Delete) - .await?; - nexus - .datastore() - .webhook_rx_secret_delete(&opctx, &authz_rx, &authz_secret) - .await?; Ok(HttpResponseDeleted()) }; apictx From 1e4ce47e9a2705ed21ee75c1579956605ab02fce Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 14:37:48 -0700 Subject: [PATCH 134/168] add event classes list/fetch APIs --- nexus/Cargo.toml | 1 + nexus/db-model/src/webhook_event_class.rs | 64 +++++++++++ nexus/db-model/src/webhook_rx.rs | 8 ++ nexus/external-api/src/lib.rs | 5 +- nexus/src/app/webhook.rs | 120 +++++++++++++++++++++ nexus/src/external_api/http_entrypoints.rs | 51 +++++---- openapi/nexus.json | 86 ++++++++------- 7 files changed, 271 insertions(+), 64 deletions(-) diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 1d573ae5091..3c609dfa0ac 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -76,6 +76,7 @@ qorb.workspace = true rand.workspace = true range-requests.workspace = true ref-cast.workspace = true +regex.workspace = true reqwest = { workspace = true, features = ["json"] } ring.workspace = true samael.workspace = true diff --git a/nexus/db-model/src/webhook_event_class.rs b/nexus/db-model/src/webhook_event_class.rs index 3654693b079..6f79cbe7748 100644 --- a/nexus/db-model/src/webhook_event_class.rs +++ b/nexus/db-model/src/webhook_event_class.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; +use nexus_types::external_api::views; use serde::de::{self, Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use std::fmt; @@ -49,6 +50,38 @@ impl WebhookEventClass { } } + /// Returns `true` if this event class is only used for testing and should + /// not be incldued in the public event class list API endpoint. + pub fn is_test(&self) -> bool { + matches!( + self, + Self::TestFoo + | Self::TestFooBar + | Self::TestFooBaz + | Self::TestQuuxBar + | Self::TestQuuxBarBaz + ) + } + + /// Returns a human-readable description string describing this event class. + pub fn description(&self) -> &'static str { + match self { + Self::Probe => { + "Synthetic events sent for webhook receiver liveness probes.\n\ + Receivers should return 2xx HTTP responses for these events, \ + but they should NOT be treated as notifications of an actual \ + event in the system." + } + Self::TestFoo + | Self::TestFooBar + | Self::TestFooBaz + | Self::TestQuuxBar + | Self::TestQuuxBarBaz => { + "This is a test of the emergency alert system" + } + } + } + /// All webhook event classes. pub const ALL_CLASSES: &'static [Self] = ::VARIANTS; @@ -98,6 +131,15 @@ impl std::str::FromStr for WebhookEventClass { } } +impl From for views::EventClass { + fn from(class: WebhookEventClass) -> Self { + Self { + name: class.to_string(), + description: class.description().to_string(), + } + } +} + #[derive(Debug, Eq, PartialEq)] pub struct EventClassParseError(()); @@ -127,4 +169,26 @@ mod tests { assert_eq!(Ok(dbg!(variant)), dbg!(variant.to_string().parse())); } } + + // This is mainly a regression test to ensure that, should anyone add new + // `test.` variants in future, the `WebhookEventClass::is_test()` method + // returns `true` for them. + #[test] + fn test_is_test() { + let problematic_variants = WebhookEventClass::ALL_CLASSES + .iter() + .copied() + .filter(|variant| { + variant.as_str().starts_with("test.") && !variant.is_test() + }) + .collect::>(); + assert_eq!( + problematic_variants, + Vec::::new(), + "you have added one or more new `test.*` webhook event class \ + variant(s), but you seem to have not updated the \ + `WebhookEventClass::is_test()` method!\nthe problematic \ + variant(s) are: {problematic_variants:?}", + ); + } } diff --git a/nexus/db-model/src/webhook_rx.rs b/nexus/db-model/src/webhook_rx.rs index b91c35c7bc0..35ec070763b 100644 --- a/nexus/db-model/src/webhook_rx.rs +++ b/nexus/db-model/src/webhook_rx.rs @@ -245,6 +245,14 @@ impl FromStr for WebhookGlob { } } +impl TryFrom for WebhookGlob { + type Error = Error; + fn try_from(glob: String) -> Result { + let regex = Self::regex_from_glob(&glob)?; + Ok(Self { glob, regex }) + } +} + impl WebhookGlob { fn regex_from_glob(glob: &str) -> Result { let seg2regex = |segment: &str, diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index eeb46149c45..701bc3e0baa 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3515,9 +3515,10 @@ pub trait NexusExternalApi { }] async fn webhook_event_class_list( rqctx: RequestContext, - query_params: Query< - PaginationParams, + pag_params: Query< + PaginationParams, >, + filter: Query, ) -> Result>, HttpError>; /// Fetch details on an event class by name. diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 007c1c778c0..66eab7eac85 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -18,6 +18,7 @@ use http::HeaderValue; use nexus_db_model::WebhookReceiver; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db; use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::SqlU8; @@ -128,6 +129,64 @@ impl super::Nexus { Ok(event) } + // + // Event class API + // + + pub fn webhook_event_class_list( + params::EventClassFilter { filter }: params::EventClassFilter, + pagparams: DataPageParams<'_, params::EventClassPage>, + ) -> ListResultVec { + let regex = if let Some(glob) = filter { + let glob = db::model::WebhookGlob::try_from(glob)?; + let re = regex::Regex::new(&glob.regex) + .map_err(|e| Error::InternalError { internal_message: format!( + "valid event class globs ({glob:?}) should always produce a valid regex: {e:?}" + )})?; + Some(re) + } else { + None + }; + let start = if let Some(params::EventClassPage { last_seen }) = + pagparams.marker + { + let start = WebhookEventClass::ALL_CLASSES + .iter() + .enumerate() + .find_map(|(idx, class)| { + if class.as_str() == last_seen { Some(idx) } else { None } + }); + match start { + Some(start) => start + 1, + None => return Ok(Vec::new()), + } + } else { + 0 + }; + if start > WebhookEventClass::ALL_CLASSES.len() { + return Ok(Vec::new()); + } + let result = WebhookEventClass::ALL_CLASSES[start..] + .iter() + .filter_map(|&class| { + // Skip test classes, as they should not be used in the public + // API, except in test builds, where we need them + // for, you know... testing... + if !cfg!(test) && class.is_test() { + return None; + } + if let Some(ref regex) = regex { + if !regex.is_match(class.as_str()) { + return None; + } + } + Some(class.into()) + }) + .take(pagparams.limit.get() as usize) + .collect::>(); + Ok(result) + } + // // Receiver configuration API methods // @@ -744,3 +803,64 @@ impl<'a> ReceiverClient<'a> { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Nexus; + use std::num::NonZeroU32; + + #[test] + fn test_event_class_list() { + #[track_caller] + fn list( + filter: Option<&str>, + last_seen: Option<&str>, + limit: u32, + ) -> Vec { + let filter = params::EventClassFilter { + filter: dbg!(filter).map(ToString::to_string), + }; + let marker = dbg!(last_seen).map(|last_seen| { + params::EventClassPage { last_seen: last_seen.to_string() } + }); + let result = Nexus::webhook_event_class_list( + filter, + DataPageParams { + marker: marker.as_ref(), + direction: dropshot::PaginationOrder::Ascending, + limit: NonZeroU32::new(dbg!(limit)).unwrap(), + }, + ); + + // Throw away the description fields + dbg!(result) + .unwrap() + .into_iter() + .map(|view| view.name) + .collect::>() + } + + // Paginated class list, without a glob filter. + let classes = list(None, None, 3); + assert_eq!(classes, &["probe", "test.foo", "test.foo.bar"]); + let classes = list(None, Some("test.foo.bar"), 3); + assert_eq!( + classes, + &["test.foo.baz", "test.quux.bar", "test.quux.bar.baz"] + ); + // Don't assert that a third list will return no more results, since + // more events may be added in the future, and we don't have a filter. + + // Try a filter for only `test.**` events. + let filter = Some("test.**"); + let classes = list(filter, None, 2); + assert_eq!(classes, &["test.foo", "test.foo.bar"]); + let classes = list(filter, Some("test.foo.bar"), 2); + assert_eq!(classes, &["test.foo.baz", "test.quux.bar"]); + let classes = list(filter, Some("test.quux.bar"), 2); + assert_eq!(classes, &["test.quux.bar.baz"]); + let classes = list(filter, Some("test.quux.bar.baz"), 2); + assert_eq!(classes, Vec::::new()); + } +} diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 11a164de93e..8f2b4f4056a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7670,21 +7670,34 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_event_class_list( rqctx: RequestContext, - _query_params: Query< - PaginationParams, + pag_params: Query< + PaginationParams, >, + filter: Query, ) -> Result>, HttpError> { let apictx = rqctx.context(); let handler = async { - let nexus = &apictx.context.nexus; - - let opctx = + let _opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let query = pag_params.into_inner(); + let filter = filter.into_inner(); + let marker = match query.page { + WhichPage::First(_) => None, + WhichPage::Next(ref addr) => Some(addr), + }; + let pag_params = DataPageParams { + limit: rqctx.page_limit(&query)?, + direction: PaginationOrder::Ascending, + marker, + }; + let event_classes = + crate::Nexus::webhook_event_class_list(filter, pag_params)?; + Ok(HttpResponseOk(ResultsPage::new( + event_classes, + &EmptyScanParams {}, + |class: &views::EventClass, _| class.name.clone(), + )?)) }; apictx .context @@ -7695,19 +7708,21 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn webhook_event_class_view( rqctx: RequestContext, - _path_params: Path, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { - let nexus = &apictx.context.nexus; - - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; + let params::EventClassSelector { name } = path_params.into_inner(); - Err(nexus - .unimplemented_todo(&opctx, crate::app::Unimpl::Public) - .await - .into()) + let event_class = name + .parse::() + .map_err(|_| { + Error::non_resourcetype_not_found(format!( + "{name:?} is not a webhook event class" + )) + })? + .into(); + Ok(HttpResponseOk(event_class)) }; apictx .context diff --git a/openapi/nexus.json b/openapi/nexus.json index 41944bb87a8..34b2293d914 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12077,15 +12077,6 @@ "summary": "List webhook event classes", "operationId": "webhook_event_class_list", "parameters": [ - { - "in": "query", - "name": "filter", - "description": "An optional glob pattern for filtering event class names.", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "limit", @@ -12105,6 +12096,15 @@ "nullable": true, "type": "string" } + }, + { + "in": "query", + "name": "filter", + "description": "An optional glob pattern for filtering event class names.", + "schema": { + "nullable": true, + "type": "string" + } } ], "responses": { @@ -12320,7 +12320,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WebhookUpdate" + "$ref": "#/components/schemas/WebhookReceiverUpdate" } } }, @@ -25659,6 +25659,38 @@ "items" ] }, + "WebhookReceiverUpdate": { + "description": "Parameters to update a webhook configuration.", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "endpoint": { + "nullable": true, + "description": "The URL that webhook notification requests should be sent to", + "type": "string", + "format": "uri" + }, + "events": { + "nullable": true, + "description": "A list of webhook event classes to subscribe to.\n\nIf this list is empty, the webhook will not be subscribed to any events.", + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "WebhookSecretCreate": { "type": "object", "properties": { @@ -25698,40 +25730,6 @@ "secrets" ] }, - "WebhookUpdate": { - "description": "Parameters to update a webhook configuration.", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "endpoint": { - "description": "The URL that webhook notification requests should be sent to", - "type": "string", - "format": "uri" - }, - "events": { - "description": "A list of webhook event classes to subscribe to.\n\nIf this list is empty, the webhook will not be subscribed to any events.", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - }, - "required": [ - "endpoint", - "events" - ] - }, "NameOrIdSortMode": { "description": "Supported set of sort modes for scanning by name or id", "oneOf": [ From 43c18421973e6941b6a79a75bdba4d4abe3058c2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 15:31:46 -0700 Subject: [PATCH 135/168] webhook event classes participate in authz now --- nexus/auth/src/authz/api_resources.rs | 41 +++++++++++++++++++ nexus/auth/src/authz/omicron.polar | 10 +++++ nexus/auth/src/authz/oso_generic.rs | 1 + nexus/src/app/webhook.rs | 19 ++++++++- nexus/src/external_api/http_entrypoints.rs | 16 ++++++-- nexus/tests/integration_tests/endpoints.rs | 16 ++++++++ .../output/uncovered-authz-endpoints.txt | 2 - 7 files changed, 98 insertions(+), 7 deletions(-) diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index b1e07b5460b..70f2dc49fb7 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -711,6 +711,47 @@ impl AuthorizedResource for TargetReleaseConfig { } } +/// Synthetic resource used for modeling access to the list of webhook event +/// classes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WebhookEventClassList; +pub const WEBHOOK_EVENT_CLASS_LIST: WebhookEventClassList = + WebhookEventClassList {}; + +impl oso::PolarClass for WebhookEventClassList { + fn get_polar_class_builder() -> oso::ClassBuilder { + // Roles are not directly attached to EventClassList + oso::Class::builder() + .with_equality_check() + .add_attribute_getter("fleet", |_| FLEET) + } +} + +impl AuthorizedResource for WebhookEventClassList { + fn load_roles<'fut>( + &'fut self, + opctx: &'fut OpContext, + authn: &'fut authn::Context, + roleset: &'fut mut RoleSet, + ) -> futures::future::BoxFuture<'fut, Result<(), Error>> { + load_roles_for_resource_tree(&FLEET, opctx, authn, roleset).boxed() + } + + fn on_unauthorized( + &self, + _: &Authz, + error: Error, + _: AnyActor, + _: Action, + ) -> Error { + error + } + + fn polar_class(&self) -> oso::Class { + Self::get_polar_class() + } +} + // Main resource hierarchy: Projects and their resources authz_resource! { diff --git a/nexus/auth/src/authz/omicron.polar b/nexus/auth/src/authz/omicron.polar index 319e6c4e04f..80abeaa1458 100644 --- a/nexus/auth/src/authz/omicron.polar +++ b/nexus/auth/src/authz/omicron.polar @@ -604,3 +604,13 @@ resource WebhookSecret { has_relation(rx: WebhookReceiver, "parent_webhook_receiver", secret: WebhookSecret) if secret.webhook_receiver = rx; + +resource WebhookEventClassList { + permissions = [ "list_children" ]; + relations = { parent_fleet: Fleet }; + + "list_children" if "viewer" on "parent_fleet"; +} + +has_relation(fleet: Fleet, "parent_fleet", collection: WebhookEventClassList) + if collection.fleet = fleet; diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index b83ab91f01c..3b47fbfdd0a 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -115,6 +115,7 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { SiloIdentityProviderList::get_polar_class(), SiloUserList::get_polar_class(), TargetReleaseConfig::get_polar_class(), + WebhookEventClassList::get_polar_class(), ]; for c in classes { oso_builder = oso_builder.register_class(c)?; diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 66eab7eac85..fcaa274cf0e 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -132,8 +132,23 @@ impl super::Nexus { // // Event class API // + pub async fn webhook_event_class_list( + &self, + opctx: &OpContext, + filter: params::EventClassFilter, + pagparams: DataPageParams<'_, params::EventClassPage>, + ) -> ListResultVec { + opctx + .authorize( + authz::Action::ListChildren, + &authz::WEBHOOK_EVENT_CLASS_LIST, + ) + .await?; + Self::actually_list_event_classes(filter, pagparams) + } - pub fn webhook_event_class_list( + // This is factored out to avoid having to make a whole Nexus to test it. + fn actually_list_event_classes( params::EventClassFilter { filter }: params::EventClassFilter, pagparams: DataPageParams<'_, params::EventClassPage>, ) -> ListResultVec { @@ -824,7 +839,7 @@ mod tests { let marker = dbg!(last_seen).map(|last_seen| { params::EventClassPage { last_seen: last_seen.to_string() } }); - let result = Nexus::webhook_event_class_list( + let result = Nexus::actually_list_event_classes( filter, DataPageParams { marker: marker.as_ref(), diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 8f2b4f4056a..dd8fd764e4a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7677,7 +7677,8 @@ impl NexusExternalApi for NexusExternalApiImpl { ) -> Result>, HttpError> { let apictx = rqctx.context(); let handler = async { - let _opctx = + let nexus = &apictx.context.nexus; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let query = pag_params.into_inner(); @@ -7691,8 +7692,9 @@ impl NexusExternalApi for NexusExternalApiImpl { direction: PaginationOrder::Ascending, marker, }; - let event_classes = - crate::Nexus::webhook_event_class_list(filter, pag_params)?; + let event_classes = nexus + .webhook_event_class_list(&opctx, filter, pag_params) + .await?; Ok(HttpResponseOk(ResultsPage::new( event_classes, &EmptyScanParams {}, @@ -7713,6 +7715,14 @@ impl NexusExternalApi for NexusExternalApiImpl { let apictx = rqctx.context(); let handler = async { let params::EventClassSelector { name } = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx + .authorize( + authz::Action::ListChildren, + &authz::WEBHOOK_EVENT_CLASS_LIST, + ) + .await?; let event_class = name .parse::() diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index f5116f2e9bb..a2451bb059e 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -1146,6 +1146,10 @@ pub static DEMO_TARGET_RELEASE: LazyLock = // Webhooks pub static WEBHOOK_RECEIVERS_URL: &'static str = "/v1/webhooks/receivers"; +pub static WEBHOOK_EVENT_CLASSES_URL: &'static str = + "/v1/webhooks/event-classes"; +pub static WEBHOOK_EVENT_CLASS_FOO_BAR_URL: &'static str = + "/v1/webhooks/event-classes/test.foo.bar"; pub static DEMO_WEBHOOK_RECEIVER_NAME: LazyLock = LazyLock::new(|| "my-great-webhook".parse().unwrap()); @@ -2830,5 +2834,17 @@ pub static VERIFY_ENDPOINTS: LazyLock> = unprivileged_access: UnprivilegedAccess::None, allowed_methods: vec![AllowedMethod::Get], }, + VerifyEndpoint { + url: &WEBHOOK_EVENT_CLASSES_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Get], + }, + VerifyEndpoint { + url: &WEBHOOK_EVENT_CLASS_FOO_BAR_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Get], + }, ] }); diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 78e1dd048f8..258d9065fe6 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -9,8 +9,6 @@ ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") networking_switch_port_status (get "/v1/system/hardware/switch-port/{port}/status") -webhook_event_class_list (get "/v1/webhooks/event-classes") -webhook_event_class_view (get "/v1/webhooks/event-classes/{name}") support_bundle_head (head "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_head_file (head "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") device_auth_request (post "/device/auth") From b91f0c9dc80dab49a76e8f226167cba3143f5c80 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 16:42:01 -0700 Subject: [PATCH 136/168] big theory statement --- nexus/src/app/webhook.rs | 159 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 6 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index fcaa274cf0e..85498e73b7c 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -6,7 +6,129 @@ //! //! # Webhooks: Theory and Practice //! -//! [RFD 538] describes the user-facing +//! [RFD 538] describes the user-facing interface for Oxide rack webhooks. +//! However, that RFD does not describe internal implementation details of the +//! webhook implementation, the key players, their roles, and interactions. +//! Instead, the implementation of webhooks are discussed here. +//! +//! ## Dramatis Personae +//! +//! There are two key elements in our webhook design: +//! +//! + **Webhook receivers** are the endpoints external to the rack to which +//! webhook requests are sent. In the context of the control plane, the term +//! "webhook receiver" refers to the configuration and state associated with +//! such an endpoint. Most other entities in the webhook API are chiild +//! resources of the [`WebhookReceiver`] API resource. +//! +//! + **Webhook events** represent events in the system for which webhook +//! notifications are generated and sent to receivers. The +//! [`Nexus::webhook_event_publish`] is called to record a new event and +//! publish it to receivers. +//! +//! Events are categorized into [event classes], as described in RFD +//! 538. Receivers *subscribe* to these classes, indicating that they wish to +//! when an event with a particular class occurs. +//! +//! Two background tasks implement the reliable persistent workflow of +//! determining what events should be sent to what receiver, and performing the +//! actual HTTP requests to send the event to the receiver: +//! +//! + The `webhook_dispatcher` task is responsible for *dispatching* events to +//! receivers. For each event that has not yet been dispatched, the task +//! queries the database for webhook receivers that have subscribed to that +//! event, and creates a *delivery record* in the `webhook_delivery` table, +//! indicating that the event should be sent to that receiver. +//! +//! + The `webhook_deliverator`[^1] task reads these delivery records and sends +//! HTTP requests to the receiver endpoint for each delivery that is +//! currently in flight. The deliverator is responsible for recording the +//! status of each *delivery attempt*. Retries and retry backoff are +//! the responsibility of the deliverator. +//! +//! ## Event Subscriptions +//! +//! A receiver's subscriptions take one of two forms: +//! +//! + **Exact** subscriptions are when a receiver subscribes to a specific event +//! class string. These are represented by entries in the +//! `webhook_rx_event_subscription` table in CockroachDB. +//! +//! + **Glob** subscriptions include wildcard segments that may match multiple +//! values. The globbing syntax is discussed in greater detail in RFD 538. +//! +//! We implement glob subscriptions by evaluating the glob against the list of +//! known webhook event classes when the glob is *created*, and creating +//! corresponding exact subscriptions for each event class that matches the +//! glob. This way, we need not perform complex pattern matching in the +//! database when dispatching an event, and can instead simply query for the +//! existence of a record in the `webhook_rx_event_subscription` table. Each +//! exact subscription entry generated by a glob records which glob it came +//! from, which is used when a receiver's subscriptions change. +//! +//! Because the generation of exact subscriptions from globs occurs when the +//! subscription is created, globs must be *reprocessed* when new event classes +//! are added to the system, generating new exact subscriptions for any +//! newly-added event classes that match the glob, and potentially removing +//! subscriptions to any defunct event classes This could occur in any software +//! release where new kinds of events are implemented. Therefore, when glob +//! subscriptions are created, we record the database schema version as part of +//! that glob subscription. Because event classes are represented as a SQL +//! `enum` type, we know that any change to the event classes should change the +//! database schema version as well. This way, we can detect whether a glob's +//! list of subscriptions are up to date. The `webhook_dispatcher` background +//! task will query the database for any globs which were last reprocessed at +//! earlier database schema versions and reprocess those globs prior to +//! attempting to dispatch events to receivers. +//! +//! ## Deliveries, Delivery Attempts, and Liveness Probes +//! +//! A *delivery* represents the process of sending HTTP request(s) representing +//! a webhook event to a receiver. Failed HTTP requests are retried up to two +//! times times, so a delivery may consist of up to three *delivery attempts*. +//! Each time the `webhook_deliverator` background task is activated, it +//! searches for deliveries which have not yet succeeded or permanently failed, +//! hich are not presently being delivered by another Nexus, and for which the +//! backoff period for any prior failed delivery attempts has elapsed. It then +//! sends an HTTP request to the webhook receiver, and records the result, +//! creating a new `webhook_delivery_attempt` record and updating the +//! `webhook_delivery` record. +//! +//! Multiple Nexii use an advisory lease mechanism to avoid attempting to +//! deliver the same event simultaneously, by setting their UUID and a timestamp +//! on the `webhook_delivery` record. Because webhook delivery is +//! at-least-once, this lease mechanism is NOT REQUIRED FOR CORRECTNESS IN ANY +//! WAY, Andrew. :) Instead, it serves only to reduce duplicate work. +//! Therefore, should a Nexus acquire a lease on a delivery and fail to either +//! complete the delivery attempt within a period of time, another Nexus is +//! permitted to clobber its lease. +//! +//! Deliveries are created either because an event occurred and a webhook +//! receiver is subscribed to it, or because we were asked to resend a previous +//! delivery that failed permanently by exhausting its retry budget. Initial +//! deliveries are created by activations of the webhook dispatcher background +//! task. When creating a delivery, the data associated with the event record +//! in the `webhook_event` table is processed to produce the data payload that +//! will actually be sent to the receiver. Data which the receiver's service +//! account is not authorized to read is filtered out of the payload.[^2] +//! +//! Re-delivery of an event can be requested either via the event resend API +//! endpoint, or by a *liveness probe* succeeding. Liveness probes are +//! synthetic delivery requests sent to a webhook receiver to check whether it's +//! actually able to receive an event. They are triggered via the +//! [`webhook_receiver_probe`] API endpoint. A probe may optionally request +//! that any events for which all past deliveries have failed be resent if it +//! succeeds. Delivery records are also created to represent the outcome of a +//! probe. +//! +//! [RFD 538]: https://rfd.shared.oxide.computer/538 +//! [event classes]: https://rfd.shared.oxide.computer/rfd/538#_event_classes +//! +//! [^1]: Read _Snow Crash_, if you haven't already. +//! [^1]: Presently, all weebhook receivers have the fleet.viewer role, so +//! this "filtering" doesn't actually do anything. When webhook receivers +//! with more restrictive permissions are implemented, please rememvber to +//! delete this footnote. use crate::app::external_dns; use anyhow::Context; @@ -56,6 +178,15 @@ use std::time::Instant; use uuid::Uuid; impl super::Nexus { + /// Publish a new webhook event, with the provided `id`, `event_class`, and + /// JSON data payload. + /// + /// If this method returns `Ok`, the event has been durably recorded in + /// CockroachDB. Once the new event record is inserted into the database, + /// the webhook dispatcher background task is activated to dispatch the + /// event to receivers. However, if (for whatever reason) this Nexus fails + /// to do that, the event remains durably in the database to be dispatched + /// and delivered by someone else. pub async fn webhook_event_publish( &self, opctx: &OpContext, @@ -75,7 +206,7 @@ impl super::Nexus { "time_created" => ?event.identity.time_created, ); - // Once the event has been isnerted, activate the dispatcher task to + // Once the event has been inserted, activate the dispatcher task to // ensure its propagated to receivers. self.background_tasks.task_webhook_dispatcher.activate(); @@ -154,14 +285,22 @@ impl super::Nexus { ) -> ListResultVec { let regex = if let Some(glob) = filter { let glob = db::model::WebhookGlob::try_from(glob)?; - let re = regex::Regex::new(&glob.regex) - .map_err(|e| Error::InternalError { internal_message: format!( - "valid event class globs ({glob:?}) should always produce a valid regex: {e:?}" - )})?; + let re = regex::Regex::new(&glob.regex).map_err(|e| { + // This oughtn't happen, provided the code for producing the + // regex for a glob is correct. + Error::InternalError { + internal_message: format!( + "valid event class globs ({glob:?}) should always \ + produce a valid regex, and yet: {e:?}" + ), + } + })?; Some(re) } else { None }; + + // If we're resuming a previous scan, figure out where to start. let start = if let Some(params::EventClassPage { last_seen }) = pagparams.marker { @@ -178,9 +317,12 @@ impl super::Nexus { } else { 0 }; + + // This shouldn't ever happen, but...don't panic I guess. if start > WebhookEventClass::ALL_CLASSES.len() { return Ok(Vec::new()); } + let result = WebhookEventClass::ALL_CLASSES[start..] .iter() .filter_map(|&class| { @@ -557,6 +699,11 @@ pub(super) fn delivery_client( .build() } +/// Everything necessary to send a delivery request to a webhook receiver. +/// +/// This is its' own thing, rather than part of the `webhook_deliverator` +/// background task, as it is used both by the deliverator RPW and by the Nexus +/// API in the liveness probe endpoint. pub(crate) struct ReceiverClient<'a> { client: &'a reqwest::Client, rx: &'a WebhookReceiver, From 485439546e11dd5e47d7ea7351b46b7ef2f683f8 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 16:49:32 -0700 Subject: [PATCH 137/168] update OMDB tests --- dev-tools/omdb/tests/env.out | 24 ++++++++++++++++++++ dev-tools/omdb/tests/successes.out | 36 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index cf5eb09cad2..0e6a62d6478 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -191,6 +191,14 @@ task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports +task: "webhook_deliverator" + sends webhook delivery requests + + +task: "webhook_dispatcher" + dispatches queued webhook events to receivers + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -379,6 +387,14 @@ task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports +task: "webhook_deliverator" + sends webhook delivery requests + + +task: "webhook_dispatcher" + dispatches queued webhook events to receivers + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -554,6 +570,14 @@ task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports +task: "webhook_deliverator" + sends webhook delivery requests + + +task: "webhook_dispatcher" + dispatches queued webhook events to receivers + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index db8a5a04eb3..ad8576a4ab5 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -399,6 +399,14 @@ task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports +task: "webhook_deliverator" + sends webhook delivery requests + + +task: "webhook_dispatcher" + dispatches queued webhook events to receivers + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -775,6 +783,20 @@ task: "vpc_route_manager" started at (s ago) and ran for ms warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {}) +task: "webhook_deliverator" + configured period: every m + currently executing: no + last completed activation: , triggered by a periodic timer firing + started at (s ago) and ran for ms +warning: unknown background task: "webhook_deliverator" (don't know how to interpret details: Object {"by_rx": Object {}, "error": Null}) + +task: "webhook_dispatcher" + configured period: every m + currently executing: no + last completed activation: , triggered by a periodic timer firing + started at (s ago) and ran for ms +warning: unknown background task: "webhook_dispatcher" (don't know how to interpret details: Object {"dispatched": Array [], "errors": Array [], "glob_version": String("130.0.0"), "globs_reprocessed": Object {}, "no_receivers": Array []}) + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -1274,6 +1296,20 @@ task: "vpc_route_manager" started at (s ago) and ran for ms warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {}) +task: "webhook_deliverator" + configured period: every m + currently executing: no + last completed activation: , triggered by a periodic timer firing + started at (s ago) and ran for ms +warning: unknown background task: "webhook_deliverator" (don't know how to interpret details: Object {"by_rx": Object {}, "error": Null}) + +task: "webhook_dispatcher" + configured period: every m + currently executing: no + last completed activation: , triggered by a periodic timer firing + started at (s ago) and ran for ms +warning: unknown background task: "webhook_dispatcher" (don't know how to interpret details: Object {"dispatched": Array [], "errors": Array [], "glob_version": String("130.0.0"), "globs_reprocessed": Object {}, "no_receivers": Array []}) + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ From f88e4fae13d68ec554db52ed390c9ada1ce47cea Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 12 Mar 2025 16:50:26 -0700 Subject: [PATCH 138/168] that's not supposed to be there, sorry --- .vscode/settings.json | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index fa39d2c768d..00000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "rust-analyzer.check.features": [], - "rust-analyzer.check.command": "check", - "rust-analyzer.completion.autoimport.enable": false -} From b5e6a0eea762f0f056fdf0287020cc9b400dc6b1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 13 Mar 2025 09:26:20 -0700 Subject: [PATCH 139/168] rustdoc fixup --- nexus/src/app/webhook.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 85498e73b7c..d308ba70175 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -22,9 +22,9 @@ //! resources of the [`WebhookReceiver`] API resource. //! //! + **Webhook events** represent events in the system for which webhook -//! notifications are generated and sent to receivers. The -//! [`Nexus::webhook_event_publish`] is called to record a new event and -//! publish it to receivers. +//! notifications are generated and sent to receivers. The control plane +//! calls the [`Nexus::webhook_event_publish`] method to record a new event +//! and publish it to receivers. //! //! Events are categorized into [event classes], as described in RFD //! 538. Receivers *subscribe* to these classes, indicating that they wish to @@ -116,20 +116,21 @@ //! endpoint, or by a *liveness probe* succeeding. Liveness probes are //! synthetic delivery requests sent to a webhook receiver to check whether it's //! actually able to receive an event. They are triggered via the -//! [`webhook_receiver_probe`] API endpoint. A probe may optionally request -//! that any events for which all past deliveries have failed be resent if it -//! succeeds. Delivery records are also created to represent the outcome of a -//! probe. +//! [`Nexus::webhook_receiver_probe`] API endpoint. A probe may optionally +//! request that any events for which all past deliveries have failed be resent +//! if it succeeds. Delivery records are also created to represent the outcome +//! of a probe. //! //! [RFD 538]: https://rfd.shared.oxide.computer/538 //! [event classes]: https://rfd.shared.oxide.computer/rfd/538#_event_classes //! //! [^1]: Read _Snow Crash_, if you haven't already. -//! [^1]: Presently, all weebhook receivers have the fleet.viewer role, so +//! [^2]: Presently, all weebhook receivers have the fleet.viewer role, so //! this "filtering" doesn't actually do anything. When webhook receivers //! with more restrictive permissions are implemented, please rememvber to //! delete this footnote. +use crate::Nexus; use crate::app::external_dns; use anyhow::Context; use chrono::TimeDelta; @@ -137,7 +138,6 @@ use chrono::Utc; use hmac::{Hmac, Mac}; use http::HeaderName; use http::HeaderValue; -use nexus_db_model::WebhookReceiver; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -151,6 +151,7 @@ use nexus_db_queries::db::model::WebhookDeliveryState; use nexus_db_queries::db::model::WebhookDeliveryTrigger; use nexus_db_queries::db::model::WebhookEvent; use nexus_db_queries::db::model::WebhookEventClass; +use nexus_db_queries::db::model::WebhookReceiver; use nexus_db_queries::db::model::WebhookReceiverConfig; use nexus_db_queries::db::model::WebhookSecret; use nexus_types::external_api::params; @@ -177,7 +178,7 @@ use std::time::Duration; use std::time::Instant; use uuid::Uuid; -impl super::Nexus { +impl Nexus { /// Publish a new webhook event, with the provided `id`, `event_class`, and /// JSON data payload. /// From 4d68e89cb544ec7a21220ed1847e7db9e4a93241 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 13 Mar 2025 10:03:27 -0700 Subject: [PATCH 140/168] update lookup expectorate --- nexus/db-macros/outputs/project.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-macros/outputs/project.txt b/nexus/db-macros/outputs/project.txt index fa08115fdb3..5bf43aa0683 100644 --- a/nexus/db-macros/outputs/project.txt +++ b/nexus/db-macros/outputs/project.txt @@ -352,7 +352,7 @@ impl<'a> Project<'a> { let (authz_silo, _) = Silo::lookup_by_id_no_authz( opctx, datastore, - &db_row.silo_id, + &db_row.silo_id.into(), ) .await?; let authz_project = Self::make_authz( From 3169e64aece0a587a6cf07efce0a92e0d47ecf75 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 13 Mar 2025 11:03:10 -0700 Subject: [PATCH 141/168] omdb background task status --- dev-tools/omdb/src/bin/omdb/db.rs | 6 +- dev-tools/omdb/src/bin/omdb/helpers.rs | 6 + dev-tools/omdb/src/bin/omdb/nexus.rs | 297 +++++++++++++++++++++++++ dev-tools/omdb/tests/successes.out | 20 +- 4 files changed, 320 insertions(+), 9 deletions(-) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 989f271d895..8eb2df42504 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -22,6 +22,7 @@ use crate::check_allow_destructive::DestructiveOperationToken; use crate::helpers::CONNECTION_OPTIONS_HEADING; use crate::helpers::DATABASE_OPTIONS_HEADING; use crate::helpers::const_max_len; +use crate::helpers::display_option_blank; use anyhow::Context; use anyhow::bail; use async_bb8_diesel::AsyncConnection; @@ -6776,11 +6777,6 @@ async fn cmd_db_vmm_list( Ok(()) } -// Display an empty cell for an Option if it's None. -fn display_option_blank(opt: &Option) -> String { - opt.as_ref().map(|x| x.to_string()).unwrap_or_else(|| "".to_string()) -} - // Format a `chrono::DateTime` in RFC3339 with milliseconds precision and using // `Z` rather than the UTC offset for UTC timestamps, to save a few characters // of line width in tabular output. diff --git a/dev-tools/omdb/src/bin/omdb/helpers.rs b/dev-tools/omdb/src/bin/omdb/helpers.rs index 2ee82ff908b..d431c807fea 100644 --- a/dev-tools/omdb/src/bin/omdb/helpers.rs +++ b/dev-tools/omdb/src/bin/omdb/helpers.rs @@ -31,3 +31,9 @@ pub(crate) const fn const_max_len(strs: &[&str]) -> usize { } max } +// Display an empty cell for an Option if it's None. +pub(crate) fn display_option_blank( + opt: &Option, +) -> String { + opt.as_ref().map(|x| x.to_string()).unwrap_or_else(|| "".to_string()) +} diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index ec903d6de23..306f4a4ae59 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -9,6 +9,7 @@ use crate::check_allow_destructive::DestructiveOperationToken; use crate::db::DbUrlOptions; use crate::helpers::CONNECTION_OPTIONS_HEADING; use crate::helpers::const_max_len; +use crate::helpers::display_option_blank; use crate::helpers::should_colorize; use anyhow::Context; use anyhow::bail; @@ -962,6 +963,12 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { "tuf_artifact_replication" => { print_task_tuf_artifact_replication(details); } + "webhook_dispatcher" => { + print_task_webhook_dispatcher(details); + } + "webhook_deliverator" => { + print_task_webhook_deliverator(details); + } _ => { println!( "warning: unknown background task: {:?} \ @@ -2259,6 +2266,296 @@ fn print_task_tuf_artifact_replication(details: &serde_json::Value) { } } +fn print_task_webhook_dispatcher(details: &serde_json::Value) { + use nexus_types::internal_api::background::WebhookDispatched; + use nexus_types::internal_api::background::WebhookDispatcherStatus; + use nexus_types::internal_api::background::WebhookGlobStatus; + + let WebhookDispatcherStatus { + globs_reprocessed, + glob_version, + errors, + dispatched, + no_receivers, + } = match serde_json::from_value::(details.clone()) + { + Err(error) => { + eprintln!( + "warning: failed to interpret task details: {:?}: {:?}", + error, details + ); + return; + } + Ok(status) => status, + }; + + if !errors.is_empty() { + println!( + " task did not complete successfully! ({} errors)", + errors.len() + ); + for line in &errors { + println!(" > {line}"); + } + } + + const DISPATCHED: &str = "events dispatched:"; + const NO_RECEIVERS: &str = "events with no receivers subscribed:"; + const OUTDATED_GLOBS: &str = "outdated glob subscriptions:"; + const GLOBS_REPROCESSED: &str = "glob subscriptions reprocessed:"; + const ALREADY_REPROCESSED: &str = + "globs already reprocessed by another Nexus:"; + const GLOB_ERRORS: &str = "globs that failed to be reprocessed"; + const WIDTH: usize = const_max_len(&[ + DISPATCHED, + NO_RECEIVERS, + OUTDATED_GLOBS, + GLOBS_REPROCESSED, + ALREADY_REPROCESSED, + GLOB_ERRORS, + ]) + 1; + const NUM_WIDTH: usize = 3; + + println!(" {DISPATCHED:NUM_WIDTH$}", dispatched.len()); + if !dispatched.is_empty() { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct DispatchedRow { + // Don't include the typed UUID's kind in the table, which the + // TypedUuid fmt::Display impl will do... + event: Uuid, + subscribed: usize, + dispatched: usize, + } + let table_rows = dispatched.iter().map( + |&WebhookDispatched { event_id, subscribed, dispatched }| { + DispatchedRow { + event: event_id.into_untyped_uuid(), + subscribed, + dispatched, + } + }, + ); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + } + + println!(" {NO_RECEIVERS:NUM_WIDTH$}", no_receivers.len()); + for event in no_receivers { + println!(" {event:?}"); + } + + let total_globs: usize = + globs_reprocessed.values().map(|globs| globs.len()).sum(); + if total_globs > 0 { + let mut reprocessed = 0; + let mut already_reprocessed = 0; + let mut glob_errors = 0; + println!(" {OUTDATED_GLOBS:NUM_WIDTH$}"); + println!(" current schema version: {glob_version}"); + for (rx_id, globs) in globs_reprocessed { + if globs.is_empty() { + continue; + } + println!(" receiver {rx_id:?}:"); + for (glob, status) in globs { + match status { + Ok(WebhookGlobStatus::AlreadyReprocessed) => { + println!(" > {glob:?}: already reprocessed"); + already_reprocessed += 1; + } + Ok(WebhookGlobStatus::Reprocessed { + created, + deleted, + prev_version, + }) => { + println!( + " > {glob:?}: previously at \ + {prev_version}\n \ + exact subscriptions: {created:>NUM_WIDTH$} \ + created, {deleted:>NUM_WIDTH$} deleted", + ); + reprocessed += 1; + } + Err(e) => { + println!(" > {glob:?}: FAILED: {e}"); + glob_errors += 1; + } + } + } + } + println!(" {GLOBS_REPROCESSED:NUM_WIDTH$}"); + println!( + " {ALREADY_REPROCESSED:NUM_WIDTH$}", + already_reprocessed + ); + println!( + "{} {GLOB_ERRORS:NUM_WIDTH$}", + warn_if_nonzero(glob_errors), + ); + } +} +fn print_task_webhook_deliverator(details: &serde_json::Value) { + use nexus_types::external_api::views::WebhookDeliveryAttemptResult; + use nexus_types::internal_api::background::WebhookDeliveratorStatus; + use nexus_types::internal_api::background::WebhookDeliveryFailure; + use nexus_types::internal_api::background::WebhookRxDeliveryStatus; + + let WebhookDeliveratorStatus { by_rx, error } = match serde_json::from_value::< + WebhookDeliveratorStatus, + >(details.clone()) + { + Err(error) => { + eprintln!( + "warning: failed to interpret task details: {:?}: {:?}", + error, details + ); + return; + } + Ok(status) => status, + }; + + if let Some(error) = error { + println!(" task did not complete successfully:\n {error}"); + } + const RECEIVERS: &str = "receivers:"; + const TOTAL_OK: &str = "successful deliveries:"; + const TOTAL_FAILED: &str = "failed deliveries:"; + const TOTAL_ALREADY_DELIVERED: &str = "already delivered by another Nexus:"; + const TOTAL_IN_PROGRESS: &str = "in progress by another Nexus:"; + const TOTAL_ERRORS: &str = "internal delivery errors:"; + const WIDTH: usize = const_max_len(&[ + RECEIVERS, + TOTAL_OK, + TOTAL_FAILED, + TOTAL_ALREADY_DELIVERED, + TOTAL_IN_PROGRESS, + TOTAL_ERRORS, + ]) + 1; + const NUM_WIDTH: usize = 3; + + let mut total_ok = 0; + let mut total_already_delivered = 0; + let mut total_in_progress = 0; + let mut total_failed = 0; + let mut total_errors = 0; + println!(" {RECEIVERS:NUM_WIDTH$}", by_rx.len()); + for (rx_id, status) in by_rx { + let WebhookRxDeliveryStatus { + ready, + delivered_ok, + already_delivered, + in_progress, + failed_deliveries, + delivery_errors, + error, + } = status; + println!(" > {rx_id:?}: {ready}"); + + const SUCCESSFUL: &str = "successfully delivered:"; + const FAILED: &str = "failed:"; + const IN_PROGRESS: &str = "in progress elsewhere:"; + const ALREADY_DELIVERED: &str = "already delivered:"; + const ERRORS: &str = "internal errors:"; + const WIDTH: usize = const_max_len(&[ + SUCCESSFUL, + FAILED, + IN_PROGRESS, + ALREADY_DELIVERED, + ERRORS, + ]) + 1; + const NUM_WIDTH: usize = 3; + + println!(" {SUCCESSFUL:NUM_WIDTH$}"); + println!( + " {ALREADY_DELIVERED:NUM_WIDTH$}", + already_delivered, + ); + println!(" {IN_PROGRESS:NUM_WIDTH$}"); + total_ok += delivered_ok; + total_already_delivered += total_already_delivered; + total_in_progress += in_progress; + let n_failed = failed_deliveries.len(); + total_failed += n_failed; + println!(" {FAILED:NUM_WIDTH$}"); + if n_failed > 0 { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct FailureRow { + event: Uuid, + delivery: Uuid, + #[tabled(rename = "#")] + attempt: usize, + result: WebhookDeliveryAttemptResult, + #[tabled(display_with = "display_option_blank")] + status: Option, + #[tabled(display_with = "display_option_blank")] + duration: Option, + } + let table_rows = failed_deliveries.into_iter().map( + |WebhookDeliveryFailure { + delivery_id, + event_id, + attempt, + result, + response_status, + response_duration, + }| FailureRow { + // Turn these into untyped `Uuid`s so that the Display impl + // doesn't include the UUID kind in the table. + delivery: delivery_id.into_untyped_uuid(), + event: event_id.into_untyped_uuid(), + attempt, + result, + status: response_status, + duration: response_duration, + }, + ); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + } + let n_internal_errors = + delivery_errors.len() + if error.is_some() { 1 } else { 0 }; + if n_internal_errors > 0 { + total_errors += n_internal_errors; + println!( + "/!\\ {ERRORS:NUM_WIDTH$}", + n_internal_errors, + ); + if let Some(error) = error { + println!(" > {error}") + } + for (id, error) in delivery_errors { + println!(" > {id:?}: {error}") + } + } + } + println!(" {TOTAL_OK:NUM_WIDTH$}"); + println!(" {TOTAL_FAILED:NUM_WIDTH$}"); + println!( + "{} {TOTAL_ERRORS:NUM_WIDTH$}", + warn_if_nonzero(total_errors), + ); + println!( + " {TOTAL_ALREADY_DELIVERED:NUM_WIDTH$}", + total_already_delivered + ); + println!( + " {TOTAL_IN_PROGRESS:NUM_WIDTH$}", + total_in_progress + ); +} + +fn warn_if_nonzero(n: usize) -> &'static str { + if n > 0 { "/!\\" } else { " " } +} + /// Summarizes an `ActivationReason` fn reason_str(reason: &ActivationReason) -> &'static str { match reason { diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index ad8576a4ab5..9debc7a49e1 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -788,14 +788,20 @@ task: "webhook_deliverator" currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms -warning: unknown background task: "webhook_deliverator" (don't know how to interpret details: Object {"by_rx": Object {}, "error": Null}) + receivers: 0 + successful deliveries: 0 + failed deliveries: 0 + internal delivery errors: 0 + already delivered by another Nexus: 0 + in progress by another Nexus: 0 task: "webhook_dispatcher" configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms -warning: unknown background task: "webhook_dispatcher" (don't know how to interpret details: Object {"dispatched": Array [], "errors": Array [], "glob_version": String("130.0.0"), "globs_reprocessed": Object {}, "no_receivers": Array []}) + events dispatched: 0 + events with no receivers subscribed: 0 --------------------------------------------- stderr: @@ -1301,14 +1307,20 @@ task: "webhook_deliverator" currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms -warning: unknown background task: "webhook_deliverator" (don't know how to interpret details: Object {"by_rx": Object {}, "error": Null}) + receivers: 0 + successful deliveries: 0 + failed deliveries: 0 + internal delivery errors: 0 + already delivered by another Nexus: 0 + in progress by another Nexus: 0 task: "webhook_dispatcher" configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms -warning: unknown background task: "webhook_dispatcher" (don't know how to interpret details: Object {"dispatched": Array [], "errors": Array [], "glob_version": String("130.0.0"), "globs_reprocessed": Object {}, "no_receivers": Array []}) + events dispatched: 0 + events with no receivers subscribed: 0 --------------------------------------------- stderr: From 032d5abc9452801b7ee9fb58c5e7ea539abe19ef Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 09:59:55 -0700 Subject: [PATCH 142/168] fix nexus-config test --- nexus-config/src/nexus_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index fcf3ea72220..2bc20eebe3e 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -1066,7 +1066,7 @@ mod test { webhook_deliverator.period_secs = 43 webhook_deliverator.lease_timeout_secs = 44 webhook_deliverator.first_retry_backoff_secs = 45 - webhook_deliverator.second_retry_backoff_secs = 45 + webhook_deliverator.second_retry_backoff_secs = 46 [default_region_allocation_strategy] type = "random" seed = 0 From d0b91e302f443ab28c0b36519792efe67f257d17 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 10:17:58 -0700 Subject: [PATCH 143/168] remove unnecessary event class get endpoint --- nexus/external-api/output/nexus_tags.txt | 1 - nexus/external-api/src/lib.rs | 11 ------- nexus/src/external_api/http_entrypoints.rs | 33 ------------------- nexus/types/src/external_api/params.rs | 6 ---- openapi/nexus.json | 38 ---------------------- 5 files changed, 89 deletions(-) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 4d5a66e6316..51f752cf687 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -271,7 +271,6 @@ OPERATION ID METHOD URL PATH webhook_delivery_list GET /v1/webhooks/deliveries webhook_delivery_resend POST /v1/webhooks/deliveries/{event_id}/resend webhook_event_class_list GET /v1/webhooks/event-classes -webhook_event_class_view GET /v1/webhooks/event-classes/{name} webhook_receiver_create POST /v1/webhooks/receivers webhook_receiver_delete DELETE /v1/webhooks/receivers/{receiver} webhook_receiver_list GET /v1/webhooks/receivers diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 701bc3e0baa..ec2cad62f26 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3521,17 +3521,6 @@ pub trait NexusExternalApi { filter: Query, ) -> Result>, HttpError>; - /// Fetch details on an event class by name. - #[endpoint { - method = GET, - path ="/v1/webhooks/event-classes/{name}", - tags = ["system/webhooks"], - }] - async fn webhook_event_class_view( - rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError>; - /// List webhook receivers. #[endpoint { method = GET, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index dd8fd764e4a..8541be9907c 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -7708,39 +7708,6 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn webhook_event_class_view( - rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let params::EventClassSelector { name } = path_params.into_inner(); - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - opctx - .authorize( - authz::Action::ListChildren, - &authz::WEBHOOK_EVENT_CLASS_LIST, - ) - .await?; - - let event_class = name - .parse::() - .map_err(|_| { - Error::non_resourcetype_not_found(format!( - "{name:?} is not a webhook event class" - )) - })? - .into(); - Ok(HttpResponseOk(event_class)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await - } - async fn webhook_receiver_list( rqctx: RequestContext, query_params: Query, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 72f5fb4f0ae..57c57cc2905 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2390,12 +2390,6 @@ pub struct EventClassPage { pub last_seen: String, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct EventClassSelector { - /// The name of the event class. - pub name: String, -} - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookReceiverSelector { /// The name or ID of the webhook receiver. diff --git a/openapi/nexus.json b/openapi/nexus.json index 34b2293d914..7db6251a47e 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12130,44 +12130,6 @@ } } }, - "/v1/webhooks/event-classes/{name}": { - "get": { - "tags": [ - "system/webhooks" - ], - "summary": "Fetch details on an event class by name.", - "operationId": "webhook_event_class_view", - "parameters": [ - { - "in": "path", - "name": "name", - "description": "The name of the event class.", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EventClass" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, "/v1/webhooks/receivers": { "get": { "tags": [ From 4f8c258fb9dadee55ab59f0b7c4ea2b207ce3606 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 10:35:33 -0700 Subject: [PATCH 144/168] improve API docs thanks @david-crespo and @benjaminleonard for advice! --- nexus/external-api/src/lib.rs | 49 +++++++++++++++++++------- nexus/types/src/external_api/params.rs | 15 ++++++++ openapi/nexus.json | 30 ++++++++++------ 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index ec2cad62f26..9e385c5ff38 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3532,7 +3532,7 @@ pub trait NexusExternalApi { query_params: Query, ) -> Result>, HttpError>; - /// Get the configuration for a webhook receiver. + /// Fetch webhook receiver #[endpoint { method = GET, path = "/v1/webhooks/receivers/{receiver}", @@ -3543,7 +3543,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Create a new webhook receiver. + /// Create webhook receiver. #[endpoint { method = POST, path = "/v1/webhooks/receivers", @@ -3554,7 +3554,11 @@ pub trait NexusExternalApi { params: TypedBody, ) -> Result, HttpError>; - /// Update the configuration of an existing webhook receiver. + /// Update webhook receiver + /// + /// Note that receiver secrets are NOT added or removed using this endpoint. + /// Instead, use the /v1/webhooks/{secrets}/?receiver={receiver} endpoint + /// to add and remove secrets. #[endpoint { method = PUT, path = "/v1/webhooks/receivers/{receiver}", @@ -3566,7 +3570,7 @@ pub trait NexusExternalApi { params: TypedBody, ) -> Result; - /// Delete a webhook receiver. + /// Delete webhook receiver. #[endpoint { method = DELETE, path = "/v1/webhooks/receivers/{receiver}", @@ -3577,10 +3581,23 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; - /// Send a liveness probe request to a webhook receiver. - // TODO(eliza): this return type isn't quite right, as the response should - // have a typed body, but any status code, as we return a resposne with the - // status code from the webhook endpoint... + /// Send liveness probe to webhook receiver + /// + /// This endpoint synchronously sends a liveness probe request to the + /// selected webhook receiver. The response message describes the outcome of + /// the probe request: either the response from the receiver endpoint, or an + /// indication of why the probe failed. + /// + /// Note that the response status is 200 OK as long as a probe request was + /// able to be sent to the receiver endpoint. If the receiver responds with + /// another status code, including an error, this will be indicated by the + /// response body, NOT the status of the response. + /// + /// The "resend" query parameter can be used to request re-delivery of + /// failed events if the liveness probe succeeds. If it is set to true and + /// the webhook receiver responds to the probe request with a 2xx status + /// code, any events for which delivery to this receiver has failed will be + /// queued for re-delivery. #[endpoint { method = POST, path = "/v1/webhooks/receivers/{receiver}/probe", @@ -3592,7 +3609,7 @@ pub trait NexusExternalApi { query_params: Query, ) -> Result, HttpError>; - /// List the IDs of secrets for a webhook receiver. + /// List webhook receiver secret IDs #[endpoint { method = GET, path = "/v1/webhooks/secrets", @@ -3603,7 +3620,7 @@ pub trait NexusExternalApi { query_params: Query, ) -> Result, HttpError>; - /// Add a secret to a webhook receiver. + /// Add secret to webhook receiver #[endpoint { method = POST, path = "/v1/webhooks/secrets", @@ -3615,7 +3632,7 @@ pub trait NexusExternalApi { params: TypedBody, ) -> Result, HttpError>; - /// Delete a secret associated with a webhook receiver by ID. + /// Remove secret from webhook receiver #[endpoint { method = DELETE, path = "/v1/webhooks/secrets/{secret_id}", @@ -3626,7 +3643,13 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; - /// List delivery attempts to a webhook receiver. + /// List delivery attempts to a webhook receiver + /// + /// Optional query parameters to this endpoint may be used to filter + /// deliveries by state. If none of the "failed", "pending", or "delivered" + /// query parameters are present, all deliveries are returned. If one or + /// more of these parameters are provided, only those which are set to + /// "true" are included in the response. #[endpoint { method = GET, path = "/v1/webhooks/deliveries", @@ -3639,7 +3662,7 @@ pub trait NexusExternalApi { pagination: Query, ) -> Result>, HttpError>; - /// Request re-delivery of a webhook event. + /// Request re-delivery of webhook event #[endpoint { method = POST, path = "/v1/webhooks/deliveries/{event_id}/resend", diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 57c57cc2905..36d7d32a3f1 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -2381,6 +2381,9 @@ pub struct DeviceAccessTokenRequest { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct EventClassFilter { /// An optional glob pattern for filtering event class names. + /// + /// If provided, only event classes which match this glob pattern will be + /// included in the response. pub filter: Option, } @@ -2432,6 +2435,7 @@ pub struct WebhookReceiverUpdate { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookSecretCreate { + /// The value of the shared secret key. pub secret: String, } @@ -2443,13 +2447,24 @@ pub struct WebhookSecretSelector { #[derive(Deserialize, JsonSchema)] pub struct WebhookEventSelector { + /// UUID of the event pub event_id: Uuid, } #[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct WebhookDeliveryStateFilter { + /// If true, include only deliveries which are currently in progress. + /// + /// A delivery is considered "pending" if it has not yet been sent at all, + /// or if a delivery attempt has failed but the delivery has retries + /// remaining. pub pending: Option, + /// If true, include only deliveries which have failed permanently. + /// + /// A delivery fails permanently when the retry limit of three total + /// attempts is reached without a successful delivery. pub failed: Option, + /// If true, include only deliveries which have succeeded. pub delivered: Option, } diff --git a/openapi/nexus.json b/openapi/nexus.json index 7db6251a47e..dbb512f2df0 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -11935,7 +11935,8 @@ "tags": [ "system/webhooks" ], - "summary": "List delivery attempts to a webhook receiver.", + "summary": "List delivery attempts to a webhook receiver", + "description": "Optional query parameters to this endpoint may be used to filter deliveries by state. If none of the \"failed\", \"pending\", or \"delivered\" query parameters are present, all deliveries are returned. If one or more of these parameters are provided, only those which are set to \"true\" are included in the response.", "operationId": "webhook_delivery_list", "parameters": [ { @@ -11950,6 +11951,7 @@ { "in": "query", "name": "delivered", + "description": "If true, include only deliveries which have succeeded.", "schema": { "nullable": true, "type": "boolean" @@ -11958,6 +11960,7 @@ { "in": "query", "name": "failed", + "description": "If true, include only deliveries which have failed permanently.\n\nA delivery fails permanently when the retry limit of three total attempts is reached without a successful delivery.", "schema": { "nullable": true, "type": "boolean" @@ -11966,6 +11969,7 @@ { "in": "query", "name": "pending", + "description": "If true, include only deliveries which are currently in progress.\n\nA delivery is considered \"pending\" if it has not yet been sent at all, or if a delivery attempt has failed but the delivery has retries remaining.", "schema": { "nullable": true, "type": "boolean" @@ -12027,12 +12031,13 @@ "tags": [ "system/webhooks" ], - "summary": "Request re-delivery of a webhook event.", + "summary": "Request re-delivery of webhook event", "operationId": "webhook_delivery_resend", "parameters": [ { "in": "path", "name": "event_id", + "description": "UUID of the event", "required": true, "schema": { "type": "string", @@ -12100,7 +12105,7 @@ { "in": "query", "name": "filter", - "description": "An optional glob pattern for filtering event class names.", + "description": "An optional glob pattern for filtering event class names.\n\nIf provided, only event classes which match this glob pattern will be included in the response.", "schema": { "nullable": true, "type": "string" @@ -12192,7 +12197,7 @@ "tags": [ "system/webhooks" ], - "summary": "Create a new webhook receiver.", + "summary": "Create webhook receiver.", "operationId": "webhook_receiver_create", "requestBody": { "content": { @@ -12229,7 +12234,7 @@ "tags": [ "system/webhooks" ], - "summary": "Get the configuration for a webhook receiver.", + "summary": "Fetch webhook receiver", "operationId": "webhook_receiver_view", "parameters": [ { @@ -12265,7 +12270,8 @@ "tags": [ "system/webhooks" ], - "summary": "Update the configuration of an existing webhook receiver.", + "summary": "Update webhook receiver", + "description": "Note that receiver secrets are NOT added or removed using this endpoint. Instead, use the /v1/webhooks/{secrets}/?receiver={receiver} endpoint to add and remove secrets.", "operationId": "webhook_receiver_update", "parameters": [ { @@ -12304,7 +12310,7 @@ "tags": [ "system/webhooks" ], - "summary": "Delete a webhook receiver.", + "summary": "Delete webhook receiver.", "operationId": "webhook_receiver_delete", "parameters": [ { @@ -12335,7 +12341,8 @@ "tags": [ "system/webhooks" ], - "summary": "Send a liveness probe request to a webhook receiver.", + "summary": "Send liveness probe to webhook receiver", + "description": "This endpoint synchronously sends a liveness probe request to the selected webhook receiver. The response message describes the outcome of the probe request: either the response from the receiver endpoint, or an indication of why the probe failed.\n\nNote that the response status is 200 OK as long as a probe request was able to be sent to the receiver endpoint. If the receiver responds with another status code, including an error, this will be indicated by the response body, NOT the status of the response.\n\nThe \"resend\" query parameter can be used to request re-delivery of failed events if the liveness probe succeeds. If it is set to true and the webhook receiver responds to the probe request with a 2xx status code, any events for which delivery to this receiver has failed will be queued for re-delivery.", "operationId": "webhook_receiver_probe", "parameters": [ { @@ -12381,7 +12388,7 @@ "tags": [ "system/webhooks" ], - "summary": "List the IDs of secrets for a webhook receiver.", + "summary": "List webhook receiver secret IDs", "operationId": "webhook_secrets_list", "parameters": [ { @@ -12417,7 +12424,7 @@ "tags": [ "system/webhooks" ], - "summary": "Add a secret to a webhook receiver.", + "summary": "Add secret to webhook receiver", "operationId": "webhook_secrets_add", "parameters": [ { @@ -12465,7 +12472,7 @@ "tags": [ "system/webhooks" ], - "summary": "Delete a secret associated with a webhook receiver by ID.", + "summary": "Remove secret from webhook receiver", "operationId": "webhook_secrets_delete", "parameters": [ { @@ -25657,6 +25664,7 @@ "type": "object", "properties": { "secret": { + "description": "The value of the shared secret key.", "type": "string" } }, From 28e851b75e3a7ba0775fa4675f4cc37d42d09bc0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 10:42:24 -0700 Subject: [PATCH 145/168] apparently markdown is okay! --- nexus/external-api/src/lib.rs | 12 ++++++------ openapi/nexus.json | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 9e385c5ff38..139303eb599 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3557,7 +3557,7 @@ pub trait NexusExternalApi { /// Update webhook receiver /// /// Note that receiver secrets are NOT added or removed using this endpoint. - /// Instead, use the /v1/webhooks/{secrets}/?receiver={receiver} endpoint + /// Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}`` endpoint /// to add and remove secrets. #[endpoint { method = PUT, @@ -3588,14 +3588,14 @@ pub trait NexusExternalApi { /// the probe request: either the response from the receiver endpoint, or an /// indication of why the probe failed. /// - /// Note that the response status is 200 OK as long as a probe request was + /// Note that the response status is `200 OK` as long as a probe request was /// able to be sent to the receiver endpoint. If the receiver responds with /// another status code, including an error, this will be indicated by the - /// response body, NOT the status of the response. + /// response body, *not* the status of the response. /// - /// The "resend" query parameter can be used to request re-delivery of + /// The `resend` query parameter can be used to request re-delivery of /// failed events if the liveness probe succeeds. If it is set to true and - /// the webhook receiver responds to the probe request with a 2xx status + /// the webhook receiver responds to the probe request with a `2xx` status /// code, any events for which delivery to this receiver has failed will be /// queued for re-delivery. #[endpoint { @@ -3646,7 +3646,7 @@ pub trait NexusExternalApi { /// List delivery attempts to a webhook receiver /// /// Optional query parameters to this endpoint may be used to filter - /// deliveries by state. If none of the "failed", "pending", or "delivered" + /// deliveries by state. If none of the `failed`, `pending` or `delivered` /// query parameters are present, all deliveries are returned. If one or /// more of these parameters are provided, only those which are set to /// "true" are included in the response. diff --git a/openapi/nexus.json b/openapi/nexus.json index dbb512f2df0..31bc0eee705 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -11936,7 +11936,7 @@ "system/webhooks" ], "summary": "List delivery attempts to a webhook receiver", - "description": "Optional query parameters to this endpoint may be used to filter deliveries by state. If none of the \"failed\", \"pending\", or \"delivered\" query parameters are present, all deliveries are returned. If one or more of these parameters are provided, only those which are set to \"true\" are included in the response.", + "description": "Optional query parameters to this endpoint may be used to filter deliveries by state. If none of the `failed`, `pending` or `delivered` query parameters are present, all deliveries are returned. If one or more of these parameters are provided, only those which are set to \"true\" are included in the response.", "operationId": "webhook_delivery_list", "parameters": [ { @@ -12271,7 +12271,7 @@ "system/webhooks" ], "summary": "Update webhook receiver", - "description": "Note that receiver secrets are NOT added or removed using this endpoint. Instead, use the /v1/webhooks/{secrets}/?receiver={receiver} endpoint to add and remove secrets.", + "description": "Note that receiver secrets are NOT added or removed using this endpoint. Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}`` endpoint to add and remove secrets.", "operationId": "webhook_receiver_update", "parameters": [ { @@ -12342,7 +12342,7 @@ "system/webhooks" ], "summary": "Send liveness probe to webhook receiver", - "description": "This endpoint synchronously sends a liveness probe request to the selected webhook receiver. The response message describes the outcome of the probe request: either the response from the receiver endpoint, or an indication of why the probe failed.\n\nNote that the response status is 200 OK as long as a probe request was able to be sent to the receiver endpoint. If the receiver responds with another status code, including an error, this will be indicated by the response body, NOT the status of the response.\n\nThe \"resend\" query parameter can be used to request re-delivery of failed events if the liveness probe succeeds. If it is set to true and the webhook receiver responds to the probe request with a 2xx status code, any events for which delivery to this receiver has failed will be queued for re-delivery.", + "description": "This endpoint synchronously sends a liveness probe request to the selected webhook receiver. The response message describes the outcome of the probe request: either the response from the receiver endpoint, or an indication of why the probe failed.\n\nNote that the response status is `200 OK` as long as a probe request was able to be sent to the receiver endpoint. If the receiver responds with another status code, including an error, this will be indicated by the response body, *not* the status of the response.\n\nThe `resend` query parameter can be used to request re-delivery of failed events if the liveness probe succeeds. If it is set to true and the webhook receiver responds to the probe request with a `2xx` status code, any events for which delivery to this receiver has failed will be queued for re-delivery.", "operationId": "webhook_receiver_probe", "parameters": [ { From c49806637f7d0dda5cb888ead2c2c54671a6a863 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 12:03:14 -0700 Subject: [PATCH 146/168] Update nexus/external-api/src/lib.rs Co-authored-by: David Crespo --- nexus/external-api/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 139303eb599..459348166d5 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3557,7 +3557,7 @@ pub trait NexusExternalApi { /// Update webhook receiver /// /// Note that receiver secrets are NOT added or removed using this endpoint. - /// Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}`` endpoint + /// Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}` endpoint /// to add and remove secrets. #[endpoint { method = PUT, From 2e7185ef7c689559bd8354a16f395222e1d79089 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 14 Mar 2025 12:59:06 -0700 Subject: [PATCH 147/168] update expectorate query --- .../tests/output/webhook_rx_list_resendable_events.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql index e1597c83fbd..6a712a2c45e 100644 --- a/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql +++ b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql @@ -2,6 +2,7 @@ SELECT DISTINCT webhook_event.id, webhook_event.time_created, + webhook_event.time_modified, webhook_event.time_dispatched, webhook_event.event_class, webhook_event.event, @@ -18,7 +19,7 @@ WHERE FROM webhook_delivery AS also_delivey WHERE - (also_delivey.event_id = webhook_event.id AND also_delivey.failed_permanently = $3) + (also_delivey.event_id = webhook_event.id AND also_delivey.state != $3) AND also_delivey.trigger != $4 ) ) From 7dbf3c485eb467506a0598d587effff558c3ee16 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 17 Mar 2025 15:47:29 -0700 Subject: [PATCH 148/168] update authz tests --- .../src/policy_test/resource_builder.rs | 4 ++ nexus/db-queries/src/policy_test/resources.rs | 32 +++++++++++ nexus/db-queries/tests/output/authz-roles.out | 56 +++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index 3d5ea068ca6..88e7b34d7a6 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -278,6 +278,9 @@ impl_dyn_authorized_resource_for_resource!(authz::TufArtifact); impl_dyn_authorized_resource_for_resource!(authz::TufRepo); impl_dyn_authorized_resource_for_resource!(authz::Vpc); impl_dyn_authorized_resource_for_resource!(authz::VpcSubnet); +impl_dyn_authorized_resource_for_resource!(authz::WebhookEvent); +impl_dyn_authorized_resource_for_resource!(authz::WebhookReceiver); +impl_dyn_authorized_resource_for_resource!(authz::WebhookSecret); impl_dyn_authorized_resource_for_resource!(authz::Zpool); impl_dyn_authorized_resource_for_global!(authz::Database); @@ -288,6 +291,7 @@ impl_dyn_authorized_resource_for_global!(authz::DnsConfig); impl_dyn_authorized_resource_for_global!(authz::IpPoolList); impl_dyn_authorized_resource_for_global!(authz::Inventory); impl_dyn_authorized_resource_for_global!(authz::TargetReleaseConfig); +impl_dyn_authorized_resource_for_global!(authz::WebhookEventClassList); impl DynAuthorizedResource for authz::SiloCertificateList { fn do_authorize<'a, 'b>( diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index b069d1df2a5..2bcadf77792 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -74,6 +74,7 @@ pub async fn make_resources( builder.new_resource(authz::INVENTORY); builder.new_resource(authz::IP_POOL_LIST); builder.new_resource(authz::TARGET_RELEASE_CONFIG); + builder.new_resource(authz::WEBHOOK_EVENT_CLASS_LIST); // Silo/organization/project hierarchy make_silo(&mut builder, "silo1", main_silo_id, true).await; @@ -171,6 +172,16 @@ pub async fn make_resources( LookupType::ById(loopback_address_id.into_untyped_uuid()), )); + let webhook_event_id = + "31cb17da-4164-4cbf-b9a3-b3e4a687c08b".parse().unwrap(); + builder.new_resource(authz::WebhookEvent::new( + authz::FLEET, + webhook_event_id, + LookupType::ById(webhook_event_id.into_untyped_uuid()), + )); + + make_webhook_rx(&mut builder).await; + builder.build() } @@ -388,6 +399,27 @@ async fn make_project( )); } +/// Helper for `make_resources()` that constructs a webhook receiver and its +/// very miniscule hierarchy (a secret). +async fn make_webhook_rx(builder: &mut ResourceBuilder<'_>) { + let webhook_rx_id = + omicron_uuid_kinds::WebhookReceiverUuid::new_v4().into(); + let webhook_rx = authz::WebhookReceiver::new( + authz::FLEET, + webhook_rx_id, + LookupType::ById(webhook_rx_id.into_untyped_uuid()), + ); + builder.new_resource(webhook_rx.clone()); + + let webhook_secret_id = + omicron_uuid_kinds::WebhookSecretUuid::new_v4().into(); + builder.new_resource(authz::WebhookSecret::new( + webhook_rx, + webhook_secret_id, + LookupType::ById(webhook_secret_id.into_untyped_uuid()), + )); +} + /// Returns the set of authz classes exempted from the coverage test pub fn exempted_authz_classes() -> BTreeSet { // Exemption list for the coverage test diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 6ff26853690..5aea4238ef6 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -124,6 +124,20 @@ resource: authz::TargetReleaseConfig silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: authz::WebhookEventClassList + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✔ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✔ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✔ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Silo "silo1" USER Q R LC RP M MP CC D @@ -1244,6 +1258,48 @@ resource: LoopbackAddress id "9efbf1b1-16f9-45ab-864a-f7ebe501ae5b" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: WebhookEvent id "31cb17da-4164-4cbf-b9a3-b3e4a687c08b" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + fleet-collaborator ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: WebhookReceiver id "73bccab2-467d-41d1-8e9d-97e9d821d029" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + fleet-collaborator ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: WebhookSecret id "0c3e55cb-fcee-46e9-a2e3-0901dbd3b997" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✔ ✘ ✔ ✔ ✔ ✘ ✔ + fleet-collaborator ✘ ✔ ✘ ✔ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✔ ✘ ✔ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + ACTIONS: Q = Query From 1d2c0375b07a65ff4f0171034b74ee24eff201eb Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 17 Mar 2025 16:34:47 -0700 Subject: [PATCH 149/168] rm unnecessary into --- nexus/db-queries/src/policy_test/resources.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 2bcadf77792..c1f389f1e1d 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -402,8 +402,7 @@ async fn make_project( /// Helper for `make_resources()` that constructs a webhook receiver and its /// very miniscule hierarchy (a secret). async fn make_webhook_rx(builder: &mut ResourceBuilder<'_>) { - let webhook_rx_id = - omicron_uuid_kinds::WebhookReceiverUuid::new_v4().into(); + let webhook_rx_id = omicron_uuid_kinds::WebhookReceiverUuid::new_v4(); let webhook_rx = authz::WebhookReceiver::new( authz::FLEET, webhook_rx_id, @@ -411,8 +410,7 @@ async fn make_webhook_rx(builder: &mut ResourceBuilder<'_>) { ); builder.new_resource(webhook_rx.clone()); - let webhook_secret_id = - omicron_uuid_kinds::WebhookSecretUuid::new_v4().into(); + let webhook_secret_id = omicron_uuid_kinds::WebhookSecretUuid::new_v4(); builder.new_resource(authz::WebhookSecret::new( webhook_rx, webhook_secret_id, From f591ac13e50e84275080ec4cf202e05701c1446e Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 18 Mar 2025 13:22:07 -0700 Subject: [PATCH 150/168] i now understand which UUIDs need to be hardcoded --- nexus/db-queries/src/policy_test/resources.rs | 9 +++++---- nexus/db-queries/tests/output/authz-roles.out | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index c1f389f1e1d..6853288ff09 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -402,15 +402,16 @@ async fn make_project( /// Helper for `make_resources()` that constructs a webhook receiver and its /// very miniscule hierarchy (a secret). async fn make_webhook_rx(builder: &mut ResourceBuilder<'_>) { - let webhook_rx_id = omicron_uuid_kinds::WebhookReceiverUuid::new_v4(); + let rx_name = "webhooked-on-phonics"; let webhook_rx = authz::WebhookReceiver::new( authz::FLEET, - webhook_rx_id, - LookupType::ById(webhook_rx_id.into_untyped_uuid()), + omicron_uuid_kinds::WebhookReceiverUuid::new_v4(), + LookupType::ByName(rx_name.to_string()), ); builder.new_resource(webhook_rx.clone()); - let webhook_secret_id = omicron_uuid_kinds::WebhookSecretUuid::new_v4(); + let webhook_secret_id = + "0c3e55cb-fcee-46e9-a2e3-0901dbd3b997".parse().unwrap(); builder.new_resource(authz::WebhookSecret::new( webhook_rx, webhook_secret_id, diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 5aea4238ef6..e83cacbe3a9 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -1272,7 +1272,7 @@ resource: WebhookEvent id "31cb17da-4164-4cbf-b9a3-b3e4a687c08b" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! -resource: WebhookReceiver id "73bccab2-467d-41d1-8e9d-97e9d821d029" +resource: WebhookReceiver "webhooked-on-phonics" USER Q R LC RP M MP CC D fleet-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ From 7223f2f0b4a1429a9e9a68a0ff755ab7a0311c90 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 18 Mar 2025 16:23:48 -0700 Subject: [PATCH 151/168] fix index naming --- schema/crdb/dbinit.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 5cacf1a437b..863d06c9ed7 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5022,12 +5022,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_receiver ( endpoint STRING(512) NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_receiver_by_id +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rx_by_id ON omicron.public.webhook_receiver (id) WHERE time_deleted IS NULL; -CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_receiver_by_name +CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rx_by_name ON omicron.public.webhook_receiver ( name ) WHERE From 9190c8c50f35d7b1df88fddd32ea8d6652ffd09a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 18 Mar 2025 16:32:16 -0700 Subject: [PATCH 152/168] reflect docs change in openapi document --- openapi/nexus.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/nexus.json b/openapi/nexus.json index 31bc0eee705..3967d78b9e4 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12271,7 +12271,7 @@ "system/webhooks" ], "summary": "Update webhook receiver", - "description": "Note that receiver secrets are NOT added or removed using this endpoint. Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}`` endpoint to add and remove secrets.", + "description": "Note that receiver secrets are NOT added or removed using this endpoint. Instead, use the `/v1/webhooks/{secrets}/?receiver={receiver}` endpoint to add and remove secrets.", "operationId": "webhook_receiver_update", "parameters": [ { From 4151a2ccf2222fcc51f757463d779b3a12dc61ae Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Tue, 18 Mar 2025 16:39:57 -0700 Subject: [PATCH 153/168] remove defunct endpoint from tests --- nexus/tests/integration_tests/endpoints.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index a2451bb059e..714da64b981 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -1148,8 +1148,6 @@ pub static DEMO_TARGET_RELEASE: LazyLock = pub static WEBHOOK_RECEIVERS_URL: &'static str = "/v1/webhooks/receivers"; pub static WEBHOOK_EVENT_CLASSES_URL: &'static str = "/v1/webhooks/event-classes"; -pub static WEBHOOK_EVENT_CLASS_FOO_BAR_URL: &'static str = - "/v1/webhooks/event-classes/test.foo.bar"; pub static DEMO_WEBHOOK_RECEIVER_NAME: LazyLock = LazyLock::new(|| "my-great-webhook".parse().unwrap()); @@ -2840,11 +2838,5 @@ pub static VERIFY_ENDPOINTS: LazyLock> = unprivileged_access: UnprivilegedAccess::None, allowed_methods: vec![AllowedMethod::Get], }, - VerifyEndpoint { - url: &WEBHOOK_EVENT_CLASS_FOO_BAR_URL, - visibility: Visibility::Public, - unprivileged_access: UnprivilegedAccess::None, - allowed_methods: vec![AllowedMethod::Get], - }, ] }); From e1ce3c67d537a5517cf5e54c72adb37730e02dfa Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 19 Mar 2025 14:45:58 -0700 Subject: [PATCH 154/168] fix accidentally empty migration step --- schema/crdb/webhooks/up03.sql | 1 - schema/crdb/webhooks/up26.sql | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/schema/crdb/webhooks/up03.sql b/schema/crdb/webhooks/up03.sql index 863bf0eaf6b..cf97de37251 100644 --- a/schema/crdb/webhooks/up03.sql +++ b/schema/crdb/webhooks/up03.sql @@ -1,4 +1,3 @@ - CREATE UNIQUE INDEX IF NOT EXISTS lookup_webhook_rx_by_name ON omicron.public.webhook_receiver ( name diff --git a/schema/crdb/webhooks/up26.sql b/schema/crdb/webhooks/up26.sql index e69de29bb2d..422ab3a6981 100644 --- a/schema/crdb/webhooks/up26.sql +++ b/schema/crdb/webhooks/up26.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_to_rx +ON omicron.public.webhook_delivery_attempt ( + rx_id +); From 9addb4eb03668a1cb9b5a41e1ab1129e197f75b0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 19 Mar 2025 18:13:27 -0700 Subject: [PATCH 155/168] Update README.adoc Co-authored-by: Sean Klein --- schema/crdb/webhooks/README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index 5d98d8689c1..96114850a07 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -7,7 +7,7 @@ This migration adds initial tables required for webhook delivery. The individual transactions in this upgrade do the following: * *Webhook receivers*: -** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores +** `up01.sql` creates the `omicron.public.webhook_receiver` table, which stores the receiver endpoints that receive webhook events. ** `up02.sql` creates the `lookup_webhook_rx_by_id` index on that table, for listing non-deleted webhook receivers. ** `up03.sql` creates the `lookup_webhook_rx_by_name` index on that table, for looking up receivers by name (and ensuring names are unique across all non-deleted receivers). From 9908b4f416f6d529381f03d83bd9a49c12995d99 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 19 Mar 2025 18:13:34 -0700 Subject: [PATCH 156/168] Update README.adoc Co-authored-by: Sean Klein --- schema/crdb/webhooks/README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index 96114850a07..791f2d581f7 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -37,7 +37,7 @@ each receiver. ** *Dispatch table*: *** `up16.sql` creates the `omicron.public.webhook_delivery_trigger` enum, which tracks why a webhook delivery was initiated. -*** `up1s7ql` creates the `omicron.public.webhook_delivery_state` enum, representing the current state of a webhook delivery. +*** `up17.sql` creates the `omicron.public.webhook_delivery_state` enum, representing the current state of a webhook delivery. *** `up18.sql` creates the table `omicron.public.webhook_delivery`, which tracks the webhook messages that have been dispatched to receivers. *** `up19.sql` creates the `one_webhook_event_dispatch_per_rx` unique index on `webhook_delivery`. + From 4ae319d3f595a208e11f7a6f2a6ea8e3e67e89d6 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 19 Mar 2025 18:13:46 -0700 Subject: [PATCH 157/168] Update nexus_config.rs Co-authored-by: Sean Klein --- nexus-config/src/nexus_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 7b661915c72..48d2b04d089 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -779,7 +779,7 @@ pub struct WebhookDeliveratorConfig { #[serde(default = "WebhookDeliveratorConfig::default_first_retry_backoff")] pub first_retry_backoff_secs: u64, - /// backoff period for the seecond retry of a failed delivery attempt. + /// backoff period for the second retry of a failed delivery attempt. /// /// this is tuneable to allow testing delivery retries without having to /// wait a long time. From f76ba0c7e903bfd0b44ff27ed6581706308f2a8b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 20 Mar 2025 09:58:34 -0700 Subject: [PATCH 158/168] make schema.rs fields match dbinit --- nexus/db-model/src/schema.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index b015882c8e8..86777670129 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2173,9 +2173,9 @@ table! { time_created -> Timestamptz, time_modified -> Timestamptz, time_deleted -> Nullable, - endpoint -> Text, secret_gen -> Int8, subscription_gen -> Int8, + endpoint -> Text, } } @@ -2184,9 +2184,9 @@ table! { id -> Uuid, time_created -> Timestamptz, time_modified -> Timestamptz, + time_deleted -> Nullable, rx_id -> Uuid, secret -> Text, - time_deleted -> Nullable, } } @@ -2204,8 +2204,8 @@ table! { rx_id -> Uuid, glob -> Text, regex -> Text, - schema_version -> Text, time_created -> Timestamptz, + schema_version -> Text, } } @@ -2225,9 +2225,9 @@ table! { id -> Uuid, time_created -> Timestamptz, time_modified -> Timestamptz, - time_dispatched -> Nullable, event_class -> crate::WebhookEventClassEnum, event -> Jsonb, + time_dispatched -> Nullable, num_dispatched -> Int8, } } From fbff13919e3f28362448d9015c9380457a54d080 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 20 Mar 2025 10:00:54 -0700 Subject: [PATCH 159/168] remove spare pipe --- nexus/db-fixed-data/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-fixed-data/src/lib.rs b/nexus/db-fixed-data/src/lib.rs index 62133405f05..696b305ca95 100644 --- a/nexus/db-fixed-data/src/lib.rs +++ b/nexus/db-fixed-data/src/lib.rs @@ -21,7 +21,7 @@ // these are valid v4 uuids, and they're as unlikely to collide with a future // uuid as any random uuid is.) // -// The specific kinds of resources to which we've assigned uuids:| +// The specific kinds of resources to which we've assigned uuids: // // UUID PREFIX RESOURCE // 001de000-05e4 built-in users ("05e4" looks a bit like "user") From 4069eea3fcf9811bdeb80506833babd8e8a54184 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Thu, 20 Mar 2025 15:21:03 -0700 Subject: [PATCH 160/168] use @david-crespo's lovely timestamp pagination from #7842 --- nexus/db-model/src/webhook_delivery.rs | 2 +- .../src/db/datastore/webhook_delivery.rs | 41 +++++++++++-------- nexus/external-api/src/lib.rs | 7 +++- .../background/tasks/webhook_dispatcher.rs | 5 ++- nexus/src/app/webhook.rs | 3 +- nexus/src/external_api/http_entrypoints.rs | 12 +++--- nexus/types/src/external_api/views.rs | 4 ++ openapi/nexus.json | 21 +++++++++- 8 files changed, 65 insertions(+), 30 deletions(-) diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 907fbf69b52..e87088afe71 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -77,7 +77,6 @@ impl WebhookDelivery { trigger: WebhookDeliveryTrigger, ) -> Self { Self { - // N.B.: perhaps we ought to use timestamp-based UUIDs for these? id: WebhookDeliveryUuid::new_v4().into(), event_id: event.id().into(), rx_id: (*rx_id).into(), @@ -134,6 +133,7 @@ impl WebhookDelivery { .iter() .map(views::WebhookDeliveryAttempt::from) .collect(), + time_started: self.time_created, }; // Make sure attempts are in order; each attempt entry also includes an // attempt number, which should be used authoritatively to determine the diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 4fa89cdd37c..b9d3537a9d5 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -17,7 +17,7 @@ use crate::db::model::WebhookDeliveryState; use crate::db::model::WebhookDeliveryTrigger; use crate::db::model::WebhookEvent; use crate::db::model::WebhookEventClass; -use crate::db::pagination::paginated; +use crate::db::pagination::paginated_multicolumn; use crate::db::schema; use crate::db::schema::webhook_delivery::dsl; use crate::db::schema::webhook_delivery_attempt::dsl as attempt_dsl; @@ -140,7 +140,7 @@ impl DataStore { rx_id: &WebhookReceiverUuid, triggers: &'static [WebhookDeliveryTrigger], only_states: Vec, - pagparams: &DataPageParams<'_, Uuid>, + pagparams: &DataPageParams<'_, (DateTime, Uuid)>, ) -> ListResultVec<( WebhookDelivery, WebhookEventClass, @@ -148,20 +148,24 @@ impl DataStore { )> { let conn = self.pool_connection_authorized(opctx).await?; // Paginate the query, ordered by delivery UUID. - let mut query = paginated(dsl::webhook_delivery, dsl::id, pagparams) - // Select only deliveries that are to the receiver we're interested in, - // and were initiated by the triggers we're interested in. - .filter( - dsl::rx_id - .eq(rx_id.into_untyped_uuid()) - .and(dsl::trigger.eq_any(triggers)), - ) - // Join with the event table on the delivery's event ID, - // so that we can grab the event class of the event that initiated - // this delivery. - .inner_join( - event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), - ); + let mut query = paginated_multicolumn( + dsl::webhook_delivery, + (dsl::time_created, dsl::id), + pagparams, + ) + // Select only deliveries that are to the receiver we're interested in, + // and were initiated by the triggers we're interested in. + .filter( + dsl::rx_id + .eq(rx_id.into_untyped_uuid()) + .and(dsl::trigger.eq_any(triggers)), + ) + // Join with the event table on the delivery's event ID, + // so that we can grab the event class of the event that initiated + // this delivery. + .inner_join( + event_dsl::webhook_event.on(dsl::event_id.eq(event_dsl::id)), + ); if !only_states.is_empty() { query = query.filter(dsl::state.eq_any(only_states)); } @@ -531,8 +535,9 @@ mod test { ) .await .unwrap(); - paginator = p - .found_batch(&deliveries, &|(d, _, _)| *d.id.as_untyped_uuid()); + paginator = p.found_batch(&deliveries, &|(d, _, _)| { + (d.time_created, *d.id.as_untyped_uuid()) + }); all_deliveries .extend(deliveries.into_iter().map(|(d, _, _)| dbg!(d).id)); } diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 459348166d5..bf205c8eccc 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -17,7 +17,10 @@ use nexus_types::{ external_api::{params, shared, views}, }; use omicron_common::api::external::{ - http_pagination::{PaginatedById, PaginatedByName, PaginatedByNameOrId}, + http_pagination::{ + PaginatedById, PaginatedByName, PaginatedByNameOrId, + PaginatedByTimeAndId, + }, *, }; use openapi_manager_types::ValidationContext; @@ -3659,7 +3662,7 @@ pub trait NexusExternalApi { rqctx: RequestContext, receiver: Query, state_filter: Query, - pagination: Query, + pagination: Query, ) -> Result>, HttpError>; /// Request re-delivery of webhook event diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 92168977008..4750ec75782 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -483,8 +483,9 @@ mod test { ) .await .unwrap(); - paginator = - p.found_batch(&batch, &|(d, _, _)| d.id.into_untyped_uuid()); + paginator = p.found_batch(&batch, &|(d, _, _)| { + (d.time_created, d.id.into_untyped_uuid()) + }); deliveries.extend(batch); } let event = diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index d308ba70175..f4eac2e35dd 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -133,6 +133,7 @@ use crate::Nexus; use crate::app::external_dns; use anyhow::Context; +use chrono::DateTime; use chrono::TimeDelta; use chrono::Utc; use hmac::{Hmac, Mac}; @@ -634,7 +635,7 @@ impl Nexus { opctx: &OpContext, rx: lookup::WebhookReceiver<'_>, filter: params::WebhookDeliveryStateFilter, - pagparams: &DataPageParams<'_, Uuid>, + pagparams: &DataPageParams<'_, (DateTime, Uuid)>, ) -> ListResultVec { let (authz_rx,) = rx.lookup_for(authz::Action::ListChildren).await?; let only_states = if filter.include_all() { diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 8541be9907c..20587d1d065 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -89,9 +89,11 @@ use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::http_pagination::PaginatedById; use omicron_common::api::external::http_pagination::PaginatedByName; use omicron_common::api::external::http_pagination::PaginatedByNameOrId; +use omicron_common::api::external::http_pagination::PaginatedByTimeAndId; use omicron_common::api::external::http_pagination::ScanById; use omicron_common::api::external::http_pagination::ScanByName; use omicron_common::api::external::http_pagination::ScanByNameOrId; +use omicron_common::api::external::http_pagination::ScanByTimeAndId; use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::http_pagination::data_page_params_for; use omicron_common::api::external::http_pagination::id_pagination; @@ -7953,7 +7955,7 @@ impl NexusExternalApi for NexusExternalApiImpl { rqctx: RequestContext, receiver: Query, filter: Query, - query: Query, + query: Query, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -7966,16 +7968,16 @@ impl NexusExternalApi for NexusExternalApiImpl { let webhook_selector = receiver.into_inner(); let filter = filter.into_inner(); let query = query.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; + let pag_params = data_page_params_for(&rqctx, &query)?; let rx = nexus.webhook_receiver_lookup(&opctx, webhook_selector)?; let deliveries = nexus - .webhook_receiver_delivery_list(&opctx, rx, filter, &pagparams) + .webhook_receiver_delivery_list(&opctx, rx, filter, &pag_params) .await?; - Ok(HttpResponseOk(ScanById::results_page( + Ok(HttpResponseOk(ScanByTimeAndId::results_page( &query, deliveries, - &|_, d| d.id, + &|_, d| (d.time_started, d.id), )?)) }; apictx diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index e65335888da..99919e8d0bc 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -1118,6 +1118,10 @@ pub struct WebhookDelivery { /// Individual attempts to deliver this webhook event, and their outcomes. pub attempts: Vec, + + /// The time at which this delivery began (i.e. the event was dispatched to + /// the receiver). + pub time_started: DateTime, } /// The state of a webhook delivery attempt. diff --git a/openapi/nexus.json b/openapi/nexus.json index 3967d78b9e4..c13f3e67b05 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -11999,7 +11999,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/TimeAndIdSortMode" } } ], @@ -25776,6 +25776,25 @@ ] } ] + }, + "TimeAndIdSortMode": { + "description": "Supported set of sort modes for scanning by timestamp and ID", + "oneOf": [ + { + "description": "sort in increasing order of timestamp and ID, i.e., earliest first", + "type": "string", + "enum": [ + "ascending" + ] + }, + { + "description": "sort in increasing order of timestamp and ID, i.e., most recent first", + "type": "string", + "enum": [ + "descending" + ] + } + ] } }, "responses": { From 1eb8aaecf14aa91b71270cee3065f1aca11918ff Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 09:41:08 -0700 Subject: [PATCH 161/168] whoops forgot to update oepnapi again --- openapi/nexus.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openapi/nexus.json b/openapi/nexus.json index c13f3e67b05..f05b6755c77 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -25318,6 +25318,11 @@ } ] }, + "time_started": { + "description": "The time at which this delivery began (i.e. the event was dispatched to the receiver).", + "type": "string", + "format": "date-time" + }, "trigger": { "description": "Why this delivery was performed.", "allOf": [ @@ -25341,6 +25346,7 @@ "event_id", "id", "state", + "time_started", "trigger", "webhook_id" ] From c1e671f1b52f9845248daf0718c8df0c0ed6d96b Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 10:14:44 -0700 Subject: [PATCH 162/168] rename `trigger` to `triggered_by` This is intended to prevent conflicts with the SQL `trigger` keyword (as suggested by @smklein [here]). [here]: https://github.com/oxidecomputer/omicron/pull/7277#discussion_r2006230891 --- nexus/db-model/src/schema.rs | 2 +- nexus/db-model/src/webhook_delivery.rs | 8 ++++---- .../src/db/datastore/webhook_delivery.rs | 6 +++--- nexus/src/app/webhook.rs | 16 ++++++++-------- schema/crdb/dbinit.sql | 2 +- schema/crdb/webhooks/up18.sql | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index ef4e5325b8f..ea945d06514 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2245,7 +2245,7 @@ table! { id -> Uuid, event_id -> Uuid, rx_id -> Uuid, - trigger -> crate::WebhookDeliveryTriggerEnum, + triggered_by -> crate::WebhookDeliveryTriggerEnum, payload -> Jsonb, attempts -> Int2, time_created -> Timestamptz, diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index e87088afe71..7f4583cb6c2 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -48,7 +48,7 @@ pub struct WebhookDelivery { pub rx_id: DbTypedUuid, /// Describes why this delivery was triggered. - pub trigger: WebhookDeliveryTrigger, + pub triggered_by: WebhookDeliveryTrigger, /// The data payload as sent to this receiver. pub payload: serde_json::Value, @@ -80,7 +80,7 @@ impl WebhookDelivery { id: WebhookDeliveryUuid::new_v4().into(), event_id: event.id().into(), rx_id: (*rx_id).into(), - trigger, + triggered_by: trigger, payload: event.event.clone(), attempts: SqlU8::new(0), time_created: Utc::now(), @@ -106,7 +106,7 @@ impl WebhookDelivery { ) .into(), rx_id: (*rx_id).into(), - trigger: WebhookDeliveryTrigger::Probe, + triggered_by: WebhookDeliveryTrigger::Probe, state: WebhookDeliveryState::Pending, payload: serde_json::json!({}), attempts: SqlU8::new(0), @@ -128,7 +128,7 @@ impl WebhookDelivery { event_class: event_class.as_str().to_owned(), event_id: self.event_id.into(), state: self.state.into(), - trigger: self.trigger.into(), + trigger: self.triggered_by.into(), attempts: attempts .iter() .map(views::WebhookDeliveryAttempt::from) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index b9d3537a9d5..72e902e58fe 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -124,7 +124,7 @@ impl DataStore { ) .filter( also_delivery - .field(dsl::trigger) + .field(dsl::triggered_by) .ne(WebhookDeliveryTrigger::Probe), ), ))) @@ -158,7 +158,7 @@ impl DataStore { .filter( dsl::rx_id .eq(rx_id.into_untyped_uuid()) - .and(dsl::trigger.eq_any(triggers)), + .and(dsl::triggered_by.eq_any(triggers)), ) // Join with the event table on the delivery's event ID, // so that we can grab the event class of the event that initiated @@ -210,7 +210,7 @@ impl DataStore { // Filter out deliveries triggered by probe requests, as those are // executed synchronously by the probe endpoint, rather than by the // webhook deliverator. - .filter(dsl::trigger.ne(WebhookDeliveryTrigger::Probe)) + .filter(dsl::triggered_by.ne(WebhookDeliveryTrigger::Probe)) // Only select deliveries that are still in progress. .filter( dsl::time_completed diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index f4eac2e35dd..5ec7d4e7b8d 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -783,7 +783,7 @@ impl<'a> ReceiverClient<'a> { id: delivery.id.into(), webhook_id: self.rx.id(), sent_at: &sent_at, - trigger: delivery.trigger.into(), + trigger: delivery.triggered_by.into(), }, }; // N.B. that we serialize the body "ourselves" rather than just @@ -801,7 +801,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "error" => %e, ); @@ -849,7 +849,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "error" => %e, "payload" => ?payload, ); @@ -871,7 +871,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "error" => %e, ); return Err(e).context(MSG); @@ -884,7 +884,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "response_status" => ?status, "response_duration" => ?duration, ); @@ -908,7 +908,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "error" => %e, ); (result, None) @@ -923,7 +923,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "response_status" => ?status, "response_duration" => ?duration, ); @@ -935,7 +935,7 @@ impl<'a> ReceiverClient<'a> { "event_id" => %delivery.event_id, "event_class" => %event_class, "delivery_id" => %delivery.id, - "delivery_trigger" => %delivery.trigger, + "delivery_trigger" => %delivery.triggered_by, "response_status" => ?status, "response_duration" => ?duration, ); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 01052385ed7..e703a118f23 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5232,7 +5232,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - trigger omicron.public.webhook_delivery_trigger NOT NULL, + triggered_by omicron.public.webhook_delivery_trigger NOT NULL, payload JSONB NOT NULL, diff --git a/schema/crdb/webhooks/up18.sql b/schema/crdb/webhooks/up18.sql index b65d5af7618..b8cd0f17a32 100644 --- a/schema/crdb/webhooks/up18.sql +++ b/schema/crdb/webhooks/up18.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- `omicron.public.webhook_rx`) rx_id UUID NOT NULL, - trigger omicron.public.webhook_delivery_trigger NOT NULL, + triggered_by omicron.public.webhook_delivery_trigger NOT NULL, payload JSONB NOT NULL, From 334249f2a28ca8038784e78aa559b70fce007902 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 10:15:53 -0700 Subject: [PATCH 163/168] rename `time_delivery_started` to `time_leased` This is intended to represent specifically the time the current lease was acquired, not when "delivery started" broadly (that's the "time_created" field). This naming should be a bit clearer --- see @smklein's comment: https://github.com/oxidecomputer/omicron/pull/7277#discussion_r2006266652 --- nexus/db-model/src/schema.rs | 2 +- nexus/db-model/src/webhook_delivery.rs | 6 ++-- .../src/db/datastore/webhook_delivery.rs | 28 ++++++++----------- schema/crdb/dbinit.sql | 4 +-- schema/crdb/webhooks/up18.sql | 4 +-- 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index ea945d06514..57df04d682b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2252,7 +2252,7 @@ table! { time_completed -> Nullable, state -> crate::WebhookDeliveryStateEnum, deliverator_id -> Nullable, - time_delivery_started -> Nullable, + time_leased -> Nullable, } } diff --git a/nexus/db-model/src/webhook_delivery.rs b/nexus/db-model/src/webhook_delivery.rs index 7f4583cb6c2..6d3aa0ec931 100644 --- a/nexus/db-model/src/webhook_delivery.rs +++ b/nexus/db-model/src/webhook_delivery.rs @@ -67,7 +67,7 @@ pub struct WebhookDelivery { pub deliverator_id: Option>, - pub time_delivery_started: Option>, + pub time_leased: Option>, } impl WebhookDelivery { @@ -86,7 +86,7 @@ impl WebhookDelivery { time_created: Utc::now(), time_completed: None, deliverator_id: None, - time_delivery_started: None, + time_leased: None, state: WebhookDeliveryState::Pending, } } @@ -113,7 +113,7 @@ impl WebhookDelivery { time_created: Utc::now(), time_completed: None, deliverator_id: Some((*deliverator_id).into()), - time_delivery_started: Some(Utc::now()), + time_leased: Some(Utc::now()), } } diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 72e902e58fe..0e85abb1c5d 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -218,14 +218,11 @@ impl DataStore { .and(dsl::state.eq(WebhookDeliveryState::Pending)), ) .filter(dsl::rx_id.eq(rx_id.into_untyped_uuid())) - .filter( - (dsl::deliverator_id.is_null()).or(dsl::time_delivery_started - .is_not_null() - .and( - dsl::time_delivery_started - .le(now.nullable() - cfg.lease_timeout), - )), - ) + .filter((dsl::deliverator_id.is_null()).or( + dsl::time_leased.is_not_null().and( + dsl::time_leased.le(now.nullable() - cfg.lease_timeout), + ), + )) .filter( // Retry backoffs: one of the following must be true: // - the delivery has not yet been attempted, @@ -234,13 +231,13 @@ impl DataStore { // - this is the first retry and the previous attempt was at // least `first_retry_backoff` ago, or .or(dsl::attempts.eq(1).and( - dsl::time_delivery_started + dsl::time_leased .le(now.nullable() - cfg.first_retry_backoff), )) // - this is the second retry, and the previous attempt was at // least `second_retry_backoff` ago. .or(dsl::attempts.eq(2).and( - dsl::time_delivery_started + dsl::time_leased .le(now.nullable() - cfg.second_retry_backoff), )), ) @@ -275,15 +272,12 @@ impl DataStore { ) .filter(dsl::id.eq(id)) .filter( - dsl::deliverator_id.is_null().or(dsl::time_delivery_started + dsl::deliverator_id.is_null().or(dsl::time_leased .is_not_null() - .and( - dsl::time_delivery_started - .le(now.nullable() - lease_timeout), - )), + .and(dsl::time_leased.le(now.nullable() - lease_timeout))), ) .set(( - dsl::time_delivery_started.eq(now.nullable()), + dsl::time_leased.eq(now.nullable()), dsl::deliverator_id.eq(nexus_id.into_untyped_uuid()), )) .check_if_exists::(id) @@ -299,7 +293,7 @@ impl DataStore { )); } - if let Some(started) = updated.found.time_delivery_started { + if let Some(started) = updated.found.time_leased { let nexus_id = updated.found.deliverator_id.ok_or_else(|| { Error::internal_error( diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e703a118f23..bbac0d5d998 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5248,12 +5248,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- Deliverator coordination bits deliverator_id UUID, - time_delivery_started TIMESTAMPTZ, + time_leased TIMESTAMPTZ, CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), CONSTRAINT active_deliveries_have_started_timestamps CHECK ( (deliverator_id IS NULL) OR ( - deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL + deliverator_id IS NOT NULL AND time_leased IS NOT NULL ) ), CONSTRAINT time_completed_iff_not_pending CHECK ( diff --git a/schema/crdb/webhooks/up18.sql b/schema/crdb/webhooks/up18.sql index b8cd0f17a32..89a2b2d4d3f 100644 --- a/schema/crdb/webhooks/up18.sql +++ b/schema/crdb/webhooks/up18.sql @@ -23,12 +23,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.webhook_delivery ( -- Deliverator coordination bits deliverator_id UUID, - time_delivery_started TIMESTAMPTZ, + time_leased TIMESTAMPTZ, CONSTRAINT attempts_is_non_negative CHECK (attempts >= 0), CONSTRAINT active_deliveries_have_started_timestamps CHECK ( (deliverator_id IS NULL) OR ( - deliverator_id IS NOT NULL AND time_delivery_started IS NOT NULL + deliverator_id IS NOT NULL AND time_leased IS NOT NULL ) ), CONSTRAINT time_completed_iff_not_pending CHECK ( From 5b42ea582a828dba338a97597b28b19563fa900a Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 10:25:56 -0700 Subject: [PATCH 164/168] remove unused default impl --- nexus/db-queries/src/db/datastore/webhook_delivery.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 0e85abb1c5d..3df65a7b73f 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -52,16 +52,6 @@ pub struct DeliveryConfig { pub lease_timeout: TimeDelta, } -impl Default for DeliveryConfig { - fn default() -> Self { - Self { - lease_timeout: TimeDelta::seconds(60), // 1 minute - first_retry_backoff: TimeDelta::seconds(60), // 1 minute - second_retry_backoff: TimeDelta::seconds(60 * 5), // 5 minutes - } - } -} - impl DataStore { pub async fn webhook_delivery_create_batch( &self, From 8c0d0ee298aabfd48bfbf95c991f1ac87f5ce789 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 10:33:58 -0700 Subject: [PATCH 165/168] Apply @smklein's suggestions Co-authored-by: Sean Klein --- nexus/external-api/src/lib.rs | 2 +- nexus/src/app/background/tasks/webhook_dispatcher.rs | 2 +- nexus/src/app/webhook.rs | 4 ++-- schema/crdb/webhooks/README.adoc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index bf205c8eccc..7c1eb9addd5 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -3527,7 +3527,7 @@ pub trait NexusExternalApi { /// List webhook receivers. #[endpoint { method = GET, - path = "/v1/webhooks/receivers/", + path = "/v1/webhooks/receivers", tags = ["system/webhooks"], }] async fn webhook_receiver_list( diff --git a/nexus/src/app/background/tasks/webhook_dispatcher.rs b/nexus/src/app/background/tasks/webhook_dispatcher.rs index 4750ec75782..7b717a8206f 100644 --- a/nexus/src/app/background/tasks/webhook_dispatcher.rs +++ b/nexus/src/app/background/tasks/webhook_dispatcher.rs @@ -54,7 +54,7 @@ impl BackgroundTask for WebhookDispatcher { ); } else { // no sense cluttering up the logs if we didn't do - // anyuthing interesting today + // anything interesting today slog::trace!( &opctx.log, "{MSG}"; diff --git a/nexus/src/app/webhook.rs b/nexus/src/app/webhook.rs index 5ec7d4e7b8d..f0fcb25012b 100644 --- a/nexus/src/app/webhook.rs +++ b/nexus/src/app/webhook.rs @@ -85,10 +85,10 @@ //! //! A *delivery* represents the process of sending HTTP request(s) representing //! a webhook event to a receiver. Failed HTTP requests are retried up to two -//! times times, so a delivery may consist of up to three *delivery attempts*. +//! times, so a delivery may consist of up to three *delivery attempts*. //! Each time the `webhook_deliverator` background task is activated, it //! searches for deliveries which have not yet succeeded or permanently failed, -//! hich are not presently being delivered by another Nexus, and for which the +//! which are not presently being delivered by another Nexus, and for which the //! backoff period for any prior failed delivery attempts has elapsed. It then //! sends an HTTP request to the webhook receiver, and records the result, //! creating a new `webhook_delivery_attempt` record and updating the diff --git a/schema/crdb/webhooks/README.adoc b/schema/crdb/webhooks/README.adoc index 791f2d581f7..0a184ea3b74 100644 --- a/schema/crdb/webhooks/README.adoc +++ b/schema/crdb/webhooks/README.adoc @@ -20,7 +20,7 @@ for looking up all secrets associated with a receiver. ** `up06.sql` creates the `omicron.public.webhook_event_class` enum type ** *Globs*: *** `up07.sql` creates the `omicron.public.webhook_rx_event_glob` table, which contains any subscriptions created by a receiver that have glob patterns. This table is used when generating exact subscription from globs. -*** `up08.sql` creates the `lookup_webhook_event_globs_for_rx` indes on `webhook_rx_event_glob`, for looking up all globs belonging to a receiver by ID. +*** `up08.sql` creates the `lookup_webhook_event_globs_for_rx` index on `webhook_rx_event_glob`, for looking up all globs belonging to a receiver by ID. *** `up09.sql` creates the `lookup_webhook_event_globs_by_schema_version` index on `webhook_rx_event_glob`, for searching for globs with outdated schema versions. ** *Subscriptions*: *** `up10.sql` creates the `omicron.public.webhook_rx_subscription` table, which tracks the event classes that a receiver is subscribed to. If a row in this table represents a subscription that was generated by a glob, this table also references the glob record. From d44d002018f6590e5cd2f3efd2f72a8c34dfe677 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 11:28:42 -0700 Subject: [PATCH 166/168] also update trigger's name where clause in index --- schema/crdb/dbinit.sql | 2 +- schema/crdb/webhooks/up19.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index bbac0d5d998..e1d6c297594 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5271,7 +5271,7 @@ ON omicron.public.webhook_delivery ( event_id, rx_id ) WHERE - trigger = 'event'; + triggered_by = 'event'; -- Index for looking up all webhook messages dispatched to a receiver ID CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_dispatched_to_rx diff --git a/schema/crdb/webhooks/up19.sql b/schema/crdb/webhooks/up19.sql index 2c825749aae..6f6ca73ee4a 100644 --- a/schema/crdb/webhooks/up19.sql +++ b/schema/crdb/webhooks/up19.sql @@ -3,4 +3,4 @@ ON omicron.public.webhook_delivery ( event_id, rx_id ) WHERE - trigger = 'event'; + triggered_by = 'event'; From 5c304d418423d5297aadaa7c4ac80b510e2da8d7 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 11:31:40 -0700 Subject: [PATCH 167/168] update query expectorate tests --- .../tests/output/webhook_rx_list_resendable_events.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql index 6a712a2c45e..f39e1afeb09 100644 --- a/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql +++ b/nexus/db-queries/tests/output/webhook_rx_list_resendable_events.sql @@ -20,6 +20,6 @@ WHERE webhook_delivery AS also_delivey WHERE (also_delivey.event_id = webhook_event.id AND also_delivey.state != $3) - AND also_delivey.trigger != $4 + AND also_delivey.triggered_by != $4 ) ) From 0b8b1b6e4cdff2710e622f453ce8d8ae52b937a0 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Fri, 21 Mar 2025 11:45:10 -0700 Subject: [PATCH 168/168] use `IncompleteOnConflictExt` as @smklein suggested in https://github.com/oxidecomputer/omicron/pull/7277#discussion_r2006220289 --- nexus/db-queries/src/db/datastore/webhook_delivery.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/webhook_delivery.rs b/nexus/db-queries/src/db/datastore/webhook_delivery.rs index 3df65a7b73f..193811686ae 100644 --- a/nexus/db-queries/src/db/datastore/webhook_delivery.rs +++ b/nexus/db-queries/src/db/datastore/webhook_delivery.rs @@ -6,6 +6,7 @@ use super::DataStore; use crate::context::OpContext; +use crate::db::IncompleteOnConflictExt; use crate::db::datastore::RunnableQuery; use crate::db::error::ErrorHandler; use crate::db::error::public_error_from_diesel; @@ -68,7 +69,9 @@ impl DataStore { // NOTHING, which is fine, becausse the only other uniqueness // constraint is the UUID primary key, and we kind of assume UUID // collisions don't happen. Oh well. - .on_conflict_do_nothing() + .on_conflict((dsl::event_id, dsl::rx_id)) + .as_partial_index() + .do_nothing() .execute_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))