From 1c9e62e719a740541afcc8f93a9a11a0bae04487 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 May 2026 17:20:58 +0200 Subject: [PATCH 1/5] Revert "feat(billing): auto-seed Hyperline products on boot when HYPERLINE_AUTO_SEED=1 (#20)" This reverts commit 5c33bdef4893233a38fb309fb008e0b6886f4940. --- bins/scrapix-api/src/lib.rs | 9 - .../examples/seed_hyperline_products.rs | 135 +++++++-- crates/scrapix-billing-hyperline/src/lib.rs | 3 - .../scrapix-billing-hyperline/src/products.rs | 259 ------------------ docs/configuration/environment-variables.mdx | 1 - docs/operations/hyperline-billing.mdx | 51 +--- 6 files changed, 119 insertions(+), 339 deletions(-) delete mode 100644 crates/scrapix-billing-hyperline/src/products.rs diff --git a/bins/scrapix-api/src/lib.rs b/bins/scrapix-api/src/lib.rs index 1d8c6b2e..a8fc7e4c 100644 --- a/bins/scrapix-api/src/lib.rs +++ b/bins/scrapix-api/src/lib.rs @@ -4512,18 +4512,9 @@ pub async fn run_with_bus( // manifest for ops cross-check. A failed ping is non-fatal: // the outbox drain worker retries on its own cadence, so a // transient Hyperline outage shouldn't crashloop the API. - // - // When HYPERLINE_AUTO_SEED=1 is set the boot additionally - // ensures one metered product per BillingEventType exists in - // the Hyperline workspace (idempotent — skips existing names - // by exact match). This closes the gap where adding a new - // BillingEventType variant required a manual seed-script run - // before the deploy landed, otherwise events 4xx and the - // outbox backs up. Seed failures log at WARN and continue. match scrapix_billing_hyperline::HyperlineClient::from_env() { Ok(client) => { let _ = scrapix_billing_hyperline::boot_self_check(&client).await; - scrapix_billing_hyperline::seed_if_enabled(&client).await; state.hyperline_client = Some(client); info!("Hyperline REST client enabled"); } diff --git a/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs b/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs index 8e68b10f..7c0a424b 100644 --- a/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs +++ b/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs @@ -1,17 +1,27 @@ -//! Idempotent seed runner: creates one `dynamic` product per +//! Idempotent seed script: creates one `dynamic` product per //! [`BillingEventType`] in the Hyperline workspace configured via //! `HYPERLINE_API_KEY` + `HYPERLINE_API_BASE`. //! -//! The actual seeding logic lives in -//! [`scrapix_billing_hyperline::products::seed_products`] so the API can -//! optionally run the same routine at boot when `HYPERLINE_AUTO_SEED=1`. -//! This binary stays useful for the one-shot ops workflow (or recovery -//! after a botched deploy where the auto-seed degraded). -//! //! Each product is a `dynamic` billing primitive whose aggregator sums the -//! `credits` property across incoming events of the matching `event_type`. -//! Seeded with placeholder `amount: 0` pricing — ops still needs to set -//! real prices in the Hyperline UI afterward. +//! `credits` property across incoming events of the matching `event_type`: +//! +//! ```json +//! { +//! "type": "dynamic", +//! "name": "Page crawled", +//! "unit_name": "credit", +//! "aggregator": { +//! "entity": "page_crawled", +//! "operation": "sum", +//! "property": "credits", +//! "type": "metered" +//! } +//! } +//! ``` +//! +//! Idempotency: we first `GET /v1/products?name__equals=` and +//! skip creation if any match is returned. Hyperline does not expose an +//! `external_id` on products, so name-equality is the only stable key we have. //! //! Usage: //! @@ -21,13 +31,39 @@ //! ``` use std::error::Error; -use std::process::ExitCode; use scrapix_billing_hyperline::client::HyperlineClient; -use scrapix_billing_hyperline::products::seed_products; +use scrapix_billing_hyperline::events::BillingEventType; +use serde::Deserialize; +use serde_json::json; + +/// Display name + slug used for the Hyperline product backing a given event. +fn product_for(ev: BillingEventType) -> (&'static str, &'static str) { + match ev { + BillingEventType::PageCrawled => ("Page crawled", "page_crawled"), + BillingEventType::BytesDownloaded => ("Bytes downloaded", "bytes_downloaded"), + BillingEventType::JsRender => ("JS render", "js_render"), + BillingEventType::ApiRequest => ("API request", "api_request"), + BillingEventType::DocumentIndexed => ("Document indexed", "document_indexed"), + BillingEventType::FeatureFormat => ("Feature format", "feature_format"), + BillingEventType::AiFeature => ("AI feature", "ai_feature"), + } +} + +#[derive(Debug, Deserialize)] +struct ProductSummary { + id: String, + #[serde(default)] + name: Option, +} + +#[derive(Debug, Deserialize)] +struct ProductList { + data: Vec, +} #[tokio::main] -async fn main() -> Result> { +async fn main() -> Result<(), Box> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() @@ -43,17 +79,72 @@ async fn main() -> Result> { }; println!("Seeding Hyperline products into {target}…"); - match seed_products(&client).await { - Ok(report) => { + let mut created = 0usize; + let mut skipped = 0usize; + + for ev in BillingEventType::all().iter().copied() { + let (name, slug) = product_for(ev); + + // 1. Look up existing products by exact name. + let existing: ProductList = client + .get_json( + "/v1/products", + &[("name__equals", name), ("status", "active")], + ) + .await?; + if let Some(p) = existing.data.first() { println!( - "Done. created={} skipped={}", - report.created, report.skipped + " [=] {slug:<18} exists ({}{})", + p.id, + p.name + .as_deref() + .map(|n| format!(" — \"{n}\"")) + .unwrap_or_default() ); - Ok(ExitCode::SUCCESS) - } - Err(e) => { - eprintln!("Seed failed: {e}"); - Ok(ExitCode::FAILURE) + skipped += 1; + continue; } + + // 2. Create the dynamic product. `unit_name` is shown on invoices. + // The `price_configurations` array is mandatory; we seed a single + // zero-amount volume tier (USD, monthly, 0 → ∞) so the product is + // valid in the dashboard but doesn't actually bill anything. Real + // pricing is set up by ops in the Hyperline UI post-cutover. + let body = json!({ + "type": "dynamic", + "name": name, + "description": format!("Scrapix billable event: {slug}"), + "unit_name": "credit", + "is_available_on_demand": true, + "is_available_on_subscription": true, + "aggregator": { + "entity": slug, + "operation": "sum", + "property": "credits", + "type": "metered", + }, + "price_configurations": [ + { + "currency": "USD", + "billing_interval": { "period": "months", "count": 1 }, + "type": "volume", + "prices": [ + { + "type": "volume", + "amount": 0, + "from": 0, + "to": null, + }, + ], + }, + ], + }); + + let created_product: ProductSummary = client.post_json("/v1/products", &body).await?; + println!(" [+] {slug:<18} created ({})", created_product.id); + created += 1; } + + println!("Done. created={created} skipped={skipped}"); + Ok(()) } diff --git a/crates/scrapix-billing-hyperline/src/lib.rs b/crates/scrapix-billing-hyperline/src/lib.rs index 3d8aaa0a..1c0cb9c5 100644 --- a/crates/scrapix-billing-hyperline/src/lib.rs +++ b/crates/scrapix-billing-hyperline/src/lib.rs @@ -10,7 +10,6 @@ //! - [`outbox`] — background drain worker that POSTs outbox rows to Hyperline. //! - [`webhooks`] — signature verification and payload parsing. //! - [`reconcile`] — wallet-balance drift checks between local ledger and Hyperline. -//! - [`products`] — idempotent product seeding (manual + boot-time auto-seed). //! - [`config`] — env-driven configuration. //! - [`error`] — crate-wide error type. @@ -19,14 +18,12 @@ pub mod config; pub mod error; pub mod events; pub mod outbox; -pub mod products; pub mod reconcile; pub mod webhooks; pub use client::{Customer, HyperlineClient, MoneyAmount, PortalLink, Wallet}; pub use config::HyperlineConfig; pub use error::HyperlineError; -pub use products::{auto_seed_enabled, seed_if_enabled, seed_products, SeedReport}; /// Boot-time self-check: pings Hyperline to validate auth+network, and /// logs the `BillingEventType` manifest that the API will emit. diff --git a/crates/scrapix-billing-hyperline/src/products.rs b/crates/scrapix-billing-hyperline/src/products.rs deleted file mode 100644 index 4e235308..00000000 --- a/crates/scrapix-billing-hyperline/src/products.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Idempotent product seeding for Hyperline. -//! -//! Hyperline's products + pricing live in the Hyperline dashboard and have -//! historically been a manual ops step before each deploy that introduces a -//! new `BillingEventType` variant. When the step is missed, usage events -//! 4xx at the ingest endpoint and the outbox backs up (see the runbook in -//! `docs/operations/hyperline-billing.mdx`). -//! -//! This module exposes the same seeding logic the `seed_hyperline_products` -//! example uses, but as a library function so the API can optionally run it -//! at boot (gated on `HYPERLINE_AUTO_SEED`, opt-in by default). -//! -//! ## Idempotency -//! -//! Hyperline does not expose an `external_id` on products, so name-equality -//! is the only stable lookup key. Each call first issues -//! `GET /v1/products?name__equals=&status=active` and skips -//! creation if a match exists. A partially-completed seed run is safe to -//! re-run; only the missing products are created. -//! -//! ## Pricing -//! -//! Seeded products are `dynamic` (metered) and use a single placeholder -//! volume tier at `amount: 0` in USD. This is intentional — the goal here -//! is to get the product *registered* so events stop 4xxing. Real prices -//! are still set by ops in the Hyperline UI after the seed. - -use serde::Deserialize; -use serde_json::json; -use tracing::{info, warn}; - -use crate::client::HyperlineClient; -use crate::error::HyperlineError; -use crate::events::BillingEventType; - -/// Display name + entity slug for the Hyperline product backing `ev`. -/// -/// The slug must equal `BillingEventType::as_str()` so the dynamic -/// aggregator's `entity` field matches the `event_type` posted to -/// `/v1/events` at ingest time. The unit test below enforces that. -pub fn product_for(ev: BillingEventType) -> (&'static str, &'static str) { - match ev { - BillingEventType::PageCrawled => ("Page crawled", "page_crawled"), - BillingEventType::BytesDownloaded => ("Bytes downloaded", "bytes_downloaded"), - BillingEventType::JsRender => ("JS render", "js_render"), - BillingEventType::ApiRequest => ("API request", "api_request"), - BillingEventType::DocumentIndexed => ("Document indexed", "document_indexed"), - BillingEventType::FeatureFormat => ("Feature format", "feature_format"), - BillingEventType::AiFeature => ("AI feature", "ai_feature"), - } -} - -/// Summary of a [`seed_products`] run. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct SeedReport { - pub created: usize, - pub skipped: usize, -} - -#[derive(Debug, Deserialize)] -struct ProductSummary { - id: String, - #[serde(default)] - name: Option, -} - -#[derive(Debug, Deserialize)] -struct ProductList { - data: Vec, -} - -/// Idempotently create one `dynamic` Hyperline product per -/// `BillingEventType` variant. -/// -/// Looks up each product by exact name match before posting. Returns the -/// counts of created vs. skipped products. Any Hyperline error short-circuits -/// the run — callers that want log-and-continue semantics should wrap with -/// [`seed_if_enabled`] (or `let _ = seed_products(&client).await`). -pub async fn seed_products(client: &HyperlineClient) -> Result { - let mut report = SeedReport::default(); - - for ev in BillingEventType::all().iter().copied() { - let (name, slug) = product_for(ev); - - let existing: ProductList = client - .get_json( - "/v1/products", - &[("name__equals", name), ("status", "active")], - ) - .await?; - if let Some(p) = existing.data.first() { - info!( - event_type = slug, - product_id = %p.id, - product_name = p.name.as_deref().unwrap_or(name), - "Hyperline product already exists — skipping" - ); - report.skipped += 1; - continue; - } - - let body = json!({ - "type": "dynamic", - "name": name, - "description": format!("Scrapix billable event: {slug}"), - "unit_name": "credit", - "is_available_on_demand": true, - "is_available_on_subscription": true, - "aggregator": { - "entity": slug, - "operation": "sum", - "property": "credits", - "type": "metered", - }, - "price_configurations": [ - { - "currency": "USD", - "billing_interval": { "period": "months", "count": 1 }, - "type": "volume", - "prices": [ - { - "type": "volume", - "amount": 0, - "from": 0, - "to": null, - }, - ], - }, - ], - }); - - let created: ProductSummary = client.post_json("/v1/products", &body).await?; - info!( - event_type = slug, - product_id = %created.id, - "Hyperline product created (placeholder zero-amount pricing — set real prices in the dashboard)" - ); - report.created += 1; - } - - Ok(report) -} - -/// Returns true when the `HYPERLINE_AUTO_SEED` env var is set to `1`/`true` -/// (case-insensitive, with surrounding whitespace trimmed). Any other value -/// — including unset — returns false. -pub fn auto_seed_enabled() -> bool { - match std::env::var("HYPERLINE_AUTO_SEED") { - Ok(v) => { - let v = v.trim().to_ascii_lowercase(); - v == "1" || v == "true" - } - Err(_) => false, - } -} - -/// Boot-time wrapper: if `HYPERLINE_AUTO_SEED` is set, run [`seed_products`] -/// and log the outcome. Failures are downgraded to a `WARN` and swallowed — -/// the API must not crashloop because of a Hyperline outage or a transient -/// product-API hiccup. The outbox keeps enqueuing either way; events will -/// still 4xx until products exist, but ops can re-run the example script -/// manually if the auto-seed degraded. -pub async fn seed_if_enabled(client: &HyperlineClient) { - if !auto_seed_enabled() { - return; - } - - info!( - sandbox = client.config().is_sandbox(), - "HYPERLINE_AUTO_SEED is set — ensuring metered products exist in Hyperline" - ); - - match seed_products(client).await { - Ok(report) => { - info!( - created = report.created, - skipped = report.skipped, - sandbox = client.config().is_sandbox(), - "Hyperline auto-seed completed" - ); - } - Err(e) => { - warn!( - error = %e, - sandbox = client.config().is_sandbox(), - "Hyperline auto-seed failed — set real prices manually or re-run `cargo run -p scrapix-billing-hyperline --example seed_hyperline_products`. Events will 4xx until products exist." - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn product_slug_matches_event_type() { - // The dynamic aggregator's `entity` field is the slug, and Hyperline - // matches it against the `event_type` we POST at ingest. If these - // ever drift the events 4xx silently from the customer's POV. - for ev in BillingEventType::all().iter().copied() { - let (_name, slug) = product_for(ev); - assert_eq!( - slug, - ev.as_str(), - "product_for({ev:?}) slug must equal BillingEventType::as_str()" - ); - } - } - - #[test] - fn product_for_covers_every_variant() { - // Catches the "added a variant, forgot the product mapping" mistake. - for ev in BillingEventType::all().iter().copied() { - let (name, slug) = product_for(ev); - assert!(!name.is_empty(), "missing display name for {ev:?}"); - assert!(!slug.is_empty(), "missing slug for {ev:?}"); - } - } - - #[test] - fn auto_seed_enabled_reads_env_var() { - // SAFETY: tests mutate process env; restore at the end. - let prev = std::env::var("HYPERLINE_AUTO_SEED").ok(); - - for (val, expected) in [ - ("1", true), - ("true", true), - ("TRUE", true), - (" 1 ", true), - ("0", false), - ("false", false), - ("yes", false), - ("", false), - ] { - unsafe { - std::env::set_var("HYPERLINE_AUTO_SEED", val); - } - assert_eq!( - auto_seed_enabled(), - expected, - "HYPERLINE_AUTO_SEED={val:?} should be {expected}" - ); - } - - unsafe { - std::env::remove_var("HYPERLINE_AUTO_SEED"); - } - assert!(!auto_seed_enabled(), "unset should be false"); - - // Restore. - unsafe { - match prev { - Some(v) => std::env::set_var("HYPERLINE_AUTO_SEED", v), - None => std::env::remove_var("HYPERLINE_AUTO_SEED"), - } - } - } -} diff --git a/docs/configuration/environment-variables.mdx b/docs/configuration/environment-variables.mdx index ec242f7a..c457c7f2 100644 --- a/docs/configuration/environment-variables.mdx +++ b/docs/configuration/environment-variables.mdx @@ -196,7 +196,6 @@ Scrapix uses Hyperline (with Stripe as the downstream payment processor) as the | `HYPERLINE_API_BASE` | auto | Control-plane base. Defaults to `https://sandbox.api.hyperline.co` for `test_` keys, `https://api.hyperline.co` for `prod_` keys. | | `HYPERLINE_INGEST_BASE` | auto | Events ingest host. Defaults to `https://sandbox.ingest.hyperline.co` / `https://ingest.hyperline.co` by the same rule. | | `HYPERLINE_WEBHOOK_SECRET` | — | `whsec_`-prefixed signing secret for `POST /webhooks/hyperline`. Required to enable webhook processing — absent means no signature verification is possible and the route returns 503. | -| `HYPERLINE_AUTO_SEED` | — | Set to `1` (or `true`) to seed one `dynamic` metered product per `BillingEventType` on API boot. Idempotent (skips products that already exist by name). See [Product Seeding in the runbook](/operations/hyperline-billing#product-seeding). | | `HYPERLINE_CENTS_PER_CREDIT` | — | How many cents one local credit is worth. Together with `HYPERLINE_DRIFT_TOLERANCE_CENTS`, enables WARN-level drift alerts in the reconcile worker. Both must be set; otherwise the worker stays in pure-observation mode. | | `HYPERLINE_DRIFT_TOLERANCE_CENTS` | — | Absolute drift tolerance in cents. Drift below this stays at INFO; drift above gets WARN. Set to ~1× a typical hot-path debit cost — see the [reconcile section of the runbook](/operations/hyperline-billing#reconcile-scan). | diff --git a/docs/operations/hyperline-billing.mdx b/docs/operations/hyperline-billing.mdx index 31e77522..220dc6b7 100644 --- a/docs/operations/hyperline-billing.mdx +++ b/docs/operations/hyperline-billing.mdx @@ -11,11 +11,10 @@ This page covers the pieces of the pipeline that ops are expected to operate: 1. Configuration surface 2. Boot self-check -3. Product seeding — manual + boot-time auto-seed -4. Outbox drain — emitting usage events to Hyperline -5. Webhooks — receiving state updates from Hyperline -6. Reconcile scan — drift detection between local ledger and Hyperline wallet -7. Replay procedures for each path +3. Outbox drain — emitting usage events to Hyperline +4. Webhooks — receiving state updates from Hyperline +5. Reconcile scan — drift detection between local ledger and Hyperline wallet +6. Replay procedures for each path ## Configuration @@ -27,7 +26,6 @@ Set these on the API and worker services (see [environment variables](/configura | `HYPERLINE_API_BASE` | no | Override control-plane host. | | `HYPERLINE_INGEST_BASE` | no | Override events ingest host. | | `HYPERLINE_WEBHOOK_SECRET` | yes, for webhooks | `whsec_…` — required to verify `POST /webhooks/hyperline`. | -| `HYPERLINE_AUTO_SEED` | no | Set to `1` to run the product seed on every API boot (idempotent). See [Product Seeding](#product-seeding). | Any missing var degrades that surface to off (not error). A missing `HYPERLINE_API_KEY` means the outbox still enqueues — rows accumulate and drain once the env var is set and a pod restarts. @@ -41,42 +39,10 @@ INFO hyperline: Hyperline event-type manifest — verify each has a metered prod INFO hyperline: Hyperline boot self-check passed sandbox=false ``` -If you add a new `BillingEventType` variant, this log reminds ops to create the matching metered product in Hyperline **before** the deploy lands — otherwise the event ingest will 4xx and the outbox will back up. The auto-seed below removes the "before" hard requirement when enabled. +If you add a new `BillingEventType` variant, this log reminds ops to create the matching metered product in Hyperline **before** the deploy lands — otherwise the event ingest will 4xx and the outbox will back up. A `WARN` instead of the second `INFO` (`Hyperline boot self-check failed — API unreachable or auth invalid`) means the key is wrong or Hyperline is down. Boot continues either way. -## Product Seeding - -Hyperline products correspond 1-to-1 with `BillingEventType` variants. Each is a `dynamic` metered product whose aggregator sums the `credits` property of its incoming events. Products are looked up by exact name match before any POST, so seeding is idempotent — re-running creates only what is missing. - -The seeded product carries placeholder **zero-amount** pricing (USD, monthly, volume tier 0 → ∞). The seed only registers the product so events stop 4xxing; real prices are still set by ops in the Hyperline UI afterward. - -### Manual one-shot - -Run before — or right after — the deploy that adds a new `BillingEventType`: - -```sh -HYPERLINE_API_KEY=prod_… cargo run -p scrapix-billing-hyperline \ - --example seed_hyperline_products -``` - -The example prints `[+]` for created products and `[=]` for ones it skipped. Safe to re-run on any schedule. - -### Boot-time auto-seed - -Set `HYPERLINE_AUTO_SEED=1` on the API service. On every boot — right after the boot self-check — the API runs the same seed routine. Logs read: - -```text -INFO hyperline: HYPERLINE_AUTO_SEED is set — ensuring metered products exist in Hyperline sandbox=false -INFO hyperline: Hyperline product already exists — skipping event_type=page_crawled product_id=prd_… -INFO hyperline: Hyperline product created (placeholder zero-amount pricing — set real prices in the dashboard) event_type=new_event_type product_id=prd_… -INFO hyperline: Hyperline auto-seed completed created=1 skipped=6 sandbox=false -``` - -A failure logs at `WARN` and is swallowed — the API still boots, the outbox still enqueues, and ops can re-run the manual seed. Events for any missing product will 4xx in the outbox until either the auto-seed retries on the next pod restart or the manual seed runs. - -**When to enable:** keep enabled in production once the workspace is initialised. The first boot does the heavy lift; subsequent boots are six `GET /v1/products?name__equals=…` calls (~ms). Leaving it on closes the deploy-order gap for future `BillingEventType` additions. - ## Outbox Drain **Shape:** `hyperline_events_outbox(id UUID PK, account_id, event_type, payload JSONB, created_at, sent_at, attempts)` — one row per billable debit, enqueued immediately after the local ledger debit returns. @@ -110,12 +76,7 @@ WHERE sent_at IS NULL AND attempts > 3 GROUP BY 1 ORDER BY 2 DESC; ``` -3. An unknown `event_type` hit the outbox — e.g. a newly added variant without a matching Hyperline product. Either: - - Run the manual seed once: `HYPERLINE_API_KEY=prod_… cargo run -p scrapix-billing-hyperline --example seed_hyperline_products` (idempotent). - - Set `HYPERLINE_AUTO_SEED=1` and restart the API — the boot will create any missing products. See [Product Seeding](#product-seeding). - - Or roll the deploy back. - - Either way, the rows will drain themselves on the next tick once the product exists. +3. An unknown `event_type` hit the outbox — e.g. a newly added variant without a matching Hyperline product. Either create the product in Hyperline's dashboard or roll the deploy back; the rows will drain themselves on the next tick once the product exists. ### Manual flush / replay From 2388be6a30da48ba8572ce6ba90055bc78c4e22b Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 May 2026 17:21:01 +0200 Subject: [PATCH 2/5] Revert "fix(billing): auto-link accounts to Hyperline customer (close SCR-68 gap) (#19)" This reverts commit 20bf9327f3cf6886a681d298369e89c276f63372. --- bins/scrapix-api/src/auth/handlers/auth.rs | 24 +-- bins/scrapix-api/src/auth/handlers/billing.rs | 130 +--------------- .../src/bin/backfill_hyperline_customers.rs | 147 ------------------ .../scrapix-billing-hyperline/src/client.rs | 39 ----- 4 files changed, 8 insertions(+), 332 deletions(-) delete mode 100644 bins/scrapix-api/src/bin/backfill_hyperline_customers.rs diff --git a/bins/scrapix-api/src/auth/handlers/auth.rs b/bins/scrapix-api/src/auth/handlers/auth.rs index cf410cac..77f3ff0b 100644 --- a/bins/scrapix-api/src/auth/handlers/auth.rs +++ b/bins/scrapix-api/src/auth/handlers/auth.rs @@ -7,9 +7,8 @@ use axum_extra::extract::CookieJar; use sha2::{Digest, Sha256}; use sqlx::Row; use std::sync::Arc; -use tracing::{info, warn}; +use tracing::info; -use super::billing::link_account_to_hyperline; use super::{ build_session_cookie, clear_session_cookie, err, AccountResponse, ApiError, ErrorBody, ForgotPasswordRequest, LoginRequest, MessageResponse, ResetPasswordRequest, SignupRequest, @@ -157,27 +156,6 @@ pub(crate) async fn signup( info!(user_id = %user_id, email = %req.email, "New user signed up"); - // Best-effort: create the Hyperline customer in the background so the - // first GET /account/billing/portal call doesn't pay the latency. If - // this fails (Hyperline outage, etc.), the portal handler's - // lazy-create path will retry on next click. - if let Some(client) = state.hyperline_client.clone() { - let pool = state.pool.clone(); - let name = account_name.clone(); - let email = req.email.clone(); - tokio::spawn(async move { - if let Err(e) = - link_account_to_hyperline(&pool, &client, account_id, &name, &email).await - { - warn!( - account_id = %account_id, - error = %e, - "hyperline eager-link on signup failed — will retry lazily on first portal click" - ); - } - }); - } - // Auto-accept pending invites for this email let pending_invites = sqlx::query( "SELECT id, account_id, role FROM account_invites \ diff --git a/bins/scrapix-api/src/auth/handlers/billing.rs b/bins/scrapix-api/src/auth/handlers/billing.rs index e1cc3620..bcbd9cac 100644 --- a/bins/scrapix-api/src/auth/handlers/billing.rs +++ b/bins/scrapix-api/src/auth/handlers/billing.rs @@ -3,8 +3,7 @@ use axum::{ http::StatusCode, Json, }; -use scrapix_billing_hyperline::HyperlineClient; -use sqlx::{PgPool, Row}; +use sqlx::Row; use std::sync::Arc; use tracing::{error, info}; @@ -15,87 +14,6 @@ use super::{ }; use crate::auth::{AuthState, AuthenticatedUser}; -/// Lazily link an account to a Hyperline customer. -/// -/// Looks up `(account_name, owner_email)`, queries Hyperline for an -/// existing customer with `external_id = account_id` (recovers from -/// crashes that orphaned a customer mid-link), creates one if absent, -/// then `UPDATE accounts SET hyperline_customer_id = COALESCE(...)` so -/// concurrent linkers converge on a single value. -/// -/// Returns the linked customer id. Network/auth failures bubble up; -/// the caller decides which HTTP status to map to. -pub(crate) async fn link_account_to_hyperline( - pool: &PgPool, - client: &HyperlineClient, - account_id: uuid::Uuid, - account_name: &str, - owner_email: &str, -) -> Result { - let external_id = account_id.to_string(); - - let customer = match client.find_customer_by_external_id(&external_id).await? { - Some(existing) => { - info!( - account_id = %account_id, - customer_id = %existing.id, - "hyperline: recovered orphaned customer by external_id" - ); - existing - } - None => { - let created = client - .create_customer(&external_id, account_name, owner_email) - .await?; - info!( - account_id = %account_id, - customer_id = %created.id, - "hyperline: created customer" - ); - created - } - }; - - // COALESCE handles the race where another request linked first; we - // adopt whatever id won and discard our just-created one (it stays - // in Hyperline as an orphan, but `find_customer_by_external_id` - // will reuse it on any future link attempt for this account). - let linked_id: String = sqlx::query_scalar( - "UPDATE accounts \ - SET hyperline_customer_id = COALESCE(hyperline_customer_id, $1) \ - WHERE id = $2 RETURNING hyperline_customer_id", - ) - .bind(&customer.id) - .bind(account_id) - .fetch_one(pool) - .await - .map_err(|e| scrapix_billing_hyperline::HyperlineError::InvalidConfig(format!("db: {e}")))?; - - Ok(linked_id) -} - -/// Owner email for an account — used as the Hyperline `email` field -/// on lazy customer creation. Picks the oldest 'owner' membership so -/// the result is stable across reruns. -pub(crate) async fn account_owner_email( - pool: &PgPool, - account_id: uuid::Uuid, -) -> Result, sqlx::Error> { - let row = sqlx::query( - "SELECT a.name AS account_name, u.email AS owner_email \ - FROM accounts a \ - JOIN account_members m ON m.account_id = a.id \ - JOIN users u ON u.id = m.user_id \ - WHERE a.id = $1 AND m.role = 'owner' \ - ORDER BY m.joined_at ASC \ - LIMIT 1", - ) - .bind(account_id) - .fetch_optional(pool) - .await?; - Ok(row.map(|r| (r.get("account_name"), r.get("owner_email")))) -} - #[utoipa::path( get, path = "/account/billing", @@ -224,46 +142,12 @@ pub(crate) async fn get_billing_portal( })? .flatten(); - let customer_id = match customer_id { - Some(id) => id, - None => { - // First portal request for this account — lazy-link to Hyperline. - // Closes the gap left by SCR-68 (no automated customer creation): - // accounts created before the migration, or whose eager-create at - // signup failed, would otherwise be stuck with a 404 forever. - let Some((account_name, owner_email)) = account_owner_email(&state.pool, account_id) - .await - .map_err(|e| { - error!(account_id = %account_id, error = %e, "owner lookup failed"); - err( - StatusCode::INTERNAL_SERVER_ERROR, - "Database error", - "internal_error", - ) - })? - else { - return Err(err( - StatusCode::NOT_FOUND, - "Account has no owner — cannot link to Hyperline", - "no_owner", - )); - }; - - link_account_to_hyperline(&state.pool, client, account_id, &account_name, &owner_email) - .await - .map_err(|e| { - error!( - account_id = %account_id, - error = %e, - "hyperline lazy-link failed" - ); - err( - StatusCode::BAD_GATEWAY, - "Failed to link account to Hyperline", - "hyperline_upstream", - ) - })? - } + let Some(customer_id) = customer_id else { + return Err(err( + StatusCode::NOT_FOUND, + "Account is not linked to Hyperline yet", + "not_linked", + )); }; match client.get_portal_url(&customer_id).await { diff --git a/bins/scrapix-api/src/bin/backfill_hyperline_customers.rs b/bins/scrapix-api/src/bin/backfill_hyperline_customers.rs deleted file mode 100644 index 0007ea78..00000000 --- a/bins/scrapix-api/src/bin/backfill_hyperline_customers.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Backfill `accounts.hyperline_customer_id` for rows that survived the -//! SCR-68 Stripe → Hyperline migration with NULL. -//! -//! Walks every account where `hyperline_customer_id IS NULL`, looks up -//! the oldest 'owner' membership for `(name, email)`, and either reuses -//! an existing Hyperline customer (matched by `external_id = -//! accounts.id`) or creates a new one — then writes the id back. -//! -//! Safe to re-run: each step is idempotent. Failures are logged per -//! account and the loop continues; rerun to retry only the survivors. -//! -//! ```sh -//! DATABASE_URL=postgres://… \ -//! HYPERLINE_API_KEY=prod_… \ -//! cargo run -p scrapix-api --bin backfill_hyperline_customers -//! ``` - -use std::error::Error; - -use scrapix_billing_hyperline::HyperlineClient; -use sqlx::{postgres::PgPoolOptions, PgPool, Row}; -use tracing::{error, info, warn}; -use uuid::Uuid; - -#[tokio::main] -async fn main() -> Result<(), Box> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - - let database_url = std::env::var("DATABASE_URL")?; - let url = if !database_url.contains("sslmode=") { - let sep = if database_url.contains('?') { "&" } else { "?" }; - format!("{database_url}{sep}sslmode=require") - } else { - database_url - }; - let pool = PgPoolOptions::new() - .max_connections(2) - .connect(&url) - .await?; - - let client = HyperlineClient::from_env()?; - let target = if client.config().is_sandbox() { - "sandbox" - } else { - "PRODUCTION" - }; - info!("Backfilling Hyperline customers into {target}…"); - - // We page through accounts; new ones get linked as the loop runs so - // restarting from the top each iteration would be wasteful. Capture - // the snapshot once and walk it. - let rows = sqlx::query( - "SELECT a.id AS account_id, a.name AS account_name, u.email AS owner_email \ - FROM accounts a \ - JOIN account_members m ON m.account_id = a.id AND m.role = 'owner' \ - JOIN users u ON u.id = m.user_id \ - WHERE a.hyperline_customer_id IS NULL \ - GROUP BY a.id, a.name, u.email, m.joined_at \ - ORDER BY a.id, m.joined_at ASC", - ) - .fetch_all(&pool) - .await?; - - // GROUP BY + ORDER BY gives duplicates per account if multiple - // owners. Dedup by account_id, keeping first (= oldest owner). - let mut seen = std::collections::HashSet::new(); - let mut targets: Vec<(Uuid, String, String)> = Vec::new(); - for row in rows { - let id: Uuid = row.get("account_id"); - if !seen.insert(id) { - continue; - } - targets.push((id, row.get("account_name"), row.get("owner_email"))); - } - - info!("Found {} unlinked account(s)", targets.len()); - - let mut linked = 0usize; - let mut reused = 0usize; - let mut failed = 0usize; - - for (account_id, account_name, owner_email) in targets { - match link_one(&pool, &client, account_id, &account_name, &owner_email).await { - Ok(LinkOutcome::Created) => { - linked += 1; - info!(account_id = %account_id, "linked (created in Hyperline)"); - } - Ok(LinkOutcome::Reused) => { - reused += 1; - info!(account_id = %account_id, "linked (reused existing Hyperline customer)"); - } - Err(e) => { - failed += 1; - error!(account_id = %account_id, error = %e, "link failed"); - } - } - } - - info!("Done. created={linked} reused={reused} failed={failed}"); - if failed > 0 { - warn!("{failed} account(s) still unlinked — rerun after addressing the root cause"); - std::process::exit(1); - } - Ok(()) -} - -enum LinkOutcome { - Created, - Reused, -} - -async fn link_one( - pool: &PgPool, - client: &HyperlineClient, - account_id: Uuid, - account_name: &str, - owner_email: &str, -) -> Result> { - let external_id = account_id.to_string(); - - let (customer_id, outcome) = match client.find_customer_by_external_id(&external_id).await? { - Some(existing) => (existing.id, LinkOutcome::Reused), - None => { - let created = client - .create_customer(&external_id, account_name, owner_email) - .await?; - (created.id, LinkOutcome::Created) - } - }; - - sqlx::query( - "UPDATE accounts \ - SET hyperline_customer_id = COALESCE(hyperline_customer_id, $1) \ - WHERE id = $2", - ) - .bind(&customer_id) - .bind(account_id) - .execute(pool) - .await?; - - Ok(outcome) -} diff --git a/crates/scrapix-billing-hyperline/src/client.rs b/crates/scrapix-billing-hyperline/src/client.rs index f1ea36fc..404f81f6 100644 --- a/crates/scrapix-billing-hyperline/src/client.rs +++ b/crates/scrapix-billing-hyperline/src/client.rs @@ -200,45 +200,6 @@ impl HyperlineClient { Ok(()) } - /// Look up a customer by its `external_id`. Returns `None` if no row - /// matches. Used by the lazy-link path on `GET /account/billing/portal` - /// to recover from a prior partial link (Hyperline customer created but - /// the `UPDATE accounts` step never landed — race or crash). Without - /// this we'd orphan a customer in Hyperline and create a duplicate on - /// the next retry. - pub async fn find_customer_by_external_id( - &self, - external_id: &str, - ) -> Result, HyperlineError> { - let page: ListResponse = self - .get_json( - "/v1/customers", - &[("external_id__equals", external_id), ("limit", "1")], - ) - .await?; - Ok(page.data.into_iter().next()) - } - - /// Create a Hyperline customer. `external_id` is the local - /// `accounts.id` (UUID) so we can recover the mapping if our DB row - /// gets cleared but Hyperline's record survives. A 4xx (e.g. - /// duplicate `external_id`) is surfaced as `HyperlineError::Api` — - /// the lazy-create path checks `find_customer_by_external_id` first - /// so this should only fire on genuinely-new customers. - pub async fn create_customer( - &self, - external_id: &str, - name: &str, - email: &str, - ) -> Result { - let body = serde_json::json!({ - "external_id": external_id, - "name": name, - "email": email, - }); - self.post_json("/v1/customers", &body).await - } - /// Fetches a wallet by its Hyperline id. /// /// Wallets are not auto-provisioned — if the customer has no wallet From 306557d07bea2b440147819ea51cfeca7f52427e Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 May 2026 17:21:05 +0200 Subject: [PATCH 3/5] Revert "SCR-68: migrate billing from Stripe to Hyperline (#6)" This reverts commit 3a94103fe128753f5873cd4f136d987f8b7bf758. --- .env.example | 12 - Cargo.lock | 281 +++- Cargo.toml | 1 - bins/scrapix-api/Cargo.toml | 13 +- bins/scrapix-api/src/auth/handlers/billing.rs | 189 +-- bins/scrapix-api/src/auth/handlers/mod.rs | 44 +- bins/scrapix-api/src/auth/mod.rs | 8 - bins/scrapix-api/src/billing.rs | 77 +- bins/scrapix-api/src/hyperline.rs | 632 --------- bins/scrapix-api/src/lib.rs | 148 +- bins/scrapix-api/src/openapi.rs | 1 - bins/scrapix-api/src/stripe.rs | 1244 +++++++++++++++++ bins/scrapix-api/tests/hyperline_webhook.rs | 581 -------- bins/scrapix/src/all.rs | 4 + console/Dockerfile | 3 +- console/heroku.yml | 1 + console/package.json | 2 + console/src/app/(marketing)/privacy/page.tsx | 15 +- console/src/app/(marketing)/terms/page.tsx | 10 +- console/src/app/dashboard/billing/page.tsx | 1012 +++++++++++--- console/src/lib/api-types.ts | 65 +- console/src/lib/api.ts | 72 +- console/src/lib/hooks.ts | 16 +- crates/scrapix-billing-hyperline/Cargo.toml | 36 - .../examples/seed_hyperline_products.rs | 150 -- .../scrapix-billing-hyperline/src/client.rs | 248 ---- .../scrapix-billing-hyperline/src/config.rs | 96 -- crates/scrapix-billing-hyperline/src/error.rs | 34 - .../scrapix-billing-hyperline/src/events.rs | 199 --- crates/scrapix-billing-hyperline/src/lib.rs | 72 - .../scrapix-billing-hyperline/src/outbox.rs | 201 --- .../src/reconcile.rs | 246 ---- .../scrapix-billing-hyperline/src/webhooks.rs | 198 --- .../tests/sandbox_roundtrip.rs | 157 --- crates/scrapix-billing/Cargo.toml | 5 +- crates/scrapix-billing/src/ledger.rs | 76 +- crates/scrapix-billing/src/lib.rs | 2 +- deploy/postgres/init.sql | 120 +- docs/configuration/environment-variables.mdx | 17 - docs/guides/billing.mdx | 14 +- docs/mint.json | 3 +- docs/operations/hyperline-billing.mdx | 218 --- 42 files changed, 2610 insertions(+), 3913 deletions(-) delete mode 100644 bins/scrapix-api/src/hyperline.rs create mode 100644 bins/scrapix-api/src/stripe.rs delete mode 100644 bins/scrapix-api/tests/hyperline_webhook.rs delete mode 100644 crates/scrapix-billing-hyperline/Cargo.toml delete mode 100644 crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs delete mode 100644 crates/scrapix-billing-hyperline/src/client.rs delete mode 100644 crates/scrapix-billing-hyperline/src/config.rs delete mode 100644 crates/scrapix-billing-hyperline/src/error.rs delete mode 100644 crates/scrapix-billing-hyperline/src/events.rs delete mode 100644 crates/scrapix-billing-hyperline/src/lib.rs delete mode 100644 crates/scrapix-billing-hyperline/src/outbox.rs delete mode 100644 crates/scrapix-billing-hyperline/src/reconcile.rs delete mode 100644 crates/scrapix-billing-hyperline/src/webhooks.rs delete mode 100644 crates/scrapix-billing-hyperline/tests/sandbox_roundtrip.rs delete mode 100644 docs/operations/hyperline-billing.mdx diff --git a/.env.example b/.env.example index 7a27cf23..b446e120 100644 --- a/.env.example +++ b/.env.example @@ -92,18 +92,6 @@ BATCH_TIMEOUT_SECS=5 # STRIPE_SECRET_KEY=sk_test_... # STRIPE_WEBHOOK_SECRET=whsec_... -# ----------------------------------------------------------------------------- -# Hyperline Billing (Migration in progress) -# ----------------------------------------------------------------------------- -# API key prefix: test_ = sandbox, prod_ = production -# HYPERLINE_API_KEY=test_... -# HYPERLINE_API_BASE=https://sandbox.api.hyperline.co -# HYPERLINE_INGEST_BASE=https://sandbox.ingest.hyperline.co -# HYPERLINE_WEBHOOK_SECRET=whsec_... -# Drift alerting (opt-in, both required): convert credits → cents and alert on |drift| > tolerance. -# HYPERLINE_CENTS_PER_CREDIT=1 -# HYPERLINE_DRIFT_TOLERANCE_CENTS=10 - # ----------------------------------------------------------------------------- # AI Features (Optional) # ----------------------------------------------------------------------------- diff --git a/Cargo.lock b/Cargo.lock index 004a613b..538d25a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,8 +376,8 @@ checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" dependencies = [ "async-task", "concurrent-queue", - "fastrand", - "futures-lite", + "fastrand 2.3.0", + "futures-lite 2.6.1", "pin-project-lite", "slab", ] @@ -393,7 +393,7 @@ dependencies = [ "async-io", "async-lock", "blocking", - "futures-lite", + "futures-lite 2.6.1", "once_cell", ] @@ -407,7 +407,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite", + "futures-lite 2.6.1", "parking", "polling", "rustix 1.1.3", @@ -477,7 +477,7 @@ dependencies = [ "blocking", "cfg-if", "event-listener 5.4.1", - "futures-lite", + "futures-lite 2.6.1", "rustix 1.1.3", ] @@ -515,7 +515,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 2.6.1", "gloo-timers", "kv-log-macro", "log", @@ -549,6 +549,30 @@ dependencies = [ "syn 2.0.113", ] +[[package]] +name = "async-stripe" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ddaa6999d246ba2c6c84d830a1ba0cd16c9234d58701988b3869f0e5bd732d" +dependencies = [ + "chrono", + "futures-util", + "hex", + "hmac", + "http-types", + "hyper 0.14.32", + "hyper-tls 0.5.0", + "serde", + "serde_json", + "serde_path_to_error", + "serde_qs 0.10.1", + "sha2", + "smart-default", + "smol_str", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "async-task" version = "4.7.1" @@ -774,9 +798,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand", + "fastrand 2.3.0", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -894,7 +924,7 @@ dependencies = [ "async-channel 2.5.0", "async-task", "futures-io", - "futures-lite", + "futures-lite 2.6.1", "piper", ] @@ -2162,6 +2192,15 @@ dependencies = [ "webdriver", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -2333,13 +2372,28 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-lite" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -2421,6 +2475,17 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -2430,7 +2495,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2480,6 +2545,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.12" @@ -2557,16 +2641,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "hdrhistogram" -version = "7.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" -dependencies = [ - "byteorder", - "num-traits", -] - [[package]] name = "heck" version = "0.4.1" @@ -2759,6 +2833,27 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel 1.9.0", + "base64 0.13.1", + "futures-lite 1.13.0", + "http 0.2.12", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs 0.8.5", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.10.1" @@ -2781,6 +2876,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -2804,7 +2900,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", + "h2 0.4.12", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -3059,6 +3155,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "inout" version = "0.1.4" @@ -3665,7 +3767,7 @@ checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -4186,7 +4288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" dependencies = [ "atomic-waker", - "fastrand", + "fastrand 2.3.0", "futures-io", ] @@ -4494,6 +4596,19 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -4526,6 +4641,16 @@ dependencies = [ "rand_core 0.10.0", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -4546,6 +4671,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -4570,6 +4704,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rangemap" version = "1.7.1" @@ -4772,7 +4915,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.4.12", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -4822,7 +4965,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.4.12", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -5320,6 +5463,7 @@ version = "0.1.0" dependencies = [ "anyhow", "argon2", + "async-stripe", "async-trait", "axum", "axum-extra", @@ -5331,7 +5475,6 @@ dependencies = [ "futures", "hex", "hmac", - "http-body-util", "jsonwebtoken", "parking_lot", "rand 0.8.5", @@ -5344,7 +5487,6 @@ dependencies = [ "scrapix-ai", "scrapix-auth", "scrapix-billing", - "scrapix-billing-hyperline", "scrapix-core", "scrapix-crawler", "scrapix-extractor", @@ -5358,7 +5500,6 @@ dependencies = [ "time", "tokio", "tokio-stream", - "tower", "tower-http", "tracing", "tracing-subscriber", @@ -5397,6 +5538,7 @@ dependencies = [ name = "scrapix-billing" version = "0.1.0" dependencies = [ + "async-stripe", "async-trait", "chrono", "scrapix-core", @@ -5409,29 +5551,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "scrapix-billing-hyperline" -version = "0.1.0" -dependencies = [ - "async-trait", - "base64 0.22.1", - "chrono", - "hmac", - "reqwest 0.12.28", - "scrapix-core", - "serde", - "serde_json", - "sha2", - "sqlx", - "subtle", - "thiserror 1.0.69", - "tokio", - "tracing", - "tracing-subscriber", - "url", - "uuid", -] - [[package]] name = "scrapix-cli" version = "0.1.0" @@ -5868,6 +5987,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_qs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cac3f1e2ca2fe333923a1ae72caca910b98ed0630bb35ef6f8c8517d6e81afa" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -6006,6 +6147,17 @@ dependencies = [ "serde", ] +[[package]] +name = "smart-default" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "smartstring" version = "1.0.1" @@ -6017,6 +6169,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "smol_str" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.5.10" @@ -6470,7 +6631,7 @@ version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ - "fastrand", + "fastrand 2.3.0", "getrandom 0.3.4", "once_cell", "rustix 1.1.3", @@ -6792,13 +6953,9 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "hdrhistogram", - "indexmap", "pin-project-lite", - "slab", "sync_wrapper", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", @@ -7197,6 +7354,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "walkdir" version = "2.5.0" @@ -7216,6 +7379,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 5faaef1d..9df0a76e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ members = [ "crates/scrapix-storage", "crates/scrapix-queue", "crates/scrapix-billing", - "crates/scrapix-billing-hyperline", "crates/scrapix-auth", "crates/scrapix-lifecycle", diff --git a/bins/scrapix-api/Cargo.toml b/bins/scrapix-api/Cargo.toml index 378c8f2e..884781a5 100644 --- a/bins/scrapix-api/Cargo.toml +++ b/bins/scrapix-api/Cargo.toml @@ -17,7 +17,6 @@ path = "src/main.rs" [dependencies] scrapix-core = { path = "../../crates/scrapix-core" } scrapix-billing = { path = "../../crates/scrapix-billing" } -scrapix-billing-hyperline = { path = "../../crates/scrapix-billing-hyperline" } scrapix-auth = { path = "../../crates/scrapix-auth" } scrapix-queue = { path = "../../crates/scrapix-queue" } scrapix-storage = { path = "../../crates/scrapix-storage" } @@ -87,18 +86,10 @@ rmcp-openapi = "0.25" # Cron scheduling croner = "3" -# Stripe integration was removed during the Hyperline migration — all -# payment-processor calls now go through scrapix-billing-hyperline. +# Stripe payments +async-stripe = { version = "0.38", default-features = false, features = ["runtime-tokio-hyper", "checkout", "billing", "connect", "webhook-events"] } hmac = "0.12" # OpenAPI utoipa = { workspace = true, features = ["axum_extras"] } utoipa-scalar = { version = "0.3", features = ["axum"] } - -[dev-dependencies] -# Integration tests for the Hyperline webhook route drive the real axum -# router in-process against a postgres pool. Tests are gated on -# TEST_DATABASE_URL so they skip cleanly when the dev DB isn't -# available; set it to the docker-compose dev DB or a scratch one. -tower = { workspace = true } -http-body-util = "0.1" diff --git a/bins/scrapix-api/src/auth/handlers/billing.rs b/bins/scrapix-api/src/auth/handlers/billing.rs index bcbd9cac..f90c6b92 100644 --- a/bins/scrapix-api/src/auth/handlers/billing.rs +++ b/bins/scrapix-api/src/auth/handlers/billing.rs @@ -9,8 +9,8 @@ use tracing::{error, info}; use super::{ err, get_user_account_id, get_user_role, require_role, ApiError, AutoTopupRequest, - BillingResponse, ErrorBody, MessageResponse, PortalResponse, SpendLimitRequest, TopupRequest, - TopupResponse, TransactionResponse, TransactionsListResponse, UpdateBillingRequest, + BillingResponse, ErrorBody, MessageResponse, SpendLimitRequest, TopupRequest, TopupResponse, + TransactionResponse, TransactionsListResponse, UpdateBillingRequest, }; use crate::auth::{AuthState, AuthenticatedUser}; @@ -32,13 +32,9 @@ pub(crate) async fn get_billing( .await .map_err(|_| err(StatusCode::NOT_FOUND, "Account not found", "not_found"))?; - // Balance-source-of-truth is still the local ledger (hot path). The - // Hyperline handles are returned so the console can deep-link to the - // hosted portal; a live wallet-balance read is a follow-up once the - // reconcile worker is in place. let row = sqlx::query( - "SELECT tier, credits_balance, monthly_spend_limit, \ - hyperline_customer_id, hyperline_wallet_id, payment_method_status \ + "SELECT tier, stripe_customer_id, credits_balance, \ + auto_topup_enabled, auto_topup_amount, auto_topup_threshold, monthly_spend_limit \ FROM accounts WHERE id = $1", ) .bind(account_id) @@ -53,121 +49,17 @@ pub(crate) async fn get_billing( })? .ok_or_else(|| err(StatusCode::NOT_FOUND, "Account not found", "not_found"))?; - let hyperline_customer_id: Option = row.get("hyperline_customer_id"); - let hyperline_wallet_id: Option = row.get("hyperline_wallet_id"); - - // Best-effort live wallet read. Failures (no Hyperline client, not - // yet linked, network error, 404) all fall through to None — the - // local `credits_balance` is still authoritative for the ledger. - let (hyperline_wallet_balance, hyperline_wallet_currency) = match ( - state.hyperline_client.as_ref(), - hyperline_wallet_id.as_deref(), - ) { - (Some(client), Some(wallet_id)) => match client.get_wallet(wallet_id).await { - Ok(wallet) => (Some(wallet.balance.amount), wallet.currency), - Err(e) => { - info!( - account_id = %account_id, - wallet_id = %wallet_id, - error = %e, - "hyperline live wallet read failed — returning local balance only" - ); - (None, None) - } - }, - _ => (None, None), - }; - Ok(Json(BillingResponse { tier: row.get("tier"), + stripe_customer_id: row.get("stripe_customer_id"), credits_balance: row.get("credits_balance"), + auto_topup_enabled: row.get("auto_topup_enabled"), + auto_topup_amount: row.get("auto_topup_amount"), + auto_topup_threshold: row.get("auto_topup_threshold"), monthly_spend_limit: row.get("monthly_spend_limit"), - hyperline_customer_id, - hyperline_wallet_id, - payment_method_status: row.get("payment_method_status"), - hyperline_wallet_balance, - hyperline_wallet_currency, })) } -/// `GET /account/billing/portal` — exchange for a Hyperline hosted-portal URL. -/// -/// Replaces the old Stripe SetupIntent + 3DS payment-method UI. Users -/// manage cards, invoices, and auto-recharge from the hosted portal; -/// we just fetch the per-customer URL and redirect. -/// -/// Returns 404 if the account hasn't been linked to Hyperline yet -/// (expected during the customer-backfill window), 503 if the -/// Hyperline REST client isn't configured, or 502 on an upstream -/// failure. -#[utoipa::path( - get, - path = "/account/billing/portal", - tag = "auth", - responses( - (status = 200, description = "Hosted-portal URL", body = PortalResponse), - (status = 404, description = "Account not linked to Hyperline", body = ErrorBody), - (status = 502, description = "Hyperline upstream error", body = ErrorBody), - (status = 503, description = "Hyperline client not configured", body = ErrorBody), - ), - security(("api_key" = [])) -)] -pub(crate) async fn get_billing_portal( - State(state): State>, - Extension(user): Extension, -) -> Result, ApiError> { - let account_id = get_user_account_id(&state.pool, user.user_id, user.selected_account_id) - .await - .map_err(|_| err(StatusCode::NOT_FOUND, "Account not found", "not_found"))?; - - let Some(client) = state.hyperline_client.as_ref() else { - return Err(err( - StatusCode::SERVICE_UNAVAILABLE, - "Hyperline client not configured", - "hyperline_disabled", - )); - }; - - let customer_id: Option = - sqlx::query_scalar("SELECT hyperline_customer_id FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_optional(&state.pool) - .await - .map_err(|_| { - err( - StatusCode::INTERNAL_SERVER_ERROR, - "Database error", - "internal_error", - ) - })? - .flatten(); - - let Some(customer_id) = customer_id else { - return Err(err( - StatusCode::NOT_FOUND, - "Account is not linked to Hyperline yet", - "not_linked", - )); - }; - - match client.get_portal_url(&customer_id).await { - Ok(link) => Ok(Json(PortalResponse { url: link.url })), - Err(e) => { - error!( - account_id = %account_id, - customer_id = %customer_id, - error = %e, - "hyperline portal URL fetch failed" - ); - Err(err( - StatusCode::BAD_GATEWAY, - "Hyperline portal URL unavailable", - "hyperline_upstream", - )) - } - } -} - #[utoipa::path( patch, path = "/account/billing", @@ -323,24 +215,63 @@ pub(crate) async fn topup_credits( tag = "auth", request_body = AutoTopupRequest, responses( - (status = 410, description = "Moved to Hyperline hosted portal", body = ErrorBody), + (status = 200, description = "Auto top-up settings updated", body = MessageResponse), + (status = 400, description = "Validation error", body = ErrorBody), + (status = 404, description = "Account not found", body = ErrorBody), ), security(("api_key" = [])) )] -/// Auto top-up moved to Hyperline wallet rules — this endpoint is a stub -/// that returns 410 Gone so the frontend knows to redirect to the hosted -/// portal instead of POSTing here. pub(crate) async fn update_auto_topup( - State(_state): State>, - Extension(_user): Extension, - Json(_req): Json, + State(state): State>, + Extension(user): Extension, + Json(req): Json, ) -> Result, ApiError> { - Err(err( - StatusCode::GONE, - "Auto top-up is now configured on the Hyperline hosted portal. \ - Use GET /account/billing/portal to obtain a portal session.", - "moved_to_hyperline", - )) + let account_id = get_user_account_id(&state.pool, user.user_id, user.selected_account_id) + .await + .map_err(|_| err(StatusCode::NOT_FOUND, "Account not found", "not_found"))?; + + if req.enabled { + let amount = req.amount.unwrap_or(5000); + let threshold = req.threshold.unwrap_or(500); + if amount <= 0 || threshold < 0 { + return Err(err( + StatusCode::BAD_REQUEST, + "Amount must be positive and threshold non-negative", + "validation_error", + )); + } + sqlx::query( + "UPDATE accounts SET auto_topup_enabled = true, auto_topup_amount = $1, auto_topup_threshold = $2 WHERE id = $3", + ) + .bind(amount) + .bind(threshold) + .bind(account_id) + .execute(&state.pool) + .await + .map_err(|_| { + err(StatusCode::INTERNAL_SERVER_ERROR, "Failed to update", "internal_error") + })?; + } else { + sqlx::query("UPDATE accounts SET auto_topup_enabled = false WHERE id = $1") + .bind(account_id) + .execute(&state.pool) + .await + .map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to update", + "internal_error", + ) + })?; + } + + Ok(Json(MessageResponse { + message: if req.enabled { + "Auto top-up enabled".to_string() + } else { + "Auto top-up disabled".to_string() + }, + })) } #[utoipa::path( diff --git a/bins/scrapix-api/src/auth/handlers/mod.rs b/bins/scrapix-api/src/auth/handlers/mod.rs index 8d321978..bc2906de 100644 --- a/bins/scrapix-api/src/auth/handlers/mod.rs +++ b/bins/scrapix-api/src/auth/handlers/mod.rs @@ -90,35 +90,12 @@ pub(crate) struct CreatedApiKeyResponse { #[derive(Serialize, utoipa::ToSchema)] pub(crate) struct BillingResponse { pub(super) tier: String, + pub(super) stripe_customer_id: Option, pub(super) credits_balance: i64, + pub(super) auto_topup_enabled: bool, + pub(super) auto_topup_amount: i64, + pub(super) auto_topup_threshold: i64, pub(super) monthly_spend_limit: Option, - /// Hyperline customer handle (null until the account has been linked - /// via the post-signup provisioning script). - pub(super) hyperline_customer_id: Option, - /// Hyperline wallet handle (null until provisioned). - pub(super) hyperline_wallet_id: Option, - /// Payment-method issue flag: `"errored"` or `"expired"`, or null - /// when the card is healthy. Set by the Hyperline payment_method.* - /// webhooks; cleared on the next invoice.settled. - pub(super) payment_method_status: Option, - /// Live Hyperline wallet balance in the smallest currency unit - /// (cents for USD). `None` when either the account isn't linked to - /// Hyperline yet, the REST client is disabled, or the live read - /// failed — in which case the UI should fall back to - /// `credits_balance`. See `hyperline_wallet_currency` for the unit. - pub(super) hyperline_wallet_balance: Option, - /// ISO 4217 code (e.g. `"USD"`) paired with - /// `hyperline_wallet_balance`. `None` when the balance is. - pub(super) hyperline_wallet_currency: Option, - // NOTE: `auto_topup_*` and `stripe_customer_id` fields were removed - // during the Hyperline migration — auto-recharge is now configured on - // the Hyperline wallet and surfaced through the hosted billing portal. -} - -/// Response from `GET /account/billing/portal` — hosted-portal deep link. -#[derive(Serialize, utoipa::ToSchema)] -pub(crate) struct PortalResponse { - pub(super) url: String, } #[derive(Deserialize, utoipa::ToSchema)] @@ -131,11 +108,7 @@ pub struct TopupRequest { pub(super) amount: i64, } -/// Retained for OpenAPI back-compat — the endpoint now returns 410 Gone -/// and redirects to the Hyperline hosted portal, but we still accept the -/// body so old frontend builds don't crash on parsing the spec. #[derive(Deserialize, utoipa::ToSchema)] -#[allow(dead_code)] pub struct AutoTopupRequest { pub(super) enabled: bool, pub(super) amount: Option, @@ -370,12 +343,12 @@ pub(crate) use auth::{ forgot_password, login, logout, resend_verification, reset_password, signup, verify_email, }; pub(crate) use billing::{ - __path_get_billing, __path_get_billing_portal, __path_list_transactions, __path_topup_credits, - __path_update_auto_topup, __path_update_billing, __path_update_spend_limit, + __path_get_billing, __path_list_transactions, __path_topup_credits, __path_update_auto_topup, + __path_update_billing, __path_update_spend_limit, }; pub(crate) use billing::{ - get_billing, get_billing_portal, list_transactions, topup_credits, update_auto_topup, - update_billing, update_spend_limit, + get_billing, list_transactions, topup_credits, update_auto_topup, update_billing, + update_spend_limit, }; pub(crate) use team::{ accept_invite, invite_member, list_invites, list_members, remove_member, revoke_invite, @@ -424,7 +397,6 @@ pub fn session_routes(state: Arc) -> Router { .route("/account/billing/auto-topup", patch(update_auto_topup)) .route("/account/billing/spend-limit", patch(update_spend_limit)) .route("/account/billing/transactions", get(list_transactions)) - .route("/account/billing/portal", get(get_billing_portal)) .layer(middleware::from_fn_with_state( state.clone(), super::validate_session, diff --git a/bins/scrapix-api/src/auth/mod.rs b/bins/scrapix-api/src/auth/mod.rs index aeeb6528..92de3aec 100644 --- a/bins/scrapix-api/src/auth/mod.rs +++ b/bins/scrapix-api/src/auth/mod.rs @@ -22,7 +22,6 @@ pub use social::{ social_auth_routes, OAuthStateStore, ProviderConfig, SocialAuthState, SocialOAuthConfig, }; -use scrapix_billing_hyperline::HyperlineClient; use sqlx::{postgres::PgPoolOptions, PgPool}; use crate::email::EmailClient; @@ -33,12 +32,6 @@ pub struct AuthState { pub pool: PgPool, pub jwt_secret: String, pub email_client: Option, - /// Hyperline REST client, set when HYPERLINE_API_KEY is configured. - /// Used by billing handlers to fetch the live wallet balance and - /// the hosted-portal URL. `None` in dev/self-hosted environments — - /// the handlers degrade gracefully (local balance only, portal - /// endpoint returns 503). - pub hyperline_client: Option, } impl AuthState { @@ -59,7 +52,6 @@ impl AuthState { pool, jwt_secret, email_client: None, - hyperline_client: None, }) } } diff --git a/bins/scrapix-api/src/billing.rs b/bins/scrapix-api/src/billing.rs index 61ac6fd9..54d52e80 100644 --- a/bins/scrapix-api/src/billing.rs +++ b/bins/scrapix-api/src/billing.rs @@ -6,9 +6,7 @@ use crate::email::EmailClient; use crate::{ApiError, ScrapeFormat}; -use scrapix_billing_hyperline::events::{enqueue_usage_event, BillingEventType}; use scrapix_core::{CrawlerType, FeaturesConfig}; -use tracing::warn; // Re-export constants from the billing crate. pub use scrapix_billing::{MAP_CREDITS, SEARCH_CREDITS}; @@ -28,10 +26,23 @@ impl From for ApiError { // Trait implementations for scrapix-billing // ============================================================================ -// NOTE: There is intentionally no `PaymentProvider` implementation here. After -// the Hyperline migration, auto-recharge is handled by Hyperline wallet -// rules — we no longer charge payment methods from our own process. Callers -// of `check_credits_and_deduct` pass `None` for the provider. +/// Implements [`scrapix_billing::PaymentProvider`] by delegating to the Stripe +/// `charge_auto_topup` function in `crate::stripe`. +pub(crate) struct StripePaymentProvider<'a> { + pub client: &'a stripe::Client, +} + +#[async_trait::async_trait] +impl scrapix_billing::PaymentProvider for StripePaymentProvider<'_> { + async fn charge_auto_topup( + &self, + pool: &sqlx::PgPool, + account_id: uuid::Uuid, + credits: i64, + ) -> Result { + crate::stripe::charge_auto_topup(self.client, pool, account_id, credits).await + } +} /// Implements [`scrapix_billing::BillingNotifier`] by sending emails via the /// Resend-backed `EmailClient`. @@ -98,64 +109,28 @@ pub(crate) async fn check_credits_and_deduct( operation: &str, description: &str, email_client: Option<&EmailClient>, + stripe_client: Option<&stripe::Client>, ) -> Result { let notifier = email_client.map(|ec| EmailBillingNotifier { email_client: ec }); let notifier_ref: Option<&dyn scrapix_billing::BillingNotifier> = notifier .as_ref() .map(|n| n as &dyn scrapix_billing::BillingNotifier); - // Post-Hyperline-migration: no local payment provider. If the local - // ledger runs out, Hyperline wallet auto-recharge is what tops it back - // up (via a `wallet.credited` webhook → ledger credit). - let new_balance = scrapix_billing::auto_topup::check_credits_and_deduct( + let provider = stripe_client.map(|client| StripePaymentProvider { client }); + let provider_ref: Option<&dyn scrapix_billing::PaymentProvider> = provider + .as_ref() + .map(|p| p as &dyn scrapix_billing::PaymentProvider); + + Ok(scrapix_billing::auto_topup::check_credits_and_deduct( pool, account_id, amount, operation, description, - None, + provider_ref, notifier_ref, ) - .await?; - - // Mirror the local debit into the Hyperline outbox. Best-effort: if - // enqueue fails we log but don't unwind the debit — the drift-detection - // job (scrapix_billing_hyperline::reconcile) is the backstop. Full - // transactional atomicity requires refactoring the ledger to accept a - // shared `&mut Transaction`; tracked for Phase 1. - emit_usage_event(pool, account_id, operation, amount, description).await; - - Ok(new_balance) -} - -/// Enqueue a single `ApiRequest` outbox row per debit site. The `operation` -/// ("scrape" / "map" / "crawl" / "search") and description are carried as -/// metadata so Hyperline can slice pricing / analytics by endpoint. -pub(crate) async fn emit_usage_event( - pool: &sqlx::PgPool, - account_id: &str, - operation: &str, - credits: i64, - description: &str, -) { - let Ok(account_uuid) = uuid::Uuid::parse_str(account_id) else { - return; - }; - let metadata = serde_json::json!({ - "operation": operation, - "description": description, - }); - if let Err(e) = enqueue_usage_event( - pool, - account_uuid, - BillingEventType::ApiRequest, - credits as f64, - Some(metadata), - ) - .await - { - warn!(account_id, operation, error = %e, "failed to enqueue Hyperline outbox row"); - } + .await?) } // ============================================================================ diff --git a/bins/scrapix-api/src/hyperline.rs b/bins/scrapix-api/src/hyperline.rs deleted file mode 100644 index f74e333a..00000000 --- a/bins/scrapix-api/src/hyperline.rs +++ /dev/null @@ -1,632 +0,0 @@ -//! Hyperline webhook receiver. -//! -//! The request-path side of the migration — signed incoming events from -//! Hyperline (wallet credited/debited, invoice settled/errored, etc.) land -//! here. Verification and payload parsing live in -//! `scrapix_billing_hyperline::webhooks`; this module is responsible for the -//! axum handler, replay dedup via `hyperline_webhook_log`, and dispatch to -//! the local ledger / email pipeline. -//! -//! The event-dispatch match is intentionally minimal for the first cut — -//! every known event type is logged, and a focused follow-up PR will wire -//! `wallet.credited` into `scrapix_billing::ledger` once the ledger's -//! idempotency key is widened beyond `stripe_payment_intent_id`. - -use axum::{ - body::Bytes, - extract::Extension, - http::{HeaderMap, StatusCode}, - routing::post, - Router, -}; -use scrapix_billing_hyperline::webhooks::{verify_signature, WebhookEnvelope, WebhookHeaders}; -use tracing::{info, warn}; - -use crate::email::{get_account_email, EmailClient}; - -// ============================================================================ -// State -// ============================================================================ - -/// Shared state for the Hyperline webhook route. -/// -/// We keep this narrow — no full `HyperlineClient`, since webhook handling -/// doesn't call the control-plane API. The secret is stored here rather -/// than re-read from env on each request. The email client is optional -/// because the webhook route should still mount even when Resend is -/// unconfigured (dev / self-hosted) — low-balance emails just get -/// skipped with a warning. -#[derive(Clone)] -pub struct HyperlineWebhookState { - pub webhook_secret: String, - pub email_client: Option, -} - -impl HyperlineWebhookState { - pub fn new(webhook_secret: String, email_client: Option) -> Self { - Self { - webhook_secret, - email_client, - } - } -} - -// ============================================================================ -// Handler -// ============================================================================ - -/// `POST /webhooks/hyperline` — receive a signed Hyperline event. -/// -/// No auth header required: Hyperline signs the body with HMAC-SHA256 over -/// `id.timestamp.body`, and we reject anything that doesn't verify. A 5-min -/// timestamp-skew tolerance is enforced by [`verify_signature`]. -/// -/// Replay protection: `hyperline_webhook_log` has `webhook_id` as primary -/// key and we `INSERT … ON CONFLICT DO NOTHING`. If the insert affected -/// zero rows, we've seen this delivery before — return 200 without -/// dispatching so Hyperline stops retrying. -async fn hyperline_webhook( - Extension(state): Extension, - Extension(pool): Extension, - headers: HeaderMap, - body: Bytes, -) -> Result { - // 1. Pull the three Svix-style headers. - let id = header_str(&headers, "webhook-id")?; - let timestamp = header_str(&headers, "webhook-timestamp")?; - let signature = header_str(&headers, "webhook-signature")?; - - // 2. Verify HMAC + timestamp skew. - let now = chrono::Utc::now().timestamp(); - verify_signature( - &state.webhook_secret, - WebhookHeaders { - id, - timestamp, - signature, - }, - &body, - now, - ) - .map_err(|e| { - warn!(webhook_id = %id, error = %e, "hyperline webhook verification failed"); - (StatusCode::BAD_REQUEST, e.to_string()) - })?; - - // 3. Parse the envelope before we persist. We still write to the log - // even on parse failure so ops can inspect the body, but we return - // 400 so Hyperline retries (in case this was transient malformed). - // - // Parse the body once as `Value` and reuse it for both the typed - // envelope and the log row — saves a second `from_slice` pass on - // the hot path and avoids the silent `unwrap_or_default()` masking - // a parse divergence. - let body_json: serde_json::Value = match serde_json::from_slice(&body) { - Ok(v) => v, - Err(e) => { - warn!(webhook_id = %id, error = %e, "hyperline webhook body is not valid JSON"); - log_raw(&pool, id, &body, Some(&format!("parse error: {e}"))).await; - return Err((StatusCode::BAD_REQUEST, format!("invalid body: {e}"))); - } - }; - let envelope: WebhookEnvelope = match serde_json::from_value(body_json.clone()) { - Ok(e) => e, - Err(e) => { - warn!(webhook_id = %id, error = %e, "hyperline webhook body is not a valid envelope"); - log_raw(&pool, id, &body, Some(&format!("envelope error: {e}"))).await; - return Err((StatusCode::BAD_REQUEST, format!("invalid envelope: {e}"))); - } - }; - - // 4. Dedupe insert. `INSERT … ON CONFLICT DO NOTHING` returns 0 rows - // when we've already processed this `webhook_id`. - let inserted = sqlx::query( - "INSERT INTO hyperline_webhook_log (webhook_id, event_type, body) \ - VALUES ($1, $2, $3) \ - ON CONFLICT (webhook_id) DO NOTHING", - ) - .bind(id) - .bind(&envelope.event_type) - .bind(&body_json) - .execute(&pool) - .await - .map_err(|e| { - warn!(webhook_id = %id, error = %e, "failed to insert hyperline_webhook_log row"); - (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) - })?; - - if inserted.rows_affected() == 0 { - info!( - webhook_id = %id, - event_type = %envelope.event_type, - "hyperline webhook already processed — skipping dispatch" - ); - return Ok(StatusCode::OK); - } - - // 5. Dispatch. Unknown events get logged and acked to stop retries. - let dispatch_result = dispatch_event(&pool, &state, &envelope).await; - - // 6. Mark processed (success or error) on the log row so ops can tell - // "we saw it and tried" from "we saw it and crashed". - let (processed_at_set, err_message) = match &dispatch_result { - Ok(_) => (true, None), - Err(e) => (false, Some(e.clone())), - }; - if let Err(e) = sqlx::query( - "UPDATE hyperline_webhook_log \ - SET processed_at = CASE WHEN $2 THEN now() ELSE processed_at END, \ - process_error = $3 \ - WHERE webhook_id = $1", - ) - .bind(id) - .bind(processed_at_set) - .bind(err_message.as_deref()) - .execute(&pool) - .await - { - warn!(webhook_id = %id, error = %e, "failed to update hyperline_webhook_log after dispatch"); - } - - // Always return 200 for a verified delivery — Hyperline retries on - // non-2xx, and retrying won't fix a bug in our dispatch code. - Ok(StatusCode::OK) -} - -async fn dispatch_event( - pool: &sqlx::PgPool, - state: &HyperlineWebhookState, - envelope: &WebhookEnvelope, -) -> Result<(), String> { - match envelope.event_type.as_str() { - "wallet.credited" | "credit.topup_transaction_created" => { - // Resolve account + idempotency key from the envelope. Values - // are extracted defensively — if anything is missing we log a - // structured warning, ack the delivery (Hyperline retries are - // pointless for schema issues), and return Ok. - let id = envelope - .data - .get("object") - .and_then(|o| o.get("id")) - .and_then(|v| v.as_str()); - let customer_id = envelope - .data - .get("object") - .and_then(|o| o.get("customer_id")) - .and_then(|v| v.as_str()); - // Credits unit is read from a custom `credits` property we - // plan to ship on the Hyperline product-aggregator side. Real - // amount→credits conversion (currency-aware) ships with the - // reconcile worker. - let credits = envelope - .data - .get("object") - .and_then(|o| o.get("credits")) - .and_then(|v| v.as_i64()); - - match (id, customer_id, credits) { - (Some(event_id), Some(cust), Some(credits_amt)) if credits_amt > 0 => { - let maybe_account: Option = sqlx::query_scalar( - "SELECT id FROM accounts WHERE hyperline_customer_id = $1", - ) - .bind(cust) - .fetch_optional(pool) - .await - .map_err(|e| format!("account lookup: {e}"))?; - - let Some(account_id) = maybe_account else { - warn!( - event_type = %envelope.event_type, - customer_id = %cust, - event_id = %event_id, - "hyperline: wallet.credited for unknown customer — acking" - ); - return Ok(()); - }; - - scrapix_billing::add_credits_from_provider( - pool, - account_id, - credits_amt, - "hyperline", - event_id, - "wallet_credit", - "Hyperline wallet credit", - ) - .await - .map_err(|e| format!("ledger credit failed: {e}"))?; - - info!( - account_id = %account_id, - event_id = %event_id, - credits = credits_amt, - "hyperline: wallet credited → ledger" - ); - } - _ => { - // Failure mode worth surfacing loudly: a real top-up - // landed at Hyperline (so the customer paid), but our - // event payload is missing the fields we need to - // mirror it into the local ledger. Returning Err - // stamps `process_error` on the webhook log row so - // the runbook's stuck-deliveries query catches it - // (vs. a warn-and-ack that disappears into log - // volume). Hyperline still gets a 200 because the - // outer handler always acks verified deliveries — - // retries can't fix a schema mismatch. - let problem = format!( - "wallet.credited missing fields (has_id={}, has_customer={}, has_credits={}) — \ - local credit not applied; backfill via reconcile or manual ledger adjustment", - id.is_some(), - customer_id.is_some(), - credits.is_some(), - ); - warn!(event_type = %envelope.event_type, %problem, "hyperline: wallet.credited unmappable"); - return Err(problem); - } - } - } - "wallet.debited" => { - // Informational only — we debit locally at request time; this - // is Hyperline's post-hoc notification. A drift check against - // the ledger lives in `scrapix_billing_hyperline::reconcile`. - info!(event_type = %envelope.event_type, "hyperline: wallet debited (mirror of local debit)"); - } - "wallet.low_projected_balance" => { - // Resolve (account, projected_balance) from the envelope and - // send a low-balance warning to the account owner. Same - // defensive extraction as wallet.credited. - let customer_id = envelope - .data - .get("object") - .and_then(|o| o.get("customer_id")) - .and_then(|v| v.as_str()); - // `projected_balance` is Hyperline's post-window projection. - // We accept either int or float; floats are rounded down. - let projected_balance = envelope - .data - .get("object") - .and_then(|o| o.get("projected_balance")) - .and_then(|v| v.as_i64().or_else(|| v.as_f64().map(|f| f as i64))); - - match (state.email_client.as_ref(), customer_id, projected_balance) { - (Some(mailer), Some(cust), Some(balance)) => { - let maybe_account: Option = sqlx::query_scalar( - "SELECT id FROM accounts WHERE hyperline_customer_id = $1", - ) - .bind(cust) - .fetch_optional(pool) - .await - .map_err(|e| format!("account lookup: {e}"))?; - - let Some(account_id) = maybe_account else { - warn!( - event_type = %envelope.event_type, - customer_id = %cust, - "hyperline: low balance for unknown customer — acking" - ); - return Ok(()); - }; - - if let Some(email) = get_account_email(pool, account_id).await { - mailer.send_low_balance_warning(&email, balance); - info!( - account_id = %account_id, - projected_balance = balance, - "hyperline: low balance email dispatched" - ); - } else { - warn!( - account_id = %account_id, - "hyperline: low balance — no owner email on file" - ); - } - } - _ => { - warn!( - event_type = %envelope.event_type, - has_email_client = state.email_client.is_some(), - has_customer = customer_id.is_some(), - has_balance = projected_balance.is_some(), - "hyperline: low balance missing fields — acking without email" - ); - } - } - } - "invoice.settled" | "invoice.errored" => { - // Record the invoice event as an informational row in the - // transactions ledger with amount = 0 — the actual credit - // change rides in a separate wallet.credited event. This - // gives the console a unified activity feed ("paid invoice - // #abc for $X" next to "+1000 credits"). - let event_id = envelope - .data - .get("object") - .and_then(|o| o.get("id")) - .and_then(|v| v.as_str()); - let customer_id = envelope - .data - .get("object") - .and_then(|o| o.get("customer_id")) - .and_then(|v| v.as_str()); - // `amount` is in the processor's minor unit (cents). Stored - // in `metadata.processor_amount` so the UI can render "$X" - // without touching `transactions.amount` (which is a credit - // delta, not a currency amount). - let processor_amount = envelope - .data - .get("object") - .and_then(|o| o.get("amount")) - .and_then(|v| v.as_i64()); - let currency = envelope - .data - .get("object") - .and_then(|o| o.get("currency")) - .and_then(|v| v.as_str()); - - let (Some(event_id), Some(cust)) = (event_id, customer_id) else { - warn!( - event_type = %envelope.event_type, - has_id = event_id.is_some(), - has_customer = customer_id.is_some(), - "hyperline: invoice event missing id/customer — acking without write" - ); - return Ok(()); - }; - - // Idempotency: same (provider, event_id) means we already - // logged this invoice event. Short-circuit. - let exists: bool = sqlx::query_scalar( - "SELECT EXISTS( \ - SELECT 1 FROM transactions \ - WHERE metadata->>'provider' = $1 \ - AND metadata->>'provider_event_id' = $2 \ - )", - ) - .bind("hyperline") - .bind(event_id) - .fetch_one(pool) - .await - .map_err(|e| format!("invoice dedupe check: {e}"))?; - if exists { - info!( - event_type = %envelope.event_type, - event_id = %event_id, - "hyperline: invoice event already logged — skipping" - ); - return Ok(()); - } - - let maybe_account: Option<(uuid::Uuid, i64)> = sqlx::query_as( - "SELECT id, credits_balance FROM accounts WHERE hyperline_customer_id = $1", - ) - .bind(cust) - .fetch_optional(pool) - .await - .map_err(|e| format!("account lookup: {e}"))?; - - let Some((account_id, balance_after)) = maybe_account else { - warn!( - event_type = %envelope.event_type, - customer_id = %cust, - "hyperline: invoice event for unknown customer — acking" - ); - return Ok(()); - }; - - let tx_type = if envelope.event_type == "invoice.settled" { - "invoice_settled" - } else { - "invoice_errored" - }; - let description = format!("Hyperline invoice {}", envelope.event_type); - let metadata = serde_json::json!({ - "provider": "hyperline", - "provider_event_id": event_id, - "processor_amount": processor_amount, - "currency": currency, - }); - - // Race-condition backstop: the EXISTS check above is not atomic - // with this INSERT; two concurrent webhook deliveries with the - // same provider event id could both pass it. The partial unique - // index `uniq_transactions_provider_event` raises 23505 in that - // case — we treat it as "already logged" and return Ok rather - // than 500ing. - let insert_result = sqlx::query( - "INSERT INTO transactions (account_id, type, amount, balance_after, description, metadata) \ - VALUES ($1, $2, 0, $3, $4, $5)", - ) - .bind(account_id) - .bind(tx_type) - .bind(balance_after) - .bind(&description) - .bind(metadata) - .execute(pool) - .await; - if let Err(sqlx::Error::Database(db_err)) = &insert_result { - if db_err.code().as_deref() == Some("23505") { - info!( - event_type = %envelope.event_type, - event_id = %event_id, - "hyperline: invoice event raced — duplicate detected by unique index" - ); - return Ok(()); - } - } - insert_result.map_err(|e| format!("insert invoice transaction: {e}"))?; - - // Successful invoice clears any prior payment_method flag — - // whatever went wrong, it's resolved now that money moved. - if envelope.event_type == "invoice.settled" { - if let Err(e) = sqlx::query( - "UPDATE accounts SET payment_method_status = NULL \ - WHERE id = $1 AND payment_method_status IS NOT NULL", - ) - .bind(account_id) - .execute(pool) - .await - { - // Non-fatal: the transaction row already landed, - // so we ack the webhook. Flag clears on the next - // successful invoice or a manual operator action. - warn!( - account_id = %account_id, - error = %e, - "hyperline: failed to clear payment_method_status on invoice.settled" - ); - } - } - - info!( - account_id = %account_id, - event_id = %event_id, - tx_type, - "hyperline: invoice event recorded in transactions" - ); - } - "payment_method.errored" | "payment_method.expired" => { - // Flag the account so the console can show a "Your card - // needs attention" banner. The actual remediation (update - // card, retry payment) happens on the Hyperline hosted - // portal — we're just surfacing the state. Clearing the - // flag is an operator action or a successful invoice.settled - // (see the invoice.settled branch for that side effect). - let customer_id = envelope - .data - .get("object") - .and_then(|o| o.get("customer_id")) - .and_then(|v| v.as_str()); - - let Some(cust) = customer_id else { - warn!( - event_type = %envelope.event_type, - "hyperline: payment_method event missing customer_id — acking without action" - ); - return Ok(()); - }; - - let status = if envelope.event_type == "payment_method.errored" { - "errored" - } else { - "expired" - }; - - let affected = sqlx::query( - "UPDATE accounts SET payment_method_status = $1 \ - WHERE hyperline_customer_id = $2", - ) - .bind(status) - .bind(cust) - .execute(pool) - .await - .map_err(|e| format!("flag payment method: {e}"))?; - - if affected.rows_affected() == 0 { - warn!( - event_type = %envelope.event_type, - customer_id = %cust, - "hyperline: payment_method event for unknown customer" - ); - } else { - info!( - customer_id = %cust, - status, - "hyperline: account payment_method_status flagged" - ); - } - } - "subscription.cancelled" => { - // Hard lock: flip `accounts.active = false`. Both the credit - // ledger (`check_credits`) and the Postgres - // `lookup_api_key_account` function gate on `active = true`, so - // this simultaneously blocks new requests and rejects API key - // auth. Re-activation is a manual operator action — we never - // flip back to true from a webhook. - let customer_id = envelope - .data - .get("object") - .and_then(|o| o.get("customer_id")) - .and_then(|v| v.as_str()); - - let Some(cust) = customer_id else { - warn!( - event_type = %envelope.event_type, - "hyperline: subscription.cancelled missing customer_id — acking without action" - ); - return Ok(()); - }; - - let affected = sqlx::query( - "UPDATE accounts SET active = false \ - WHERE hyperline_customer_id = $1 AND active = true", - ) - .bind(cust) - .execute(pool) - .await - .map_err(|e| format!("deactivate account: {e}"))?; - - if affected.rows_affected() == 0 { - warn!( - event_type = %envelope.event_type, - customer_id = %cust, - "hyperline: subscription.cancelled for unknown or already-inactive customer" - ); - } else { - info!( - customer_id = %cust, - "hyperline: account deactivated on subscription.cancelled" - ); - } - } - other => { - info!(event_type = %other, "hyperline: unknown event type — acked and logged"); - } - } - Ok(()) -} - -async fn log_raw(pool: &sqlx::PgPool, id: &str, body: &[u8], err: Option<&str>) { - let body_json: serde_json::Value = serde_json::from_slice(body).unwrap_or_default(); - if let Err(e) = sqlx::query( - "INSERT INTO hyperline_webhook_log (webhook_id, event_type, body, process_error) \ - VALUES ($1, 'unknown', $2, $3) \ - ON CONFLICT (webhook_id) DO NOTHING", - ) - .bind(id) - .bind(body_json) - .bind(err) - .execute(pool) - .await - { - warn!(webhook_id = %id, error = %e, "failed to log unparseable hyperline webhook"); - } -} - -fn header_str<'a>(headers: &'a HeaderMap, name: &str) -> Result<&'a str, (StatusCode, String)> { - headers - .get(name) - .and_then(|v| v.to_str().ok()) - .ok_or_else(|| (StatusCode::BAD_REQUEST, format!("missing header: {name}"))) -} - -// ============================================================================ -// Route -// ============================================================================ - -/// Build the `/webhooks/hyperline` router. -/// -/// No auth middleware — verification is per-request against the HMAC secret -/// in [`HyperlineWebhookState`]. Wire into the main app in `lib.rs` via -/// `app = app.merge(hyperline::webhook_route(pool, secret))`. -pub fn webhook_route( - pool: sqlx::PgPool, - webhook_secret: String, - email_client: Option, -) -> Router { - let state = HyperlineWebhookState::new(webhook_secret, email_client); - Router::new() - .route("/webhooks/hyperline", post(hyperline_webhook)) - .layer(Extension(state)) - .layer(Extension(pool)) -} diff --git a/bins/scrapix-api/src/lib.rs b/bins/scrapix-api/src/lib.rs index a8fc7e4c..34753204 100644 --- a/bins/scrapix-api/src/lib.rs +++ b/bins/scrapix-api/src/lib.rs @@ -47,14 +47,11 @@ pub mod configs; pub mod email; pub mod email_scheduler; pub mod engines; -pub mod hyperline; pub mod jobs_db; pub mod mcp; pub mod openapi; -// NOTE: `mod stripe;` was removed during the Hyperline migration. All -// payment-processor plumbing now lives in `scrapix_billing_hyperline` and -// the thin webhook handler in `hyperline.rs`. pub mod rate_limit; +pub mod stripe; use axum::{ extract::{ @@ -134,6 +131,14 @@ pub struct Args { #[arg(long, env = "REDIS_URL")] pub redis_url: Option, + /// Stripe secret key (enables payment processing) + #[arg(long, env = "STRIPE_SECRET_KEY")] + pub stripe_secret_key: Option, + + /// Stripe webhook signing secret (for verifying webhook events) + #[arg(long, env = "STRIPE_WEBHOOK_SECRET")] + pub stripe_webhook_secret: Option, + /// Resend API key for transactional emails (optional, disables emails if not set) #[arg(long, env = "RESEND_API_KEY")] pub resend_api_key: Option, @@ -224,6 +229,8 @@ struct AppState { db_pool: Option, /// Optional email client for transactional emails email_client: Option, + /// Optional Stripe client for payment-backed auto-topup + stripe_client: Option<::stripe::Client>, /// Optional ClickHouse analytics store (used for event history queries) analytics_store: Option>, } @@ -247,6 +254,7 @@ impl AppState { ai_service: Option>, db_pool: Option, email_client: Option, + stripe_client: Option<::stripe::Client>, analytics_store: Option>, ) -> Self { let (event_tx, _) = broadcast::channel(10_000); @@ -275,6 +283,7 @@ impl AppState { ai_service, db_pool, email_client, + stripe_client, analytics_store, } } @@ -636,6 +645,7 @@ impl AppState { let pool = pool.clone(); let acct_id = acct_id.clone(); let job_id = job_id.to_string(); + let stripe_cl = self.stripe_client.clone(); tokio::spawn(async move { match billing::check_credits_and_deduct( &pool, @@ -647,6 +657,7 @@ impl AppState { job_id, total_pages, cost_per_page ), None, + stripe_cl.as_ref(), ) .await { @@ -2070,6 +2081,7 @@ async fn scrape_url( "scrape", &format!("{} ({} credits)", final_url, scrape_cost), None, + state.stripe_client.as_ref(), ) .await { @@ -3214,6 +3226,7 @@ async fn map_url( "map", &request.url, None, + state.stripe_client.as_ref(), ) .await { @@ -3443,6 +3456,7 @@ async fn search_url( "search", &format!("{} q={}", request.url, request.q), None, + state.stripe_client.as_ref(), ) .await { @@ -4503,26 +4517,6 @@ pub async fn run_with_bus( info!("Transactional emails disabled (RESEND_API_KEY not set)"); } - // Initialize Hyperline client if HYPERLINE_API_KEY is set. - // The billing handlers use this to fetch live wallet balance - // and generate hosted-portal URLs; absence degrades gracefully - // (handlers fall back to local ledger state only). - // - // Boot self-check pings Hyperline and logs the event-type - // manifest for ops cross-check. A failed ping is non-fatal: - // the outbox drain worker retries on its own cadence, so a - // transient Hyperline outage shouldn't crashloop the API. - match scrapix_billing_hyperline::HyperlineClient::from_env() { - Ok(client) => { - let _ = scrapix_billing_hyperline::boot_self_check(&client).await; - state.hyperline_client = Some(client); - info!("Hyperline REST client enabled"); - } - Err(e) => { - info!(error = %e, "Hyperline REST client disabled (HYPERLINE_API_KEY not set or invalid)"); - } - } - info!("Authentication enabled via PostgreSQL"); Some(Arc::new(state)) } @@ -4623,6 +4617,7 @@ pub async fn run_with_bus( }; let db_pool = auth_state.as_ref().map(|a| a.pool.clone()); let email_client = auth_state.as_ref().and_then(|a| a.email_client.clone()); + let stripe_client = args.stripe_secret_key.as_ref().map(::stripe::Client::new); let state = Arc::new(AppState::new( producer, config, @@ -4635,6 +4630,7 @@ pub async fn run_with_bus( ai_service, db_pool, email_client, + stripe_client, analytics_state.clone(), )); @@ -4659,78 +4655,6 @@ pub async fn run_with_bus( } } - // Spawn the Hyperline outbox drain worker when configured. Silent no-op - // otherwise — the enqueue side still writes outbox rows, so turning - // HYPERLINE_API_KEY on later replays any backlog on the next boot. - // - // The reconcile scan worker shares the same (pool, client) and spins - // up alongside — it periodically pairs local credits with live - // Hyperline balance for shadow-mode observability (SCR-68 Phase 1). - let (hyperline_drain_handle, hyperline_reconcile_handle) = match ( - state.db_pool.clone(), - scrapix_billing_hyperline::HyperlineClient::from_env(), - ) { - (Some(pool), Ok(client)) => { - info!( - sandbox = client.config().is_sandbox(), - "Hyperline outbox drain worker starting (interval=5s, batch=100)" - ); - let drain = scrapix_billing_hyperline::outbox::spawn_drain_worker( - pool.clone(), - client.clone(), - std::time::Duration::from_secs(5), - 100, - ); - // Reconcile interval is intentionally long: scan every 15 min. - // Hyperline is authoritative via webhooks on the millisecond - // scale; this worker is a drift backstop, not the primary - // sync path, so a tight loop would burn API quota for no gain. - // - // Drift alerting is opt-in: set both HYPERLINE_CENTS_PER_CREDIT - // and HYPERLINE_DRIFT_TOLERANCE_CENTS to enable WARN-level alerts - // on accounts whose paired balances diverge beyond tolerance. - // Without these the worker runs in pure-observation mode (INFO - // observations only). - let threshold = match ( - std::env::var("HYPERLINE_CENTS_PER_CREDIT") - .ok() - .and_then(|s| s.parse::().ok()), - std::env::var("HYPERLINE_DRIFT_TOLERANCE_CENTS") - .ok() - .and_then(|s| s.parse::().ok()), - ) { - (Some(cents_per_credit), Some(tolerance_cents)) - if cents_per_credit > 0 && tolerance_cents >= 0 => - { - Some(scrapix_billing_hyperline::reconcile::DriftThreshold { - cents_per_credit, - tolerance_cents, - }) - } - _ => None, - }; - info!( - drift_alerts = threshold.is_some(), - "Hyperline reconcile scan worker starting (interval=15m)" - ); - let reconcile = scrapix_billing_hyperline::reconcile::spawn_scan_worker( - pool, - client, - std::time::Duration::from_secs(15 * 60), - threshold, - ); - (Some(drain), Some(reconcile)) - } - (None, _) => { - info!("Hyperline workers disabled: no DB pool"); - (None, None) - } - (_, Err(e)) => { - info!(reason = %e, "Hyperline workers disabled: client not configured"); - (None, None) - } - }; - // Shutdown coordination let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); @@ -5228,17 +5152,21 @@ pub async fn run_with_bus( } info!("Auth routes enabled (/auth/signup, /auth/login, /auth/me, /account/*, /oauth/*)"); - // Hyperline webhook receiver. - // Enabled whenever HYPERLINE_WEBHOOK_SECRET is set — independent of - // the ingest-side `HYPERLINE_API_KEY`, since you can verify incoming - // deliveries in one env without egressing events from it. - if let Ok(secret) = std::env::var("HYPERLINE_WEBHOOK_SECRET") { - app = app.merge(hyperline::webhook_route( - auth.pool.clone(), - secret, - auth.email_client.clone(), - )); - info!("Hyperline webhook route enabled (/webhooks/hyperline)"); + // Stripe payment routes + if let Some(ref stripe_key) = args.stripe_secret_key { + let stripe_state = + stripe::StripeState::new(stripe_key, args.stripe_webhook_secret.clone()); + app = app + .merge(stripe::stripe_session_routes( + auth.clone(), + stripe_state.clone(), + )) + .merge(stripe::stripe_webhook_route( + auth.pool.clone(), + stripe_state, + auth.email_client.clone(), + )); + info!("Stripe routes enabled (/account/billing/setup-intent, /account/billing/payment-methods, /account/billing/purchase, /webhooks/stripe)"); } // MCP HTTP endpoint with Bearer token auth @@ -5429,12 +5357,6 @@ pub async fn run_with_bus( if let Some(handle) = ai_receiver_handle { handle.abort(); } - if let Some(handle) = hyperline_drain_handle { - handle.abort(); - } - if let Some(handle) = hyperline_reconcile_handle { - handle.abort(); - } info!("Shutdown complete"); Ok(()) diff --git a/bins/scrapix-api/src/openapi.rs b/bins/scrapix-api/src/openapi.rs index 8be8b79a..303d5bf7 100644 --- a/bins/scrapix-api/src/openapi.rs +++ b/bins/scrapix-api/src/openapi.rs @@ -76,7 +76,6 @@ use utoipa::OpenApi; crate::auth::handlers::create_api_key, crate::auth::handlers::revoke_api_key, crate::auth::handlers::get_billing, - crate::auth::handlers::get_billing_portal, crate::auth::handlers::update_billing, crate::auth::handlers::topup_credits, crate::auth::handlers::update_auto_topup, diff --git a/bins/scrapix-api/src/stripe.rs b/bins/scrapix-api/src/stripe.rs new file mode 100644 index 00000000..5a3d173c --- /dev/null +++ b/bins/scrapix-api/src/stripe.rs @@ -0,0 +1,1244 @@ +//! Stripe payment integration. +//! +//! Handles customer creation, payment methods, credit purchases via Invoices, +//! and webhook processing. All UI is custom — Stripe is used purely as a backend +//! payment engine. + +use axum::{ + body::Bytes, + extract::{Extension, Path, State}, + http::{HeaderMap, StatusCode}, + middleware, + routing::{delete, get, post}, + Json, Router, +}; +use serde::{Deserialize, Serialize}; +use sqlx::Row; +use std::sync::Arc; +use stripe::{ + Client as StripeClient, CreateCustomer, CreateInvoice, CreateInvoiceItem, CreateSetupIntent, + Currency, Customer, CustomerId, EventObject, EventType, Invoice, + InvoicePendingInvoiceItemsBehavior, InvoiceStatus, ListInvoices, ListPaymentMethods, + PaymentIntent, PaymentIntentStatus, PaymentMethod, PaymentMethodId, PaymentMethodTypeFilter, + SetupIntent, Webhook, +}; +use tracing::{error, info, warn}; + +use crate::auth::{AuthState, AuthenticatedUser}; +use crate::email::EmailClient; + +// ============================================================================ +// Volume-based tiered pricing +// ============================================================================ + +// Re-export pricing from the billing crate. +pub use scrapix_billing::calculate_price_cents; + +// ============================================================================ +// State +// ============================================================================ + +/// Shared Stripe state, injected into routes as an Extension. +#[derive(Clone)] +pub struct StripeState { + pub client: StripeClient, + pub webhook_secret: Option, +} + +impl StripeState { + pub fn new(secret_key: &str, webhook_secret: Option) -> Self { + Self { + client: StripeClient::new(secret_key), + webhook_secret, + } + } +} + +// ============================================================================ +// Request / Response types +// ============================================================================ + +#[derive(Serialize)] +pub(crate) struct SetupIntentResponse { + client_secret: String, +} + +#[derive(Serialize)] +pub(crate) struct PaymentMethodResponse { + id: String, + brand: Option, + last4: Option, + exp_month: Option, + exp_year: Option, + is_default: bool, +} + +#[derive(Deserialize)] +pub struct PurchaseCreditsRequest { + credits: i64, + payment_method_id: Option, +} + +#[derive(Serialize)] +pub(crate) struct PurchaseResponse { + status: String, + client_secret: Option, + credits: i64, + amount_cents: i64, + message: String, +} + +#[derive(Deserialize)] +pub struct SetDefaultPaymentMethodRequest { + payment_method_id: String, +} + +#[derive(Serialize)] +pub(crate) struct MessageResponse { + message: String, +} + +#[derive(Serialize)] +pub(crate) struct InvoiceResponse { + id: String, + number: Option, + amount_cents: i64, + credits: Option, + status: String, + description: Option, + created_at: String, + invoice_pdf: Option, + hosted_invoice_url: Option, +} + +#[derive(Serialize)] +pub(crate) struct PricingTier { + up_to: Option, + unit_price_cents: f64, + per_1k: f64, +} + +type ApiError = (StatusCode, Json); + +#[derive(Debug, Serialize)] +pub(crate) struct StripeErrorBody { + error: String, + code: String, +} + +fn err(status: StatusCode, msg: &str, code: &str) -> ApiError { + ( + status, + Json(StripeErrorBody { + error: msg.to_string(), + code: code.to_string(), + }), + ) +} + +// ============================================================================ +// Helpers +// ============================================================================ + +/// Get or create a Stripe customer for the given account. +async fn get_or_create_customer( + stripe: &StripeClient, + pool: &sqlx::PgPool, + account_id: uuid::Uuid, +) -> Result { + // Check if we already have a stripe_customer_id + let existing: Option = + sqlx::query_scalar("SELECT stripe_customer_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(pool) + .await + .map_err(|e| { + error!(error = %e, "Failed to query stripe_customer_id"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + if let Some(cid) = existing { + return cid.parse::().map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Invalid stripe customer ID in database", + "internal_error", + ) + }); + } + + // Fetch account name and email for the customer + let row = sqlx::query( + "SELECT a.name, u.email FROM accounts a \ + JOIN account_members m ON m.account_id = a.id \ + JOIN users u ON u.id = m.user_id \ + WHERE a.id = $1 LIMIT 1", + ) + .bind(account_id) + .fetch_optional(pool) + .await + .map_err(|e| { + error!(error = %e, "Failed to query account for Stripe customer"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .ok_or_else(|| err(StatusCode::NOT_FOUND, "Account not found", "not_found"))?; + + let name: String = row.get("name"); + let email: String = row.get("email"); + + // Create Stripe customer + let mut params = CreateCustomer::new(); + params.name = Some(&name); + params.email = Some(&email); + params.metadata = Some( + [("scrapix_account_id".to_string(), account_id.to_string())] + .into_iter() + .collect(), + ); + + let customer = Customer::create(stripe, params).await.map_err(|e| { + error!(error = %e, "Failed to create Stripe customer"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to create Stripe customer", + "stripe_error", + ) + })?; + + // Store the customer ID + sqlx::query("UPDATE accounts SET stripe_customer_id = $1 WHERE id = $2") + .bind(customer.id.as_str()) + .bind(account_id) + .execute(pool) + .await + .map_err(|e| { + error!(error = %e, "Failed to store stripe_customer_id"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })?; + + info!(account_id = %account_id, customer_id = %customer.id, "Created Stripe customer"); + + Ok(customer.id) +} + +/// Get the user's account_id (re-exported from auth for convenience). +async fn get_account_id( + pool: &sqlx::PgPool, + user: &AuthenticatedUser, +) -> Result { + crate::auth::get_user_account_id(pool, user.user_id, user.selected_account_id) + .await + .map_err(|_| err(StatusCode::NOT_FOUND, "Account not found", "not_found")) +} + +// ============================================================================ +// Handlers +// ============================================================================ + +/// POST /account/billing/setup-intent +/// +/// Create a SetupIntent for the frontend to collect a payment method +/// (card details) via Stripe Elements. Returns a client_secret. +async fn create_setup_intent( + State(state): State>, + Extension(user): Extension, + Extension(stripe_state): Extension, +) -> Result, ApiError> { + let account_id = get_account_id(&state.pool, &user).await?; + let customer_id = get_or_create_customer(&stripe_state.client, &state.pool, account_id).await?; + + let mut params = CreateSetupIntent::new(); + params.customer = Some(customer_id); + params.payment_method_types = Some(vec!["card".to_string()]); + params.metadata = Some( + [("scrapix_account_id".to_string(), account_id.to_string())] + .into_iter() + .collect(), + ); + + let setup_intent = SetupIntent::create(&stripe_state.client, params) + .await + .map_err(|e| { + error!(error = %e, "Failed to create SetupIntent"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to create setup intent", + "stripe_error", + ) + })?; + + Ok(Json(SetupIntentResponse { + client_secret: setup_intent.client_secret.ok_or_else(|| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Missing client_secret", + "stripe_error", + ) + })?, + })) +} + +/// GET /account/billing/payment-methods +/// +/// List all saved payment methods for the account's Stripe customer. +async fn list_payment_methods( + State(state): State>, + Extension(user): Extension, + Extension(stripe_state): Extension, +) -> Result>, ApiError> { + let account_id = get_account_id(&state.pool, &user).await?; + + // Get stripe customer id — if none, return empty list + let customer_id: Option = + sqlx::query_scalar("SELECT stripe_customer_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(&state.pool) + .await + .map_err(|e| { + error!(error = %e, "DB error"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + let customer_id = match customer_id { + Some(c) => c, + None => return Ok(Json(vec![])), + }; + + let cid: CustomerId = customer_id.parse().map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Invalid stripe customer ID", + "internal_error", + ) + })?; + + // Get default payment method from our DB + let default_pm: Option = + sqlx::query_scalar("SELECT stripe_default_payment_method_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(&state.pool) + .await + .map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + let mut params = ListPaymentMethods::new(); + params.customer = Some(cid); + params.type_ = Some(PaymentMethodTypeFilter::Card); + + let methods = PaymentMethod::list(&stripe_state.client, ¶ms) + .await + .map_err(|e| { + error!(error = %e, "Failed to list payment methods"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to list payment methods", + "stripe_error", + ) + })?; + + let result: Vec = methods + .data + .iter() + .map(|pm| { + let card = pm.card.as_ref(); + PaymentMethodResponse { + id: pm.id.to_string(), + brand: card.map(|c| format!("{:?}", c.brand).to_lowercase()), + last4: card.map(|c| c.last4.clone()), + exp_month: card.map(|c| c.exp_month as i32), + exp_year: card.map(|c| c.exp_year as i32), + is_default: default_pm.as_deref() == Some(pm.id.as_str()), + } + }) + .collect(); + + Ok(Json(result)) +} + +/// DELETE /account/billing/payment-methods/{id} +/// +/// Detach a payment method from the customer. +async fn delete_payment_method( + State(state): State>, + Extension(user): Extension, + Extension(stripe_state): Extension, + Path(pm_id): Path, +) -> Result, ApiError> { + let account_id = get_account_id(&state.pool, &user).await?; + + // Verify the payment method belongs to this account's customer + let customer_id: Option = + sqlx::query_scalar("SELECT stripe_customer_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(&state.pool) + .await + .map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + let customer_id = customer_id + .ok_or_else(|| err(StatusCode::BAD_REQUEST, "No Stripe customer", "no_customer"))?; + + let pm_id: PaymentMethodId = pm_id.parse().map_err(|_| { + err( + StatusCode::BAD_REQUEST, + "Invalid payment method ID", + "validation_error", + ) + })?; + + // Fetch the payment method to verify ownership + let pm = PaymentMethod::retrieve(&stripe_state.client, &pm_id, &[]) + .await + .map_err(|e| { + error!(error = %e, "Failed to retrieve payment method"); + err( + StatusCode::NOT_FOUND, + "Payment method not found", + "not_found", + ) + })?; + + // Verify it belongs to this customer + if pm.customer.as_ref().map(|c| c.id().to_string()) != Some(customer_id) { + return Err(err( + StatusCode::FORBIDDEN, + "Payment method does not belong to this account", + "forbidden", + )); + } + + PaymentMethod::detach(&stripe_state.client, &pm.id) + .await + .map_err(|e| { + error!(error = %e, "Failed to detach payment method"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to remove payment method", + "stripe_error", + ) + })?; + + // If this was the default, clear it + let default_pm: Option = + sqlx::query_scalar("SELECT stripe_default_payment_method_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(&state.pool) + .await + .ok() + .flatten() + .flatten(); + + if default_pm.as_deref() == Some(pm.id.as_str()) { + sqlx::query("UPDATE accounts SET stripe_default_payment_method_id = NULL WHERE id = $1") + .bind(account_id) + .execute(&state.pool) + .await + .ok(); + } + + info!(account_id = %account_id, payment_method = %pm.id, "Payment method detached"); + + Ok(Json(MessageResponse { + message: "Payment method removed".to_string(), + })) +} + +/// PATCH /account/billing/default-payment-method +/// +/// Set the default payment method for the account. +async fn set_default_payment_method( + State(state): State>, + Extension(user): Extension, + Json(req): Json, +) -> Result, ApiError> { + let account_id = get_account_id(&state.pool, &user).await?; + + sqlx::query("UPDATE accounts SET stripe_default_payment_method_id = $1 WHERE id = $2") + .bind(&req.payment_method_id) + .bind(account_id) + .execute(&state.pool) + .await + .map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to update", + "internal_error", + ) + })?; + + Ok(Json(MessageResponse { + message: "Default payment method updated".to_string(), + })) +} + +/// POST /account/billing/purchase +/// +/// Purchase a credit pack. Creates a Stripe Invoice with line items, finalizes +/// and pays it. This generates a proper invoice with PDF. If 3D Secure is +/// required, returns `requires_action` with a `client_secret` for the frontend. +async fn purchase_credits( + State(state): State>, + Extension(user): Extension, + Extension(stripe_state): Extension, + Json(req): Json, +) -> Result, ApiError> { + if req.credits < 100 { + return Err(err( + StatusCode::BAD_REQUEST, + "Minimum purchase is 100 credits", + "validation_error", + )); + } + + let amount_cents = calculate_price_cents(req.credits); + + let account_id = get_account_id(&state.pool, &user).await?; + let customer_id = get_or_create_customer(&stripe_state.client, &state.pool, account_id).await?; + + // Determine payment method: explicit or default + let pm_id = match req.payment_method_id { + Some(ref id) => id.clone(), + None => { + let default: Option = sqlx::query_scalar( + "SELECT stripe_default_payment_method_id FROM accounts WHERE id = $1", + ) + .bind(account_id) + .fetch_optional(&state.pool) + .await + .map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + default.ok_or_else(|| { + err( + StatusCode::BAD_REQUEST, + "No payment method on file. Please add a card first.", + "no_payment_method", + ) + })? + } + }; + + // Create the invoice, pay it, and add credits + let invoice = create_and_pay_invoice( + &stripe_state.client, + customer_id, + account_id, + &pm_id, + req.credits, + amount_cents, + "credit_purchase", + ) + .await?; + + // Check the payment intent status on the paid invoice + let pi_status = invoice + .payment_intent + .as_ref() + .and_then(|pi| pi.as_object()) + .map(|pi| pi.status); + + match pi_status { + Some(PaymentIntentStatus::Succeeded) => { + let pi_id = invoice + .payment_intent + .as_ref() + .map(|pi| pi.id().to_string()) + .unwrap_or_default(); + + add_credits_for_payment( + &state.pool, + account_id, + req.credits, + &pi_id, + "Credit purchase", + ) + .await?; + + Ok(Json(PurchaseResponse { + status: "succeeded".to_string(), + client_secret: None, + credits: req.credits, + amount_cents, + message: format!("{} credits added to your account", req.credits), + })) + } + Some(PaymentIntentStatus::RequiresAction) => { + let client_secret = invoice + .payment_intent + .as_ref() + .and_then(|pi| pi.as_object()) + .and_then(|pi| pi.client_secret.clone()); + + Ok(Json(PurchaseResponse { + status: "requires_action".to_string(), + client_secret, + credits: req.credits, + amount_cents, + message: "Additional authentication required".to_string(), + })) + } + other => { + warn!(status = ?other, invoice_id = %invoice.id, "Unexpected payment status on invoice"); + Err(err( + StatusCode::BAD_REQUEST, + "Payment could not be processed", + "payment_failed", + )) + } + } +} + +/// Create a Stripe Invoice with a line item, finalize it, and pay it. +/// Returns the paid Invoice object (with `invoice_pdf`, `hosted_invoice_url`, etc.). +async fn create_and_pay_invoice( + stripe: &StripeClient, + customer_id: CustomerId, + account_id: uuid::Uuid, + payment_method_id: &str, + credits: i64, + amount_cents: i64, + purchase_type: &str, +) -> Result { + // 1. Create an invoice item (pending, attached to customer) + let item_description = format!("Scrapix: {} credits", credits); + let mut item_params = CreateInvoiceItem::new(customer_id.clone()); + item_params.amount = Some(amount_cents); + item_params.currency = Some(Currency::USD); + item_params.description = Some(&item_description); + + stripe::InvoiceItem::create(stripe, item_params) + .await + .map_err(|e| { + error!(error = %e, "Failed to create InvoiceItem"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to create invoice item", + "stripe_error", + ) + })?; + + // 2. Create a draft invoice (picks up the pending invoice item) + let description = format!("Scrapix: {} credits", credits); + let mut invoice_params = CreateInvoice::new(); + invoice_params.customer = Some(customer_id); + invoice_params.collection_method = Some(stripe::CollectionMethod::ChargeAutomatically); + invoice_params.auto_advance = Some(false); // we'll finalize and pay manually + invoice_params.default_payment_method = Some(payment_method_id); + invoice_params.description = Some(&description); + invoice_params.pending_invoice_items_behavior = + Some(InvoicePendingInvoiceItemsBehavior::Include); + invoice_params.metadata = Some( + [ + ("scrapix_account_id".to_string(), account_id.to_string()), + ("credits".to_string(), credits.to_string()), + ("type".to_string(), purchase_type.to_string()), + ] + .into_iter() + .collect(), + ); + + let invoice = Invoice::create(stripe, invoice_params).await.map_err(|e| { + error!(error = %e, "Failed to create Invoice"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to create invoice", + "stripe_error", + ) + })?; + + // 3. Finalize the invoice + let finalize_params: std::collections::HashMap<&str, &str> = + [("auto_advance", "false")].into_iter().collect(); + let invoice: Invoice = stripe + .post_form( + &format!("/invoices/{}/finalize", invoice.id), + finalize_params, + ) + .await + .map_err(|e| { + error!(error = %e, invoice_id = %invoice.id, "Failed to finalize invoice"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to finalize invoice", + "stripe_error", + ) + })?; + + // 4. Pay the invoice — expands the payment_intent so we can check its status + let pay_params: std::collections::HashMap<&str, &str> = + [("expand[]", "payment_intent")].into_iter().collect(); + let invoice: Invoice = stripe + .post_form(&format!("/invoices/{}/pay", invoice.id), pay_params) + .await + .map_err(|e| { + error!(error = %e, invoice_id = %invoice.id, "Failed to pay invoice"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Payment failed. Please try again or use a different card.", + "stripe_error", + ) + })?; + + info!( + account_id = %account_id, + invoice_id = %invoice.id, + credits, + amount_cents, + "Invoice created and paid" + ); + + Ok(invoice) +} + +/// Add credits to an account after a successful payment. +/// Delegates to `scrapix_billing::add_credits_for_payment`. +async fn add_credits_for_payment( + pool: &sqlx::PgPool, + account_id: uuid::Uuid, + credits: i64, + payment_intent_id: &str, + description: &str, +) -> Result<(), ApiError> { + scrapix_billing::add_credits_for_payment( + pool, + account_id, + credits, + payment_intent_id, + description, + ) + .await + .map_err(|e| { + error!(error = %e, "Failed to add credits for payment"); + err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string(), e.code()) + }) +} + +// ============================================================================ +// Webhook handler +// ============================================================================ + +/// POST /webhooks/stripe +/// +/// Receives Stripe webhook events. No auth required (verified by signature). +async fn stripe_webhook( + Extension(stripe_state): Extension, + Extension(pool): Extension, + Extension(email_client): Extension>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let signature = headers + .get("stripe-signature") + .and_then(|v| v.to_str().ok()) + .ok_or(( + StatusCode::BAD_REQUEST, + "Missing stripe-signature header".to_string(), + ))?; + + let webhook_secret = stripe_state.webhook_secret.as_deref().ok_or(( + StatusCode::INTERNAL_SERVER_ERROR, + "Webhook secret not configured".to_string(), + ))?; + + let payload = std::str::from_utf8(&body).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid payload encoding".to_string(), + ) + })?; + + let event = Webhook::construct_event(payload, signature, webhook_secret).map_err(|e| { + warn!(error = %e, "Webhook signature verification failed"); + ( + StatusCode::BAD_REQUEST, + "Webhook signature verification failed".to_string(), + ) + })?; + + match event.type_ { + EventType::InvoicePaid => { + if let EventObject::Invoice(inv) = event.data.object { + handle_invoice_paid(&pool, &inv, email_client.as_ref()).await; + } + } + EventType::PaymentIntentSucceeded => { + if let EventObject::PaymentIntent(pi) = event.data.object { + handle_payment_intent_succeeded(&pool, &pi, email_client.as_ref()).await; + } + } + EventType::PaymentIntentPaymentFailed => { + if let EventObject::PaymentIntent(pi) = event.data.object { + warn!( + pi_id = %pi.id, + "Payment failed for PaymentIntent" + ); + } + } + EventType::SetupIntentSucceeded => { + if let EventObject::SetupIntent(si) = event.data.object { + handle_setup_intent_succeeded(&pool, &si).await; + } + } + _ => { + // Ignore events we don't handle + } + } + + Ok(StatusCode::OK) +} + +async fn handle_invoice_paid( + pool: &sqlx::PgPool, + inv: &Invoice, + email_client: Option<&EmailClient>, +) { + let metadata = match &inv.metadata { + Some(m) => m, + None => { + warn!(invoice_id = %inv.id, "Invoice missing metadata"); + return; + } + }; + + let account_id_str = match metadata.get("scrapix_account_id") { + Some(id) => id.clone(), + None => { + // Not a Scrapix invoice — ignore + return; + } + }; + + let credits_str = match metadata.get("credits") { + Some(c) => c.clone(), + None => { + warn!(invoice_id = %inv.id, "Invoice missing credits metadata"); + return; + } + }; + + let account_id: uuid::Uuid = match account_id_str.parse() { + Ok(id) => id, + Err(_) => { + warn!(invoice_id = %inv.id, "Invalid account_id in invoice metadata"); + return; + } + }; + + let credits: i64 = match credits_str.parse() { + Ok(c) => c, + Err(_) => { + warn!(invoice_id = %inv.id, "Invalid credits in invoice metadata"); + return; + } + }; + + // Use the invoice's payment_intent ID for idempotency + let pi_id = inv + .payment_intent + .as_ref() + .map(|pi| pi.id().to_string()) + .unwrap_or_else(|| inv.id.to_string()); + + if let Err(e) = add_credits_for_payment( + pool, + account_id, + credits, + &pi_id, + "Credit purchase (Invoice)", + ) + .await + { + error!(error = ?e, invoice_id = %inv.id, "Failed to add credits from invoice webhook"); + return; + } + + // Send payment receipt via the reliable queue + if let Some(mailer) = email_client { + let amount_cents = inv.amount_paid.unwrap_or(0); + if let Some(email) = crate::email::get_account_email(pool, account_id).await { + mailer + .queue_payment_receipt(pool, &email, credits, amount_cents) + .await; + } + } +} + +async fn handle_payment_intent_succeeded( + pool: &sqlx::PgPool, + pi: &PaymentIntent, + email_client: Option<&EmailClient>, +) { + let metadata = &pi.metadata; + + let account_id_str = match metadata.get("scrapix_account_id") { + Some(id) => id.clone(), + None => { + warn!(pi_id = %pi.id, "PaymentIntent missing scrapix_account_id metadata"); + return; + } + }; + + let credits_str = match metadata.get("credits") { + Some(c) => c.clone(), + None => { + warn!(pi_id = %pi.id, "PaymentIntent missing credits metadata"); + return; + } + }; + + let account_id: uuid::Uuid = match account_id_str.parse() { + Ok(id) => id, + Err(_) => { + warn!(pi_id = %pi.id, "Invalid account_id in metadata"); + return; + } + }; + + let credits: i64 = match credits_str.parse() { + Ok(c) => c, + Err(_) => { + warn!(pi_id = %pi.id, "Invalid credits in metadata"); + return; + } + }; + + if let Err(e) = add_credits_for_payment( + pool, + account_id, + credits, + pi.id.as_ref(), + "Credit purchase (Stripe)", + ) + .await + { + error!(error = ?e, pi_id = %pi.id, "Failed to add credits from webhook"); + return; + } + + // Send payment receipt via the reliable queue + if let Some(mailer) = email_client { + let amount_cents = pi.amount; + if let Some(email) = crate::email::get_account_email(pool, account_id).await { + mailer + .queue_payment_receipt(pool, &email, credits, amount_cents) + .await; + } + } +} + +async fn handle_setup_intent_succeeded(pool: &sqlx::PgPool, si: &SetupIntent) { + // When a setup intent succeeds, set the payment method as default if the account + // doesn't have one yet + let metadata = match &si.metadata { + Some(m) => m, + None => return, + }; + + let account_id_str = match metadata.get("scrapix_account_id") { + Some(id) => id.clone(), + None => return, + }; + + let account_id: uuid::Uuid = match account_id_str.parse() { + Ok(id) => id, + Err(_) => return, + }; + + let pm_id = match &si.payment_method { + Some(pm) => pm.id().to_string(), + None => return, + }; + + // Set as default only if no default exists yet + let result = sqlx::query( + "UPDATE accounts SET stripe_default_payment_method_id = $1 \ + WHERE id = $2 AND stripe_default_payment_method_id IS NULL", + ) + .bind(&pm_id) + .bind(account_id) + .execute(pool) + .await; + + match result { + Ok(r) if r.rows_affected() > 0 => { + info!(account_id = %account_id, pm = %pm_id, "Set default payment method from SetupIntent"); + } + Ok(_) => {} // already had a default + Err(e) => { + warn!(error = %e, "Failed to set default payment method from webhook"); + } + } +} + +// ============================================================================ +// Auto-topup with Stripe +// ============================================================================ + +/// Charge the account's saved payment method for an auto-topup. +/// Called from `billing::maybe_auto_topup` when a real payment is needed. +pub async fn charge_auto_topup( + stripe: &StripeClient, + pool: &sqlx::PgPool, + account_id: uuid::Uuid, + credits: i64, +) -> Result { + // Get customer ID and default payment method + let row = sqlx::query( + "SELECT stripe_customer_id, stripe_default_payment_method_id FROM accounts WHERE id = $1", + ) + .bind(account_id) + .fetch_optional(pool) + .await + .map_err(|e| format!("DB error: {e}"))? + .ok_or("Account not found")?; + + let customer_id: Option = row.get("stripe_customer_id"); + let pm_id: Option = row.get("stripe_default_payment_method_id"); + + let customer_id = customer_id.ok_or("No Stripe customer")?; + let pm_id = pm_id.ok_or("No default payment method for auto-topup")?; + + let cid: CustomerId = customer_id.parse().map_err(|_| "Invalid customer ID")?; + + let amount_cents = calculate_price_cents(credits); + + let invoice = create_and_pay_invoice( + stripe, + cid, + account_id, + &pm_id, + credits, + amount_cents, + "auto_topup", + ) + .await + .map_err(|e| format!("Invoice error: {}", e.1.error))?; + + let pi_status = invoice + .payment_intent + .as_ref() + .and_then(|pi| pi.as_object()) + .map(|pi| pi.status); + + if pi_status == Some(PaymentIntentStatus::Succeeded) { + let pi_id = invoice + .payment_intent + .as_ref() + .map(|pi| pi.id().to_string()) + .unwrap_or_default(); + + add_credits_for_payment(pool, account_id, credits, &pi_id, "Auto top-up (Stripe)") + .await + .map_err(|e| format!("Failed to add credits: {}", e.0))?; + + Ok(pi_id) + } else { + Err(format!("Auto-topup payment status: {:?}", pi_status)) + } +} + +// ============================================================================ +// Invoices +// ============================================================================ + +/// GET /account/billing/invoices +/// +/// List actual Stripe Invoices for the account, with PDF download links. +async fn list_invoices( + State(state): State>, + Extension(user): Extension, + Extension(stripe_state): Extension, +) -> Result>, ApiError> { + let account_id = get_account_id(&state.pool, &user).await?; + + let customer_id: Option = + sqlx::query_scalar("SELECT stripe_customer_id FROM accounts WHERE id = $1") + .bind(account_id) + .fetch_optional(&state.pool) + .await + .map_err(|e| { + error!(error = %e, "DB error fetching stripe_customer_id"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Database error", + "internal_error", + ) + })? + .flatten(); + + let customer_id = match customer_id { + Some(c) => c, + None => return Ok(Json(vec![])), + }; + + let cid: CustomerId = customer_id.parse().map_err(|_| { + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Invalid stripe customer ID", + "internal_error", + ) + })?; + + let mut params = ListInvoices::new(); + params.customer = Some(cid); + params.status = Some(InvoiceStatus::Paid); + params.limit = Some(50); + + let stripe_invoices = Invoice::list(&stripe_state.client, ¶ms) + .await + .map_err(|e| { + error!(error = %e, "Failed to list invoices"); + err( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to list invoices", + "stripe_error", + ) + })?; + + let invoices: Vec = stripe_invoices + .data + .iter() + .map(|inv| { + let credits = inv + .metadata + .as_ref() + .and_then(|m| m.get("credits")) + .and_then(|c| c.parse::().ok()); + + let status = inv + .status + .map(|s| s.as_str().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + InvoiceResponse { + id: inv.id.to_string(), + number: inv.number.clone(), + amount_cents: inv.amount_paid.unwrap_or(0), + credits, + status, + description: inv.description.clone(), + created_at: inv + .created + .and_then(|ts| chrono::DateTime::from_timestamp(ts, 0)) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_default(), + invoice_pdf: inv.invoice_pdf.clone(), + hosted_invoice_url: inv.hosted_invoice_url.clone(), + } + }) + .collect(); + + Ok(Json(invoices)) +} + +// ============================================================================ +// Pricing +// ============================================================================ + +/// GET /account/billing/pricing +/// +/// Returns the volume-based pricing tiers. +async fn get_pricing() -> Json> { + Json(vec![ + PricingTier { + up_to: Some(999), + unit_price_cents: 1.0, + per_1k: 10.0, + }, + PricingTier { + up_to: Some(4_999), + unit_price_cents: 0.8, + per_1k: 8.0, + }, + PricingTier { + up_to: Some(9_999), + unit_price_cents: 0.7, + per_1k: 7.0, + }, + PricingTier { + up_to: None, + unit_price_cents: 0.5, + per_1k: 5.0, + }, + ]) +} + +// ============================================================================ +// Router +// ============================================================================ + +/// Stripe-related routes that require session auth. +pub fn stripe_session_routes(state: Arc, stripe_state: StripeState) -> Router { + Router::new() + .route("/account/billing/setup-intent", post(create_setup_intent)) + .route( + "/account/billing/payment-methods", + get(list_payment_methods), + ) + .route( + "/account/billing/payment-methods/{id}", + delete(delete_payment_method), + ) + .route( + "/account/billing/default-payment-method", + axum::routing::patch(set_default_payment_method), + ) + .route("/account/billing/purchase", post(purchase_credits)) + .route("/account/billing/invoices", get(list_invoices)) + .route("/account/billing/pricing", get(get_pricing)) + .layer(Extension(stripe_state)) + .layer(middleware::from_fn_with_state( + state.clone(), + crate::auth::validate_session, + )) + .with_state(state) +} + +/// Stripe webhook route (no auth — verified by Stripe signature). +pub fn stripe_webhook_route( + pool: sqlx::PgPool, + stripe_state: StripeState, + email_client: Option, +) -> Router { + Router::new() + .route("/webhooks/stripe", post(stripe_webhook)) + .layer(Extension(stripe_state)) + .layer(Extension(pool)) + .layer(Extension(email_client)) +} diff --git a/bins/scrapix-api/tests/hyperline_webhook.rs b/bins/scrapix-api/tests/hyperline_webhook.rs deleted file mode 100644 index b3be8a3c..00000000 --- a/bins/scrapix-api/tests/hyperline_webhook.rs +++ /dev/null @@ -1,581 +0,0 @@ -//! DB-bound integration tests for `POST /webhooks/hyperline`. -//! -//! These drive the real axum router in-process against a Postgres pool, -//! signing each payload with the same HMAC-SHA256 algorithm Hyperline -//! uses so the verifier sees real bytes end-to-end. We don't stub -//! `verify_signature` — that's the whole point of this suite. -//! -//! Gating: tests are `#[ignore]` by default and require -//! `TEST_DATABASE_URL` to point at a Postgres whose schema matches -//! `deploy/postgres/init.sql`. The simplest way to run them: -//! -//! ```sh -//! docker compose up -d postgres -//! TEST_DATABASE_URL=postgres://scrapix:scrapix@localhost:5433/scrapix \ -//! cargo test -p scrapix-api --test hyperline_webhook -- --ignored --nocapture -//! ``` -//! -//! Tests don't clean up after themselves; they use fresh UUIDs for -//! every row so re-runs never collide. The dev DB accumulates harmless -//! rows; `docker compose down -v` resets it. -//! -//! What's covered: -//! - Signature verification rejects invalid signatures and stale -//! timestamps (matches the unit tests in the verifier crate, but -//! exercised through the axum extract layer — catches regressions in -//! header parsing / extractor wiring). -//! - `wallet.credited` → ledger credit + `transactions` row. -//! - Webhook-id dedupe: same delivery replayed is a no-op. -//! - Ledger-level dedupe: new webhook_id but same `provider_event_id` -//! credits only once. -//! - `invoice.settled` clears `payment_method_status`. -//! - `invoice.errored` records a transaction without clearing the flag. -//! - `payment_method.expired` sets `payment_method_status`. -//! - `subscription.cancelled` flips `accounts.active = false`. -//! - Unknown event type → 200, logged (so Hyperline stops retrying). - -use axum::{ - body::Body, - http::{Request, StatusCode}, - Router, -}; -use base64::{engine::general_purpose::STANDARD as B64, Engine}; -use hmac::{Hmac, Mac}; -use http_body_util::BodyExt; -use scrapix_api::hyperline; -use sha2::Sha256; -use sqlx::{PgPool, Row}; -use std::sync::Once; -use tower::ServiceExt; -use uuid::Uuid; - -type HmacSha256 = Hmac; - -/// HMAC key bytes shared by every test in this file. We derive the -/// `whsec_` secret string from it so the handler and the test agree. -const WEBHOOK_KEY_RAW: &[u8] = b"scrapix-test-webhook-key-do-not-use-in-prod"; - -static INIT: Once = Once::new(); - -/// Connect to `TEST_DATABASE_URL` or return `None` (tests then skip). -/// -/// We don't panic on a missing var — `#[ignore]` is doing the real -/// gating, but making the helper return Option keeps the "no DB -/// available" message readable. -async fn try_pool() -> Option { - let url = std::env::var("TEST_DATABASE_URL").ok()?; - let pool = sqlx::PgPool::connect(&url) - .await - .expect("TEST_DATABASE_URL is set but the connection failed"); - - // Run the schema once per process. init.sql is fully idempotent (IF - // NOT EXISTS everywhere), so running it multiple times is safe — but - // doing it once saves a few ms per test. - INIT.call_once(|| { - let pool = pool.clone(); - // Best-effort: if schema application fails the tests will surface - // the real error downstream, no need to bubble it up here. - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async move { - let sql = std::fs::read_to_string(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../deploy/postgres/init.sql" - )) - .expect("reading deploy/postgres/init.sql"); - // sqlx can only run one statement per `execute`, so we - // walk the file. Splitting on `;\n` is coarse but good - // enough for init.sql (no function bodies with embedded - // semicolons, which is the usual gotcha). - for stmt in sql.split(";\n") { - let trimmed = stmt.trim(); - if trimmed.is_empty() || trimmed.starts_with("--") { - continue; - } - // Ignore failures here (e.g. re-running in a dirty - // DB): init.sql is entirely IF NOT EXISTS / IF NOT - // EXISTS wrapped, but manual DB surgery between - // runs can leave it in a state where a subset - // errors. Tests then fail loudly on real assertions. - let _ = sqlx::query(trimmed).execute(&pool).await; - } - }); - }); - }); - - Some(pool) -} - -/// Build the `whsec_`-prefixed secret string the handler expects. -fn webhook_secret() -> String { - format!("whsec_{}", B64.encode(WEBHOOK_KEY_RAW)) -} - -/// Sign a body per Hyperline's Svix-style HMAC over `id.timestamp.body`. -fn sign(id: &str, ts: &str, body: &[u8]) -> String { - let mut mac = HmacSha256::new_from_slice(WEBHOOK_KEY_RAW).expect("hmac key"); - mac.update(id.as_bytes()); - mac.update(b"."); - mac.update(ts.as_bytes()); - mac.update(b"."); - mac.update(body); - let b64 = B64.encode(mac.finalize().into_bytes()); - format!("v1,{b64}") -} - -/// Build the webhook router with a fresh state. No email client so the -/// low-balance branch is exercised without a real mailer (it logs and -/// moves on — the transactions/accounts side effects still land). -fn router(pool: PgPool) -> Router { - hyperline::webhook_route(pool, webhook_secret(), None) -} - -/// Seed a user → account pair. Returns `(account_id, hyperline_customer_id)`. -/// -/// Each account gets a unique `hyperline_customer_id` so tests don't -/// collide on the UNIQUE constraint, and the returned handle is used -/// as the `customer_id` in webhook payloads. -async fn seed_account(pool: &PgPool) -> (Uuid, String) { - let customer_id = format!("cus_test_{}", Uuid::new_v4().simple()); - let account_id: Uuid = sqlx::query_scalar( - "INSERT INTO accounts (name, hyperline_customer_id, credits_balance) \ - VALUES ($1, $2, $3) RETURNING id", - ) - .bind(format!("test-account-{}", Uuid::new_v4().simple())) - .bind(&customer_id) - .bind(100_i64) - .fetch_one(pool) - .await - .expect("seed_account"); - (account_id, customer_id) -} - -/// Construct a signed `Request` for the webhook route. -fn signed_request( - id: &str, - timestamp: i64, - body: serde_json::Value, - signature_override: Option<&str>, -) -> Request { - let ts_str = timestamp.to_string(); - let body_bytes = serde_json::to_vec(&body).expect("serialize body"); - let sig = signature_override - .map(str::to_owned) - .unwrap_or_else(|| sign(id, &ts_str, &body_bytes)); - Request::builder() - .method("POST") - .uri("/webhooks/hyperline") - .header("content-type", "application/json") - .header("webhook-id", id) - .header("webhook-timestamp", &ts_str) - .header("webhook-signature", &sig) - .body(Body::from(body_bytes)) - .expect("build request") -} - -/// Drive the router once and return the status code. Body is consumed -/// so tests can follow up with DB assertions. -async fn send(router: &Router, req: Request) -> StatusCode { - let resp = router.clone().oneshot(req).await.expect("router oneshot"); - let status = resp.status(); - // Drain the body so the connection "finishes" cleanly. - let _ = resp.into_body().collect().await; - status -} - -// ============================================================================ -// Tests -// ============================================================================ - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn rejects_invalid_signature() { - let Some(pool) = try_pool().await else { return }; - let app = router(pool); - - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - serde_json::json!({"type": "wallet.debited", "data": {}}), - Some("v1,bogus-signature-bytes"), - ); - assert_eq!(send(&app, req).await, StatusCode::BAD_REQUEST); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn rejects_stale_timestamp() { - let Some(pool) = try_pool().await else { return }; - let app = router(pool); - - // 10 minutes in the past — well outside the 5-min skew window. - let stale = chrono::Utc::now().timestamp() - 600; - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - stale, - serde_json::json!({"type": "wallet.debited", "data": {}}), - None, - ); - assert_eq!(send(&app, req).await, StatusCode::BAD_REQUEST); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn wallet_credited_adds_credits_and_records_transaction() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let event_id = format!("evt_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "wallet.credited", - "data": { - "object": { - "id": event_id, - "customer_id": customer_id, - "credits": 500, - } - } - }); - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - body, - None, - ); - assert_eq!(send(&app, req).await, StatusCode::OK); - - // Balance bumped by 500 (from the seed's 100 → 600). - let balance: i64 = sqlx::query_scalar("SELECT credits_balance FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(balance, 600); - - // One `wallet_credit` transaction row with the provider event id. - let row = sqlx::query( - "SELECT type, amount, metadata FROM transactions \ - WHERE account_id = $1 AND metadata->>'provider_event_id' = $2", - ) - .bind(account_id) - .bind(&event_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(row.get::("type"), "wallet_credit"); - assert_eq!(row.get::("amount"), 500); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn replay_same_webhook_id_is_no_op() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let webhook_id = format!("msg_{}", Uuid::new_v4().simple()); - let event_id = format!("evt_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "wallet.credited", - "data": { - "object": { - "id": event_id, - "customer_id": customer_id, - "credits": 250, - } - } - }); - let ts = chrono::Utc::now().timestamp(); - - // First delivery — accepted, credits applied. - let req1 = signed_request(&webhook_id, ts, body.clone(), None); - assert_eq!(send(&app, req1).await, StatusCode::OK); - - // Second delivery with the *same* webhook_id — dedupe should kick - // in at the hyperline_webhook_log layer before dispatch runs. - let req2 = signed_request(&webhook_id, ts, body, None); - assert_eq!(send(&app, req2).await, StatusCode::OK); - - let balance: i64 = sqlx::query_scalar("SELECT credits_balance FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - // Still 350 (100 seed + 250 once), not 600. - assert_eq!(balance, 350); - - let tx_count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM transactions \ - WHERE account_id = $1 AND metadata->>'provider_event_id' = $2", - ) - .bind(account_id) - .bind(&event_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(tx_count, 1); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn replay_new_webhook_id_same_event_id_is_ledger_idempotent() { - // Covers the dual idempotency model: if Hyperline re-broadcasts the - // same business event under a new delivery id (which bypasses the - // webhook_log dedup), the ledger's (provider, provider_event_id) - // check keeps us honest. - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let event_id = format!("evt_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "wallet.credited", - "data": { - "object": { - "id": event_id, - "customer_id": customer_id, - "credits": 777, - } - } - }); - - // Two deliveries with *different* webhook_ids but the same event_id. - let ts = chrono::Utc::now().timestamp(); - let r1 = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - ts, - body.clone(), - None, - ); - let r2 = signed_request(&format!("msg_{}", Uuid::new_v4().simple()), ts, body, None); - assert_eq!(send(&app, r1).await, StatusCode::OK); - assert_eq!(send(&app, r2).await, StatusCode::OK); - - let balance: i64 = sqlx::query_scalar("SELECT credits_balance FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(balance, 100 + 777); - - let tx_count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM transactions \ - WHERE account_id = $1 AND metadata->>'provider_event_id' = $2", - ) - .bind(account_id) - .bind(&event_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(tx_count, 1); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn invoice_settled_clears_payment_method_status() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - // Pre-flag the account so we can verify the clearing side effect. - sqlx::query("UPDATE accounts SET payment_method_status = 'errored' WHERE id = $1") - .bind(account_id) - .execute(&pool) - .await - .unwrap(); - - let app = router(pool.clone()); - - let event_id = format!("inv_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "invoice.settled", - "data": { - "object": { - "id": event_id, - "customer_id": customer_id, - "amount": 2500, - "currency": "USD", - } - } - }); - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - body, - None, - ); - assert_eq!(send(&app, req).await, StatusCode::OK); - - let status: Option = - sqlx::query_scalar("SELECT payment_method_status FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(status, None, "invoice.settled should clear the flag"); - - // Transaction row landed with amount=0 and processor_amount stashed in metadata. - let row = sqlx::query( - "SELECT type, amount, metadata FROM transactions \ - WHERE account_id = $1 AND metadata->>'provider_event_id' = $2", - ) - .bind(account_id) - .bind(&event_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(row.get::("type"), "invoice_settled"); - assert_eq!(row.get::("amount"), 0); - let meta: serde_json::Value = row.get("metadata"); - assert_eq!(meta["processor_amount"], 2500); - assert_eq!(meta["currency"], "USD"); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn invoice_errored_records_transaction_and_keeps_flag() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - // Errored flag should survive an invoice.errored — only settled clears. - sqlx::query("UPDATE accounts SET payment_method_status = 'errored' WHERE id = $1") - .bind(account_id) - .execute(&pool) - .await - .unwrap(); - - let app = router(pool.clone()); - - let event_id = format!("inv_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "invoice.errored", - "data": { - "object": { - "id": event_id, - "customer_id": customer_id, - "amount": 1000, - "currency": "USD", - } - } - }); - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - body, - None, - ); - assert_eq!(send(&app, req).await, StatusCode::OK); - - let status: Option = - sqlx::query_scalar("SELECT payment_method_status FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(status.as_deref(), Some("errored")); - - let tx_type: String = sqlx::query_scalar( - "SELECT type FROM transactions \ - WHERE account_id = $1 AND metadata->>'provider_event_id' = $2", - ) - .bind(account_id) - .bind(&event_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(tx_type, "invoice_errored"); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn payment_method_expired_flags_account() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let body = serde_json::json!({ - "type": "payment_method.expired", - "data": { - "object": { - "id": format!("pm_{}", Uuid::new_v4().simple()), - "customer_id": customer_id, - } - } - }); - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - body, - None, - ); - assert_eq!(send(&app, req).await, StatusCode::OK); - - let status: Option = - sqlx::query_scalar("SELECT payment_method_status FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(status.as_deref(), Some("expired")); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn subscription_cancelled_deactivates_account() { - let Some(pool) = try_pool().await else { return }; - let (account_id, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let body = serde_json::json!({ - "type": "subscription.cancelled", - "data": { - "object": { - "id": format!("sub_{}", Uuid::new_v4().simple()), - "customer_id": customer_id, - } - } - }); - let req = signed_request( - &format!("msg_{}", Uuid::new_v4().simple()), - chrono::Utc::now().timestamp(), - body, - None, - ); - assert_eq!(send(&app, req).await, StatusCode::OK); - - let active: bool = sqlx::query_scalar("SELECT active FROM accounts WHERE id = $1") - .bind(account_id) - .fetch_one(&pool) - .await - .unwrap(); - assert!(!active); -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires TEST_DATABASE_URL"] -async fn unknown_event_type_is_acked_and_logged() { - let Some(pool) = try_pool().await else { return }; - let (_, customer_id) = seed_account(&pool).await; - let app = router(pool.clone()); - - let webhook_id = format!("msg_{}", Uuid::new_v4().simple()); - let body = serde_json::json!({ - "type": "some.future.event", - "data": { - "object": { - "id": "evt_unknown", - "customer_id": customer_id, - } - } - }); - let req = signed_request(&webhook_id, chrono::Utc::now().timestamp(), body, None); - // Must 200 — a non-2xx would cause Hyperline to retry forever. - assert_eq!(send(&app, req).await, StatusCode::OK); - - // Row landed in the log for audit purposes. - let logged: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM hyperline_webhook_log WHERE webhook_id = $1)", - ) - .bind(&webhook_id) - .fetch_one(&pool) - .await - .unwrap(); - assert!(logged); -} diff --git a/bins/scrapix/src/all.rs b/bins/scrapix/src/all.rs index d506cee3..d86a01f0 100644 --- a/bins/scrapix/src/all.rs +++ b/bins/scrapix/src/all.rs @@ -126,6 +126,8 @@ async fn run_all_channels(args: &AllArgs) -> anyhow::Result<()> { database_url: args.database_url.clone(), jwt_secret: args.jwt_secret.clone(), redis_url: std::env::var("REDIS_URL").ok(), + stripe_secret_key: std::env::var("STRIPE_SECRET_KEY").ok(), + stripe_webhook_secret: std::env::var("STRIPE_WEBHOOK_SECRET").ok(), resend_api_key: std::env::var("RESEND_API_KEY").ok(), google_client_id: std::env::var("GOOGLE_CLIENT_ID").ok(), google_client_secret: std::env::var("GOOGLE_CLIENT_SECRET").ok(), @@ -370,6 +372,8 @@ async fn run_all_kafka(args: &AllArgs, brokers: &str) -> anyhow::Result<()> { database_url: args.database_url.clone(), jwt_secret: args.jwt_secret.clone(), redis_url: std::env::var("REDIS_URL").ok(), + stripe_secret_key: std::env::var("STRIPE_SECRET_KEY").ok(), + stripe_webhook_secret: std::env::var("STRIPE_WEBHOOK_SECRET").ok(), resend_api_key: std::env::var("RESEND_API_KEY").ok(), google_client_id: std::env::var("GOOGLE_CLIENT_ID").ok(), google_client_secret: std::env::var("GOOGLE_CLIENT_SECRET").ok(), diff --git a/console/Dockerfile b/console/Dockerfile index ad842333..3a936c83 100644 --- a/console/Dockerfile +++ b/console/Dockerfile @@ -15,7 +15,8 @@ ENV NEXT_TELEMETRY_DISABLED=1 # Next.js inlines NEXT_PUBLIC_* vars at build time. In Docker builds # (e.g. Heroku), runtime config vars aren't available during build, -# so we hardcode defaults here. +# so we hardcode defaults here. The publishable key is public by design. +ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51TAwQdRGrCEv0wspTJFC1nDzrfhVwg2pxBtfAplfdjdTDkB8kOxrVAodNPYjehdMhIzTCY4HUnJ6I21ELXuKiXDq00pDyrnwvj ENV NEXT_PUBLIC_SCRAPIX_API_URL=https://api.scrapix.meilisearch.com RUN npm run build diff --git a/console/heroku.yml b/console/heroku.yml index f9e8171c..04370bf7 100644 --- a/console/heroku.yml +++ b/console/heroku.yml @@ -2,4 +2,5 @@ build: docker: web: Dockerfile config: + NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: pk_test_51TAwQdRGrCEv0wspTJFC1nDzrfhVwg2pxBtfAplfdjdTDkB8kOxrVAodNPYjehdMhIzTCY4HUnJ6I21ELXuKiXDq00pDyrnwvj NEXT_PUBLIC_SCRAPIX_API_URL: https://api.scrapix.meilisearch.com diff --git a/console/package.json b/console/package.json index 7b127f20..3b9f12d3 100644 --- a/console/package.json +++ b/console/package.json @@ -17,6 +17,8 @@ "@radix-ui/react-separator": "^1.1.8", "@radix-ui/react-slot": "^1.2.4", "@radix-ui/react-tabs": "^1.1.13", + "@stripe/react-stripe-js": "^5.6.1", + "@stripe/stripe-js": "^8.11.0", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-query": "^5.90.21", "class-variance-authority": "^0.7.1", diff --git a/console/src/app/(marketing)/privacy/page.tsx b/console/src/app/(marketing)/privacy/page.tsx index 62cb256e..af1f4669 100644 --- a/console/src/app/(marketing)/privacy/page.tsx +++ b/console/src/app/(marketing)/privacy/page.tsx @@ -81,11 +81,9 @@ export default function PrivacyPolicyPage() {

3.3 Payment Data

- Billing is handled by our billing provider Hyperline, which uses - Stripe, Inc. as its downstream payment processor. We do not store - credit card numbers or bank account details. We retain the billing - provider's customer and wallet identifiers for billing - purposes. + Payment processing is handled by Stripe, Inc. We do not store credit + card numbers or bank account details. We retain your Stripe customer + ID and payment method identifiers for billing purposes.

3.4 Technical Data

@@ -165,10 +163,9 @@ export default function PrivacyPolicyPage() {
  • Service providers — - Hyperline (billing) with Stripe as its downstream payment - processor, Resend (transactional emails), Heroku/cloud - infrastructure providers (hosting). These processors act on - our instructions and are bound by data processing agreements. + Stripe (payment processing), Resend (transactional emails), + Heroku/cloud infrastructure providers (hosting). These processors + act on our instructions and are bound by data processing agreements.
  • Affiliates — companies diff --git a/console/src/app/(marketing)/terms/page.tsx b/console/src/app/(marketing)/terms/page.tsx index a1a0b135..da01b88e 100644 --- a/console/src/app/(marketing)/terms/page.tsx +++ b/console/src/app/(marketing)/terms/page.tsx @@ -117,12 +117,10 @@ export default function TermsOfServicePage() {

    5.2 Payment

    - Payments are processed by our billing provider Hyperline, which - uses Stripe, Inc. as its downstream payment processor. By - providing payment information, you authorize us to charge your - payment method for credits purchased or auto-top-up amounts - configured. All amounts are in US Dollars unless otherwise - specified. + Payments are processed by Stripe, Inc. By providing payment + information, you authorize us to charge your payment method for + credits purchased or auto-top-up amounts configured. All amounts are + in US Dollars unless otherwise specified.

    5.3 Refunds

    diff --git a/console/src/app/dashboard/billing/page.tsx b/console/src/app/dashboard/billing/page.tsx index d4532bc8..c40eecdb 100644 --- a/console/src/app/dashboard/billing/page.tsx +++ b/console/src/app/dashboard/billing/page.tsx @@ -1,8 +1,16 @@ "use client"; -import { useState, useMemo } from "react"; -import { useBilling, useTransactions, useAllTransactions, useMe } from "@/lib/hooks"; -import { fetchBillingPortal, updateSpendLimit } from "@/lib/api"; +import { useState, useMemo, useCallback } from "react"; +import { useBilling, useTransactions, useAllTransactions, useMe, usePaymentMethods, useInvoices } from "@/lib/hooks"; +import { + topupCredits, + updateAutoTopup, + updateSpendLimit, + createSetupIntent, + deletePaymentMethod, + setDefaultPaymentMethod, + purchaseCredits, +} from "@/lib/api"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; import { @@ -13,7 +21,18 @@ import { CardTitle, } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; import { Skeleton } from "@/components/ui/skeleton"; +import { Badge } from "@/components/ui/badge"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; import { Sheet, SheetContent, @@ -39,52 +58,70 @@ import { RefreshCw, ChevronLeft, ChevronRight, + Plus, + Trash2, + Star, Loader2, - Info, - ExternalLink, - AlertTriangle, Receipt, + ExternalLink, + FileDown, + Info, } from "lucide-react"; import { toast } from "sonner"; +import { cn } from "@/lib/utils"; import { formatDistanceToNow, format, parseISO, eachDayOfInterval, startOfDay } from "date-fns"; import dynamic from "next/dynamic"; +import { loadStripe } from "@stripe/stripe-js"; +import { + Elements, + PaymentElement, + useStripe, + useElements, +} from "@stripe/react-stripe-js"; const DailyCostChart = dynamic(() => import("./daily-cost-chart"), { ssr: false, }); -const TX_PAGE_SIZE = 5; +const stripePromise = process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY + ? loadStripe(process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY) + : null; -// Format a smallest-currency-unit amount (cents for USD) as a human-readable -// string. Falls back gracefully when `currency` is missing. -function formatMoney(amount: number, currency: string | null | undefined): string { - const code = (currency ?? "USD").toUpperCase(); - try { - return new Intl.NumberFormat("en-US", { - style: "currency", - currency: code, - }).format(amount / 100); - } catch { - // Non-ISO currency code — render the raw amount and the code. - return `${(amount / 100).toFixed(2)} ${code}`; - } +const PRICING_TIERS = [ + { upTo: 999, rate: 0.01, per1k: "$10" }, + { upTo: 4_999, rate: 0.008, per1k: "$8" }, + { upTo: 9_999, rate: 0.007, per1k: "$7" }, + { upTo: null as number | null, rate: 0.005, per1k: "$5" }, +]; + +const QUICK_AMOUNTS = [1_000, 5_000, 10_000, 50_000]; + +function calculatePrice(credits: number): number { + if (credits >= 10_000) return credits * 0.005; + if (credits >= 5_000) return credits * 0.007; + if (credits >= 1_000) return credits * 0.008; + return credits * 0.01; +} + +function getActiveTier(credits: number): number { + if (credits >= 10_000) return 3; + if (credits >= 5_000) return 2; + if (credits >= 1_000) return 1; + return 0; } +const TX_PAGE_SIZE = 5; + function transactionIcon(type: string) { switch (type) { case "initial_deposit": return ; case "manual_topup": - case "wallet_credit": return ; case "auto_topup": return ; case "usage_deduction": return ; - case "invoice_settled": - return ; - case "invoice_errored": - return ; default: return ; } @@ -100,12 +137,6 @@ function transactionLabel(type: string) { return "Auto Top-up"; case "usage_deduction": return "Usage"; - case "wallet_credit": - return "Wallet Credit"; - case "invoice_settled": - return "Invoice Paid"; - case "invoice_errored": - return "Invoice Failed"; case "refund": return "Refund"; case "adjustment": @@ -115,6 +146,67 @@ function transactionLabel(type: string) { } } +function cardBrandName(brand: string | null) { + if (!brand) return "Card"; + const names: Record = { + visa: "Visa", + mastercard: "Mastercard", + amex: "Amex", + discover: "Discover", + diners: "Diners", + jcb: "JCB", + unionpay: "UnionPay", + }; + return names[brand] ?? brand; +} + +// ============================================================================ +// Add Card Form (rendered inside ) +// ============================================================================ + +function AddCardForm({ onSuccess, onCancel }: { onSuccess: () => void; onCancel: () => void }) { + const stripe = useStripe(); + const elements = useElements(); + const [loading, setLoading] = useState(false); + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!stripe || !elements) return; + + setLoading(true); + const { error } = await stripe.confirmSetup({ + elements, + confirmParams: { + return_url: window.location.href, + }, + redirect: "if_required", + }); + + setLoading(false); + if (error) { + toast.error(error.message ?? "Failed to save card"); + } else { + toast.success("Card saved successfully"); + onSuccess(); + } + }; + + return ( +
    + +
    + + +
    + + ); +} + // ============================================================================ // Main Billing Page // ============================================================================ @@ -124,26 +216,122 @@ export default function BillingPage() { const { data: user, isLoading: userLoading } = useMe(); const isOwner = user?.account?.role === "owner"; const { data: billing, isLoading: billingLoading } = useBilling(); + const { data: paymentMethods, isLoading: pmLoading } = usePaymentMethods(); + const { data: invoices, isLoading: invoicesLoading } = useInvoices(); const [txOffset, setTxOffset] = useState(0); const { data: txData, isLoading: txLoading } = useTransactions(TX_PAGE_SIZE, txOffset); const { data: allTxData } = useAllTransactions(); + // Add card form state + const [showAddCard, setShowAddCard] = useState(false); + const [setupClientSecret, setSetupClientSecret] = useState(null); + + // Purchase state + const [purchaseAmount, setPurchaseAmount] = useState("1000"); + const [purchasingPack, setPurchasingPack] = useState(null); + const [showPurchaseConfirm, setShowPurchaseConfirm] = useState(false); + + // Auto top-up form state + const [autoTopupAmount, setAutoTopupAmount] = useState(""); + const [autoTopupThreshold, setAutoTopupThreshold] = useState(""); + // Spend limit form state const [spendLimit, setSpendLimit] = useState(""); const isLoading = userLoading || billingLoading; + const hasStripe = !!stripePromise; + const hasPaymentMethod = (paymentMethods?.length ?? 0) > 0; + + const invalidateBilling = useCallback(() => { + queryClient.invalidateQueries({ queryKey: ["billing"] }); + queryClient.invalidateQueries({ queryKey: ["transactions"] }); + queryClient.invalidateQueries({ queryKey: ["me"] }); + queryClient.invalidateQueries({ queryKey: ["payment-methods"] }); + }, [queryClient]); + + // Start adding a card + const handleAddCard = async () => { + try { + const { client_secret } = await createSetupIntent(); + setSetupClientSecret(client_secret); + setShowAddCard(true); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to start card setup"); + } + }; + + // Purchase credits via Stripe + const purchaseMutation = useMutation({ + mutationFn: async (credits: number) => { + const result = await purchaseCredits(credits); + if (result.status === "requires_action" && result.client_secret) { + // Handle 3D Secure + const stripe = await stripePromise; + if (!stripe) throw new Error("Stripe not loaded"); + const { error } = await stripe.confirmPayment({ + clientSecret: result.client_secret, + confirmParams: { return_url: window.location.href }, + redirect: "if_required", + }); + if (error) throw new Error(error.message ?? "Payment authentication failed"); + return { ...result, status: "succeeded" as const, message: `${credits.toLocaleString()} credits added to your account` }; + } + return result; + }, + onSuccess: (data) => { + toast.success(data.message); + invalidateBilling(); + setPurchasingPack(null); + }, + onError: (error: Error) => { + toast.error(error.message); + setPurchasingPack(null); + }, + }); - // Hit the portal endpoint on click rather than on mount. This keeps the - // session URL fresh (Hyperline URLs are short-lived) and avoids calling - // the upstream unnecessarily on page loads. - const portalMutation = useMutation({ - mutationFn: fetchBillingPortal, + // Fallback topup (no Stripe, for dev/free) + const topupMutation = useMutation({ + mutationFn: (amount: number) => topupCredits(amount), onSuccess: (data) => { - // Full-page redirect — Hyperline portal is not iframe-safe. - window.location.href = data.url; + toast.success(data.message); + invalidateBilling(); }, onError: (error: Error) => { - toast.error(error.message || "Failed to open billing portal"); + toast.error(error.message); + }, + }); + + const deletePmMutation = useMutation({ + mutationFn: (id: string) => deletePaymentMethod(id), + onSuccess: () => { + toast.success("Card removed"); + queryClient.invalidateQueries({ queryKey: ["payment-methods"] }); + }, + onError: (error: Error) => { + toast.error(error.message); + }, + }); + + const setDefaultPmMutation = useMutation({ + mutationFn: (id: string) => setDefaultPaymentMethod(id), + onSuccess: () => { + toast.success("Default card updated"); + queryClient.invalidateQueries({ queryKey: ["payment-methods"] }); + }, + onError: (error: Error) => { + toast.error(error.message); + }, + }); + + const autoTopupMutation = useMutation({ + mutationFn: (config: { enabled: boolean; amount?: number; threshold?: number }) => + updateAutoTopup(config), + onSuccess: () => { + toast.success("Auto top-up settings updated"); + queryClient.invalidateQueries({ queryKey: ["billing"] }); + }, + onError: (error: Error) => { + toast.error(error.message); }, }); @@ -194,10 +382,18 @@ export default function BillingPage() { } const creditsBalance = billing?.credits_balance ?? user?.account?.credits_balance ?? 0; - const hyperlineLinked = !!billing?.hyperline_customer_id; - const walletBalance = billing?.hyperline_wallet_balance ?? null; - const walletCurrency = billing?.hyperline_wallet_currency ?? null; - const pmStatus = billing?.payment_method_status ?? null; + + const handlePurchase = (credits: number) => { + if (hasStripe && hasPaymentMethod) { + setPurchasingPack(credits); + purchaseMutation.mutate(credits); + } else if (hasStripe && !hasPaymentMethod) { + toast.error("Please add a payment method first"); + } else { + // Fallback: free topup (dev mode) + topupMutation.mutate(credits); + } + }; return (
    @@ -208,42 +404,6 @@ export default function BillingPage() {

    - {/* Payment method status banner — surfaced from Hyperline - payment_method.errored / payment_method.expired webhooks. */} - {pmStatus && ( - - - -
    -

    - {pmStatus === "expired" - ? "Your payment method has expired" - : "Your payment method is not working"} -

    -

    - Update your card on the billing portal to avoid service - interruption when your next top-up runs. -

    -
    - {isOwner && hyperlineLinked && ( - - )} -     -     - )} - {/* Credits Balance */} @@ -384,127 +544,545 @@ export default function BillingPage() { - +
    {Number(creditsBalance).toLocaleString()} credits remaining
    - {/* Hyperline wallet mirror: the account's cash wallet on the - billing provider. Shown when the account is linked and the - live balance read succeeded. */} - {hyperlineLinked && walletBalance !== null && ( -
    -
    - Wallet balance - (on Hyperline) -
    -
    - {formatMoney(walletBalance, walletCurrency)} -
    -
    - )}     - {/* Hyperline hosted portal — single entry point for card management, - invoices, auto-recharge, and one-off top-ups. Owners only. */} - {isOwner && ( + {/* Payment Methods */} + {hasStripe && ( - - - Payment methods & invoices - - - Manage your card, view and download invoices, configure - auto-recharge, and buy credits on the hosted billing portal. - +
    +
    + + + Payment Methods + + + Manage your cards for credit purchases and auto top-up + +
    + {!showAddCard && isOwner && ( + + )} +
    - {hyperlineLinked ? ( - - ) : ( -

    - Your account isn't linked to the billing provider yet. - Contact support if you need to purchase credits. + { + setShowAddCard(false); + setSetupClientSecret(null); + queryClient.invalidateQueries({ queryKey: ["payment-methods"] }); + }} + onCancel={() => { + setShowAddCard(false); + setSetupClientSecret(null); + }} + /> + + ) : pmLoading ? ( +

    + + +
    + ) : !paymentMethods?.length ? ( +

    + No payment methods yet. Add a card to purchase credits.

    + ) : ( +
    + {paymentMethods.map((pm) => ( +
    +
    + +
    +
    + + {cardBrandName(pm.brand)} ending in {pm.last4} + + {pm.is_default && ( + + Default + + )} +
    + + Expires {pm.exp_month}/{pm.exp_year} + +
    +
    +
    + {isOwner && !pm.is_default && ( + + )} + {isOwner && ( + + )} +
    +
    + ))} +
    )}     )} - {/* Monthly Spend Limit — owner only. Kept local: no equivalent - Hyperline primitive, and we want the hard stop to live in our - ledger rather than round-tripping to the provider. */} - {isOwner && ( - - - Monthly Spend Limit - - Set a maximum amount of credits that can be added per month via top-ups. - Auto top-up will stop when this limit is reached. - - - -
    + {/* Buy Credits — owner only */} + {isOwner && <> + +
    +
    + + + {hasStripe ? "Buy Credits" : "Add Credits"} + + + {hasStripe + ? "Purchase credits with your saved payment method" + : "Add credits to your account"} + +
    + + + + + + + Volume pricing + + The more credits you buy, the less you pay per credit. + + +
    +
    + + + + Volume + Per credit + Per 1K + + + + {PRICING_TIERS.map((tier, i) => ( + + + {tier.upTo ? `1 – ${tier.upTo.toLocaleString()}` : "10,000+"} + + + ${tier.rate.toFixed(3)} + + + {tier.per1k} + + + ))} + +
    +
    +

    + Volume-based pricing: the entire purchase is priced at the tier rate for the total quantity. +

    +
    +     +     +
    +     + + {/* Preset amounts + custom */} +
    + {QUICK_AMOUNTS.map((amt) => { + const price = calculatePrice(amt); + const isSelected = purchaseAmount === String(amt); + return ( + + ); + })} + +
    + + {/* Custom amount input (always visible but highlighted when Custom is selected) */} + {!QUICK_AMOUNTS.includes(parseInt(purchaseAmount)) && ( +
    + setSpendLimit(e.target.value)} - className="max-w-[200px]" + min={100} + step={100} + value={purchaseAmount} + onChange={(e) => setPurchaseAmount(e.target.value)} + placeholder="Enter number of credits (min 100)" /> - credits / month +
    + )} + + {/* Order summary */} + {(() => { + const amt = parseInt(purchaseAmount) || 0; + if (amt < 100) return null; + const price = calculatePrice(amt); + const per1k = (price / amt) * 1000; + return ( +
    +
    +
    +
    + {amt.toLocaleString()} credits × ${(price / amt).toFixed(3)}/credit +
    +
    + ${per1k.toFixed(2)} per 1,000 credits +
    +
    +
    +
    ${price.toFixed(2)}
    +
    +
    +
    + +
    +
    + ); + })()} + + {hasStripe && !hasPaymentMethod && ( +

    + Add a payment method above to purchase credits. +

    + )} +     +     + + {/* Purchase confirmation dialog */} + + + + Confirm purchase + + You are about to purchase credits. Your default card will be charged. + + + {(() => { + const amt = parseInt(purchaseAmount) || 0; + const price = calculatePrice(amt); + return ( +
    +
    + Credits + {amt.toLocaleString()} +
    +
    + Rate + ${(price / amt).toFixed(3)} / credit +
    +
    + Total + ${price.toFixed(2)} +
    +
    + ); + })()} + + + + +     +     } + + {/* Auto Top-up — owner only */} + {isOwner && + + + + + Auto Top-up + + + Automatically add credits when your balance drops below a threshold + {hasStripe && hasPaymentMethod && " — charges your default card"} + + + +
    + { + const amount = autoTopupAmount ? parseInt(autoTopupAmount) : billing?.auto_topup_amount; + const threshold = autoTopupThreshold ? parseInt(autoTopupThreshold) : billing?.auto_topup_threshold; + autoTopupMutation.mutate({ enabled, amount, threshold }); + }} + /> + +
    + {billing?.auto_topup_enabled && ( +
    +
    + +
    + setAutoTopupAmount(e.target.value)} + /> + +
    +

    + Currently: {billing.auto_topup_amount.toLocaleString()} credits +

    +
    +
    + +
    + setAutoTopupThreshold(e.target.value)} + /> + +
    +

    + Currently: {billing.auto_topup_threshold.toLocaleString()} credits +

    +
    +
    + )} +     +     } + + {/* Monthly Spend Limit — owner only */} + {isOwner && + + Monthly Spend Limit + + Set a maximum amount of credits that can be added per month via top-ups. + Auto top-up will stop when this limit is reached. + + + +
    + setSpendLimit(e.target.value)} + className="max-w-[200px]" + /> + credits / month + + {billing?.monthly_spend_limit && ( - {billing?.monthly_spend_limit && ( - - )} -
    - {billing?.monthly_spend_limit && ( -

    - Current limit: {billing.monthly_spend_limit.toLocaleString()} credits / month -

    )} -     -     - )} +
    + {billing?.monthly_spend_limit && ( +

    + Current limit: {billing.monthly_spend_limit.toLocaleString()} credits / month +

    + )} +     +     } {/* Daily Cost Chart */} {dailyCostData.length > 0 && ( @@ -573,9 +1151,7 @@ export default function BillingPage() { className={ tx.amount > 0 ? "text-green-600 dark:text-green-400" - : tx.amount < 0 - ? "text-red-600 dark:text-red-400" - : "text-muted-foreground" + : "text-red-600 dark:text-red-400" } > {tx.amount > 0 ? "+" : ""} @@ -599,7 +1175,7 @@ export default function BillingPage() { {txData.total > TX_PAGE_SIZE && (

    - Showing {txOffset + 1}–{Math.min(txOffset + TX_PAGE_SIZE, txData.total)} of{" "} + Showing {txOffset + 1}\u2013{Math.min(txOffset + TX_PAGE_SIZE, txData.total)} of{" "} {txData.total}

    @@ -626,6 +1202,94 @@ export default function BillingPage() { )} + + {/* Invoices */} + {hasStripe && ( + + + + + Invoices + + + Download invoices and receipts + + + + {invoicesLoading ? ( +
    + {Array.from({ length: 3 }).map((_, i) => ( + + ))} +
    + ) : !invoices?.length ? ( +

    + No invoices yet +

    + ) : ( + + + + Invoice + Date + Description + Amount + Credits + PDF + + + + {invoices.map((inv) => ( + + + {inv.number ?? "—"} + + + {format(new Date(inv.created_at), "MMM d, yyyy")} + + + {inv.description || "Credit purchase"} + + + ${(inv.amount_cents / 100).toFixed(2)} + + + {inv.credits?.toLocaleString() ?? "—"} + + +
    + {inv.invoice_pdf && ( + + + + )} + {inv.hosted_invoice_url && ( + + + + )} +
    +     +     + ))} +     +
    + )} +     +     + )}
    ); } diff --git a/console/src/lib/api-types.ts b/console/src/lib/api-types.ts index 6dff1fd9..6c15a3d9 100644 --- a/console/src/lib/api-types.ts +++ b/console/src/lib/api-types.ts @@ -268,20 +268,12 @@ export interface MapResult { export interface BillingInfo { tier: string; + stripe_customer_id: string | null; credits_balance: number; + auto_topup_enabled: boolean; + auto_topup_amount: number; + auto_topup_threshold: number; monthly_spend_limit: number | null; - // Hyperline handles — present once the account is linked via the - // post-signup provisioning path. - hyperline_customer_id: string | null; - hyperline_wallet_id: string | null; - // Live Hyperline wallet balance in the smallest currency unit (cents - // for USD). Null when the account isn't linked yet or the live read - // failed; the UI should fall back to `credits_balance` in that case. - hyperline_wallet_balance: number | null; - hyperline_wallet_currency: string | null; - // Set by the Hyperline payment_method.* webhooks. Drives the - // "update your card" banner. - payment_method_status: "errored" | "expired" | null; } export interface Transaction { @@ -298,15 +290,52 @@ export interface TransactionsListResponse { total: number; } +export interface TopupResponse { + credits_balance: number; + transaction_id: string; + message: string; +} + // ============================================================================ -// Hyperline hosted portal +// Stripe / Payment Methods // ============================================================================ -/// Response from `GET /account/billing/portal`. Redirect the user to this -/// URL to let them manage cards, invoices, and auto-recharge on the -/// Hyperline hosted portal. -export interface PortalResponse { - url: string; +export interface SetupIntentResponse { + client_secret: string; +} + +export interface PaymentMethodInfo { + id: string; + brand: string | null; + last4: string | null; + exp_month: number | null; + exp_year: number | null; + is_default: boolean; +} + +export interface PurchaseCreditsRequest { + credits: number; + payment_method_id?: string; +} + +export interface PurchaseResponse { + status: "succeeded" | "requires_action"; + client_secret: string | null; + credits: number; + amount_cents: number; + message: string; +} + +export interface InvoiceInfo { + id: string; + number: string | null; + amount_cents: number; + credits: number | null; + status: string; + description: string | null; + created_at: string; + invoice_pdf: string | null; + hosted_invoice_url: string | null; } // ============================================================================ diff --git a/console/src/lib/api.ts b/console/src/lib/api.ts index f2232e8a..17c742cc 100644 --- a/console/src/lib/api.ts +++ b/console/src/lib/api.ts @@ -26,7 +26,11 @@ import type { TopDomainRow, BillingInfo, TransactionsListResponse, - PortalResponse, + TopupResponse, + SetupIntentResponse, + PaymentMethodInfo, + PurchaseResponse, + InvoiceInfo, AccountListItem, MemberInfo, InviteInfo, @@ -366,6 +370,26 @@ export async function fetchBilling(): Promise { return request("/account/billing"); } +export async function topupCredits(amount: number): Promise { + return request("/account/billing/topup", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ amount }), + }); +} + +export async function updateAutoTopup(config: { + enabled: boolean; + amount?: number; + threshold?: number; +}): Promise { + await request("/account/billing/auto-topup", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(config), + }); +} + export async function updateSpendLimit(monthly_spend_limit: number | null): Promise { await request("/account/billing/spend-limit", { method: "PATCH", @@ -379,23 +403,39 @@ export async function fetchTransactions(limit: number = 50, offset: number = 0): } // ============================================================================ -// Hyperline hosted portal +// Stripe / Payment Methods // ============================================================================ -// Card management, invoice history, auto-recharge configuration, and -// one-off top-ups all live on the Hyperline hosted portal. The console -// exchanges for the portal URL via this endpoint, then redirects. -// -// Replaces the pre-Hyperline set of endpoints: -// - POST /account/billing/topup (one-off top-up) -// - PATCH /account/billing/auto-topup (auto-recharge config) -// - POST /account/billing/setup-intent (add payment method UI) -// - GET /account/billing/payment-methods (list cards) -// - DELETE /account/billing/payment-methods/:id (remove card) -// - POST /account/billing/purchase (charge + credit) -// - GET /account/billing/invoices (history) -export async function fetchBillingPortal(): Promise { - return request("/account/billing/portal"); +export async function createSetupIntent(): Promise { + return request("/account/billing/setup-intent", { method: "POST" }); +} + +export async function fetchPaymentMethods(): Promise { + return request("/account/billing/payment-methods"); +} + +export async function deletePaymentMethod(id: string): Promise { + await request(`/account/billing/payment-methods/${id}`, { method: "DELETE" }); +} + +export async function setDefaultPaymentMethod(paymentMethodId: string): Promise { + await request("/account/billing/default-payment-method", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ payment_method_id: paymentMethodId }), + }); +} + +export async function purchaseCredits(credits: number, paymentMethodId?: string): Promise { + return request("/account/billing/purchase", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ credits, payment_method_id: paymentMethodId }), + }); +} + +export async function fetchInvoices(): Promise { + return request("/account/billing/invoices"); } // ============================================================================ diff --git a/console/src/lib/hooks.ts b/console/src/lib/hooks.ts index 6a8b632c..6c918a1b 100644 --- a/console/src/lib/hooks.ts +++ b/console/src/lib/hooks.ts @@ -1,6 +1,6 @@ import { useQuery } from "@tanstack/react-query"; import { getMe } from "./auth"; -import { fetchApiKeys, fetchServiceHealth, fetchBilling, fetchTransactions, fetchMyAccounts, fetchMembers, fetchInvites } from "./api"; +import { fetchApiKeys, fetchServiceHealth, fetchBilling, fetchTransactions, fetchPaymentMethods, fetchInvoices, fetchMyAccounts, fetchMembers, fetchInvites } from "./api"; export function useMe() { return useQuery({ @@ -52,6 +52,20 @@ export function useAllTransactions() { }); } +export function usePaymentMethods() { + return useQuery({ + queryKey: ["payment-methods"], + queryFn: fetchPaymentMethods, + }); +} + +export function useInvoices() { + return useQuery({ + queryKey: ["invoices"], + queryFn: fetchInvoices, + }); +} + export function useMyAccounts() { return useQuery({ queryKey: ["my-accounts"], diff --git a/crates/scrapix-billing-hyperline/Cargo.toml b/crates/scrapix-billing-hyperline/Cargo.toml deleted file mode 100644 index 7823d6a7..00000000 --- a/crates/scrapix-billing-hyperline/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "scrapix-billing-hyperline" -description = "Hyperline billing provider: typed client, webhook verification, usage event outbox" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true - -[dependencies] -scrapix-core = { path = "../scrapix-core" } - -reqwest = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -thiserror = { workspace = true } -async-trait = { workspace = true } -url = { workspace = true } -uuid = { workspace = true } -chrono = { workspace = true } -sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "uuid", "chrono", "json", "tls-rustls"] } - -# Webhook signature verification (HMAC-SHA256) -hmac = "0.12" -sha2 = { workspace = true } -base64 = { workspace = true } -subtle = "2.6" - -[dev-dependencies] -tokio = { workspace = true } -tracing-subscriber = { workspace = true } - -[[example]] -name = "seed_hyperline_products" -path = "examples/seed_hyperline_products.rs" diff --git a/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs b/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs deleted file mode 100644 index 7c0a424b..00000000 --- a/crates/scrapix-billing-hyperline/examples/seed_hyperline_products.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! Idempotent seed script: creates one `dynamic` product per -//! [`BillingEventType`] in the Hyperline workspace configured via -//! `HYPERLINE_API_KEY` + `HYPERLINE_API_BASE`. -//! -//! Each product is a `dynamic` billing primitive whose aggregator sums the -//! `credits` property across incoming events of the matching `event_type`: -//! -//! ```json -//! { -//! "type": "dynamic", -//! "name": "Page crawled", -//! "unit_name": "credit", -//! "aggregator": { -//! "entity": "page_crawled", -//! "operation": "sum", -//! "property": "credits", -//! "type": "metered" -//! } -//! } -//! ``` -//! -//! Idempotency: we first `GET /v1/products?name__equals=` and -//! skip creation if any match is returned. Hyperline does not expose an -//! `external_id` on products, so name-equality is the only stable key we have. -//! -//! Usage: -//! -//! ```sh -//! HYPERLINE_API_KEY=test_… cargo run -p scrapix-billing-hyperline \ -//! --example seed_hyperline_products -//! ``` - -use std::error::Error; - -use scrapix_billing_hyperline::client::HyperlineClient; -use scrapix_billing_hyperline::events::BillingEventType; -use serde::Deserialize; -use serde_json::json; - -/// Display name + slug used for the Hyperline product backing a given event. -fn product_for(ev: BillingEventType) -> (&'static str, &'static str) { - match ev { - BillingEventType::PageCrawled => ("Page crawled", "page_crawled"), - BillingEventType::BytesDownloaded => ("Bytes downloaded", "bytes_downloaded"), - BillingEventType::JsRender => ("JS render", "js_render"), - BillingEventType::ApiRequest => ("API request", "api_request"), - BillingEventType::DocumentIndexed => ("Document indexed", "document_indexed"), - BillingEventType::FeatureFormat => ("Feature format", "feature_format"), - BillingEventType::AiFeature => ("AI feature", "ai_feature"), - } -} - -#[derive(Debug, Deserialize)] -struct ProductSummary { - id: String, - #[serde(default)] - name: Option, -} - -#[derive(Debug, Deserialize)] -struct ProductList { - data: Vec, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - - let client = HyperlineClient::from_env()?; - let target = if client.config().is_sandbox() { - "sandbox" - } else { - "PRODUCTION" - }; - println!("Seeding Hyperline products into {target}…"); - - let mut created = 0usize; - let mut skipped = 0usize; - - for ev in BillingEventType::all().iter().copied() { - let (name, slug) = product_for(ev); - - // 1. Look up existing products by exact name. - let existing: ProductList = client - .get_json( - "/v1/products", - &[("name__equals", name), ("status", "active")], - ) - .await?; - if let Some(p) = existing.data.first() { - println!( - " [=] {slug:<18} exists ({}{})", - p.id, - p.name - .as_deref() - .map(|n| format!(" — \"{n}\"")) - .unwrap_or_default() - ); - skipped += 1; - continue; - } - - // 2. Create the dynamic product. `unit_name` is shown on invoices. - // The `price_configurations` array is mandatory; we seed a single - // zero-amount volume tier (USD, monthly, 0 → ∞) so the product is - // valid in the dashboard but doesn't actually bill anything. Real - // pricing is set up by ops in the Hyperline UI post-cutover. - let body = json!({ - "type": "dynamic", - "name": name, - "description": format!("Scrapix billable event: {slug}"), - "unit_name": "credit", - "is_available_on_demand": true, - "is_available_on_subscription": true, - "aggregator": { - "entity": slug, - "operation": "sum", - "property": "credits", - "type": "metered", - }, - "price_configurations": [ - { - "currency": "USD", - "billing_interval": { "period": "months", "count": 1 }, - "type": "volume", - "prices": [ - { - "type": "volume", - "amount": 0, - "from": 0, - "to": null, - }, - ], - }, - ], - }); - - let created_product: ProductSummary = client.post_json("/v1/products", &body).await?; - println!(" [+] {slug:<18} created ({})", created_product.id); - created += 1; - } - - println!("Done. created={created} skipped={skipped}"); - Ok(()) -} diff --git a/crates/scrapix-billing-hyperline/src/client.rs b/crates/scrapix-billing-hyperline/src/client.rs deleted file mode 100644 index 404f81f6..00000000 --- a/crates/scrapix-billing-hyperline/src/client.rs +++ /dev/null @@ -1,248 +0,0 @@ -use std::time::Duration; - -use reqwest::header; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tracing::debug; -use url::Url; - -use crate::config::HyperlineConfig; -use crate::error::HyperlineError; -use crate::events::UsageEvent; - -/// Envelope returned by Hyperline list endpoints: `{ meta, data }`. -#[derive(Debug, Deserialize)] -pub struct ListResponse { - pub meta: ListMeta, - pub data: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct ListMeta { - pub total: u64, - pub taken: u64, - pub skipped: u64, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Customer { - pub id: String, - #[serde(default)] - pub external_id: Option, - #[serde(default)] - pub name: Option, - #[serde(default)] - pub email: Option, -} - -/// Hyperline wallet — response shape for `GET /v1/wallets/{id}`. -/// -/// `balance.amount` and `projected_balance.amount` are integers in the -/// smallest currency unit (cents for USD). `projected_balance` is -/// Hyperline's forecast given active subscriptions; the current spendable -/// balance is `balance.amount`. -#[derive(Debug, Deserialize, Serialize)] -pub struct Wallet { - pub id: String, - pub customer_id: String, - #[serde(default)] - pub state: Option, - #[serde(default)] - pub currency: Option, - pub balance: MoneyAmount, - #[serde(default)] - pub projected_balance: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MoneyAmount { - pub amount: i64, -} - -/// Customer portal response shape for `GET /v1/customers/{id}/portal`. -/// -/// The URL embeds a signed JWT (observed in sandbox: HS256 with a -/// `type=portal,customerId,iat` payload), so **do not cache it** — treat -/// each call as minting a fresh session token. The console hits this -/// endpoint on click, not on page load, for exactly this reason. -#[derive(Debug, Deserialize, Serialize)] -pub struct PortalLink { - pub url: String, -} - -#[derive(Debug, Clone)] -pub struct HyperlineClient { - http: reqwest::Client, - config: HyperlineConfig, -} - -impl HyperlineClient { - pub fn new(config: HyperlineConfig) -> Result { - let mut headers = header::HeaderMap::new(); - let auth_value = format!("Bearer {}", config.api_key); - let mut auth = header::HeaderValue::from_str(&auth_value) - .map_err(|e| HyperlineError::InvalidConfig(format!("invalid api key: {e}")))?; - auth.set_sensitive(true); - headers.insert(header::AUTHORIZATION, auth); - headers.insert( - header::ACCEPT, - header::HeaderValue::from_static("application/json"), - ); - - let http = reqwest::Client::builder() - .default_headers(headers) - .timeout(Duration::from_secs(15)) - .build()?; - - Ok(Self { http, config }) - } - - pub fn from_env() -> Result { - Self::new(HyperlineConfig::from_env()?) - } - - pub fn config(&self) -> &HyperlineConfig { - &self.config - } - - fn api_url(&self, path: &str) -> Result { - let trimmed = path.trim_start_matches('/'); - Ok(self.config.api_base.join(trimmed)?) - } - - fn ingest_url(&self, path: &str) -> Result { - let trimmed = path.trim_start_matches('/'); - Ok(self.config.ingest_base.join(trimmed)?) - } - - pub async fn get_json( - &self, - path: &str, - query: &[(&str, &str)], - ) -> Result { - let url = self.api_url(path)?; - debug!(%url, "hyperline GET"); - let resp = self.http.get(url).query(query).send().await?; - parse::(resp).await - } - - /// POSTs `body` as JSON to `path` on the control-plane API (not the ingest - /// host) and deserializes the response. Used by seed/admin tooling. - pub async fn post_json( - &self, - path: &str, - body: &B, - ) -> Result { - let url = self.api_url(path)?; - debug!(%url, "hyperline POST"); - let resp = self.http.post(url).json(body).send().await?; - parse::(resp).await - } - - /// POSTs a single billable event to `/v1/events`. This is the endpoint - /// the outbox drain worker uses — one row → one request. Dedupe is keyed - /// on `event.record.id` (the outbox UUID), so retries are safe. - pub async fn ingest_event(&self, event: &UsageEvent<'_>) -> Result<(), HyperlineError> { - let url = self.ingest_url("/v1/events")?; - debug!(%url, "hyperline ingest one"); - let resp = self.http.post(url).json(event).send().await?; - Self::check_ok(resp).await - } - - /// POSTs a batch of billable events to `/v1/events/batch` (max 5000 - /// per call, per Hyperline's schema). Retained for ops/replay tooling. - pub async fn ingest_events_batch( - &self, - events: &[UsageEvent<'_>], - ) -> Result<(), HyperlineError> { - if events.is_empty() { - return Ok(()); - } - if events.len() > 5000 { - return Err(HyperlineError::InvalidConfig(format!( - "batch too large: {} > 5000", - events.len() - ))); - } - let url = self.ingest_url("/v1/events/batch")?; - debug!(%url, count = events.len(), "hyperline ingest batch"); - let resp = self.http.post(url).json(events).send().await?; - Self::check_ok(resp).await - } - - async fn check_ok(resp: reqwest::Response) -> Result<(), HyperlineError> { - let status = resp.status(); - if status.is_success() { - return Ok(()); - } - let bytes = resp.bytes().await?; - let message = std::str::from_utf8(&bytes) - .unwrap_or("") - .to_string(); - Err(HyperlineError::Api { - status: status.as_u16(), - message: truncate(&message, 512), - }) - } - - /// Lists customers. Useful as a connectivity check. - pub async fn list_customers( - &self, - limit: u32, - ) -> Result, HyperlineError> { - let limit_str = limit.to_string(); - self.get_json("/v1/customers", &[("limit", &limit_str)]) - .await - } - - /// Lightweight liveness probe — `GET /v1/customers?limit=1`. - pub async fn ping(&self) -> Result<(), HyperlineError> { - let _ = self.list_customers(1).await?; - Ok(()) - } - - /// Fetches a wallet by its Hyperline id. - /// - /// Wallets are not auto-provisioned — if the customer has no wallet - /// yet the call 404s. Callers should treat a NotFound-style `Api` - /// error as "no wallet yet" rather than a hard failure. - pub async fn get_wallet(&self, wallet_id: &str) -> Result { - let path = format!("/v1/wallets/{wallet_id}"); - self.get_json(&path, &[]).await - } - - /// Fetches the hosted customer-portal URL. - /// - /// Unlike Stripe's portal sessions this is a GET, but the returned - /// URL embeds a signed JWT (observed in sandbox) so the link is - /// session-scoped, not a permalink. Callers should hit this on-demand - /// (e.g. on button click) and redirect the browser immediately - /// rather than caching the URL. - pub async fn get_portal_url(&self, customer_id: &str) -> Result { - let path = format!("/v1/customers/{customer_id}/portal"); - self.get_json(&path, &[]).await - } -} - -async fn parse(resp: reqwest::Response) -> Result { - let status = resp.status(); - let bytes = resp.bytes().await?; - if status.is_success() { - let value: T = serde_json::from_slice(&bytes)?; - return Ok(value); - } - let message = std::str::from_utf8(&bytes) - .unwrap_or("") - .to_string(); - Err(HyperlineError::Api { - status: status.as_u16(), - message: truncate(&message, 512), - }) -} - -fn truncate(s: &str, max: usize) -> String { - if s.len() <= max { - s.to_string() - } else { - format!("{}…", &s[..max]) - } -} diff --git a/crates/scrapix-billing-hyperline/src/config.rs b/crates/scrapix-billing-hyperline/src/config.rs deleted file mode 100644 index faf74cd2..00000000 --- a/crates/scrapix-billing-hyperline/src/config.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::env; - -use url::Url; - -use crate::error::HyperlineError; - -pub const DEFAULT_API_BASE: &str = "https://api.hyperline.co"; -pub const DEFAULT_INGEST_BASE: &str = "https://ingest.hyperline.co"; -pub const SANDBOX_API_BASE: &str = "https://sandbox.api.hyperline.co"; -pub const SANDBOX_INGEST_BASE: &str = "https://sandbox.ingest.hyperline.co"; - -#[derive(Debug, Clone)] -pub struct HyperlineConfig { - pub api_key: String, - pub api_base: Url, - pub ingest_base: Url, - pub webhook_secret: Option, -} - -impl HyperlineConfig { - /// Load configuration from environment variables: - /// - `HYPERLINE_API_KEY` (required) — prefixed `test_` or `prod_`. - /// - `HYPERLINE_API_BASE` (optional) — defaults to sandbox for `test_` keys, - /// production for `prod_` keys. - /// - `HYPERLINE_INGEST_BASE` (optional) — same default rule. - /// - `HYPERLINE_WEBHOOK_SECRET` (optional) — required to verify webhooks. - pub fn from_env() -> Result { - let api_key = env::var("HYPERLINE_API_KEY") - .map_err(|_| HyperlineError::MissingEnv("HYPERLINE_API_KEY"))?; - - let is_sandbox = api_key.starts_with("test_"); - let default_api = if is_sandbox { - SANDBOX_API_BASE - } else { - DEFAULT_API_BASE - }; - let default_ingest = if is_sandbox { - SANDBOX_INGEST_BASE - } else { - DEFAULT_INGEST_BASE - }; - - let api_base = env::var("HYPERLINE_API_BASE").unwrap_or_else(|_| default_api.into()); - let ingest_base = - env::var("HYPERLINE_INGEST_BASE").unwrap_or_else(|_| default_ingest.into()); - - Ok(Self { - api_key, - api_base: Url::parse(&api_base)?, - ingest_base: Url::parse(&ingest_base)?, - webhook_secret: env::var("HYPERLINE_WEBHOOK_SECRET").ok(), - }) - } - - pub fn is_sandbox(&self) -> bool { - self.api_key.starts_with("test_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sandbox_defaults_from_test_prefix() { - // Guard against a developer having the var set in their shell. - let prev = env::var("HYPERLINE_API_KEY").ok(); - let prev_base = env::var("HYPERLINE_API_BASE").ok(); - - // SAFETY: tests may mutate process env; we restore at the end. - unsafe { - env::set_var("HYPERLINE_API_KEY", "test_abc"); - env::remove_var("HYPERLINE_API_BASE"); - env::remove_var("HYPERLINE_INGEST_BASE"); - } - - let cfg = HyperlineConfig::from_env().unwrap(); - assert!(cfg.is_sandbox()); - assert_eq!(cfg.api_base.as_str(), "https://sandbox.api.hyperline.co/"); - assert_eq!( - cfg.ingest_base.as_str(), - "https://sandbox.ingest.hyperline.co/" - ); - - // Restore. - unsafe { - match prev { - Some(v) => env::set_var("HYPERLINE_API_KEY", v), - None => env::remove_var("HYPERLINE_API_KEY"), - } - if let Some(v) = prev_base { - env::set_var("HYPERLINE_API_BASE", v); - } - } - } -} diff --git a/crates/scrapix-billing-hyperline/src/error.rs b/crates/scrapix-billing-hyperline/src/error.rs deleted file mode 100644 index 9f5e284b..00000000 --- a/crates/scrapix-billing-hyperline/src/error.rs +++ /dev/null @@ -1,34 +0,0 @@ -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum HyperlineError { - #[error("missing environment variable: {0}")] - MissingEnv(&'static str), - - #[error("invalid configuration: {0}")] - InvalidConfig(String), - - #[error("http transport error: {0}")] - Http(#[from] reqwest::Error), - - #[error("url parse error: {0}")] - Url(#[from] url::ParseError), - - #[error("hyperline api error: {status} {message}")] - Api { status: u16, message: String }, - - #[error("decode error: {0}")] - Decode(#[from] serde_json::Error), - - #[error("database error: {0}")] - Database(#[from] sqlx::Error), - - #[error("invalid webhook signature")] - InvalidSignature, - - #[error("webhook timestamp outside 5-minute tolerance")] - StaleTimestamp, - - #[error("malformed webhook header: {0}")] - MalformedHeader(&'static str), -} diff --git a/crates/scrapix-billing-hyperline/src/events.rs b/crates/scrapix-billing-hyperline/src/events.rs deleted file mode 100644 index 7f212484..00000000 --- a/crates/scrapix-billing-hyperline/src/events.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Usage-event catalog and outbox enqueue helper. -//! -//! Events are written to `hyperline_events_outbox` in the same Postgres -//! transaction as the credit-ledger debit, so a debited credit always has a -//! matching outbox row. A background drain worker (see `outbox.rs`) POSTs the -//! rows to Hyperline and uses the outbox row's UUID as the `record.id` — -//! giving us free idempotency across retries. -//! -//! Wire format matches Hyperline's `BillableEvent` schema: -//! ```json -//! { -//! "customer_id": "cus_…", -//! "event_type": "api_request", -//! "timestamp": "2026-04-17T…Z", -//! "record": { "id": "", "credits": 5, …extra scalars } -//! } -//! ``` -//! `record.id` is the dedupe key on Hyperline's side; arbitrary scalars in -//! `record` become aggregatable properties (e.g. `sum(record.credits)` for -//! a dynamic product). - -use serde::{Deserialize, Serialize}; -use sqlx::PgExecutor; -use uuid::Uuid; - -use crate::error::HyperlineError; - -/// Event types mirroring the existing credit-cost rules in -/// `crates/scrapix-billing/src/credits.rs`. One variant per rule so Hyperline -/// can price each independently without us redeploying. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BillingEventType { - PageCrawled, - BytesDownloaded, - JsRender, - ApiRequest, - DocumentIndexed, - FeatureFormat, - AiFeature, -} - -impl BillingEventType { - pub fn as_str(&self) -> &'static str { - match self { - Self::PageCrawled => "page_crawled", - Self::BytesDownloaded => "bytes_downloaded", - Self::JsRender => "js_render", - Self::ApiRequest => "api_request", - Self::DocumentIndexed => "document_indexed", - Self::FeatureFormat => "feature_format", - Self::AiFeature => "ai_feature", - } - } - - /// All variants, for bootstrap/seed tooling. - pub fn all() -> &'static [Self] { - &[ - Self::PageCrawled, - Self::BytesDownloaded, - Self::JsRender, - Self::ApiRequest, - Self::DocumentIndexed, - Self::FeatureFormat, - Self::AiFeature, - ] - } -} - -/// Wire shape posted to Hyperline's ingest API. `record` is a pre-built JSON -/// object containing `id` (dedupe key), `credits` (aggregator property), and -/// any flattened metadata scalars — see [`build_record`]. -#[derive(Debug, Serialize)] -pub struct UsageEvent<'a> { - pub customer_id: &'a str, - pub event_type: &'a str, - pub timestamp: chrono::DateTime, - pub record: serde_json::Value, -} - -/// Build the `record` JSON object for a `UsageEvent`. -/// -/// - `id` becomes Hyperline's dedupe key (`record.id`). -/// - `credits` is exposed as an aggregatable numeric property. -/// - Any scalar keys in `metadata` (strings, numbers, booleans, nulls) are -/// flattened into the record. Non-scalar values are dropped to stay within -/// the `BillableEvent.record` `additionalProperties` shape accepted by -/// Hyperline. Existing keys (`id`, `credits`) are never overwritten. -pub fn build_record( - id: &str, - credits: f64, - metadata: Option<&serde_json::Value>, -) -> serde_json::Value { - let mut obj = serde_json::Map::new(); - obj.insert("id".to_string(), serde_json::Value::String(id.to_string())); - obj.insert("credits".to_string(), serde_json::Value::from(credits)); - if let Some(serde_json::Value::Object(meta)) = metadata { - for (k, v) in meta { - if obj.contains_key(k) { - continue; - } - if v.is_string() || v.is_number() || v.is_boolean() || v.is_null() { - obj.insert(k.clone(), v.clone()); - } - } - } - serde_json::Value::Object(obj) -} - -/// Inline payload persisted in the outbox. The drain worker reads this back -/// and builds a `UsageEvent.record` at send time via [`build_record`]. -#[derive(Debug, Serialize, Deserialize)] -pub struct OutboxPayload { - pub quantity: f64, - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata: Option, -} - -/// Enqueue a usage event on the outbox. Accepts any `PgExecutor`, so it can -/// be called either on a pool (standalone) or — crucially — on -/// `&mut *transaction` to stay atomic with the ledger debit. -pub async fn enqueue_usage_event<'c, E>( - executor: E, - account_id: Uuid, - event_type: BillingEventType, - quantity: f64, - metadata: Option, -) -> Result -where - E: PgExecutor<'c>, -{ - let id = Uuid::new_v4(); - let payload = serde_json::to_value(OutboxPayload { quantity, metadata })?; - - sqlx::query( - "INSERT INTO hyperline_events_outbox (id, account_id, event_type, payload) \ - VALUES ($1, $2, $3, $4)", - ) - .bind(id) - .bind(account_id) - .bind(event_type.as_str()) - .bind(payload) - .execute(executor) - .await?; - - Ok(id) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn build_record_minimal() { - let rec = build_record("abc", 5.0, None); - assert_eq!(rec["id"], "abc"); - assert_eq!(rec["credits"], 5.0); - assert_eq!(rec.as_object().unwrap().len(), 2); - } - - #[test] - fn build_record_flattens_scalar_metadata() { - let meta = serde_json::json!({ - "operation": "scrape", - "description": "…", - "js_rendered": true, - }); - let rec = build_record("abc", 3.0, Some(&meta)); - assert_eq!(rec["operation"], "scrape"); - assert_eq!(rec["description"], "…"); - assert_eq!(rec["js_rendered"], true); - } - - #[test] - fn build_record_drops_non_scalar_metadata() { - let meta = serde_json::json!({ - "nested": { "a": 1 }, - "arr": [1, 2, 3], - "keep_me": "yes", - }); - let rec = build_record("abc", 1.0, Some(&meta)); - assert!(rec.get("nested").is_none()); - assert!(rec.get("arr").is_none()); - assert_eq!(rec["keep_me"], "yes"); - } - - #[test] - fn build_record_never_overwrites_reserved_keys() { - let meta = serde_json::json!({ - "id": "shadow", - "credits": 999, - "real": "kept", - }); - let rec = build_record("outbox-id", 1.0, Some(&meta)); - assert_eq!(rec["id"], "outbox-id"); - assert_eq!(rec["credits"], 1.0); - assert_eq!(rec["real"], "kept"); - } -} diff --git a/crates/scrapix-billing-hyperline/src/lib.rs b/crates/scrapix-billing-hyperline/src/lib.rs deleted file mode 100644 index 1c0cb9c5..00000000 --- a/crates/scrapix-billing-hyperline/src/lib.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Scrapix — Hyperline billing provider. -//! -//! Replaces the direct Stripe integration in `bins/scrapix-api/src/stripe.rs` -//! by treating Hyperline as the billing system of record while Stripe acts as -//! a downstream payment processor configured inside Hyperline. -//! -//! Layout: -//! - [`client`] — typed REST client over `HYPERLINE_API_BASE`. -//! - [`events`] — usage event types + outbox-backed emission helper. -//! - [`outbox`] — background drain worker that POSTs outbox rows to Hyperline. -//! - [`webhooks`] — signature verification and payload parsing. -//! - [`reconcile`] — wallet-balance drift checks between local ledger and Hyperline. -//! - [`config`] — env-driven configuration. -//! - [`error`] — crate-wide error type. - -pub mod client; -pub mod config; -pub mod error; -pub mod events; -pub mod outbox; -pub mod reconcile; -pub mod webhooks; - -pub use client::{Customer, HyperlineClient, MoneyAmount, PortalLink, Wallet}; -pub use config::HyperlineConfig; -pub use error::HyperlineError; - -/// Boot-time self-check: pings Hyperline to validate auth+network, and -/// logs the `BillingEventType` manifest that the API will emit. -/// -/// Hyperline's product/pricing config lives in the Hyperline dashboard and -/// has no read-only introspection endpoint, so true bidirectional parity -/// isn't available. Instead we: -/// 1. Ping `/v1/customers?limit=1` — proves the API key is live and the -/// configured base URL is reachable. -/// 2. Emit an INFO log listing every `event_type` this process will POST -/// to `/v1/events`. Ops reads this off startup logs and cross-checks -/// that each event_type has a matching metered product in Hyperline. -/// -/// On ping failure we log at WARN and return the error; the caller decides -/// whether to fail the boot (prod) or proceed in degraded local-only mode -/// (dev/CI). A transient outage shouldn't DoS the API — the outbox drain -/// worker already handles Hyperline downtime via retries — so production -/// deploys should log-and-continue rather than crashloop. -pub async fn boot_self_check(client: &HyperlineClient) -> Result<(), HyperlineError> { - use events::BillingEventType; - - let manifest: Vec<&'static str> = BillingEventType::all().iter().map(|t| t.as_str()).collect(); - tracing::info!( - sandbox = client.config().is_sandbox(), - event_types = ?manifest, - "Hyperline event-type manifest — verify each has a metered product in the Hyperline dashboard" - ); - - match client.ping().await { - Ok(()) => { - tracing::info!( - sandbox = client.config().is_sandbox(), - "Hyperline boot self-check passed" - ); - Ok(()) - } - Err(e) => { - tracing::warn!( - error = %e, - sandbox = client.config().is_sandbox(), - "Hyperline boot self-check failed — API unreachable or auth invalid. Outbox will keep enqueuing and retry once Hyperline is reachable." - ); - Err(e) - } - } -} diff --git a/crates/scrapix-billing-hyperline/src/outbox.rs b/crates/scrapix-billing-hyperline/src/outbox.rs deleted file mode 100644 index 8269b3df..00000000 --- a/crates/scrapix-billing-hyperline/src/outbox.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Background drain worker for the Hyperline usage-event outbox. -//! -//! One tokio task loops every `interval`, pulls the oldest unsent rows whose -//! account already has a `hyperline_customer_id`, POSTs them to the ingest -//! API, and marks `sent_at` on success (or bumps `attempts` + `last_error` -//! on failure). The outbox row UUID doubles as Hyperline's `record_id`, so -//! duplicate deliveries are deduped server-side. - -use std::time::Duration; - -use sqlx::postgres::PgRow; -use sqlx::{PgPool, Row}; -use tokio::task::JoinHandle; -use tracing::{debug, error, warn}; -use uuid::Uuid; - -use crate::client::HyperlineClient; -use crate::error::HyperlineError; -use crate::events::{build_record, OutboxPayload, UsageEvent}; - -/// Max attempts per row before we stop retrying. Further retries require a -/// manual replay (reset `attempts` / `sent_at` in the DB). -const MAX_ATTEMPTS: i32 = 10; - -#[derive(Debug, Default, Clone, Copy)] -pub struct DrainStats { - pub succeeded: u32, - pub failed: u32, - pub skipped: u32, -} - -struct OutboxRow { - id: Uuid, - hyperline_customer_id: String, - event_type: String, - payload: serde_json::Value, - created_at: chrono::DateTime, -} - -impl OutboxRow { - fn from_pg(row: PgRow) -> Result { - Ok(Self { - id: row.try_get("id")?, - hyperline_customer_id: row.try_get("hyperline_customer_id")?, - event_type: row.try_get("event_type")?, - payload: row.try_get("payload")?, - created_at: row.try_get("created_at")?, - }) - } -} - -/// Drain up to `batch_size` outbox rows once. Intended for both the -/// background worker and ops/replay tooling. -pub async fn drain_once( - pool: &PgPool, - client: &HyperlineClient, - batch_size: i64, -) -> Result { - let rows = sqlx::query( - "SELECT o.id, o.event_type, o.payload, o.created_at, \ - a.hyperline_customer_id \ - FROM hyperline_events_outbox o \ - JOIN accounts a ON a.id = o.account_id \ - WHERE o.sent_at IS NULL \ - AND o.attempts < $1 \ - AND a.hyperline_customer_id IS NOT NULL \ - ORDER BY o.created_at ASC \ - LIMIT $2", - ) - .bind(MAX_ATTEMPTS) - .bind(batch_size) - .fetch_all(pool) - .await?; - - let mut stats = DrainStats::default(); - for raw in rows { - let row = OutboxRow::from_pg(raw)?; - match deliver_row(pool, client, &row).await { - Ok(()) => stats.succeeded += 1, - Err(DrainError::Permanent(e)) => { - stats.failed += 1; - error!(id = %row.id, error = %e, "outbox row permanently failed"); - } - Err(DrainError::Transient(e)) => { - stats.failed += 1; - warn!(id = %row.id, error = %e, "outbox row transient failure — will retry"); - } - } - } - debug!(?stats, "outbox drain pass complete"); - Ok(stats) -} - -/// Spawn the drain loop as a tokio task. Returns the `JoinHandle` so callers -/// can `abort()` on shutdown. -pub fn spawn_drain_worker( - pool: PgPool, - client: HyperlineClient, - interval: Duration, - batch_size: i64, -) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(interval); - // Align to the interval start, skipping missed ticks under load. - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - if let Err(e) = drain_once(&pool, &client, batch_size).await { - error!(error = %e, "outbox drain pass failed"); - } - } - }) -} - -enum DrainError { - Transient(HyperlineError), - Permanent(HyperlineError), -} - -async fn deliver_row( - pool: &PgPool, - client: &HyperlineClient, - row: &OutboxRow, -) -> Result<(), DrainError> { - let payload: OutboxPayload = match serde_json::from_value(row.payload.clone()) { - Ok(p) => p, - Err(e) => { - // Malformed payload won't fix itself on retry — drop it. - mark_failed(pool, row.id, &e.to_string(), /* final */ true).await; - return Err(DrainError::Permanent(HyperlineError::Decode(e))); - } - }; - - let id_string = row.id.to_string(); - let record = build_record(&id_string, payload.quantity, payload.metadata.as_ref()); - let event = UsageEvent { - customer_id: &row.hyperline_customer_id, - event_type: &row.event_type, - timestamp: row.created_at, - record, - }; - - match client.ingest_event(&event).await { - Ok(()) => { - mark_sent(pool, row.id) - .await - .map_err(DrainError::Permanent)?; - Ok(()) - } - Err(e) => { - let permanent = matches!( - &e, - HyperlineError::Api { status, .. } if *status >= 400 && *status < 500 && *status != 429 - ); - mark_failed(pool, row.id, &e.to_string(), permanent).await; - if permanent { - Err(DrainError::Permanent(e)) - } else { - Err(DrainError::Transient(e)) - } - } - } -} - -async fn mark_sent(pool: &PgPool, id: Uuid) -> Result<(), HyperlineError> { - sqlx::query("UPDATE hyperline_events_outbox SET sent_at = now() WHERE id = $1") - .bind(id) - .execute(pool) - .await?; - Ok(()) -} - -async fn mark_failed(pool: &PgPool, id: Uuid, err: &str, permanent: bool) { - // On a permanent 4xx we push `attempts` to MAX_ATTEMPTS so the query - // filter excludes the row from future picks; transient bumps by 1. - let result = if permanent { - sqlx::query( - "UPDATE hyperline_events_outbox \ - SET attempts = $1, last_error = $2 \ - WHERE id = $3", - ) - .bind(MAX_ATTEMPTS) - .bind(err) - .bind(id) - .execute(pool) - .await - } else { - sqlx::query( - "UPDATE hyperline_events_outbox \ - SET attempts = attempts + 1, last_error = $1 \ - WHERE id = $2", - ) - .bind(err) - .bind(id) - .execute(pool) - .await - }; - if let Err(e) = result { - error!(id = %id, error = %e, "failed to update outbox row failure state"); - } -} diff --git a/crates/scrapix-billing-hyperline/src/reconcile.rs b/crates/scrapix-billing-hyperline/src/reconcile.rs deleted file mode 100644 index fd818bb1..00000000 --- a/crates/scrapix-billing-hyperline/src/reconcile.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! Wallet-balance reconciliation between the local credit ledger and Hyperline. -//! -//! Two layers: -//! -//! - [`scan_once`] / [`spawn_scan_worker`] — for-every-linked-account pair -//! the local `credits_balance` with the live Hyperline wallet balance and -//! emit a paired observation. The worker never blocks or retries; it -//! logs and moves on. -//! -//! - [`BalanceDrift`] — the paired observation shape. `is_within_tolerance` -//! is deliberately **not** provided at the type level: local credits are -//! our internal unit and Hyperline's `balance.amount` is the processor's -//! minor currency unit (cents). Converting between the two is a -//! pricing-policy decision that lives outside this module — callers that -//! want drift alerts supply their own conversion when comparing. -//! -//! Per SCR-68 Phase 1: shadow-mode observability. No action is taken on -//! drift beyond logging — production alerting is a downstream concern that -//! consumes the scan output. -//! -//! Units are **not** normalized here on purpose. `local_credits` is in the -//! credit unit used by the ledger (`accounts.credits_balance`); both -//! `hyperline_balance` and `hyperline_projected_balance` are in the -//! smallest currency unit (cents for USD). The scan worker ships the raw -//! numbers; conversion is the consumer's job. - -use std::time::Duration; - -use serde::Serialize; -use sqlx::{PgPool, Row}; -use tokio::task::JoinHandle; -use tracing::{error, info, warn}; - -use crate::client::HyperlineClient; -use crate::error::HyperlineError; - -/// Paired observation: local credits vs. Hyperline wallet balance for a -/// single account. See module docs for the unit caveat. -#[derive(Debug, Clone, Serialize)] -pub struct BalanceDrift { - pub account_id: uuid::Uuid, - /// Local `accounts.credits_balance` (credits, our internal unit). - pub local_credits: i64, - /// Hyperline `Wallet.balance.amount` — smallest currency unit. - pub hyperline_balance: i64, - /// Hyperline `Wallet.projected_balance.amount` — same unit as - /// `hyperline_balance`. `None` when Hyperline didn't return it. - pub hyperline_projected_balance: Option, - /// ISO 4217 (e.g. `"USD"`) from the wallet, when present. - pub currency: Option, -} - -/// Drift-alert configuration. Local credits and the Hyperline balance live -/// in different units (credits vs. cents), so the comparator needs a ratio -/// and an absolute tolerance to be meaningful. -/// -/// The semantics: convert `local_credits` to cents via -/// `local_credits * cents_per_credit`, then alert when -/// `|local_cents - hyperline_balance| > tolerance_cents`. -#[derive(Debug, Clone, Copy)] -pub struct DriftThreshold { - /// How many cents one local credit is worth. Site-specific pricing — - /// no sensible default, must be set per deployment. - pub cents_per_credit: i64, - /// Tolerance in cents. Drift below this is normal noise (in-flight - /// outbox rows, webhook lag) and stays at INFO; drift above gets a - /// WARN line that ops dashboards alert on. - pub tolerance_cents: i64, -} - -impl DriftThreshold { - /// Convert a paired observation into "drift in cents" using this - /// threshold's ratio. Negative means the local ledger is *ahead* of - /// Hyperline (we've debited more than they've billed); positive means - /// behind (Hyperline has more credit than the ledger reflects). - pub fn delta_cents(&self, drift: &BalanceDrift) -> i64 { - drift - .local_credits - .saturating_mul(self.cents_per_credit) - .saturating_sub(drift.hyperline_balance) - } - - /// `true` when the absolute drift exceeds the tolerance. - pub fn breached(&self, drift: &BalanceDrift) -> bool { - self.delta_cents(drift).saturating_abs() > self.tolerance_cents - } -} - -/// Scan every account with a linked Hyperline wallet and return paired -/// balance observations. A per-account failure (wallet 404, network blip) -/// logs a warning and is skipped — one bad account never aborts the scan. -/// -/// When `threshold` is `Some`, accounts whose drift exceeds the configured -/// tolerance get a `WARN` line per scan pass (alert-friendly). When `None` -/// the worker stays in pure observation mode — paired observations only, -/// no drift alerts. -pub async fn scan_once( - pool: &PgPool, - client: &HyperlineClient, - threshold: Option, -) -> Result, HyperlineError> { - let rows = sqlx::query( - "SELECT id, credits_balance, hyperline_wallet_id \ - FROM accounts \ - WHERE hyperline_wallet_id IS NOT NULL \ - AND active = true", - ) - .fetch_all(pool) - .await?; - - let mut results = Vec::with_capacity(rows.len()); - for row in rows { - let account_id: uuid::Uuid = row.try_get("id")?; - let local_credits: i64 = row.try_get("credits_balance")?; - let wallet_id: String = row.try_get("hyperline_wallet_id")?; - - match client.get_wallet(&wallet_id).await { - Ok(wallet) => { - let drift = BalanceDrift { - account_id, - local_credits, - hyperline_balance: wallet.balance.amount, - hyperline_projected_balance: wallet.projected_balance.map(|p| p.amount), - currency: wallet.currency, - }; - if let Some(t) = threshold { - let delta = t.delta_cents(&drift); - if t.breached(&drift) { - warn!( - account_id = %drift.account_id, - local_credits = drift.local_credits, - hyperline_balance = drift.hyperline_balance, - cents_per_credit = t.cents_per_credit, - tolerance_cents = t.tolerance_cents, - delta_cents = delta, - currency = ?drift.currency, - "reconcile: balance drift exceeds tolerance" - ); - } else { - info!( - account_id = %drift.account_id, - local_credits = drift.local_credits, - hyperline_balance = drift.hyperline_balance, - delta_cents = delta, - currency = ?drift.currency, - "reconcile: paired balance observation (within tolerance)" - ); - } - } else { - info!( - account_id = %drift.account_id, - local_credits = drift.local_credits, - hyperline_balance = drift.hyperline_balance, - hyperline_projected = ?drift.hyperline_projected_balance, - currency = ?drift.currency, - "reconcile: paired balance observation" - ); - } - results.push(drift); - } - Err(e) => { - warn!( - account_id = %account_id, - wallet_id = %wallet_id, - error = %e, - "reconcile: wallet fetch failed — skipping account" - ); - } - } - } - Ok(results) -} - -/// Spawn the scan loop as a tokio task. Returns the `JoinHandle` so -/// callers can `abort()` on shutdown. A scan pass that errors at the DB -/// layer (not per-account) logs and the loop continues on the next tick. -/// -/// `threshold` controls drift alerting — see [`scan_once`]. -pub fn spawn_scan_worker( - pool: PgPool, - client: HyperlineClient, - interval: Duration, - threshold: Option, -) -> JoinHandle<()> { - tokio::spawn(async move { - // Sensible first tick: give the app a moment after boot rather - // than firing at t=0. - let mut ticker = tokio::time::interval_at( - tokio::time::Instant::now() + Duration::from_secs(30), - interval, - ); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - match scan_once(&pool, &client, threshold).await { - Ok(drifts) => { - info!(count = drifts.len(), "reconcile: scan pass complete"); - } - Err(e) => { - error!(error = %e, "reconcile: scan pass failed"); - } - } - } - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn drift(local: i64, hyperline: i64) -> BalanceDrift { - BalanceDrift { - account_id: uuid::Uuid::nil(), - local_credits: local, - hyperline_balance: hyperline, - hyperline_projected_balance: None, - currency: Some("USD".to_string()), - } - } - - #[test] - fn delta_cents_handles_ratio() { - let t = DriftThreshold { - cents_per_credit: 1, - tolerance_cents: 1, - }; - // 100 credits at 1¢/credit = 100¢; Hyperline says 100¢ → no drift. - assert_eq!(t.delta_cents(&drift(100, 100)), 0); - // 100 credits at 1¢/credit = 100¢; Hyperline says 90¢ → +10¢ ahead. - assert_eq!(t.delta_cents(&drift(100, 90)), 10); - // 100 credits at 1¢/credit = 100¢; Hyperline says 110¢ → -10¢ behind. - assert_eq!(t.delta_cents(&drift(100, 110)), -10); - } - - #[test] - fn breached_respects_tolerance() { - let t = DriftThreshold { - cents_per_credit: 1, - tolerance_cents: 5, - }; - assert!(!t.breached(&drift(100, 100))); // 0 ≤ 5 - assert!(!t.breached(&drift(100, 95))); // 5 ≤ 5 - assert!(t.breached(&drift(100, 94))); // 6 > 5 - assert!(t.breached(&drift(100, 106))); // 6 > 5 (negative) - } -} diff --git a/crates/scrapix-billing-hyperline/src/webhooks.rs b/crates/scrapix-billing-hyperline/src/webhooks.rs deleted file mode 100644 index 3aeb07da..00000000 --- a/crates/scrapix-billing-hyperline/src/webhooks.rs +++ /dev/null @@ -1,198 +0,0 @@ -//! Webhook signature verification and event parsing. -//! -//! Hyperline signs webhooks with HMAC-SHA256 over `id.timestamp.body` using a -//! base64-encoded secret that starts with `whsec_`. Headers arrive as -//! `webhook-id`, `webhook-timestamp` (Unix seconds), and `webhook-signature` -//! (space-separated list of `v1,` signatures — any valid one accepts). - -use base64::engine::general_purpose::STANDARD as B64; -use base64::Engine; -use hmac::{Hmac, Mac}; -use serde::Deserialize; -use sha2::Sha256; -use subtle::ConstantTimeEq; - -use crate::error::HyperlineError; - -type HmacSha256 = Hmac; - -const TIMESTAMP_TOLERANCE_SECS: i64 = 300; -const SECRET_PREFIX: &str = "whsec_"; - -pub struct WebhookHeaders<'a> { - pub id: &'a str, - pub timestamp: &'a str, - pub signature: &'a str, -} - -/// Verify a Hyperline webhook delivery. Returns `Ok(())` on a valid signature -/// within the 5-minute timestamp tolerance; otherwise returns the failure -/// reason. -pub fn verify_signature( - secret: &str, - headers: WebhookHeaders<'_>, - body: &[u8], - now_unix: i64, -) -> Result<(), HyperlineError> { - let raw = secret - .strip_prefix(SECRET_PREFIX) - .ok_or(HyperlineError::InvalidConfig( - "webhook secret must start with whsec_".into(), - ))?; - let key = B64 - .decode(raw) - .map_err(|_| HyperlineError::InvalidConfig("webhook secret is not valid base64".into()))?; - - let ts: i64 = headers - .timestamp - .parse() - .map_err(|_| HyperlineError::MalformedHeader("webhook-timestamp"))?; - if (now_unix - ts).abs() > TIMESTAMP_TOLERANCE_SECS { - return Err(HyperlineError::StaleTimestamp); - } - - // Canonical signed string per Hyperline / Svix spec. - let mut mac = HmacSha256::new_from_slice(&key).map_err(|_| HyperlineError::InvalidSignature)?; - mac.update(headers.id.as_bytes()); - mac.update(b"."); - mac.update(headers.timestamp.as_bytes()); - mac.update(b"."); - mac.update(body); - let expected = mac.finalize().into_bytes(); - - // `webhook-signature` is a space-separated list of `v,`. - for part in headers.signature.split_whitespace() { - let Some((_version, b64)) = part.split_once(',') else { - continue; - }; - let Ok(candidate) = B64.decode(b64) else { - continue; - }; - if candidate.len() == expected.len() - && candidate.ct_eq(expected.as_slice()).unwrap_u8() == 1 - { - return Ok(()); - } - } - Err(HyperlineError::InvalidSignature) -} - -/// Minimal shape for dispatching a verified webhook payload. -#[derive(Debug, Deserialize)] -pub struct WebhookEnvelope { - #[serde(rename = "type")] - pub event_type: String, - #[serde(default)] - pub data: serde_json::Value, -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Canonical fixture: known key, payload, timestamp, id → known signature. - fn sign(key: &[u8], id: &str, ts: &str, body: &[u8]) -> String { - let mut mac = HmacSha256::new_from_slice(key).unwrap(); - mac.update(id.as_bytes()); - mac.update(b"."); - mac.update(ts.as_bytes()); - mac.update(b"."); - mac.update(body); - B64.encode(mac.finalize().into_bytes()) - } - - #[test] - fn verifies_good_signature() { - let key = b"super-secret-key-bytes"; - let secret = format!("{SECRET_PREFIX}{}", B64.encode(key)); - let id = "msg_01HY"; - let ts = "1712000000"; - let body = br#"{"type":"wallet.credited"}"#; - let sig_b64 = sign(key, id, ts, body); - let signature = format!("v1,{sig_b64}"); - - verify_signature( - &secret, - WebhookHeaders { - id, - timestamp: ts, - signature: &signature, - }, - body, - 1712000010, - ) - .unwrap(); - } - - #[test] - fn rejects_tampered_body() { - let key = b"super-secret-key-bytes"; - let secret = format!("{SECRET_PREFIX}{}", B64.encode(key)); - let id = "msg_01HY"; - let ts = "1712000000"; - let body = br#"{"type":"wallet.credited"}"#; - let sig_b64 = sign(key, id, ts, body); - let signature = format!("v1,{sig_b64}"); - - let err = verify_signature( - &secret, - WebhookHeaders { - id, - timestamp: ts, - signature: &signature, - }, - br#"{"type":"wallet.debited"}"#, - 1712000010, - ) - .unwrap_err(); - assert!(matches!(err, HyperlineError::InvalidSignature)); - } - - #[test] - fn rejects_stale_timestamp() { - let key = b"super-secret-key-bytes"; - let secret = format!("{SECRET_PREFIX}{}", B64.encode(key)); - let id = "msg_01HY"; - let ts = "1712000000"; - let body = br#"{}"#; - let sig_b64 = sign(key, id, ts, body); - let signature = format!("v1,{sig_b64}"); - - let err = verify_signature( - &secret, - WebhookHeaders { - id, - timestamp: ts, - signature: &signature, - }, - body, - 1712000000 + TIMESTAMP_TOLERANCE_SECS + 1, - ) - .unwrap_err(); - assert!(matches!(err, HyperlineError::StaleTimestamp)); - } - - #[test] - fn accepts_second_signature_in_rotation() { - // Hyperline may rotate secrets by sending multiple signatures. - let key = b"current-key"; - let secret = format!("{SECRET_PREFIX}{}", B64.encode(key)); - let id = "msg_01HY"; - let ts = "1712000000"; - let body = br#"{}"#; - let good = sign(key, id, ts, body); - let signature = format!("v1,bogus-old v1,{good}"); - - verify_signature( - &secret, - WebhookHeaders { - id, - timestamp: ts, - signature: &signature, - }, - body, - 1712000010, - ) - .unwrap(); - } -} diff --git a/crates/scrapix-billing-hyperline/tests/sandbox_roundtrip.rs b/crates/scrapix-billing-hyperline/tests/sandbox_roundtrip.rs deleted file mode 100644 index 7988335e..00000000 --- a/crates/scrapix-billing-hyperline/tests/sandbox_roundtrip.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! Live Hyperline sandbox round-trip. -//! -//! These tests are `#[ignore]`d by default and only run against the live -//! sandbox when explicitly invoked. They exercise the wire contract — URL -//! paths, header names, payload shapes, serde envelopes — in the one way -//! that mocks and unit tests cannot. -//! -//! Run with: -//! -//! ```sh -//! HYPERLINE_API_KEY=test_... \ -//! cargo test -p scrapix-billing-hyperline \ -//! --test sandbox_roundtrip -- --ignored --nocapture -//! ``` -//! -//! Each test is independent and uses a fresh external_id so re-runs don't -//! collide. A 4xx from Hyperline is reported verbatim so wire drift is -//! loud (rather than silently turning into `None` via `#[serde(default)]`). - -use scrapix_billing_hyperline::events::{build_record, UsageEvent}; -use scrapix_billing_hyperline::{boot_self_check, Customer, HyperlineClient}; -use uuid::Uuid; - -fn client() -> HyperlineClient { - let c = HyperlineClient::from_env().expect("HYPERLINE_API_KEY must be set"); - assert!( - c.config().is_sandbox(), - "sandbox roundtrip must run against a test_ key, got prod_" - ); - c -} - -/// Connectivity + auth + JSON envelope — the cheapest signal that the -/// client is wired correctly. -#[tokio::test] -#[ignore = "requires HYPERLINE_API_KEY to hit live sandbox"] -async fn lists_customers_in_sandbox() { - let c = client(); - let page = c.list_customers(1).await.expect("list_customers failed"); - assert!(page.meta.taken <= 1, "limit=1 was not honored"); - println!( - "[ok] list_customers → meta={{total:{}, taken:{}, skipped:{}}}", - page.meta.total, page.meta.taken, page.meta.skipped - ); -} - -/// The boot self-check the API runs during startup. Verifies the ping -/// returns 2xx and that the function emits its event-type manifest log. -/// Run as a single-test smoke to catch regressions in the exact -/// log-and-ping contract the API deploy relies on. -#[tokio::test] -#[ignore = "requires HYPERLINE_API_KEY to hit live sandbox"] -async fn boot_self_check_passes_in_sandbox() { - let c = client(); - boot_self_check(&c) - .await - .expect("boot_self_check failed against live sandbox"); - println!("[ok] boot_self_check → ping 2xx + manifest logged"); -} - -/// Create a fresh customer, then verify we can fetch their portal URL. -/// -/// This is the exact path `GET /account/billing/portal` walks in prod: -/// the backend holds `hyperline_customer_id`, we call -/// `get_portal_url(customer_id)`, and we redirect the browser to the -/// returned URL. -#[tokio::test] -#[ignore = "requires HYPERLINE_API_KEY to hit live sandbox"] -async fn creates_customer_and_fetches_portal_url() { - let c = client(); - - // Unique per run; `external_id` is our account UUID in prod. - let external_id = format!("smoke-{}", Uuid::new_v4()); - let body = serde_json::json!({ - "external_id": external_id, - "name": "Scrapix smoke test", - "email": format!("{external_id}@example.com"), - }); - - let created: Customer = c - .post_json("/v1/customers", &body) - .await - .expect("create_customer failed"); - assert_eq!(created.external_id.as_deref(), Some(external_id.as_str())); - println!( - "[ok] create_customer → id={} external_id={:?}", - created.id, created.external_id - ); - - let portal = c - .get_portal_url(&created.id) - .await - .expect("get_portal_url failed"); - assert!( - portal.url.starts_with("http"), - "portal URL must be absolute: {}", - portal.url - ); - println!("[ok] get_portal_url → {}", portal.url); -} - -/// Post a single billable event to the ingest host. -/// -/// Mirrors exactly what the outbox drain worker does: builds a -/// `UsageEvent` with a UUID `record.id` and POSTs it to `/v1/events`. -/// A 2xx is the success criterion — Hyperline's ingest returns an -/// empty body on accept. Repeated POSTs with the same `record.id` -/// validate our idempotency assumption (one of the flagged risks). -#[tokio::test] -#[ignore = "requires HYPERLINE_API_KEY to hit live sandbox"] -async fn ingests_event_and_is_idempotent_on_record_id() { - let c = client(); - - // Need a customer to attach the event to. - let external_id = format!("smoke-{}", Uuid::new_v4()); - let body = serde_json::json!({ - "external_id": external_id, - "name": "Scrapix smoke event", - "email": format!("{external_id}@example.com"), - }); - let customer: Customer = c - .post_json("/v1/customers", &body) - .await - .expect("create_customer failed"); - - // One outbox-shaped event. - let record_id = Uuid::new_v4().to_string(); - let record = build_record( - &record_id, - 3.0, - Some(&serde_json::json!({ - "operation": "scrape", - "smoke": true, - })), - ); - let event = UsageEvent { - customer_id: &customer.id, - event_type: "api_request", - timestamp: chrono::Utc::now(), - record, - }; - - c.ingest_event(&event) - .await - .expect("ingest_event #1 failed"); - println!("[ok] ingest_event #1 → 2xx (record.id={record_id})"); - - // Same record.id → expect idempotent (2xx) rather than 409 / duplicate. - // If Hyperline doesn't dedupe on record.id this will still 2xx, but a - // follow-up `/v1/events/list` query (not implemented in our client - // yet) would show two rows — this test only validates the HTTP - // contract, not the server-side behavior. - c.ingest_event(&event) - .await - .expect("ingest_event #2 (replay) failed"); - println!("[ok] ingest_event #2 (replay, same record.id) → 2xx"); -} diff --git a/crates/scrapix-billing/Cargo.toml b/crates/scrapix-billing/Cargo.toml index 29fb7ee9..cf2582af 100644 --- a/crates/scrapix-billing/Cargo.toml +++ b/crates/scrapix-billing/Cargo.toml @@ -19,6 +19,5 @@ tokio = { workspace = true } thiserror = { workspace = true } async-trait = { workspace = true } -# NOTE: async-stripe dependency was removed during the Hyperline migration. -# The ledger no longer knows or cares about Stripe specifically — it accepts -# a provider-agnostic (provider, event_id) pair for idempotency. +# Stripe payments +async-stripe = { version = "0.38", default-features = false, features = ["runtime-tokio-hyper", "checkout", "billing", "connect", "webhook-events"] } diff --git a/crates/scrapix-billing/src/ledger.rs b/crates/scrapix-billing/src/ledger.rs index c00f9d35..46fdc9db 100644 --- a/crates/scrapix-billing/src/ledger.rs +++ b/crates/scrapix-billing/src/ledger.rs @@ -134,50 +134,26 @@ pub async fn check_spend_limit( Ok(()) } -/// Add credits to an account in response to a billing-provider event -/// (typically a Hyperline `wallet.credited` webhook). -/// -/// Idempotency is keyed on `(provider, provider_event_id)` stored in the -/// transaction's metadata JSONB — re-delivery of the same webhook is a -/// no-op. The pair is namespaced by `provider` so we can accept events -/// from multiple sources without collisions on short numeric IDs. -/// -/// `tx_type` is written to the `transactions.type` column (e.g. -/// "wallet_credit" for wallet.credited, "manual_topup" for an admin- -/// triggered adjustment). -pub async fn add_credits_from_provider( +/// Add credits to an account after a successful payment. +/// Idempotent: checks if a transaction with this `stripe_payment_intent_id` already exists. +pub async fn add_credits_for_payment( pool: &sqlx::PgPool, account_id: uuid::Uuid, credits: i64, - provider: &str, - provider_event_id: &str, - tx_type: &str, + payment_intent_id: &str, description: &str, ) -> Result<(), BillingError> { - // Idempotency check — same provider+event_id means we've already - // credited these credits. Short-circuit without an update so - // concurrent redeliveries don't double-count. + // Idempotency check let exists: bool = sqlx::query_scalar( - "SELECT EXISTS( \ - SELECT 1 FROM transactions \ - WHERE account_id = $1 \ - AND metadata->>'provider' = $2 \ - AND metadata->>'provider_event_id' = $3 \ - )", + "SELECT EXISTS(SELECT 1 FROM transactions WHERE account_id = $1 AND metadata->>'stripe_payment_intent_id' = $2)", ) .bind(account_id) - .bind(provider) - .bind(provider_event_id) + .bind(payment_intent_id) .fetch_one(pool) .await?; if exists { - info!( - account_id = %account_id, - provider, - event_id = %provider_event_id, - "Provider event already processed, skipping" - ); + info!(account_id = %account_id, pi = %payment_intent_id, "Payment already processed, skipping"); return Ok(()); } @@ -192,42 +168,20 @@ pub async fn add_credits_from_provider( .await?; let metadata = serde_json::json!({ - "provider": provider, - "provider_event_id": provider_event_id, + "stripe_payment_intent_id": payment_intent_id, }); - let insert_result = sqlx::query( + sqlx::query( "INSERT INTO transactions (account_id, type, amount, balance_after, description, metadata) \ - VALUES ($1, $2, $3, $4, $5, $6)", + VALUES ($1, 'manual_topup', $2, $3, $4, $5)", ) .bind(account_id) - .bind(tx_type) .bind(credits) .bind(new_balance) .bind(description) .bind(metadata) .execute(&mut *tx) - .await; - - // Race-condition backstop: if a concurrent caller raced past our - // `SELECT EXISTS` check and inserted the same `(provider, event_id)` - // first, the partial unique index `uniq_transactions_provider_event` - // raises a 23505. Roll back the wallet bump (so we don't double-credit) - // and treat the situation as "already processed" — same outcome as - // hitting the EXISTS branch above. - if let Err(sqlx::Error::Database(db_err)) = &insert_result { - if db_err.code().as_deref() == Some("23505") { - tx.rollback().await?; - info!( - account_id = %account_id, - provider, - event_id = %provider_event_id, - "Provider event raced — duplicate detected by unique index, rolling back" - ); - return Ok(()); - } - } - insert_result?; + .await?; tx.commit().await?; @@ -235,10 +189,8 @@ pub async fn add_credits_from_provider( account_id = %account_id, credits, new_balance, - provider, - event_id = %provider_event_id, - tx_type, - "Credits added from provider event" + pi = %payment_intent_id, + "Credits added via payment" ); Ok(()) diff --git a/crates/scrapix-billing/src/lib.rs b/crates/scrapix-billing/src/lib.rs index 4a013d11..82ff0ed6 100644 --- a/crates/scrapix-billing/src/lib.rs +++ b/crates/scrapix-billing/src/lib.rs @@ -13,5 +13,5 @@ pub mod pricing; pub use auto_topup::{BillingNotifier, PaymentProvider}; pub use credits::{crawl_credits_per_page, scrape_credits, MAP_CREDITS, SEARCH_CREDITS}; pub use error::BillingError; -pub use ledger::{add_credits_from_provider, check_credits, check_spend_limit, deduct_credits}; +pub use ledger::{add_credits_for_payment, check_credits, check_spend_limit, deduct_credits}; pub use pricing::calculate_price_cents; diff --git a/deploy/postgres/init.sql b/deploy/postgres/init.sql index 0bf8bec7..20b0c00e 100644 --- a/deploy/postgres/init.sql +++ b/deploy/postgres/init.sql @@ -28,13 +28,12 @@ CREATE TABLE IF NOT EXISTS accounts ( name TEXT NOT NULL, tier TEXT NOT NULL DEFAULT 'free' CHECK (tier IN ('free', 'starter', 'pro', 'enterprise')), active BOOLEAN NOT NULL DEFAULT true, - hyperline_customer_id TEXT UNIQUE, - hyperline_wallet_id TEXT, - -- Surfaced by the Hyperline payment_method.errored / .expired - -- webhooks. NULL means "no issue on file". The console reads this - -- to show a "Your card expired — update it on the portal" banner. - payment_method_status TEXT CHECK (payment_method_status IN ('errored', 'expired')), + stripe_customer_id TEXT, + stripe_default_payment_method_id TEXT, credits_balance BIGINT NOT NULL DEFAULT 100, + auto_topup_enabled BOOLEAN NOT NULL DEFAULT false, + auto_topup_amount BIGINT NOT NULL DEFAULT 5000, + auto_topup_threshold BIGINT NOT NULL DEFAULT 500, monthly_spend_limit BIGINT, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() @@ -252,14 +251,9 @@ END $$; CREATE TABLE IF NOT EXISTS transactions ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - -- type is the ledger semantic tag. `wallet_credit`, `invoice_settled` - -- and `invoice_errored` are Hyperline webhook-driven rows (see - -- bins/scrapix-api/src/hyperline.rs). `auto_topup` is retained for - -- historical rows from the pre-Hyperline auto-topup cron. type TEXT NOT NULL CHECK (type IN ( 'initial_deposit', 'manual_topup', 'auto_topup', - 'usage_deduction', 'refund', 'adjustment', - 'wallet_credit', 'invoice_settled', 'invoice_errored' + 'usage_deduction', 'refund', 'adjustment' )), amount BIGINT NOT NULL, balance_after BIGINT NOT NULL, @@ -331,49 +325,20 @@ DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'credits_balance') THEN ALTER TABLE accounts ADD COLUMN credits_balance BIGINT NOT NULL DEFAULT 100; END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'monthly_spend_limit') THEN - ALTER TABLE accounts ADD COLUMN monthly_spend_limit BIGINT; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'auto_topup_enabled') THEN + ALTER TABLE accounts ADD COLUMN auto_topup_enabled BOOLEAN NOT NULL DEFAULT false; END IF; - - -- Hyperline migration (SCR-68): replace Stripe + local auto-topup with Hyperline customer/wallet IDs. - -- Safe to drop Stripe columns directly — zero customers as of 2026-04-17. - ALTER TABLE accounts DROP COLUMN IF EXISTS stripe_customer_id; - ALTER TABLE accounts DROP COLUMN IF EXISTS stripe_default_payment_method_id; - ALTER TABLE accounts DROP COLUMN IF EXISTS auto_topup_enabled; - ALTER TABLE accounts DROP COLUMN IF EXISTS auto_topup_amount; - ALTER TABLE accounts DROP COLUMN IF EXISTS auto_topup_threshold; - - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'hyperline_customer_id') THEN - ALTER TABLE accounts ADD COLUMN hyperline_customer_id TEXT UNIQUE; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'auto_topup_amount') THEN + ALTER TABLE accounts ADD COLUMN auto_topup_amount BIGINT NOT NULL DEFAULT 5000; END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'hyperline_wallet_id') THEN - ALTER TABLE accounts ADD COLUMN hyperline_wallet_id TEXT; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'auto_topup_threshold') THEN + ALTER TABLE accounts ADD COLUMN auto_topup_threshold BIGINT NOT NULL DEFAULT 500; END IF; - - -- Flag surfaced by the Hyperline payment_method webhooks. NULL - -- means "no issue"; 'errored' / 'expired' come from the webhook - -- handler in bins/scrapix-api/src/hyperline.rs. - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'payment_method_status') THEN - ALTER TABLE accounts ADD COLUMN payment_method_status TEXT - CHECK (payment_method_status IN ('errored', 'expired')); + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'monthly_spend_limit') THEN + ALTER TABLE accounts ADD COLUMN monthly_spend_limit BIGINT; END IF; - - -- Widen transactions.type CHECK constraint for Hyperline webhook rows. - -- The webhook handler writes wallet_credit (from wallet.credited) and - -- invoice_settled / invoice_errored (from invoice.*). Drop + re-add - -- is safe here because we migrate the constraint definition, not row - -- data — existing rows all use the pre-existing values. - IF EXISTS ( - SELECT 1 FROM information_schema.check_constraints - WHERE constraint_name = 'transactions_type_check' - ) THEN - ALTER TABLE transactions DROP CONSTRAINT IF EXISTS transactions_type_check; - ALTER TABLE transactions ADD CONSTRAINT transactions_type_check - CHECK (type IN ( - 'initial_deposit', 'manual_topup', 'auto_topup', - 'usage_deduction', 'refund', 'adjustment', - 'wallet_credit', 'invoice_settled', 'invoice_errored' - )); + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'stripe_default_payment_method_id') THEN + ALTER TABLE accounts ADD COLUMN stripe_default_payment_method_id TEXT; END IF; -- Add 'viewer' role to account_members if not already present @@ -476,56 +441,3 @@ BEGIN ALTER TABLE users ALTER COLUMN password_hash DROP NOT NULL; END IF; END $$; - --- ============================================================================ --- Hyperline Events Outbox (reliable, idempotent usage-event delivery) --- ============================================================================ --- Each row is written in the same Postgres transaction as the credit ledger --- debit. A background worker drains unsent rows to POST /v1/events, using --- `id` as the Hyperline `record_id` (natural dedupe key across retries). -CREATE TABLE IF NOT EXISTS hyperline_events_outbox ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - account_id UUID NOT NULL REFERENCES accounts (id) ON DELETE CASCADE, - event_type TEXT NOT NULL, - payload JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - sent_at TIMESTAMPTZ, - attempts INT NOT NULL DEFAULT 0, - last_error TEXT -); - -CREATE INDEX IF NOT EXISTS idx_hyperline_outbox_unsent - ON hyperline_events_outbox (created_at) WHERE sent_at IS NULL; -CREATE INDEX IF NOT EXISTS idx_hyperline_outbox_account - ON hyperline_events_outbox (account_id); - --- ============================================================================ --- Hyperline Webhook Log (replay protection + audit trail) --- ============================================================================ --- `webhook_id` is the `webhook-id` header value; INSERT ... ON CONFLICT DO --- NOTHING gives us dedupe. Rows are retained for ops replay. -CREATE TABLE IF NOT EXISTS hyperline_webhook_log ( - webhook_id TEXT PRIMARY KEY, - event_type TEXT NOT NULL, - body JSONB NOT NULL, - received_at TIMESTAMPTZ NOT NULL DEFAULT now(), - processed_at TIMESTAMPTZ, - process_error TEXT -); - -CREATE INDEX IF NOT EXISTS idx_hyperline_webhook_log_received - ON hyperline_webhook_log (received_at DESC); - --- ============================================================================ --- Provider event idempotency backstop --- ============================================================================ --- `add_credits_from_provider` (and the invoice.* webhook branches) gate on a --- SELECT EXISTS check against `metadata->>'provider' / 'provider_event_id'`, --- but the read+insert isn't atomic — two concurrent webhook deliveries with --- the same provider event id could both pass the check and both insert. --- This partial unique index makes that race fail fast at the DB layer rather --- than silently double-crediting; callers should treat unique-violation --- errors as "already processed" and short-circuit. -CREATE UNIQUE INDEX IF NOT EXISTS uniq_transactions_provider_event - ON transactions ((metadata ->> 'provider'), (metadata ->> 'provider_event_id')) - WHERE metadata ? 'provider_event_id'; diff --git a/docs/configuration/environment-variables.mdx b/docs/configuration/environment-variables.mdx index c457c7f2..6ba9baaf 100644 --- a/docs/configuration/environment-variables.mdx +++ b/docs/configuration/environment-variables.mdx @@ -186,23 +186,6 @@ Setting `CLICKHOUSE_URL` automatically enables the analytics API at `/analytics/ | `DEDUP_MINHASH_THRESHOLD` | `0.85` | MinHash Jaccard similarity threshold | | `DEDUP_MAX_FINGERPRINTS` | `10000000` | Maximum stored fingerprints | -## Billing (Hyperline) - -Scrapix uses Hyperline (with Stripe as the downstream payment processor) as the billing system of record. The local credit ledger is still the hot-path gate — these variables wire the API and workers to Hyperline's control plane, ingest host, and webhook signer. All are optional: if `HYPERLINE_API_KEY` is unset the API runs in local-only mode and `/account/billing/portal` returns 503. - -| Variable | Default | Description | -|----------|---------|-------------| -| `HYPERLINE_API_KEY` | — | Hyperline API key. Prefixed `test_` for sandbox or `prod_` for production; the prefix auto-selects the default base URLs. | -| `HYPERLINE_API_BASE` | auto | Control-plane base. Defaults to `https://sandbox.api.hyperline.co` for `test_` keys, `https://api.hyperline.co` for `prod_` keys. | -| `HYPERLINE_INGEST_BASE` | auto | Events ingest host. Defaults to `https://sandbox.ingest.hyperline.co` / `https://ingest.hyperline.co` by the same rule. | -| `HYPERLINE_WEBHOOK_SECRET` | — | `whsec_`-prefixed signing secret for `POST /webhooks/hyperline`. Required to enable webhook processing — absent means no signature verification is possible and the route returns 503. | -| `HYPERLINE_CENTS_PER_CREDIT` | — | How many cents one local credit is worth. Together with `HYPERLINE_DRIFT_TOLERANCE_CENTS`, enables WARN-level drift alerts in the reconcile worker. Both must be set; otherwise the worker stays in pure-observation mode. | -| `HYPERLINE_DRIFT_TOLERANCE_CENTS` | — | Absolute drift tolerance in cents. Drift below this stays at INFO; drift above gets WARN. Set to ~1× a typical hot-path debit cost — see the [reconcile section of the runbook](/operations/hyperline-billing#reconcile-scan). | - - -On boot the API pings Hyperline once and logs the event-type manifest so ops can cross-check that every event we emit has a matching metered product in the Hyperline dashboard. A failed ping is non-fatal — the outbox drain worker retries on its own cadence — so a transient Hyperline outage does not crash the API. - - ## Object Storage (S3-compatible) | Variable | Default | Description | diff --git a/docs/guides/billing.mdx b/docs/guides/billing.mdx index 11166ed6..875785ae 100644 --- a/docs/guides/billing.mdx +++ b/docs/guides/billing.mdx @@ -7,8 +7,6 @@ description: Understand billing tiers, usage limits, and how to monitor resource Scrapix uses a tier-based billing model that controls crawl limits, rate limits, and feature access per account. Billing enforcement is automatic when authentication is enabled. -Billing of record is handled by [Hyperline](https://hyperline.co) — invoices, payment methods, auto-recharge, and the hosted customer portal are all managed there (Stripe sits behind Hyperline as the payment processor). The local credit ledger is still the runtime gate on every request; Hyperline mirrors it via webhook. Operators should also read the [Hyperline billing runbook](/operations/hyperline-billing). - ## Billing Tiers | Limit | Free | Starter | Pro | Enterprise | @@ -68,20 +66,10 @@ curl http://localhost:8080/account/billing -b cookies.txt ```json { "tier": "starter", - "credits_balance": 12345, - "monthly_spend_limit": null, - "payment_method_status": null, - "hyperline_customer_id": "cus_…", - "hyperline_wallet_id": "wal_…", - "hyperline_wallet_balance": 12345, - "hyperline_wallet_currency": "USD" + "stripe_customer_id": null } ``` - -`credits_balance` is the local ledger balance (authoritative for rate-gating). `hyperline_wallet_balance` is the live Hyperline wallet amount in the smallest currency unit (cents for USD). The two should match within a few seconds — if they diverge, see the [drift backstop section of the runbook](/operations/hyperline-billing#reconcile-scan). - - ### Account Details ```bash diff --git a/docs/mint.json b/docs/mint.json index ffd40fd6..c4a6befa 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -106,8 +106,7 @@ "pages": [ "operations/monitoring", "operations/scaling", - "operations/troubleshooting", - "operations/hyperline-billing" + "operations/troubleshooting" ] } ], diff --git a/docs/operations/hyperline-billing.mdx b/docs/operations/hyperline-billing.mdx deleted file mode 100644 index 220dc6b7..00000000 --- a/docs/operations/hyperline-billing.mdx +++ /dev/null @@ -1,218 +0,0 @@ ---- -title: Hyperline Billing Runbook -description: Operating the Hyperline-backed billing pipeline — webhooks, outbox, reconciliation, and replay procedures. ---- - -# Hyperline Billing Runbook - -Scrapix uses Hyperline as the billing system of record (with Stripe configured as Hyperline's downstream payment processor). The local credit ledger remains the runtime gate on every `/scrape`, `/crawl`, and `/map` request; Hyperline owns invoices, wallet balances, payment methods, and the hosted customer portal. - -This page covers the pieces of the pipeline that ops are expected to operate: - -1. Configuration surface -2. Boot self-check -3. Outbox drain — emitting usage events to Hyperline -4. Webhooks — receiving state updates from Hyperline -5. Reconcile scan — drift detection between local ledger and Hyperline wallet -6. Replay procedures for each path - -## Configuration - -Set these on the API and worker services (see [environment variables](/configuration/environment-variables#billing-hyperline)): - -| Variable | Required | Notes | -|----------|----------|-------| -| `HYPERLINE_API_KEY` | yes | `test_` or `prod_` — prefix auto-selects base URLs. | -| `HYPERLINE_API_BASE` | no | Override control-plane host. | -| `HYPERLINE_INGEST_BASE` | no | Override events ingest host. | -| `HYPERLINE_WEBHOOK_SECRET` | yes, for webhooks | `whsec_…` — required to verify `POST /webhooks/hyperline`. | - -Any missing var degrades that surface to off (not error). A missing `HYPERLINE_API_KEY` means the outbox still enqueues — rows accumulate and drain once the env var is set and a pod restarts. - -## Boot Self-Check - -On startup the API logs a single manifest line: - -```text -INFO hyperline: Hyperline event-type manifest — verify each has a metered product in the Hyperline dashboard - sandbox=false event_types=["page_crawled","bytes_downloaded","js_render","api_request","document_indexed","feature_format","ai_feature"] -INFO hyperline: Hyperline boot self-check passed sandbox=false -``` - -If you add a new `BillingEventType` variant, this log reminds ops to create the matching metered product in Hyperline **before** the deploy lands — otherwise the event ingest will 4xx and the outbox will back up. - -A `WARN` instead of the second `INFO` (`Hyperline boot self-check failed — API unreachable or auth invalid`) means the key is wrong or Hyperline is down. Boot continues either way. - -## Outbox Drain - -**Shape:** `hyperline_events_outbox(id UUID PK, account_id, event_type, payload JSONB, created_at, sent_at, attempts)` — one row per billable debit, enqueued immediately after the local ledger debit returns. - - -The enqueue is intentionally *not* atomic with the ledger debit yet — `bins/scrapix-api/src/billing.rs::emit_usage_event` runs after `auto_topup::check_credits_and_deduct`. If the enqueue fails after a successful debit, you'll see a `failed to enqueue Hyperline outbox row` warning in the API logs and the customer was debited but no usage event was emitted. The reconcile scan (below) is the backstop until the ledger is refactored to accept a shared transaction handle. - - -**Drain worker:** `spawn_drain_worker(pool, client, interval=5s, batch=100)` — runs inside the API process. Each iteration claims rows with `sent_at IS NULL`, POSTs them to `{ingest_base}/v1/events`, and stamps `sent_at = now()` on 2xx. Failures bump `attempts`; exponential backoff is implicit via the next interval tick. - -**Dedupe key:** `record.id = `. Hyperline dedupes on this field, so a retry after a partial failure will not double-bill. - -### Healthy state - -```sql -SELECT count(*) FROM hyperline_events_outbox WHERE sent_at IS NULL; --- expect < 500 in steady state; < 5000 during burst recovery -``` - -### Symptom: outbox backlog climbing - -Likely causes (in order of frequency): - -1. Hyperline API key expired / rotated — check the boot self-check log on the most recent API restart. -2. Hyperline ingest is returning 4xx — inspect attempts: - -```sql -SELECT event_type, count(*), max(attempts) -FROM hyperline_events_outbox -WHERE sent_at IS NULL AND attempts > 3 -GROUP BY 1 ORDER BY 2 DESC; -``` - -3. An unknown `event_type` hit the outbox — e.g. a newly added variant without a matching Hyperline product. Either create the product in Hyperline's dashboard or roll the deploy back; the rows will drain themselves on the next tick once the product exists. - -### Manual flush / replay - -Rows are idempotent on `record.id`, so re-running the drain is safe: - -```bash -# Nudge a restart; the worker picks up unsent rows on first tick. -kubectl rollout restart deploy/scrapix-api -n scrapix -``` - -For surgical replay of a specific window, clear `attempts` so they aren't throttled: - -```sql -UPDATE hyperline_events_outbox -SET attempts = 0 -WHERE sent_at IS NULL AND created_at > now() - interval '1 hour'; -``` - -## Webhooks - -**Endpoint:** `POST /webhooks/hyperline` - -**Signature scheme:** Svix-style. Hyperline sends `webhook-id`, `webhook-timestamp`, and `webhook-signature` headers. We verify HMAC-SHA256 over `id.timestamp.body` using the base64-decoded portion of `HYPERLINE_WEBHOOK_SECRET` (after stripping the `whsec_` prefix). - -**Anti-replay:** -- Timestamp skew > 300s → 400. -- Existing `webhook-id` → `INSERT ... ON CONFLICT DO NOTHING` on `hyperline_webhook_log` returns no-op and the handler early-exits. Second line of defense: `add_credits_from_provider` also checks `(provider, provider_event_id)` in the ledger. - -**Dispatched events:** - -| Event type | Side effect | -|------------|-------------| -| `wallet.credited`, `credit.topup_transaction_created` | Credit the local ledger, write a `wallet_credit` transaction row. **Requires a `credits` field on the event `data.object`** — until Hyperline's metered products surface this, the handler returns an error rather than silently acking, and `process_error` on the webhook log row will read `wallet.credited missing fields …`. See the [pricing field section](#wallet-credited-pricing-field) below. | -| `wallet.debited` | Reconciliation + drift log (we already debited locally). | -| `wallet.low_projected_balance` | Email the account owner via Resend (if configured). | -| `invoice.settled` | Record transaction, clear `payment_method_status`. | -| `invoice.errored` | Record transaction, keep `payment_method_status` set. | -| `payment_method.errored`, `payment_method.expired` | Set `accounts.payment_method_status = 'errored'\|'expired'`. | -| `subscription.cancelled` | Set `accounts.active = false`. | -| _unknown_ | ACK 200 + log. We don't want Hyperline to retry forever on events we haven't wired up. | - -### Symptom: webhook dashboard shows delivery failures - -Inspect `hyperline_webhook_log` for what actually landed: - -```sql -SELECT event_type, received_at, processed_at -FROM hyperline_webhook_log -ORDER BY received_at DESC -LIMIT 20; -``` - -If `processed_at IS NULL` for a row, the handler errored after dedupe insert — check the API logs for that `webhook-id`. - -### Manual replay - -The safest replay is re-sending the webhook from Hyperline's dashboard. On our side: - -```sql --- Clear dedupe for a specific webhook-id so the next delivery processes again. -DELETE FROM hyperline_webhook_log WHERE webhook_id = 'evt_…'; -``` - - -Deleting a row from `hyperline_webhook_log` bypasses dedupe. The ledger's own `(provider, provider_event_id)` idempotency check still prevents double-crediting for `wallet.credited`, but `invoice.*` and `payment_method.*` side effects can re-apply. Prefer Hyperline's dashboard replay. - - -## Reconcile Scan - -**Purpose:** drift backstop. Every 15 minutes a worker pages through accounts with a `hyperline_wallet_id` and pairs the local `credits_balance` with the live wallet `balance.amount`. - -**Tune:** interval is hard-coded to 15 minutes; Hyperline is authoritative via webhooks on the millisecond scale, so this worker is only a backstop. Tighter intervals waste API quota for no gain. - -### Drift alerting (opt-in) - -Local credits and Hyperline `balance.amount` live in different units (credits vs. cents), so the worker can't compare them without a ratio. Drift WARN logs are gated on two env vars: - -| Variable | Description | -|----------|-------------| -| `HYPERLINE_CENTS_PER_CREDIT` | How many cents one local credit is worth. Site-specific pricing. | -| `HYPERLINE_DRIFT_TOLERANCE_CENTS` | Absolute tolerance (cents). Drift below this stays at INFO; drift above gets WARN. | - -When **both** are set, the comparator is `local_credits × cents_per_credit − hyperline_balance`. The `delta_cents` value appears on every paired observation log line (positive = local ledger ahead, negative = behind). Set the tolerance to ~1× a typical hot-path debit cost — that's enough to absorb in-flight outbox rows without crying wolf. - -When either var is unset (default) the worker stays in pure-observation mode: paired observations log at INFO, no WARNs. - -**Healthy log line:** - -```text -INFO reconcile: paired balance observation (within tolerance) - account_id=… local_credits=12345 hyperline_balance=12345 delta_cents=0 currency="USD" -``` - -**Drift breach:** - -```text -WARN reconcile: balance drift exceeds tolerance - account_id=… local_credits=12345 hyperline_balance=12000 - cents_per_credit=1 tolerance_cents=5 delta_cents=345 currency="USD" -``` - -A drift warning means one of: - -1. An outbox row hasn't drained yet (check `sent_at IS NULL` count). -2. A webhook replay applied to the ledger but not the wallet (rare — Hyperline owns the wallet side). -3. A manual ledger edit (admin credit) without a matching Hyperline adjustment. - -### Correcting drift - -If (3) — reconcile by issuing the matching move in Hyperline's dashboard (admin credit note / debit). If (1) — nudge the drain worker (see above). If (2) — contact Hyperline support with the `webhook-id`. - -## Wallet-credited pricing field - -`wallet.credited` and `credit.topup_transaction_created` events must carry a `credits` integer on `data.object` so the webhook handler can mirror the Hyperline-side credit into the local ledger. This is a custom property surfaced by the metered-product config in Hyperline's dashboard. - -**If the field is missing on a real top-up, the customer paid but the local ledger doesn't move.** The handler refuses to silently ack — it returns an error, which lands in `hyperline_webhook_log.process_error` as `wallet.credited missing fields …`. Find stuck rows with: - -```sql -SELECT webhook_id, event_type, received_at, process_error -FROM hyperline_webhook_log -WHERE process_error LIKE 'wallet.credited missing fields%' -ORDER BY received_at DESC; -``` - -To recover, either add the missing custom property in the Hyperline product config and replay the webhook, or apply the credit manually via the admin ledger and dismiss the row. - -## Failure Matrix - -| Failure | Blast radius | Caller impact | -|---------|--------------|---------------| -| `HYPERLINE_API_KEY` invalid | Outbox backs up, live wallet reads in `GET /account/billing` return local-only | Users see "Billing portal unavailable" on `GET /account/billing/portal` (503). Scrape / crawl / map still work. | -| Hyperline API down | Outbox backs up; `GET /account/billing/portal` 502s | Same as above. Local ledger is unaffected. | -| `HYPERLINE_WEBHOOK_SECRET` wrong | All `POST /webhooks/hyperline` return 400 | Top-ups won't credit until the secret is fixed and webhooks are re-delivered from the Hyperline dashboard. | -| Postgres down | Everything degrades | Handled by the existing API DB-outage runbook, not specific to Hyperline. | - -## Related - -- [Environment variables](/configuration/environment-variables#billing-hyperline) -- [Billing & usage overview](/guides/billing) From ca08a029d969fafcbe206c2425bf0e87589e756e Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 May 2026 17:30:15 +0200 Subject: [PATCH 4/5] fix(db): re-add stripe_customer_id idempotent migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SCR-68 Hyperline migration explicitly DROPPED stripe_customer_id from accounts. The reverted init.sql only had idempotent ADD COLUMNs for the other dropped columns (stripe_default_payment_method_id, auto_topup_*), not for stripe_customer_id itself — so production deploy would 500 on the first /account/billing request because stripe.rs heavily queries this column. Adds an IF NOT EXISTS block so the next API boot self-heals the schema. --- deploy/postgres/init.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/postgres/init.sql b/deploy/postgres/init.sql index 20b0c00e..891bc8ba 100644 --- a/deploy/postgres/init.sql +++ b/deploy/postgres/init.sql @@ -337,6 +337,12 @@ DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'monthly_spend_limit') THEN ALTER TABLE accounts ADD COLUMN monthly_spend_limit BIGINT; END IF; + -- Re-add Stripe columns dropped by the SCR-68 Hyperline migration that this + -- revert is rolling back. Safe to re-add empty: zero customers as of revert, + -- and stripe.rs lazily mints customers on first /account/billing request. + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'stripe_customer_id') THEN + ALTER TABLE accounts ADD COLUMN stripe_customer_id TEXT; + END IF; IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'accounts' AND column_name = 'stripe_default_payment_method_id') THEN ALTER TABLE accounts ADD COLUMN stripe_default_payment_method_id TEXT; END IF; From 82ffa007b8ef20819c9f887cb4dffb37c2dc391d Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 May 2026 10:08:32 +0200 Subject: [PATCH 5/5] feat(console): use live Stripe publishable key for production builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NEXT_PUBLIC_* vars are inlined by Next.js at build time, so this needs to be a Dockerfile ENV (not a Fly runtime secret). Publishable keys are public by design — safe to commit, they appear in every browser bundle anyway. --- console/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/console/Dockerfile b/console/Dockerfile index 3a936c83..e25c3d1d 100644 --- a/console/Dockerfile +++ b/console/Dockerfile @@ -16,7 +16,7 @@ ENV NEXT_TELEMETRY_DISABLED=1 # Next.js inlines NEXT_PUBLIC_* vars at build time. In Docker builds # (e.g. Heroku), runtime config vars aren't available during build, # so we hardcode defaults here. The publishable key is public by design. -ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51TAwQdRGrCEv0wspTJFC1nDzrfhVwg2pxBtfAplfdjdTDkB8kOxrVAodNPYjehdMhIzTCY4HUnJ6I21ELXuKiXDq00pDyrnwvj +ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_live_51TAwQVEi7PX2EfAShr2ErTsNBnMoU4QFOSnAUowQtdrm7hITOoN1Hw0wRdOiE5O7ENEsuMRiFwGWOk6sAUzRQdAg00wyGy7e1r ENV NEXT_PUBLIC_SCRAPIX_API_URL=https://api.scrapix.meilisearch.com RUN npm run build