feat(validator-debt): better summarized debt (malbeclabs/doublezero-offchain#218)

## Summary of Changes
This PR adds in more details for when debt collection is posted to slack
and also summarizes debt collection for all epochs. It moves some of the
debt-related slack composition to the validator-debt crate so we can
reuse the existing structs instead of a many-arity function. It also
adds total debt, total paid, and insufficient payment counts to the pay
debt GenServer which are then posted to slack at the end of a debt
collection run.

Closes #2256.

## Testing Verification
* This was tested locally in a private slack channel:

<img width="1269" height="702" alt="Screenshot 2025-12-02 at 17 19 46"
src="https://github.com/user-attachments/assets/a8f75951-01e0-4ba4-a4f6-9d8cf2657dab"
/>

Author: bgm-malbeclabs

offchain/Cargo.lock

@@ -39,7 +39,7 @@ pub async fn post_distribution_to_slack(
         transaction.unwrap_or("No transaction details".to_string()),
     ];
 
-    post_to_slack(filepath, client, header, table_header, table_values).await?;
+    post_to_slack(filepath, &client, header, table_header, table_values).await?;
 
     Ok(())
 }
@@ -60,51 +60,14 @@ pub async fn post_finalized_distribution_to_slack(
 
     let table_values = vec![dz_epoch.to_string(), finalized_sig.to_string()];
 
-    post_to_slack(None, client, header, table_header, table_values).await?;
+    post_to_slack(None, &client, header, table_header, table_values).await?;
 
     Ok(())
 }
 
-pub async fn post_debt_collection_to_slack(
-    total_transactions: usize,
-    total_success: usize,
-    insufficient_funds: usize,
-    already_paid: usize,
-    dz_epoch: u64,
-    filepath: Option<String>,
-    dry_run: bool,
-) -> Result<()> {
-    let client = reqwest::Client::new();
-    let header = if dry_run {
-        "DRY RUN Debt Collected DRY RUN"
-    } else {
-        "Debt Collected"
-    };
-
-    let table_header = vec![
-        "DoubleZero Epoch".to_string(),
-        "Total Attempted Transactions".to_string(),
-        "Successful Transactions".to_string(),
-        "Insufficient Funds".to_string(),
-        "Already Paid".to_string(),
-    ];
-
-    let table_values = vec![
-        dz_epoch.to_string(),
-        total_transactions.to_string(),
-        total_success.to_string(),
-        insufficient_funds.to_string(),
-        already_paid.to_string(),
-    ];
-
-    post_to_slack(filepath, client, header, table_header, table_values).await?;
-
-    Ok(())
-}
-
-async fn post_to_slack(
+pub async fn post_to_slack(
     filepath: Option<String>,
-    client: Client,
+    client: &Client,
     header: &str,
     mut table_header: Vec<String>,
     mut table_values: Vec<String>,
@@ -121,7 +84,7 @@ async fn post_to_slack(
 
     let payload = serde_json::to_string(&msg)?;
     let body = Body::from(payload);
-    let request = slack::build_message_request(&client, body, slack_webhook()?)?;
+    let request = slack::build_message_request(client, body, slack_webhook()?)?;
     let _resp = request.send().await?;
 
     Ok(())

offchain/crates/slack-notifier/src/validator_debt.rs

@@ -13,7 +13,6 @@ use doublezero_solana_client_tools::{
     rpc::DoubleZeroLedgerConnection,
 };
 use doublezero_solana_validator_debt::worker;
-use slack_notifier::validator_debt;
 
 #[derive(Debug, Clone, ValueEnum)]
 pub enum ExportFormat {
@@ -89,23 +88,15 @@ async fn execute_pay_solana_validator_debt(
         };
         let mut writer = csv::Writer::from_path(string_filename.clone())?;
 
-        for tx_result in tx_results.collection_results {
+        for tx_result in tx_results.collection_results.clone() {
             writer.serialize(tx_result)?;
         }
         filename = Some(string_filename);
         writer.flush()?;
     };
+
     if let Some(ExportFormat::Slack) = export {
-        validator_debt::post_debt_collection_to_slack(
-            tx_results.total_transactions_attempted,
-            tx_results.successful_transactions,
-            tx_results.insufficient_funds,
-            tx_results.already_paid,
-            epoch,
-            filename,
-            dry_run,
-        )
-        .await?;
+        worker::post_debt_collection_to_slack(tx_results, dry_run, filename).await?;
     }
 
     Ok(())

offchain/crates/solana-cli/src/command/revenue_distribution/relay/mod.rs

@@ -42,16 +42,16 @@ pub struct Transaction {
     pub force: bool,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub struct DebtCollectionResults {
+    pub dz_epoch: u64,
     pub collection_results: Vec<DebtCollectionResult>,
-    pub total_transactions_attempted: usize,
-    pub successful_transactions: usize,
-    pub insufficient_funds: usize,
-    pub already_paid: usize,
+    pub insufficient_funds: Vec<DebtCollectionResult>,
+    pub already_paid: Vec<DebtCollectionResult>,
+    pub successful_transactions: Vec<DebtCollectionResult>,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub struct DebtCollectionResult {
     pub validator_id: String,
     pub amount: u64,
@@ -362,23 +362,24 @@ impl Transaction {
             }
         }
 
-        let total_transactions = debt_collection_result.len();
-        let total_success = debt_collection_result
-            .iter()
+        let successful_transactions: Vec<DebtCollectionResult> = debt_collection_result
+            .clone()
+            .into_iter()
             .filter(|pr| pr.success)
-            .count();
+            .collect();
 
-        let insufficient_funds_count =
-            count_failed_debt_collection("Insufficient funds", debt_collection_result.as_ref());
-        let already_paid_count =
-            count_failed_debt_collection("Merkle leaf", debt_collection_result.as_ref());
+        let already_paid =
+            count_failed_debt_collection("Merkle leaf", debt_collection_result.clone());
+
+        let insufficient_funds =
+            count_failed_debt_collection("Insufficient funds", debt_collection_result.clone());
 
         let debt_collection_results = DebtCollectionResults {
+            dz_epoch,
+            successful_transactions,
             collection_results: debt_collection_result,
-            total_transactions_attempted: total_transactions,
-            successful_transactions: total_success,
-            insufficient_funds: insufficient_funds_count,
-            already_paid: already_paid_count,
+            insufficient_funds,
+            already_paid,
         };
         Ok(debt_collection_results)
     }
@@ -402,14 +403,17 @@ impl Transaction {
     }
 }
 
-fn count_failed_debt_collection(error_type: &str, dcr: &[DebtCollectionResult]) -> usize {
-    dcr.iter()
+fn count_failed_debt_collection(
+    error_type: &str,
+    dcr: Vec<DebtCollectionResult>,
+) -> Vec<DebtCollectionResult> {
+    dcr.into_iter()
         .filter(|pr| {
             pr.result
                 .as_ref()
                 .is_some_and(|res| res.contains(error_type))
         })
-        .count()
+        .collect()
 }
 
 fn parse_program_logs(

offchain/crates/validator-debt/src/transaction.rs

@@ -522,6 +522,81 @@ pub async fn initialize_distribution(
     Ok(())
 }
 
+pub async fn post_debt_collection_to_slack(
+    debt_collection_results: DebtCollectionResults,
+    dry_run: bool,
+    filepath: Option<String>,
+) -> Result<()> {
+    let client = reqwest::Client::new();
+    let header = if dry_run {
+        "DRY RUN Debt Collected DRY RUN"
+    } else {
+        "Debt Collected"
+    };
+
+    let table_header = vec![
+        "DoubleZero Epoch".to_string(),
+        "Total Paid".to_string(),
+        "Total Debt".to_string(),
+        "Percentage Paid".to_string(),
+        "Total Attempted Transactions".to_string(),
+        "Successful Transactions".to_string(),
+        "Insufficient Funds".to_string(),
+        "Already Paid".to_string(),
+    ];
+
+    let total_attempted_transactions_count = debt_collection_results.collection_results.len();
+    let successful_transactions_count = debt_collection_results.successful_transactions.len();
+    let already_paid_count = debt_collection_results.already_paid.len();
+
+    let percentage_paid: f64 = if total_attempted_transactions_count == 0 {
+        0.0
+    } else {
+        (already_paid_count + successful_transactions_count) as f64
+            / total_attempted_transactions_count as f64
+    };
+
+    // the total amount paid for an epoch is `total_collected_this_run` + `already_paid`
+    let already_paid_total: u64 = debt_collection_results
+        .already_paid
+        .iter()
+        .map(|ap| ap.amount)
+        .sum();
+    let total_collected_this_run: u64 = debt_collection_results
+        .successful_transactions
+        .iter()
+        .map(|ap| ap.amount)
+        .sum();
+
+    let total_paid = already_paid_total + total_collected_this_run;
+    let total_debt: u64 = debt_collection_results
+        .collection_results
+        .iter()
+        .map(|cr| cr.amount)
+        .sum();
+    let table_values = vec![
+        debt_collection_results.dz_epoch.to_string(),
+        total_paid.to_string(),
+        total_debt.to_string(),
+        format!("{:.2}%", percentage_paid * 100.0),
+        total_attempted_transactions_count.to_string(),
+        successful_transactions_count.to_string(),
+        debt_collection_results.insufficient_funds.len().to_string(),
+        already_paid_count.to_string(),
+    ];
+
+    slack_notifier::validator_debt::post_to_slack(
+        filepath,
+        &client,
+        header,
+        table_header,
+        table_values,
+    )
+    .await?;
+
+    Ok(())
+}
+
 async fn create_or_validate_ledger_record(
     solana_debt_calculator: &impl ValidatorRewards,
     transaction: &Transaction,

offchain/crates/validator-debt/src/worker.rs

@@ -5,11 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+- feat(summarize_debt): Summarize debt for each epoch and then for all epochs ([#218](https://github.com/doublezerofoundation/doublezero-offchain/pull/218))
 
 ## [v0.1.3]
-- feat(deploy service): Add deploy steps through actions and goreleaser ([#205]https://github.com/doublezerofoundation/doublezero-offchain/pull/205)
-- feat(automate_finalize_distribution):
-- feat(automate_calculate_distribution): Add GenServer and Rust NIF to automatically calculate a distribution on a configurable interval ([#197]https://github.com/doublezerofoundation/doublezero-offchain/pull/197)
-- feat(automate_initialize_distribution): Add GenServer and Rust NIF to automatically initialize a distribution on a configurable interval ([#197]https://github.com/doublezerofoundation/doublezero-offchain/pull/197)
-- feat(automate_debt_payment): Add GenServer and Rust NIF to automatically collect debt on a configurable interval ([#183]https://github.com/doublezerofoundation/doublezero-offchain/pull/183)
-- feat(scheduler): add Elixir app that manages scheduling and executing Rust processes for debt collection and payment  ([#183]https://github.com/doublezerofoundation/doublezero-offchain/pull/183)
+- feat(deploy service): Add deploy steps through actions and goreleaser ([#205](https://github.com/doublezerofoundation/doublezero-offchain/pull/205))
+- feat(automate_finalize_distribution): Update calculate distribution GenServer to finalize distribution through a Rust NIF ([#200](https://github.com/doublezerofoundation/doublezero-offchain/pull/200))
+- feat(automate_calculate_distribution): Add GenServer and Rust NIF to automatically calculate a distribution on a configurable interval ([#199](https://github.com/doublezerofoundation/doublezero-offchain/pull/199))
+- feat(automate_initialize_distribution): Add GenServer and Rust NIF to automatically initialize a distribution on a configurable interval ([#197](https://github.com/doublezerofoundation/doublezero-offchain/pull/197))
+- feat(automate_debt_payment): Add GenServer and Rust NIF to automatically collect debt on a configurable interval ([#183](https://github.com/doublezerofoundation/doublezero-offchain/pull/183))
+- feat(scheduler): add Elixir app that manages scheduling and executing Rust processes for debt collection and payment  ([#183](https://github.com/doublezerofoundation/doublezero-offchain/pull/183)

offchain/scheduler/CHANGELOG.md

@@ -11,4 +11,7 @@ defmodule Scheduler.DoubleZero do
     do: :erlang.nif_error(:nif_not_loaded)
 
   def current_dz_epoch(_ledger_rpc), do: :erlang.nif_error(:nif_not_loaded)
+
+  def post_debt_summary(_insufficient_funds_count, _total_debt, _total_paid),
+    do: :erlang.nif_error(:nif_not_loaded)
 end

offchain/scheduler/lib/scheduler/scheduler_doublezero.ex

@@ -0,0 +1,10 @@
+defmodule Scheduler.ValidatorDebt.DebtCollection do
+  defstruct total_paid: 0,
+            total_debt: 0,
+            total_validators: 0,
+            insufficient_funds_count: 0
+end
+
+defmodule Scheduler.ValidatorDebt.Debt do
+  defstruct [:validator_id, :amount, :result, :success]
+end

offchain/scheduler/lib/scheduler/validator_debt/debt_collection.ex

@@ -4,7 +4,14 @@ defmodule Scheduler.Worker.PayDebt do
   require Logger
 
   def start_link(_var \\ []) do
-    state = %{genesis_epoch: genesis_epoch(), current_epoch: genesis_epoch()}
+    state = %{
+      genesis_epoch: genesis_epoch(),
+      current_epoch: genesis_epoch(),
+      total_debt: 0,
+      total_paid: 0,
+      insufficient_funds_count: 0
+    }
+
     GenServer.start_link(__MODULE__, state, name: __MODULE__)
   end
 
@@ -24,18 +31,34 @@ defmodule Scheduler.Worker.PayDebt do
         if String.contains?(error, "Record account not found at address") ||
              String.contains?(error, "Failed to fetch record") do
           Logger.info("scheduler completed sweep at epoch #{state.current_epoch}")
+
+          Scheduler.DoubleZero.post_debt_summary(
+            state.insufficient_funds_count,
+            state.total_debt,
+            state.total_paid
+          )
+
           {:stop, :shutdown, state}
         else
           Logger.error(
             "scheduler encountered unexpected error at epoch #{state.current_epoch}: #{inspect(error)}"
           )
 
-          {:stop,:shutdown, state}
+          {:stop, :shutdown, state}
         end
 
-      _ ->
+      debt ->
         Logger.info("completed epoch #{state.current_epoch}")
-        state = %{state | current_epoch: state.current_epoch + 1}
+
+        state = %{
+          state
+          | current_epoch: state.current_epoch + 1,
+            total_debt: state.total_debt + debt.total_debt,
+            total_paid: state.total_paid + debt.total_paid,
+            insufficient_funds_count:
+              state.insufficient_funds_count + debt.insufficient_funds_count
+        }
+
         {:noreply, state, {:continue, :queue_debt_payment}}
     end
   end

offchain/scheduler/lib/scheduler/worker/pay_debt.ex

@@ -7,7 +7,7 @@ defmodule Scheduler.MixProject do
       version: "0.1.0",
       elixir: "~> 1.18",
       start_permanent: Mix.env() == :prod,
-      deps: deps(),
+      deps: deps()
     ]
   end