feat: implement fallback rpcs and assets to hold redistribution feature

README.md

@@ -62,7 +62,7 @@ The `engine` crate contains the lending model, the reactor loop, and the trait d
 
 **`liquidator`** models obligations and oracle prices per market, estimates net profit after gas, swap costs, and `liquidator_min_profit_margin_cents`, and picks the most profitable (borrow, deposit) pair to liquidate. It chooses between three execution modes based on the keeper's on-hand liquidity: `Direct` (the keeper already holds enough of the repayment asset), `PreSwap` (swap a held asset into the repayment asset first, then liquidate), or `Flash` (flash-borrow the repayment asset from the pool, seize the collateral, swap it back, and repay the flash loan atomically). Non-target collateral received from liquidations is later swapped by the rebalancer; assets in `assets_to_hold` are kept.
 
-**`rebalancer`** (the `Balancer` strategy, config prefix `balancer_*`) runs every `balancer_refresh_interval_blocks` ledgers. It walks the wallet and swaps each non-target asset whose dollar value exceeds `balancer_min_swap_amount_value_cents` into the rebalancer target (the first entry of `assets_to_hold`). Trade size is capped so on-chain price impact stays under `balancer_max_price_impact_bps`, probing progressively smaller sizes up to `balancer_max_swap_provider_probes` times per provider; `balancer_max_allowed_swap_slippage_bps` is applied on top when constructing `min_amount_out`. Retries up to `balancer_max_retries` times on failure.
+**`rebalancer`** (the `Balancer` strategy, config prefix `balancer_*`) runs every `balancer_refresh_interval_blocks` ledgers. It walks the wallet and swaps each non-target asset whose dollar value exceeds `balancer_min_swap_amount_value_cents` into the rebalancer target (the first entry of `assets_to_hold`). Trade size is capped so on-chain price impact stays under `balancer_max_price_impact_bps`, probing progressively smaller sizes up to `balancer_max_swap_provider_halving_probes` times per provider; `balancer_max_allowed_swap_slippage_bps` is applied on top when constructing `min_amount_out`. Retries up to `balancer_max_retries` times on failure.
 
 **`withdrawer`** watches the keeper's own deposits and pulls idle supply out of pools once it can do so without pushing utilisation past `withdrawer_utilization_safety_margin_bps`. Withdrawals below `withdrawer_min_withdraw_value_cents` are skipped.
 

config.example.json

@@ -0,0 +1,47 @@
+{
+    "rpc_url": "https://soroban-testnet.stellar.org",
+    "fallback_rpc_urls": ["https://sorban-testnet.stellar.org"],
+    "rpc_max_call_duration_secs": 30,
+    "db_path": "./data.db",
+    "markets": [
+        "CB5IJJRIYYF2BJESDFIEERGCX7DCDWRWBTYXSMSQF2TXSZ2MVQYWXJYT"
+    ],
+    "xlm_address": "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC",
+    "hub_address": "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA",
+    "xlm_safety_margin": 2000000000,
+    "default_simulation_fee": 100000,
+    "network_passphrase": "Test SDF Network ; September 2015",
+    "assets_to_hold": {
+        "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC": 1000,
+        "CDNVQW44C3HALYNVQ4SOBXY5EWYTGVYXX6JPESOLQDABJI5FC5LTRRUE": 4000
+    },
+    "swap_providers": [
+        "CCRIEORD53E4TSYS33JYSQVN5N6AQCB7JUSVROWFHHKQUBDDDZTXOLU7",
+        "CCBMBQGNR3NCR2FPN3AYQMJ65UMGQHCOJUB34RQCGIYMX77D5HUUSQDG"
+    ],
+    "metrics_bind_addr": "0.0.0.0:9000",
+    "readiness_staleness_budget_secs": 120,
+    "event_collector_start_ledger": 3724633,
+    "ledger_collector_polling_interval_secs": 2,
+    "keeper_capital_balance_ttl_secs": 10,
+    "keeper_capital_reservation_ttl_secs": 30,
+    "bad_debt_request_initiator_max_retries": 3,
+    "bad_debt_request_initiator_refresh_interval_blocks": 5,
+    "withdrawer_max_retries": 3,
+    "withdrawer_refresh_interval_blocks": 5,
+    "withdrawer_min_withdraw_value_cents": 500,
+    "withdrawer_utilization_safety_margin_bps": 500,
+    "liquidator_max_retries": 3,
+    "liquidator_refresh_interval_blocks": 2,
+    "liquidator_min_profit_margin_cents": 0,
+    "liquidator_max_allowed_swap_slippage_bps": 0,
+    "balancer_max_retries": 3,
+    "balancer_refresh_interval_blocks": 2,
+    "balancer_max_allowed_swap_slippage_bps": 30,
+    "balancer_max_execution_impact_bps": 1200,
+    "balancer_rebalance_threshold_bps": 500,
+    "balancer_max_swaps_per_batch": 5,
+    "balancer_max_swap_provider_halving_probes": 3,
+    "balancer_min_swap_amount_value_cents": 100,
+    "balancer_max_oracle_price_spread_bps": 200
+}

config.example.toml

@@ -10,6 +10,17 @@
 # latency directly caps how fast the bot reacts. Use one you trust.
 rpc_url = "https://soroban-testnet.stellar.org"
 
+# Optional fallback Soroban RPC endpoints. When the primary `rpc_url` is
+# unreachable (times out / connection reset / 5xx), the keeper fails over to
+# these in order and sticks to the last-known-good node. Leave empty to run
+# against a single endpoint.
+fallback_rpc_urls = []
+
+# Wall-clock budget for a single logical RPC call across all failover attempts.
+# Once exceeded, the failover loop stops trying further endpoints and returns
+# the last transport error. Optional; defaults to 30.
+rpc_max_call_duration_secs = 30
+
 # Local SQLite file: event cursor + obligation cache. Deleting it forces a
 # full event re-sync from `event_collector_start_ledger` on next start.
 db_path = "./data.db"
@@ -22,6 +33,12 @@ markets = ["CCUDRNK5K5UR6NLYKAQROCBUQJBVY66DF62NDMKVH7CQ7W46ASJAJPBI"]
 # `xlm_safety_margin` applies to.
 xlm_address = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"
 
+# Stablecoin "hub" the Balancer prices the portfolio in and routes every
+# rebalance swap through. It has no explicit target weight — its share is the
+# residual left after the assets_to_hold targets. Must NOT appear in
+# assets_to_hold.
+hub_address = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"
+
 # Min XLM (stroops) held back for inclusion fees and reserves. 1 XLM =
 # 10_000_000 stroops, so 2000000000 = 200 XLM. Usable XLM = balance − margin.
 xlm_safety_margin = 2000000000
@@ -33,12 +50,12 @@ default_simulation_fee = 10000
 # Mainnet: "Public Global Stellar Network ; September 2015".
 network_passphrase = "Test SDF Network ; September 2015"
 
-# Assets the keeper keeps on its balance. The FIRST entry is the Balancer's
-# swap target — anything not in this list is rebalanced into assets_to_hold[0].
-assets_to_hold = [
-    "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA",
-    "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC",
-]
+# Assets the keeper keeps on its balance, mapped to a target share of held
+# value in basis points (bps; 10000 = 100%). These are the *volatile* assets;
+# the hub (hub_address) is NOT listed here. The distributions must sum to at
+# most 10000 — the remainder is the implied hub floor. Anything held but not in
+# this map (nor the hub) is sold into the hub.
+assets_to_hold = { CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC = 4000 }
 
 # DEX-adapter contract IDs the keeper may route swaps through. More providers
 # = better fills but more RPC calls per evaluation.
@@ -102,17 +119,38 @@ liquidator_min_profit_margin_cents = 0
 liquidator_max_allowed_swap_slippage_bps = 0
 
 # --- Balancer ---
-# Rebalances stray assets into assets_to_hold[0].
+# Rebalances the held portfolio back toward the assets_to_hold target weights,
+# using the hub asset as the swap counterparty. Sells surpluses into the hub,
+# then buys deficits with the hub, in one atomic batch per cycle.
 
 # Max submission attempts (1–50).
 balancer_max_retries = 3
 # Re-scan cadence in ledgers (≥ 1).
 balancer_refresh_interval_blocks = 2
 # Max swap slippage tolerated on a rebalance (bps, 0–10000).
 balancer_max_allowed_swap_slippage_bps = 30
-# Max price impact tolerated on a rebalance (bps, 0–10000).
-balancer_max_price_impact_bps = 1200
-# How many swap providers to probe when pricing a rebalance (≥ 1).
-balancer_max_swap_provider_probes = 3
+# Max execution price impact tolerated on a rebalance leg vs oracle (bps,
+# 0–10000). Legs are halved until they clear this ceiling.
+balancer_max_execution_impact_bps = 1200
+# Tolerance band around each asset's target weight before it's rebalanced
+# (bps, 0–10000; 500 = 5 percentage points).
+balancer_rebalance_threshold_bps = 500
+# Max swap legs packed into one atomic rebalance batch (≥ 1).
+balancer_max_swaps_per_batch = 5
+# How many times a leg's input is halved while searching for a size that clears
+# the execution-impact ceiling (≥ 1).
+balancer_max_swap_provider_halving_probes = 3
 # Skip rebalances below this value (USD cents, ≥ 0).
 balancer_min_swap_amount_value_cents = 100
+# Abort the whole cycle if any asset's cross-market oracle spread exceeds this (bps, 0-10000).
+balancer_max_oracle_price_spread_bps = 200
+
+# Optional: which strategies to run. Omit or leave empty to run all.
+# Valid values: "liquidator", "withdrawer", "balancer", "bad-debt".
+# A `--strategies` CLI flag overrides this.
+# strategies = ["liquidator", "balancer"]
+
+# Optional: which liquidation types the liquidator may use. Omit/empty = all.
+# Valid values: "direct", "preswap", "flash".
+# A `--liquidation-types` CLI flag overrides this.
+# liquidation_types = ["direct", "flash"]

docs/configuration.md

@@ -21,10 +21,11 @@ Shared infrastructure and cross-strategy capital settings. These are consumed in
 | `db_path` | path | — | Local SQLite file. Persists the event cursor and the obligation cache. Losing it forces a full event re-sync from `event_collector_start_ledger` on the next start. |
 | `markets` | strkey[] | ≥ 1 entry, each a 56-char `C…`/`G…` address | Alula pool (money-market) contract IDs to watch. The liquidator, withdrawer, and bad-debt initiator fan out across **all** of them(currently, one only single market is supported).|
 | `xlm_address` | strkey | exactly 56 chars | SAC contract address for native XLM on the chosen network. Identifies which token the `xlm_safety_margin` reserve applies to; balances of this asset are shaved by the margin before the liquidator or balancer will spend them. |
+| `hub_address` | strkey | exactly 56 chars, **not** a key in `assets_to_hold` | Stablecoin "hub" the Balancer prices the whole portfolio in and routes every rebalance swap through (surpluses are sold into it, deficits bought with it). It has **no explicit target weight** — its share is the residual left after the `assets_to_hold` targets. |
 | `xlm_safety_margin` | i128 (stroops) | ≥ 1 | Minimum XLM balance to hold back for inclusion fees and trustline reserves. 1 XLM = 10 000 000 stroops (`2000000000` = 200 XLM). Whenever XLM is a spend/swap source, the usable amount is `balance − xlm_safety_margin`.|
 | `default_simulation_fee` | u32 (stroops) | ≥ 100 | Starting `fee` on every transaction the executor builds, and a **floor** on the final fee.|
 | `network_passphrase` | string | non-empty | Stellar network passphrase; hashed into the signature payload, so it must match the network `rpc_url` points at or every submission is rejected. Testnet: `Test SDF Network ; September 2015`. Mainnet: `Public Global Stellar Network ; September 2015`. |
-| `assets_to_hold` | strkey[] | ≥ 1 entry, valid addresses | Assets the keeper wants to keep on its balance. **The first entry is the Balancer's swap target(for now)** — everything not in this list gets rebalanced into `assets_to_hold[0]`. The liquidator also treats these as the candidate *source* assets for pre-swap liquidations (swaps a held asset into the repay asset). |
+| `assets_to_hold` | map `strkey -> u16` (bps) | ≥ 1 entry, valid addresses, values 0–10000 and summing to **at most** 10000 | Target weights for the *volatile* assets the keeper holds, in basis points of total held value. The hub (`hub_address`) is **not** listed here; its share is the residual up to 10000. Each cycle the Balancer sells assets whose weight is above target (by more than the threshold) into the hub, then buys assets below target with the hub. The liquidator also treats these as the candidate *source* assets for pre-swap liquidations. |
 | `swap_providers` | strkey[] | ≥ 1 entry, valid addresses | DEX-adapter contract IDs the keeper may route swaps through. The liquidator probes each to price its flash/pre-swap legs; the Balancer probes each to price rebalances. More providers = better fills but more RPC calls per evaluation. |
 | `metrics_bind_addr` | `host:port` | valid socket addr | Address the Prometheus `/metrics` endpoint binds to. Use `0.0.0.0:9000` inside docker compose so the `prometheus` container can scrape it; use `127.0.0.1:9000` for local-only exposure. The same address also serves `/healthz` (liveness) and `/readyz` (readiness). |
 | `readiness_staleness_budget_secs` | u64 (seconds) | ≥ 1, **optional** (default `120`) | How long the keeper may go without completing a scan tick before `/readyz` returns `503`. Set it comfortably above your slowest `*_refresh_interval_blocks` cadence (blocks × ~5 s) so an idle-but-healthy keeper doesn't flap between ready and not-ready. |
@@ -84,20 +85,23 @@ The core strategy: scans cached obligations each interval, finds under-collatera
 
 ## Balancer
 
-Converts non-target assets in the keeper's wallet back into the target asset (`assets_to_hold[0]`) via on-chain AMMs, so collateral seized in liquidations doesn't sit around as odd tokens. **Operates on `markets[0]` only.** Runs each interval and also reacts to the keeper's own `Liquidate`/`Withdraw` events.
+Rebalances the held portfolio back toward the `assets_to_hold` target weights, using `hub_address` as the swap counterparty. Each cycle it values the whole portfolio in the hub's numeraire, finds assets that have drifted past the threshold, **sells** surpluses into the hub then **buys** deficits with the hub — all in a single atomic `submit_requests_batch` of up to `balancer_max_swaps_per_batch` legs. **Operates on `markets[0]` only.** Runs each interval and also reacts to the keeper's own `Liquidate`/`Withdraw` events.
 
 | Field | Type / Unit | Constraints | Description |
 |---|---|---|---|
 | `balancer_max_retries` | u32 | 1–50 | Max submission attempts for a rebalance swap transaction. |
 | `balancer_refresh_interval_blocks` | u32 (ledgers) | ≥ 1 | Re-evaluation cadence; fires on ledgers that are exact multiples of this value. `2` ⇒ every other ledger (~10 s on testnet). |
 | `balancer_max_allowed_swap_slippage_bps` | i128 (bps) | 0–10000 | External slippage buffer applied **after** the price-impact check, when constructing `min_amount_out` (`amount_out × (10000 − bps)/10000`). Protects the swap from moving between quote and execution. `30` = 0.30 %. |
-| `balancer_max_price_impact_bps` | i128 (bps) | 0–10000 | Hard cap on a swap's price impact **relative to the oracle price**. For each provider the balancer starts at the full swappable balance and, if the quote's impact exceeds this cap, halves the size and re-probes. `1200` = 12 %.|
-| `balancer_max_swap_provider_probes` | u32 | ≥ 1 | How many halving attempts the size-search makes per provider before giving up on that route. With `max_price_impact_bps`, this bounds the binary-search-style descent: `3` probes tries full, ½, ¼ of the balance. Higher = more chances to find a fitting size, at more RPC calls. |
+| `balancer_max_execution_impact_bps` | i128 (bps) | 0–10000 | Hard cap on a single leg's execution price impact **relative to the oracle price**. The balancer sizes a leg to fully correct the drift and, if the quote's impact exceeds this cap, halves the size and re-probes, taking the largest slice that clears the cap. `1200` = 12 %.|
+| `balancer_rebalance_threshold_bps` | u16 (bps) | 0–10000 | Tolerance band around each asset's target weight. An asset is only rebalanced once `|current_weight − target_weight|` reaches this many bps. `500` = 5 percentage points. |
+| `balancer_max_swaps_per_batch` | u32 | ≥ 1 | Upper bound on swap legs packed into one atomic rebalance batch (sells ordered before buys). Largest-drift assets are serviced first when the cap truncates. |
+| `balancer_max_swap_provider_halving_probes` | u32 | ≥ 1 | How many halving attempts the size-search makes per provider before giving up on that route. With `max_execution_impact_bps`, this bounds the descent: `3` probes tries full, ½, ¼ of the intended size. Higher = more chances to find a fitting size, at more RPC calls. |
 | `balancer_min_swap_amount_value_cents` | i128 (USD cents) | ≥ 0 | Skip rebalance swaps whose input value is below this. Stops the keeper paying fees on dust.|
 
 ★ Insight ─────────────────────────────────────
-- **`max_price_impact_bps` and `max_swap_provider_probes` work together as a sizing search.** The cap says "don't move the price more than X"; the probe count says "how hard to look for a size that fits under X". A tight cap with only 1 probe often yields `no_viable_provider` because the first (full-size) quote blows the cap and there's no room to shrink and retry.
-- **Price impact is measured against the oracle, not against zero.** A positive impact means the DEX fill is *worse* than the oracle-implied price. That's why the cap is meaningful even for large, "liquid" pairs — it's gating oracle/DEX divergence, not just raw slippage.
+- **`max_execution_impact_bps` and `max_swap_provider_halving_probes` work together as a sizing search.** The cap says "don't move the price more than X"; the probe count says "how hard to look for a size that fits under X". A tight cap with only 1 probe often yields `no_viable_provider` because the first (full-size) quote blows the cap and there's no room to shrink and retry.
+- **One leg per asset per batch is forced by AMM mechanics.** 
+- **Sells settle before buys inside the atomic batch.** The hub minted by selling surpluses funds the buys in the *same* transaction.
 ─────────────────────────────────────────────────
 
 ---

engine/src/lending_model/liquidation.rs

@@ -61,15 +61,15 @@ pub fn compute_is_liquidatable(
         };
         let real_debt = fixed_mul_ceil(borrow.d_tokens.0, pool.d_token_rate_ceil_bps);
         let decimals_divisor = 10_i128.checked_pow(pool.token_decimals).m_ou()?;
-        let value = real_debt
-            .checked_mul(pool.oracle_asset_price)
-            .m_ou()?
-            .checked_add(decimals_divisor - 1)
-            .m_ou()?
-            .checked_div(decimals_divisor)
-            .m_ou()?;
-        let scaled = fixed_mul_ceil(value, pool.liability_factor_bps);
-        debt_value = debt_value.checked_add(scaled).m_ou()?;
+
+        let numerator_pre = real_debt.checked_mul(pool.oracle_asset_price).m_ou()?;
+        let numerator = numerator_pre.checked_add(decimals_divisor - 1).m_ou()?;
+        let denominator = decimals_divisor;
+
+        let value = numerator.checked_div(denominator).m_ou()?;
+        let value_scaled_w_liability_factor = fixed_mul_ceil(value, pool.liability_factor_bps);
+
+        debt_value = debt_value.checked_add(value_scaled_w_liability_factor).m_ou()?;
     }
     if debt_value == 0 {
         return Ok(false);
@@ -88,13 +88,14 @@ pub fn compute_is_liquidatable(
         let real_supply = fixed_mul_floor(deposit.j_tokens.0, pool.j_token_rate_floor_bps);
         let total_tokens = real_supply.checked_add(deposit.collateral.0).m_ou()?;
         let decimals_divisor = 10_i128.checked_pow(pool.token_decimals).m_ou()?;
-        let value = total_tokens
-            .checked_mul(pool.oracle_asset_price)
-            .m_ou()?
-            .checked_div(decimals_divisor)
-            .m_ou()?;
-        let scaled = fixed_mul_floor(value, pool.close_ltv_bps);
-        collateral_value = collateral_value.checked_add(scaled).m_ou()?;
+
+        let numerator = total_tokens.checked_mul(pool.oracle_asset_price).m_ou()?;
+        let denominator = decimals_divisor;
+
+        let value = numerator.checked_div(denominator).m_ou()?;
+        let value_scaled_w_close_ltv = fixed_mul_floor(value, pool.close_ltv_bps);
+
+        collateral_value = collateral_value.checked_add(value_scaled_w_close_ltv).m_ou()?;
     }
 
     let min_collateral_value_cents = market_data.min_collateral_value_cents;