Split MPP by maximizing expected delivered amount

docs/release-notes/eclair-vnext.md

@@ -4,7 +4,11 @@
 
 ## Major changes
 
-<insert changes>
+### New MPP splitting strategy
+
+Eclair can send large payments using multiple low-capacity routes by sending as much as it can through each route (if `randomize-route-selection = false`) or some random fraction (if `randomize-route-selection = true`).
+These splitting strategies are now specified using `mpp.splitting-strategy = "full-capacity"` or `mpp.splitting-strategy = "randomize"`.
+In addition, a new strategy is available: `mpp.splitting-strategy = "max-expected-amount"` will send through each route the amount that maximizes the expected delivered amount (amount sent times success probability).
 
 ### Remove support for legacy channel codecs
 

eclair-core/src/main/resources/reference.conf

@@ -461,6 +461,7 @@ eclair {
         mpp {
           min-amount-satoshis = 15000 // minimum amount sent via partial HTLCs
           max-parts = 5 // maximum number of HTLCs sent per payment: increasing this value will impact performance
+          splitting-strategy = "randomize" // Can be either "full-capacity", "randomize" or "max-expected-amount"
         }
 
         boundaries {

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -468,7 +468,12 @@ object NodeParams extends Logging {
         usePastRelaysData = config.getBoolean("use-past-relay-data")),
       mpp = MultiPartParams(
         Satoshi(config.getLong("mpp.min-amount-satoshis")).toMilliSatoshi,
-        config.getInt("mpp.max-parts")),
+        config.getInt("mpp.max-parts"),
+        config.getString("mpp.splitting-strategy") match {
+          case "full-capacity" => MultiPartParams.FullCapacity
+          case "randomize" => MultiPartParams.Randomize
+          case "max-expected-amount" => MultiPartParams.MaxExpectedAmount
+        }),
       experimentName = name,
       experimentPercentage = config.getInt("percentage"))
 

eclair-core/src/main/scala/fr/acinq/eclair/payment/send/MultiPartPaymentLifecycle.scala

@@ -27,6 +27,7 @@ import fr.acinq.eclair.payment.PaymentSent.PartialPayment
 import fr.acinq.eclair.payment._
 import fr.acinq.eclair.payment.send.PaymentInitiator.SendPaymentConfig
 import fr.acinq.eclair.payment.send.PaymentLifecycle.SendPaymentToRoute
+import fr.acinq.eclair.router.Router.MultiPartParams.FullCapacity
 import fr.acinq.eclair.router.Router._
 import fr.acinq.eclair.{FSMDiagnosticActorLogging, Logs, MilliSatoshiLong, NodeParams, TimestampMilli}
 import scodec.bits.ByteVector
@@ -57,7 +58,7 @@ class MultiPartPaymentLifecycle(nodeParams: NodeParams, cfg: SendPaymentConfig,
 
   when(WAIT_FOR_PAYMENT_REQUEST) {
     case Event(r: SendMultiPartPayment, _) =>
-      val routeParams = r.routeParams.copy(randomize = false) // we don't randomize the first attempt, regardless of configuration choices
+      val routeParams = r.routeParams.copy(randomize = false, mpp = r.routeParams.mpp.copy(splittingStrategy = FullCapacity)) // we don't randomize the first attempt, regardless of configuration choices
       log.debug("sending {} with maximum fee {}", r.recipient.totalAmount, r.routeParams.getMaxFee(r.recipient.totalAmount))
       val d = PaymentProgress(r, r.maxAttempts, Map.empty, Ignore.empty, retryRouteRequest = false, failures = Nil)
       router ! createRouteRequest(self, nodeParams, routeParams, d, cfg)

eclair-core/src/main/scala/fr/acinq/eclair/remote/EclairInternalsSerializer.scala

@@ -64,13 +64,23 @@ object EclairInternalsSerializer {
       ("useLogProbability" | bool(8)) ::
       ("usePastRelaysData" | bool(8))).as[HeuristicsConstants]
 
-  val weightRatiosCodec: Codec[WeightRatios[PaymentPathWeight]] =
-    discriminated[WeightRatios[PaymentPathWeight]].by(uint8)
+  val weightRatiosCodec: Codec[HeuristicsConstants] =
+    discriminated[HeuristicsConstants].by(uint8)
       .typecase(0xff, heuristicsConstantsCodec)
 
   val multiPartParamsCodec: Codec[MultiPartParams] = (
     ("minPartAmount" | millisatoshi) ::
-      ("maxParts" | int32)).as[MultiPartParams]
+      ("maxParts" | int32) ::
+      ("splittingStrategy" | int8.narrow[MultiPartParams.SplittingStrategy]({
+        case 0 => Attempt.successful(MultiPartParams.FullCapacity)
+        case 1 => Attempt.successful(MultiPartParams.Randomize)
+        case 2 => Attempt.successful(MultiPartParams.MaxExpectedAmount)
+        case n => Attempt.failure(Err(s"Invalid value $n"))
+      }, {
+        case MultiPartParams.FullCapacity => 0
+        case MultiPartParams.Randomize => 1
+        case MultiPartParams.MaxExpectedAmount => 2
+      }))).as[MultiPartParams]
 
   val pathFindingConfCodec: Codec[PathFindingConf] = (
     ("randomize" | bool(8)) ::

eclair-core/src/main/scala/fr/acinq/eclair/router/BalanceEstimate.scala

@@ -216,7 +216,7 @@ case class BalanceEstimate private(low: MilliSatoshi,
    * - probability that it can relay a payment of high is decay(high, 0, highTimestamp) which is close to 0 if highTimestamp is recent
    * - probability that it can relay a payment of maxCapacity is 0
    */
-  def canSend(amount: MilliSatoshi, now: TimestampSecond): Double = {
+  def canSendAndDerivative(amount: MilliSatoshi, now: TimestampSecond): (Double, Double) = {
     val a = amount.toLong.toDouble
     val l = low.toLong.toDouble
     val h = high.toLong.toDouble
@@ -227,15 +227,17 @@ case class BalanceEstimate private(low: MilliSatoshi,
     val pHigh = decay(high, 0, highTimestamp, now)
 
     if (amount < low) {
-      (l - a * (1.0 - pLow)) / l
+      ((l - a * (1.0 - pLow)) / l, (pLow - 1.0) / l)
     } else if (amount < high) {
-      ((h - a) * pLow + (a - l) * pHigh) / (h - l)
+      (((h - a) * pLow + (a - l) * pHigh) / (h - l), (pHigh - pLow) / (h - l))
     } else if (h < c) {
-      ((c - a) * pHigh) / (c - h)
+      (((c - a) * pHigh) / (c - h), (-pHigh) / (c - h))
     } else {
-      0
+      (0.0, 0.0)
     }
   }
+
+  def canSend(amount: MilliSatoshi, now: TimestampSecond): Double = canSendAndDerivative(amount, now)._1
 }
 
 object BalanceEstimate {

eclair-core/src/main/scala/fr/acinq/eclair/router/RouteCalculation.scala

@@ -390,14 +390,7 @@ object RouteCalculation {
                          pendingHtlcs: Seq[Route] = Nil,
                          routeParams: RouteParams,
                          currentBlockHeight: BlockHeight): Try[Seq[Route]] = Try {
-    val result = findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams, currentBlockHeight) match {
-      case Right(routes) => Right(routes)
-      case Left(RouteNotFound) if routeParams.randomize =>
-        // If we couldn't find a randomized solution, fallback to a deterministic one.
-        findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams.copy(randomize = false), currentBlockHeight)
-      case Left(ex) => Left(ex)
-    }
-    result match {
+    findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams, currentBlockHeight) match {
       case Right(routes) => routes
       case Left(ex) => return Failure(ex)
     }
@@ -413,7 +406,8 @@ object RouteCalculation {
                                          ignoredVertices: Set[PublicKey] = Set.empty,
                                          pendingHtlcs: Seq[Route] = Nil,
                                          routeParams: RouteParams,
-                                         currentBlockHeight: BlockHeight): Either[RouterException, Seq[Route]] = {
+                                         currentBlockHeight: BlockHeight,
+                                         now: TimestampSecond = TimestampSecond.now()): Either[RouterException, Seq[Route]] = {
     // We use Yen's k-shortest paths to find many paths for chunks of the total amount.
     // When the recipient is a direct peer, we have complete visibility on our local channels so we can use more accurate MPP parameters.
     val routeParams1 = {
@@ -429,48 +423,101 @@ object RouteCalculation {
       // We want to ensure that the set of routes we find have enough capacity to allow sending the total amount,
       // without excluding routes with small capacity when the total amount is small.
       val minPartAmount = routeParams.mpp.minPartAmount.max(amount / numRoutes).min(amount)
-      routeParams.copy(mpp = MultiPartParams(minPartAmount, numRoutes))
+      routeParams.copy(mpp = MultiPartParams(minPartAmount, numRoutes, routeParams.mpp.splittingStrategy))
     }
     findRouteInternal(g, localNodeId, targetNodeId, routeParams1.mpp.minPartAmount, maxFee, routeParams1.mpp.maxParts, extraEdges, ignoredEdges, ignoredVertices, routeParams1, currentBlockHeight) match {
-      case Right(routes) =>
+      case Right(paths) =>
         // We use these shortest paths to find a set of non-conflicting HTLCs that send the total amount.
-        split(amount, mutable.Queue(routes: _*), initializeUsedCapacity(pendingHtlcs), routeParams1) match {
+        split(amount, mutable.Queue(paths: _*), initializeUsedCapacity(pendingHtlcs), routeParams1, g.balances, now) match {
           case Right(routes) if validateMultiPartRoute(amount, maxFee, routes, routeParams.includeLocalChannelCost) => Right(routes)
+          case Right(_) if routeParams.randomize =>
+            // We've found a multipart route, but it's too expensive. We try again without randomization to prioritize cheaper paths.
+            val sortedPaths = paths.sortBy(_.weight.weight)
+            split(amount, mutable.Queue(sortedPaths: _*), initializeUsedCapacity(pendingHtlcs), routeParams1, g.balances, now) match {
+              case Right(routes) if validateMultiPartRoute(amount, maxFee, routes, routeParams.includeLocalChannelCost) => Right(routes)
+              case _ => Left(RouteNotFound)
+            }
           case _ => Left(RouteNotFound)
         }
       case Left(ex) => Left(ex)
     }
   }
 
-  @tailrec
-  private def split(amount: MilliSatoshi, paths: mutable.Queue[WeightedPath[PaymentPathWeight]], usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], routeParams: RouteParams, selectedRoutes: Seq[Route] = Nil): Either[RouterException, Seq[Route]] = {
-    if (amount == 0.msat) {
-      Right(selectedRoutes)
-    } else if (paths.isEmpty) {
-      Left(RouteNotFound)
-    } else {
+  private case class CandidateRoute(routeWithMaximumAmount: Route, chosenAmount: MilliSatoshi)
+
+  private def split(amount: MilliSatoshi, paths: mutable.Queue[WeightedPath[PaymentPathWeight]], usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], routeParams: RouteParams, balances: BalancesEstimates, now: TimestampSecond): Either[RouterException, Seq[Route]] = {
+    var amountLeft = amount
+    var candidates: List[CandidateRoute] = Nil
+    while(paths.nonEmpty && amountLeft > 0.msat) {
       val current = paths.dequeue()
       val candidate = computeRouteMaxAmount(current.path, usedCapacity)
-      if (candidate.amount < routeParams.mpp.minPartAmount.min(amount)) {
-        // this route doesn't have enough capacity left: we remove it and continue.
-        split(amount, paths, usedCapacity, routeParams, selectedRoutes)
-      } else {
-        val route = if (routeParams.randomize) {
-          // randomly choose the amount to be between 20% and 100% of the available capacity.
-          val randomizedAmount = candidate.amount * ((20d + Random.nextInt(81)) / 100)
-          if (randomizedAmount < routeParams.mpp.minPartAmount) {
-            candidate.copy(amount = routeParams.mpp.minPartAmount.min(amount))
-          } else {
-            candidate.copy(amount = randomizedAmount.min(amount))
-          }
+      if (candidate.amount >= routeParams.mpp.minPartAmount.min(amountLeft)) {
+        val routeFullCapacity = candidate.copy(amount = candidate.amount.min(amountLeft))
+        val chosenAmount = routeParams.mpp.splittingStrategy match {
+          case MultiPartParams.Randomize =>
+            // randomly choose the amount to be between 20% and 100% of the available capacity.
+            val randomizedAmount = candidate.amount * ((20d + Random.nextInt(81)) / 100)
+            if (randomizedAmount < routeParams.mpp.minPartAmount) {
+              routeParams.mpp.minPartAmount.min(amountLeft)
+            } else {
+              randomizedAmount.min(amountLeft)
+            }
+          case MultiPartParams.MaxExpectedAmount =>
+            val bestAmount = optimizeExpectedValue(current.path, candidate.amount, usedCapacity, routeParams.heuristics.usePastRelaysData, balances, now)
+            bestAmount.max(routeParams.mpp.minPartAmount).min(amountLeft)
+          case MultiPartParams.FullCapacity =>
+            routeFullCapacity.amount
+        }
+        updateUsedCapacity(routeFullCapacity, usedCapacity)
+        candidates = CandidateRoute(routeFullCapacity, chosenAmount) :: candidates
+        amountLeft = amountLeft - chosenAmount
+        paths.enqueue(current)
+      }
+    }
+    val totalChosen = candidates.map(_.chosenAmount).sum
+    val totalMaximum = candidates.map(_.routeWithMaximumAmount.amount).sum
+    if (totalMaximum < amount) {
+      Left(RouteNotFound)
+    } else {
+      val additionalFraction = if (totalMaximum > totalChosen) (amount - totalChosen).toLong.toDouble / (totalMaximum - totalChosen).toLong.toDouble else 0.0
+      var routes: List[Route] = Nil
+      var amountLeft = amount
+      candidates.foreach { case CandidateRoute(route, chosenAmount) =>
+        if (amountLeft > 0.msat) {
+          val additionalAmount = MilliSatoshi(((route.amount - chosenAmount).toLong * additionalFraction).ceil.toLong)
+          val amountToSend = (chosenAmount + additionalAmount).min(amountLeft)
+          routes = route.copy(amount = amountToSend) :: routes
+          amountLeft = amountLeft - amountToSend
+        }
+      }
+      Right(routes)
+    }
+  }
+
+  private def optimizeExpectedValue(route: Seq[GraphEdge], capacity: MilliSatoshi, usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], usePastRelaysData: Boolean, balances: BalancesEstimates, now: TimestampSecond): MilliSatoshi = {
+    // We search the maximum value of a polynomial between its two smallest roots (0 and the minimum channel capacity on the path).
+    // We use binary search to find where the derivative changes sign.
+    var low = 1L
+    var high = capacity.toLong
+    while (high - low > 1L) {
+      val x = (high + low) / 2
+      val d = route.drop(1).foldLeft(1.0 / x.toDouble) { case (total, edge) =>
+        // We compute the success probability `p` for this edge, and its derivative `dp`.
+        val (p, dp) = if (usePastRelaysData) {
+          balances.get(edge).canSendAndDerivative(MilliSatoshi(x), now)
         } else {
-          candidate.copy(amount = candidate.amount.min(amount))
+          val c = (edge.capacity - usedCapacity.getOrElse(edge.desc.shortChannelId, 0 msat)).toLong.toDouble
+          (1.0 - x.toDouble / c, -1.0 / c)
         }
-        updateUsedCapacity(route, usedCapacity)
-        // NB: we re-enqueue the current path, it may still have capacity for a second HTLC.
-        split(amount - route.amount, paths.enqueue(current), usedCapacity, routeParams, route +: selectedRoutes)
+        total + dp / p
+      }
+      if (d > 0.0) {
+        low = x
+      } else {
+        high = x
       }
     }
+    MilliSatoshi(high)
   }
 
   /** Compute the maximum amount that we can send through the given route. */

eclair-core/src/main/scala/fr/acinq/eclair/router/Router.scala

@@ -38,7 +38,7 @@ import fr.acinq.eclair.payment.send.Recipient
 import fr.acinq.eclair.payment.{Bolt11Invoice, Invoice}
 import fr.acinq.eclair.remote.EclairInternalsSerializer.RemoteTypes
 import fr.acinq.eclair.router.Graph.GraphStructure.DirectedGraph
-import fr.acinq.eclair.router.Graph.MessageWeightRatios
+import fr.acinq.eclair.router.Graph.{HeuristicsConstants, MessageWeightRatios}
 import fr.acinq.eclair.router.Monitoring.Metrics
 import fr.acinq.eclair.wire.protocol._
 
@@ -354,7 +354,7 @@ object Router {
 
   case class PathFindingConf(randomize: Boolean,
                              boundaries: SearchBoundaries,
-                             heuristics: Graph.WeightRatios[Graph.PaymentPathWeight],
+                             heuristics: HeuristicsConstants,
                              mpp: MultiPartParams,
                              experimentName: String,
                              experimentPercentage: Int) {
@@ -577,11 +577,24 @@ object Router {
     override def fee(amount: MilliSatoshi): MilliSatoshi = fee
   }
 
-  case class MultiPartParams(minPartAmount: MilliSatoshi, maxParts: Int)
+  object MultiPartParams {
+    sealed trait SplittingStrategy
+
+    /** Send the full capacity of the route */
+    object FullCapacity extends SplittingStrategy
+
+    /** Send between 20% and 100% of the capacity of the route */
+    object Randomize extends SplittingStrategy
+
+    /** Maximize the expected delivered amount */
+    object MaxExpectedAmount extends SplittingStrategy
+  }
+
+  case class MultiPartParams(minPartAmount: MilliSatoshi, maxParts: Int, splittingStrategy: MultiPartParams.SplittingStrategy)
 
   case class RouteParams(randomize: Boolean,
                          boundaries: SearchBoundaries,
-                         heuristics: Graph.WeightRatios[Graph.PaymentPathWeight],
+                         heuristics: HeuristicsConstants,
                          mpp: MultiPartParams,
                          experimentName: String,
                          includeLocalChannelCost: Boolean) {

eclair-core/src/test/scala/fr/acinq/eclair/TestConstants.scala

@@ -231,6 +231,7 @@ object TestConstants {
           mpp = MultiPartParams(
             minPartAmount = 15000000 msat,
             maxParts = 10,
+            splittingStrategy = MultiPartParams.FullCapacity
           ),
           experimentName = "alice-test-experiment",
           experimentPercentage = 100))),
@@ -422,6 +423,7 @@ object TestConstants {
           mpp = MultiPartParams(
             minPartAmount = 15000000 msat,
             maxParts = 10,
+            splittingStrategy = MultiPartParams.FullCapacity
           ),
           experimentName = "bob-test-experiment",
           experimentPercentage = 100))),

eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala

@@ -64,7 +64,7 @@ abstract class IntegrationSpec extends TestKitBaseClass with BitcoindService wit
       useLogProbability = false,
       usePastRelaysData = false
     ),
-    mpp = MultiPartParams(15000000 msat, 6),
+    mpp = MultiPartParams(15000000 msat, 6, MultiPartParams.FullCapacity),
     experimentName = "my-test-experiment",
     experimentPercentage = 100
   ).getDefaultRouteParams

eclair-core/src/test/scala/fr/acinq/eclair/payment/MultiPartPaymentLifecycleSpec.scala

@@ -697,7 +697,7 @@ object MultiPartPaymentLifecycleSpec {
       6,
       CltvExpiryDelta(1008)),
     HeuristicsConstants(0, RelayFees(0 msat, 0), RelayFees(0 msat, 0), useLogProbability = false, usePastRelaysData = false),
-    MultiPartParams(1000 msat, 5),
+    MultiPartParams(1000 msat, 5, MultiPartParams.FullCapacity),
     experimentName = "my-test-experiment",
     experimentPercentage = 100
   ).getDefaultRouteParams

eclair-core/src/test/scala/fr/acinq/eclair/payment/PaymentLifecycleSpec.scala

@@ -302,7 +302,7 @@ class PaymentLifecycleSpec extends BaseRouterSpec {
       randomize = false,
       boundaries = SearchBoundaries(100 msat, 0.0, 20, CltvExpiryDelta(2016)),
       HeuristicsConstants(0, RelayFees(0 msat, 0), RelayFees(0 msat, 0), useLogProbability = false, usePastRelaysData = false),
-      MultiPartParams(10_000 msat, 5),
+      MultiPartParams(10_000 msat, 5, MultiPartParams.FullCapacity),
       "my-test-experiment",
       experimentPercentage = 100
     ).getDefaultRouteParams