fixes

Author: thomash-acinq

docs/release-notes/eclair-vnext.md

@@ -4,7 +4,11 @@
 
 ## Major changes
 
-<insert changes>
+### New MPP splitting strategy
+
+Eclair can send large payments using multiple low-capacity routes by sending as much as it can through each route (if `randomize-route-selection = false`) or some random fraction (if `randomize-route-selection = true`).
+These splitting strategies are now specified using `mpp.splitting-strategy = "full-capacity"` or `mpp.splitting-strategy = "randomize"`.
+In addition, a new strategy is available: `mpp.splitting-strategy = "max-expected-amount"` will send through each route the amount that maximizes the expected delivered amount (amount sent times success probability).
 
 ### Remove support for legacy channel codecs
 

eclair-core/src/main/scala/fr/acinq/eclair/payment/send/MultiPartPaymentLifecycle.scala

@@ -27,6 +27,7 @@ import fr.acinq.eclair.payment.PaymentSent.PartialPayment
 import fr.acinq.eclair.payment._
 import fr.acinq.eclair.payment.send.PaymentInitiator.SendPaymentConfig
 import fr.acinq.eclair.payment.send.PaymentLifecycle.SendPaymentToRoute
+import fr.acinq.eclair.router.Router.MultiPartParams.FullCapacity
 import fr.acinq.eclair.router.Router._
 import fr.acinq.eclair.{FSMDiagnosticActorLogging, Logs, MilliSatoshiLong, NodeParams, TimestampMilli}
 import scodec.bits.ByteVector
@@ -57,7 +58,7 @@ class MultiPartPaymentLifecycle(nodeParams: NodeParams, cfg: SendPaymentConfig,
 
   when(WAIT_FOR_PAYMENT_REQUEST) {
     case Event(r: SendMultiPartPayment, _) =>
-      val routeParams = r.routeParams.copy(randomize = false) // we don't randomize the first attempt, regardless of configuration choices
+      val routeParams = r.routeParams.copy(randomize = false, mpp = r.routeParams.mpp.copy(splittingStrategy = FullCapacity)) // we don't randomize the first attempt, regardless of configuration choices
       log.debug("sending {} with maximum fee {}", r.recipient.totalAmount, r.routeParams.getMaxFee(r.recipient.totalAmount))
       val d = PaymentProgress(r, r.maxAttempts, Map.empty, Ignore.empty, retryRouteRequest = false, failures = Nil)
       router ! createRouteRequest(self, nodeParams, routeParams, d, cfg)

eclair-core/src/main/scala/fr/acinq/eclair/remote/EclairInternalsSerializer.scala

@@ -64,8 +64,8 @@ object EclairInternalsSerializer {
       ("useLogProbability" | bool(8)) ::
       ("usePastRelaysData" | bool(8))).as[HeuristicsConstants]
 
-  val weightRatiosCodec: Codec[WeightRatios[PaymentPathWeight]] =
-    discriminated[WeightRatios[PaymentPathWeight]].by(uint8)
+  val weightRatiosCodec: Codec[HeuristicsConstants] =
+    discriminated[HeuristicsConstants].by(uint8)
       .typecase(0xff, heuristicsConstantsCodec)
 
   val multiPartParamsCodec: Codec[MultiPartParams] = (

eclair-core/src/main/scala/fr/acinq/eclair/router/BalanceEstimate.scala

@@ -216,7 +216,7 @@ case class BalanceEstimate private(low: MilliSatoshi,
    * - probability that it can relay a payment of high is decay(high, 0, highTimestamp) which is close to 0 if highTimestamp is recent
    * - probability that it can relay a payment of maxCapacity is 0
    */
-  def canSend(amount: MilliSatoshi, now: TimestampSecond): Double = {
+  def canSendAndDerivative(amount: MilliSatoshi, now: TimestampSecond): (Double, Double) = {
     val a = amount.toLong.toDouble
     val l = low.toLong.toDouble
     val h = high.toLong.toDouble
@@ -227,15 +227,17 @@ case class BalanceEstimate private(low: MilliSatoshi,
     val pHigh = decay(high, 0, highTimestamp, now)
 
     if (amount < low) {
-      (l - a * (1.0 - pLow)) / l
+      ((l - a * (1.0 - pLow)) / l, (pLow - 1.0) / l)
     } else if (amount < high) {
-      ((h - a) * pLow + (a - l) * pHigh) / (h - l)
+      (((h - a) * pLow + (a - l) * pHigh) / (h - l), (pHigh - pLow) / (h - l))
     } else if (h < c) {
-      ((c - a) * pHigh) / (c - h)
+      (((c - a) * pHigh) / (c - h), (-pHigh) / (c - h))
     } else {
-      0
+      (0.0, 0.0)
     }
   }
+
+  def canSend(amount: MilliSatoshi, now: TimestampSecond): Double = canSendAndDerivative(amount, now)._1
 }
 
 object BalanceEstimate {

eclair-core/src/main/scala/fr/acinq/eclair/router/RouteCalculation.scala

@@ -390,14 +390,7 @@ object RouteCalculation {
                          pendingHtlcs: Seq[Route] = Nil,
                          routeParams: RouteParams,
                          currentBlockHeight: BlockHeight): Try[Seq[Route]] = Try {
-    val result = findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams, currentBlockHeight) match {
-      case Right(routes) => Right(routes)
-      case Left(RouteNotFound) if routeParams.randomize =>
-        // If we couldn't find a randomized solution, fallback to a deterministic one.
-        findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams.copy(randomize = false), currentBlockHeight)
-      case Left(ex) => Left(ex)
-    }
-    result match {
+    findMultiPartRouteInternal(g, localNodeId, targetNodeId, amount, maxFee, extraEdges, ignoredEdges, ignoredVertices, pendingHtlcs, routeParams, currentBlockHeight) match {
       case Right(routes) => routes
       case Left(ex) => return Failure(ex)
     }
@@ -413,7 +406,8 @@ object RouteCalculation {
                                          ignoredVertices: Set[PublicKey] = Set.empty,
                                          pendingHtlcs: Seq[Route] = Nil,
                                          routeParams: RouteParams,
-                                         currentBlockHeight: BlockHeight): Either[RouterException, Seq[Route]] = {
+                                         currentBlockHeight: BlockHeight,
+                                         now: TimestampSecond = TimestampSecond.now()): Either[RouterException, Seq[Route]] = {
     // We use Yen's k-shortest paths to find many paths for chunks of the total amount.
     // When the recipient is a direct peer, we have complete visibility on our local channels so we can use more accurate MPP parameters.
     val routeParams1 = {
@@ -432,62 +426,95 @@ object RouteCalculation {
       routeParams.copy(mpp = MultiPartParams(minPartAmount, numRoutes, routeParams.mpp.splittingStrategy))
     }
     findRouteInternal(g, localNodeId, targetNodeId, routeParams1.mpp.minPartAmount, maxFee, routeParams1.mpp.maxParts, extraEdges, ignoredEdges, ignoredVertices, routeParams1, currentBlockHeight) match {
-      case Right(routes) =>
+      case Right(paths) =>
         // We use these shortest paths to find a set of non-conflicting HTLCs that send the total amount.
-        split(amount, mutable.Queue(routes: _*), initializeUsedCapacity(pendingHtlcs), routeParams1) match {
+        split(amount, mutable.Queue(paths: _*), initializeUsedCapacity(pendingHtlcs), routeParams1, g.balances, now) match {
           case Right(routes) if validateMultiPartRoute(amount, maxFee, routes, routeParams.includeLocalChannelCost) => Right(routes)
+          case Right(_) if routeParams.randomize =>
+            // We've found a multipart route, but it's too expensive. We try again without randomization to prioritize cheaper paths.
+            val sortedPaths = paths.sortBy(_.weight.weight)
+            split(amount, mutable.Queue(sortedPaths: _*), initializeUsedCapacity(pendingHtlcs), routeParams1, g.balances, now) match {
+              case Right(routes) if validateMultiPartRoute(amount, maxFee, routes, routeParams.includeLocalChannelCost) => Right(routes)
+              case _ => Left(RouteNotFound)
+            }
           case _ => Left(RouteNotFound)
         }
       case Left(ex) => Left(ex)
     }
   }
 
-  @tailrec
-  private def split(amount: MilliSatoshi, paths: mutable.Queue[WeightedPath[PaymentPathWeight]], usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], routeParams: RouteParams, selectedRoutes: Seq[Route] = Nil): Either[RouterException, Seq[Route]] = {
-    if (amount == 0.msat) {
-      Right(selectedRoutes)
-    } else if (paths.isEmpty) {
-      Left(RouteNotFound)
-    } else {
+  private case class CandidateRoute(routeWithMaximumAmount: Route, chosenAmount: MilliSatoshi)
+
+  private def split(amount: MilliSatoshi, paths: mutable.Queue[WeightedPath[PaymentPathWeight]], usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], routeParams: RouteParams, balances: BalancesEstimates, now: TimestampSecond): Either[RouterException, Seq[Route]] = {
+    var amountLeft = amount
+    var candidates: List[CandidateRoute] = Nil
+    while(paths.nonEmpty && amountLeft > 0.msat) {
       val current = paths.dequeue()
       val candidate = computeRouteMaxAmount(current.path, usedCapacity)
-      if (candidate.amount < routeParams.mpp.minPartAmount.min(amount)) {
-        // this route doesn't have enough capacity left: we remove it and continue.
-        split(amount, paths, usedCapacity, routeParams, selectedRoutes)
-      } else {
-        val route = routeParams.mpp.splittingStrategy match {
+      if (candidate.amount >= routeParams.mpp.minPartAmount.min(amountLeft)) {
+        val routeFullCapacity = candidate.copy(amount = candidate.amount.min(amountLeft))
+        val chosenAmount = routeParams.mpp.splittingStrategy match {
           case MultiPartParams.Randomize =>
             // randomly choose the amount to be between 20% and 100% of the available capacity.
             val randomizedAmount = candidate.amount * ((20d + Random.nextInt(81)) / 100)
-            candidate.copy(amount = randomizedAmount.max(routeParams.mpp.minPartAmount).min(amount))
+            if (randomizedAmount < routeParams.mpp.minPartAmount) {
+              routeParams.mpp.minPartAmount.min(amountLeft)
+            } else {
+              randomizedAmount.min(amountLeft)
+            }
           case MultiPartParams.MaxExpectedAmount =>
-            val bestAmount = optimizeExpectedValue(current.path, candidate.amount, usedCapacity)
-            candidate.copy(amount = bestAmount.max(routeParams.mpp.minPartAmount).min(amount))
+            val bestAmount = optimizeExpectedValue(current.path, candidate.amount, usedCapacity, routeParams.heuristics.usePastRelaysData, balances, now)
+            bestAmount.max(routeParams.mpp.minPartAmount).min(amountLeft)
           case MultiPartParams.FullCapacity =>
-            candidate.copy(amount = candidate.amount.min(amount))
+            routeFullCapacity.amount
         }
-        updateUsedCapacity(route, usedCapacity)
-        // NB: we re-enqueue the current path, it may still have capacity for a second HTLC.
-        split(amount - route.amount, paths.enqueue(current), usedCapacity, routeParams, route +: selectedRoutes)
+        updateUsedCapacity(routeFullCapacity, usedCapacity)
+        candidates = CandidateRoute(routeFullCapacity, chosenAmount) :: candidates
+        amountLeft = amountLeft - chosenAmount
+        paths.enqueue(current)
       }
     }
+    val totalChosen = candidates.map(_.chosenAmount).sum
+    val totalMaximum = candidates.map(_.routeWithMaximumAmount.amount).sum
+    if (totalMaximum < amount) {
+      Left(RouteNotFound)
+    } else {
+      val additionalFraction = if (totalMaximum > totalChosen) (amount - totalChosen).toLong.toDouble / (totalMaximum - totalChosen).toLong.toDouble else 0.0
+      var routes: List[Route] = Nil
+      var amountLeft = amount
+      candidates.foreach { case CandidateRoute(route, chosenAmount) =>
+        if (amountLeft > 0.msat) {
+          val additionalAmount = MilliSatoshi(((route.amount - chosenAmount).toLong * additionalFraction).ceil.toLong)
+          val amountToSend = (chosenAmount + additionalAmount).min(amountLeft)
+          routes = route.copy(amount = amountToSend) :: routes
+          amountLeft = amountLeft - amountToSend
+        }
+      }
+      Right(routes)
+    }
   }
 
-  private def optimizeExpectedValue(route: Seq[GraphEdge], capacity: MilliSatoshi, usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi]): MilliSatoshi = {
+  private def optimizeExpectedValue(route: Seq[GraphEdge], capacity: MilliSatoshi, usedCapacity: mutable.Map[ShortChannelId, MilliSatoshi], usePastRelaysData: Boolean, balances: BalancesEstimates, now: TimestampSecond): MilliSatoshi = {
     // We search the maximum value of a polynomial between its two smallest roots (0 and the minimum channel capacity on the path).
     // We use binary search to find where the derivative changes sign.
     var low = 1L
     var high = capacity.toLong
     while (high - low > 1L) {
-      val mid = (high + low) / 2
-      val d = route.drop(1).foldLeft(1.0 / mid) { case (x, edge) =>
-        val availableCapacity = edge.capacity - usedCapacity.getOrElse(edge.desc.shortChannelId, 0 msat)
-        x - 1.0 / (availableCapacity.toLong - mid)
+      val x = (high + low) / 2
+      val d = route.drop(1).foldLeft(1.0 / x.toDouble) { case (total, edge) =>
+        // We compute the success probability `p` for this edge, and its derivative `dp`.
+        val (p, dp) = if (usePastRelaysData) {
+          balances.get(edge).canSendAndDerivative(MilliSatoshi(x), now)
+        } else {
+          val c = (edge.capacity - usedCapacity.getOrElse(edge.desc.shortChannelId, 0 msat)).toLong.toDouble
+          (1.0 - x.toDouble / c, -1.0 / c)
+        }
+        total + dp / p
       }
       if (d > 0.0) {
-        low = mid
+        low = x
       } else {
-        high = mid
+        high = x
       }
     }
     MilliSatoshi(high)

eclair-core/src/main/scala/fr/acinq/eclair/router/Router.scala

@@ -38,7 +38,7 @@ import fr.acinq.eclair.payment.send.Recipient
 import fr.acinq.eclair.payment.{Bolt11Invoice, Invoice}
 import fr.acinq.eclair.remote.EclairInternalsSerializer.RemoteTypes
 import fr.acinq.eclair.router.Graph.GraphStructure.DirectedGraph
-import fr.acinq.eclair.router.Graph.MessageWeightRatios
+import fr.acinq.eclair.router.Graph.{HeuristicsConstants, MessageWeightRatios}
 import fr.acinq.eclair.router.Monitoring.Metrics
 import fr.acinq.eclair.wire.protocol._
 
@@ -354,7 +354,7 @@ object Router {
 
   case class PathFindingConf(randomize: Boolean,
                              boundaries: SearchBoundaries,
-                             heuristics: Graph.WeightRatios[Graph.PaymentPathWeight],
+                             heuristics: HeuristicsConstants,
                              mpp: MultiPartParams,
                              experimentName: String,
                              experimentPercentage: Int) {
@@ -594,7 +594,7 @@ object Router {
 
   case class RouteParams(randomize: Boolean,
                          boundaries: SearchBoundaries,
-                         heuristics: Graph.WeightRatios[Graph.PaymentPathWeight],
+                         heuristics: HeuristicsConstants,
                          mpp: MultiPartParams,
                          experimentName: String,
                          includeLocalChannelCost: Boolean) {