Revived branch from growing utxo benchmark (#2443)

Revive of #2431 

This was changing too much things and resulted in a too annoying
developer experience of using `cabal bench hydra-cluster`. So we tried
again...

Fixes #2441

---

<!-- Consider each and tick it off one way or the other -->
* [ ] CHANGELOG updated or not needed
* [ ] Documentation updated or not needed
* [ ] Haddocks updated or not needed
* [ ] No new TODOs introduced or explained herafter

Author: noonio

.github/workflows/ci-nix.yaml

@@ -91,7 +91,7 @@ jobs:
             options: '-o $(pwd)/../benchmarks/ledger-bench.html'
           - package: hydra-cluster
             bench: bench-e2e
-            options: 'datasets datasets/1-node.json datasets/3-nodes.json --output-directory $(pwd)/../benchmarks --timeout 1000s'
+            options: 'single datasets/1-node.json datasets/3-nodes.json --output-directory $(pwd)/../benchmarks --timeout 1000s'
     steps:
     - name: 📥 Checkout repository
       uses: actions/checkout@v4

.github/workflows/network-test.yaml

@@ -108,7 +108,7 @@ jobs:
         nix run .#legacyPackages.x86_64-linux.hydra-cluster.components.benchmarks.bench-e2e -- \
           demo \
           --output-directory=benchmarks \
-          --scaling-factor="$scaling_factor" \
+          --number-of-txs="$scaling_factor" \
           --timeout=1200s \
           --testnet-magic 42 \
           --node-socket=demo/devnet/node.socket \

hydra-cluster/README.md

@@ -130,41 +130,63 @@ produces a `results.csv` file in a work directory. To plot the transaction
 confirmation times you can use the `bench/plot.sh` script, passing it the
 directory containing the benchmark's results.
 
-To run and plot results of the benchmark:
+For the benchmarks, you can choose between generating either a constant-size
+UTxO set or a growing UTxO set.
+
+Constant UTxO set:
+Each transaction spends one input and creates exactly one new output (1-in-1-out), so the total number of
+UTxOs in the set remains roughly the same over time.
+
+Growing UTxO set:
+Each transaction spends one input but creates two outputs gradually increasing the total number of UTxOs as more
+transactions are processed. For this we use the `--number-of-txs` argument.
+
+This distinction allows you to measure performance under different realistic UTxO-set growth scenarios on Cardano.
+
+
+To generate, run and then plot results of the benchmark:
 
 ```sh
-cabal run bench-e2e -- single --output-directory out"
-bench/plot.sh out
+cabal run bench-e2e -- datasets --number-of-txs 10 --output-directory out
+./hydra-cluster/bench/plot.sh out
 ```
 
 Which will produce an output like:
 
 ```
-Generating dataset with scaling factor: 10
 Writing dataset to: out/dataset.json
+Saved dataset in: out/dataset.json
 Test logs available in: out/test.log
 Starting benchmark
 Seeding network
 Fund scenario from faucet
-Fuel node key "16e61ed92346eb0b0bd1c6d8c0f924b4d1278996a61043a0a42afad193e5f3fb"
+Fuel node key "006ba2f18d2e08f1cb96d3a425090768e3b6dc5e7f613a882509a02af668e6d7"
+Fuel node key "33184090500d0c26994df825800d169021e6dc32ecf1633d0903c28eecd87830"
+Fuel node key "d7f2a66d3f7bc9bdf135ad28b5106ee751aa5725d767336a2aa1ee19a5532c00"
 Publishing hydra scripts
 Starting hydra cluster in out
 Initializing Head
 Committing initialUTxO from dataset
 HeadIsOpen
-Client 1 (node 0): 0/300 (0.00%)
-Client 1 (node 0): 266/300 (88.67%)
+Client 1 (node 0): 1/10 (10.00%)
+Client 2 (node 1): 1/10 (10.00%)
+Client 3 (node 2): 1/10 (10.00%)
+All transactions confirmed. Sweet!
+All transactions confirmed. Sweet!
 All transactions confirmed. Sweet!
 Closing the Head
-Finalizing the Head
 Writing results to: out/results.csv
-Confirmed txs/Total expected txs: 300/300 (100.00 %)
-Average confirmation time (ms): 18.747147496
-P99: 23.100851369999994ms
-P95: 19.81722345ms
-P50: 18.532922ms
+Finalizing the Head
+Confirmed txs/Total expected txs: 30/30 (100.00 %)
+Average confirmation time (ms): 59.977068200
+P99: 75.43316676ms
+P95: 70.41318959999998ms
+P50: 60.638328ms
 Invalid txs: 0
+Fanout outputs: 3
 Writing report to: out/end-to-end-benchmarks.md
+
+./hydra-cluster/bench/plot.sh out
          line 0: warning: Cannot find or open file "out/system.csv"
 Created plot: out/results.png
 ```

hydra-cluster/bench/Bench/EndToEnd.hs

@@ -6,6 +6,7 @@ import Hydra.Prelude
 import Test.Hydra.Prelude
 
 import Bench.Summary (Summary (..), SystemStats, makeQuantiles)
+import Cardano.Api.UTxO qualified as UTxO
 import CardanoNode (findRunningCardanoNode', withCardanoNodeDevnet)
 import Control.Concurrent.Class.MonadSTM (
   MonadSTM (readTVarIO),
@@ -19,14 +20,15 @@ import Control.Lens (to, (^..), (^?))
 import Control.Monad.Class.MonadAsync (mapConcurrently)
 import Data.Aeson (Result (Error, Success), Value, encode, fromJSON, (.=))
 import Data.Aeson.Lens (key, values, _JSON, _Number, _String)
+import Data.Aeson.Types (parseEither)
 import Data.ByteString.Lazy qualified as LBS
 import Data.List qualified as List
 import Data.Map qualified as Map
 import Data.Scientific (Scientific)
 import Data.Set ((\\))
 import Data.Set qualified as Set
 import Data.Time (UTCTime (UTCTime), utctDayTime)
-import Hydra.Cardano.Api (NetworkId, SocketPath, Tx, TxId, UTxO, getVerificationKey, lovelaceToValue, signTx)
+import Hydra.Cardano.Api (Era, NetworkId, SocketPath, Tx, TxId, UTxO, getVerificationKey, lovelaceToValue, signTx)
 import Hydra.Chain.Backend (ChainBackend)
 import Hydra.Chain.Backend qualified as Backend
 import Hydra.Cluster.Faucet (FaucetLog (..), publishHydraScriptsAs, returnFundsToFaucet', seedFromFaucet)
@@ -177,21 +179,23 @@ scenario hydraTracer backend workDir Dataset{clientDatasets, title, description}
     guard $ v ^? key "headId" == Just (toJSON headId)
     v ^? key "contestationDeadline" . _JSON
 
+  -- Write the results already in case we cannot finalize
+  let res = mapMaybe analyze . Map.toList $ processedTransactions
+      aggregates = movingAverage res
+
+  writeResultsCsv (workDir </> "results.csv") aggregates
+
   -- Expect to see ReadyToFanout within 3 seconds after deadline
   remainingTime <- diffUTCTime deadline <$> getCurrentTime
   waitFor hydraTracer (remainingTime + 3) [leader] $
     output "ReadyToFanout" ["headId" .= headId]
 
   putTextLn "Finalizing the Head"
   send leader $ input "Fanout" []
-  waitMatch 100 leader $ \v -> do
+  finalUTxOJSON <- waitMatch 100 leader $ \v -> do
     guard (v ^? key "tag" == Just "HeadIsFinalized")
     guard $ v ^? key "headId" == Just (toJSON headId)
-
-  let res = mapMaybe analyze . Map.toList $ processedTransactions
-      aggregates = movingAverage res
-
-  writeResultsCsv (workDir </> "results.csv") aggregates
+    v ^? key "utxo"
 
   let confTimes = map (\(_, _, a) -> a) res
       numberOfTxs = length confTimes
@@ -200,6 +204,10 @@ scenario hydraTracer backend workDir Dataset{clientDatasets, title, description}
       quantiles = makeQuantiles confTimes
       summaryTitle = fromMaybe "Baseline Scenario" title
       summaryDescription = fromMaybe defaultDescription description
+      numberOfFanoutOutputs =
+        case parseEither (parseJSON @(UTxO.UTxO Era)) finalUTxOJSON of
+          Left _ -> error "Failed to decode Fanout UTxO"
+          Right fanoutUTxO -> UTxO.size fanoutUTxO
 
   pure $
     Summary
@@ -211,6 +219,7 @@ scenario hydraTracer backend workDir Dataset{clientDatasets, title, description}
       , summaryTitle
       , summaryDescription
       , numberOfInvalidTxs
+      , numberOfFanoutOutputs
       }
 
 defaultDescription :: Text

hydra-cluster/bench/Bench/Options.hs

@@ -29,23 +29,26 @@ import Options.Applicative (
  )
 import Options.Applicative.Builder (argument)
 
+data UTxOSize = Constant | Growing deriving (Eq, Show, Read)
+
 data Options
   = StandaloneOptions
-      { scalingFactor :: Int
-      , clusterSize :: Word64
+      { datasetFiles :: [FilePath]
       , outputDirectory :: Maybe FilePath
       , timeoutSeconds :: NominalDiffTime
       , startingNodeId :: Int
       }
   | DatasetOptions
-      { datasetFiles :: [FilePath]
-      , outputDirectory :: Maybe FilePath
+      { outputDirectory :: Maybe FilePath
       , timeoutSeconds :: NominalDiffTime
+      , datasetUTxO :: UTxOSize
+      , numberOfTxs :: Int
+      , clusterSize :: Word64
       , startingNodeId :: Int
       }
   | DemoOptions
       { outputDirectory :: Maybe FilePath
-      , scalingFactor :: Int
+      , numberOfTxs :: Int
       , timeoutSeconds :: NominalDiffTime
       , networkId :: NetworkId
       , nodeSocket :: SocketPath
@@ -78,13 +81,12 @@ standaloneOptionsInfo :: ParserInfo Options
 standaloneOptionsInfo =
   info
     standaloneOptionsParser
-    (progDesc "Runs a single scenario, generating or reusing a previous dataset from some directory.")
+    (progDesc "Runs a scenario reusing a previous dataset/s from some directory.")
 
 standaloneOptionsParser :: Parser Options
 standaloneOptionsParser =
   StandaloneOptions
-    <$> scalingFactorParser
-    <*> clusterSizeParser
+    <$> many filepathParser
     <*> optional outputDirectoryParser
     <*> timeoutParser
     <*> startingNodeIdParser
@@ -99,14 +101,14 @@ outputDirectoryParser =
           \ If not set, raw text summary will be printed to the console. (default: none)"
     )
 
-scalingFactorParser :: Parser Int
-scalingFactorParser =
+numberOfTxsParser :: Parser Int
+numberOfTxsParser =
   option
     auto
-    ( long "scaling-factor"
+    ( long "number-of-txs"
         <> value 100
         <> metavar "INT"
-        <> help "The scaling factor to apply to transactions generator (default: 100)"
+        <> help "Number of txs to generate (default: 100)"
     )
 
 timeoutParser :: Parser NominalDiffTime
@@ -146,6 +148,19 @@ startingNodeIdParser =
           \ benchmark conflicts with default ports allocation scheme (default: 0)"
     )
 
+utxoSizeParser :: Parser UTxOSize
+utxoSizeParser =
+  option
+    auto
+    ( long "utxo-size"
+        <> value Constant
+        <> metavar "UTxOSize"
+        <> help
+          "Generated UTxO size. This can be 'Constant' where UTxO set has constant size \
+          \ depending on the number of generated txs or 'Growing' where each new generated \
+          \ transaction produces one extra output which makes the UTxO in the Head grow."
+    )
+
 demoOptionsInfo :: ParserInfo Options
 demoOptionsInfo =
   info
@@ -161,7 +176,7 @@ demoOptionsParser :: Parser Options
 demoOptionsParser =
   DemoOptions
     <$> optional outputDirectoryParser
-    <*> scalingFactorParser
+    <*> numberOfTxsParser
     <*> timeoutParser
     <*> networkIdParser
     <*> nodeSocketParser
@@ -192,9 +207,11 @@ datasetOptionsInfo =
 datasetOptionsParser :: Parser Options
 datasetOptionsParser =
   DatasetOptions
-    <$> many filepathParser
-    <*> optional outputDirectoryParser
+    <$> optional outputDirectoryParser
     <*> timeoutParser
+    <*> utxoSizeParser
+    <*> numberOfTxsParser
+    <*> clusterSizeParser
     <*> startingNodeIdParser
 
 filepathParser :: Parser FilePath

hydra-cluster/bench/Bench/Summary.hs

@@ -29,6 +29,7 @@ data Summary = Summary
   , summaryTitle :: Text
   , summaryDescription :: Text
   , quantiles :: Vector Double
+  , numberOfFanoutOutputs :: Int
   }
   deriving stock (Generic, Eq, Show)
 
@@ -44,6 +45,7 @@ errorSummary Dataset{title, clientDatasets} (HUnitFailure sourceLocation reason)
     , summaryDescription =
         pack $ "Benchmark failed " <> formatLocation sourceLocation <> ": " <> formatFailureReason reason
     , quantiles = mempty
+    , numberOfFanoutOutputs = 0
     }
  where
   formatLocation = maybe "" (\loc -> "at " <> prettySrcLoc loc)
@@ -54,7 +56,7 @@ makeQuantiles times =
   Statistics.quantilesVec def (fromList [0 .. 99]) 100 (fromList $ map (fromRational . (* 1000) . toRational . nominalDiffTimeToSeconds) times)
 
 textReport :: (Summary, SystemStats) -> [Text]
-textReport (Summary{totalTxs, numberOfTxs, averageConfirmationTime, quantiles, numberOfInvalidTxs}, systemStats) =
+textReport (Summary{totalTxs, numberOfTxs, averageConfirmationTime, quantiles, numberOfInvalidTxs, numberOfFanoutOutputs}, systemStats) =
   let frac :: Double
       frac = 100 * fromIntegral numberOfTxs / fromIntegral totalTxs
    in [ pack $ printf "Confirmed txs/Total expected txs: %d/%d (%.2f %%)" numberOfTxs totalTxs frac
@@ -69,6 +71,7 @@ textReport (Summary{totalTxs, numberOfTxs, averageConfirmationTime, quantiles, n
               else []
            )
         ++ ["Invalid txs: " <> show numberOfInvalidTxs]
+        ++ ["Fanout outputs: " <> show numberOfFanoutOutputs]
         ++ if null systemStats then [] else "\n### Memory data \n" : [unlines systemStats]
 
 markdownReport :: UTCTime -> [(Summary, SystemStats)] -> [Text]