Merge pull request #5020 from IntersectMBO/nm/ledger-state-benchmarks

Update and fix `ledger-state` benchmarks

Author: neilmayhew

libs/ledger-state/README.md

@@ -9,64 +9,107 @@ the memory overhead.
 In order to be able to use the tool we need to get ahold of current ledger
 state. For this we need to start a cardano node and wait for it to sync.
 
-```haskell
-$ export CARDANO_DATA="${HOME}/iohk/chain/mainnet"
-$ mkdir -p "${CARDANO_DATA}"/db
+```shell
+$ export CARDANO_DATA=${HOME}/iohk/chain/mainnet
+$ mkdir -p "${CARDANO_DATA}/db"
 $ cd "${CARDANO_DATA}"
 ```
 
-Download all the [mainnet related config files](https://developers.cardano.org/docs/get-started/running-cardano/#mainnet--production):
+Download all the [mainnet related config files](https://developers.cardano.org/docs/get-started/cardano-node/running-cardano/#configuration-files):
+
+```shell
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/config.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/db-sync-config.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/submit-api-config.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/topology.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/byron-genesis.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/shelley-genesis.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/alonzo-genesis.json
+curl -O -J https://book.play.dev.cardano.org/environments/mainnet/conway-genesis.json
 ```
-curl -O -J https://hydra.iohk.io/build/7370192/download/1/mainnet-config.json
-curl -O -J https://hydra.iohk.io/build/7370192/download/1/mainnet-byron-genesis.json
-curl -O -J https://hydra.iohk.io/build/7370192/download/1/mainnet-shelley-genesis.json
-curl -O -J https://hydra.iohk.io/build/7370192/download/1/mainnet-alonzo-genesis.json
-curl -O -J https://hydra.iohk.io/build/7370192/download/1/mainnet-topology.json
+
+Download or build copies of `cardano-node` and `cardano-cli`. A convenient way of doing this is to download the assets from one of the `cardano-node` [releases](https://github.com/IntersectMBO/cardano-node/releases) on GitHub. The Linux executables are statically linked, so will run on any system.
+
+Download a snapshot of the node db using [mithril](https://mithril.network/doc/manual/getting-started/bootstrap-cardano-node/#bootstrap-a-cardano-node-from-a-testnet-mithril-cardano-db-snapshot). This will greatly speed up the process of syncing the node. There's a convenient nix-based script for doing it in `scripts/mithril-download.sh`:
+
+```shell
+$ scripts/mithril-download.sh -d "${CARDANO_DATA}/db" mainnet
 ```
 
+Note that you will need to get a snapshot that's compatible with the version of `cardano-node` you're using.
+
 Start the node and wait for it to fully sync
 
-```
+```shell
 $ export CARDANO_NODE_SOCKET_PATH="${CARDANO_DATA}/db/node.socket"
-$ cardano-node run
-  --topology "${CARDANO_DATA}/mainnet-topology.json" \
-  --database-path "${CARDANO_DATA}/db" \
-  --socket-path "${CARDANO_NODE_SOCKET_PATH}" \
-  --host-addr 127.0.0.1 \
-  --port 3001 \
-  --config "${CARDANO_DATA}/mainnet-config.json" &
+$ cardano-node run \
+    --topology "${CARDANO_DATA}/topology.json" \
+    --config "${CARDANO_DATA}/config.json" \
+    --database-path "${CARDANO_DATA}/db" \
+    --socket-path "${CARDANO_NODE_SOCKET_PATH}" \
+    --host-addr 0.0.0.0 \
+    --port 3001 &
+```
+
+Dump the ledger state and utxo:
+
+```shell
+$ cardano-cli query ledger-state --mainnet \
+    --socket-path "${CARDANO_NODE_SOCKET_PATH}" \
+    --out-file "${CARDANO_DATA}/new-epoch-state.bin"
+$ cardano-cli query utxo --mainnet --whole-utxo --output-cbor \
+    --socket-path "${CARDANO_NODE_SOCKET_PATH}" \
+    --out-file "${CARDANO_DATA}/utxo.hex"
 ```
 
-Dump the ledger state and focus back onto the node:
+Bring the node back into the foreground and use Ctrl-C to stop it:
 
 ```shell
-$ cardano-cli query ledger-state --mainnet --out-file "${CARDANO_DATA}/ledger-state.bin"
 $ fg
+$ ^C
 ```
-Hit Ctr-C to stop the node
 
-## Populate sqlite db
+## Populate sqlite db from `NewEpochState` file
 
 ```shell
-$ cabal run -- ledger-state:ledger-state --new-epoch-state-cbor="${CARDANO_DATA}/ledger-state.bin" --sqlite-db="${CARDANO_DATA}/ledger-state.sqlite"
+$ cabal run -- ledger-state:ledger-state \
+    --new-epoch-state-cbor="${CARDANO_DATA}/new-epoch-state.bin" \
+    --sqlite-db="${CARDANO_DATA}/epoch-state.sqlite"
 ```
 
-## Create NewEpochState from sqlite db
+## Create `EpochState` file from sqlite db
 
 ```shell
-$ cabal run -- ledger-state:ledger-state --epoch-state-cbor="${CARDANO_DATA}/ledger-state.bin" --sqlite-db="${CARDANO_DATA}/ledger-state.sqlite"
+$ cabal run -- ledger-state:ledger-state \
+    --epoch-state-cbor="${CARDANO_DATA}/epoch-state.bin" \
+    --sqlite-db="${CARDANO_DATA}/epoch-state.sqlite"
 ```
 
 ## Running benchmarks
 
 ### Memory
 
 ```shell
-$ cabal bench ledger-state:memory --benchmark-options="--new-epoch-state-cbor=\"${CARDANO_DATA}/ledger-state.bin\" --new-epoch-state-sqlite=\"${CARDANO_DATA}/ledger-state.sqlite\""
+$ cabal build ledger-state:memory
+$ cabal bench ledger-state:memory -v0 \
+    --benchmark-option="--new-epoch-state-cbor=${CARDANO_DATA}/new-epoch-state.bin" \
+    --benchmark-option="--epoch-state-cbor=${CARDANO_DATA}/epoch-state.bin" \
+    --benchmark-option="--sqlite-db=${CARDANO_DATA}/epoch-state.sqlite" |
+  tee ledger-state:memory.txt
 ```
 ### Performance
 
 Performance benchmarks need an actual mainnet ledger state and genesis config
 file to run properly. It is not possible to add extra arguments to criterion cli
 menu, therefore paths to those files must be supplied as environment variables.
 
+```shell
+$ export BENCH_GENESIS_PATH=${CARDANO_DATA}/shelley-genesis.json
+$ export BENCH_LEDGER_STATE_PATH=${CARDANO_DATA}/new-epoch-state.bin
+$ export BENCH_UTXO_PATH=${CARDANO_DATA}/utxo.hex
+$ cabal bench ledger-state:performance --benchmark-option=--csv=ledger-state:performance.csv
+```
+
+The csv file will be saved in the `libs/ledger-state` directory.
+
+Since the `performance` benchmark uses only `new-epoch-state.bin` you don't need to run the `sqlite.db` steps above if you want to run only the `performance` benchmark.

libs/ledger-state/app/Main.hs

@@ -14,7 +14,7 @@ import System.IO
 -- | Insight into options:
 --
 -- * `optsNewEpochStateBinaryFile` is for reading a previously serialized
--- * `NewEpochState` produced by cadano-cli` and is used to populate sqlite
+-- * `NewEpochState` produced by cardano-cli` and is used to populate sqlite
 -- * database
 --
 -- * `optsEpochStateBinaryFile` is used for grabbing data from sqlite,
@@ -26,7 +26,7 @@ data Opts = Opts
   -- load into sqlite database
   , optsEpochStateBinaryFile :: Maybe FilePath
   -- ^ Path to the CBOR encoded EpochState data type, which will have data
-  -- from sqlite database written into.
+  -- from sqlite database written into it.
   , optsSqliteDbFile :: Maybe FilePath
   -- ^ Path to Sqlite database file.
   }
@@ -62,7 +62,7 @@ optsParser =
           <> value Nothing
           <> help
             ( "Path to Sqlite database file. When supplied then new-epoch-state "
-                <> "will be loaded into the databse. Requires --new-epoch-state-cbor"
+                <> "will be loaded into the database. Requires --new-epoch-state-cbor"
             )
       )
 
@@ -92,7 +92,7 @@ main = do
       epochState <- loadEpochState dbFp
       putStrLn "Loaded EpochState from the database"
       writeEpochState binFp epochState
-      putStrLn $ "Written EpochState into: " ++ dbFpStr
+      putStrLn $ "Written EpochState into: " ++ binFp
 
 -- forM_ (optsSqliteDbFile opts) $ \dbFpStr -> do
 --   let dbFp = T.pack dbFpStr

libs/ledger-state/bench/Performance.hs

@@ -1,5 +1,6 @@
 {-# LANGUAGE BangPatterns #-}
 {-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE TypeApplications #-}
@@ -11,6 +12,10 @@ import Cardano.Ledger.Api.Era
 import Cardano.Ledger.Api.State.Query (queryStakePoolDelegsAndRewards)
 import Cardano.Ledger.BaseTypes
 import Cardano.Ledger.Binary
+import Cardano.Ledger.Conway.Rules (
+  ConwayLedgerPredFailure (ConwayUtxowFailure),
+  ConwayUtxowPredFailure (InvalidWitnessesUTXOW),
+ )
 import Cardano.Ledger.Core
 import Cardano.Ledger.Shelley.API.Mempool
 import Cardano.Ledger.Shelley.API.Wallet (getFilteredUTxO, getUTxO)
@@ -21,54 +26,77 @@ import Cardano.Ledger.Shelley.Genesis (
  )
 import Cardano.Ledger.Shelley.LedgerState
 import Cardano.Ledger.State
-import Cardano.Ledger.State.UTxO (CurrentEra, readNewEpochState)
+import Cardano.Ledger.State.UTxO (CurrentEra, readHexUTxO, readNewEpochState)
 import Cardano.Ledger.UMap
 import Cardano.Ledger.Val
 import Cardano.Slotting.EpochInfo (fixedEpochInfo)
 import Cardano.Slotting.Time (mkSlotLength)
 import Control.DeepSeq
+import Control.Monad (when)
 import Criterion.Main
 import Data.Aeson
-import Data.Bifunctor (first)
+import Data.Bifunctor (bimap, first)
 import Data.ByteString.Base16.Lazy as BSL16
 import Data.ByteString.Lazy (ByteString)
 import Data.Foldable as F
+import Data.List.NonEmpty (NonEmpty ((:|)))
 import Data.Map.Strict (Map)
 import qualified Data.Map.Strict as Map
 import Data.MapExtras (extractKeys, extractKeysSmallSet)
 import Data.Set (Set)
 import qualified Data.Set as Set
-import Lens.Micro ((^.))
+import GHC.Stack (HasCallStack)
+import Lens.Micro ((&), (.~), (^.))
 import System.Environment (getEnv)
+import System.Exit (die)
 import System.Random.Stateful
 import Test.Cardano.Ledger.Api.State.Query (getFilteredDelegationsAndRewardAccounts)
 import Test.Cardano.Ledger.Core.Arbitrary (uniformSubSet)
 
 main :: IO ()
 main = do
-  let ledgerVarName = "BENCH_LEDGER_STATE_PATH"
-      genesisVarName = "BENCH_GENESIS_PATH"
-  ledgerStateFilePath <- getEnv ledgerVarName
+  let genesisVarName = "BENCH_GENESIS_PATH"
+      utxoVarName = "BENCH_UTXO_PATH"
+      ledgerStateVarName = "BENCH_LEDGER_STATE_PATH"
   genesisFilePath <- getEnv genesisVarName
+  utxoFilePath <- getEnv utxoVarName
+  ledgerStateFilePath <- getEnv ledgerStateVarName
+
   genesis <- either error id <$> eitherDecodeFileStrict' genesisFilePath
+  putStrLn $ "Importing UTxO from: " ++ show utxoFilePath
+  utxo <- readHexUTxO utxoFilePath
+  putStrLn "Done importing UTxO"
+  putStrLn $ "Importing NewEpochState from: " ++ show ledgerStateFilePath
+  es' <- readNewEpochState ledgerStateFilePath
+  putStrLn "Done importing NewEpochState"
+
+  let nesUTxOL = nesEsL . esLStateL . lsUTxOStateL . utxoL
+      es = es' & nesUTxOL .~ utxo
+      utxoMap = unUTxO utxo
+      utxoSize = Map.size utxoMap
+      largeKeysNum = 100000
+      stdGen = mkStdGen 2022
 
   let toMempoolState :: NewEpochState CurrentEra -> MempoolState CurrentEra
       toMempoolState NewEpochState {nesEs = EpochState {esLState}} = esLState
       !globals = mkGlobals genesis
       !slotNo = SlotNo 55733343
+      restrictError = \case
+        ApplyTxError (ConwayUtxowFailure (InvalidWitnessesUTXOW [_]) :| []) -> ()
+        otherErr -> error . show $ otherErr
       applyTx' mempoolEnv mempoolState =
-        either (error . show) seqTuple
+        -- TODO: revert this to `either (error . show) seqTuple` after tx's are fixed
+        bimap restrictError seqTuple
           . applyTx globals mempoolEnv mempoolState
-      reapplyTx' mempoolEnv mempoolState tx =
-        case reapplyTx globals mempoolEnv mempoolState tx of
-          Left err -> error (show err)
-          Right st -> st
-  putStrLn $ "Importing NewEpochState from: " ++ show ledgerStateFilePath
-  es <- readNewEpochState ledgerStateFilePath
-  putStrLn "Done importing NewEpochState"
-  let largeKeysNum = 100000
-      stdGen = mkStdGen 2022
-  largeKeys <- selectRandomMapKeys 100000 stdGen (unUTxO (getUTxO es))
+      reapplyTx' mempoolEnv mempoolState =
+        either (error . show) id
+          . reapplyTx globals mempoolEnv mempoolState
+
+  when (utxoSize < largeKeysNum) $
+    die $
+      "UTxO size is too small (" <> show utxoSize <> " < " <> show largeKeysNum <> ")"
+  largeKeys <- selectRandomMapKeys 100000 stdGen utxoMap
+
   defaultMain
     [ env (pure (mkMempoolEnv es slotNo, toMempoolState es)) $ \ ~(mempoolEnv, mempoolState) ->
         bgroup
@@ -92,22 +120,26 @@ main = do
               bench "Tx2" . whnf (applyTx' mempoolEnv mempoolState)
           , env (pure (extractTx validatedTx3)) $
               bench "Tx3" . whnf (applyTx' mempoolEnv mempoolState)
+          , env
+              (pure [validatedTx1, validatedTx2, validatedTx3])
+              $ bench "Tx1+Tx2+Tx3"
+                -- TODO: revert this to `foldl'` without `fmap` after tx's are fixed
+                . whnf (F.foldlM (\ms -> fmap fst . applyTx' mempoolEnv ms . extractTx) mempoolState)
           ]
-    , env (pure (getUTxO es)) $ \utxo ->
+    , env (pure utxo) $ \utxo' ->
         bgroup
           "UTxO"
-          [ bench "balance" $ nf balance utxo
-          , bench "coinBalance" $ nf coinBalance utxo
+          [ bench "balance" $ nf balance utxo'
+          , bench "coinBalance" $ nf coinBalance utxo'
           , -- We need to filter out all multi-assets to prevent `areAllAdaOnly`
             -- from short circuiting and producing results that are way better
             -- than the worst case
-            env (pure $ Map.filter (\txOut -> isAdaOnly (txOut ^. valueTxOutL)) $ unUTxO utxo) $
+            env (pure $ Map.filter (\txOut -> isAdaOnly (txOut ^. valueTxOutL)) $ unUTxO utxo') $
               bench "areAllAdaOnly" . nf areAllAdaOnly
           ]
     , env (pure es) $ \newEpochState ->
-        let utxo = getUTxO es
-            (_, minTxOut) = Map.findMin $ unUTxO utxo
-            (_, maxTxOut) = Map.findMax $ unUTxO utxo
+        let (_, minTxOut) = Map.findMin utxoMap
+            (_, maxTxOut) = Map.findMax utxoMap
             setAddr =
               Set.fromList [minTxOut ^. addrTxOutL, maxTxOut ^. addrTxOutL]
          in bgroup
@@ -137,10 +169,10 @@ main = do
               ]
     , bgroup
         "DeleteTxOuts"
-        [ extractKeysBench (unUTxO (getUTxO es)) largeKeysNum largeKeys
-        , extractKeysBench (unUTxO (getUTxO es)) 9 (Set.take 9 largeKeys)
-        , extractKeysBench (unUTxO (getUTxO es)) 5 (Set.take 5 largeKeys)
-        , extractKeysBench (unUTxO (getUTxO es)) 2 (Set.take 2 largeKeys)
+        [ extractKeysBench utxoMap largeKeysNum largeKeys
+        , extractKeysBench utxoMap 9 (Set.take 9 largeKeys)
+        , extractKeysBench utxoMap 5 (Set.take 5 largeKeys)
+        , extractKeysBench utxoMap 2 (Set.take 2 largeKeys)
         ]
     ]
 
@@ -176,10 +208,12 @@ selectRandomMapKeys n gen m = runStateGenT_ gen $ \g ->
 extractKeysNaive :: Ord k => Map k a -> Set.Set k -> (Map k a, Map k a)
 extractKeysNaive sm s = (Map.withoutKeys sm s, Map.restrictKeys sm s)
 
-decodeTx :: ByteString -> Tx CurrentEra
+decodeTx :: HasCallStack => ByteString -> Tx CurrentEra
 decodeTx hex = either error id $ do
   bsl <- BSL16.decode hex
-  first show $ decodeFull (eraProtVerHigh @CurrentEra) bsl
+  tx <- first show $ decodeFull (eraProtVerHigh @BabbageEra) bsl
+  -- TODO: remove this after the transactions below are updated
+  first show $ upgradeTx tx
 
 -- | Most basic ada-only transaction:
 --
@@ -235,8 +269,8 @@ validatedTx3 =
       \424643546f6b656e1a006cc9f2021a0002afe90e81581c780648b89ea2f11fa9bbdd67\
       \552db5dd020eda1c9a54142dd9f1b136a10081825820cf2477066091b565f87f044581\
       \7c4df726900b29af3f05d229309afdbf94296d584088444a5845b198a2d255175770be\
-      \7120c2d3482751b14f06dd41d7ff023eeae6e63933b097c023c1ed19df6a061173c45aa\
-      \54cceb568ff1886e2716e84e6260df5f6"
+      \7120c2d3482751b14f06dd41d7ff023eeae6e63933b097c023c1ed19df6a061173c45a\
+      \a54cceb568ff1886e2716e84e6260df5f6"
 
 mkGlobals :: ShelleyGenesis -> Globals
 mkGlobals genesis =

libs/ledger-state/ledger-state.cabal

@@ -36,6 +36,7 @@ library
 
   build-depends:
     base >=4.14 && <5,
+    base16-bytestring,
     bytestring,
     cardano-crypto-class,
     cardano-ledger-alonzo,
@@ -124,6 +125,7 @@ benchmark performance
     cardano-data,
     cardano-ledger-api:{cardano-ledger-api, testlib},
     cardano-ledger-binary,
+    cardano-ledger-conway,
     cardano-ledger-core:{cardano-ledger-core, testlib},
     cardano-ledger-shelley,
     cardano-slotting,