Skip to content

Commit 8089a6e

Browse files
mgmeierfmaste
mgmeier
authored and
fmaste
committed
try: cardano-profile: use LMDB on nomad
1 parent 3f7adf6 commit 8089a6e

File tree

8 files changed

+5556
-3513
lines changed

8 files changed

+5556
-3513
lines changed

‎bench/cardano-profile/data/all-profiles-coay.json‎

Lines changed: 5440 additions & 3448 deletions
Large diffs are not rendered by default.

‎bench/cardano-profile/src/Cardano/Benchmarking/Profile/Builtin/Cloud.hs‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,16 @@ profilesNoEraCloud =
139139
-- PParams overlays and calibration for 4 tx per block memory full.
140140
blockMem15x = P.budgetBlockMemoryOneAndAHalf . P.overlay Pl.calibrateLoopBlockMemx15
141141
blockMem2x = P.budgetBlockMemoryDouble . P.overlay Pl.calibrateLoopBlockMemx2
142+
-- LMDB helper. Node config add the "hostvolume"s as a cluster constraint.
143+
lmdb = P.lmdb
144+
. -- The name of the defined volume in the Nomad Client config and where to
145+
-- mount it inside the isolated chroot.
146+
-- If the volume is not present the deployment will fail!
147+
P.nomadHostVolume (Types.ByNodeType {
148+
Types.producer = [Types.HostVolume "/ephemeral" False "ephemeral"]
149+
, Types.explorer = Nothing
150+
})
151+
. P.ssdDirectory "/ephemeral"
142152
in [
143153
-- Value (pre-Voltaire profiles)
144154
value & P.name "value-nomadperf" . P.dreps 0 . P.newTracing . P.p2pOn
@@ -151,6 +161,7 @@ profilesNoEraCloud =
151161
-- Value (post-Voltaire profiles)
152162
, valueVolt & P.name "value-volt-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn
153163
, valueVolt & P.name "value-volt-rtsqg1-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . P.rtsGcParallel . P.rtsGcLoadBalance
164+
, valueVolt & P.name "value-volt-lmdb-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . lmdb
154165
-- Plutus (pre-Voltaire profiles)
155166
, loop & P.name "plutus-nomadperf" . P.dreps 0 . P.newTracing . P.p2pOn
156167
, loop & P.name "plutus-nomadperf-nop2p" . P.dreps 0 . P.newTracing . P.p2pOff
@@ -165,6 +176,7 @@ profilesNoEraCloud =
165176
, loopVolt & P.name "plutus-volt-memx15-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . blockMem15x
166177
, loopVolt & P.name "plutus-volt-memx2-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . blockMem2x
167178
, loopVolt & P.name "plutus-volt-rtsqg1-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . P.rtsGcParallel . P.rtsGcLoadBalance
179+
, loopVolt & P.name "plutus-volt-lmdb-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . lmdb
168180
-- TODO: scaling the BLST workload only works well for 4 txns/block instead of 8. However, comparing it to other steps-constrained workloads, requires 8txns/block (like all of those).
169181
, blst & P.name "plutusv3-blst-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . P.v10Preview
170182
, blst & P.name "plutusv3-blst-stepx15-nomadperf" . P.dreps 10000 . P.newTracing . P.p2pOn . P.v10Preview . P.budgetBlockStepsOneAndAHalf
@@ -281,34 +293,48 @@ profilesNoEraCloud =
281293

282294
nomadPerf :: Types.Profile -> Types.Profile
283295
nomadPerf =
296+
-- Exact regions with availability zone (AZ) to match.
284297
P.regions
285298
[
286299
Types.AWS Types.EU_CENTRAL_1
287300
, Types.AWS Types.US_EAST_1
288301
, Types.AWS Types.AP_SOUTHEAST_2
289302
]
290303
.
291-
P.nomadNamespace "perf"
292-
.
293-
P.nomadClass "perf"
304+
-- Logical cluster separation. To avoid conflicts with same-server machines.
305+
P.nomadNamespace "perf" . P.nomadClass "perf"
294306
.
307+
-- This will be used as constraints at the Task level.
295308
P.nomadResources (Types.ByNodeType {
296-
Types.producer = Types.Resources 8 15400 16000
297-
, Types.explorer = Just $ Types.Resources 16 32000 64000
309+
Types.producer = Types.Resources {
310+
Types.cores = 8
311+
, Types.memory = 15400
312+
, Types.memory_max = 16000
313+
}
314+
, Types.explorer = Just $ Types.Resources {
315+
Types.cores = 16
316+
, Types.memory = 32000
317+
, Types.memory_max = 64000
318+
}
298319
})
299320
.
300-
P.nomadSSHLogsOn
301-
.
302-
P.clusterKeepRunningOn
303-
.
321+
-- Instance types will be used as Group "constraints".
304322
P.awsInstanceTypes (Types.ByNodeType {
305323
Types.producer = "c5d.2xlarge"
306324
, Types.explorer = Just "m5.4xlarge"
307325
})
308326
.
327+
-- Force all network related stuff to "attr.unique.platform.aws.public-ipv4".
309328
P.usePublicRouting
310329
.
330+
-- Nomad cloud backend Jobs won't start below these levels.
311331
P.clusterMinimunStorage (Just $ Types.ByNodeType {
312332
Types.producer = 12582912
313333
, Types.explorer = Just 14155776
314334
})
335+
.
336+
-- Flag to use SSH instead of `nomad exec` to fetch the logs.
337+
P.nomadSSHLogsOn
338+
.
339+
-- Don't stop the Nomad Job when finished.
340+
P.clusterKeepRunningOn

‎bench/cardano-profile/src/Cardano/Benchmarking/Profile/Extra/Scaling.hs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,16 @@ clusterNomadSsdNoRegions =
100100
, Types.explorer = Just $ Types.Resources 16 120000 124000
101101
})
102102
.
103-
P.nomadHostVolume (Types.HostVolume "/ssd2" False "ssd2")
104-
.
105-
P.nomadHostVolume (Types.HostVolume "/ssd1" False "ssd1")
103+
P.nomadHostVolume (
104+
let hostVolumes =
105+
[ Types.HostVolume "/ssd1" False "ssd1"
106+
, Types.HostVolume "/ssd2" False "ssd2"
107+
]
108+
in Types.ByNodeType {
109+
Types.producer = hostVolumes
110+
, Types.explorer = Just hostVolumes
111+
}
112+
)
106113
.
107114
P.nomadSSHLogsOn
108115
.

‎bench/cardano-profile/src/Cardano/Benchmarking/Profile/Primitives.hs‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ module Cardano.Benchmarking.Profile.Primitives (
113113

114114
) where
115115

116-
import Prelude hiding (id)
116+
import Prelude
117117
import Data.Maybe (isJust)
118118
import GHC.Stack (HasCallStack)
119119
-- Package: aeson.
@@ -877,13 +877,8 @@ nomadClass nc = nomad (\n -> n {Types.nomad_class = nc})
877877
nomadResources :: Types.ByNodeType Types.Resources -> Types.Profile -> Types.Profile
878878
nomadResources r = nomad (\n -> n {Types.resources = r})
879879

880-
nomadHostVolume :: Types.HostVolume -> Types.Profile -> Types.Profile
881-
nomadHostVolume hv = nomad (\n ->
882-
let mhvs = case Types.host_volumes n of
883-
Nothing -> Just [hv]
884-
(Just hvs) -> Just $ hvs ++ [hv]
885-
in n {Types.host_volumes = mhvs}
886-
)
880+
nomadHostVolume :: Types.ByNodeType [Types.HostVolume] -> Types.Profile -> Types.Profile
881+
nomadHostVolume h = nomad (\n -> n {Types.host_volumes = Just h})
887882

888883
nomadSSHLogsOn :: Types.Profile -> Types.Profile
889884
nomadSSHLogsOn = nomad (\n -> n {Types.fetch_logs_ssh = True})

‎bench/cardano-profile/src/Cardano/Benchmarking/Profile/Types.hs‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ data ClusterNomad = ClusterNomad
627627
{ namespace :: String
628628
, nomad_class :: String
629629
, resources :: ByNodeType Resources
630-
, host_volumes :: Maybe [HostVolume]
630+
, host_volumes :: Maybe (ByNodeType [HostVolume])
631631
, fetch_logs_ssh :: Bool
632632
}
633633
deriving (Eq, Show, Generic)
@@ -652,19 +652,6 @@ instance Aeson.FromJSON ClusterNomad where
652652
<*> o Aeson..: "host_volumes"
653653
<*> o Aeson..: "fetch_logs_ssh"
654654

655-
data HostVolume = HostVolume
656-
{ destination :: String
657-
, read_only :: Bool
658-
, source :: String
659-
}
660-
deriving (Eq, Show, Generic)
661-
662-
instance Aeson.ToJSON HostVolume
663-
664-
instance Aeson.FromJSON HostVolume where
665-
parseJSON = Aeson.genericParseJSON
666-
(Aeson.defaultOptions {Aeson.rejectUnknownFields = True})
667-
668655
data ClusterAWS = ClusterAWS
669656
{ instance_type :: ByNodeType String
670657
, use_public_routing :: Bool
@@ -687,6 +674,7 @@ instance Aeson.ToJSON a => Aeson.ToJSON (ByNodeType a)
687674

688675
instance Aeson.FromJSON a => Aeson.FromJSON (ByNodeType a)
689676

677+
-- These matches Nomad "resources" inside each Job Task.
690678
data Resources = Resources
691679
{ cores :: Integer
692680
, memory :: Integer
@@ -700,6 +688,27 @@ instance Aeson.FromJSON Resources where
700688
parseJSON = Aeson.genericParseJSON
701689
(Aeson.defaultOptions {Aeson.rejectUnknownFields = True})
702690

691+
-- The is used in the Nomad Job to define "volume" at the Group level and
692+
-- "volume_mount" at the Task level.
693+
data HostVolume = HostVolume
694+
{ -- Used at the Task level to create the "volume_mount" property.
695+
-- The destination is where it'll appear inside the Task's isolated chroot.
696+
destination :: String
697+
-- How it should be mounted inside the Task's isolated chroot.
698+
-- Independent of how it's defined in the Nomad Client config.
699+
, read_only :: Bool
700+
-- Used at the Group level to create the "volume" property.
701+
-- This name matches the Nomad Client config (client.host_volume.NAME).
702+
, source :: String
703+
}
704+
deriving (Eq, Show, Generic)
705+
706+
instance Aeson.ToJSON HostVolume
707+
708+
instance Aeson.FromJSON HostVolume where
709+
parseJSON = Aeson.genericParseJSON
710+
(Aeson.defaultOptions {Aeson.rejectUnknownFields = True})
711+
703712
--------------------------------------------------------------------------------
704713

705714
data Analysis = Analysis

‎nix/workbench/backend/nomad-job.nix‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -465,22 +465,29 @@ let
465465
//
466466
# If it needs host volumes add the constraints (can't be "null" or "[]".)
467467
### - https://developer.hashicorp.com/nomad/tutorials/stateful-workloads/stateful-workloads-host-volumes
468-
(lib.optionalAttrs ((profile.cluster or null) != null && profile.cluster.nomad.host_volumes != null) {
469-
volume = lib.listToAttrs (lib.lists.imap0
470-
(i: v: {
471-
# Internal name, reference to mount in this group's tasks below.
472-
name = "volume-${taskName}-${toString i}";
473-
value = {
474-
type = "host"; # We only support type "host".
475-
read_only = v.read_only;
476-
# How it is named in the Nomad Client's config.
477-
# https://developer.hashicorp.com/nomad/docs/configuration/client#host_volume-block
478-
source = v.source;
479-
};
480-
})
481-
profile.cluster.nomad.host_volumes
482-
);
483-
})
468+
(
469+
let
470+
# JSON Object like `{"explorer": null, "producers": [...]}`.
471+
nodeType = if nodeSpec.name == "explorer" then "explorer" else "producer";
472+
volumesList = profile.cluster.nomad.host_volumes.${nodeType} or null;
473+
in
474+
(lib.attrsets.optionalAttrs (volumesList != null)
475+
{ volume = lib.listToAttrs (lib.lists.imap0
476+
(i: v: {
477+
# Internal name, reference to mount in this group's tasks below.
478+
name = "volume-${taskName}-${toString i}";
479+
value = {
480+
type = "host"; # We only support type "host".
481+
read_only = v.read_only;
482+
# How it is named in the Nomad Client's config.
483+
# https://developer.hashicorp.com/nomad/docs/configuration/client#host_volume-block
484+
source = v.source;
485+
};
486+
})
487+
volumesList
488+
);}
489+
)
490+
)
484491
//
485492
{
486493
# The task stanza creates an individual unit of work, such as a Docker
@@ -591,17 +598,23 @@ let
591598
};
592599

593600
# If it needs host volumes mount them (defined above if any).
594-
volume_mount = if (profile.cluster or null) != null && profile.cluster.nomad.host_volumes != null
595-
then lib.lists.imap0
596-
(i: v: {
597-
# Internal name, defined above in the group's specification.
598-
volume = "volume-${taskName}-${toString i}";
599-
# Where it is going to be mounted inside the Task.
600-
destination = v.destination;
601-
read_only = v.read_only;
602-
})
603-
profile.cluster.nomad.host_volumes
604-
else null
601+
volume_mount =
602+
let
603+
# JSON Object like `{"explorer": null, "producers": [...]}`.
604+
nodeType = if nodeSpec.name == "explorer" then "explorer" else "producer";
605+
volumesList = profile.cluster.nomad.host_volumes.${nodeType} or null;
606+
in
607+
if volumesList != null
608+
then lib.lists.imap0
609+
(i: v: {
610+
# Internal name, defined above in the group's specification.
611+
volume = "volume-${taskName}-${toString i}";
612+
# Where it is going to be mounted inside the Task.
613+
destination = v.destination;
614+
read_only = v.read_only;
615+
})
616+
volumesList
617+
else null
605618
;
606619

607620
# Specifies the set of templates to render for the task. Templates can

‎nix/workbench/service/nodes.nix‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,9 @@ let
7373
topology = "topology.json";
7474
nodeConfigFile = "config.json";
7575

76-
# Allow for local clusters to have multiple LMDB directories in the same physical ssd_directory
77-
withUtxoHdLmdb = profile.node.utxo_lmdb;
76+
# Allow for local clusters to have multiple LMDB directories in the same physical ssd_directory;
77+
# non-block producers (like the explorer node) keep using the in-memory backend
78+
withUtxoHdLmdb = profile.node.utxo_lmdb && isProducer;
7879
lmdbDatabasePath = liveTablesPath i;
7980

8081
## Combine:
@@ -117,7 +118,7 @@ let
117118
ShelleyGenesisFile = "../genesis/genesis-shelley.json";
118119
AlonzoGenesisFile = "../genesis/genesis.alonzo.json";
119120
ConwayGenesisFile = "../genesis/genesis.conway.json";
120-
} // optionalAttrs profile.node.utxo_lmdb
121+
} // optionalAttrs (profile.node.utxo_lmdb && isProducer)
121122
{
122123
LedgerDB = {
123124
Backend = "V1LMDB";

‎wb_profiles.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ PROFILES_K3 := k3-3ep-5kTx-10000kU-1300kD-64kbs-fixed-loaded k3-3ep-9kTx-10000kU
77
PROFILES_SCENARIOS := chainsync-early-byron chainsync-early-byron-notracer chainsync-early-byron-oldtracing chainsync-early-alonzo chainsync-early-alonzo-notracer chainsync-early-alonzo-p2p chainsync-early-alonzo-oldtracing devops idle tracer-only
88
PROFILES_LEGACY := ci-test-dense10 dish dish-10M dish-plutus dish-10M-plutus
99
PROFILES_SCALING := faststartup-24M
10-
PROFILES_NOMAD_PERF := value-nomadperf value-nomadperf-nop2p value-drep1k-nomadperf value-drep10k-nomadperf value-drep100k-nomadperf value-oldtracing-nomadperf value-oldtracing-nomadperf-nop2p value-volt-nomadperf value-volt-rtsqg1-nomadperf plutus-nomadperf plutus-nomadperf-nop2p plutus-drep1k-nomadperf plutus-drep10k-nomadperf plutus-drep100k-nomadperf plutus24-nomadperf plutus-secp-ecdsa-nomadperf plutus-secp-schnorr-nomadperf plutus-volt-nomadperf plutus-volt-memx15-nomadperf plutus-volt-memx2-nomadperf plutus-volt-rtsqg1-nomadperf plutusv3-blst-nomadperf plutusv3-blst-stepx15-nomadperf plutusv3-blst-stepx2-nomadperf plutusv3-ripemd-nomadperf plutusv3-ripemd-stepx15-nomadperf plutusv3-ripemd-stepx2-nomadperf value-voting-utxo-volt-nomadperf value-voting-volt-nomadperf value-voting-double-volt-nomadperf plutus-voting-utxo-volt-nomadperf plutus-voting-volt-nomadperf plutus-voting-double-volt-nomadperf latency-nomadperf fast-nomadperf fast-nomadperf-nop2p ci-test-nomadperf ci-test-nomadperf-nop2p ci-test-oldtracing-nomadperf default-nomadperf-nop2p default-nomadperf oldtracing-nomadperf oldtracing-nomadperf-nop2p ci-bench-nomadperf ci-bench-nomadperf-nop2p ci-bench-oldtracing-nomadperf
10+
PROFILES_NOMAD_PERF := value-nomadperf value-nomadperf-nop2p value-drep1k-nomadperf value-drep10k-nomadperf value-drep100k-nomadperf value-oldtracing-nomadperf value-oldtracing-nomadperf-nop2p value-volt-nomadperf value-volt-rtsqg1-nomadperf value-volt-lmdb-nomadperf plutus-nomadperf plutus-nomadperf-nop2p plutus-drep1k-nomadperf plutus-drep10k-nomadperf plutus-drep100k-nomadperf plutus24-nomadperf plutus-secp-ecdsa-nomadperf plutus-secp-schnorr-nomadperf plutus-volt-nomadperf plutus-volt-memx15-nomadperf plutus-volt-memx2-nomadperf plutus-volt-rtsqg1-nomadperf plutus-volt-lmdb-nomadperf plutusv3-blst-nomadperf plutusv3-blst-stepx15-nomadperf plutusv3-blst-stepx2-nomadperf plutusv3-ripemd-nomadperf plutusv3-ripemd-stepx15-nomadperf plutusv3-ripemd-stepx2-nomadperf value-voting-utxo-volt-nomadperf value-voting-volt-nomadperf value-voting-double-volt-nomadperf plutus-voting-utxo-volt-nomadperf plutus-voting-volt-nomadperf plutus-voting-double-volt-nomadperf latency-nomadperf fast-nomadperf fast-nomadperf-nop2p ci-test-nomadperf ci-test-nomadperf-nop2p ci-test-oldtracing-nomadperf default-nomadperf-nop2p default-nomadperf oldtracing-nomadperf oldtracing-nomadperf-nop2p ci-bench-nomadperf ci-bench-nomadperf-nop2p ci-bench-oldtracing-nomadperf
1111
PROFILES_NOMAD_PERFSSD := utxoscale-solo-12M16G-nomadperfssd utxoscale-solo-12M64G-nomadperfssd utxoscale-solo-24M64G-nomadperfssd fast-nomadperfssd value-nomadperfssd latency-nomadperfssd
1212

1313
LOCAL_PROFILES += $(PROFILES_EMPTY)

0 commit comments

Comments
 (0)