cardano-testnet: allow to take node configuration file as input

cardano-testnet/cardano-testnet.cabal

@@ -91,6 +91,7 @@ library
                       , transformers
                       , transformers-except
                       , vector
+                      , yaml
 
   hs-source-dirs:       src
   exposed-modules:      Cardano.Testnet

cardano-testnet/src/Cardano/Testnet.hs

@@ -11,6 +11,7 @@ module Cardano.Testnet (
   -- ** Testnet options
   CardanoTestnetOptions(..),
   TestnetNodeOptions(..),
+  AutomaticNodeOption(..),
   cardanoDefaultTestnetNodeOptions,
   getDefaultAlonzoGenesis,
   getDefaultShelleyGenesis,

cardano-testnet/src/Parsers/Cardano.hs

@@ -1,3 +1,5 @@
+{-# LANGUAGE ScopedTypeVariables #-}
+
 module Parsers.Cardano
   ( cmdCardano
   ) where
@@ -9,6 +11,7 @@ import           Cardano.CLI.EraBased.Options.Common hiding (pNetworkId)
 
 import           Prelude
 
+import           Control.Applicative
 import           Data.Default.Class
 import           Data.Functor
 import qualified Data.List as L
@@ -20,6 +23,7 @@ import           Testnet.Start.Cardano
 import           Testnet.Start.Types
 import           Testnet.Types (readNodeLoggingFormat)
 
+{- HLINT ignore "Use asum" -}
 
 optsTestnet :: EnvCli -> Parser CardanoTestnetCliOptions
 optsTestnet envCli = CardanoTestnetCliOptions
@@ -28,7 +32,7 @@ optsTestnet envCli = CardanoTestnetCliOptions
 
 pCardanoTestnetCliOptions :: EnvCli -> Parser CardanoTestnetOptions
 pCardanoTestnetCliOptions envCli = CardanoTestnetOptions
-  <$> pNumSpoNodes
+  <$> pTestnetNodeOptions
   <*> pAnyShelleyBasedEra'
   <*> pMaxLovelaceSupply
   <*> OA.option auto
@@ -47,7 +51,7 @@ pCardanoTestnetCliOptions envCli = CardanoTestnetOptions
       )
   <*> OA.option auto
       (   OA.long "num-dreps"
-      <>  OA.help "Number of delegate representatives (DReps) to generate"
+      <>  OA.help "Number of delegate representatives (DReps) to generate. Ignored if a custom Conway genesis file is passed."
       <>  OA.metavar "NUMBER"
       <>  OA.showDefault
       <>  OA.value 3
@@ -67,19 +71,27 @@ pCardanoTestnetCliOptions envCli = CardanoTestnetOptions
     pAnyShelleyBasedEra' =
       pAnyShelleyBasedEra envCli <&> (\(EraInEon x) -> AnyShelleyBasedEra x)
 
-pNumSpoNodes :: Parser [TestnetNodeOptions]
-pNumSpoNodes =
-  -- We don't support passing custom node configurations files on the CLI.
-  -- So we use a default node configuration for all nodes.
-  (`L.replicate` defaultSpoOptions) <$>
-    OA.option auto
-    (   OA.long "num-pool-nodes"
-    <>  OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes."
-    <>  OA.metavar "COUNT"
-    <>  OA.showDefault
-    <>  OA.value 1)
+pTestnetNodeOptions :: Parser TestnetNodeOptions
+pTestnetNodeOptions =
+  asum' [
+      AutomaticNodeOptions . (`L.replicate` defaultSpoOptions) <$>
+        OA.option auto
+        (   OA.long "num-pool-nodes"
+        <>  OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes."
+        <>  OA.metavar "COUNT"
+        <>  OA.showDefault
+        <>  OA.value 1)
+    , UserProvidedNodeOptions
+        <$> strOption ( long "node-config"
+                        <> metavar "FILEPATH"
+                        <> help "Path to the node's configuration file (which is generated otherwise). If you use this option, you should also pass all the genesis files (files pointed to by the fields \"AlonzoGenesisFile\", \"ShelleyGenesisFile\", etc.).")
+    ]
   where
     defaultSpoOptions = SpoNodeOptions []
+    -- \| Because asum is not available GHC 8.10.7's base (4.14.3.0). This can be removed
+    -- when oldest version of GHC we use is >= 9.0 (base >= 4.15)
+    asum' :: (Foldable t, Alternative f) => t (f a) -> f a
+    asum' = foldr (<|>) empty
 
 pGenesisOptions :: Parser GenesisOptions
 pGenesisOptions =
@@ -92,23 +104,26 @@ pGenesisOptions =
     pEpochLength =
       OA.option auto
         (   OA.long "epoch-length"
-        <>  OA.help "Epoch length, in number of slots"
+        -- TODO Check that this flag is not used when a custom Shelley genesis file is passed
+        <>  OA.help "Epoch length, in number of slots. Ignored if a custom Shelley genesis file is passed."
         <>  OA.metavar "SLOTS"
         <>  OA.showDefault
         <>  OA.value (genesisEpochLength def)
         )
     pSlotLength =
       OA.option auto
         (   OA.long "slot-length"
-        <>  OA.help "Slot length"
+        -- TODO Check that this flag is not used when a custom Shelley genesis file is passed
+        <>  OA.help "Slot length. Ignored if a custom Shelley genesis file is passed."
         <>  OA.metavar "SECONDS"
         <>  OA.showDefault
         <>  OA.value (genesisSlotLength def)
         )
     pActiveSlotCoeffs =
       OA.option auto
         (   OA.long "active-slots-coeff"
-        <>  OA.help "Active slots co-efficient"
+        -- TODO Check that this flag is not used when a custom Shelley genesis file is passed
+        <>  OA.help "Active slots coefficient. Ignored if a custom Shelley genesis file is passed."
         <>  OA.metavar "DOUBLE"
         <>  OA.showDefault
         <>  OA.value (genesisActiveSlotsCoeff def)
@@ -129,7 +144,8 @@ pMaxLovelaceSupply :: Parser Word64
 pMaxLovelaceSupply =
   option auto
       (   long "max-lovelace-supply"
-      <>  help "Max lovelace supply that your testnet starts with."
+      -- TODO Check that this flag is not used when a custom Shelley genesis file is passed
+      <>  help "Max lovelace supply that your testnet starts with. Ignored if a custom Shelley genesis file is passed."
       <>  metavar "WORD64"
       <>  showDefault
       <>  value (cardanoMaxSupply def)

cardano-testnet/src/Parsers/Run.hs

@@ -1,5 +1,6 @@
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE ScopedTypeVariables #-}
 
 module Parsers.Run
   ( commands
@@ -8,14 +9,27 @@ module Parsers.Run
   , opts
   ) where
 
+import           Cardano.Api.Ledger (StandardCrypto)
+import           Cardano.Api.Shelley (ShelleyGenesis)
+
 import           Cardano.CLI.Environment
+import           Cardano.Ledger.Alonzo.Genesis (AlonzoGenesis)
+import           Cardano.Ledger.Conway.Genesis (ConwayGenesis)
+import           Cardano.Node.Configuration.POM
+import           Cardano.Node.Types
 
+import qualified Data.Aeson as Aeson
 import           Data.Foldable
+import           Data.Monoid
+import           Data.Yaml (decodeFileThrow)
 import           Options.Applicative
 import qualified Options.Applicative as Opt
+import qualified System.Directory as System
+import           System.FilePath (takeDirectory, (</>))
 
 import           Testnet.Property.Run
 import           Testnet.Start.Cardano
+import           Testnet.Start.Types
 
 import           Parsers.Cardano
 import           Parsers.Help
@@ -52,5 +66,49 @@ runTestnetCmd = \case
 
 
 runCardanoOptions :: CardanoTestnetCliOptions -> IO ()
-runCardanoOptions (CardanoTestnetCliOptions testnetOptions shelleyOptions) =
-  runTestnet testnetOptions $ cardanoTestnetDefault testnetOptions shelleyOptions
+runCardanoOptions (CardanoTestnetCliOptions testnetOptions genesisOptions) =
+    case cardanoNodes testnetOptions of
+      AutomaticNodeOptions _ -> runTestnet testnetOptions $ cardanoTestnetDefault testnetOptions genesisOptions
+      UserProvidedNodeOptions nodeInputConfigFile -> do
+        nodeConfigFile <- readNodeConfigurationFile nodeInputConfigFile
+        let protocolConfig :: NodeProtocolConfiguration =
+              case getLast $ pncProtocolConfig nodeConfigFile of
+                Nothing -> error $ "Genesis files not specified in node configuration file: " <> nodeInputConfigFile
+                Just x -> x
+            adjustedProtocolConfig =
+              -- Make all the files be relative to the location of the config file.
+              adjustFilePaths (takeDirectory nodeInputConfigFile </>) protocolConfig
+            (shelley, alonzo, conway) = getShelleyGenesises adjustedProtocolConfig
+        shelleyGenesis :: UserProvidedData (ShelleyGenesis StandardCrypto) <-
+          genesisFilepathToData $ npcShelleyGenesisFile shelley
+        alonzoGenesis :: UserProvidedData AlonzoGenesis <-
+          genesisFilepathToData $ npcAlonzoGenesisFile alonzo
+        conwayGenesis :: UserProvidedData (ConwayGenesis StandardCrypto) <-
+          genesisFilepathToData $ npcConwayGenesisFile conway
+        runTestnet testnetOptions $ cardanoTestnet
+          testnetOptions
+          genesisOptions
+          shelleyGenesis
+          alonzoGenesis
+          conwayGenesis
+  where
+    getShelleyGenesises (NodeProtocolConfigurationCardano _byron shelley alonzo conway _hardForkConfig _checkPointConfig) =
+      (shelley, alonzo, conway)
+    readNodeConfigurationFile :: FilePath -> IO PartialNodeConfiguration
+    readNodeConfigurationFile file = do
+      errOrNodeConfig <- Aeson.eitherDecodeFileStrict' file
+      case errOrNodeConfig of
+        Left err -> error $ "Error reading node configuration file: " <> err
+        Right nodeConfig -> pure nodeConfig
+    genesisFilepathToData :: Aeson.FromJSON a => GenesisFile -> IO (UserProvidedData a)
+    genesisFilepathToData (GenesisFile filepath) = do
+      exists <- System.doesFileExist filepath
+      if exists
+        then UserProvidedData <$> decodeFileThrow filepath
+        else
+          -- Here we forbid mixing defaulting of genesis files with providing a user node configuration file:
+          --
+          -- Because of this check, either the user passes a node configuration file AND he passes all the genesis files.
+          -- Or the user doesn't pass a node configuration file AND cardano-testnet generates all the genesis files.
+          -- See the discussion here: https://github.com/IntersectMBO/cardano-node/pull/6103#discussion_r1995431437
+          error $ "Genesis file specified in node configuration file does not exist: " <> filepath

cardano-testnet/src/Testnet/Runtime.hs

@@ -142,16 +142,13 @@ startNode tp node ipv4 port _testnetMagic nodeCmd = GHC.withFrozenCallStack $ do
        left MaxSprocketLengthExceededError
 
     let socketAbsPath = H.sprocketSystemName sprocket
+        completeNodeCmd =  nodeCmd ++
+                             [ "--socket-path", H.sprocketArgumentName sprocket
+                             , "--port", show port
+                             , "--host-addr", showIpv4Address ipv4
+                             ]
 
-    nodeProcess
-      <- newExceptT . fmap (first ExecutableRelatedFailure) . try
-           $ procNode $ mconcat
-                         [ nodeCmd
-                         , [ "--socket-path", H.sprocketArgumentName sprocket
-                           , "--port", show port
-                           , "--host-addr", showIpv4Address ipv4
-                           ]
-                         ]
+    nodeProcess <- newExceptT . fmap (first ExecutableRelatedFailure) . try $ procNode completeNodeCmd
 
     -- The port number if it is obtained using 'H.randomPort', it is firstly bound to and then closed. The closing
     -- and release in the operating system is done asynchronously and can be slow. Here we wait until the port