Enhance start up logs (#2486)

<!-- Describe your change here -->

Closes #2483

🔑 Motivation

Improve node observability by introducing structured lifecycle and
sync-status logs.
These logs make startup, hydration, and sync transitions explicit, and
provide better insight into drift before going out of sync.

Overall, this PR makes node startup phases and sync health easier to
trace and reason about, while reducing noise when fully in sync.

> This PR also fixes the smoke tests by allowing enough time (depending
on the actual chain backend used during tests) to observe the node
becoming `InSync` before starting the scenario.

🔄 Changes

**New Logs Introduced**

* `LoadedChainState`
Emitted during hydration with the last known chain point loaded into
`chainStateHistory`.

* `NodeHydrated`
  Emitted once node hydration completes.

* `StartingDecision`
Emitted by the chain layer to report the first chain point used as
prefix when selecting an intersection during chain-sync.

* `ChainBackendStarted`
  Emitted after successfully establishing the chain layer connection.

* `NetworkStarted`
  Emitted after successfully establishing the network layer connection.

* `EnteringMainloop`
  Emitted right before starting the `runHydraNode` main loop.

* `SyncedStatusReport`
  Emitted:
  * On every tick while `CatchingUp`
* On every tick when drift exceeds 80% of the unsynced policy while
`InSync`

---

<!-- Consider each and tick it off one way or the other -->
* [x] CHANGELOG updated or not needed
* [x] Documentation updated or not needed
* [x] Haddocks updated or not needed
* [x] No new TODOs introduced or explained herafter

Author: noonio

hydra-cluster/bench/Bench/EndToEnd.hs

@@ -73,7 +73,7 @@ bench startingNodeId timeoutSeconds workDir dataset = do
         let cardanoKeys = hydraNodeKeys dataset <&> \sk -> (getVerificationKey sk, sk)
         let hydraKeys = generateSigningKey . show <$> [1 .. toInteger (length cardanoKeys)]
         statsTvar <- newLabelledTVarIO "bench-stats" mempty
-        scenarioData <- withCardanoNodeDevnet (contramap FromCardanoNode tracer) workDir $ \_ backend -> do
+        scenarioData <- withCardanoNodeDevnet (contramap FromCardanoNode tracer) workDir $ \blockTime backend -> do
           let nodeSocket' = case Backend.getOptions backend of
                 Direct DirectOptions{nodeSocket} -> nodeSocket
                 _ -> error "Unexpected Blockfrost backend"
@@ -83,8 +83,7 @@ bench startingNodeId timeoutSeconds workDir dataset = do
           hydraScriptsTxId <- publishHydraScriptsAs backend Faucet
           putStrLn $ "Starting hydra cluster in " <> workDir
           let hydraTracer = contramap FromHydraNode tracer
-
-          withHydraCluster hydraTracer workDir nodeSocket' startingNodeId cardanoKeys hydraKeys hydraScriptsTxId 10 $ \clients -> do
+          withHydraCluster hydraTracer blockTime workDir nodeSocket' startingNodeId cardanoKeys hydraKeys hydraScriptsTxId 10 $ \clients -> do
             waitForNodesConnected hydraTracer 20 clients
             scenario hydraTracer backend workDir dataset clients
         systemStats <- readTVarIO statsTvar

hydra-cluster/src/Hydra/Cluster/Scenarios.hs

@@ -281,6 +281,7 @@ getMetrics HydraClient{hydraNodeId, apiHost = Host{hostname}} = do
 withHydraCluster ::
   HasCallStack =>
   Tracer IO HydraNodeLog ->
+  NominalDiffTime ->
   FilePath ->
   SocketPath ->
   -- | First node id
@@ -294,7 +295,7 @@ withHydraCluster ::
   ContestationPeriod ->
   (NonEmpty HydraClient -> IO a) ->
   IO a
-withHydraCluster tracer workDir nodeSocket firstNodeId allKeys hydraKeys hydraScriptsTxId contestationPeriod action = do
+withHydraCluster tracer blockTime workDir nodeSocket firstNodeId allKeys hydraKeys hydraScriptsTxId contestationPeriod action = do
   when (clusterSize == 0) $
     failure "Cannot run a cluster with 0 number of nodes"
   when (length allKeys /= length hydraKeys) $
@@ -332,6 +333,7 @@ withHydraCluster tracer workDir nodeSocket firstNodeId allKeys hydraKeys hydraSc
                 }
       withHydraNode
         tracer
+        blockTime
         chainConfig
         workDir
         nodeId
@@ -438,18 +440,19 @@ prepareHydraNode chainConfig workDir hydraNodeId hydraSKey hydraVKeys allNodeIds
 withPreparedHydraNodeInSync ::
   HasCallStack =>
   Tracer IO HydraNodeLog ->
+  NominalDiffTime ->
   FilePath ->
   Int ->
   RunOptions ->
   (HydraClient -> IO a) ->
   IO a
-withPreparedHydraNodeInSync tracer workDir hydraNodeId runOptions action =
-  withPreparedHydraNode tracer workDir hydraNodeId runOptions action'
+withPreparedHydraNodeInSync tracer blockTime workDir hydraNodeId runOptions action = do
+  let waitTime = blockTime * waitFactor
+  withPreparedHydraNode tracer workDir hydraNodeId runOptions (action' waitTime)
  where
-  action' client = do
-    getHydraBackend >>= \case
-      DirectBackendType -> waitForNodesSynced 5 $ client :| []
-      BlockfrostBackendType -> waitForNodesSynced 10 $ client :| []
+  waitFactor = 5
+  action' waitTime client = do
+    waitForNodesSynced waitTime $ client :| []
     action client
 
 -- | Run a hydra-node with given 'RunOptions'.
@@ -496,6 +499,7 @@ withPreparedHydraNode tracer workDir hydraNodeId runOptions action =
 withHydraNode ::
   HasCallStack =>
   Tracer IO HydraNodeLog ->
+  NominalDiffTime ->
   ChainConfig ->
   FilePath ->
   Int ->
@@ -504,9 +508,9 @@ withHydraNode ::
   [Int] ->
   (HydraClient -> IO a) ->
   IO a
-withHydraNode tracer chainConfig workDir hydraNodeId hydraSKey hydraVKeys allNodeIds action = do
+withHydraNode tracer blockTime chainConfig workDir hydraNodeId hydraSKey hydraVKeys allNodeIds action = do
   opts <- prepareHydraNode chainConfig workDir hydraNodeId hydraSKey hydraVKeys allNodeIds id
-  withPreparedHydraNodeInSync tracer workDir hydraNodeId opts action
+  withPreparedHydraNodeInSync tracer blockTime workDir hydraNodeId opts action
 
 -- | Run a hydra-node with given 'ChainConfig' and using the config from
 -- config and catching up with chain backend/.

hydra-cluster/src/HydraNode.hs

@@ -41,13 +41,13 @@ spec = do
       showLogsOnFailure "ChainObserverSpec" $ \tracer -> do
         withTempDir "hydra-cluster" $ \tmpDir -> do
           -- Start a cardano devnet
-          withCardanoNodeDevnet (contramap FromCardanoNode tracer) tmpDir $ \_ backend -> do
+          withCardanoNodeDevnet (contramap FromCardanoNode tracer) tmpDir $ \blockTime backend -> do
             -- Prepare a hydra-node
             let hydraTracer = contramap FromHydraNode tracer
             hydraScriptsTxId <- publishHydraScriptsAs backend Faucet
             (aliceCardanoVk, _) <- keysFor Alice
             aliceChainConfig <- chainConfigFor Alice tmpDir backend hydraScriptsTxId [] cperiod
-            withHydraNode hydraTracer aliceChainConfig tmpDir 1 aliceSk [] [1] $ \hydraNode -> do
+            withHydraNode hydraTracer blockTime aliceChainConfig tmpDir 1 aliceSk [] [1] $ \hydraNode -> do
               withChainObserver backend $ \observer -> do
                 seedFromFaucet_ backend aliceCardanoVk 100_000_000 (contramap FromFaucet tracer)