diff --git a/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal b/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal index 061513b2b2..b52b784bcd 100644 --- a/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal +++ b/ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal @@ -542,6 +542,7 @@ library unstable-cardano-tools filepath, fs-api ^>=0.3, githash, + lz4, microlens, mtl, network, diff --git a/ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBAnalyser/Analysis.hs b/ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBAnalyser/Analysis.hs index 6dfd65d6b2..615774d800 100644 --- a/ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBAnalyser/Analysis.hs +++ b/ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBAnalyser/Analysis.hs @@ -25,6 +25,8 @@ module Cardano.Tools.DBAnalyser.Analysis ( , runAnalysis ) where +import qualified Codec.Compression.LZ4 as LZ4 + import qualified Cardano.Slotting.Slot as Slotting import qualified Cardano.Tools.DBAnalyser.Analysis.BenchmarkLedgerOps.FileWriting as F import qualified Cardano.Tools.DBAnalyser.Analysis.BenchmarkLedgerOps.SlotDataPoint as DP @@ -38,6 +40,7 @@ import Control.Monad (unless, void, when) import Control.Monad.Except (runExcept) import Control.ResourceRegistry import Control.Tracer (Tracer (..), nullTracer, traceWith) +import qualified Data.ByteString.Lazy as BL import Data.Int (Int64) import Data.List (intercalate) import qualified Data.Map.Strict as Map @@ -174,11 +177,15 @@ data TraceEvent blk = -- * slot number when the block was forged -- * cumulative tx output -- * count tx output - | EbbEvent (HeaderHash blk) (ChainHash blk) Bool + | EbbEvent (HeaderHash blk) (ChainHash blk) Bool Int SizeInBytes SizeInBytes SizeInBytes -- ^ triggered when EBB block has been found, it holds: -- * its hash, -- * hash of previous block -- * flag whether the EBB is known + -- * number of preceding EBBs + -- * header size + -- * its size + -- * its size after LZ4 | CountedBlocksEvent Int -- ^ triggered once during CountBLocks analysis, -- when blocks were counted @@ -233,10 +240,14 @@ instance (HasAnalysis blk, LedgerSupportsProtocol blk) => Show (TraceEvent blk) , "cumulative: " <> show cumulative , "count: " <> show count ] - show (EbbEvent ebb previous known) = intercalate "\t" [ + show (EbbEvent ebb previous known i hsz sz sz2) = intercalate "\t" [ "EBB: " <> show ebb , "Prev: " <> show previous , "Known: " <> show known + , "Index: " <> show i + , "ByteSizeHeader: " <> show (getSizeInBytes hsz) + , "ByteSize: " <> show (getSizeInBytes sz) + , "ByteSizeLZ4: " <> show (getSizeInBytes sz2) ] show (CountedBlocksEvent counted) = "Counted " <> show counted <> " blocks." show (HeaderSizeEvent bn sn hSz bSz) = intercalate "\t" $ [ @@ -355,20 +366,25 @@ showBlockTxsSize AnalysisEnv { db, registry, startFrom, limit, tracer } = do showEBBs :: forall blk. HasAnalysis blk => Analysis blk StartFromPoint showEBBs AnalysisEnv { db, registry, startFrom, limit, tracer } = do - processAll_ db registry GetBlock startFrom limit process + _ <- processAll db registry ((,,) <$> GetBlock <*> GetRawBlock <*> GetRawHeader) startFrom limit (0, BL.empty, BL.empty) process pure Nothing where - process :: blk -> IO () - process blk = + process :: (Int, BL.ByteString, BL.ByteString) -> (blk, BL.ByteString, BL.ByteString) -> IO (Int, BL.ByteString, BL.ByteString) + process (!i, cacc, hacc) (blk, bbytes, hbytes) = case blockIsEBB blk of Just _epoch -> do + let cbytes = maybe undefined BL.fromStrict $ LZ4.compressHC $ BL.toStrict bbytes let known = Map.lookup (blockHash blk) (HasAnalysis.knownEBBs (Proxy @blk)) == Just (blockPrevHash blk) - event = EbbEvent (blockHash blk) (blockPrevHash blk) known + event = EbbEvent (blockHash blk) (blockPrevHash blk) known i (SizeInBytes $ fromIntegral $ BL.length hbytes) (SizeInBytes $ fromIntegral $ BL.length bbytes) (SizeInBytes $ fromIntegral $ BL.length cbytes) traceWith tracer event - _otherwise -> return () -- Skip regular blocks + when (i == 175) $ do + BL.writeFile "EbbsHcConcat.bin" (cacc <> cbytes) + BL.writeFile "EbbHeadersConcat.bin" (hacc <> hbytes) + pure $ (i+1, cacc <> cbytes, hacc <> hbytes) + _otherwise -> return (i, cacc, hacc) -- Skip regular blocks {------------------------------------------------------------------------------- Analysis: store a ledger at specific slot diff --git a/ouroboros-consensus/ouroboros-consensus.cabal b/ouroboros-consensus/ouroboros-consensus.cabal index 27552f9e20..c056d96c5b 100644 --- a/ouroboros-consensus/ouroboros-consensus.cabal +++ b/ouroboros-consensus/ouroboros-consensus.cabal @@ -70,7 +70,6 @@ library exposed-modules: Ouroboros.Consensus.Block Ouroboros.Consensus.Block.Abstract - Ouroboros.Consensus.Block.EBB Ouroboros.Consensus.Block.Forging Ouroboros.Consensus.Block.NestedContent Ouroboros.Consensus.Block.RealPoint diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block.hs index e9cccfdc24..12d9e7517f 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block.hs @@ -2,7 +2,6 @@ module Ouroboros.Consensus.Block (module X) where import Ouroboros.Consensus.Block.Abstract as X -import Ouroboros.Consensus.Block.EBB as X import Ouroboros.Consensus.Block.Forging as X import Ouroboros.Consensus.Block.NestedContent as X import Ouroboros.Consensus.Block.RealPoint as X diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/Abstract.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/Abstract.hs index ff7f160206..f6d2a70bf5 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/Abstract.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/Abstract.hs @@ -19,12 +19,9 @@ module Ouroboros.Consensus.Block.Abstract ( -- * Working with headers , GetHeader (..) , Header - , blockIsEBB - , blockToIsEBB , getBlockHeaderFields , headerHash , headerPoint - , headerToIsEBB -- * Raw hash , ConvertRawHash (..) , decodeRawHash @@ -73,10 +70,8 @@ import qualified Data.ByteString as Strict import Data.ByteString.Short (ShortByteString) import qualified Data.ByteString.Short as Short import Data.Kind (Type) -import Data.Maybe (isJust) import Data.Word (Word32, Word64) import NoThunks.Class (NoThunks) -import Ouroboros.Consensus.Block.EBB import Ouroboros.Network.Block (ChainHash (..), HasHeader (..), HeaderFields (..), HeaderHash, Point, StandardHash, blockHash, blockNo, blockPoint, blockSlot, castHash, @@ -133,19 +128,6 @@ class HasHeader (Header blk) => GetHeader blk where -- header matches that of the block. blockMatchesHeader :: Header blk -> blk -> Bool - -- | When the given header is the header of an Epoch Boundary Block, returns - -- its epoch number. - headerIsEBB :: Header blk -> Maybe EpochNo - -headerToIsEBB :: GetHeader blk => Header blk -> IsEBB -headerToIsEBB = toIsEBB . isJust . headerIsEBB - -blockIsEBB :: GetHeader blk => blk -> Maybe EpochNo -blockIsEBB = headerIsEBB . getHeader - -blockToIsEBB :: GetHeader blk => blk -> IsEBB -blockToIsEBB = headerToIsEBB . getHeader - type instance BlockProtocol (Header blk) = BlockProtocol blk {------------------------------------------------------------------------------- diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/EBB.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/EBB.hs deleted file mode 100644 index 72fb1daa0d..0000000000 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block/EBB.hs +++ /dev/null @@ -1,48 +0,0 @@ -{-# LANGUAGE DeriveAnyClass #-} -{-# LANGUAGE DeriveGeneric #-} - --- | Generic infrastructure for working with EBBs -module Ouroboros.Consensus.Block.EBB ( - IsEBB (..) - , fromIsEBB - , toIsEBB - ) where - -import Codec.Serialise (Serialise (..)) -import GHC.Generics (Generic) -import NoThunks.Class (NoThunks) -import Ouroboros.Consensus.Util.Condense - -{------------------------------------------------------------------------------- - EBBs --------------------------------------------------------------------------------} - --- | Whether a block is an Epoch Boundary Block (EBB) --- --- See "Ouroboros.Storage.ImmutableDB.API" for a discussion of EBBs. Key --- idiosyncracies: --- --- * An EBB carries no unique information. --- --- * An EBB has the same 'BlockNo' as its predecessor. --- --- * EBBs are vestigial. As of Shelley, nodes no longer forge EBBs: they are --- only a legacy/backwards-compatibility concern. -data IsEBB - = IsEBB - | IsNotEBB - deriving (Eq, Show, Generic, NoThunks) - -instance Serialise IsEBB where - encode = encode . fromIsEBB - decode = toIsEBB <$> decode - -instance Condense IsEBB where - condense = show - -toIsEBB :: Bool -> IsEBB -toIsEBB b = if b then IsEBB else IsNotEBB - -fromIsEBB :: IsEBB -> Bool -fromIsEBB IsEBB = True -fromIsEBB IsNotEBB = False diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HardFork/Combinator/Block.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HardFork/Combinator/Block.hs index a61c174f78..29f2e8363e 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HardFork/Combinator/Block.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HardFork/Combinator/Block.hs @@ -68,12 +68,6 @@ instance CanHardFork xs => GetHeader (HardForkBlock xs) where matchesSingle :: GetHeader blk => Product Header I blk -> K Bool blk matchesSingle (Pair hdr (I blk)) = K (blockMatchesHeader hdr blk) - headerIsEBB = - hcollapse - . hcmap proxySingle (K . headerIsEBB) - . getOneEraHeader - . getHardForkHeader - {------------------------------------------------------------------------------- HasHeader -------------------------------------------------------------------------------} @@ -229,9 +223,6 @@ instance CanHardFork xs => BasicEnvelopeValidation (HardForkBlock xs) where case isNonEmpty (Proxy @xs) of ProofNonEmpty p _ -> minimumPossibleSlotNo p - -- TODO: If the block is from a different era as the current tip, we just - -- expect @succ b@. This may not be sufficient: if we ever transition /to/ - -- an era with EBBs, this is not correct. expectedNextBlockNo _ (OneEraTipInfo oldTip) (OneEraTipInfo newBlock) b = case Match.matchNS oldTip newBlock of Right matched -> hcollapse $ hcmap proxySingle aux matched @@ -243,20 +234,6 @@ instance CanHardFork xs => BasicEnvelopeValidation (HardForkBlock xs) where aux (Pair (WrapTipInfo old) (WrapTipInfo new)) = K $ expectedNextBlockNo (Proxy @blk) old new b - -- TODO: If the block is from a different era as the current tip, we just - -- expect @succ s@. This may not be sufficient: if we ever transition /to/ - -- an era with EBBs, this is not correct. - minimumNextSlotNo _ (OneEraTipInfo oldTip) (OneEraTipInfo newBlock) s = - case Match.matchNS oldTip newBlock of - Right matched -> hcollapse $ hcmap proxySingle aux matched - Left _mismatch -> succ s - where - aux :: forall blk. SingleEraBlock blk - => Product WrapTipInfo WrapTipInfo blk - -> K SlotNo blk - aux (Pair (WrapTipInfo old) (WrapTipInfo new)) = K $ - minimumNextSlotNo (Proxy @blk) old new s - {------------------------------------------------------------------------------- Other instances (primarily for the benefit of tests) -------------------------------------------------------------------------------} diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderStateHistory.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderStateHistory.hs index 0fc4be977d..e8956da7f7 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderStateHistory.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderStateHistory.hs @@ -103,10 +103,6 @@ cast (HeaderStateHistory history) = -- -- We also return the oldest 'HeaderStateWithTime' that was rewound, if any. -- --- NOTE: we don't distinguish headers of regular blocks from headers of EBBs. --- Whenever we use \"header\" it can be either. In practice, EBB headers do not --- affect the 'ChainDepState', but they /do/ affect the 'AnnTip'. --- -- PRECONDITION: the point to rewind to must correspond to a header (or -- 'GenesisPoint') that was previously applied to the header state history. -- diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderValidation.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderValidation.hs index cff3545b16..fd90586af2 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderValidation.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HeaderValidation.hs @@ -44,14 +44,10 @@ module Ouroboros.Consensus.HeaderValidation ( -- * Errors , HeaderError (..) , castHeaderError - -- * TipInfoIsEBB - , TipInfoIsEBB (..) -- * Serialization - , decodeAnnTipIsEBB , decodeHeaderState , defaultDecodeAnnTip , defaultEncodeAnnTip - , encodeAnnTipIsEBB , encodeHeaderState -- * Type family instances , Ticked (..) @@ -87,9 +83,7 @@ import qualified Ouroboros.Consensus.Util.CBOR as Util.CBOR -- | Annotated information about the tip of the chain -- -- The annotation is the additional information we need to validate the --- header envelope. Under normal circumstances no additional information is --- required, but for instance for Byron we need to know if the previous header --- was an EBB. +-- header envelope. data AnnTip blk = AnnTip { annTipSlotNo :: !SlotNo , annTipBlockNo :: !BlockNo @@ -275,13 +269,6 @@ class ( HasHeader (Header blk) minimumPossibleSlotNo :: Proxy blk -> SlotNo minimumPossibleSlotNo _ = SlotNo 0 - -- | Minimum next slot number - minimumNextSlotNo :: proxy blk - -> TipInfo blk -- ^ Old tip - -> TipInfo blk -- ^ New block - -> SlotNo -> SlotNo - minimumNextSlotNo _ _ _ = succ - -- | Validate header envelope class ( BasicEnvelopeValidation blk , GetPrevHash blk @@ -311,8 +298,8 @@ validateEnvelope :: forall blk. (ValidateEnvelope blk) validateEnvelope cfg ledgerView oldTip hdr = do unless (actualBlockNo == expectedBlockNo) $ throwError $ UnexpectedBlockNo expectedBlockNo actualBlockNo - unless (actualSlotNo >= expectedSlotNo) $ - throwError $ UnexpectedSlotNo expectedSlotNo actualSlotNo + unless (actualSlotNo >= minimumSlotNo) $ + throwError $ UnexpectedSlotNo minimumSlotNo actualSlotNo unless (checkPrevHash' (annTipHash <$> oldTip) actualPrevHash) $ throwError $ UnexpectedPrevHash (annTipHash <$> oldTip) actualPrevHash validateIfCheckpoint (topLevelConfigCheckpoints cfg) hdr @@ -334,13 +321,11 @@ validateEnvelope cfg ledgerView oldTip hdr = do actualBlockNo = blockNo hdr actualPrevHash = headerPrevHash hdr - expectedSlotNo :: SlotNo -- Lower bound only - expectedSlotNo = + minimumSlotNo :: SlotNo + minimumSlotNo = case oldTip of Origin -> minimumPossibleSlotNo p - NotOrigin tip -> minimumNextSlotNo p (annTipInfo tip) - (getTipInfo hdr) - (annTipSlotNo tip) + NotOrigin tip -> succ $ annTipSlotNo tip expectedBlockNo :: BlockNo expectedBlockNo = @@ -486,19 +471,6 @@ revalidateHeader cfg ledgerView hdr st = (untickedHeaderStateTip st) hdr -{------------------------------------------------------------------------------- - TipInfoIsEBB --------------------------------------------------------------------------------} - --- | Reusable strict data type for 'TipInfo' in case the 'TipInfo' should --- contain 'IsEBB' in addition to the 'HeaderHash'. -data TipInfoIsEBB blk = TipInfoIsEBB !(HeaderHash blk) !IsEBB - deriving (Generic) - -deriving instance StandardHash blk => Eq (TipInfoIsEBB blk) -deriving instance StandardHash blk => Show (TipInfoIsEBB blk) -deriving instance StandardHash blk => NoThunks (TipInfoIsEBB blk) - {------------------------------------------------------------------------------- Serialisation -------------------------------------------------------------------------------} @@ -523,36 +495,6 @@ defaultDecodeAnnTip decodeHash = do annTipBlockNo <- decode return AnnTip{..} -encodeAnnTipIsEBB :: TipInfo blk ~ TipInfoIsEBB blk - => (HeaderHash blk -> Encoding) - -> (AnnTip blk -> Encoding) -encodeAnnTipIsEBB encodeHash AnnTip{..} = mconcat [ - encodeListLen 4 - , encode annTipSlotNo - , encodeHash hash - , encode annTipBlockNo - , encodeInfo isEBB - ] - where - TipInfoIsEBB hash isEBB = annTipInfo - - encodeInfo :: IsEBB -> Encoding - encodeInfo = encode - -decodeAnnTipIsEBB :: TipInfo blk ~ TipInfoIsEBB blk - => (forall s. Decoder s (HeaderHash blk)) - -> (forall s. Decoder s (AnnTip blk)) -decodeAnnTipIsEBB decodeHash = do - enforceSize "AnnTip" 4 - annTipSlotNo <- decode - hash <- decodeHash - annTipBlockNo <- decode - isEBB <- decodeInfo - return AnnTip{annTipInfo = TipInfoIsEBB hash isEBB, ..} - where - decodeInfo :: forall s. Decoder s IsEBB - decodeInfo = decode - encodeHeaderState :: (ChainDepState (BlockProtocol blk) -> Encoding) -> (AnnTip blk -> Encoding) -> (HeaderState blk -> Encoding) diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Basics.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Basics.hs index b7b6eca434..06a69681aa 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Basics.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Basics.hs @@ -146,7 +146,7 @@ class ( -- Requirements on the ledger state itself -- that as soon as a certain slot was reached, /any/ block would be invalid. -- -- PRECONDITION: The slot number must be strictly greater than the slot at - -- the tip of the ledger (except for EBBs, obviously..). + -- the tip of the ledger. -- -- NOTE: 'applyChainTickLedgerResult' should /not/ change the tip of the -- underlying ledger state, which should still refer to the most recent diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Dual.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Dual.hs index a5dc517634..48fd7110c7 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Dual.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Dual.hs @@ -101,7 +101,7 @@ import Ouroboros.Consensus.Util.Condense -- -- The auxiliary block is optional; this can be used if some " main " blocks -- should have no effect on the auxiliary ledger state at all. The motivating --- example is EBBs: if the main blocks are real Byron blocks, and the auxiliary +-- example is EBBs (before they were removed from the codebase): if the main blocks are real Byron blocks, and the auxiliary -- blocks are Byron spec blocks, then regular Byron blocks correspond to Byron -- spec blocks, but EBBs don't correspond to a spec block at all and should -- leave the Byron spec ledger state unchanged. @@ -144,9 +144,6 @@ instance Bridge m a => GetHeader (DualBlock m a) where blockMatchesHeader hdr = blockMatchesHeader (dualHeaderMain hdr) . dualBlockMain - -- We can look at the concrete header to see if this is an EBB - headerIsEBB = headerIsEBB . dualHeaderMain - type DualHeader m a = Header (DualBlock m a) deriving instance Show (Header m) => Show (DualHeader m a) @@ -462,7 +459,6 @@ instance Bridge m a => BasicEnvelopeValidation (DualBlock m a) where expectedFirstBlockNo _ = expectedFirstBlockNo (Proxy @m) expectedNextBlockNo _ = expectedNextBlockNo (Proxy @m) minimumPossibleSlotNo _ = minimumPossibleSlotNo (Proxy @m) - minimumNextSlotNo _ = minimumNextSlotNo (Proxy @m) instance Bridge m a => ValidateEnvelope (DualBlock m a) where type OtherHeaderEnvelopeError (DualBlock m a) = OtherHeaderEnvelopeError m diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/MiniProtocol/BlockFetch/ClientInterface.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/MiniProtocol/BlockFetch/ClientInterface.hs index 7748ab7ac7..53b540bb9a 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/MiniProtocol/BlockFetch/ClientInterface.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/MiniProtocol/BlockFetch/ClientInterface.hs @@ -308,12 +308,10 @@ mkBlockFetchConsensusInterface -- precondition holds. -- -- 4. Our chain's anchor can only move forward. We can detect this by - -- looking at the block/slot numbers of the anchors: When the anchor - -- advances, either the block number increases (usual case), or the - -- block number stays the same, but the slot number increases (EBB - -- case). + -- looking at the block numbers of the anchors: when the anchor + -- advances, the block number increases. -- - | anchorBlockNoAndSlot cand < anchorBlockNoAndSlot ours -- (4) + | anchorBlockNo cand < anchorBlockNo ours -- (4) = case (AF.null ours, AF.null cand) of -- Both are non-empty, the precondition trivially holds. (False, False) -> preferAnchoredCandidate bcfg ours cand @@ -333,13 +331,10 @@ mkBlockFetchConsensusInterface | otherwise = preferAnchoredCandidate bcfg ours cand where - anchorBlockNoAndSlot :: + anchorBlockNo :: AnchoredFragment (Header blk) -> (WithOrigin BlockNo, WithOrigin SlotNo) - anchorBlockNoAndSlot frag = - (AF.anchorToBlockNo a, AF.anchorToSlotNo a) - where - a = AF.anchor frag + anchorBlockNo = AF.anchorToBlockNo . AF.anchor compareCandidateChains :: AnchoredFragment (Header blk) -> AnchoredFragment (Header blk) diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Node/InitStorage.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Node/InitStorage.hs index 73f979bb92..da5508d42b 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Node/InitStorage.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Node/InitStorage.hs @@ -29,7 +29,7 @@ class NodeInitStorage blk where -- production. -- -- This function can be used to, for example, create the genesis EBB in case - -- the chain(DB) is empty. + -- the chain(DB) is empty (as was the case before EBBs were removed). -- -- We only provide a limited interface to the chain DB. This is primarily -- useful for the definition of combinators (which may need to turn a diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT.hs index 8859ba501f..b73aa57edc 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT.hs @@ -38,8 +38,7 @@ module Ouroboros.Consensus.Protocol.PBFT ( , PBftMockCrypto , PBftMockVerKeyHash (..) , PBftValidateView (..) - , pbftValidateBoundary - , pbftValidateRegular + , pbftValidate -- * CannotForge , PBftCannotForge (..) , pbftCheckCanForge @@ -97,47 +96,31 @@ instance (PBftCrypto c, Typeable toSign) => NoThunks (PBftFields c toSign) -- | Part of the header that we validate data PBftValidateView c = - -- | Regular block + -- | Blocks are signed, and so we need to validate them. -- - -- Regular blocks are signed, and so we need to validate them. -- We also need to know the slot number of the block forall signed. Signable (PBftDSIGN c) signed - => PBftValidateRegular + => PBftValidate (PBftFields c signed) signed (ContextDSIGN (PBftDSIGN c)) - -- | Boundary block (EBB) - -- - -- EBBs are not signed and they do not affect the consensus state. - | PBftValidateBoundary - -- | Convenience constructor for 'PBftValidateView' for regular blocks -pbftValidateRegular :: ( SignedHeader hdr +pbftValidate :: ( SignedHeader hdr , Signable (PBftDSIGN c) (Signed hdr) ) => ContextDSIGN (PBftDSIGN c) -> (hdr -> PBftFields c (Signed hdr)) -> (hdr -> PBftValidateView c) -pbftValidateRegular contextDSIGN getFields hdr = - PBftValidateRegular +pbftValidate contextDSIGN getFields hdr = + PBftValidate (getFields hdr) (headerSigned hdr) contextDSIGN --- | Convenience constructor for 'PBftValidateView' for boundary blocks -pbftValidateBoundary :: hdr -> PBftValidateView c -pbftValidateBoundary _hdr = PBftValidateBoundary - -- | Part of the header required for chain selection --- --- EBBs share a block number with regular blocks, and so for chain selection --- we need to know if a block is an EBB or not (because a chain ending on an --- EBB with a particular block number is longer than a chain on a regular --- block with that same block number). data PBftSelectView = PBftSelectView { pbftSelectViewBlockNo :: BlockNo - , pbftSelectViewIsEBB :: IsEBB } deriving stock (Show, Eq, Generic) deriving anyclass (NoThunks) @@ -145,27 +128,14 @@ data PBftSelectView = PBftSelectView { mkPBftSelectView :: GetHeader blk => Header blk -> PBftSelectView mkPBftSelectView hdr = PBftSelectView { - pbftSelectViewBlockNo = blockNo hdr - , pbftSelectViewIsEBB = headerToIsEBB hdr + pbftSelectViewBlockNo = blockNo hdr } instance Ord PBftSelectView where - compare (PBftSelectView lBlockNo lIsEBB) (PBftSelectView rBlockNo rIsEBB) = - mconcat [ + compare (PBftSelectView lBlockNo) (PBftSelectView rBlockNo) = -- Prefer the highest block number, as it is a proxy for chain length lBlockNo `compare` rBlockNo - -- If the block numbers are the same, check if one of them is an EBB. - -- An EBB has the same block number as the block before it, so the - -- chain ending with an EBB is actually longer than the one ending - -- with a regular block. - , score lIsEBB `compare` score rIsEBB - ] - where - score :: IsEBB -> Int - score IsEBB = 1 - score IsNotEBB = 0 - {------------------------------------------------------------------------------- Block forging -------------------------------------------------------------------------------} @@ -325,9 +295,7 @@ instance PBftCrypto c => ConsensusProtocol (PBft c) where slot (TickedPBftState (PBftLedgerView dms) state) = case toValidate of - PBftValidateBoundary -> - return state - PBftValidateRegular PBftFields{..} signed contextDSIGN -> do + PBftValidate PBftFields{..} signed contextDSIGN -> do -- Check that the issuer signature verifies, and that it's a delegate of a -- genesis key, and that genesis key hasn't voted too many times. case verifySignedDSIGN @@ -338,10 +306,7 @@ instance PBftCrypto c => ConsensusProtocol (PBft c) where Right () -> return () Left err -> throwError $ PBftInvalidSignature (Text.pack err) - -- FIXME confirm that non-strict inequality is ok in general. - -- It's here because EBBs have the same slot as the first block of their - -- epoch. - unless (NotOrigin slot >= S.lastSignedSlot state) + unless (NotOrigin slot > S.lastSignedSlot state) $ throwError PBftInvalidSlot case Bimap.lookupR (hashVerKey pbftIssuer) dms of @@ -362,8 +327,7 @@ instance PBftCrypto c => ConsensusProtocol (PBft c) where slot (TickedPBftState (PBftLedgerView dms) state) = case toValidate of - PBftValidateBoundary -> state - PBftValidateRegular PBftFields{pbftIssuer} _ _ -> + PBftValidate PBftFields{pbftIssuer} _ _ -> case Bimap.lookupR (hashVerKey pbftIssuer) dms of Nothing -> error $ show $ PBftNotGenesisDelegate diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT/State.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT/State.hs index 28d8848935..2ea32be6d7 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT/State.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Protocol/PBFT/State.hs @@ -181,8 +181,6 @@ countSignedBy PBftState{..} gk = Map.findWithDefault 0 gk counts -- -- Returns 'Origin' if there are no signatures in the window (this will happen -- exactly at genesis only). --- --- Unaffected by EBBs, since they're not signed. lastSignedSlot :: PBftState c -> WithOrigin SlotNo lastSignedSlot PBftState{..} = case inWindow of diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/API.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/API.hs index dfc656e4c3..b24e40f674 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/API.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/API.hs @@ -57,7 +57,6 @@ module Ouroboros.Consensus.Storage.ChainDB.API ( , traverseFollower -- * Recovery , ChainDbFailure (..) - , IsEBB (..) -- * Exceptions , ChainDbError (..) -- * Genesis diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/ChainSel.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/ChainSel.hs index 2a25cfcdd1..1cb3b28aaa 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/ChainSel.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/ChainSel.hs @@ -352,7 +352,7 @@ chainSelSync cdb@CDB {..} (ChainSelAddBlock BlockToAdd { blockToAdd = b, .. }) = -- We follow the steps from section "## Adding a block" in ChainDB.md if - | olderThanK hdr isEBB immBlockNo -> do + | olderThanK hdr immBlockNo -> do lift $ traceWith addBlockTracer $ IgnoreBlockOlderThanK (blockRealPoint b) lift $ deliverWrittenToDisk False @@ -372,7 +372,7 @@ chainSelSync cdb@CDB {..} (ChainSelAddBlock BlockToAdd { blockToAdd = b, .. }) = -- The remaining cases | otherwise -> do - let traceEv = AddedBlockToVolatileDB (blockRealPoint b) (blockNo b) isEBB + let traceEv = AddedBlockToVolatileDB (blockRealPoint b) (blockNo b) lift $ encloseWith (traceEv >$< addBlockTracer) $ VolatileDB.putBlock cdbVolatileDB b lift $ deliverWrittenToDisk True @@ -388,9 +388,6 @@ chainSelSync cdb@CDB {..} (ChainSelAddBlock BlockToAdd { blockToAdd = b, .. }) = hdr :: Header blk hdr = getHeader b - isEBB :: IsEBB - isEBB = headerToIsEBB hdr - -- | Fill in the 'TMVar' for the 'varBlockWrittenToDisk' of the block's -- 'AddBlockPromise' with the given 'Bool'. deliverWrittenToDisk :: Bool -> m () @@ -407,34 +404,15 @@ chainSelSync cdb@CDB {..} (ChainSelAddBlock BlockToAdd { blockToAdd = b, .. }) = -- because it is too old, i.e., we wouldn't be able to switch to a chain -- containing the corresponding block because its block number is more than -- @k@ blocks or exactly @k@ blocks back. --- --- Special case: the header corresponds to an EBB which has the same block --- number as the block @k@ blocks back (the most recent \"immutable\" block). --- As EBBs share their block number with the block before them, the EBB is not --- too old in that case and can be adopted as part of our chain. --- --- This special case can occur, for example, when the VolatileDB is empty --- (because of corruption). The \"immutable\" block is then also the tip of --- the chain. If we then try to add the EBB after it, it will have the same --- block number, so we must allow it. olderThanK :: HasHeader (Header blk) => Header blk -- ^ Header of the block to add - -> IsEBB - -- ^ Whether the block is an EBB or not -> WithOrigin BlockNo -- ^ The block number of the most recent \"immutable\" block, i.e., the -- block @k@ blocks back. -> Bool -olderThanK hdr isEBB immBlockNo - | NotOrigin bNo == immBlockNo - , isEBB == IsEBB - = False - | otherwise - = NotOrigin bNo <= immBlockNo - where - bNo = blockNo hdr +olderThanK hdr immBlockNo = NotOrigin (blockNo hdr) <= immBlockNo -- | When we switch to a new selected chain, we are either extending the current -- chain by adding blocks on top or we are switching to a fork. @@ -538,7 +516,7 @@ chainSelectionForBlock cdb@CDB{..} blockCache hdr punish = electric $ do if -- The chain might have grown since we added the block such that the -- block is older than @k@. - | olderThanK hdr isEBB immBlockNo -> do + | olderThanK hdr immBlockNo -> do traceWith addBlockTracer $ IgnoreBlockOlderThanK p -- The block is invalid @@ -579,9 +557,6 @@ chainSelectionForBlock cdb@CDB{..} blockCache hdr punish = electric $ do p :: RealPoint blk p = headerRealPoint hdr - isEBB :: IsEBB - isEBB = headerToIsEBB hdr - addBlockTracer :: Tracer m (TraceAddBlockEvent blk) addBlockTracer = TraceAddBlockEvent >$< cdbTracer @@ -640,18 +615,7 @@ chainSelectionForBlock cdb@CDB{..} blockCache hdr punish = electric $ do $ fmap Diff.extend $ NE.toList candidates -- All candidates are longer than the current chain, so they will be - -- preferred over it, /unless/ the block we just added is an EBB, - -- which has the same 'BlockNo' as the block before it, so when - -- using the 'BlockNo' as the proxy for the length (note that some - -- protocols might do it differently), the candidate with the EBB - -- appended will not be preferred over the current chain. - -- - -- The consequence of this is that when adding an EBB, it will not - -- be selected by chain selection and thus not appended to the chain - -- until the block after it is added, which will again result in a - -- candidate preferred over the current chain. In this case, the - -- candidate will be a two-block (the EBB and the new block) - -- extension of the current chain. + -- preferred over it. case chainDiffs of Nothing -> return () Just chainDiffs' -> @@ -1183,17 +1147,7 @@ ledgerValidateCandidate chainSelEnv chainDiff@(ChainDiff rollback suffix) = $ if addedPt == pt then InvalidBlockPunishment.BlockItself else InvalidBlockPunishment.BlockPrefix - case realPointSlot pt `compare` realPointSlot addedPt of - LT -> m - GT -> pure () - EQ -> when (lastValid /= realPointToPoint addedPt) m - -- If pt and addedPt have the same slot, and addedPt is the tip of - -- the ledger that pt was validated against, then addedPt is an - -- EBB and is valid. - -- - -- Otherwise, either pt == addedPt or addedPt comes after pt, so - -- we should punish. (Tacit assumption made here: it's impossible - -- three blocks in a row have the same slot.) + when (realPointSlot pt <= realPointSlot addedPt) m return $ ValidatedDiff.new chainDiff' ledger' diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Follower.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Follower.hs index 8f6366e3f9..3b43876ba2 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Follower.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Follower.hs @@ -300,7 +300,6 @@ instructionHelper registry varFollower chainType blockComponent fromMaybeSTM CDB GetRawHeader -> return $ rawHdr GetHash -> return $ headerHash hdr GetSlot -> return $ blockSlot hdr - GetIsEBB -> return $ headerToIsEBB hdr GetBlockSize -> getBlockComponent GetBlockSize -- We could look up the header size in the index of the VolatileDB, -- but getting the serialisation is cheap because we keep the @@ -360,8 +359,11 @@ instructionHelper registry varFollower chainType blockComponent fromMaybeSTM CDB -- The ImmutableDB somehow rolled back GT -> error "follower streamed beyond tip of the ImmutableDB" + EQ | pt /= pointAtImmutableDBTip + -> error "follower streamed different tip of the ImmutableDB" + -- The tip is still the same, so switch to the in-memory chain - EQ | pt == pointAtImmutableDBTip + | otherwise -> do trace $ FollowerSwitchToMem pt slotNoAtImmutableDBTip fupdate <- atomically $ fromMaybeSTM $ do @@ -374,15 +376,9 @@ instructionHelper registry varFollower chainType blockComponent fromMaybeSTM CDB -- block component. headerUpdateToBlockComponentUpdate fupdate - -- Two possibilities: - -- - -- 1. (EQ): the tip changed, but the slot number is the same. This - -- is only possible when an EBB was at the tip and the regular - -- block in the same slot was appended to the ImmutableDB. - -- - -- 2. (LT): the tip of the ImmutableDB has progressed since we - -- opened the iterator. - _ -> do + -- The tip of the ImmutableDB has progressed since we opened the + -- iterator. + LT -> do trace $ FollowerNewImmIterator pt slotNoAtImmutableDBTip immIt' <- ImmutableDB.streamAfterKnownPoint cdbImmutableDB registry ((,) <$> getPoint <*> blockComponent) pt diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Iterator.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Iterator.hs index cbd08b2769..47c60782ab 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Iterator.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Iterator.hs @@ -18,8 +18,8 @@ module Ouroboros.Consensus.Storage.ChainDB.Impl.Iterator ( ) where import Control.Monad (unless, when) -import Control.Monad.Except (ExceptT (..), catchError, runExceptT, - throwError, withExceptT) +import Control.Monad.Except (ExceptT (..), runExceptT, throwError, + withExceptT) import Control.Monad.Trans.Class (lift) import Control.ResourceRegistry (ResourceRegistry) import Control.Tracer @@ -257,7 +257,7 @@ newIterator itEnv@IteratorEnv{..} getItEnv registry blockComponent from to = do => ExceptT (UnknownRange blk) m (Iterator m blk b) start = lift (atomically (ImmutableDB.getTip itImmutableDB)) >>= \case Origin -> findPathInVolatileDB - NotOrigin ImmutableDB.Tip { tipSlotNo, tipHash, tipIsEBB } -> + NotOrigin ImmutableDB.Tip { tipSlotNo, tipHash } -> case realPointSlot endPoint `compare` tipSlotNo of -- The end point is < the tip of the ImmutableDB LT -> streamFromImmutableDB @@ -266,60 +266,13 @@ newIterator itEnv@IteratorEnv{..} getItEnv registry blockComponent from to = do -- The end point == the tip of the ImmutableDB -> streamFromImmutableDB - -- The end point /= the tip of the ImmutableDB. - -- - -- The end point can be a regular block or EBB. So can the tip of - -- the ImmutableDB. We distinguish the following for cases where - -- each block and EBB has the same slot number, and a block or - -- EBB /not/ on the current chain is indicated with a '. - -- - -- 1. ImmutableDB: .. :> EBB :> B - -- end point: B' - -- desired outcome: ForkTooOld - -- - -- 2. ImmutableDB: .. :> EBB :> B - -- end point: EBB' - -- desired outcome: ForkTooOld - -- - -- 3. ImmutableDB: .. :> EBB :> B - -- end point: EBB - -- desired outcome: stream from ImmutableDB - -- - -- 4. ImmutableDB: .. :> EBB - -- end point: B - -- desired outcome: find path in the VolatileDB - -- - -- 5. ImmutableDB: .. :> EBB - -- end point: B' - -- desired outcome: ForkTooOld - -- - -- 6. ImmutableDB: .. :> EBB - -- end point: EBB' - -- desired outcome: ForkTooOld - -- - -- We don't know upfront whether the given end point refers to a - -- block or EBB nor whether it is part of the current chain or - -- not. This means we don't know yet with which case we are - -- dealing. The only thing we know for sure, is whether the - -- ImmutableDB tip ends with a regular block (1-3) or an EBB - -- (4-6). - - | IsNotEBB <- tipIsEBB -- Cases 1-3 - -> streamFromImmutableDB `catchError` - -- We also use 'streamFromImmutableDB' to check whether the - -- block or EBB is in the ImmutableDB. If that's not the case, - -- 'streamFromImmutableDB' will return 'MissingBlock'. Instead - -- of returning that, we should return 'ForkTooOld', which is - -- more correct. - const (throwError $ ForkTooOld from) - | otherwise -- Cases 4-6 - -> findPathInVolatileDB + | otherwise + -> throwError $ ForkTooOld from -- The end point is > the tip of the ImmutableDB GT -> findPathInVolatileDB - -- | PRECONDITION: the upper bound >= the tip of the ImmutableDB. - -- Greater or /equal/, because of EBBs :( + -- | PRECONDITION: the upper bound > the tip of the ImmutableDB. findPathInVolatileDB :: HasCallStack => ExceptT (UnknownRange blk) m (Iterator m blk b) findPathInVolatileDB = do diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Paths.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Paths.hs index c98f94d042..7b7deca476 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Paths.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Paths.hs @@ -191,7 +191,7 @@ computePath lookupBlockInfo from to = | StreamFromInclusive _ <- from -> Just $ PartiallyInVolatileDB hash acc - volPath' ::> (flds, _isEBB) + volPath' ::> flds | StreamFromExclusive GenesisPoint <- from -> go (addToAcc flds acc) volPath' | StreamFromExclusive (BlockPoint _ hash') <- from @@ -274,27 +274,14 @@ data ReversePath blk = -- Since block numbers are consecutive, we subtract 1 from the block -- number of the last block to obtain the block number corresponding to -- this hash. - -- - -- EBBs share their block number with their predecessor: - -- - -- > block: regular block 1 | EBB | regular block 2 - -- > block number: X | X | X + 1 - -- - -- So when the hash refers to regular block 1, we see that the successor - -- block is an EBB and use its block number without subtracting 1. - -- - -- Edge case: if there are two or more consecutive EBBs, we might - -- predict the wrong block number, but there are no consecutive EBBs in - -- practice, they are one epoch apart. | StoppedAt (HeaderHash blk) BlockNo -- | Snoc: the block with the given 'HeaderFields' is in the VolatileDB. - -- We also track whether it is an EBB or not. -- -- NOTE: we are intentionally lazy in the spine, as constructing the -- path requires lookups in the VolatileDB's in-memory indices, which -- are logarithmic in the size of the index. - | (ReversePath blk) ::> (HeaderFields blk, IsEBB) + | (ReversePath blk) ::> (HeaderFields blk) -- | Lazily compute the 'ReversePath' that starts (i.e., ends) with the given -- 'HeaderHash'. @@ -310,43 +297,31 @@ computeReversePath computeReversePath lookupBlockInfo endHash = case lookupBlockInfo endHash of Nothing -> Nothing - Just blockInfo@VolatileDB.BlockInfo { biBlockNo, biIsEBB, biPrevHash } -> Just $ - go biPrevHash biBlockNo biIsEBB ::> (headerFieldsFromBlockInfo blockInfo, biIsEBB) + Just blockInfo@VolatileDB.BlockInfo { biBlockNo, biPrevHash } -> Just $ + go biPrevHash biBlockNo ::> (headerFieldsFromBlockInfo blockInfo) where go :: ChainHash blk -- ^ The predecessor of the last block added to the path. Not -- necessarily in the VolatileDB. -> BlockNo -- ^ The block number of the last block - -> IsEBB -- ^ Whether the last block is an EBB or not -> ReversePath blk - go predecessor lastBlockNo lastIsEBB = case predecessor of + go predecessor lastBlockNo = case predecessor of GenesisHash -> StoppedAtGenesis BlockHash prevHash -> case lookupBlockInfo prevHash of Nothing -> - StoppedAt prevHash (prevBlockNo lastBlockNo lastIsEBB) - Just blockInfo@VolatileDB.BlockInfo { biBlockNo, biIsEBB, biPrevHash } -> - go biPrevHash biBlockNo biIsEBB ::> (headerFieldsFromBlockInfo blockInfo, biIsEBB) + StoppedAt prevHash (prevBlockNo lastBlockNo) + Just blockInfo@VolatileDB.BlockInfo { biBlockNo, biPrevHash } -> + go biPrevHash biBlockNo ::> headerFieldsFromBlockInfo blockInfo -- | Predict the block number of the missing predecessor. -- - -- PRECONDITION: the block number and 'IsEBB' correspond to a block that + -- PRECONDITION: the block number corresponds to a block that -- has a predecessor. - -- - -- For regular blocks, this is just block number - 1, EBBs are special of - -- course: they share their block number with their predecessor: - -- - -- > block: regular block 1 | EBB | regular block 2 - -- > block number: X | X | X + 1 - -- - -- Edge case: if there are two or more consecutive EBBs, we might predict - -- the wrong block number, but there are no consecutive EBBs in practice - -- (nor in the tests), they are one epoch apart. - prevBlockNo :: BlockNo -> IsEBB -> BlockNo - prevBlockNo bno isEBB = case (bno, isEBB) of - (0, IsNotEBB) -> error "precondition violated" - (_, IsNotEBB) -> bno - 1 - (_, IsEBB) -> bno + prevBlockNo :: BlockNo -> BlockNo + prevBlockNo bno = case bno of + 0 -> error "precondition violated" + _ -> bno - 1 {------------------------------------------------------------------------------- Reachability @@ -398,7 +373,6 @@ isReachable lookupBlockInfo = \chain b -> -- thus the same slot. Both the chain and the path are ordered by slots, -- so we compare the slots and drop the largest one until we have a match -- in slot, then we check hashes. If those don't match, we drop both. - -- Note: EBBs complicate things, see 'ebbAwareCompare'. go :: AnchoredFragment (Header blk) -- ^ Prefix of the current chain @@ -417,7 +391,7 @@ isReachable lookupBlockInfo = \chain b -> | otherwise -> Nothing - (AF.Empty anchor, path' ::> (flds, _)) + (AF.Empty anchor, path' ::> flds) | AF.anchorToHeaderFields (AF.castAnchor anchor) == NotOrigin flds -> Just (ChainDiff rollback (AF.fromOldestFirst (AF.castAnchor anchor) acc)) | AF.anchorToBlockNo anchor > NotOrigin (headerFieldBlockNo flds) @@ -442,13 +416,13 @@ isReachable lookupBlockInfo = \chain b -> | otherwise -> Nothing - (chain' AF.:> hdr, path' ::> (flds, ptIsEBB)) -> - case hdr `ebbAwareCompare` (headerFieldBlockNo flds, ptIsEBB) of + (chain' AF.:> hdr, path' ::> flds) -> + case hdr `cmp` headerFieldBlockNo flds of -- Drop from the path LT -> go chain path' rollback (flds:acc) -- Drop from the current chain fragment GT -> go chain' path (rollback + 1) acc - -- Same slot and value for 'IsEBB' + -- Check hash EQ | blockHash hdr == headerFieldHash flds , let anchor = AF.castAnchor (AF.anchorFromBlock hdr) -- Found a match @@ -457,14 +431,5 @@ isReachable lookupBlockInfo = \chain b -> | otherwise -> go chain' path' (rollback + 1) (flds:acc) - -- | EBBs have the same block number as their predecessor, which means - -- that in case we have an EBB and a regular block with the same slot, the - -- EBB comes /after/ the regular block. - ebbAwareCompare :: Header blk -> (BlockNo, IsEBB) -> Ordering - ebbAwareCompare hdr (ptBlockNo, ptIsEBB) = - compare (blockNo hdr) ptBlockNo `mappend` - case (headerToIsEBB hdr, ptIsEBB) of - (IsEBB, IsNotEBB) -> GT - (IsNotEBB, IsEBB) -> LT - (IsEBB, IsEBB) -> EQ - (IsNotEBB, IsNotEBB) -> EQ + cmp :: Header blk -> BlockNo -> Ordering + cmp hdr ptBlockNo = compare (blockNo hdr) ptBlockNo diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Types.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Types.hs index 03e880f16a..1e6696f766 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Types.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ChainDB/Impl/Types.hs @@ -740,7 +740,7 @@ data TraceAddBlockEvent blk = | PoppedReprocessLoEBlocksFromQueue -- | A block was added to the Volatile DB - | AddedBlockToVolatileDB (RealPoint blk) BlockNo IsEBB Enclosing + | AddedBlockToVolatileDB (RealPoint blk) BlockNo Enclosing -- | The block fits onto the current chain, we'll try to use it to extend -- our chain. diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/Common.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/Common.hs index 6da304ff11..498699b6f2 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/Common.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/Common.hs @@ -123,15 +123,9 @@ newtype StreamTo blk = validBounds :: StandardHash blk => StreamFrom blk -> StreamTo blk -> Bool validBounds from (StreamToInclusive (RealPoint sto hto)) = case from of - StreamFromExclusive GenesisPoint -> True - -- EBBs spoil the fun again: when 'StreamFromExclusive' refers to an EBB - -- in slot X and 'StreamToInclusive' to the regular block in the same slot - -- X, the bound is still valid. Without EBBs, we would have @sfrom < sto@. - -- - -- We /can/ rule out streaming exclusively from the block to the same - -- block. - StreamFromExclusive (BlockPoint sfrom hfrom) -> hfrom /= hto && sfrom <= sto - StreamFromInclusive (RealPoint sfrom _) -> sfrom <= sto + StreamFromExclusive GenesisPoint -> True + StreamFromExclusive (BlockPoint sfrom hfrom) -> hfrom /= hto && sfrom < sto + StreamFromInclusive (RealPoint sfrom _) -> sfrom < sto {------------------------------------------------------------------------------- BlockComponent @@ -152,7 +146,6 @@ data BlockComponent blk a where GetRawHeader :: BlockComponent blk ByteString GetHash :: BlockComponent blk (HeaderHash blk) GetSlot :: BlockComponent blk SlotNo - GetIsEBB :: BlockComponent blk IsEBB -- TODO: use `SizeInBytes` rather than Word32 GetBlockSize :: BlockComponent blk SizeInBytes GetHeaderSize :: BlockComponent blk Word16 diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/API.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/API.hs index b35eb013e8..4a9332bb11 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/API.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/API.hs @@ -88,23 +88,19 @@ type SecondaryOffset = Word32 -- | API for the 'ImmutableDB'. -- -- The 'ImmutableDB' stores blocks in 'SlotNo's. Nevertheless, lookups use --- 'RealPoint', primarily because Epoch Boundary Blocks (EBBs) have the same --- 'SlotNo' as the regular block after them (unless that slot is empty), so that --- we have to use the hash of the block to distinguish the two (hence --- 'RealPoint'). But also to avoid reading the wrong block, i.e., when we expect +-- 'RealPoint', to avoid reading the wrong block, i.e., when we expect -- a block with a different hash. -- -- The database is append-only, so you cannot append a block to a slot in the -- past. You can, however, skip slots, e.g., append to slot 0 and then to slot -- 5, but afterwards, you can no longer append to slots 1-4. You can only store --- at most one block in each slot, except for EBBs, which are stored separately, --- at the start of each epoch/chunk. +-- at most one block in each slot. -- -- The block stored in a slot can be queried with 'getBlockComponent'. Block -- components can also be streamed using 'Iterator's, see 'stream'. -- -- The 'Tip' of the database can be queried with 'getTip'. This tip will --- always point to a filled slot or an EBB that is present. +-- always point to a filled slot. -- -- The database can be explicitly closed, but can also be automatically closed -- in case of an 'UnexpectedFailure'. @@ -118,16 +114,14 @@ data ImmutableDB m blk = ImmutableDB { -- | Return the tip of the database. -- - -- The tip of the database will never point to an unfilled slot or missing - -- EBB. + -- The tip of the database will never point to an unfilled slot. -- -- Throws a 'ClosedDBError' if the database is closed. , getTip_ :: HasCallStack => STM m (WithOrigin (Tip blk)) -- | Get the block component of the block with the given 'Point'. -- - -- The hash of the point is used to distinguish a potential EBB from the - -- regular block in the same slot. + -- The hash of the point is used only for error checking. -- -- Returns a 'MissingBlockError' if no block was stored with the given -- 'Point', either because the slot was empty or because the block stored @@ -183,7 +177,7 @@ data Iterator m blk b = Iterator { -- | Steps an 'Iterator' yielding an 'IteratorResult'. -- -- After returning the block component as an 'IteratorResult', the - -- iterator is advanced to the next non-empty slot or non-empty EBB. + -- iterator is advanced to the next non-empty slot. -- -- Throws a 'ClosedDBError' if the database is closed. -- @@ -253,7 +247,6 @@ emptyIterator = Iterator { -- | Information about the tip of the ImmutableDB. data Tip blk = Tip { tipSlotNo :: !SlotNo - , tipIsEBB :: !IsEBB , tipBlockNo :: !BlockNo , tipHash :: !(HeaderHash blk) } @@ -281,7 +274,6 @@ tipToAnchor = \case headerToTip :: GetHeader blk => Header blk -> Tip blk headerToTip hdr = Tip { tipSlotNo = blockSlot hdr - , tipIsEBB = headerToIsEBB hdr , tipBlockNo = blockNo hdr , tipHash = blockHash hdr } @@ -289,7 +281,7 @@ headerToTip hdr = Tip { blockToTip :: GetHeader blk => blk -> Tip blk blockToTip = headerToTip . getHeader --- | newtype with an 'Ord' instance that only uses 'tipSlotNo' and 'tipIsEBB' +-- | newtype with an 'Ord' instance that only uses 'tipSlotNo' -- and ignores the other fields. newtype CompareTip blk = CompareTip { getCompareTip :: Tip blk } @@ -297,16 +289,7 @@ instance Eq (CompareTip blk) where a == b = compare a b == EQ instance Ord (CompareTip blk) where - compare = mconcat [ - compare `on` tipSlotNo . getCompareTip - , compareIsEBB `on` tipIsEBB . getCompareTip - ] - where - -- When a block and an EBB share a slot number, the EBB is "older". - compareIsEBB :: IsEBB -> IsEBB -> Ordering - compareIsEBB IsEBB IsNotEBB = LT - compareIsEBB IsNotEBB IsEBB = GT - compareIsEBB _ _ = EQ + compare = compare `on` (tipSlotNo . getCompareTip) {------------------------------------------------------------------------------- Errors @@ -416,7 +399,7 @@ data MissingBlock blk (Maybe (StrictSeq SecondaryOffset)) -- ^ Which offsets are known if we are looking at the current (probably cached) chunk - -- | The block and/or EBB in the slot of the given point have a different + -- | The block in the slot of the given point have a different -- hash. We return the 'HeaderHash' for each block we found with the -- corresponding slot number. | WrongHash (RealPoint blk) (NonEmpty (HeaderHash blk)) diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks.hs index d05a94d39e..162e005186 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks.hs @@ -3,7 +3,7 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks (module X) where import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal as X (ChunkInfo (..), ChunkNo, ChunkSize (..), - chunkInfoSupportsEBBs, chunksBetween, compareRelativeSlot, + chunksBetween, compareRelativeSlot, countChunks, firstChunkNo, getChunkSize, mkRelativeSlot, nextChunkNo, prevChunkNo, simpleChunkInfo, singleChunkInfo) import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Layout as X diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Internal.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Internal.hs index 15b29b51d4..22f81b45c6 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Internal.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Internal.hs @@ -8,7 +8,6 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal ( ChunkInfo (..) - , chunkInfoSupportsEBBs , simpleChunkInfo , singleChunkInfo -- * Chunk number @@ -20,8 +19,6 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal ( , firstChunkNo , nextChunkNo , prevChunkNo - , unsafeChunkNoToEpochNo - , unsafeEpochNoToChunkNo -- * Chunk size , ChunkSize (..) , getChunkSize @@ -33,7 +30,6 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal ( , mkRelativeSlot -- * Assertions , ChunkAssertionFailure - , assertChunkCanContainEBB , assertSameChunk , assertWithinBounds ) where @@ -55,11 +51,6 @@ import Ouroboros.Consensus.Util.RedundantConstraints -- data ChunkInfo = -- | A single, uniform, chunk size - -- - -- If EBBs are present, the chunk size must line up precisely with the - -- epoch size (that is, the number of regular blocks in the chunk must equal - -- the number of regular blocks in an epoch). - -- UniformChunkSize !ChunkSize deriving stock (Show, Generic) deriving anyclass (NoThunks) @@ -70,7 +61,7 @@ data ChunkInfo = -- 'ChunkSize': the translation from 'EpochSize' to 'ChunkSize' (number of -- available entries in a chunk) should not be done by client code. simpleChunkInfo :: EpochSize -> ChunkInfo -simpleChunkInfo (EpochSize sz) = UniformChunkSize (ChunkSize True sz) +simpleChunkInfo (EpochSize sz) = UniformChunkSize (ChunkSize sz) -- | 'ChunkInfo' for a single 'ChunkSize' -- @@ -78,28 +69,16 @@ simpleChunkInfo (EpochSize sz) = UniformChunkSize (ChunkSize True sz) singleChunkInfo :: ChunkSize -> ChunkInfo singleChunkInfo = UniformChunkSize --- | Can we store EBBs in the chunks described by this 'ChunkInfo'? --- --- This is only used for tests. This API will need to change (and the tests will --- become more complicated) once we support non-uniform 'ChunkInfo'. -chunkInfoSupportsEBBs :: ChunkInfo -> Bool -chunkInfoSupportsEBBs (UniformChunkSize chunkSize) = - chunkCanContainEBB chunkSize - {------------------------------------------------------------------------------- Queries -------------------------------------------------------------------------------} -- | Size of a chunk -- --- The total number of slots available in a chunk is equal to 'numRegularBlocks' --- if @not@ 'chunkCanContainEBB', and 'numRegularBlocks' @+ 1@ otherwise. +-- The total number of slots available in a chunk is equal to 'numBlocks' data ChunkSize = ChunkSize { - -- | Does this chunk also accomodate an EBB? - chunkCanContainEBB :: !Bool - - -- | The number of regular blocks in this chunk - , numRegularBlocks :: !Word64 + -- | The number of blocks in this chunk + numBlocks :: !Word64 } deriving stock (Show, Generic) deriving anyclass (NoThunks) @@ -146,21 +125,6 @@ chunksBetween :: ChunkNo -> ChunkNo -> [ChunkNo] chunksBetween (ChunkNo a) (ChunkNo b) = map ChunkNo $ if a >= b then [a .. b] else [b .. a] --- | Translate 'EpochNo' to 'ChunkNo' --- --- This should /ONLY/ be used to translate the 'EpochNo' of an EBB, since the --- invariant says EBBs can only exist in the first period of the DB, where the --- chunk size must equal the epoch size. See 'ChunkInfo' for details. -unsafeEpochNoToChunkNo :: EpochNo -> ChunkNo -unsafeEpochNoToChunkNo (EpochNo n) = ChunkNo n - --- | Translate 'ChunkNo' to 'EpochNo' --- --- This should /ONLY/ be used for chunks that contain EBBs. --- See 'unsafeEpochNoToChunkNo' and 'ChunkInfo' for details. -unsafeChunkNoToEpochNo :: ChunkNo -> EpochNo -unsafeChunkNoToEpochNo (ChunkNo n) = EpochNo n - getChunkSize :: ChunkInfo -> ChunkNo -> ChunkSize getChunkSize chunkInfo _chunk = case chunkInfo of @@ -184,8 +148,7 @@ data RelativeSlot = RelativeSlot { -- | The size of the chunk that this slot is in -- - -- We record this for bounds checking as well as to be able to answer - -- questions such as 'relativeSlotIsEBB'. + -- We record this for bounds checking , relativeSlotChunkSize :: !ChunkSize -- | The index within the chunk @@ -196,9 +159,7 @@ data RelativeSlot = RelativeSlot { -- | Maximum relative index within a chunk maxRelativeIndex :: ChunkSize -> Word64 -maxRelativeIndex ChunkSize{..} - | chunkCanContainEBB = numRegularBlocks - | otherwise = numRegularBlocks - 1 +maxRelativeIndex ChunkSize{..} = numBlocks - 1 -- | Smart constructor for 'RelativeSlot' mkRelativeSlot :: HasCallStack => ChunkInfo -> ChunkNo -> Word64 -> RelativeSlot @@ -245,7 +206,6 @@ assertRelativeSlotInChunk chunk relSlot = data ChunkAssertionFailure = NotSameChunk ChunkNo ChunkNo PrettyCallStack | NotWithinBounds Word64 ChunkSize PrettyCallStack - | ChunkCannotContainEBBs ChunkNo PrettyCallStack deriving (Show) instance Exception ChunkAssertionFailure @@ -271,14 +231,3 @@ assertWithinBounds _ _ = id #endif where _ = keepRedundantConstraint (Proxy @HasCallStack) - -assertChunkCanContainEBB :: HasCallStack => ChunkNo -> ChunkSize -> a -> a -#if ENABLE_ASSERTIONS -assertChunkCanContainEBB chunk size - | chunkCanContainEBB size = id - | otherwise = throw $ ChunkCannotContainEBBs chunk prettyCallStack -#else -assertChunkCanContainEBB _ _ = id -#endif - where - _ = keepRedundantConstraint (Proxy @HasCallStack) diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Layout.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Layout.hs index bf0b72d59c..e23ef47516 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Layout.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Chunks/Layout.hs @@ -14,11 +14,10 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Layout ( -- * Relative slots NextRelativeSlot (..) - , firstBlockOrEBB + , firstBlock , maxRelativeSlot , nextRelativeSlot - , nthBlockOrEBB - , relativeSlotIsEBB + , nthBlock , unsafeNextRelativeSlot -- ** Opaque , RelativeSlot @@ -28,30 +27,20 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Layout ( , ChunkSlot (..) , pattern ChunkSlot -- ** Translation /to/ 'ChunkSlot' - , chunkSlotForBlockOrEBB - , chunkSlotForBoundaryBlock - , chunkSlotForRegularBlock + , chunkSlotForSlot , chunkSlotForRelativeSlot , chunkSlotForTip , chunkSlotForUnknownBlock -- ** Translation /from/ 'ChunkSlot' - , chunkSlotToBlockOrEBB , chunkSlotToSlot - -- ** Support for EBBs - , slotMightBeEBB - , slotNoOfBlockOrEBB - , slotNoOfEBB ) where -import Control.Monad import GHC.Generics (Generic) import GHC.Stack import NoThunks.Class (NoThunks) import Ouroboros.Consensus.Block import Ouroboros.Consensus.Storage.ImmutableDB.API (Tip (..)) import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal -import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Types - (BlockOrEBB (..)) {------------------------------------------------------------------------------- Relative slots @@ -64,27 +53,14 @@ maxRelativeSlot ci chunk = where size = getChunkSize ci chunk --- | Is this relative slot reserved for an EBB? -relativeSlotIsEBB :: RelativeSlot -> IsEBB -relativeSlotIsEBB RelativeSlot{..} - | relativeSlotIndex == 0 - , chunkCanContainEBB relativeSlotChunkSize - = IsEBB - | otherwise - = IsNotEBB - -- | The @n@'th relative slot for an arbitrary block --- --- NOTE: Offset @0@ refers to an EBB only if the 'ChunkSize' supports it. -nthBlockOrEBB :: (HasCallStack, Integral a) - => ChunkInfo -> ChunkNo -> a -> RelativeSlot -nthBlockOrEBB ci chunk = mkRelativeSlot ci chunk . fromIntegral +nthBlock :: (HasCallStack, Integral a) + => ChunkInfo -> ChunkNo -> a -> RelativeSlot +nthBlock ci chunk = mkRelativeSlot ci chunk . fromIntegral -- | The first relative slot --- --- NOTE: This refers to an EBB only if the 'ChunkSize' supports it. -firstBlockOrEBB :: ChunkInfo -> ChunkNo -> RelativeSlot -firstBlockOrEBB ci chunk = mkRelativeSlot ci chunk 0 +firstBlock :: ChunkInfo -> ChunkNo -> RelativeSlot +firstBlock ci chunk = mkRelativeSlot ci chunk 0 -- | Result of 'nextRelativeSlot' data NextRelativeSlot = @@ -119,7 +95,7 @@ unsafeNextRelativeSlot s@RelativeSlot{..} = chunkIndexOfSlot :: ChunkInfo -> SlotNo -> ChunkNo chunkIndexOfSlot (UniformChunkSize ChunkSize{..}) (SlotNo slot) = ChunkNo $ - slot `div` numRegularBlocks + slot `div` numBlocks {------------------------------------------------------------------------------- Slot within an epoch @@ -155,64 +131,30 @@ instance Show ChunkSlot where -------------------------------------------------------------------------------} -- | Chunk slot for an unknown block --- --- This returns /two/ 'ChunkSlot's: one in case the block could be an EBB, --- and one in case the block is a regular block. In addition, it also returns --- the 'ChunkNo' that both of these 'ChunkSlot's must necessarily share. -chunkSlotForUnknownBlock :: HasCallStack - => ChunkInfo +chunkSlotForUnknownBlock :: ChunkInfo -> SlotNo - -> (ChunkNo, Maybe ChunkSlot, ChunkSlot) + -> (ChunkNo, ChunkSlot) chunkSlotForUnknownBlock ci slot = ( - (case mIfBoundary of - Nothing -> id - Just ifBoundary -> assertSameChunk (chunkIndex ifBoundary) - (chunkIndex ifRegular)) $ - chunkIndex ifRegular - , mIfBoundary - , ifRegular + chunkIndex cslot + , cslot ) where - ifRegular = chunkSlotForRegularBlock ci slot - mIfBoundary = chunkSlotForBoundaryBlock ci <$> slotMightBeEBB ci slot + cslot = chunkSlotForSlot ci slot --- | Chunk slot for a regular block (i.e., not an EBB) -chunkSlotForRegularBlock :: ChunkInfo -> SlotNo -> ChunkSlot -chunkSlotForRegularBlock (UniformChunkSize sz@ChunkSize{..}) (SlotNo slot) = +-- | Chunk slot for a slot +chunkSlotForSlot :: ChunkInfo -> SlotNo -> ChunkSlot +chunkSlotForSlot (UniformChunkSize sz@ChunkSize{..}) (SlotNo slot) = UnsafeChunkSlot { chunkIndex = ChunkNo chunk - , chunkRelative = RelativeSlot (ChunkNo chunk) sz $ - if chunkCanContainEBB - then withinChunk + 1 - else withinChunk + , chunkRelative = RelativeSlot (ChunkNo chunk) sz $ withinChunk } where - (chunk, withinChunk) = slot `divMod` numRegularBlocks - --- | Chunk slot for EBB -chunkSlotForBoundaryBlock :: HasCallStack => ChunkInfo -> EpochNo -> ChunkSlot -chunkSlotForBoundaryBlock ci epoch = - assertChunkCanContainEBB chunk size $ - UnsafeChunkSlot chunk $ firstBlockOrEBB ci chunk - where - chunk = unsafeEpochNoToChunkNo epoch - size = getChunkSize ci chunk - --- | Chunk slot for 'BlockOrEBB' -chunkSlotForBlockOrEBB :: ChunkInfo -> BlockOrEBB -> ChunkSlot -chunkSlotForBlockOrEBB ci = \case - Block slot -> chunkSlotForRegularBlock ci slot - EBB epoch -> chunkSlotForBoundaryBlock ci epoch + (chunk, withinChunk) = slot `divMod` numBlocks -- | Chunk slot for 'Tip' chunkSlotForTip :: ChunkInfo -> Tip blk -> ChunkSlot -chunkSlotForTip ci Tip { tipSlotNo, tipIsEBB } = case tipIsEBB of - IsNotEBB -> chunkSlotForRegularBlock ci tipSlotNo - IsEBB -> assertChunkCanContainEBB chunkIndex relativeSlotChunkSize $ - UnsafeChunkSlot chunkIndex $ firstBlockOrEBB ci chunkIndex - where - UnsafeChunkSlot{..} = chunkSlotForRegularBlock ci tipSlotNo - RelativeSlot{..} = chunkRelative +chunkSlotForTip ci Tip { tipSlotNo } = + chunkSlotForSlot ci tipSlotNo chunkSlotForRelativeSlot :: ChunkNo -> RelativeSlot -> ChunkSlot chunkSlotForRelativeSlot chunk relSlot = @@ -221,49 +163,12 @@ chunkSlotForRelativeSlot chunk relSlot = {------------------------------------------------------------------------------- Translation /from/ 'ChunkSlot' - - Reminder: - - * EBB shares its slot number with its successor - * EBB shares its block number with its predecessor -------------------------------------------------------------------------------} -- | From relative to absolute slot --- --- This can be used for EBBs and regular blocks, since they don't share a --- relative slot. chunkSlotToSlot :: ChunkInfo -> ChunkSlot -> SlotNo chunkSlotToSlot (UniformChunkSize ChunkSize{..}) UnsafeChunkSlot{..} = SlotNo $ - chunk * numRegularBlocks - + case (chunkCanContainEBB, relativeSlotIndex) of - (_ , 0) -> 0 - (True , n) -> n - 1 - (False, n) -> n + chunk * numBlocks + relativeSlotIndex where ChunkNo chunk = chunkIndex RelativeSlot{..} = chunkRelative - -chunkSlotToBlockOrEBB :: ChunkInfo -> ChunkSlot -> BlockOrEBB -chunkSlotToBlockOrEBB chunkInfo chunkSlot@(ChunkSlot chunk relSlot) = - case relativeSlotIsEBB relSlot of - IsEBB -> EBB $ unsafeChunkNoToEpochNo chunk - IsNotEBB -> Block $ chunkSlotToSlot chunkInfo chunkSlot - -{------------------------------------------------------------------------------- - Support for EBBs --------------------------------------------------------------------------------} - -slotNoOfEBB :: HasCallStack => ChunkInfo -> EpochNo -> SlotNo -slotNoOfEBB ci = chunkSlotToSlot ci . chunkSlotForBoundaryBlock ci - -slotMightBeEBB :: ChunkInfo -> SlotNo -> Maybe EpochNo -slotMightBeEBB ci slot = do - guard $ chunkCanContainEBB relativeSlotChunkSize && relativeSlotIndex == 1 - return $ unsafeChunkNoToEpochNo chunkIndex - where - UnsafeChunkSlot{..} = chunkSlotForRegularBlock ci slot - RelativeSlot{..} = chunkRelative - -slotNoOfBlockOrEBB :: ChunkInfo -> BlockOrEBB -> SlotNo -slotNoOfBlockOrEBB _ (Block slot) = slot -slotNoOfBlockOrEBB ci (EBB epoch) = slotNoOfEBB ci epoch diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl.hs index d4b4716b84..d8146ef160 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl.hs @@ -13,8 +13,7 @@ -- = Internal format -- -- The API of the ImmutableDB uses 'SlotNo' to indicate a location in the --- chain\/immutable database. To distinguish EBBs from regular blocks, the hash --- is used (together they form a 'RealPoint'). The contents of the database are +-- chain\/immutable database. The contents of the database are -- not stored in one big file that is appended to in eternity, but a separate -- file is created for each 'ChunkNo'. -- @@ -33,9 +32,7 @@ -- > │ │ │ │ │ │ │ │ │ │ │ -- > └───┴───┴───┴───┴───┘ └───┴───┴───┴───┘ -- > 'RelativeSlot': 0 1 2 3 4 0 1 2 3 --- > 'SlotNo': EBB 0 1 2 3 EBB 4 5 6 --- --- Not all chunks can contain EBBs; see 'ChunkInfo' for details. +-- > 'SlotNo': 0 1 2 3 4 5 6 7 8 -- -- = Errors -- @@ -76,7 +73,7 @@ -- nothing is stored for empty slots. -- -- * A \"secondary index file\" that stores information about each block: its --- hash, the slot number or epoch number in case of an EBB, a checksum of +-- hash, the slot number, a checksum of -- the block, the offset of the block in the chunk file, and more. This -- index is sparse to save space. -- @@ -202,8 +199,7 @@ data Internal m blk = Internal { -- -- Throws a 'ClosedDBError' if the database is closed. deleteAfter_ :: HasCallStack => WithOrigin (Tip blk) -> m () - -- | Get the hash of the block in the given slot. If the slot contains both - -- an EBB and a non-EBB, return the hash of the non-EBB. + -- | Get the hash of the block in the given slot. , getHashForSlot_ :: HasCallStack => SlotNo -> m (Maybe (HeaderHash blk)) } @@ -364,12 +360,11 @@ deleteAfterImpl dbEnv@ImmutableDBEnv { tracer, chunkInfo } newTip = primaryIndex <- Primary.load (Proxy @blk) hasFS chunk Primary.truncateToSlotFS hasFS chunk relSlot let lastSecondaryOffset = Primary.offsetOfSlot primaryIndex relSlot - isEBB = relativeSlotIsEBB relSlot -- Retrieve the needed info from the secondary index file and then -- truncate it. (entry :: Secondary.Entry blk, blockSize) <- - Secondary.readEntry hasFS chunk isEBB lastSecondaryOffset + Secondary.readEntry hasFS chunk lastSecondaryOffset Secondary.truncateToEntry (Proxy @blk) hasFS chunk lastSecondaryOffset -- Truncate the chunk file. @@ -397,8 +392,7 @@ getHashForSlotImpl dbEnv slot = readOffset offset = lift $ Index.readOffset index chunk (chunkRelative offset) - (chunk, mIfBoundary, ifRegular) = - chunkSlotForUnknownBlock chunkInfo slot + (chunk, chunkSlot) = chunkSlotForUnknownBlock chunkInfo slot -- Check that the slot is not beyond the tip. case currentTip of @@ -407,18 +401,13 @@ getHashForSlotImpl dbEnv slot = -> pure () _ -> exitEarly - -- Primary index: test whether the slot contains a non-EBB, or an EBB as a - -- fallback. - (offset, isEBB) <- readOffset ifRegular >>= \case - (Just offset, _) -> pure (offset, IsNotEBB) - (Nothing, _) -> case mIfBoundary of - Nothing -> exitEarly - Just ifBoundary -> readOffset ifBoundary >>= \case - (Just offset, _) -> pure (offset, IsEBB) - (Nothing, _) -> exitEarly + -- Read offset from primary index. + offset <- readOffset chunkSlot >>= \case + (Just offset, _) -> pure offset + (Nothing, _) -> exitEarly -- Read hash from secondary index. - (entry, _) <- lift $ Index.readEntry index chunk isEBB offset + (entry, _) <- lift $ Index.readEntry index chunk offset pure $ Secondary.headerHash entry where ImmutableDBEnv { chunkInfo } = dbEnv @@ -539,9 +528,9 @@ appendBlockImpl dbEnv blk = -- If we had to start a new chunk, we start with slot 0. Note that -- in this case the 'currentTip' will refer to something in a -- chunk before 'currentChunk'. - then firstBlockOrEBB chunkInfo chunk + then firstBlock chunkInfo chunk else case currentTip of - Origin -> firstBlockOrEBB chunkInfo firstChunkNo + Origin -> firstBlock chunkInfo firstChunkNo -- Invariant: the currently open chunk is never full NotOrigin tip -> unsafeNextRelativeSlot . chunkRelative $ chunkSlotForTip chunkInfo tip @@ -560,7 +549,7 @@ appendBlockImpl dbEnv blk = , headerSize = HeaderSize headerSize , checksum = crc , headerHash = tipHash blockTip - , blockOrEBB = blockOrEBB + , entrySlot = blockSlot blk } entrySize <- fromIntegral <$> @@ -589,15 +578,7 @@ appendBlockImpl dbEnv blk = where ImmutableDBEnv { chunkInfo, codecConfig } = dbEnv - newBlockIsEBB :: Maybe EpochNo - newBlockIsEBB = blockIsEBB blk - - blockOrEBB :: BlockOrEBB - blockOrEBB = case newBlockIsEBB of - Just epochNo -> EBB epochNo - Nothing -> Block (blockSlot blk) - - ChunkSlot chunk relSlot = chunkSlotForBlockOrEBB chunkInfo blockOrEBB + ChunkSlot chunk relSlot = chunkSlotForSlot chunkInfo (blockSlot blk) blockTip :: Tip blk blockTip = blockToTip blk @@ -624,13 +605,13 @@ startNewChunk hasFS index chunkInfo tipChunk = do let nextFreeRelSlot :: NextRelativeSlot nextFreeRelSlot = case currentTip of Origin -> - NextRelativeSlot $ firstBlockOrEBB chunkInfo firstChunkNo + NextRelativeSlot $ firstBlock chunkInfo firstChunkNo NotOrigin tip -> if tipChunk == currentChunk then let ChunkSlot _ relSlot = chunkSlotForTip chunkInfo tip in nextRelativeSlot relSlot else - NextRelativeSlot $ firstBlockOrEBB chunkInfo currentChunk + NextRelativeSlot $ firstBlock chunkInfo currentChunk let backfillOffsets = Primary.backfillChunk chunkInfo diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index.hs index 65b6156121..7e8237fee3 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index.hs @@ -25,7 +25,7 @@ import Data.Typeable (Typeable) import Data.Word (Word64) import GHC.Stack (HasCallStack) import NoThunks.Class (OnlyCheckWhnfNamed (..)) -import Ouroboros.Consensus.Block (ConvertRawHash, IsEBB, StandardHash) +import Ouroboros.Consensus.Block (ConvertRawHash, StandardHash) import Ouroboros.Consensus.Storage.ImmutableDB.Chunks import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index.Cache (CacheConfig (..)) @@ -80,7 +80,7 @@ data Index m blk h = Index , readEntries :: forall t. (HasCallStack, Traversable t) => ChunkNo - -> t (IsEBB, SecondaryOffset) + -> t SecondaryOffset -> m (t (Secondary.Entry blk, BlockSize)) -- | See 'Secondary.readAllEntries' @@ -90,7 +90,6 @@ data Index m blk h = Index -> ChunkNo -> (Secondary.Entry blk -> Bool) -> Word64 - -> IsEBB -> m [WithBlockSize (Secondary.Entry blk)] -- | See 'Secondary.appendEntry' @@ -134,11 +133,10 @@ readEntry :: Functor m => Index m blk h -> ChunkNo - -> IsEBB -> SecondaryOffset -> m (Secondary.Entry blk, BlockSize) -readEntry index chunk isEBB slotOffset = runIdentity <$> - readEntries index chunk (Identity (isEBB, slotOffset)) +readEntry index chunk slotOffset = runIdentity <$> + readEntries index chunk (Identity slotOffset) {------------------------------------------------------------------------------ File-backed index diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Cache.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Cache.hs index ae55a1e18b..e82bd3c297 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Cache.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Cache.hs @@ -54,12 +54,11 @@ import qualified Data.Vector as Vector import Data.Void (Void) import Data.Word (Word32, Word64) import GHC.Generics (Generic) -import Ouroboros.Consensus.Block (ConvertRawHash, IsEBB (..), +import Ouroboros.Consensus.Block (ConvertRawHash, StandardHash) import Ouroboros.Consensus.Storage.ImmutableDB.API (UnexpectedFailure (..), throwUnexpectedFailure) import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal -import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Layout import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index.Primary (PrimaryIndex, SecondaryOffset) import qualified Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index.Primary as Primary @@ -471,20 +470,9 @@ readPrimaryIndex :: -> HasFS m h -> ChunkInfo -> ChunkNo - -> m (PrimaryIndex, IsEBB) - -- ^ The primary index and whether it starts with an EBB or not -readPrimaryIndex pb hasFS chunkInfo chunk = do - primaryIndex <- Primary.load pb hasFS chunk - let firstIsEBB - | Primary.containsSlot primaryIndex firstRelativeSlot - , Primary.isFilledSlot primaryIndex firstRelativeSlot - = relativeSlotIsEBB firstRelativeSlot - | otherwise - = IsNotEBB - return (primaryIndex, firstIsEBB) - where - firstRelativeSlot :: RelativeSlot - firstRelativeSlot = firstBlockOrEBB chunkInfo chunk + -> m PrimaryIndex +readPrimaryIndex pb hasFS _chunkInfo chunk = do + Primary.load pb hasFS chunk readSecondaryIndex :: ( HasCallStack @@ -495,12 +483,15 @@ readSecondaryIndex :: ) => HasFS m h -> ChunkNo - -> IsEBB -> m [Entry blk] -readSecondaryIndex hasFS@HasFS { hGetSize } chunk firstIsEBB = do +readSecondaryIndex hasFS@HasFS { hGetSize } chunk = do !chunkFileSize <- withFile hasFS chunkFile ReadMode hGetSize - Secondary.readAllEntries hasFS secondaryOffset - chunk stopCondition chunkFileSize firstIsEBB + Secondary.readAllEntries + hasFS + secondaryOffset + chunk + stopCondition + chunkFileSize where chunkFile = fsPathChunkFile chunk -- Read from the start @@ -525,9 +516,8 @@ loadCurrentChunkInfo hasFS chunkInfo chunk = do -- index file will also exist chunkExists <- doesFileExist hasFS primaryIndexFile if chunkExists then do - (primaryIndex, firstIsEBB) <- - readPrimaryIndex (Proxy @blk) hasFS chunkInfo chunk - entries <- readSecondaryIndex hasFS chunk firstIsEBB + primaryIndex <- readPrimaryIndex (Proxy @blk) hasFS chunkInfo chunk + entries <- readSecondaryIndex hasFS chunk return CurrentChunkInfo { currentChunkNo = chunk , currentChunkOffsets = @@ -553,8 +543,8 @@ loadPastChunkInfo :: -> ChunkNo -> m (PastChunkInfo blk) loadPastChunkInfo hasFS chunkInfo chunk = do - (primaryIndex, firstIsEBB) <- readPrimaryIndex (Proxy @blk) hasFS chunkInfo chunk - entries <- readSecondaryIndex hasFS chunk firstIsEBB + primaryIndex <- readPrimaryIndex (Proxy @blk) hasFS chunkInfo chunk + entries <- readSecondaryIndex hasFS chunk return PastChunkInfo { pastChunkOffsets = primaryIndex , pastChunkEntries = Vector.fromList $ forceElemsToWHNF entries @@ -792,17 +782,17 @@ readEntries :: ) => CacheEnv m blk h -> ChunkNo - -> t (IsEBB, SecondaryOffset) + -> t SecondaryOffset -> m (t (Secondary.Entry blk, BlockSize)) readEntries cacheEnv chunk toRead = getChunkInfo cacheEnv chunk >>= \case Left CurrentChunkInfo { currentChunkEntries } -> - forM toRead $ \(_isEBB, secondaryOffset) -> + forM toRead $ \secondaryOffset -> case currentChunkEntries Seq.!? indexForOffset secondaryOffset of Just (WithBlockSize size entry) -> return (entry, BlockSize size) Nothing -> noEntry secondaryOffset Right PastChunkInfo { pastChunkEntries } -> - forM toRead $ \(_isEBB, secondaryOffset) -> + forM toRead $ \secondaryOffset -> case pastChunkEntries Vector.!? indexForOffset secondaryOffset of Just (WithBlockSize size entry) -> return (entry, BlockSize size) Nothing -> noEntry secondaryOffset @@ -835,10 +825,9 @@ readAllEntries :: -> ChunkNo -> (Secondary.Entry blk -> Bool) -> Word64 - -> IsEBB -> m [WithBlockSize (Secondary.Entry blk)] readAllEntries cacheEnv secondaryOffset chunk stopCondition - _chunkFileSize _firstIsEBB = + _chunkFileSize = getChunkInfo cacheEnv chunk <&> \case Left CurrentChunkInfo { currentChunkEntries } -> takeUntil (stopCondition . withoutBlockSize) $ diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Primary.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Primary.hs index c97fcac64f..fcd6c3f070 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Primary.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Primary.hs @@ -230,7 +230,7 @@ readFirstFilledSlot :: readFirstFilledSlot pb hasFS@HasFS { hSeek, hGetSome } chunkInfo chunk = withFile hasFS primaryIndexFile ReadMode $ \pHnd -> do hSeek pHnd AbsoluteSeek skip - go pHnd $ NextRelativeSlot (firstBlockOrEBB chunkInfo chunk) + go pHnd $ NextRelativeSlot (firstBlock chunkInfo chunk) where primaryIndexFile = fsPathPrimaryIndexFile chunk @@ -239,7 +239,7 @@ readFirstFilledSlot pb hasFS@HasFS { hSeek, hGetSome } chunkInfo chunk = + fromIntegral secondaryOffsetSize -- | Read offset per offset until we find a non-zero one. In the - -- Byron-era, the first slot is always filled with an EBB, so we only need + -- Byron-era, the first slot is almost always filled, so we'd only need -- to read one 4-byte offset. In the Shelley era, approximately one in ten -- slots is filled, so on average we need to read 5 4-byte offsets. The OS -- will buffer this anyway. @@ -436,7 +436,7 @@ lastOffset (MkPrimaryIndex _ offsets) getLastSlot :: ChunkInfo -> PrimaryIndex -> Maybe RelativeSlot getLastSlot chunkInfo (MkPrimaryIndex chunk offsets) = do guard $ V.length offsets >= 2 - return $ nthBlockOrEBB chunkInfo chunk (V.length offsets - 2) + return $ nthBlock chunkInfo chunk (V.length offsets - 2) -- | Check whether the given slot is within the primary index. containsSlot :: PrimaryIndex -> RelativeSlot -> Bool @@ -505,7 +505,7 @@ nextFilledSlot chunkInfo primary@(MkPrimaryIndex chunk offsets) relSlot = | offsets ! i == offsets ! (i + 1) = go (i + 1) | otherwise - = Just (nthBlockOrEBB chunkInfo chunk i) + = Just (nthBlock chunkInfo chunk i) -- | Find the first filled (length > zero) slot in the primary index. If there -- is none, return 'Nothing'. @@ -531,7 +531,7 @@ firstFilledSlot chunkInfo (MkPrimaryIndex chunk offsets) = go 1 | offsets ! i == 0 = go (i + 1) | otherwise - = Just (nthBlockOrEBB chunkInfo chunk (i - 1)) + = Just (nthBlock chunkInfo chunk (i - 1)) -- | Return a list of all the filled (length > zero) slots in the primary -- index. @@ -553,7 +553,7 @@ lastFilledSlot chunkInfo (MkPrimaryIndex chunk offsets) = | offsets ! i == offsets ! (i - 1) = go (i - 1) | otherwise - = Just (nthBlockOrEBB chunkInfo chunk (i - 1)) + = Just (nthBlock chunkInfo chunk (i - 1)) -- | Return the slots to backfill the primary index file with. -- diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Secondary.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Secondary.hs index a0d6f065a1..7388d67617 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Secondary.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Index/Secondary.hs @@ -41,7 +41,7 @@ import Ouroboros.Consensus.Storage.ImmutableDB.Chunks import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index.Primary (SecondaryOffset) import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Types - (BlockOrEBB (..), WithBlockSize (..)) + (WithBlockSize (..)) import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Util (fsPathSecondaryIndexFile, runGet, runGetWithUnconsumed) import Ouroboros.Consensus.Util.IOLike @@ -76,14 +76,11 @@ instance Binary HeaderSize where get = HeaderSize <$> Get.getWord16be put = Put.putWord16be . unHeaderSize -getBlockOrEBB :: IsEBB -> Get BlockOrEBB -getBlockOrEBB IsEBB = EBB . EpochNo <$> Get.getWord64be -getBlockOrEBB IsNotEBB = Block . SlotNo <$> Get.getWord64be +getBlock :: Get SlotNo +getBlock = SlotNo <$> Get.getWord64be -putBlockOrEBB :: BlockOrEBB -> Put -putBlockOrEBB blockOrEBB = Put.putWord64be $ case blockOrEBB of - Block slotNo -> unSlotNo slotNo - EBB epochNo -> unEpochNo epochNo +putBlock :: SlotNo -> Put +putBlock slotNo = Put.putWord64be $ unSlotNo slotNo {------------------------------------------------------------------------------ Entry @@ -95,7 +92,7 @@ data Entry blk = Entry { , headerSize :: !HeaderSize , checksum :: !CRC , headerHash :: !(HeaderHash blk) - , blockOrEBB :: !BlockOrEBB + , entrySlot :: !SlotNo } deriving (Generic) @@ -103,14 +100,14 @@ deriving instance StandardHash blk => Eq (Entry blk) deriving instance StandardHash blk => Show (Entry blk) deriving instance StandardHash blk => NoThunks (Entry blk) -getEntry :: forall blk. ConvertRawHash blk => IsEBB -> Get (Entry blk) -getEntry isEBB = do +getEntry :: forall blk. ConvertRawHash blk => Get (Entry blk) +getEntry = do blockOffset <- get headerOffset <- get headerSize <- get checksum <- CRC <$> Get.getWord32be headerHash <- getHash pb - blockOrEBB <- getBlockOrEBB isEBB + entrySlot <- getBlock return Entry {..} where pb :: Proxy blk @@ -123,7 +120,7 @@ putEntry Entry {..} = mconcat [ , put headerSize , Put.putWord32be (getCRC checksum) , putHash pb headerHash - , putBlockOrEBB blockOrEBB + , putBlock entrySlot ] where pb :: Proxy blk @@ -136,7 +133,7 @@ entrySize pb = + size 2 "headerSize" headerSize + size 4 "checksum" checksum + hashSize pb - + 8 -- blockOrEBB + + 8 -- entrySlot where size :: Storable a => Word32 -> String -> (Entry blk -> a) -> Word32 size expected name field = assert (expected == actual) actual @@ -150,8 +147,7 @@ data BlockSize -- offset after it that we can use to calculate the size of the block. deriving (Eq, Show, Generic, NoThunks) --- | Read the entry at the given 'SecondaryOffset'. Interpret it as an EBB --- depending on the given 'IsEBB'. +-- | Read the entry at the given 'SecondaryOffset'. readEntry :: forall m blk h. ( HasCallStack @@ -162,11 +158,10 @@ readEntry :: ) => HasFS m h -> ChunkNo - -> IsEBB -> SecondaryOffset -> m (Entry blk, BlockSize) -readEntry hasFS chunk isEBB slotOffset = runIdentity <$> - readEntries hasFS chunk (Identity (isEBB, slotOffset)) +readEntry hasFS chunk slotOffset = runIdentity <$> + readEntries hasFS chunk (Identity slotOffset) -- | Same as 'readEntry', but for multiple entries. -- @@ -184,13 +179,13 @@ readEntries :: ) => HasFS m h -> ChunkNo - -> t (IsEBB, SecondaryOffset) + -> t SecondaryOffset -> m (t (Entry blk, BlockSize)) readEntries hasFS chunk toRead = withFile hasFS secondaryIndexFile ReadMode $ \sHnd -> do -- TODO can we avoid this call to 'hGetSize'? size <- hGetSize sHnd - forM toRead $ \(isEBB, slotOffset) -> do + forM toRead $ \slotOffset -> do let offset = AbsOffset (fromIntegral slotOffset) -- Is there another entry after the entry we need to read so that -- we can read its 'blockOffset' that will allow us to calculate @@ -201,14 +196,14 @@ readEntries hasFS chunk toRead = (entry, nextBlockOffset) <- hGetExactlyAt hasFS sHnd (nbBytes + nbBlockOffsetBytes) offset >>= runGet (Proxy @blk) secondaryIndexFile - ((,) <$> getEntry isEBB <*> get) + ((,) <$> getEntry <*> get) let blockSize = fromIntegral $ unBlockOffset nextBlockOffset - unBlockOffset (blockOffset entry) return (entry, BlockSize blockSize) else do entry <- hGetExactlyAt hasFS sHnd nbBytes offset >>= - runGet (Proxy @blk) secondaryIndexFile (getEntry isEBB) + runGet (Proxy @blk) secondaryIndexFile getEntry return (entry, LastEntry) where secondaryIndexFile = fsPathSecondaryIndexFile chunk @@ -234,29 +229,27 @@ readAllEntries :: -> (Entry blk -> Bool) -- ^ Stop condition: stop after this entry -> Word64 -- ^ The size of the chunk file, used to compute -- the size of the last block. - -> IsEBB -- ^ Is the first entry to read an EBB? -> m [WithBlockSize (Entry blk)] -readAllEntries hasFS secondaryOffset chunk stopAfter chunkFileSize = \isEBB -> +readAllEntries hasFS secondaryOffset chunk stopAfter chunkFileSize = withFile hasFS secondaryIndexFile ReadMode $ \sHnd -> do bl <- hGetAllAt hasFS sHnd (AbsOffset (fromIntegral secondaryOffset)) - go isEBB bl [] Nothing + go bl [] Nothing where secondaryIndexFile = fsPathSecondaryIndexFile chunk - go :: IsEBB -- ^ Interpret the next entry as an EBB? - -> Lazy.ByteString + go :: Lazy.ByteString -> [WithBlockSize (Entry blk)] -- ^ Accumulator -> Maybe (Entry blk) -- ^ The previous entry we read. We can only add it to the -- accumulator when we know its block size, which we compute based -- on the next entry's offset. -> m [WithBlockSize (Entry blk)] - go isEBB bl acc mbPrevEntry + go bl acc mbPrevEntry | Lazy.null bl = return $ reverse $ (addBlockSize chunkFileSize <$> mbPrevEntry) `consMaybe` acc | otherwise = do (remaining, entry) <- - runGetWithUnconsumed (Proxy @blk) secondaryIndexFile (getEntry isEBB) bl + runGetWithUnconsumed (Proxy @blk) secondaryIndexFile getEntry bl let offsetAfterPrevBlock = unBlockOffset (blockOffset entry) acc' = (addBlockSize offsetAfterPrevBlock <$> mbPrevEntry) `consMaybe` acc @@ -276,9 +269,7 @@ readAllEntries hasFS secondaryOffset chunk stopAfter chunkFileSize = \isEBB -> return $ reverse $ addBlockSize nextBlockOffset entry : acc' else - -- Pass 'IsNotEBB' because there can only be one EBB and that must - -- be the first one in the file. - go IsNotEBB remaining acc' (Just entry) + go remaining acc' (Just entry) -- | Add the block size to an entry, it is computed by subtracting the -- entry's block offset from the offset after the entry's block, i.e., diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Iterator.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Iterator.hs index ac897e5e6a..44be1eddd9 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Iterator.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Iterator.hs @@ -134,17 +134,10 @@ streamImpl dbEnv registry blockComponent = \from to -> checkLowerBound currentIndex currentTip from lift $ do - -- 'validBounds' will catch nearly all invalid ranges, except for one: - -- streaming from the regular block to the EBB in the same slot. The - -- EBB comes before the regular block, so these bounds are invalid. - -- However, to distinguish the EBB from the regular block, as both - -- have the same slot number, we need to look at the hashes. - -- 'validateBounds' doesn't have enough information to do that. - when (startChunkSlot > endChunkSlot) $ + when (startChunkSlot >= endChunkSlot) $ throwApiMisuse $ InvalidIteratorRangeError from to - let ChunkSlot startChunk startRelSlot = startChunkSlot - startIsEBB = relativeSlotIsEBB startRelSlot + let ChunkSlot startChunk _startRelSlot = startChunkSlot currentChunkInfo = CurrentChunkInfo currentChunk currentChunkOffset endHash = case to of StreamToInclusive (RealPoint _slot hash) -> hash @@ -158,7 +151,6 @@ streamImpl dbEnv registry blockComponent = \from to -> endHash startChunk secondaryOffset - startIsEBB varIteratorState <- newTVarIO $ IteratorStateOpen iteratorState @@ -246,12 +238,12 @@ streamImpl dbEnv registry blockComponent = \from to -> Nothing -> go (nextChunkNo chunk) Just relSlot -> return (0, chunkSlotForRelativeSlot chunk relSlot) --- | Get information about the block or EBB at the given slot with the given +-- | Get information about the block at the given slot with the given -- hash. If no such block exists, because the slot is empty, it contains a block --- and/or EBB with a different hash, or it is newer than the current tip, return +-- with a different hash, or it is newer than the current tip, return -- a 'MissingBlock'. -- --- Return the 'ChunkSlot' corresponding to the block or EBB, the corresponding +-- Return the 'ChunkSlot' corresponding to the block, the corresponding -- entry (and 'BlockSize') from the secondary index file, and the -- 'SecondaryOffset' of that entry. -- @@ -270,7 +262,7 @@ getSlotInfo :: , SecondaryOffset ) getSlotInfo chunkInfo index currentTip pt@(RealPoint slot hash) = do - let (chunk, mIfBoundary, ifRegular) = + let (chunk, chunkSlot) = chunkSlotForUnknownBlock chunkInfo slot case currentTip of @@ -281,30 +273,15 @@ getSlotInfo chunkInfo index currentTip pt@(RealPoint slot hash) = do -> throwError $ NewerThanTip pt (tipToPoint currentTip) -- Obtain the offsets in the secondary index file from the primary index - -- file. The block /could/ still correspond to an EBB, a regular block or - -- both. We will know which one it is when we can check the hashes from - -- the secondary index file with the hash we have. - toRead :: NonEmpty (IsEBB, SecondaryOffset) <- case mIfBoundary of - Just ifBoundary -> do - let relatives@(Two relb relr) = chunkRelative <$> Two ifBoundary ifRegular - (offsets, s) <- lift $ Index.readOffsets index chunk relatives - case offsets of - Two Nothing Nothing -> - throwError $ EmptySlot pt chunk [relb, relr] s - Two (Just ebbOffset) (Just blkOffset) -> - return ((IsEBB, ebbOffset) NE.:| [(IsNotEBB, blkOffset)]) - Two (Just ebbOffset) Nothing -> - return ((IsEBB, ebbOffset) NE.:| []) - Two Nothing (Just blkOffset) -> - return ((IsNotEBB, blkOffset) NE.:| []) - Nothing -> do - let relr = chunkRelative ifRegular + -- file. + toRead :: NonEmpty SecondaryOffset <- do + let relr = chunkRelative chunkSlot (offset, s) <- lift $ Index.readOffset index chunk relr case offset of Nothing -> throwError $ EmptySlot pt chunk [relr] s Just blkOffset -> - return ((IsNotEBB, blkOffset) NE.:| []) + return (blkOffset NE.:| []) entriesWithBlockSizes :: NonEmpty (Secondary.Entry blk, BlockSize) <- lift $ Index.readEntries index chunk toRead @@ -313,17 +290,12 @@ getSlotInfo chunkInfo index currentTip pt@(RealPoint slot hash) = do -- expected hash. (secondaryOffset, (entry, blockSize)) <- case find ((== hash) . Secondary.headerHash . fst . snd) - (NE.zip (fmap snd toRead) entriesWithBlockSizes) of + (NE.zip toRead entriesWithBlockSizes) of Just found -> return found Nothing -> throwError $ WrongHash pt hashes where hashes = Secondary.headerHash . fst <$> entriesWithBlockSizes - -- Use the secondary index entry to determine whether the slot + hash - -- correspond to an EBB or a regular block. - let chunkSlot = case (mIfBoundary, Secondary.blockOrEBB entry) of - (Just ifBoundary, EBB _) -> ifBoundary - _otherwise -> ifRegular return (chunkSlot, (entry, blockSize), secondaryOffset) @@ -377,16 +349,14 @@ stepIterator registry currentChunkInfo -- cannot loop forever as an error would be thrown when opening the -- index file(s) of a non-existing chunk. Nothing -> openNextChunk (nextChunkNo chunk) - Just relSlot -> do - -- Note that the only reason we actually open the primary index file - -- is to see whether the first block in the chunk is an EBB or not. - -- To see whether the chunk is empty, we could open the secondary - -- index file directly and see whether it contains any blocks. The - -- 'secondaryOffset' will be 0, as the first entry in the secondary - -- index file always starts at offset 0. The same is true for - -- 'findFirstFilledSlot'. - let firstIsEBB = relativeSlotIsEBB relSlot - secondaryOffset = 0 + Just{} -> do + -- TODO Skip opening the primary index merely to check for emptiness; + -- that can be noticed when opening the secondary index instead. + -- + -- The 'secondaryOffset' will be 0, as the first entry in the + -- secondary index file always starts at offset 0. The same is true + -- for 'findFirstFilledSlot'. + let secondaryOffset = 0 iteratorStateForChunk ithHasFS @@ -396,7 +366,6 @@ stepIterator registry currentChunkInfo ithEndHash chunk secondaryOffset - firstIsEBB iteratorNextImpl :: @@ -446,7 +415,7 @@ iteratorHasNextImpl :: => ImmutableDBEnv m blk -> IteratorHandle m blk h -> STM m (Maybe (RealPoint blk)) -iteratorHasNextImpl ImmutableDBEnv { chunkInfo } IteratorHandle { ithVarState } = +iteratorHasNextImpl _immutableDBEnv IteratorHandle { ithVarState } = readTVar ithVarState <&> \case IteratorStateExhausted -> Nothing IteratorStateOpen IteratorState { itsChunkEntries } -> @@ -455,7 +424,7 @@ iteratorHasNextImpl ImmutableDBEnv { chunkInfo } IteratorHandle { ithVarState } WithBlockSize _ nextEntry NE.:| _ = itsChunkEntries slotNo :: SlotNo - slotNo = slotNoOfBlockOrEBB chunkInfo (Secondary.blockOrEBB nextEntry) + slotNo = Secondary.entrySlot nextEntry iteratorCloseImpl :: (HasCallStack, IOLike m) @@ -496,12 +465,10 @@ iteratorStateForChunk :: -> ChunkNo -> SecondaryOffset -- ^ Where to start in the secondary index - -> IsEBB - -- ^ Whether the first expected block will be an EBB or not. -> m (IteratorState m blk h) iteratorStateForChunk hasFS index registry (CurrentChunkInfo curChunk curChunkOffset) endHash - chunk secondaryOffset firstIsEBB = do + chunk secondaryOffset = do -- Open the chunk file. Allocate the handle in the registry so that it -- will be closed in case of an exception. (key, eHnd) <- allocate @@ -539,7 +506,7 @@ iteratorStateForChunk hasFS index registry else hGetSize eHnd entries <- Index.readAllEntries index secondaryOffset chunk - ((== endHash) . Secondary.headerHash) chunkFileSize firstIsEBB + ((== endHash) . Secondary.headerHash) chunkFileSize case NE.nonEmpty entries of -- We still haven't encountered the end bound, so it cannot be @@ -574,14 +541,13 @@ extractBlockComponent :: -> WithBlockSize (Secondary.Entry blk) -> BlockComponent blk b -> m b -extractBlockComponent hasFS chunkInfo chunk ccfg checkIntegrity eHnd +extractBlockComponent hasFS _chunkInfo chunk ccfg checkIntegrity eHnd (WithBlockSize blockSize entry) = go where go :: forall b'. BlockComponent blk b' -> m b' go = \case GetHash -> return headerHash GetSlot -> return slotNo - GetIsEBB -> return $ isBlockOrEBB blockOrEBB GetBlockSize -> return $ SizeInBytes blockSize GetHeaderSize -> return $ fromIntegral $ Secondary.unHeaderSize headerSize GetRawBlock -> readBlock @@ -605,11 +571,11 @@ extractBlockComponent hasFS chunkInfo chunk ccfg checkIntegrity eHnd , headerHash , headerSize , headerOffset - , blockOrEBB + , entrySlot } = entry slotNo :: SlotNo - slotNo = slotNoOfBlockOrEBB chunkInfo blockOrEBB + slotNo = entrySlot pt :: RealPoint blk pt = RealPoint slotNo headerHash diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Parser.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Parser.hs index 59315baa43..750d8ac04b 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Parser.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Parser.hs @@ -161,9 +161,7 @@ parseChunkFile ccfg hasFS isNotCorrupt fsPath expectedChecksums k = , headerSize = Secondary.HeaderSize headerSize , checksum = checksum , headerHash = blockHash blk - , blockOrEBB = case blockIsEBB blk of - Just epoch -> EBB epoch - Nothing -> Block (blockSlot blk) + , entrySlot = blockSlot blk } !blockSummary = BlockSummary { summaryEntry = entry diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Types.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Types.hs index 17bc5cd5df..5d3cf478d8 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Types.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Types.hs @@ -4,9 +4,7 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Impl.Types ( -- * Misc types - BlockOrEBB (..) - , WithBlockSize (..) - , isBlockOrEBB + WithBlockSize (..) -- * Validation policy , ValidationPolicy (..) -- * Chunk file error @@ -31,15 +29,6 @@ import Ouroboros.Consensus.Util.CBOR (ReadIncrementalErr) Misc types ------------------------------------------------------------------------------} -data BlockOrEBB = - Block !SlotNo - | EBB !EpochNo - deriving (Eq, Show, Generic, NoThunks) - -isBlockOrEBB :: BlockOrEBB -> IsEBB -isBlockOrEBB (Block _) = IsNotEBB -isBlockOrEBB (EBB _) = IsEBB - data WithBlockSize a = WithBlockSize { blockSize :: !Word32 , withoutBlockSize :: !a diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Util.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Util.hs index bbd7e1bb93..0cddd0e89a 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Util.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Util.hs @@ -8,8 +8,7 @@ module Ouroboros.Consensus.Storage.ImmutableDB.Impl.Util ( -- * Utilities - Two (..) - , checkChecksum + checkChecksum , dbFilesOnDisk , fsPathChunkFile , fsPathPrimaryIndexFile @@ -47,11 +46,6 @@ import Text.Read (readMaybe) Utilities ------------------------------------------------------------------------------} --- | Useful when you have exactly two values of some type and want to --- 'traverse' over both of them (which is not possible with a tuple). -data Two a = Two a a - deriving (Functor, Foldable, Traversable) - fsPathChunkFile :: ChunkNo -> FsPath fsPathChunkFile = renderFile "chunk" diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Validation.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Validation.hs index 6ab3e82d46..4b225ad576 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Validation.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/ImmutableDB/Impl/Validation.hs @@ -30,8 +30,6 @@ import GHC.Stack (HasCallStack) import Ouroboros.Consensus.Block hiding (hashSize) import Ouroboros.Consensus.Storage.ImmutableDB.API import Ouroboros.Consensus.Storage.ImmutableDB.Chunks -import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal - (unChunkNo, unsafeEpochNoToChunkNo) import Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index (cachedIndex) import qualified Ouroboros.Consensus.Storage.ImmutableDB.Impl.Index as Index @@ -315,7 +313,7 @@ data ShouldBeFinalised = -- -- * When an invalid block needs to be truncated, trailing empty slots are -- also truncated so that the tip of the database will always point to a --- valid block or EBB. +-- valid block. -- -- * All but the most recent chunk in the database should be finalised, i.e. -- padded to the size of the chunk. @@ -360,12 +358,12 @@ validateChunk ValidateEnv{..} shouldBeFinalised chunk mbPrevHash validationTrace -- Note the 'maxBound': it is used to calculate the block size for -- each entry, but we don't care about block sizes here, so we use -- some dummy value. - (Secondary.readAllEntries hasFS 0 chunk (const False) maxBound IsEBB) >>= \case + (Secondary.readAllEntries hasFS 0 chunk (const False) maxBound) >>= \case Left _ -> do traceWith validationTracer $ InvalidSecondaryIndex chunk return [] Right entriesFromFile -> - return $ fixupEBB (map withoutBlockSize entriesFromFile) + return $ map withoutBlockSize entriesFromFile else do traceWith validationTracer $ MissingSecondaryIndex chunk return [] @@ -428,7 +426,7 @@ validateChunk ValidateEnv{..} shouldBeFinalised chunk mbPrevHash validationTrace chunkInfo shouldBeFinalised chunk - (map Secondary.blockOrEBB entries) + (map Secondary.entrySlot entries) primaryIndexFileExists <- doesFileExist primaryIndexFile primaryIndexFileMatches <- if primaryIndexFileExists then tryJust isInvalidFileError (Primary.load (Proxy @blk) hasFS chunk) >>= \case @@ -455,7 +453,6 @@ validateChunk ValidateEnv{..} shouldBeFinalised chunk mbPrevHash validationTrace summaryToTipInfo :: BlockSummary blk -> Tip blk summaryToTipInfo BlockSummary {..} = Tip { tipSlotNo = summarySlotNo - , tipIsEBB = isBlockOrEBB $ Secondary.blockOrEBB summaryEntry , tipBlockNo = summaryBlockNo , tipHash = Secondary.headerHash summaryEntry } @@ -467,51 +464,6 @@ validateChunk ValidateEnv{..} shouldBeFinalised chunk mbPrevHash validationTrace UnexpectedFailure (InvalidFileError {}) -> Just () _ -> Nothing - -- | When reading the entries from the secondary index file, we need to - -- pass in a value of type 'IsEBB' so we know whether the first entry - -- corresponds to an EBB or a regular block. We need this information to - -- correctly interpret the deserialised 'Word64' as a 'BlockOrEBB': if - -- it's an EBB, it's the 'EpochNo' ('Word64'), if it's a regular block, - -- it's a 'SlotNo' ('Word64'). - -- - -- However, at the point we are reading the secondary index file, we don't - -- yet know whether the first block will be an EBB or a regular block. We - -- will find that out when we read the actual block from the chunk file. - -- - -- Fortunately, we can make a /very/ good guess: if the 'Word64' of the - -- 'BlockOrEBB' matches the chunk number, it is almost certainly an EBB, - -- as the slot numbers increase @10k@ times faster than chunk numbers - -- (remember that for EBBs, chunk numbers and epoch numbers must line up). - -- Property: for every chunk @e > 0@, for all slot numbers @s@ in chunk - -- @e@ we have @s > e@. The only exception is chunk 0, which contains a - -- slot number 0. From this follows that it's an EBB if and only if the - -- 'Word64' matches the chunk number. - -- - -- E.g., the first slot number in chunk 1 will be 21600 if @k = 2160@. We - -- could only make the wrong guess in the first very first chunk, i.e., - -- chunk 0, as the first slot number is also 0. However, we know that the - -- real blockchain starts with an EBB, so even in that case we're fine. - -- - -- If the chunk size were 1, then we would make the wrong guess for each - -- chunk that contains an EBB, which is a rather unrealistic scenario. - -- - -- Note that even making the wrong guess is not a problem. The (CRC) - -- checksums are the only thing we extract from the secondary index file. - -- These are passed to the 'ChunkFileParser'. We then reconstruct the - -- secondary index using the output of the 'ChunkFileParser'. If that - -- output doesn't match the parsed secondary index file, we will overwrite - -- the secondary index file. - -- - -- So the only thing that wouldn't go according to plan is that we will - -- needlessly overwrite the secondary index file. - fixupEBB :: forall hash. [Secondary.Entry hash] -> [Secondary.Entry hash] - fixupEBB = \case - entry@Secondary.Entry { blockOrEBB = EBB epoch' }:rest - | let chunk' = unsafeEpochNoToChunkNo epoch' - , chunk' /= chunk - -> entry { Secondary.blockOrEBB = Block (SlotNo (unChunkNo chunk')) }:rest - entries -> entries - -- | Reconstruct a 'PrimaryIndex' based on a list of 'Secondary.Entry's. reconstructPrimaryIndex :: forall blk. (ConvertRawHash blk, HasCallStack) @@ -519,13 +471,13 @@ reconstructPrimaryIndex :: -> ChunkInfo -> ShouldBeFinalised -> ChunkNo - -> [BlockOrEBB] + -> [SlotNo] -> PrimaryIndex -reconstructPrimaryIndex pb chunkInfo shouldBeFinalised chunk blockOrEBBs = +reconstructPrimaryIndex pb chunkInfo shouldBeFinalised chunk entrySlots = fromMaybe (error nonIncreasing) $ Primary.mk chunk . (0:) $ - go (NextRelativeSlot (firstBlockOrEBB chunkInfo chunk)) 0 $ - map (chunkRelative . chunkSlotForBlockOrEBB chunkInfo) blockOrEBBs + go (NextRelativeSlot (firstBlock chunkInfo chunk)) 0 $ + map (chunkRelative . chunkSlotForSlot chunkInfo) entrySlots where nonIncreasing :: String nonIncreasing = "blocks have non-increasing slot numbers" @@ -594,7 +546,7 @@ reconstructPrimaryIndex pb chunkInfo shouldBeFinalised chunk blockOrEBBs = -- * The only difference with the version after it was that chunk files were -- named "XXXXX.epoch" instead of "XXXXX.chunk". The contents of all files -- remain identical because we chose the chunk size to be equal to the Byron --- epoch size and allowed EBBs in the chunk. +-- epoch size. -- -- We don't include versions before the first release, as we don't have to -- migrate from them. diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/Snapshots.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/Snapshots.hs index e3c335b065..2631a2925b 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/Snapshots.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/Snapshots.hs @@ -106,11 +106,6 @@ data TraceSnapshotEvent blk -- If a snapshot with the same number already exists on disk or if the tip is at -- genesis, no snapshot is taken. -- --- Note that an EBB can have the same slot number and thus snapshot number as --- the block after it. This doesn't matter. The one block difference in the --- ledger state doesn't warrant an additional snapshot. The number in the name --- of the snapshot is only indicative, we don't rely on it being correct. --- -- NOTE: This is a lower-level API that takes a snapshot independent from -- whether this snapshot corresponds to a state that is more than @k@ back. -- diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/API.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/API.hs index 20cbce8849..3e8aa3b508 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/API.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/API.hs @@ -107,11 +107,11 @@ data VolatileDB m blk = VolatileDB { -- after all. Say we have mistakenly garbage collected such a block, in -- that case the following would be true: -- - -- 1. The block has a slot number older than the immutable block's slot + -- 1. The block has a slot number older than or equal to the immutable block's slot -- number: otherwise we wouldn't have mistakenly garbage collected -- it. -- - -- 2. The block has a block number greater than the immutable block's + -- 2. The block has a block number greater than or equal to the immutable block's -- block number: otherwise we wouldn't want to adopt it, as it would -- have been older than @k@. -- @@ -121,19 +121,9 @@ data VolatileDB m blk = VolatileDB { -- -- As slot numbers grow monotonically within a chain, all forks starting -- after the immutable block will only contain blocks with slot numbers - -- greater (or equal to in case of EBBs) than the immutable block's slot + -- greater than the immutable block's slot -- number. This directly contradicts (1), so we will /never/ garbage -- collect a block that we might still want to adopt. - -- - -- = Less than vs. less than or equal to - -- - -- Note that we remove blocks with a slot number /less than/ the given - -- slot number, but not /equal to/ it. In practice, this off-by-one - -- difference will not matter in terms of disk space usage, because as - -- soon as the chain grows again by at least one block, those blocks - -- will be removed anyway. The reason for @<@ opposed to @<=@ is to - -- avoid issues with /EBBs/, which have the same slot number as the - -- block after it. , garbageCollect :: HasCallStack => SlotNo -> m () -- | Return the highest slot number ever stored by the VolatileDB. , getMaxSlotNo :: HasCallStack => STM m MaxSlotNo @@ -150,7 +140,6 @@ data BlockInfo blk = BlockInfo { , biSlotNo :: !SlotNo , biBlockNo :: !BlockNo , biPrevHash :: !(ChainHash blk) - , biIsEBB :: !IsEBB , biHeaderOffset :: !Word16 , biHeaderSize :: !Word16 } diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl.hs index 6d1d09cfcd..4c010df5dc 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl.hs @@ -268,7 +268,6 @@ getBlockComponentImpl env@VolatileDBEnv { codecConfig, checkIntegrity } blockCom getBlockComponent hasFS ibi = \case GetHash -> return hash GetSlot -> return biSlotNo - GetIsEBB -> return biIsEBB GetBlockSize -> return $ fromIntegral $ unBlockSize ibiBlockSize GetHeaderSize -> return biHeaderSize GetPure a -> return a @@ -397,7 +396,7 @@ putBlockImpl env@VolatileDBEnv{ maxBlocksPerFile, tracer, codecConfig } , currentMaxSlotNo = currentMaxSlotNo `max` MaxSlotNo biSlotNo } --- | Garbage collect all files of which the highest slot is less than the +-- | Garbage collect all files of which the highest slot is less than or equal to the -- given slot. -- -- We first check whether we actually can garbage collect any file. If we can, diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl/Parser.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl/Parser.hs index c71827de63..4307853152 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl/Parser.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/VolatileDB/Impl/Parser.hs @@ -108,7 +108,6 @@ extractBlockInfo blk = BlockInfo { biHash = blockHash blk , biSlotNo = blockSlot blk , biBlockNo = blockNo blk - , biIsEBB = blockToIsEBB blk , biPrevHash = blockPrevHash blk , biHeaderOffset = headerOffset , biHeaderSize = headerSize diff --git a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Util/AnchoredFragment.hs b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Util/AnchoredFragment.hs index a08d3bdc5c..61415f1d86 100644 --- a/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Util/AnchoredFragment.hs +++ b/ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Util/AnchoredFragment.hs @@ -36,17 +36,6 @@ import qualified Ouroboros.Network.AnchoredFragment as AF -- -- A fragment with a head is always \"greater\" than one without. When both -- fragments have no head (i.e. are empty), they are 'EQ'. --- --- Note that an EBB can share its @BlockNo@ with another regular block. If --- such an EBB is the head of one fragment and the regular block with the same --- @BlockNo@ is the head of the other fragment, then this function will say --- they are 'EQ', while in fact one fragment should be preferred over the --- other. --- --- This is not a big deal as we won't be seeing new EBBs, so they will not be --- the head of a fragment very often anyway, only when catching up. As soon as --- a new block/header is added to the fragment, the right decision will be --- made again ('GT' or 'LT'). compareHeadBlockNo :: HasHeader b => AnchoredFragment b