Add NTN CDDLs

.github/workflows/ci.yml

@@ -55,6 +55,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Check CDDLs up-to-date with Blueprints
+      continue-on-error: true
+      run: |
+        ./scripts/ci/diff-cddls.sh
+
     - name: Install base libraries
       uses: input-output-hk/actions/base@latest
       with:
@@ -138,6 +143,17 @@ jobs:
     - name: Build projects [build]
       run: cabal build all -j
 
+    - name: Install Ruby for CDDL
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+        bundler-cache: true
+
+    - name: Install cddl tools
+      run: |
+        gem install cddlc
+        cabal install cuddle-0.5.0.0
+
     - name: Test
       if: matrix.test-set == 'all'
       run: cabal test all -j --test-show-details=streaming

nix/cddlc/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'cddlc'

nix/cddlc/Gemfile.lock

@@ -0,0 +1,19 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cddlc (0.4.2)
+      neatjson (~> 0.10)
+      treetop (~> 1)
+    neatjson (0.10.5)
+    polyglot (0.3.5)
+    treetop (1.6.14)
+      polyglot (~> 0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cddlc
+
+BUNDLED WITH
+   2.6.2

nix/cddlc/gemset.nix

@@ -0,0 +1,47 @@
+{
+  cddlc = {
+    dependencies = [
+      "neatjson"
+      "treetop"
+    ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1s3fbgd5yqgji162zsmlwnva1v1r3zc1qiyv6im7karv5f08r8m3";
+      type = "gem";
+    };
+    version = "0.4.2";
+  };
+  neatjson = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "0wm1lq8yl6rzysh3wg6fa55w5534k6ppiz0qb7jyvdy582mk5i0s";
+      type = "gem";
+    };
+    version = "0.10.5";
+  };
+  polyglot = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1bqnxwyip623d8pr29rg6m8r0hdg08fpr2yb74f46rn1wgsnxmjr";
+      type = "gem";
+    };
+    version = "0.3.5";
+  };
+  treetop = {
+    dependencies = [ "polyglot" ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1m5fqy7vq6y7bgxmw7jmk7y6pla83m16p7lb41lbqgg53j8x2cds";
+      type = "gem";
+    };
+    version = "1.6.14";
+  };
+}

nix/cddlc/package.nix

@@ -0,0 +1,23 @@
+{ lib
+, bundlerApp
+, bundlerUpdateScript
+}:
+
+bundlerApp {
+  pname = "cddlc";
+
+  gemdir = ./.;
+
+  exes = [ "cddlc" ];
+
+  passthru.updateScript = bundlerUpdateScript "cddlc";
+
+  meta = {
+    description = "CDDL conversion utilities";
+    homepage = "https://github.com/cabo/cddlc";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ amesgen ];
+    platforms = lib.platforms.unix;
+    mainProgram = "cddlc";
+  };
+}

nix/haskell.nix

@@ -38,6 +38,19 @@ let
       ({ pkgs, lib, ... }: lib.mkIf pkgs.stdenv.hostPlatform.isWindows {
         # https://github.com/input-output-hk/haskell.nix/issues/2332
         packages.basement.configureFlags = [ "--hsc2hs-option=--cflag=-Wno-int-conversion" ];
+        # We can't cross-compile the ruby gem `cddlc` so we decided to skip this
+        # test on Windows in Hydra.
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test.preCheck = ''
+          export DISABLE_CDDLC=1
+        '';
+      })
+      ({ pkgs, ... }: lib.mkIf (!pkgs.stdenv.hostPlatform.isWindows) {
+        # Tools for CBOR/CDDL tests:
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test = {
+          build-tools =
+            [ pkgs.cddlc pkgs.cuddle ];
+          extraSrcFiles = [ "cddl/**/*" ];
+        };
       })
     ];
     flake.variants = {

nix/shell.nix

@@ -16,6 +16,8 @@ hsPkgs.shellFor {
     pkgs.ghcid
     pkgs.xrefcheck
     pkgs.fourmolu
+    pkgs.cuddle
+    pkgs.cddlc
 
     # release management
     pkgs.scriv

nix/tools.nix

@@ -38,6 +38,18 @@ in
 
   fourmolu = tool "fourmolu" "0.18.0.0" { };
 
+  cuddle = tool "cuddle" "git" {
+    src = final.fetchFromGitHub {
+      owner = "input-output-hk";
+      repo = "cuddle";
+      rev = "cuddle-0.5.0.0";
+      hash = "sha256-06a9N1IAh0kKW/xPu1qiLK22HpXyARnipA1YJxY4jOQ=";
+    };
+  };
+
+  # remove once our nixpkgs contains https://github.com/NixOS/nixpkgs/pull/394873
+  cddlc = final.callPackage ./cddlc/package.nix { };
+
   haskellBuildUtils = prev.haskellBuildUtils.override {
     inherit (final.hsPkgs.args) compiler-nix-name;
     index-state = tool-index-state;

ouroboros-consensus-cardano/cddl/base.cddl

@@ -0,0 +1,52 @@
+telescope7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [pastEra, pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<conway>] /
+    [pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<babbage>] /
+    [pastEra, pastEra, pastEra, pastEra, currentEra<alonzo>] /
+    [pastEra, pastEra, pastEra, currentEra<mary>] /
+    [pastEra, pastEra, currentEra<allegra>] /
+    [pastEra, currentEra<shelley>] /
+    [currentEra<byron>]
+
+ns7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [6, conway] /
+    [5, babbage] /
+    [4, alonzo] /
+    [3, mary] /
+    [2, allegra] /
+    [1, shelley] /
+    [0, byron]
+
+;; Blockchain types
+pastEra = [bound, bound]
+currentEra<st> = [bound, st]
+bound = [relativeTime, slotno, epochno]
+eraIdx = word8
+individualPoolStake = [stake, hash]
+nonce = [0] / [1, hash]
+point = [] / [ slotno, hash ]
+poolDistr = map<keyhash, individualPoolStake>
+slotno = word64
+stake = rational
+
+withOrigin<v> = [] / [v]
+
+;; Collections
+either<x, y> = [0, x] / [1, y]
+map<x, y> = { * x => y }
+maybe<x> = [] / [x]
+seq<x> = [*23 x] / [24* x] ; encoded with indefinite-length encoding
+set<x> = #6.258([* x])
+
+;; Types from other packages
+blockno = word64
+epochno = word64
+coin = word64
+rational = [int, int]
+keyhash = bstr .size 28
+hash = bstr .size 32
+relativeTime = int
+
+;; Base word types
+word8  = uint .size 1
+word32 = uint .size 4
+word64 = uint .size 8

ouroboros-consensus-cardano/cddl/node-to-node/blockfetch/block.cddl

@@ -0,0 +1,17 @@
+serialisedCardanoBlock = #6.24(bytes .cbor cardanoBlock)
+
+cardanoBlock = byron.block
+             / [2, shelley.block]
+             / [3, allegra.block]
+             / [4, mary.block]
+             / [5, alonzo.block]
+             / [6, babbage.block]
+             / [7, conway.block]
+
+;# import byron as byron
+;# import shelley as shelley
+;# import allegra as allegra
+;# import mary as mary
+;# import alonzo as alonzo
+;# import babbage as babbage
+;# import conway as conway

ouroboros-consensus-cardano/cddl/node-to-node/chainsync/header.cddl

@@ -0,0 +1,25 @@
+header
+ = base.ns7<byronHeader,
+            serialisedShelleyHeader<shelley.header>,
+            serialisedShelleyHeader<allegra.header>,
+            serialisedShelleyHeader<mary.header>,
+            serialisedShelleyHeader<alonzo.header>,
+            serialisedShelleyHeader<babbage.header>,
+            serialisedShelleyHeader<conway.header>>
+
+byronHeader = [byronRegularIdx, #6.24(bytes .cbor byron.blockhead)]
+            / [byronBoundaryIdx, #6.24(bytes .cbor byron.ebbhead)]
+
+byronBoundaryIdx = [0, base.word32]
+byronRegularIdx = [1, base.word32]
+
+serialisedShelleyHeader<era> = #6.24(bytes .cbor era)
+
+;# include byron as byron
+;# include shelley as shelley
+;# include allegra as allegra
+;# include mary as mary
+;# include alonzo as alonzo
+;# include babbage as babbage
+;# include conway as conway
+;# import base as base

ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/tx.cddl

@@ -0,0 +1,23 @@
+tx =
+  base.ns7<byron.transaction,
+           serialisedShelleyTx<shelley.transaction>,
+           serialisedShelleyTx<allegra.transaction>,
+           serialisedShelleyTx<mary.transaction>,
+           serialisedShelleyTx<alonzo.transaction>,
+           serialisedShelleyTx<babbage.transaction>,
+           serialisedShelleyTx<conway.transaction>>
+
+serialisedShelleyTx<era> = #6.24(bytes .cbor era)
+
+;# include byron as byron
+
+; See https://github.com/IntersectMBO/cardano-ledger/issues/5124
+byron.transaction = [0, [byron.tx, [+ byron.twit]]] / [1, any] / [2, any] / [3, any]
+
+;# include shelley as shelley
+;# include allegra as allegra
+;# include mary as mary
+;# include alonzo as alonzo
+;# include babbage as babbage
+;# include conway as conway
+;# import base as base

ouroboros-consensus-cardano/cddl/node-to-node/txsubmission2/txId.cddl

@@ -0,0 +1,22 @@
+txId =
+  base.ns7<byronTxId,
+           shelley.transaction_id,
+           allegra.transaction_id,
+           mary.transaction_id,
+           alonzo.transaction_id,
+           conway.transaction_id,
+           babbage.transaction_id>
+
+byronTxId = [0, byron.txid]
+          / [1, byron.certificateid]
+          / [2, byron.updid]
+          / [3, byron.voteid]
+
+;# include byron as byron
+;# include shelley as shelley
+;# include allegra as allegra
+;# include mary as mary
+;# include alonzo as alonzo
+;# include babbage as babbage
+;# include conway as conway
+;# import base as base

ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal

@@ -23,6 +23,10 @@ extra-doc-files:
   CHANGELOG.md
   README.md
 
+data-files:
+  cddl/**/*.cddl
+  cddl/base.cddl
+
 source-repository head
   type: git
   location: https://github.com/IntersectMBO/ouroboros-consensus
@@ -435,6 +439,7 @@ test-suite cardano-test
   main-is: Main.hs
   other-modules:
     Test.Consensus.Cardano.DiffusionPipelining
+    Test.Consensus.Cardano.GenCDDLs
     Test.Consensus.Cardano.Golden
     Test.Consensus.Cardano.MiniProtocol.LocalTxSubmission.ByteStringTxParser
     Test.Consensus.Cardano.MiniProtocol.LocalTxSubmission.Server
@@ -448,27 +453,34 @@ test-suite cardano-test
     Test.ThreadNet.MaryAlonzo
     Test.ThreadNet.ShelleyAllegra
 
+  other-modules: Paths_ouroboros_consensus_cardano
   build-depends:
     QuickCheck,
     base,
     base16-bytestring,
     bytestring,
+    cardano-ledger-allegra:testlib,
     cardano-ledger-alonzo,
+    cardano-ledger-alonzo:testlib,
     cardano-ledger-alonzo-test,
     cardano-ledger-api,
+    cardano-ledger-babbage:testlib,
     cardano-ledger-babbage-test,
     cardano-ledger-binary:testlib,
-    cardano-ledger-byron,
+    cardano-ledger-byron:{cardano-ledger-byron, testlib},
     cardano-ledger-conway:testlib,
     cardano-ledger-core:{cardano-ledger-core, testlib},
+    cardano-ledger-mary:testlib,
     cardano-ledger-shelley,
+    cardano-ledger-shelley:testlib,
     cardano-ledger-shelley-test,
     cardano-protocol-tpraos,
     cardano-slotting,
     cborg,
     constraints,
     containers,
     contra-tracer,
+    directory,
     filepath,
     microlens,
     ouroboros-consensus:{ouroboros-consensus, unstable-consensus-testlib, unstable-mempool-test-utils},
@@ -478,12 +490,14 @@ test-suite cardano-test
     ouroboros-network-api,
     ouroboros-network-protocols:{ouroboros-network-protocols, testlib},
     pretty-simple,
+    process-extras,
     sop-core,
     sop-extras,
     strict-sop-core,
     tasty,
     tasty-hunit,
     tasty-quickcheck,
+    temporary,
     typed-protocols ^>=0.3,
     unstable-byron-testlib,
     unstable-cardano-testlib,