Merge pull request #17 from greg-szabo/greg/blobs

feat(consensus): blobs feature

Author: greg-szabo

.golangci.yml

@@ -73,6 +73,25 @@ linters-settings:
           - github.com/stretchr/testify/require
           - github.com/syndtr/goleveldb
           - github.com/decred/dcrd/dcrec/secp256k1/v4
+          - google.golang.org/protobuf/proto
+          - google.golang.org/protobuf/types/known/timestamppb
+          - golang.org/x/sync/errgroup
+          - golang.org/x/net/netutil
+          - golang.org/x/crypto/chacha20poly1305
+          - golang.org/x/net/netutil
+          - google.golang.org/grpc
+          - golang.org/x/net/context
+          - google.golang.org/grpc
+          - golang.org/x/crypto/openpgp/armor
+          - gonum.org/v1/gonum/stat
+          - golang.org/x/crypto/chacha20poly1305
+          - golang.org/x/crypto/curve25519
+          - golang.org/x/crypto/hkdf
+          - golang.org/x/crypto/nacl/box
+          - google.golang.org/grpc
+          - google.golang.org/grpc/credentials/insecure
+          - golang.org/x/crypto/nacl/secretbox
+          - golang.org/x/sync/semaphore
       test:
         files:
           - "$test"
@@ -94,6 +113,11 @@ linters-settings:
           - github.com/spf13
           - github.com/stretchr/testify
           - github.com/decred/dcrd/dcrec/secp256k1/v4
+          - google.golang.org/grpc
+          - golang.org/x/net/context
+          - golang.org/x/crypto/bcrypt
+          - golang.org/x/crypto/chacha20poly1305
+          - google.golang.org/protobuf/types/known/timestamppb
 
   revive:
     enable-all-rules: true

abci/example/kvstore/kvstore.go

@@ -22,6 +22,8 @@ import (
 var (
 	stateKey        = []byte("stateKey")
 	kvPairPrefixKey = []byte("kvPairKey:")
+
+	_testBlob = []byte("testBlob")
 )
 
 const (
@@ -49,6 +51,9 @@ type Application struct {
 	// If true, the app will generate block events in BeginBlock. Used to test the event indexer
 	// Should be false by default to avoid generating too much data.
 	genBlockEvents bool
+
+	// Generate blobs
+	generateBlobs bool
 }
 
 // NewApplication creates an instance of the kvstore from the provided database
@@ -80,6 +85,17 @@ func (app *Application) SetGenBlockEvents() {
 	app.genBlockEvents = true
 }
 
+func (app *Application) SetGenerateBlobs() {
+	app.generateBlobs = true
+}
+
+// TestBlob returns the blob that the app returns in PrepareProposal.
+// TestBlob is only used in testing, and is not part of the abci.Application
+// interface.
+func (*Application) TestBlob() []byte {
+	return _testBlob
+}
+
 // Info returns information about the state of the application. This is generally used everytime a Tendermint instance
 // begins and let's the application know what Tendermint versions it's interacting with. Based from this information,
 // Tendermint will ensure it is in sync with the application by potentially replaying the blocks it has. If the
@@ -162,6 +178,9 @@ func isValidTx(tx []byte) bool {
 // quite a trivial example of transaction modification.
 // NOTE: we assume that CometBFT will never provide more transactions than can fit in a block.
 func (app *Application) PrepareProposal(ctx context.Context, req *types.RequestPrepareProposal) (*types.ResponsePrepareProposal, error) {
+	if app.generateBlobs {
+		return &types.ResponsePrepareProposal{Txs: app.formatTxs(ctx, req.Txs), Blob: _testBlob}, nil
+	}
 	return &types.ResponsePrepareProposal{Txs: app.formatTxs(ctx, req.Txs)}, nil
 }
 
@@ -186,6 +205,10 @@ func (app *Application) ProcessProposal(ctx context.Context, req *types.RequestP
 			return &types.ResponseProcessProposal{Status: types.ResponseProcessProposal_REJECT}, nil
 		}
 	}
+
+	if app.generateBlobs && !bytes.Equal(req.Blob, _testBlob) {
+		return &types.ResponseProcessProposal{Status: types.ResponseProcessProposal_REJECT}, nil
+	}
 	return &types.ResponseProcessProposal{Status: types.ResponseProcessProposal_ACCEPT}, nil
 }
 

abci/types/types.pb.go

@@ -479,7 +479,7 @@ FOR_LOOP:
 			// Try again quickly next loop.
 			didProcessCh <- struct{}{}
 
-			firstParts, err := first.MakePartSet(types.BlockPartSizeBytes)
+			firstParts, err := first.MakePartSet(types.PartSizeBytes)
 			if err != nil {
 				bcR.Logger.Error("failed to make ",
 					"height", first.Height,

blocksync/reactor.go

@@ -140,7 +140,7 @@ func newReactor(
 
 		thisBlock := state.MakeBlock(blockHeight, nil, lastExtCommit.ToCommit(), nil, state.Validators.Proposer.Address)
 
-		thisParts, err := thisBlock.MakePartSet(types.BlockPartSizeBytes)
+		thisParts, err := thisBlock.MakePartSet(types.PartSizeBytes)
 		require.NoError(t, err)
 		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
 

blocksync/reactor_test.go

@@ -210,10 +210,10 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 		}
 		proposerAddr := lazyProposer.privValidatorPubKey.Address()
 
-		block, err := lazyProposer.blockExec.CreateProposalBlock(
+		block, _, err := lazyProposer.blockExec.CreateProposalBlock(
 			ctx, lazyProposer.Height, lazyProposer.state, extCommit, proposerAddr)
 		require.NoError(t, err)
-		blockParts, err := block.MakePartSet(types.BlockPartSizeBytes)
+		blockParts, err := block.MakePartSet(types.PartSizeBytes)
 		require.NoError(t, err)
 
 		// Flush the WAL. Otherwise, we may not recompute the same proposal to sign,
@@ -224,7 +224,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 
 		// Make proposal
 		propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
-		proposal := types.NewProposal(height, round, lazyProposer.ValidRound, propBlockID)
+		proposal := types.NewProposal(height, round, lazyProposer.ValidRound, propBlockID, types.BlobID{})
 		p := proposal.ToProto()
 		if err := lazyProposer.privValidator.SignProposal(lazyProposer.state.ChainID, p); err == nil {
 			proposal.Signature = p.Signature
@@ -304,7 +304,7 @@ func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	app := newKVStore
+	app := newKVStoreWithBlob
 	css, cleanup := randConsensusNet(t, N, "consensus_byzantine_test", newMockTickerFunc(false), app)
 	defer cleanup()
 
@@ -342,7 +342,7 @@ func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
 			}
 			// We are setting the prevote function to do nothing because the prevoting
 			// and precommitting are done alongside the proposal.
-			css[i].doPrevote = func(height int64, round int32) {}
+			css[i].doPrevote = func(_ int64, _ int32) {}
 		}
 
 		eventBus := css[i].eventBus
@@ -461,12 +461,16 @@ func byzantineDecideProposalFunc(ctx context.Context, t *testing.T, height int64
 	// Avoid sending on internalMsgQueue and running consensus state.
 
 	// Create a new proposal block from state/txs from the mempool.
-	block1, err := cs.createProposalBlock(ctx)
-	require.NoError(t, err)
-	blockParts1, err := block1.MakePartSet(types.BlockPartSizeBytes)
-	require.NoError(t, err)
-	polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()}
-	proposal1 := types.NewProposal(height, round, polRound, propBlockID)
+	block1, blockParts1, propBlockID, blob := createProposalBlockAndBlob(t, cs)
+
+	blobParts := types.NewPartSetFromData(blob, types.PartSizeBytes)
+	blobID    := types.BlobID{
+		Hash:          blob.Hash(),
+		PartSetHeader: blobParts.Header(),
+	}
+
+	polRound := cs.ValidRound
+	proposal1 := types.NewProposal(height, round, polRound, propBlockID, blobID)
 	p1 := proposal1.ToProto()
 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil {
 		t.Error(err)
@@ -478,12 +482,12 @@ func byzantineDecideProposalFunc(ctx context.Context, t *testing.T, height int64
 	deliverTxsRange(t, cs, 0, 1)
 
 	// Create a new proposal block from state/txs from the mempool.
-	block2, err := cs.createProposalBlock(ctx)
+	block2, _, err := cs.createProposalBlock(ctx)
 	require.NoError(t, err)
-	blockParts2, err := block2.MakePartSet(types.BlockPartSizeBytes)
+	blockParts2, err := block2.MakePartSet(types.PartSizeBytes)
 	require.NoError(t, err)
 	polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()}
-	proposal2 := types.NewProposal(height, round, polRound, propBlockID)
+	proposal2 := types.NewProposal(height, round, polRound, propBlockID, blobID)
 	p2 := proposal2.ToProto()
 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil {
 		t.Error(err)
@@ -499,9 +503,9 @@ func byzantineDecideProposalFunc(ctx context.Context, t *testing.T, height int64
 	t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers))
 	for i, peer := range peers {
 		if i < len(peers)/2 {
-			go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1)
+			go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1, blobParts)
 		} else {
-			go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2)
+			go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2, blobParts)
 		}
 	}
 }
@@ -513,17 +517,17 @@ func sendProposalAndParts(
 	peer p2p.Peer,
 	proposal *types.Proposal,
 	blockHash []byte,
-	parts *types.PartSet,
+	blockParts, blobParts *types.PartSet,
 ) {
 	// proposal
 	peer.Send(p2p.Envelope{
 		ChannelID: DataChannel,
 		Message:   &cmtcons.Proposal{Proposal: *proposal.ToProto()},
 	})
 
-	// parts
-	for i := 0; i < int(parts.Total()); i++ {
-		part := parts.GetPart(i)
+	// block parts
+	for i := 0; i < int(blockParts.Total()); i++ {
+		part := blockParts.GetPart(i)
 		pp, err := part.ToProto()
 		if err != nil {
 			panic(err) // TODO: wbanfield better error handling
@@ -538,10 +542,29 @@ func sendProposalAndParts(
 		})
 	}
 
+	// blob parts
+	if blobParts != nil {
+		for i := 0; i < int(blobParts.Total()); i++ {
+			part := blobParts.GetPart(i)
+			pp, err := part.ToProto()
+			if err != nil {
+				panic(err)
+			}
+			peer.Send(p2p.Envelope{
+				ChannelID: DataChannel,
+				Message: &cmtcons.BlobPart{
+					Height: height, // This tells peer that this part applies to us.
+					Round:  round,  // This tells peer that this part applies to us.
+					Part:   *pp,
+				},
+			})
+		}
+	}
+
 	// votes
 	cs.mtx.Lock()
-	prevote, _ := cs.signVote(cmtproto.PrevoteType, blockHash, parts.Header(), nil)
-	precommit, _ := cs.signVote(cmtproto.PrecommitType, blockHash, parts.Header(), nil)
+	prevote, _ := cs.signVote(cmtproto.PrevoteType, blockHash, blockParts.Header(), nil)
+	precommit, _ := cs.signVote(cmtproto.PrecommitType, blockHash, blockParts.Header(), nil)
 	cs.mtx.Unlock()
 	peer.Send(p2p.Envelope{
 		ChannelID: VoteChannel,