Cell Level Dissemination / Partial columns

beacon-chain/blockchain/BUILD.bazel

@@ -167,6 +167,7 @@ go_test(
         "//beacon-chain/operations/slashings:go_default_library",
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
+        "//beacon-chain/p2p/partial:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
         "//beacon-chain/startup:go_default_library",
         "//beacon-chain/state:go_default_library",

beacon-chain/blockchain/setup_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/operations/attestations"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/operations/blstoexec"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/p2p"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/p2p/partial"
 	p2pTesting "github.com/OffchainLabs/prysm/v6/beacon-chain/p2p/testing"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/startup"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/state/stategen"
@@ -89,7 +90,7 @@ func (mb *mockBroadcaster) BroadcastLightClientFinalityUpdate(_ context.Context,
 	return nil
 }
 
-func (mb *mockBroadcaster) BroadcastDataColumnSidecars(_ context.Context, _ []blocks.VerifiedRODataColumn) error {
+func (mb *mockBroadcaster) BroadcastDataColumnSidecars(_ context.Context, _ []blocks.VerifiedRODataColumn, _ []partial.PartialColumn) error {
 	mb.broadcastCalled = true
 	return nil
 }

beacon-chain/core/peerdas/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/blockchain/kzg:go_default_library",
+        "//beacon-chain/p2p/partial:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
@@ -33,6 +34,7 @@ go_library(
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@org_golang_x_sync//errgroup:go_default_library",
     ],
 )

beacon-chain/core/peerdas/reconstruction.go

@@ -10,6 +10,7 @@ import (
 	pb "github.com/OffchainLabs/prysm/v6/proto/engine/v1"
 	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/go-bitfield"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -237,35 +238,38 @@ func ComputeCellsAndProofsFromFlat(blobs [][]byte, cellProofs [][]byte) ([]kzg.C
 }
 
 // ComputeCellsAndProofs computes the cells and proofs from blobs and cell proofs.
-func ComputeCellsAndProofsFromStructured(blobsAndProofs []*pb.BlobAndProofV2) ([]kzg.CellsAndProofs, error) {
+func ComputeCellsAndProofsFromStructured(commitmentCount uint64, blobsAndProofs []*pb.BlobAndProofV2) (included bitfield.Bitlist, _ []kzg.CellsAndProofs, _ error) {
 	numberOfColumns := params.BeaconConfig().NumberOfColumns
 
 	cellsAndProofs := make([]kzg.CellsAndProofs, 0, len(blobsAndProofs))
-	for _, blobAndProof := range blobsAndProofs {
+	included = bitfield.NewBitlist(commitmentCount)
+
+	for i, blobAndProof := range blobsAndProofs {
 		if blobAndProof == nil {
-			return nil, ErrNilBlobAndProof
+			continue
 		}
+		included.SetBitAt(uint64(i), true)
 
 		var kzgBlob kzg.Blob
 		if copy(kzgBlob[:], blobAndProof.Blob) != len(kzgBlob) {
-			return nil, errors.New("wrong blob size - should never happen")
+			return nil, nil, errors.New("wrong blob size - should never happen")
 		}
 
 		// Compute the extended cells from the (non-extended) blob.
 		cells, err := kzg.ComputeCells(&kzgBlob)
 		if err != nil {
-			return nil, errors.Wrap(err, "compute cells")
+			return nil, nil, errors.Wrap(err, "compute cells")
 		}
 
-		kzgProofs := make([]kzg.Proof, 0, numberOfColumns*kzg.BytesPerProof)
+		kzgProofs := make([]kzg.Proof, 0, numberOfColumns)
 		for _, kzgProofBytes := range blobAndProof.KzgProofs {
 			if len(kzgProofBytes) != kzg.BytesPerProof {
-				return nil, errors.New("wrong KZG proof size - should never happen")
+				return nil, nil, errors.New("wrong KZG proof size - should never happen")
 			}
 
 			var kzgProof kzg.Proof
 			if copy(kzgProof[:], kzgProofBytes) != len(kzgProof) {
-				return nil, errors.New("wrong copied KZG proof size - should never happen")
+				return nil, nil, errors.New("wrong copied KZG proof size - should never happen")
 			}
 
 			kzgProofs = append(kzgProofs, kzgProof)
@@ -275,7 +279,7 @@ func ComputeCellsAndProofsFromStructured(blobsAndProofs []*pb.BlobAndProofV2) ([
 		cellsAndProofs = append(cellsAndProofs, cellsProofs)
 	}
 
-	return cellsAndProofs, nil
+	return included, cellsAndProofs, nil
 }
 
 // blobSidecarsFromDataColumnSidecars converts verified data column sidecars to verified blob sidecars.

beacon-chain/core/peerdas/reconstruction_test.go

@@ -380,8 +380,9 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
 
 func TestComputeCellsAndProofsFromStructured(t *testing.T) {
 	t.Run("nil blob and proof", func(t *testing.T) {
-		_, err := peerdas.ComputeCellsAndProofsFromStructured([]*pb.BlobAndProofV2{nil})
-		require.ErrorIs(t, err, peerdas.ErrNilBlobAndProof)
+		included, _, err := peerdas.ComputeCellsAndProofsFromStructured(0, []*pb.BlobAndProofV2{nil})
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), included.Count())
 	})
 
 	t.Run("nominal", func(t *testing.T) {
@@ -432,14 +433,16 @@ func TestComputeCellsAndProofsFromStructured(t *testing.T) {
 		require.NoError(t, err)
 
 		// Test ComputeCellsAndProofs
-		actualCellsAndProofs, err := peerdas.ComputeCellsAndProofsFromStructured(blobsAndProofs)
+		included, actualCellsAndProofs, err := peerdas.ComputeCellsAndProofsFromStructured(uint64(len(blobsAndProofs)), blobsAndProofs)
+		require.Equal(t, included.Count(), uint64(len(actualCellsAndProofs)))
 		require.NoError(t, err)
 		require.Equal(t, blobCount, len(actualCellsAndProofs))
 
 		// Verify the results match expected
 		for i := range blobCount {
 			require.Equal(t, len(expectedCellsAndProofs[i].Cells), len(actualCellsAndProofs[i].Cells))
 			require.Equal(t, len(expectedCellsAndProofs[i].Proofs), len(actualCellsAndProofs[i].Proofs))
+			require.Equal(t, len(expectedCellsAndProofs[i].Proofs), cap(actualCellsAndProofs[i].Proofs))
 
 			// Compare cells
 			for j, expectedCell := range expectedCellsAndProofs[i].Cells {

beacon-chain/core/peerdas/validator.go

@@ -1,16 +1,19 @@
 package peerdas
 
 import (
+	"slices"
 	"time"
 
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/kzg"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/p2p/partial"
 	beaconState "github.com/OffchainLabs/prysm/v6/beacon-chain/state"
 	fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams"
 	"github.com/OffchainLabs/prysm/v6/config/params"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
 	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/go-bitfield"
 )
 
 var (
@@ -119,16 +122,21 @@ func DataColumnSidecars(rows []kzg.CellsAndProofs, src ConstructionPopulator) ([
 	for idx := range maxIdx {
 		sidecar := &ethpb.DataColumnSidecar{
 			Index:                        idx,
-			Column:                       cells[idx],
 			KzgCommitments:               info.kzgCommitments,
-			KzgProofs:                    proofs[idx],
+			Column:                       make([][]byte, len(cells[idx])),
+			KzgProofs:                    make([][]byte, len(proofs[idx])),
 			SignedBlockHeader:            info.signedBlockHeader,
 			KzgCommitmentsInclusionProof: info.kzgInclusionProof,
 		}
 
 		if len(sidecar.KzgCommitments) != len(sidecar.Column) || len(sidecar.KzgCommitments) != len(sidecar.KzgProofs) {
 			return nil, ErrSizeMismatch
 		}
+		for i := range cells[idx] {
+			sidecar.Column[i] = cells[idx][i][:]
+			// The above check verifies len(cells) == len(proofs)
+			sidecar.KzgProofs[i] = proofs[idx][i][:]
+		}
 
 		roSidecar, err := blocks.NewRODataColumnWithRoot(sidecar, src.Root())
 		if err != nil {
@@ -141,6 +149,49 @@ func DataColumnSidecars(rows []kzg.CellsAndProofs, src ConstructionPopulator) ([
 	return roSidecars, nil
 }
 
+func PartialColumns(included bitfield.Bitlist, rows []kzg.CellsAndProofs, src ConstructionPopulator) ([]partial.PartialColumn, error) {
+	if len(rows) == 0 {
+		return nil, nil
+	}
+	numColumns := params.BeaconConfig().NumberOfColumns
+	cells, proofs, err := rotateRowsToCols(rows, numColumns)
+	if err != nil {
+		return nil, errors.Wrap(err, "rotate cells and proofs")
+	}
+	info, err := src.extract()
+	if err != nil {
+		return nil, errors.Wrap(err, "extract block info")
+	}
+
+	cols := make([]partial.PartialColumn, numColumns)
+	commitments := make([]kzg.Commitment, len(info.kzgCommitments))
+	for i := range commitments {
+		if copy(commitments[i][:], info.kzgCommitments[i]) != len(commitments[i]) {
+			return nil, errors.New("size mismatch of kzg commitments")
+		}
+	}
+	blockInfo, err := src.extract()
+	if err != nil {
+		return nil, errors.Wrap(err, "extract block info")
+	}
+
+	root := src.Root()
+	for idx := range numColumns {
+		cols[idx] = partial.NewPartialColumn(
+			blockInfo.signedBlockHeader,
+			blockInfo.kzgInclusionProof,
+			root[:],
+			commitments,
+			// Clone because it becomes owned by the partial column.
+			slices.Clone(included),
+			cells[idx],
+			proofs[idx],
+		)
+	}
+
+	return cols, nil
+}
+
 // Slot returns the slot of the source
 func (s *BlockReconstructionSource) Slot() primitives.Slot {
 	return s.Block().Slot()
@@ -197,12 +248,12 @@ func (b *BlockReconstructionSource) extract() (*blockInfo, error) {
 
 // rotateRowsToCols takes a 2D slice of cells and proofs, where the x is rows (blobs) and y is columns,
 // and returns a 2D slice where x is columns and y is rows.
-func rotateRowsToCols(rows []kzg.CellsAndProofs, numCols uint64) ([][][]byte, [][][]byte, error) {
+func rotateRowsToCols(rows []kzg.CellsAndProofs, numCols uint64) ([][]kzg.Cell, [][]kzg.Proof, error) {
 	if len(rows) == 0 {
 		return nil, nil, nil
 	}
-	cellCols := make([][][]byte, numCols)
-	proofCols := make([][][]byte, numCols)
+	cellCols := make([][]kzg.Cell, numCols)
+	proofCols := make([][]kzg.Proof, numCols)
 	for i, cp := range rows {
 		if uint64(len(cp.Cells)) != numCols {
 			return nil, nil, errors.Wrap(ErrNotEnoughDataColumnSidecars, "not enough cells")
@@ -212,11 +263,11 @@ func rotateRowsToCols(rows []kzg.CellsAndProofs, numCols uint64) ([][][]byte, []
 		}
 		for j := uint64(0); j < numCols; j++ {
 			if i == 0 {
-				cellCols[j] = make([][]byte, len(rows))
-				proofCols[j] = make([][]byte, len(rows))
+				cellCols[j] = make([]kzg.Cell, len(rows))
+				proofCols[j] = make([]kzg.Proof, len(rows))
 			}
-			cellCols[j][i] = cp.Cells[j][:]
-			proofCols[j][i] = cp.Proofs[j][:]
+			cellCols[j][i] = cp.Cells[j]
+			proofCols[j][i] = cp.Proofs[j]
 		}
 	}
 	return cellCols, proofCols, nil

beacon-chain/execution/BUILD.bazel

@@ -36,6 +36,7 @@ go_library(
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/execution/types:go_default_library",
+        "//beacon-chain/p2p/partial:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
@@ -72,6 +73,7 @@ go_library(
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_k8s_client_go//tools/cache:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",