feat!(share/shwap): implement get range request over shwap

nodebuilder/share/share.go

@@ -3,27 +3,17 @@ package share
 import (
 	"context"
 
-	"github.com/tendermint/tendermint/types"
-
 	libshare "github.com/celestiaorg/go-square/v2/share"
 	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/celestiaorg/celestia-node/header"
 	headerServ "github.com/celestiaorg/celestia-node/nodebuilder/header"
 	"github.com/celestiaorg/celestia-node/share"
-	"github.com/celestiaorg/celestia-node/share/eds"
 	"github.com/celestiaorg/celestia-node/share/shwap"
 )
 
 var _ Module = (*API)(nil)
 
-// GetRangeResult wraps the return value of the GetRange endpoint
-// because Json-RPC doesn't support more than two return values.
-type GetRangeResult struct {
-	Shares []libshare.Share
-	Proof  *types.ShareProof
-}
-
 // Module provides access to any data square or block share on the network.
 //
 // All Get methods provided on Module follow the following flow:
@@ -58,7 +48,9 @@ type Module interface {
 		ctx context.Context, height uint64, namespace libshare.Namespace,
 	) (shwap.NamespaceData, error)
 	// GetRange gets a list of shares and their corresponding proof.
-	GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error)
+	GetRange(
+		ctx context.Context, ns libshare.Namespace, height uint64, from, to shwap.SampleCoords, proofsOnly bool,
+	) (shwap.RangeNamespaceData, error)
 }
 
 // API is a wrapper around Module for the RPC.
@@ -91,9 +83,11 @@ type API struct {
 		) (shwap.NamespaceData, error) `perm:"read"`
 		GetRange func(
 			ctx context.Context,
+			ns libshare.Namespace,
 			height uint64,
-			start, end int,
-		) (*GetRangeResult, error) `perm:"read"`
+			from, to shwap.SampleCoords,
+			proofsOnly bool,
+		) (shwap.RangeNamespaceData, error) `perm:"read"`
 	}
 }
 
@@ -119,8 +113,10 @@ func (api *API) GetRow(ctx context.Context, height uint64, rowIdx int) (shwap.Ro
 	return api.Internal.GetRow(ctx, height, rowIdx)
 }
 
-func (api *API) GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) {
-	return api.Internal.GetRange(ctx, height, start, end)
+func (api *API) GetRange(
+	ctx context.Context, ns libshare.Namespace, height uint64, from, to shwap.SampleCoords, proofsOnly bool,
+) (shwap.RangeNamespaceData, error) {
+	return api.Internal.GetRange(ctx, ns, height, from, to, proofsOnly)
 }
 
 func (api *API) GetNamespaceData(
@@ -175,25 +171,18 @@ func (m module) SharesAvailable(ctx context.Context, height uint64) error {
 	return m.avail.SharesAvailable(ctx, header)
 }
 
-func (m module) GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) {
-	extendedDataSquare, err := m.GetEDS(ctx, height)
-	if err != nil {
-		return nil, err
-	}
-
-	proof, err := eds.ProveShares(extendedDataSquare, start, end)
-	if err != nil {
-		return nil, err
-	}
-
-	shares, err := libshare.FromBytes(extendedDataSquare.FlattenedODS()[start:end])
+func (m module) GetRange(
+	ctx context.Context,
+	ns libshare.Namespace,
+	height uint64,
+	from, to shwap.SampleCoords,
+	proofsOnly bool,
+) (shwap.RangeNamespaceData, error) {
+	header, err := m.hs.GetByHeight(ctx, height)
 	if err != nil {
-		return nil, err
+		return shwap.RangeNamespaceData{}, err
 	}
-	return &GetRangeResult{
-		Shares: shares,
-		Proof:  proof,
-	}, nil
+	return m.getter.GetRangeNamespaceData(ctx, header, ns, from, to, proofsOnly)
 }
 
 func (m module) GetNamespaceData(

nodebuilder/tests/share_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 func TestShareModule(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
 	t.Cleanup(cancel)
 	sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second*1))
 	blobSize := 128
@@ -44,6 +44,7 @@ func TestShareModule(t *testing.T) {
 	require.NoError(t, fullNode.Start(ctx))
 
 	addrsFull, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(fullNode.Host))
+
 	require.NoError(t, err)
 
 	lightCfg := sw.DefaultTestConfig(node.Light)
@@ -207,7 +208,36 @@ func TestShareModule(t *testing.T) {
 					require.NoError(t, err)
 					// compare commitments
 					require.Equal(t, nodeBlob[0].Commitment, blb[0].Commitment)
+				}
+			},
+		},
+		{
+			name: "GetRangeNamespaceData",
+			doFn: func(t *testing.T) {
+				dah := hdr.DAH
+				blobLength, err := sampledBlob.Length()
+				require.NoError(t, err)
+				from, to, err := shwap.RangeCoordsFromIdx(sampledBlob.Index(), blobLength, len(dah.RowRoots))
+				require.NoError(t, err)
+				for _, client := range clients {
+					rng, err := client.Share.GetRange(ctx, nodeBlob[0].Namespace(), height, from, to, false)
+					require.NoError(t, err)
+					err = rng.Verify(nodeBlob[0].Namespace(), from, to, dah.Hash())
+					require.NoError(t, err)
+
+					shrs := rng.Flatten()
+					blbs, err := libshare.ParseBlobs(shrs)
+					require.NoError(t, err)
 
+					parsedBlob, err := blob.ToNodeBlobs(blbs...)
+					require.NoError(t, err)
+					require.Equal(t, nodeBlob[0].Commitment, parsedBlob[0].Commitment)
+
+					rngProofsOnly, err := client.Share.GetRange(ctx, nodeBlob[0].Namespace(), height, from, to, true)
+					require.NoError(t, err)
+					assert.Empty(t, rngProofsOnly.Shares)
+					err = rngProofsOnly.VerifyShares(rng.Shares, nodeBlob[0].Namespace(), from, to, dah.Hash())
+					require.NoError(t, err)
 				}
 			},
 		},

share/availability/light/availability_test.go

@@ -321,6 +321,16 @@ func (g successGetter) GetNamespaceData(
 	panic("not implemented")
 }
 
+func (g successGetter) GetRangeNamespaceData(
+	_ context.Context,
+	_ *header.ExtendedHeader,
+	_ libshare.Namespace,
+	_, _ shwap.SampleCoords,
+	_ bool,
+) (shwap.RangeNamespaceData, error) {
+	panic("not implemented")
+}
+
 func TestPruneAll(t *testing.T) {
 	const size = 8
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)

share/eds/accessor.go

@@ -33,6 +33,13 @@ type Accessor interface {
 	RowNamespaceData(ctx context.Context, namespace libshare.Namespace, rowIdx int) (shwap.RowNamespaceData, error)
 	// Shares returns data (ODS) shares extracted from the Accessor.
 	Shares(ctx context.Context) ([]libshare.Share, error)
+	// RangeNamespaceData returns data(ODS) shares along with their proofs from the requested range
+	// from the Accessor. Response might have only proofs.
+	RangeNamespaceData(
+		ctx context.Context,
+		ns libshare.Namespace,
+		from, to shwap.SampleCoords,
+	) (shwap.RangeNamespaceData, error)
 }
 
 // AccessorStreamer is an interface that groups Accessor and Streamer interfaces.

share/eds/close_once.go

@@ -93,6 +93,17 @@ func (c *closeOnce) Shares(ctx context.Context) ([]libshare.Share, error) {
 	return c.f.Shares(ctx)
 }
 
+func (c *closeOnce) RangeNamespaceData(
+	ctx context.Context,
+	ns libshare.Namespace,
+	from, to shwap.SampleCoords,
+) (shwap.RangeNamespaceData, error) {
+	if c.closed.Load() {
+		return shwap.RangeNamespaceData{}, errAccessorClosed
+	}
+	return c.f.RangeNamespaceData(ctx, ns, from, to)
+}
+
 func (c *closeOnce) Reader() (io.Reader, error) {
 	if c.closed.Load() {
 		return nil, errAccessorClosed

share/eds/close_once_test.go

@@ -75,6 +75,14 @@ func (s *stubEdsAccessorCloser) RowNamespaceData(
 	return shwap.RowNamespaceData{}, nil
 }
 
+func (s *stubEdsAccessorCloser) RangeNamespaceData(
+	_ context.Context,
+	_ libshare.Namespace,
+	_, _ shwap.SampleCoords,
+) (shwap.RangeNamespaceData, error) {
+	return shwap.RangeNamespaceData{}, nil
+}
+
 func (s *stubEdsAccessorCloser) Shares(context.Context) ([]libshare.Share, error) {
 	return nil, nil
 }

share/eds/proofs_cache.go

@@ -225,7 +225,6 @@ func (c *proofsCache) RowNamespaceData(
 	if err != nil {
 		return shwap.RowNamespaceData{}, fmt.Errorf("shares by namespace %s for row %v: %w", namespace.String(), rowIdx, err)
 	}
-
 	return shwap.RowNamespaceData{
 		Shares: row,
 		Proof:  proof,
@@ -255,6 +254,64 @@ func (c *proofsCache) Shares(ctx context.Context) ([]libshare.Share, error) {
 	return shares, nil
 }
 
+// RangeNamespaceData tries to find all complete rows in cache. For all incomplete rows,
+// it uses the inner accessor to build the namespace data
+func (c *proofsCache) RangeNamespaceData(
+	ctx context.Context,
+	ns libshare.Namespace,
+	from, to shwap.SampleCoords,
+) (shwap.RangeNamespaceData, error) {
+	odsSize := c.Size(ctx) / 2
+
+	roots, err := c.AxisRoots(ctx)
+	if err != nil {
+		return shwap.RangeNamespaceData{}, fmt.Errorf("accessing axis roots: %w", err)
+	}
+	rngdata := shwap.RangeNamespaceData{
+		Start:  from.Row,
+		Shares: make([][]libshare.Share, 0, to.Row-from.Row+1),
+		Proof:  make([]*shwap.Proof, 0, to.Row-from.Row+1),
+	}
+
+	// iterate over each row in the range [from.Row; to.Row].
+	// All complete rows(from.Col = 0 && to.Col = odsSize-1) is
+	// requested using `RowNamespaceData` that uses cache.
+	// Other cases are handled using `RangeNamespaceData` as these rows are incomplete.
+	for row := from.Row; row <= to.Row; row++ {
+		startCol := from.Col
+		endCol := to.Col
+		if row != to.Row {
+			endCol = odsSize - 1
+		}
+
+		if startCol == 0 && endCol == odsSize-1 {
+			// request full row using RowNamespaceData
+			rowData, err := c.RowNamespaceData(ctx, ns, row)
+			if err != nil {
+				return shwap.RangeNamespaceData{}, err
+			}
+			rngdata.Shares = append(rngdata.Shares, rowData.Shares)
+			rngdata.Proof = append(rngdata.Proof, shwap.NewProof(row, rowData.Proof, roots))
+			continue
+		}
+
+		// Otherwise, fetch the partial range
+		data, err := c.inner.RangeNamespaceData(
+			ctx, ns,
+			shwap.SampleCoords{Row: row, Col: startCol},
+			shwap.SampleCoords{Row: row, Col: endCol},
+		)
+		if err != nil {
+			return shwap.RangeNamespaceData{}, err
+		}
+		rngdata.Shares = append(rngdata.Shares, data.Shares[0])
+		rngdata.Proof = append(rngdata.Proof, data.Proof[0])
+		// Reset column for subsequent rows
+		from.Col = 0
+	}
+	return rngdata, nil
+}
+
 func (c *proofsCache) Reader() (io.Reader, error) {
 	odsSize := c.Size(context.TODO()) / 2
 	reader := NewShareReader(odsSize, c.getShare)

share/eds/proofs_cache_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestCache(t *testing.T) {
 	ODSSize := 16
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	t.Cleanup(cancel)
 
 	newAccessor := func(tb testing.TB, inner *rsmt2d.ExtendedDataSquare) Accessor {

share/eds/rsmt2d.go

@@ -116,6 +116,29 @@ func (eds *Rsmt2D) RowNamespaceData(
 	return shwap.RowNamespaceDataFromShares(sh, namespace, rowIdx)
 }
 
+// RangeNamespaceData builds a namespace range from the given coordinates and the length of the
+// range.
+func (eds *Rsmt2D) RangeNamespaceData(
+	ctx context.Context,
+	ns libshare.Namespace,
+	from, to shwap.SampleCoords,
+) (shwap.RangeNamespaceData, error) {
+	rawShares := make([][]libshare.Share, 0, to.Row-from.Row+1)
+	for row := from.Row; row <= to.Row; row++ {
+		rawShare := eds.Row(uint(row))
+		sh, err := libshare.FromBytes(rawShare)
+		if err != nil {
+			return shwap.RangeNamespaceData{}, err
+		}
+		rawShares = append(rawShares, sh)
+	}
+	roots, err := eds.AxisRoots(ctx)
+	if err != nil {
+		return shwap.RangeNamespaceData{}, err
+	}
+	return shwap.RangedNamespaceDataFromShares(rawShares, ns, roots, from, to)
+}
+
 // Shares returns data (ODS) shares extracted from the EDS. It returns new copy of the shares each
 // time.
 func (eds *Rsmt2D) Shares(_ context.Context) ([]libshare.Share, error) {