Experimental as default (#67)

Author: tarassh

.vscode/launch.json

@@ -17,7 +17,7 @@
       "request": "launch",
       "mode": "auto",
       "program": "${workspaceFolder}/cmd/pinner/main.go",
-      "buildFlags": ["-tags", "experimental"],
+      // "buildFlags": ["-tags", "experimental"],
       "args": [
         "--w3-agent-key",
         "MgCY1PrWhMRwMBFuRpZMX2Ds7NyoHPnrDY3hrd54kWpS3ue0BR363tSycSPmI1YiaNkek4QXCsUjZ1cx80NcmuiHbsYs=", // sample key - change to your key

README.md

@@ -58,12 +58,6 @@ Build binaries:
 make
 ```
 
-Note: To build das-pinner with experimental features (stacking erasure coded cells together), run:
-  
-```sh
-make build-pinner EXTRA_TAGS="-tags experimental"
-```
-
 The binaries will be built in the `bin` directory.
 
 ### Clean Up

internal/pinner/ipfs-node/extract.go

@@ -1,5 +1,3 @@
-//go:build !experimental
-
 package ipfsnode
 
 import (
@@ -32,26 +30,26 @@ func (ipfsNode *IPFSNode) ExtractData(ctx context.Context, cidStr string) ([]byt
 
 	// Start processing each root link in parallel
 	for i, link := range root.Links {
-		// wg.Add(1)
-		// go func(i int, link internal.Link) {
-		// defer wg.Done()
-
-		// Fetch the next set of links (128 cells per link)
-		var blobLinks []internal.Link
-		if err := ipfsNode.GetData(ctx, link.CID, &blobLinks); err != nil {
-			select {
-			case errorChan <- err:
-			default:
+		wg.Add(1)
+		go func(i int, link internal.Link) {
+			defer wg.Done()
+
+			// Fetch the next set of links (128 cells per link)
+			var blobLinks []internal.Link
+			if err := ipfsNode.GetData(ctx, link.CID, &blobLinks); err != nil {
+				select {
+				case errorChan <- err:
+				default:
+				}
+				return
 			}
-			// return
-		}
 
-		// Download up to 64 cells from the blob links
-		err := downloadCells(ctx, byteCells, ipfsNode, i, blobLinks, errorChan, 64)
-		if err != nil {
-			// return
-		}
-		// }(i, link)
+			// Download up to 64 cells from the blob links
+			err := downloadCells(ctx, byteCells, ipfsNode, i, blobLinks, errorChan, 64)
+			if err != nil {
+				return
+			}
+		}(i, link)
 	}
 
 	// Goroutine to close error channel when all downloads are done
@@ -65,13 +63,14 @@ func (ipfsNode *IPFSNode) ExtractData(ctx context.Context, cidStr string) ([]byt
 	case err := <-errorChan:
 		// If an error occurs, cancel the remaining operations
 		cancel()
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		// If no error, combine them into a block
+		return combineDownloadedCells(root, byteCells)
 	case <-ctx.Done():
 		// If the context is canceled, return an error
 		return nil, errors.New("context canceled")
-	case <-errorChan:
-		// All downloads completed successfully, combine them into a block
-		return combineDownloadedCells(root, byteCells)
 	}
 }
 
@@ -103,16 +102,20 @@ func downloadCells(ctx context.Context, byteCells [][][]byte, ipfsNode *IPFSNode
 			}
 
 			// Allocate space for each byte slice within the cell
-			cellBytes := make([]byte, len(cell.Cell.Nested.Bytes))
+			cellBytes := make([][2048]byte, len(cell.Cell.Nested.Bytes)/2048)
 
 			mu.Lock()
 			defer mu.Unlock()
 
 			// Insert the cell at the correct index and increment the count
 			if count < limit {
-				copy(cellBytes, cell.Cell.Nested.Bytes)
-				byteCells[blobIndex][i] = cellBytes
-				count++
+				for z := 0; z < len(cellBytes); z++ {
+					copy(cellBytes[z][:], cell.Cell.Nested.Bytes[z*2048:(z+1)*2048])
+					log.Debugf("Downloaded blob [%3d] cell [%3d] byte [%3d] stackSize", blobIndex, i, z, internal.StackSize)
+					byteCells[blobIndex][i*internal.StackSize+z] = cellBytes[z][:]
+
+					count++
+				}
 
 				log.Infof("Downloaded blob [%3d] cell [%3d] total [%3d/%3d]", blobIndex, i, count, limit)
 			}

internal/pinner/ipfs-node/extract_experimental.go

@@ -1,5 +1,3 @@
-//go:build !experimental
-
 package ipldencoder
 
 import (
@@ -59,36 +57,47 @@ func (b *IPLDDataBlock) Encode(block internal.DataBlock) error {
 
 func (b *IPLDDataBlock) encodeDataNodes(block internal.DataBlock) error {
 	_, nBlobs, nCells := block.Describe()
+	cellsInNode := nCells / internal.StackSize
 
 	b.DataNodes = make([][]datamodel.Node, nBlobs)
 	for nBlob := uint64(0); nBlob < nBlobs; nBlob++ {
-		b.DataNodes[nBlob] = make([]datamodel.Node, nCells)
+		b.DataNodes[nBlob] = make([]datamodel.Node, cellsInNode)
 
-		for nCell := uint64(0); nCell < nCells; nCell++ {
-			proof, cell, err := block.ProofAndCell(nBlob, nCell)
-			if err != nil {
-				return err
+		for nCellStack := uint64(0); nCellStack < cellsInNode; nCellStack++ {
+			var stackedProof []byte
+			var stackedCell []byte
+
+			for i := uint64(0); i < internal.StackSize; i++ {
+				proof, cell, err := block.ProofAndCell(nBlob, nCellStack*internal.StackSize+i)
+				if err != nil {
+					return err
+				}
+
+				stackedProof = append(stackedProof, proof...)
+				stackedCell = append(stackedCell, cell...)
 			}
 
-			node, err := createCellNode(proof, cell)
+			node, err := createCellNode(stackedProof, stackedCell)
 			if err != nil {
 				return err
 			}
 
-			b.DataNodes[nBlob][nCell] = node
+			b.DataNodes[nBlob][nCellStack] = node
 		}
 	}
 
 	return nil
 }
 
 func (b *IPLDDataBlock) encodeLinks(lsys *linking.LinkSystem, block internal.DataBlock) error {
-	_, nBlobs, nCells := block.Describe()
+	_, nBlobs, _ := block.Describe()
 
 	b.Links = make([][]datamodel.Link, nBlobs)
 	for i := uint64(0); i < nBlobs; i++ {
+		nCells := len(b.DataNodes[i])
+
 		b.Links[i] = make([]datamodel.Link, nCells)
-		for j := uint64(0); j < nCells; j++ {
+		for j := 0; j < nCells; j++ {
 			link, err := b.createLink(lsys, b.DataNodes[i][j])
 			if err != nil {
 				return err
@@ -124,8 +133,8 @@ func (b *IPLDDataBlock) encodeRoot(lsys *linking.LinkSystem, block internal.Data
 
 	// Create the root DAG-CBOR object
 	rootNode, err := qp.BuildMap(basicnode.Prototype.Map, -1, func(ma datamodel.MapAssembler) {
-		qp.MapEntry(ma, "version", qp.String("v0.1.0"))
-		qp.MapEntry(ma, "length", qp.Int(int64(nCell)))
+		qp.MapEntry(ma, "version", qp.String("v0.2.0"))
+		qp.MapEntry(ma, "length", qp.Int(int64(nCell/internal.StackSize)))
 		qp.MapEntry(ma, "size", qp.Int(int64(size)))
 		qp.MapEntry(ma, "commitments", qp.List(int64(nBlob), func(la ipld.ListAssembler) {
 			for i := uint64(0); i < nBlob; i++ {