Merge pull request #1 from Dev43/feat/batcher

Feat/batcher

Author: Dev43

README.md

@@ -5,50 +5,187 @@
 [![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](https://godoc.org/github.com/Dev43/arweave-go)
 [![Go Report Card](https://goreportcard.com/badge/github.com/Dev43/arweave-go)](https://goreportcard.com/report/github.com/Dev43/arweave-go)
 
-Golang SDK for the Arweave client.
+Golang Client to interact with the Arweave Blockchain.
 
-Example of use:
+## Usage
+
+### Wallet
+
+In the current version, you can load the Arweave wallet file created from the Arweave server or the plugin.
 
 ```golang
+// create a new wallet instance
+w := wallet.NewWallet()
+// extract the key from the wallet instance
+err = w.LoadKeyFromFile("./arweave.json")
+if err != nil {
+	//...
+}
+```
+
+You can directly load the key by using it's filepath or pass it as an array of bytes using `LoadKey([]byte)`.
+
+With the wallet struct, you can sign and verify a message:
+
+```golang
+// sign the message "example"
+msg := []byte("example")
+sig, err := w.Sign(msg))
+if err != nil {
+	//...
+}
+err = w.Verify(msg, sig)
+if err != nil {
+	// message signature is not valid...
+}
+// message signature is valid
+```
+
+### API
+
+You can call all of the Arweave HTTP api endpoints using the api package. First you must give it the node url or IP address.
+
+```golang
+ipAddress := "127.0.0.1"
+c, err := api.Dial(ipAddress)
+if err != nil {
+	// problem connecting
+}
+```
+
+To call the endpoints, you will need to pass in a context.
+
+```golang
+c.GetBalance(context.TODO(), "1seRanklLU_1VTGkEk7P0xAwMJfA7owA1JHW5KyZKlY")
+```
 
-	// create a new arweave client
-    ar, err := transactor.NewTransactor("209.97.142.169")
-    if err != nil {
-        log.Fatal(err)
-    }
+### Transactions
+
+To create a new transaction, you will need to interact with the `transactor` package. The transactor package has 3 main functions, creating, sending and waiting for a transaction.
+
+```golang
+	// create a new transactor client
+	ar, err := transactor.NewTransactor("127.0.0.1")
+	if err != nil {
+		//...
+	}
 
 	// create a new wallet instance
 	w := wallet.NewWallet()
 	// extract the key from the wallet instance
-	err = w.ExtractKey("./arweave.json")
+	err = w.LoadKeyFromFile("./arweave.json")
 	if err != nil {
-		log.Fatal(err)
+		//...
 	}
 	// create a transaction
-	txBuilder, err := ar.CreateTransaction(context.TODO(), w, "0", []byte(""), "xblmNxr6cqDT0z7QIWBCo8V0UfJLd3CRDffDhF5Uh9g")
+	txBuilder, err := ar.CreateTransaction(context.TODO(), w, "0", []byte(""), "1seRanklLU_1VTGkEk7P0xAwMJfA7owA1JHW5KyZKlY")
 	if err != nil {
-		log.Fatal(err)
+		//...
 	}
-
+	
 	// sign the transaction
 	txn, err := txBuilder.Sign(w)
 	if err != nil {
-		log.Fatal(err)
+		//...
 	}
 
 	// send the transaction
 	resp, err := ar.SendTransaction(context.TODO(), txn)
 	if err != nil {
-		log.Fatal(err)
+		//...
 	}
 
-	fmt.Println(resp)
-
 	// wait for the transaction to get mined
-	pendingTx, err := ar.WaitMined(context.TODO(), txn)
+	finalTx, err := ar.WaitMined(context.TODO(), txn)
+	if err != nil {
+		//...
+	}
+	// get the hash of the transaction
+	fmt.Println(finalTx.Hash())
+```
+
+### Batching, chunking and recombining
+
+Arweave currently has a 3MB transaction limit. To let you be able to upload more than 3MB of data, this library lets you chunk your data into multiples chunks, and upload these chunks to the weave. These chunks are all backlinked to their previous chunk (except the first one) so all you need to use to retrieve all the chunks is the last chunk's address (the "tip" of this linked list).
+
+So far the library has been tested with Tarred and Gzipped files (it's a good idea to compress your files, you'll be paying less fees to the network!)
+
+#### Creating chunks
+
+To create chunks, use the `chunker` package. Currently the chunker takes 500KB chunks, encodes it to the same encoding as the Arweave (Raw Base64 URL encoding) and adds it to a transaction.
+
+```golang
+	f, err := os.Open("example.tar.gz")
 	if err != nil {
-		log.Fatal(err)
+		//...
 	}
-	fmt.Println(pendingTx)
+	info, err := f.Stat()
+	if err != nil {
+		//...
+	}
+	chunker, err := chunker.NewChunker(f, info.Size())
+	if err != nil {
+		//...
+	}
+	chunks, err := chunker.ChunkAll()
+	if err != nil {
+		//...
+	}
+```
+
+#### Sending a batch of transactions
+
+To directly do both the chunking and the batch sending of transaction, you can use the `batchchunker` package. The package needs to wait for a single transaction to be mined  before sending a new one. This will take a significant amount of time as the Arweave has a ~2 minute block time.
+
+```golang
+	f, err := os.Open("example.tar.gz")
+	if err != nil {
+		//...
+	}
+	info, err := f.Stat()
+	if err != nil {
+		//...
+	}
+
+	// creates a new batch
+	newB := batchchunker.NewBatch(ar, w, f, info.Size())
+
+	// sends all the transactions
+	list, err := newB.SendBatchTransaction()
+	if err != nil {
+		//...
+	}
+```
+
+#### Recombining
+
+Recombining all the chunks needs to be done from the tip of the batch chain (the last transaction sent). The function will grab all the chunks and recombine it into an io.Writer.
+
+```golang
+	// in this example, we will save the data as a file
+	f, err := os.Create("example.tar.gz")
+	if err != nil {
+		//...
+	}
+	// create a new combiner
+	newBCombiner := combiner.NewBatchCombiner(ar.Client)
 
+	// grab all the chunks starting from chunk at this address
+	liveChunks, err := newBCombiner.GetAllChunks("fBwTnPGNSAtHTNr6pGvqiYpLmBvTjjJOu3kBxuuit1c")
+	if err != nil {
+		//...
+	}
+	// recombine all the chunks into our file
+	err = combiner.Recombine(liveChunks, f)
+	if err != nil {
+		//...
+	}
 ```
+
+If you actually run the last example, it will retrieve a folder with the Bitcoin, Ethereum and Arweave whitepaper
+
+
+If you enjoy the library, please consider donating:
+
+- **Arweave Address**: `pfJXiTwwjQwSJF9VT1ZK6kauvobWuKKLUzjz29R1gbQ`
+- **Ethereum Address**: `0x3E42b8b399dca71b5c004921Fc6eFfa8dDc9409d`

batchchunker/batchchunker.go

@@ -0,0 +1,102 @@
+package batchchunker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/Dev43/arweave-go"
+	"github.com/Dev43/arweave-go/chunker"
+	"github.com/Dev43/arweave-go/transactor"
+	"github.com/Dev43/arweave-go/wallet"
+)
+
+const chunkerVersion = "0.0.1"
+
+// BatchMaker struct
+type BatchMaker struct {
+	ar        *transactor.Transactor
+	wallet    *wallet.Wallet
+	reader    io.Reader
+	totalSize int64
+}
+
+// ChunkInformation is the extra data we add to the tags to inform us about the last chunk, it's position, whether it's the
+// head of the chunk and it's data version
+type ChunkInformation struct {
+	PreviousChunk string `json:"previous_chunk"`
+	IsHead        bool   `json:"is_head"`
+	Version       string `json:"version"`
+	Position      int64  `json:"position"`
+}
+
+// NewBatch creates a NewBatch struct
+func NewBatch(ar *transactor.Transactor, w *wallet.Wallet, reader io.Reader, totalSize int64) *BatchMaker {
+	return &BatchMaker{
+		ar:        ar,
+		wallet:    w,
+		reader:    reader,
+		totalSize: totalSize,
+	}
+}
+
+// SendBatchTransaction chunks, sends and waits for all the transactions to get mined
+func (b *BatchMaker) SendBatchTransaction() ([]string, error) {
+	txList := []string{}
+	ch, err := chunker.NewChunker(b.reader, b.totalSize)
+	if err != nil {
+		return nil, err
+	}
+	for i := int64(0); i < ch.TotalChunks(); i++ {
+		chunk, err := ch.Next()
+		if err != nil {
+			return nil, err
+		}
+		data, err := json.Marshal(chunk)
+		if err != nil {
+			return nil, err
+		}
+
+		txBuilder, err := b.ar.CreateTransaction(context.TODO(), b.wallet, "0", data, "")
+		if err != nil {
+			return nil, err
+		}
+		previousChunk := ""
+		if len(txList) > 0 {
+			previousChunk = txList[len(txList)-1]
+		}
+		isHead := false
+		if i+1 == ch.TotalChunks() {
+			isHead = true
+		}
+		chunkerInfo := ChunkInformation{
+			PreviousChunk: previousChunk,
+			IsHead:        isHead,
+			Version:       chunkerVersion,
+			Position:      chunk.Position,
+		}
+		tagValue, err := json.Marshal(chunkerInfo)
+		if err != nil {
+			return nil, err
+		}
+		txBuilder.AddTag(arweave.BatchChunkerAppName, string(tagValue))
+		tx, err := txBuilder.Sign(b.wallet)
+		if err != nil {
+			return nil, err
+		}
+		resp, err := b.ar.SendTransaction(context.TODO(), tx)
+		if err != nil {
+			return nil, err
+		}
+		fmt.Println(resp)
+		minedTx, err := b.ar.WaitMined(context.TODO(), tx)
+		txList = append(txList, minedTx.Hash())
+		fmt.Printf("Successfully sent transaction %d/%d with hash %s \n", chunk.Position+1, ch.TotalChunks(), minedTx.Hash())
+
+	}
+	fmt.Printf("Successfully sent batch transactions with head transaction %s and list of transactions: \n - %s \n", txList[len(txList)-1], strings.Join(txList, "\n - "))
+
+	return txList, nil
+}