Merge pull request #280 from ddvk/sync15concurrency

sync15: use parallel requests when getting remote tree

Author: ddvk

README.md

@@ -209,3 +209,4 @@ rMAPI will set the exit code to `0` if the command succeedes, or `1` if it fails
 - `RMAPI_AUTH`: override the default authorization url
 - `RMAPI_DOC`: override the default document storage url
 - `RMAPI_HOST`: override all urls 
+- `RMAPI_CONCURRENT`: sync15: maximum number of goroutines/http requests to use (default: 20)

api/api.go

@@ -24,6 +24,7 @@ type ApiCtx interface {
 	DeleteEntry(node *model.Node) error
 	SyncComplete() error
 	Nuke() error
+	Refresh() error
 }
 
 type UserToken struct {

api/sync10/api.go

@@ -26,6 +26,10 @@ func (ctx *ApiCtx) Filetree() *filetree.FileTreeCtx {
 	return ctx.ft
 }
 
+func (ctx *ApiCtx) Refresh() (err error) {
+	return errors.New("not implemented")
+}
+
 // Nuke removes all documents from the account
 func (ctx *ApiCtx) Nuke() error {
 	documents := make([]model.Document, 0)

api/sync15/apictx.go

@@ -6,8 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/google/uuid"
@@ -19,8 +19,6 @@ import (
 	"github.com/juruen/rmapi/util"
 )
 
-var ErrorNotImplemented = errors.New("not implemented")
-
 // An ApiCtx allows you interact with the remote reMarkable API
 type ApiCtx struct {
 	Http        *transport.HttpClientCtx
@@ -29,10 +27,45 @@ type ApiCtx struct {
 	hashTree    *HashTree
 }
 
+// max number of concurrent requests
+var concurrent = 20
+
+func init() {
+	c := os.Getenv("RMAPI_CONCURRENT")
+	if u, err := strconv.Atoi(c); err == nil {
+		concurrent = u
+	}
+}
+
+func CreateCtx(http *transport.HttpClientCtx) (*ApiCtx, error) {
+	apiStorage := NewBlobStorage(http)
+	cacheTree, err := loadTree()
+	if err != nil {
+		fmt.Print(err)
+		return nil, err
+	}
+	err = cacheTree.Mirror(apiStorage, concurrent)
+	if err != nil {
+		return nil, err
+	}
+	saveTree(cacheTree)
+	tree := DocumentsFileTree(cacheTree)
+	return &ApiCtx{http, tree, apiStorage, cacheTree}, nil
+}
+
 func (ctx *ApiCtx) Filetree() *filetree.FileTreeCtx {
 	return ctx.ft
 }
 
+func (ctx *ApiCtx) Refresh() error {
+	err := ctx.hashTree.Mirror(ctx.blobStorage, concurrent)
+	if err != nil {
+		return err
+	}
+	ctx.ft = DocumentsFileTree(ctx.hashTree)
+	return nil
+}
+
 // Nuke removes all documents from the account
 func (ctx *ApiCtx) Nuke() (err error) {
 	err = Sync(ctx.blobStorage, ctx.hashTree, func(t *HashTree) error {
@@ -54,7 +87,7 @@ func (ctx *ApiCtx) FetchDocument(docId, dstPath string) error {
 		return err
 	}
 
-	tmp, err := ioutil.TempFile("", "rmapizip")
+	tmp, err := os.CreateTemp("", "rmapizip")
 
 	if err != nil {
 		log.Error.Println("failed to create tmpfile for zip dir", err)
@@ -104,7 +137,7 @@ func (ctx *ApiCtx) CreateDir(parentId, name string, notify bool) (*model.Documen
 
 	files := &archive.DocumentFiles{}
 
-	tmpDir, err := ioutil.TempDir("", "rmupload")
+	tmpDir, err := os.MkdirTemp("", "rmupload")
 	if err != nil {
 		return nil, err
 	}
@@ -219,10 +252,11 @@ func Sync(b *BlobStorage, tree *HashTree, operation func(t *HashTree) error) err
 
 		log.Info.Println("wrong generation, re-reading remote tree")
 		//resync and try again
-		err = tree.Mirror(b)
+		err = tree.Mirror(b, concurrent)
 		if err != nil {
 			return err
 		}
+		log.Warning.Println("remote tree has changed, refresh the file tree")
 	}
 	return saveTree(tree)
 }
@@ -258,10 +292,10 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
 		if err != nil {
 			return err
 		}
-		doc.MetadataFile.Version += 1
-		doc.MetadataFile.DocName = name
-		doc.MetadataFile.Parent = dstDir.Id()
-		doc.MetadataFile.MetadataModified = true
+		doc.Metadata.Version += 1
+		doc.Metadata.DocName = name
+		doc.Metadata.Parent = dstDir.Id()
+		doc.Metadata.MetadataModified = true
 
 		hashStr, reader, err := doc.MetadataHashAndReader()
 		if err != nil {
@@ -306,7 +340,7 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
 		return nil, err
 	}
 
-	return &model.Node{d.ToDocument(), src.Children, dstDir}, nil
+	return &model.Node{Document: d.ToDocument(), Children: src.Children, Parent: dstDir}, nil
 }
 
 // UploadDocument uploads a local document given by sourceDocPath under the parentId directory
@@ -324,7 +358,7 @@ func (ctx *ApiCtx) UploadDocument(parentId string, sourceDocPath string, notify
 
 	var err error
 
-	tmpDir, err := ioutil.TempDir("", "rmupload")
+	tmpDir, err := os.MkdirTemp("", "rmupload")
 	if err != nil {
 		return nil, err
 	}
@@ -391,33 +425,14 @@ func (ctx *ApiCtx) UploadDocument(parentId string, sourceDocPath string, notify
 	return doc.ToDocument(), nil
 }
 
-func CreateCtx(http *transport.HttpClientCtx) (*ApiCtx, error) {
-	apiStorage := &BlobStorage{http}
-	cacheTree, err := loadTree()
-	if err != nil {
-		fmt.Print(err)
-		return nil, err
-	}
-	err = cacheTree.Mirror(apiStorage)
-	if err != nil {
-		return nil, err
-	}
-	saveTree(cacheTree)
-	tree, err := DocumentsFileTree(cacheTree)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch document tree %v", err)
-	}
-	return &ApiCtx{http, tree, apiStorage, cacheTree}, nil
-}
-
 // DocumentsFileTree reads your remote documents and builds a file tree
 // structure to represent them
-func DocumentsFileTree(tree *HashTree) (*filetree.FileTreeCtx, error) {
+func DocumentsFileTree(tree *HashTree) *filetree.FileTreeCtx {
 
 	documents := make([]*model.Document, 0)
 	for _, d := range tree.Docs {
 		//dont show deleted (already cached)
-		if d.Deleted {
+		if d.Metadata.Deleted {
 			continue
 		}
 		doc := d.ToDocument()
@@ -434,7 +449,7 @@ func DocumentsFileTree(tree *HashTree) (*filetree.FileTreeCtx, error) {
 		log.Trace.Println(d.Name(), d.IsFile())
 	}
 
-	return &fileTree, nil
+	return &fileTree
 }
 
 // SyncComplete notfies that somethings has changed (triggers tablet sync)

api/sync15/blobdoc.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"io/ioutil"
 	"sort"
 	"strconv"
 	"strings"
@@ -22,12 +21,12 @@ import (
 type BlobDoc struct {
 	Files []*Entry
 	Entry
-	archive.MetadataFile
+	Metadata archive.MetadataFile
 }
 
 func NewBlobDoc(name, documentId, colType, parentId string) *BlobDoc {
 	return &BlobDoc{
-		MetadataFile: archive.MetadataFile{
+		Metadata: archive.MetadataFile{
 			DocName:        name,
 			CollectionType: colType,
 			LastModified:   archive.UnixTimestamp(),
@@ -52,7 +51,7 @@ func (d *BlobDoc) Rehash() error {
 }
 
 func (d *BlobDoc) MetadataHashAndReader() (hash string, reader io.Reader, err error) {
-	jsn, err := json.Marshal(d.MetadataFile)
+	jsn, err := json.Marshal(d.Metadata)
 	if err != nil {
 		return
 	}
@@ -121,7 +120,7 @@ func (d *BlobDoc) ReadMetadata(fileEntry *Entry, r RemoteStorage) error {
 			return err
 		}
 		defer meta.Close()
-		content, err := ioutil.ReadAll(meta)
+		content, err := io.ReadAll(meta)
 		if err != nil {
 			return err
 		}
@@ -130,7 +129,7 @@ func (d *BlobDoc) ReadMetadata(fileEntry *Entry, r RemoteStorage) error {
 			log.Error.Printf("cannot read metadata %s %v", fileEntry.DocumentID, err)
 		}
 		log.Trace.Println("name from metadata: ", metadata.DocName)
-		d.MetadataFile = metadata
+		d.Metadata = metadata
 	}
 
 	return nil
@@ -208,10 +207,10 @@ func (d *BlobDoc) Mirror(e *Entry, r RemoteStorage) error {
 }
 func (d *BlobDoc) ToDocument() *model.Document {
 	var lastModified string
-	unixTime, err := strconv.ParseInt(d.MetadataFile.LastModified, 10, 64)
+	unixTime, err := strconv.ParseInt(d.Metadata.LastModified, 10, 64)
 	if err == nil {
 		//HACK: convert wrong nano timestamps to millis
-		if len(d.MetadataFile.LastModified) > 18 {
+		if len(d.Metadata.LastModified) > 18 {
 			unixTime /= 1000000
 		}
 
@@ -220,11 +219,11 @@ func (d *BlobDoc) ToDocument() *model.Document {
 	}
 	return &model.Document{
 		ID:             d.DocumentID,
-		VissibleName:   d.MetadataFile.DocName,
-		Version:        d.MetadataFile.Version,
-		Parent:         d.MetadataFile.Parent,
-		Type:           d.MetadataFile.CollectionType,
-		CurrentPage:    d.MetadataFile.LastOpenedPage,
+		VissibleName:   d.Metadata.DocName,
+		Version:        d.Metadata.Version,
+		Parent:         d.Metadata.Parent,
+		Type:           d.Metadata.CollectionType,
+		CurrentPage:    d.Metadata.LastOpenedPage,
 		ModifiedClient: lastModified,
 	}
 }

api/sync15/blobstorage.go

@@ -3,7 +3,6 @@ package sync15
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/juruen/rmapi/config"
@@ -13,7 +12,14 @@ import (
 )
 
 type BlobStorage struct {
-	http *transport.HttpClientCtx
+	http        *transport.HttpClientCtx
+	concurrency int
+}
+
+func NewBlobStorage(http *transport.HttpClientCtx) *BlobStorage {
+	return &BlobStorage{
+		http: http,
+	}
 }
 
 const ROOT_NAME = "root"
@@ -111,7 +117,7 @@ func (b *BlobStorage) GetRootIndex() (string, int64, error) {
 	if err != nil {
 		return "", 0, err
 	}
-	content, err := ioutil.ReadAll(blob)
+	content, err := io.ReadAll(blob)
 	if err != nil {
 		return "", 0, err
 	}

api/sync15/common.go

@@ -4,7 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"path"
 	"sort"
@@ -42,14 +41,16 @@ func getCachedTreePath() (string, error) {
 	return cacheFile, nil
 }
 
+const cacheVersion = 2
+
 func loadTree() (*HashTree, error) {
-	tree := &HashTree{}
 	cacheFile, err := getCachedTreePath()
 	if err != nil {
 		return nil, err
 	}
+	tree := &HashTree{}
 	if _, err := os.Stat(cacheFile); err == nil {
-		b, err := ioutil.ReadFile(cacheFile)
+		b, err := os.ReadFile(cacheFile)
 		if err != nil {
 			return nil, err
 		}
@@ -58,8 +59,10 @@ func loadTree() (*HashTree, error) {
 			log.Error.Println("cache corrupt")
 			return tree, nil
 		}
-	} else {
-		os.Create(cacheFile)
+		if tree.CacheVersion != cacheVersion {
+			log.Info.Println("wrong cache file version, resync")
+			return &HashTree{}, nil
+		}
 	}
 	log.Info.Println("cache loaded: ", cacheFile)
 
@@ -72,10 +75,11 @@ func saveTree(tree *HashTree) error {
 	if err != nil {
 		return err
 	}
+	tree.CacheVersion = cacheVersion
 	b, err := json.MarshalIndent(tree, "", "")
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(cacheFile, b, 0644)
+	err = os.WriteFile(cacheFile, b, 0644)
 	return err
 }