Skip to content
This repository was archived by the owner on Jul 2, 2024. It is now read-only.

Commit 58deab4

Browse files
ddvkddvk
authored
Merge pull request #260 from ddvk/fix_urls
update 1.5 sync urls, implement v2 api #259
2 parents f3599d1 + 8098251 commit 58deab4

File tree

14 files changed

+157
-113
lines changed

14 files changed

+157
-113
lines changed

‎api/api.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package api
22

33
import (
4+
"fmt"
5+
"os"
46
"strings"
57

68
"github.com/golang-jwt/jwt"
@@ -28,7 +30,7 @@ type UserToken struct {
2830
}
2931

3032
// CreateApiCtx initializes an instance of ApiCtx
31-
func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, isSync15 bool, err error) {
33+
func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, err error) {
3234
userToken := http.Tokens.UserToken
3335
claims := UserToken{}
3436
jwt.ParseWithClaims(userToken, &claims, func(token *jwt.Token) (interface{}, error) {
@@ -38,6 +40,8 @@ func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, isSync15 bool, err
3840
return
3941
}
4042
fld := strings.Fields(claims.Scopes)
43+
isSync15 := false
44+
4145
forloop:
4246
for _, f := range fld {
4347
switch f {
@@ -51,6 +55,7 @@ forloop:
5155
}
5256
}
5357
if isSync15 {
58+
fmt.Fprintln(os.Stderr, `WARNING!!! Using the new 1.5 sync, this has not been fully tested yet!!! first sync can be slow`)
5459
ctx, err = sync15.CreateCtx(http)
5560
} else {
5661
ctx, err = sync10.CreateCtx(http)

‎api/sync10/api.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func (ctx *ApiCtx) Nuke() error {
3737
for _, d := range documents {
3838
log.Info.Println("Deleting: ", d.VissibleName)
3939

40-
err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, d, nil)
40+
err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, util.InSlice(d), nil)
4141
if err != nil {
4242
log.Error.Println("failed to remove entry", err)
4343
return err
@@ -141,7 +141,7 @@ func (ctx *ApiCtx) CreateDir(parentId, name string) (*model.Document, error) {
141141

142142
metaDoc := model.CreateUploadDocumentMeta(uploadRsp.ID, model.DirectoryType, parentId, name)
143143

144-
err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
144+
err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
145145

146146
if err != nil {
147147
log.Error.Println("failed to move entry", err)
@@ -162,7 +162,7 @@ func (ctx *ApiCtx) DeleteEntry(node *model.Node) error {
162162

163163
deleteDoc := node.Document.ToDeleteDocument()
164164

165-
err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, deleteDoc, nil)
165+
err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, util.InSlice(deleteDoc), nil)
166166

167167
if err != nil {
168168
log.Error.Println("failed to remove entry", err)
@@ -186,7 +186,7 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
186186
metaDoc.VissibleName = name
187187
metaDoc.Parent = dstDir.Id()
188188

189-
err := ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
189+
err := ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
190190

191191
if err != nil {
192192
log.Error.Println("failed to move entry", err)
@@ -258,7 +258,7 @@ func (ctx *ApiCtx) UploadDocument(parentId string, sourceDocPath string, notify
258258

259259
metaDoc := model.CreateUploadDocumentMeta(uploadRsp.ID, model.DocumentType, parentId, name)
260260

261-
err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
261+
err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
262262

263263
if err != nil {
264264
log.Error.Println("failed to move entry", err)
@@ -274,7 +274,7 @@ func (ctx *ApiCtx) uploadRequest(id string, entryType string) (model.UploadDocum
274274
uploadReq := model.CreateUploadDocumentRequest(id, entryType)
275275
uploadRsp := make([]model.UploadDocumentResponse, 0)
276276

277-
err := ctx.Http.Put(transport.UserBearer, config.UploadRequest, uploadReq, &uploadRsp)
277+
err := ctx.Http.Put(transport.UserBearer, config.UploadRequest, util.InSlice(uploadReq), &uploadRsp)
278278

279279
if err != nil {
280280
log.Error.Println("failed to to send upload request", err)

‎api/sync15/apictx.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,9 +196,8 @@ func Sync(b *BlobStorage, tree *HashTree, operation func(t *HashTree) error) err
196196

197197
newGeneration, err := b.WriteRootIndex(tree.Hash, tree.Generation)
198198

199-
log.Info.Println("wrote root, new gen: ", newGeneration)
200-
201199
if err == nil {
200+
log.Info.Println("wrote root, new gen: ", newGeneration)
202201
tree.Generation = newGeneration
203202
break
204203
}
@@ -388,7 +387,6 @@ func CreateCtx(http *transport.HttpClientCtx) (*ApiCtx, error) {
388387
fmt.Print(err)
389388
return nil, err
390389
}
391-
fmt.Fprintln(os.Stderr, "Refreshing tree...")
392390
err = cacheTree.Mirror(apiStorage)
393391
if err != nil {
394392
return nil, err
@@ -426,7 +424,7 @@ func DocumentsFileTree(tree *HashTree) (*filetree.FileTreeCtx, error) {
426424

427425
// SyncComplete notfies that somethings has changed (triggers tablet sync)
428426
func (ctx *ApiCtx) SyncComplete() error {
429-
err := ctx.blobStorage.SyncComplete()
427+
err := ctx.blobStorage.SyncComplete(ctx.hashTree.Generation)
430428
if err != nil {
431429
log.Error.Printf("cannot send sync %v", err)
432430
}

‎api/sync15/blobdoc.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"encoding/hex"
88
"encoding/json"
99
"errors"
10-
"fmt"
1110
"io"
1211
"io/ioutil"
1312
"sort"
@@ -218,7 +217,6 @@ func (d *BlobDoc) ToDocument() *model.Document {
218217

219218
t := time.Unix(unixTime/1000, 0)
220219
lastModified = t.UTC().Format(time.RFC3339Nano)
221-
fmt.Println(" Time " + lastModified + " " + d.MetadataFile.DocName)
222220
}
223221
return &model.Document{
224222
ID: d.DocumentID,

‎api/sync15/blobstorage.go‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"bytes"
55
"io"
66
"io/ioutil"
7-
"strconv"
7+
"net/http"
88

99
"github.com/juruen/rmapi/config"
1010
"github.com/juruen/rmapi/log"
@@ -16,26 +16,40 @@ type BlobStorage struct {
1616
http *transport.HttpClientCtx
1717
}
1818

19-
func (b *BlobStorage) PutUrl(hash string, gen int64) (string, error) {
19+
const ROOT_NAME = "root"
20+
21+
func (b *BlobStorage) PutRootUrl(hash string, gen int64) (string, int64, error) {
22+
log.Trace.Println("fetching ROOT url for: " + hash)
23+
req := model.BlobRootStorageRequest{
24+
Method: http.MethodPut,
25+
RelativePath: ROOT_NAME,
26+
RootSchema: hash,
27+
Generation: gen,
28+
}
29+
var res model.BlobStorageResponse
30+
31+
if err := b.http.Post(transport.UserBearer, config.UploadBlob, req, &res); err != nil {
32+
return "", 0, err
33+
}
34+
return res.Url, res.MaxUploadSizeBytes, nil
35+
}
36+
func (b *BlobStorage) PutUrl(hash string) (string, int64, error) {
2037
log.Trace.Println("fetching PUT blob url for: " + hash)
2138
var req model.BlobStorageRequest
2239
var res model.BlobStorageResponse
23-
req.Method = "PUT"
40+
req.Method = http.MethodPut
2441
req.RelativePath = hash
25-
if gen > 0 {
26-
req.Generation = strconv.FormatInt(gen, 10)
27-
}
2842
if err := b.http.Post(transport.UserBearer, config.UploadBlob, req, &res); err != nil {
29-
return "", err
43+
return "", 0, err
3044
}
31-
return res.Url, nil
45+
return res.Url, res.MaxUploadSizeBytes, nil
3246
}
3347

3448
func (b *BlobStorage) GetUrl(hash string) (string, error) {
3549
log.Trace.Println("fetching GET blob url for: " + hash)
3650
var req model.BlobStorageRequest
3751
var res model.BlobStorageResponse
38-
req.Method = "GET"
52+
req.Method = http.MethodGet
3953
req.RelativePath = hash
4054
if err := b.http.Post(transport.UserBearer, config.DownloadBlob, req, &res); err != nil {
4155
return "", err
@@ -55,37 +69,36 @@ func (b *BlobStorage) GetReader(hash string) (io.ReadCloser, error) {
5569
}
5670

5771
func (b *BlobStorage) UploadBlob(hash string, reader io.Reader) error {
58-
url, err := b.PutUrl(hash, -1)
72+
url, size, err := b.PutUrl(hash)
5973
if err != nil {
6074
return err
6175
}
6276
log.Trace.Println("put url: " + url)
6377

64-
_, err = b.http.PutBlobStream(url, -1, reader)
65-
return err
78+
return b.http.PutBlobStream(url, reader, size)
6679
}
6780

6881
// SyncComplete notifies that the sync is done
69-
func (b *BlobStorage) SyncComplete() error {
70-
return b.http.Post(transport.UserBearer, config.SyncComplete, nil, nil)
82+
func (b *BlobStorage) SyncComplete(gen int64) error {
83+
req := model.SyncCompletedRequest{
84+
Generation: gen,
85+
}
86+
return b.http.Post(transport.UserBearer, config.SyncComplete, req, nil)
7187
}
7288

7389
func (b *BlobStorage) WriteRootIndex(roothash string, gen int64) (int64, error) {
7490
log.Info.Println("writing root with gen: ", gen)
75-
url, err := b.PutUrl("root", gen)
91+
url, maxRequestSize, err := b.PutRootUrl(roothash, gen)
7692
if err != nil {
7793
return 0, err
7894
}
7995
log.Trace.Println("got root url:", url)
8096
reader := bytes.NewBufferString(roothash)
8197

82-
gen, err = b.http.PutBlobStream(url, gen, reader)
83-
return gen, err
84-
98+
return b.http.PutRootBlobStream(url, gen, maxRequestSize, reader)
8599
}
86100
func (b *BlobStorage) GetRootIndex() (string, int64, error) {
87-
88-
url, err := b.GetUrl("root")
101+
url, err := b.GetUrl(ROOT_NAME)
89102
if err != nil {
90103
return "", 0, err
91104
}

‎api/sync15/tree.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ func (t *HashTree) Rehash() error {
161161
return nil
162162
}
163163

164-
/// Mirror makes the tree look like the storage
164+
// / Mirror makes the tree look like the storage
165165
func (t *HashTree) Mirror(r RemoteStorage) error {
166166
rootHash, gen, err := r.GetRootIndex()
167167
if err != nil {

‎cloud/client_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
//go:build withauth
12
// +build withauth
23

34
package cloud

‎config/url.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ var SyncComplete string
1616
func init() {
1717
docHost := "https://document-storage-production-dot-remarkable-production.appspot.com"
1818
authHost := "https://webapp-production-dot-remarkable-production.appspot.com"
19-
blobHost := "https://rm-blob-storage-prod.appspot.com"
19+
syncHost := "https://internal.cloud.remarkable.com"
2020

2121
host := os.Getenv("RMAPI_DOC")
2222
if host != "" {
@@ -33,7 +33,7 @@ func init() {
3333
if host != "" {
3434
authHost = host
3535
docHost = host
36-
blobHost = host
36+
syncHost = host
3737
}
3838

3939
NewTokenDevice = authHost + "/token/json/2/device/new"
@@ -43,7 +43,7 @@ func init() {
4343
UploadRequest = docHost + "/document-storage/json/2/upload/request"
4444
DeleteEntry = docHost + "/document-storage/json/2/delete"
4545

46-
UploadBlob = blobHost + "/api/v1/signed-urls/uploads"
47-
DownloadBlob = blobHost + "/api/v1/signed-urls/downloads"
48-
SyncComplete = blobHost + "/api/v1/sync-complete"
46+
UploadBlob = syncHost + "/sync/v2/signed-urls/uploads"
47+
DownloadBlob = syncHost + "/sync/v2/signed-urls/downloads"
48+
SyncComplete = syncHost + "/sync/v2/sync-complete"
4949
}

‎encoding/rm/rm.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
//
2424
// To try to be as idiomatic as possible, this package implements the two following interfaces
2525
// of the default encoding package (https://golang.org/pkg/encoding/).
26-
// - BinaryMarshaler
27-
// - BinaryUnmarshaler
26+
// - BinaryMarshaler
27+
// - BinaryUnmarshaler
2828
//
2929
// The scope of this package is defined as just the encoding/decoding of the .rm format.
3030
// It will only deal with bytes and not files (one must take care of unzipping the archive

‎main.go‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package main
22

33
import (
44
"flag"
5-
"fmt"
65
"os"
76

87
"github.com/juruen/rmapi/api"
@@ -31,17 +30,11 @@ func main() {
3130
var ctx api.ApiCtx
3231
var err error
3332
for i := 0; i < AUTH_RETRIES; i++ {
34-
isSync15 := false
35-
ctx, isSync15, err = api.CreateApiCtx(api.AuthHttpCtx(i > 0, *ni))
33+
ctx, err = api.CreateApiCtx(api.AuthHttpCtx(i > 0, *ni))
3634

3735
if err != nil {
3836
log.Trace.Println(err)
3937
} else {
40-
if isSync15 {
41-
fmt.Fprintln(os.Stderr, `WARNING!!!
42-
Using the new 1.5 sync, this has not been fully tested yet!!!
43-
Make sure you have a backup, in case there is a bug that could cause data loss!`)
44-
}
4538
break
4639
}
4740
}

0 commit comments

Comments
 (0)