Merge pull request #1480 from nono/hint-for-initial-files-sync

Add informations about the number of files for initial sync

Author: nono

docs/realtime.md

@@ -156,11 +156,11 @@ Then messages are sent using json
 
 ```
 client > {"method": "AUTH",
-          "payload": "xxAppOrAuthTokenxx="
+          "payload": "xxAppOrAuthTokenxx="}
 client > {"method": "SUBSCRIBE",
-          "payload": {"type": "io.cozy.files"}
+          "payload": {"type": "io.cozy.files"}}
 client > {"method": "SUBSCRIBE",
-          "payload": {"type": "io.cozy.contacts"}
+          "payload": {"type": "io.cozy.contacts"}}
 server > {"event": "UPDATED",
           "payload": {"id": "idA", "rev": "2-705...", "type": "io.cozy.contacts", "doc": {embeded doc ...}}}
 server > {"event": "DELETED",

docs/sharing-design.md

@@ -307,6 +307,9 @@ revision (4-4aa). The resolution takes 4 steps:
   * `false` if only the owner can add a new recipient
 * Some technical data (`created_at`, `updated_at`, `app_slug`, `preview_path`,
   `triggers`, `credentials`)
+* a number of files to synchronize for the initial sync,
+  `initial_number_of_files_to_sync` (if there are no files to sync or the
+  initial replication has finished, the field won't be here)
 * A list of sharing `rules`, each rule being composed of:
   * a `title`, that will be displayed to the recipients before they accept the
     sharing

docs/sharing.md

@@ -296,6 +296,7 @@ Content-Type: application/vnd.api+json
       "owner": true,
       "created_at": "2018-01-04T12:35:08Z",
       "updated_at": "2018-01-04T13:45:43Z",
+      "initial_number_of_files_to_sync": 42,
       "members": [
         {
           "status": "owner",
@@ -1267,3 +1268,43 @@ Authorization: Bearer ...
 ```http
 HTTP/1.1 204 No Content
 ```
+
+### DELETE /sharings/:sharing-id/initial
+
+This internal route is used by the sharer to inform a recipient's cozy that
+the initial sync is finished.
+
+```http
+DELETE /sharings/ce8835a061d0ef68947afe69a0046722/initial HTTP/1.1
+Host: bob.example.net
+Authorization: Bearer ...
+```
+
+#### Response
+
+```http
+HTTP/1.1 204 No Content
+```
+
+## Real-time via websockets
+
+You can subscribe to the [realtime](realtime.md) API for the normal doctypes,
+but also for a special `io.cozy.sharings.initial-sync` doctype. For this
+doctype, you can give the id of a sharing and you will be notified when a file
+will be received during the initial synchronisation (`UPDATED`), and when the
+sync will be done (`DELETED`).
+
+### Example
+
+```
+client > {"method": "AUTH",
+          "payload": "xxAppOrAuthTokenxx="}
+client > {"method": "SUBSCRIBE",
+          "payload": {"type": "io.cozy.sharings.initial-sync", "id": "ce8835a061d0ef68947afe69a0046722"}
+server > {"event": "UPDATED",
+          "payload": {"id": "ce8835a061d0ef68947afe69a0046722", "type": "io.cozy.sharings.initial-sync", "doc": {"count": 12}}}
+server > {"event": "UPDATED",
+          "payload": {"id": "ce8835a061d0ef68947afe69a0046722", "type": "io.cozy.sharings.initial-sync", "doc": {"count": 13}}}
+server > {"event": "DELETED",
+          "payload": {"id": "ce8835a061d0ef68947afe69a0046722", "type": "io.cozy.sharings.initial-sync"}}
+```

pkg/consts/consts.go

@@ -54,6 +54,9 @@ const (
 	Sharings = "io.cozy.sharings"
 	// SharingsAnswer doc type for credentials exchange for sharings
 	SharingsAnswer = "io.cozy.sharings.answer"
+	// SharingsInitialSync doc type for real-time events for initial sync of a
+	// sharing
+	SharingsInitialSync = "io.cozy.sharings.initial-sync"
 	// Triggers doc type for triggers, jobs launchers
 	Triggers = "io.cozy.triggers"
 	// TriggersState doc type for triggers current state, jobs launchers

pkg/sharing/oauth.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cozy/cozy-stack/pkg/instance"
 	"github.com/cozy/cozy-stack/pkg/oauth"
 	"github.com/cozy/cozy-stack/pkg/permissions"
+	"github.com/cozy/cozy-stack/pkg/vfs"
 	"github.com/cozy/cozy-stack/web/jsonapi"
 )
 
@@ -61,6 +62,7 @@ func (m *Member) CreateSharingRequest(inst *instance.Instance, s *Sharing, c *Cr
 			UpdatedAt:   s.UpdatedAt,
 			Rules:       rules,
 			Members:     members,
+			NbFiles:     s.countFiles(inst),
 		},
 		nil,
 		nil,
@@ -112,6 +114,40 @@ func clearAppInHost(host string) string {
 	return parts[0] + "." + domain
 }
 
+// countFiles returns the number of files that should be uploaded on the
+// initial synchronisation.
+func (s *Sharing) countFiles(inst *instance.Instance) int {
+	count := 0
+	for _, rule := range s.Rules {
+		if rule.DocType != consts.Files || rule.Local || len(rule.Values) == 0 {
+			continue
+		}
+		if rule.Selector == "" || rule.Selector == "id" {
+			for _, fileID := range rule.Values {
+				vfs.WalkByID(inst.VFS(), fileID, func(name string, dir *vfs.DirDoc, file *vfs.FileDoc, err error) error {
+					if err != nil {
+						return err
+					}
+					if file != nil {
+						count++
+					}
+					return nil
+				})
+			}
+		} else {
+			var resCount couchdb.ViewResponse
+			for _, val := range rule.Values {
+				reqCount := &couchdb.ViewRequest{Key: val, Reduce: true}
+				err := couchdb.ExecView(inst, consts.FilesReferencedByView, reqCount, &resCount)
+				if err == nil && len(resCount.Rows) > 0 {
+					count += int(resCount.Rows[0].Value.(float64))
+				}
+			}
+		}
+	}
+	return count
+}
+
 // RegisterCozyURL saves a new Cozy URL for a member
 func (s *Sharing) RegisterCozyURL(inst *instance.Instance, m *Member, cozyURL string) error {
 	if !s.Owner {

pkg/sharing/sharing.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cozy/cozy-stack/pkg/jobs"
 	"github.com/cozy/cozy-stack/pkg/permissions"
 	"github.com/cozy/cozy-stack/pkg/prefixer"
+	"github.com/cozy/cozy-stack/pkg/realtime"
 	multierror "github.com/hashicorp/go-multierror"
 )
 
@@ -43,6 +44,7 @@ type Sharing struct {
 	PreviewPath string    `json:"preview_path,omitempty"`
 	CreatedAt   time.Time `json:"created_at"`
 	UpdatedAt   time.Time `json:"updated_at"`
+	NbFiles     int       `json:"initial_number_of_files_to_sync,omitempty"`
 
 	Rules []Rule `json:"rules"`
 
@@ -514,4 +516,21 @@ func (s *Sharing) RedirectAfterAuthorizeURL(inst *instance.Instance) *url.URL {
 	return inst.SubDomain(app.Slug())
 }
 
+// EndInitial is used to finish the initial sync phase of a sharing
+func (s *Sharing) EndInitial(inst *instance.Instance) error {
+	if s.NbFiles == 0 {
+		return nil
+	}
+	s.NbFiles = 0
+	if err := couchdb.UpdateDoc(inst, s); err != nil {
+		return err
+	}
+	doc := couchdb.JSONDoc{
+		Type: consts.SharingsInitialSync,
+		M:    map[string]interface{}{"_id": s.SID},
+	}
+	realtime.GetHub().Publish(inst, realtime.EventDelete, doc, nil)
+	return nil
+}
+
 var _ couchdb.Doc = &Sharing{}

pkg/sharing/upload.go

@@ -3,6 +3,7 @@ package sharing
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -14,6 +15,7 @@ import (
 	"github.com/cozy/cozy-stack/pkg/couchdb"
 	"github.com/cozy/cozy-stack/pkg/instance"
 	"github.com/cozy/cozy-stack/pkg/lock"
+	"github.com/cozy/cozy-stack/pkg/realtime"
 	"github.com/cozy/cozy-stack/pkg/vfs"
 	multierror "github.com/hashicorp/go-multierror"
 )
@@ -81,14 +83,45 @@ func (s *Sharing) InitialUpload(inst *instance.Instance, m *Member) error {
 			return err
 		}
 		if !more {
-			return nil
+			return s.sendInitialEndNotif(inst, m)
 		}
 	}
 
 	s.pushJob(inst, "share-upload")
 	return nil
 }
 
+// sendInitialEndNotif sends a notification to the recipient that the initial
+// sync is finished
+func (s *Sharing) sendInitialEndNotif(inst *instance.Instance, m *Member) error {
+	u, err := url.Parse(m.Instance)
+	if err != nil {
+		return err
+	}
+	c := s.FindCredentials(m)
+	if c == nil {
+		return ErrInvalidSharing
+	}
+	opts := &request.Options{
+		Method: http.MethodDelete,
+		Scheme: u.Scheme,
+		Domain: u.Host,
+		Path:   fmt.Sprintf("/sharings/%s/initial", s.SID),
+		Headers: request.Headers{
+			"Authorization": "Bearer " + c.AccessToken.AccessToken,
+		},
+	}
+	res, err := request.Req(opts)
+	if err != nil {
+		return err
+	}
+	res.Body.Close()
+	if res.StatusCode/100 != 2 {
+		return ErrInternalServerError
+	}
+	return nil
+}
+
 // UploadTo uploads one file to the given member. It returns false if there
 // are no more files to upload to this member currently.
 func (s *Sharing) UploadTo(inst *instance.Instance, m *Member) (bool, error) {
@@ -517,9 +550,52 @@ func (s *Sharing) UploadNewFile(inst *instance.Instance, target *FileDocWithRevi
 			Debugf("Cannot create file: %s", err)
 		return err
 	}
+	if s.NbFiles > 0 {
+		defer s.countReceivedFiles(inst)
+	}
 	return copyFileContent(inst, file, body)
 }
 
+// countReceivedFiles counts the number of files received during the initial
+// sync, and pushs an event to the real-time system with this count
+func (s *Sharing) countReceivedFiles(inst *instance.Instance) {
+	count := 0
+	var req = &couchdb.ViewRequest{
+		Key:         s.SID,
+		IncludeDocs: true,
+	}
+	var res couchdb.ViewResponse
+	err := couchdb.ExecView(inst, consts.SharedDocsBySharingID, req, &res)
+	if err == nil {
+		for _, row := range res.Rows {
+			var doc SharedRef
+			if err = json.Unmarshal(row.Doc, &doc); err != nil {
+				continue
+			}
+			if doc.Infos[s.SID].Binary {
+				count++
+			}
+		}
+	}
+
+	if count >= s.NbFiles {
+		if err = s.EndInitial(inst); err != nil {
+			inst.Logger().WithField("nspace", "sharing").
+				Errorf("Can't save sharing %v: %s", s, err)
+		}
+		return
+	}
+
+	doc := couchdb.JSONDoc{
+		Type: consts.SharingsInitialSync,
+		M: map[string]interface{}{
+			"_id":   s.SID,
+			"count": count,
+		},
+	}
+	realtime.GetHub().Publish(inst, realtime.EventUpdate, doc, nil)
+}
+
 // UploadExistingFile is used to receive new content for an existing file.
 //
 // Note: if file was renamed + its content has changed, we modify the content

tests/sharing/Gemfile

@@ -2,11 +2,10 @@ source 'https://rubygems.org'
 
 gem "awesome_print"
 gem "faker"
+gem "faye-websocket"
 gem "mimemagic"
 gem "minitest"
 gem "pry"
 gem "pry-rescue"
+gem "pry-stack_explorer"
 gem "rest-client"
-
-# Added at 2018-05-21 09:30:04 +0200 by nono:
-gem "pry-stack_explorer", "~> 0.4.9"

tests/sharing/Gemfile.lock

@@ -9,8 +9,12 @@ GEM
     debug_inspector (0.0.3)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
+    eventmachine (1.2.7)
     faker (1.8.7)
       i18n (>= 0.7)
+    faye-websocket (0.10.7)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.5.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (1.0.0)
@@ -39,18 +43,22 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.5)
+    websocket-driver (0.7.0)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.3)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   awesome_print
   faker
+  faye-websocket
   mimemagic
   minitest
   pry
   pry-rescue
-  pry-stack_explorer (~> 0.4.9)
+  pry-stack_explorer
   rest-client
 
 BUNDLED WITH