Merge pull request #726 from nono/worker-unzip

Worker unzip

Author: aenario

docs/workers.md

@@ -12,6 +12,42 @@ arguments.
 The `log` worker will just print in the log file the job sent to it. It can
 useful for debugging for example.
 
+## unzip worker
+
+The `unzip` worker can take a zip archive from the VFS, and will unzip the
+files inside it to a directory of the VFS. The options are:
+
+- `zip`: the ID of the zip file
+- `destination`: the ID of the directory where the files will be unzipped.
+
+### Example
+
+```json
+{
+  "zip": "8737b5d6-51b6-11e7-9194-bf5b64b3bc9e",
+  "destination": "88750a84-51b6-11e7-ba90-4f0b1cb62b7b"
+}
+```
+
+### Permissions
+
+To use this worker from a client-side application, you will need to ask the
+permission. It is done by adding this to the manifest:
+
+```json
+{
+  "permissions": {
+    "unzip-to-a-directory": {
+      "description": "Required to unzip a file inside the cozy",
+      "type": "io.cozy.jobs",
+      "verbs": ["POST"],
+      "selector": "worker",
+      "values": ["unzip"]
+    }
+  }
+}
+```
+
 ## sendmail worker
 
 The `sendmail` worker can be used to send mail from the stack. It implies that

pkg/vfs/vfs.go

@@ -78,6 +78,7 @@ type Fs interface {
 // file for reading or writing.
 type File interface {
 	io.Reader
+	io.ReaderAt
 	io.Seeker
 	io.Writer
 	io.Closer

pkg/vfs/vfsafero/impl.go

@@ -407,6 +407,10 @@ func (f *aferoFileOpen) Read(p []byte) (int, error) {
 	return f.f.Read(p)
 }
 
+func (f *aferoFileOpen) ReadAt(p []byte, off int64) (int, error) {
+	return f.f.ReadAt(p, off)
+}
+
 func (f *aferoFileOpen) Seek(offset int64, whence int) (int64, error) {
 	return f.f.Seek(offset, whence)
 }
@@ -441,6 +445,10 @@ func (f *aferoFileCreation) Read(p []byte) (int, error) {
 	return 0, os.ErrInvalid
 }
 
+func (f *aferoFileCreation) ReadAt(p []byte, off int64) (int, error) {
+	return 0, os.ErrInvalid
+}
+
 func (f *aferoFileCreation) Seek(offset int64, whence int) (int64, error) {
 	return 0, os.ErrInvalid
 }

pkg/vfs/vfsswift/impl.go

@@ -1,9 +1,11 @@
 package vfsswift
 
 import (
+	"bytes"
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -329,7 +331,7 @@ func (sfs *swiftVFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &swiftFileOpen{f}, nil
+	return &swiftFileOpen{f, nil}, nil
 }
 
 // UpdateFileDoc overrides the indexer's one since the swift fs indexes files
@@ -460,6 +462,10 @@ func (f *swiftFileCreation) Read(p []byte) (int, error) {
 	return 0, os.ErrInvalid
 }
 
+func (f *swiftFileCreation) ReadAt(p []byte, off int64) (int, error) {
+	return 0, os.ErrInvalid
+}
+
 func (f *swiftFileCreation) Seek(offset int64, whence int) (int64, error) {
 	return 0, os.ErrInvalid
 }
@@ -584,13 +590,26 @@ func (f *swiftFileCreation) Close() (err error) {
 }
 
 type swiftFileOpen struct {
-	f *swift.ObjectOpenFile
+	f  *swift.ObjectOpenFile
+	br *bytes.Reader
 }
 
 func (f *swiftFileOpen) Read(p []byte) (int, error) {
 	return f.f.Read(p)
 }
 
+func (f *swiftFileOpen) ReadAt(p []byte, off int64) (int, error) {
+	// TODO find something smarter than keeping the whole file in memory
+	if f.br == nil {
+		buf, err := ioutil.ReadAll(f.f)
+		if err != nil {
+			return 0, err
+		}
+		f.br = bytes.NewReader(buf)
+	}
+	return f.br.ReadAt(p, off)
+}
+
 func (f *swiftFileOpen) Seek(offset int64, whence int) (int64, error) {
 	return f.f.Seek(offset, whence)
 }

pkg/workers/unzip/unzip.go

@@ -0,0 +1,113 @@
+package unzip
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"runtime"
+	"time"
+
+	"github.com/cozy/cozy-stack/pkg/couchdb"
+	"github.com/cozy/cozy-stack/pkg/instance"
+	"github.com/cozy/cozy-stack/pkg/jobs"
+	"github.com/cozy/cozy-stack/pkg/logger"
+	"github.com/cozy/cozy-stack/pkg/vfs"
+)
+
+type zipMessage struct {
+	Zip         string `json:"zip"`
+	Destination string `string:"destination"`
+}
+
+func init() {
+	jobs.AddWorker("unzip", &jobs.WorkerConfig{
+		Concurrency:  (runtime.NumCPU() + 1) / 2,
+		MaxExecCount: 2,
+		Timeout:      30 * time.Second,
+		WorkerFunc:   Worker,
+	})
+}
+
+// Worker is a worker that unzip a file.
+func Worker(ctx context.Context, m *jobs.Message) error {
+	msg := &zipMessage{}
+	if err := m.Unmarshal(msg); err != nil {
+		return err
+	}
+	domain := ctx.Value(jobs.ContextDomainKey).(string)
+	log := logger.WithDomain(domain)
+	log.Infof("[jobs] unzip %s in %s", msg.Zip, msg.Destination)
+	i, err := instance.Get(domain)
+	if err != nil {
+		return err
+	}
+	fs := i.VFS()
+	return unzip(fs, msg.Zip, msg.Destination)
+}
+
+func unzip(fs vfs.VFS, zipID, destination string) error {
+	zipDoc, err := fs.FileByID(zipID)
+	if err != nil {
+		return err
+	}
+	dstDoc, err := fs.DirByID(destination)
+	if err != nil {
+		return err
+	}
+
+	fr, err := fs.OpenFile(zipDoc)
+	if err != nil {
+		return err
+	}
+	defer fr.Close()
+	r, err := zip.NewReader(fr, zipDoc.ByteSize)
+	if err != nil {
+		return err
+	}
+	for _, f := range r.File {
+		name := path.Base(f.Name)
+		dirname := path.Dir(f.Name)
+		dir := dstDoc
+		if dirname != "." {
+			dirname = path.Join(dstDoc.Fullpath, dirname)
+			dir, err = vfs.MkdirAll(fs, dirname, nil)
+			if err != nil {
+				return err
+			}
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+
+		size := int64(f.UncompressedSize64)
+		mime, class := vfs.ExtractMimeAndClassFromFilename(f.Name)
+		now := time.Now()
+		doc, err := vfs.NewFileDoc(name, dir.ID(), size, nil, mime, class, now, false, false, nil)
+		if err != nil {
+			return err
+		}
+		file, err := fs.CreateFile(doc, nil)
+		if err != nil {
+			if couchdb.IsConflictError(err) {
+				doc.DocName = fmt.Sprintf("%s - conflict - %d", doc.DocName, time.Now().Unix())
+				file, err = fs.CreateFile(doc, nil)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		_, err = io.Copy(file, rc)
+		cerr := file.Close()
+		if err != nil {
+			return err
+		}
+		if cerr != nil {
+			return cerr
+		}
+	}
+	return nil
+}

pkg/workers/unzip/unzip_test.go

@@ -0,0 +1,61 @@
+package unzip
+
+import (
+	"io"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/cozy/cozy-stack/pkg/config"
+	"github.com/cozy/cozy-stack/pkg/consts"
+	"github.com/cozy/cozy-stack/pkg/instance"
+	"github.com/cozy/cozy-stack/pkg/vfs"
+	"github.com/cozy/cozy-stack/tests/testutils"
+	"github.com/stretchr/testify/assert"
+)
+
+var inst *instance.Instance
+
+func TestUnzip(t *testing.T) {
+	fs := inst.VFS()
+	dst, err := vfs.Mkdir(fs, "/destination", nil)
+	assert.NoError(t, err)
+	_, err = vfs.Mkdir(fs, "/destination/foo", nil)
+	assert.NoError(t, err)
+
+	fd, err := os.Open("../../../tests/fixtures/logos.zip")
+	assert.NoError(t, err)
+	defer fd.Close()
+	zip, err := vfs.NewFileDoc("logos.zip", consts.RootDirID, -1, nil, "application/zip", "application", time.Now(), false, false, nil)
+	assert.NoError(t, err)
+	file, err := fs.CreateFile(zip, nil)
+	assert.NoError(t, err)
+	_, err = io.Copy(file, fd)
+	assert.NoError(t, err)
+	assert.NoError(t, file.Close())
+
+	_, err = fs.OpenFile(zip)
+	assert.NoError(t, err)
+
+	err = unzip(fs, zip.ID(), dst.ID())
+	assert.NoError(t, err)
+
+	blue, err := fs.FileByPath("/destination/blue.svg")
+	assert.NoError(t, err)
+	assert.Equal(t, int64(2029), blue.ByteSize)
+
+	white, err := fs.FileByPath("/destination/white.svg")
+	assert.NoError(t, err)
+	assert.Equal(t, int64(2030), white.ByteSize)
+
+	baz, err := fs.FileByPath("/destination/foo/bar/baz")
+	assert.NoError(t, err)
+	assert.Equal(t, int64(4), baz.ByteSize)
+}
+
+func TestMain(m *testing.M) {
+	config.UseTestFile()
+	setup := testutils.NewSetup(m, "unzip_test")
+	inst = setup.GetTestInstance()
+	os.Exit(setup.Run())
+}

tests/fixtures/logos.zip

@@ -20,6 +20,7 @@ import (
 	_ "github.com/cozy/cozy-stack/pkg/workers/mails"
 	_ "github.com/cozy/cozy-stack/pkg/workers/sharings"
 	_ "github.com/cozy/cozy-stack/pkg/workers/thumbnail"
+	_ "github.com/cozy/cozy-stack/pkg/workers/unzip"
 )
 
 type (