sync: add alrternative CopyDir implementation

Add sync.CopyDir2 which is coping files from directory
to directory based on absolute paths from the root of
file systems.

Author: ajankovic

fs/fs.go

@@ -1061,6 +1061,7 @@ type Objects []Object
 // operation.
 type ObjectPair struct {
 	Src, Dst Object
+	Name     string
 }
 
 // UnWrapFs unwraps f as much as possible and returns the base Fs

fs/march/march.go

@@ -24,8 +24,9 @@ type March struct {
 	// parameters
 	Ctx           context.Context // context for background goroutines
 	Fdst          fs.Fs           // source Fs
+	DstDir        string          // destination directory
 	Fsrc          fs.Fs           // dest Fs
-	Dir           string          // directory
+	SrcDir        string          // source directory
 	NoTraverse    bool            // don't traverse the destination
 	SrcIncludeAll bool            // don't include all files in the src
 	DstIncludeAll bool            // don't include all files in the destination
@@ -49,9 +50,9 @@ type Marcher interface {
 
 // init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
 func (m *March) init() {
-	m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll)
+	m.srcListDir = m.makeListDir(m.Fsrc, m.SrcDir, m.SrcIncludeAll)
 	if !m.NoTraverse {
-		m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll)
+		m.dstListDir = m.makeListDir(m.Fdst, m.DstDir, m.DstIncludeAll)
 	}
 	// Now create the matching transform
 	// ..normalise the UTF8 first
@@ -72,7 +73,7 @@ type listDirFn func(dir string) (entries fs.DirEntries, err error)
 
 // makeListDir makes constructs a listing function for the given fs
 // and includeAll flags for marching through the file system.
-func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
+func (m *March) makeListDir(f fs.Fs, remote string, includeAll bool) listDirFn {
 	if !(fs.Config.UseListR && f.Features().ListR != nil) && // !--fast-list active and
 		!(fs.Config.NoTraverse && filter.Active.HaveFilesFrom()) { // !(--files-from and --no-traverse)
 		return func(dir string) (entries fs.DirEntries, err error) {
@@ -92,7 +93,7 @@ func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
 		mu.Lock()
 		defer mu.Unlock()
 		if !started {
-			dirs, dirsErr = walk.NewDirTree(m.Ctx, f, m.Dir, includeAll, fs.Config.MaxDepth)
+			dirs, dirsErr = walk.NewDirTree(m.Ctx, f, remote, includeAll, fs.Config.MaxDepth)
 			started = true
 		}
 		if dirsErr != nil {
@@ -185,9 +186,9 @@ func (m *March) Run() error {
 	// Start the process
 	traversing.Add(1)
 	in <- listDirJob{
-		srcRemote: m.Dir,
+		srcRemote: m.SrcDir,
 		srcDepth:  srcDepth - 1,
-		dstRemote: m.Dir,
+		dstRemote: m.DstDir,
 		dstDepth:  dstDepth - 1,
 		noDst:     m.NoCheckDest,
 	}

fs/march/march_test.go

@@ -195,8 +195,9 @@ func TestMarch(t *testing.T) {
 			m := &March{
 				Ctx:           ctx,
 				Fdst:          r.Fremote,
+				DstDir:        "",
 				Fsrc:          r.Flocal,
-				Dir:           "",
+				SrcDir:        "",
 				NoTraverse:    mt.noTraverse,
 				Callback:      mt,
 				DstIncludeAll: filter.Active.Opt.DeleteExcluded,
@@ -262,8 +263,9 @@ func TestMarchNoTraverse(t *testing.T) {
 			m := &March{
 				Ctx:           ctx,
 				Fdst:          r.Fremote,
+				DstDir:        "",
 				Fsrc:          r.Flocal,
-				Dir:           "",
+				SrcDir:        "",
 				NoTraverse:    mt.noTraverse,
 				Callback:      mt,
 				DstIncludeAll: filter.Active.Opt.DeleteExcluded,

fs/operations/operations.go

@@ -869,7 +869,6 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
 		Ctx:      ctx,
 		Fdst:     fdst,
 		Fsrc:     fsrc,
-		Dir:      "",
 		Callback: c,
 	}
 	fs.Infof(fdst, "Waiting for checks to finish")

fs/sync/sync.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"path"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -27,7 +28,8 @@ type syncCopyMove struct {
 	DoMove             bool
 	copyEmptySrcDirs   bool
 	deleteEmptySrcDirs bool
-	dir                string
+	dstDir             string
+	srcDir             string
 	// internal state
 	ctx             context.Context        // internal context for controlling go-routines
 	cancel          func()                 // cancel the context
@@ -64,9 +66,10 @@ type syncCopyMove struct {
 	renameCheck     []fs.Object            // accumulate files to check for rename here
 	compareCopyDest fs.Fs                  // place to check for files to server side copy
 	backupDir       fs.Fs                  // place to store overwrites/deletes
+	useSrcBaseName  bool                   // use only source basename when coping or moving to destination directory
 }
 
-func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
+func newSyncCopyMove(ctx context.Context, fdst fs.Fs, remoteDst string, fsrc fs.Fs, remoteSrc string, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, useSrcBaseName bool) (*syncCopyMove, error) {
 	if (deleteMode != fs.DeleteModeOff || DoMove) && operations.Overlapping(fdst, fsrc) {
 		return nil, fserrors.FatalError(fs.ErrorOverlapping)
 	}
@@ -77,7 +80,8 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		DoMove:             DoMove,
 		copyEmptySrcDirs:   copyEmptySrcDirs,
 		deleteEmptySrcDirs: deleteEmptySrcDirs,
-		dir:                "",
+		dstDir:             remoteDst,
+		srcDir:             remoteSrc,
 		srcFilesChan:       make(chan fs.Object, fs.Config.Checkers+fs.Config.Transfers),
 		srcFilesResult:     make(chan error, 1),
 		dstFilesResult:     make(chan error, 1),
@@ -89,6 +93,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		trackRenames:       fs.Config.TrackRenames,
 		commonHash:         fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
 		trackRenamesCh:     make(chan fs.Object, fs.Config.Checkers),
+		useSrcBaseName:     useSrcBaseName,
 	}
 	var err error
 	s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog)
@@ -325,15 +330,32 @@ func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs,
 			return
 		}
 		src := pair.Src
+		name := s.dstObjectName(pair)
 		if s.DoMove {
-			_, err = operations.Move(ctx, fdst, pair.Dst, src.Remote(), src)
+			_, err = operations.Move(ctx, fdst, pair.Dst, name, src)
 		} else {
-			_, err = operations.Copy(ctx, fdst, pair.Dst, src.Remote(), src)
+			_, err = operations.Copy(ctx, fdst, pair.Dst, name, src)
 		}
 		s.processError(err)
 	}
 }
 
+// dstObjectName returns full name of the object as it should be on the
+// destination after copy or move.
+func (s *syncCopyMove) dstObjectName(pair fs.ObjectPair) string {
+	if !s.useSrcBaseName {
+		return pair.Src.Remote()
+	}
+
+	// Example when coping:
+	// data:test_keyspace/snapshots/123/bla/somefile.file
+	// to
+	// s3:bucket/backup/long/path/to/123
+	// this should return backup/long/path/to/123/bla/somefile.file
+	// So we get full path on the destination fs.
+	return path.Join(s.dstDir, strings.TrimPrefix(pair.Src.Remote(), s.srcDir))
+}
+
 // This starts the background checkers.
 func (s *syncCopyMove) startCheckers() {
 	s.checkerWg.Add(fs.Config.Checkers)
@@ -696,8 +718,9 @@ func (s *syncCopyMove) run() error {
 	m := &march.March{
 		Ctx:           s.ctx,
 		Fdst:          s.fdst,
+		DstDir:        s.dstDir,
 		Fsrc:          s.fsrc,
-		Dir:           s.dir,
+		SrcDir:        s.srcDir,
 		NoTraverse:    s.noTraverse,
 		Callback:      s,
 		DstIncludeAll: filter.Active.Opt.DeleteExcluded,
@@ -899,7 +922,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
 // If DoMove is true then files will be moved instead of copied
 //
 // dir is the start directory, "" for root
-func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
+func runSyncCopyMove(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
 	if deleteMode != fs.DeleteModeOff && DoMove {
 		return fserrors.FatalError(errors.New("can't delete and move at the same time"))
 	}
@@ -909,7 +932,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 			return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))
 		}
 		// only delete stuff during in this pass
-		do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs)
+		do, err := newSyncCopyMove(ctx, fdst, "", fsrc, "", fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs, false)
 		if err != nil {
 			return err
 		}
@@ -920,7 +943,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		// Next pass does a copy only
 		deleteMode = fs.DeleteModeOff
 	}
-	do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs)
+	do, err := newSyncCopyMove(ctx, fdst, "", fsrc, "", deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs, false)
 	if err != nil {
 		return err
 	}
@@ -937,6 +960,15 @@ func CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error
 	return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs)
 }
 
+// CopyDir2 copies files from  fsrc/remoteSrc into fdst/remoteDst.
+func CopyDir2(ctx context.Context, fdst fs.Fs, remoteDst string, fsrc fs.Fs, remoteSrc string, copyEmptySrcDirs bool) error {
+	do, err := newSyncCopyMove(ctx, fdst, remoteDst, fsrc, remoteSrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs, true)
+	if err != nil {
+		return err
+	}
+	return do.run()
+}
+
 // moveDir moves fsrc into fdst
 func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
 	return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)