2018-01-12 16:30:54 +00:00
|
|
|
// Package sync is the implementation of sync/copy/move
|
|
|
|
package sync
|
2016-07-04 12:12:33 +00:00
|
|
|
|
|
|
|
import (
|
2018-04-06 18:13:27 +00:00
|
|
|
"context"
|
2017-01-03 17:35:12 +00:00
|
|
|
"fmt"
|
2017-08-09 20:06:39 +00:00
|
|
|
"sort"
|
2016-07-04 12:12:33 +00:00
|
|
|
"sync"
|
2016-11-05 18:03:55 +00:00
|
|
|
|
2018-01-12 16:30:54 +00:00
|
|
|
"github.com/ncw/rclone/fs"
|
|
|
|
"github.com/ncw/rclone/fs/accounting"
|
|
|
|
"github.com/ncw/rclone/fs/filter"
|
|
|
|
"github.com/ncw/rclone/fs/fserrors"
|
|
|
|
"github.com/ncw/rclone/fs/hash"
|
|
|
|
"github.com/ncw/rclone/fs/march"
|
|
|
|
"github.com/ncw/rclone/fs/operations"
|
2016-11-05 18:03:55 +00:00
|
|
|
"github.com/pkg/errors"
|
2016-07-04 12:12:33 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type syncCopyMove struct {
|
|
|
|
// parameters
|
2018-01-12 16:30:54 +00:00
|
|
|
fdst fs.Fs
|
|
|
|
fsrc fs.Fs
|
|
|
|
deleteMode fs.DeleteMode // how we are doing deletions
|
2017-11-27 11:42:02 +00:00
|
|
|
DoMove bool
|
|
|
|
deleteEmptySrcDirs bool
|
|
|
|
dir string
|
2016-07-04 12:12:33 +00:00
|
|
|
// internal state
|
2018-01-12 16:30:54 +00:00
|
|
|
ctx context.Context // internal context for controlling go-routines
|
|
|
|
cancel func() // cancel the context
|
|
|
|
deletersWg sync.WaitGroup // for delete before go routine
|
|
|
|
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
|
|
|
|
trackRenames bool // set if we should do server side renames
|
|
|
|
dstFilesMu sync.Mutex // protect dstFiles
|
|
|
|
dstFiles map[string]fs.Object // dst files, always filled
|
|
|
|
srcFiles map[string]fs.Object // src files, only used if deleteBefore
|
|
|
|
srcFilesChan chan fs.Object // passes src objects
|
|
|
|
srcFilesResult chan error // error result of src listing
|
|
|
|
dstFilesResult chan error // error result of dst listing
|
|
|
|
dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs
|
|
|
|
dstEmptyDirs []fs.DirEntry // potentially empty directories
|
|
|
|
srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs
|
|
|
|
srcEmptyDirs []fs.DirEntry // potentially empty directories
|
|
|
|
checkerWg sync.WaitGroup // wait for checkers
|
|
|
|
toBeChecked fs.ObjectPairChan // checkers channel
|
|
|
|
transfersWg sync.WaitGroup // wait for transfers
|
|
|
|
toBeUploaded fs.ObjectPairChan // copiers channel
|
|
|
|
errorMu sync.Mutex // Mutex covering the errors variables
|
|
|
|
err error // normal error from copy process
|
|
|
|
noRetryErr error // error with NoRetry set
|
|
|
|
fatalErr error // fatal error
|
|
|
|
commonHash hash.Type // common hash type between src and dst
|
|
|
|
renameMapMu sync.Mutex // mutex to protect the below
|
|
|
|
renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames
|
|
|
|
renamerWg sync.WaitGroup // wait for renamers
|
|
|
|
toBeRenamed fs.ObjectPairChan // renamers channel
|
|
|
|
trackRenamesWg sync.WaitGroup // wg for background track renames
|
|
|
|
trackRenamesCh chan fs.Object // objects are pumped in here
|
|
|
|
renameCheck []fs.Object // accumulate files to check for rename here
|
|
|
|
backupDir fs.Fs // place to store overwrites/deletes
|
|
|
|
suffix string // suffix to add to files placed in backupDir
|
|
|
|
}
|
|
|
|
|
|
|
|
func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool) (*syncCopyMove, error) {
|
2016-07-04 12:12:33 +00:00
|
|
|
s := &syncCopyMove{
|
2017-11-27 11:42:02 +00:00
|
|
|
fdst: fdst,
|
|
|
|
fsrc: fsrc,
|
|
|
|
deleteMode: deleteMode,
|
|
|
|
DoMove: DoMove,
|
|
|
|
deleteEmptySrcDirs: deleteEmptySrcDirs,
|
|
|
|
dir: "",
|
2018-01-12 16:30:54 +00:00
|
|
|
srcFilesChan: make(chan fs.Object, fs.Config.Checkers+fs.Config.Transfers),
|
2017-11-27 11:42:02 +00:00
|
|
|
srcFilesResult: make(chan error, 1),
|
|
|
|
dstFilesResult: make(chan error, 1),
|
2018-01-12 16:30:54 +00:00
|
|
|
toBeChecked: make(fs.ObjectPairChan, fs.Config.Transfers),
|
|
|
|
toBeUploaded: make(fs.ObjectPairChan, fs.Config.Transfers),
|
|
|
|
deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
|
|
|
|
trackRenames: fs.Config.TrackRenames,
|
2017-11-27 11:42:02 +00:00
|
|
|
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
|
2018-01-12 16:30:54 +00:00
|
|
|
toBeRenamed: make(fs.ObjectPairChan, fs.Config.Transfers),
|
|
|
|
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
2017-09-01 14:47:32 +00:00
|
|
|
s.ctx, s.cancel = context.WithCancel(context.Background())
|
2017-01-02 16:37:59 +00:00
|
|
|
if s.trackRenames {
|
2017-01-10 20:03:55 +00:00
|
|
|
// Don't track renames for remotes without server-side move support.
|
2018-01-12 16:30:54 +00:00
|
|
|
if !operations.CanServerSideMove(fdst) {
|
|
|
|
fs.Errorf(fdst, "Ignoring --track-renames as the destination does not support server-side move or copy")
|
2017-01-02 16:37:59 +00:00
|
|
|
s.trackRenames = false
|
|
|
|
}
|
2018-01-18 20:27:52 +00:00
|
|
|
if s.commonHash == hash.None {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(fdst, "Ignoring --track-renames as the source and destination do not have a common hash")
|
2017-01-02 16:37:59 +00:00
|
|
|
s.trackRenames = false
|
|
|
|
}
|
2018-03-17 10:31:23 +00:00
|
|
|
if s.deleteMode == fs.DeleteModeOff {
|
|
|
|
fs.Errorf(fdst, "Ignoring --track-renames as it doesn't work with copy or move, only sync")
|
|
|
|
s.trackRenames = false
|
|
|
|
}
|
2017-01-02 16:37:59 +00:00
|
|
|
}
|
2017-01-25 19:35:14 +00:00
|
|
|
if s.trackRenames {
|
|
|
|
// track renames needs delete after
|
2018-01-12 16:30:54 +00:00
|
|
|
if s.deleteMode != fs.DeleteModeOff {
|
|
|
|
s.deleteMode = fs.DeleteModeAfter
|
2017-01-25 19:35:14 +00:00
|
|
|
}
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2017-01-10 21:47:03 +00:00
|
|
|
// Make Fs for --backup-dir if required
|
2018-01-12 16:30:54 +00:00
|
|
|
if fs.Config.BackupDir != "" {
|
2017-01-10 21:47:03 +00:00
|
|
|
var err error
|
2018-01-12 16:30:54 +00:00
|
|
|
s.backupDir, err = fs.NewFs(fs.Config.BackupDir)
|
2017-01-10 21:47:03 +00:00
|
|
|
if err != nil {
|
2018-01-12 16:30:54 +00:00
|
|
|
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", fs.Config.BackupDir, err))
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
if !operations.CanServerSideMove(s.backupDir) {
|
|
|
|
return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server side move or copy"))
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
if !operations.SameConfig(fdst, s.backupDir) {
|
|
|
|
return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination"))
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
if operations.Overlapping(fdst, s.backupDir) {
|
|
|
|
return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap"))
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
if operations.Overlapping(fsrc, s.backupDir) {
|
|
|
|
return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap"))
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
s.suffix = fs.Config.Suffix
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
|
|
|
return s, nil
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
2017-09-01 14:47:32 +00:00
|
|
|
// Check to see if the context has been cancelled
|
2016-07-04 12:12:33 +00:00
|
|
|
func (s *syncCopyMove) aborting() bool {
|
|
|
|
select {
|
2017-09-01 14:47:32 +00:00
|
|
|
case <-s.ctx.Done():
|
2016-07-04 12:12:33 +00:00
|
|
|
return true
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2017-01-03 17:35:12 +00:00
|
|
|
// This reads the map and pumps it into the channel passed in, closing
|
|
|
|
// the channel at the end
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) pumpMapToChan(files map[string]fs.Object, out chan<- fs.Object) {
|
2016-07-04 12:12:33 +00:00
|
|
|
outer:
|
2017-01-03 17:35:12 +00:00
|
|
|
for _, o := range files {
|
2016-07-04 12:12:33 +00:00
|
|
|
if s.aborting() {
|
|
|
|
break outer
|
|
|
|
}
|
|
|
|
select {
|
2017-01-03 17:35:12 +00:00
|
|
|
case out <- o:
|
2017-09-01 14:47:32 +00:00
|
|
|
case <-s.ctx.Done():
|
2016-07-04 12:12:33 +00:00
|
|
|
break outer
|
|
|
|
}
|
|
|
|
}
|
2017-01-03 17:35:12 +00:00
|
|
|
close(out)
|
|
|
|
s.srcFilesResult <- nil
|
|
|
|
}
|
|
|
|
|
2016-07-04 12:12:33 +00:00
|
|
|
// This checks the types of errors returned while copying files
|
|
|
|
func (s *syncCopyMove) processError(err error) {
|
|
|
|
if err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
s.errorMu.Lock()
|
|
|
|
defer s.errorMu.Unlock()
|
|
|
|
switch {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fserrors.IsFatalError(err):
|
2016-08-18 17:56:57 +00:00
|
|
|
if !s.aborting() {
|
2017-09-01 14:47:32 +00:00
|
|
|
s.cancel()
|
2016-08-18 17:56:57 +00:00
|
|
|
}
|
2016-07-04 12:12:33 +00:00
|
|
|
s.fatalErr = err
|
2018-01-12 16:30:54 +00:00
|
|
|
case fserrors.IsNoRetryError(err):
|
2016-07-04 12:12:33 +00:00
|
|
|
s.noRetryErr = err
|
|
|
|
default:
|
|
|
|
s.err = err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-03 17:35:12 +00:00
|
|
|
// Returns the current error (if any) in the order of prececedence
|
|
|
|
// fatalErr
|
|
|
|
// normal error
|
|
|
|
// noRetryErr
|
|
|
|
func (s *syncCopyMove) currentError() error {
|
|
|
|
s.errorMu.Lock()
|
|
|
|
defer s.errorMu.Unlock()
|
|
|
|
if s.fatalErr != nil {
|
|
|
|
return s.fatalErr
|
|
|
|
}
|
|
|
|
if s.err != nil {
|
|
|
|
return s.err
|
|
|
|
}
|
|
|
|
return s.noRetryErr
|
|
|
|
}
|
|
|
|
|
2016-07-04 12:12:33 +00:00
|
|
|
// pairChecker reads Objects~s on in send to out if they need transferring.
|
|
|
|
//
|
|
|
|
// FIXME potentially doing lots of hashes at once
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) pairChecker(in fs.ObjectPairChan, out fs.ObjectPairChan, wg *sync.WaitGroup) {
|
2016-07-04 12:12:33 +00:00
|
|
|
defer wg.Done()
|
|
|
|
for {
|
|
|
|
if s.aborting() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case pair, ok := <-in:
|
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
src := pair.Src
|
|
|
|
accounting.Stats.Checking(src.Remote())
|
2016-10-03 18:58:44 +00:00
|
|
|
// Check to see if can store this
|
|
|
|
if src.Storable() {
|
2018-01-12 16:30:54 +00:00
|
|
|
if operations.NeedTransfer(pair.Dst, pair.Src) {
|
2017-09-02 08:29:01 +00:00
|
|
|
// If files are treated as immutable, fail if destination exists and does not match
|
2018-01-12 16:30:54 +00:00
|
|
|
if fs.Config.Immutable && pair.Dst != nil {
|
|
|
|
fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
|
|
|
|
s.processError(fs.ErrorImmutableModified)
|
2017-09-02 08:29:01 +00:00
|
|
|
} else {
|
|
|
|
// If destination already exists, then we must move it into --backup-dir if required
|
2018-01-12 16:30:54 +00:00
|
|
|
if pair.Dst != nil && s.backupDir != nil {
|
|
|
|
remoteWithSuffix := pair.Dst.Remote() + s.suffix
|
2017-09-02 08:29:01 +00:00
|
|
|
overwritten, _ := s.backupDir.NewObject(remoteWithSuffix)
|
2018-01-12 16:30:54 +00:00
|
|
|
_, err := operations.Move(s.backupDir, overwritten, remoteWithSuffix, pair.Dst)
|
2017-09-02 08:29:01 +00:00
|
|
|
if err != nil {
|
|
|
|
s.processError(err)
|
|
|
|
} else {
|
|
|
|
// If successful zero out the dst as it is no longer there and copy the file
|
2018-01-12 16:30:54 +00:00
|
|
|
pair.Dst = nil
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case out <- pair:
|
|
|
|
}
|
2017-09-02 08:29:01 +00:00
|
|
|
}
|
2017-01-10 21:47:03 +00:00
|
|
|
} else {
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case out <- pair:
|
|
|
|
}
|
2017-01-10 21:47:03 +00:00
|
|
|
}
|
|
|
|
}
|
2016-10-03 18:58:44 +00:00
|
|
|
} else {
|
|
|
|
// If moving need to delete the files we don't need to copy
|
|
|
|
if s.DoMove {
|
|
|
|
// Delete src if no error on copy
|
2018-01-12 16:30:54 +00:00
|
|
|
s.processError(operations.DeleteFile(src))
|
2016-10-03 18:58:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
accounting.Stats.DoneChecking(src.Remote())
|
2017-09-01 14:47:32 +00:00
|
|
|
case <-s.ctx.Done():
|
2016-07-04 12:12:33 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-03 17:35:12 +00:00
|
|
|
// pairRenamer reads Objects~s on in and attempts to rename them,
|
|
|
|
// otherwise it sends them out if they need transferring.
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) pairRenamer(in fs.ObjectPairChan, out fs.ObjectPairChan, wg *sync.WaitGroup) {
|
2017-01-03 17:35:12 +00:00
|
|
|
defer wg.Done()
|
|
|
|
for {
|
|
|
|
if s.aborting() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case pair, ok := <-in:
|
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
src := pair.Src
|
2017-01-03 17:35:12 +00:00
|
|
|
if !s.tryRename(src) {
|
|
|
|
// pass on if not renamed
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case out <- pair:
|
|
|
|
}
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2017-09-01 14:47:32 +00:00
|
|
|
case <-s.ctx.Done():
|
2017-01-03 17:35:12 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-10-22 16:53:10 +00:00
|
|
|
// pairCopyOrMove reads Objects on in and moves or copies them.
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) pairCopyOrMove(in fs.ObjectPairChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
2016-07-04 12:12:33 +00:00
|
|
|
defer wg.Done()
|
2016-10-22 16:53:10 +00:00
|
|
|
var err error
|
2016-07-04 12:12:33 +00:00
|
|
|
for {
|
|
|
|
if s.aborting() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case pair, ok := <-in:
|
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
src := pair.Src
|
|
|
|
accounting.Stats.Transferring(src.Remote())
|
2016-10-22 16:53:10 +00:00
|
|
|
if s.DoMove {
|
2018-01-12 16:30:54 +00:00
|
|
|
_, err = operations.Move(fdst, pair.Dst, src.Remote(), src)
|
2016-07-04 12:12:33 +00:00
|
|
|
} else {
|
2018-01-12 16:30:54 +00:00
|
|
|
_, err = operations.Copy(fdst, pair.Dst, src.Remote(), src)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
2016-10-22 16:53:10 +00:00
|
|
|
s.processError(err)
|
2018-01-12 16:30:54 +00:00
|
|
|
accounting.Stats.DoneTransferring(src.Remote(), err == nil)
|
2017-09-01 14:47:32 +00:00
|
|
|
case <-s.ctx.Done():
|
2016-07-04 12:12:33 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This starts the background checkers.
|
|
|
|
func (s *syncCopyMove) startCheckers() {
|
2018-01-12 16:30:54 +00:00
|
|
|
s.checkerWg.Add(fs.Config.Checkers)
|
|
|
|
for i := 0; i < fs.Config.Checkers; i++ {
|
2016-07-04 12:12:33 +00:00
|
|
|
go s.pairChecker(s.toBeChecked, s.toBeUploaded, &s.checkerWg)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This stops the background checkers
|
|
|
|
func (s *syncCopyMove) stopCheckers() {
|
|
|
|
close(s.toBeChecked)
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(s.fdst, "Waiting for checks to finish")
|
2016-07-04 12:12:33 +00:00
|
|
|
s.checkerWg.Wait()
|
|
|
|
}
|
|
|
|
|
|
|
|
// This starts the background transfers
|
|
|
|
func (s *syncCopyMove) startTransfers() {
|
2018-01-12 16:30:54 +00:00
|
|
|
s.transfersWg.Add(fs.Config.Transfers)
|
|
|
|
for i := 0; i < fs.Config.Transfers; i++ {
|
2016-10-22 16:53:10 +00:00
|
|
|
go s.pairCopyOrMove(s.toBeUploaded, s.fdst, &s.transfersWg)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This stops the background transfers
|
|
|
|
func (s *syncCopyMove) stopTransfers() {
|
|
|
|
close(s.toBeUploaded)
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(s.fdst, "Waiting for transfers to finish")
|
2016-10-22 16:53:10 +00:00
|
|
|
s.transfersWg.Wait()
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
2017-01-03 17:35:12 +00:00
|
|
|
// This starts the background renamers.
|
|
|
|
func (s *syncCopyMove) startRenamers() {
|
|
|
|
if !s.trackRenames {
|
|
|
|
return
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
s.renamerWg.Add(fs.Config.Checkers)
|
|
|
|
for i := 0; i < fs.Config.Checkers; i++ {
|
2017-01-03 17:35:12 +00:00
|
|
|
go s.pairRenamer(s.toBeRenamed, s.toBeUploaded, &s.renamerWg)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This stops the background renamers
|
|
|
|
func (s *syncCopyMove) stopRenamers() {
|
|
|
|
if !s.trackRenames {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
close(s.toBeRenamed)
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(s.fdst, "Waiting for renames to finish")
|
2017-01-03 17:35:12 +00:00
|
|
|
s.renamerWg.Wait()
|
|
|
|
}
|
|
|
|
|
2017-01-24 11:04:09 +00:00
|
|
|
// This starts the collection of possible renames
|
|
|
|
func (s *syncCopyMove) startTrackRenames() {
|
|
|
|
if !s.trackRenames {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
s.trackRenamesWg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer s.trackRenamesWg.Done()
|
|
|
|
for o := range s.trackRenamesCh {
|
|
|
|
s.renameCheck = append(s.renameCheck, o)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
// This stops the background rename collection
|
|
|
|
func (s *syncCopyMove) stopTrackRenames() {
|
|
|
|
if !s.trackRenames {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
close(s.trackRenamesCh)
|
|
|
|
s.trackRenamesWg.Wait()
|
|
|
|
}
|
|
|
|
|
2017-01-25 19:59:53 +00:00
|
|
|
// This starts the background deletion of files for --delete-during
|
|
|
|
func (s *syncCopyMove) startDeleters() {
|
2018-01-12 16:30:54 +00:00
|
|
|
if s.deleteMode != fs.DeleteModeDuring && s.deleteMode != fs.DeleteModeOnly {
|
2017-01-25 19:59:53 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
s.deletersWg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer s.deletersWg.Done()
|
2018-01-12 16:30:54 +00:00
|
|
|
err := operations.DeleteFilesWithBackupDir(s.deleteFilesCh, s.backupDir)
|
2017-01-25 19:59:53 +00:00
|
|
|
s.processError(err)
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
// This stops the background deleters
|
|
|
|
func (s *syncCopyMove) stopDeleters() {
|
2018-01-12 16:30:54 +00:00
|
|
|
if s.deleteMode != fs.DeleteModeDuring && s.deleteMode != fs.DeleteModeOnly {
|
2017-01-25 19:59:53 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
close(s.deleteFilesCh)
|
|
|
|
s.deletersWg.Wait()
|
|
|
|
}
|
|
|
|
|
2016-07-04 12:12:33 +00:00
|
|
|
// This deletes the files in the dstFiles map. If checkSrcMap is set
|
|
|
|
// then it checks to see if they exist first in srcFiles the source
|
|
|
|
// file map, otherwise it unconditionally deletes them. If
|
|
|
|
// checkSrcMap is clear then it assumes that the any source files that
|
|
|
|
// have been found have been removed from dstFiles already.
|
|
|
|
func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
|
2018-03-12 23:40:19 +00:00
|
|
|
if accounting.Stats.Errored() && !fs.Config.IgnoreErrors {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)
|
|
|
|
return fs.ErrorNotDeleting
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Delete the spare files
|
2018-01-12 16:30:54 +00:00
|
|
|
toDelete := make(fs.ObjectsChan, fs.Config.Transfers)
|
2016-07-04 12:12:33 +00:00
|
|
|
go func() {
|
2018-04-21 21:01:27 +00:00
|
|
|
outer:
|
2016-07-04 12:12:33 +00:00
|
|
|
for remote, o := range s.dstFiles {
|
|
|
|
if checkSrcMap {
|
|
|
|
_, exists := s.srcFiles[remote]
|
|
|
|
if exists {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if s.aborting() {
|
|
|
|
break
|
|
|
|
}
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
break outer
|
|
|
|
case toDelete <- o:
|
|
|
|
}
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
close(toDelete)
|
|
|
|
}()
|
2018-01-12 16:30:54 +00:00
|
|
|
return operations.DeleteFilesWithBackupDir(toDelete, s.backupDir)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
2017-08-09 20:06:39 +00:00
|
|
|
// This deletes the empty directories in the slice passed in. It
|
|
|
|
// ignores any errors deleting directories
|
2018-01-12 16:30:54 +00:00
|
|
|
func deleteEmptyDirectories(f fs.Fs, entries fs.DirEntries) error {
|
2017-08-09 20:06:39 +00:00
|
|
|
if len(entries) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
2018-03-12 23:40:19 +00:00
|
|
|
if accounting.Stats.Errored() && !fs.Config.IgnoreErrors {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(f, "%v", fs.ErrorNotDeletingDirs)
|
|
|
|
return fs.ErrorNotDeletingDirs
|
2017-08-09 20:06:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Now delete the empty directories starting from the longest path
|
|
|
|
sort.Sort(entries)
|
|
|
|
var errorCount int
|
|
|
|
var okCount int
|
|
|
|
for i := len(entries) - 1; i >= 0; i-- {
|
|
|
|
entry := entries[i]
|
2018-01-12 16:30:54 +00:00
|
|
|
dir, ok := entry.(fs.Directory)
|
2017-08-09 20:06:39 +00:00
|
|
|
if ok {
|
|
|
|
// TryRmdir only deletes empty directories
|
2018-01-12 16:30:54 +00:00
|
|
|
err := operations.TryRmdir(f, dir.Remote())
|
2017-08-09 20:06:39 +00:00
|
|
|
if err != nil {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(fs.LogDirName(f, dir.Remote()), "Failed to Rmdir: %v", err)
|
2017-08-09 20:06:39 +00:00
|
|
|
errorCount++
|
|
|
|
} else {
|
|
|
|
okCount++
|
|
|
|
}
|
|
|
|
} else {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(f, "Not a directory: %v", entry)
|
2017-08-09 20:06:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if errorCount > 0 {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(f, "failed to delete %d directories", errorCount)
|
2017-08-09 20:06:39 +00:00
|
|
|
}
|
|
|
|
if okCount > 0 {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(f, "deleted %d directories", okCount)
|
2017-08-09 20:06:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-01-03 17:35:12 +00:00
|
|
|
// renameHash makes a string with the size and the hash for rename detection
|
|
|
|
//
|
|
|
|
// it may return an empty string in which case no hash could be made
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) renameHash(obj fs.Object) (hash string) {
|
2017-01-03 23:03:20 +00:00
|
|
|
var err error
|
2017-01-03 17:35:12 +00:00
|
|
|
hash, err = obj.Hash(s.commonHash)
|
|
|
|
if err != nil {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(obj, "Hash failed: %v", err)
|
2017-01-03 23:03:20 +00:00
|
|
|
return ""
|
2016-12-18 10:03:56 +00:00
|
|
|
}
|
2017-01-03 17:35:12 +00:00
|
|
|
if hash == "" {
|
2017-01-03 23:03:20 +00:00
|
|
|
return ""
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2017-01-03 23:03:20 +00:00
|
|
|
return fmt.Sprintf("%d,%s", obj.Size(), hash)
|
|
|
|
}
|
|
|
|
|
|
|
|
// pushRenameMap adds the object with hash to the rename map
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) pushRenameMap(hash string, obj fs.Object) {
|
2017-01-03 23:03:20 +00:00
|
|
|
s.renameMapMu.Lock()
|
|
|
|
s.renameMap[hash] = append(s.renameMap[hash], obj)
|
|
|
|
s.renameMapMu.Unlock()
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2016-12-18 10:03:56 +00:00
|
|
|
|
2017-01-03 23:03:20 +00:00
|
|
|
// popRenameMap finds the object with hash and pop the first match from
|
|
|
|
// renameMap or returns nil if not found.
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) popRenameMap(hash string) (dst fs.Object) {
|
2017-01-03 23:03:20 +00:00
|
|
|
s.renameMapMu.Lock()
|
|
|
|
dsts, ok := s.renameMap[hash]
|
|
|
|
if ok && len(dsts) > 0 {
|
|
|
|
dst, dsts = dsts[0], dsts[1:]
|
|
|
|
if len(dsts) > 0 {
|
|
|
|
s.renameMap[hash] = dsts
|
|
|
|
} else {
|
|
|
|
delete(s.renameMap, hash)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
s.renameMapMu.Unlock()
|
|
|
|
return dst
|
|
|
|
}
|
|
|
|
|
|
|
|
// makeRenameMap builds a map of the destination files by hash that
|
2017-01-24 11:04:09 +00:00
|
|
|
// match sizes in the slice of objects in s.renameCheck
|
|
|
|
func (s *syncCopyMove) makeRenameMap() {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(s.fdst, "Making map for --track-renames")
|
2016-12-18 10:03:56 +00:00
|
|
|
|
2017-01-03 23:03:20 +00:00
|
|
|
// first make a map of possible sizes we need to check
|
|
|
|
possibleSizes := map[int64]struct{}{}
|
2017-01-24 11:04:09 +00:00
|
|
|
for _, obj := range s.renameCheck {
|
2017-01-03 23:03:20 +00:00
|
|
|
possibleSizes[obj.Size()] = struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// pump all the dstFiles into in
|
2018-01-12 16:30:54 +00:00
|
|
|
in := make(chan fs.Object, fs.Config.Checkers)
|
2017-01-03 17:35:12 +00:00
|
|
|
go s.pumpMapToChan(s.dstFiles, in)
|
2016-12-18 10:03:56 +00:00
|
|
|
|
2017-01-03 23:03:20 +00:00
|
|
|
// now make a map of size,hash for all dstFiles
|
2018-01-12 16:30:54 +00:00
|
|
|
s.renameMap = make(map[string][]fs.Object)
|
2017-01-03 17:35:12 +00:00
|
|
|
var wg sync.WaitGroup
|
2018-01-12 16:30:54 +00:00
|
|
|
wg.Add(fs.Config.Transfers)
|
|
|
|
for i := 0; i < fs.Config.Transfers; i++ {
|
2016-12-18 10:03:56 +00:00
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
2017-01-03 23:03:20 +00:00
|
|
|
for obj := range in {
|
2018-01-12 16:30:54 +00:00
|
|
|
// only create hash for dst fs.Object if its size could match
|
2017-01-03 23:03:20 +00:00
|
|
|
if _, found := possibleSizes[obj.Size()]; found {
|
2018-01-12 16:30:54 +00:00
|
|
|
accounting.Stats.Checking(obj.Remote())
|
2017-01-03 23:03:20 +00:00
|
|
|
hash := s.renameHash(obj)
|
|
|
|
if hash != "" {
|
|
|
|
s.pushRenameMap(hash, obj)
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
accounting.Stats.DoneChecking(obj.Remote())
|
2016-12-18 10:03:56 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
wg.Wait()
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(s.fdst, "Finished making map for --track-renames")
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
|
|
|
|
2017-01-03 23:03:20 +00:00
|
|
|
// tryRename renames a src object when doing track renames if
|
|
|
|
// possible, it returns true if the object was renamed.
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) tryRename(src fs.Object) bool {
|
|
|
|
accounting.Stats.Checking(src.Remote())
|
|
|
|
defer accounting.Stats.DoneChecking(src.Remote())
|
2016-12-18 10:03:56 +00:00
|
|
|
|
2017-02-22 19:28:22 +00:00
|
|
|
// Calculate the hash of the src object
|
2017-01-03 23:03:20 +00:00
|
|
|
hash := s.renameHash(src)
|
2017-01-03 17:35:12 +00:00
|
|
|
if hash == "" {
|
2017-01-03 23:03:20 +00:00
|
|
|
return false
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2017-01-03 23:03:20 +00:00
|
|
|
|
2017-02-22 19:28:22 +00:00
|
|
|
// Get a match on fdst
|
2017-01-03 23:03:20 +00:00
|
|
|
dst := s.popRenameMap(hash)
|
|
|
|
if dst == nil {
|
|
|
|
return false
|
2017-01-03 17:35:12 +00:00
|
|
|
}
|
2017-01-03 23:03:20 +00:00
|
|
|
|
2017-02-22 19:28:22 +00:00
|
|
|
// Find dst object we are about to overwrite if it exists
|
|
|
|
dstOverwritten, _ := s.fdst.NewObject(src.Remote())
|
|
|
|
|
|
|
|
// Rename dst to have name src.Remote()
|
2018-01-12 16:30:54 +00:00
|
|
|
_, err := operations.Move(s.fdst, dstOverwritten, src.Remote(), dst)
|
2017-01-03 23:03:20 +00:00
|
|
|
if err != nil {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(src, "Failed to rename to %q: %v", dst.Remote(), err)
|
2017-01-03 23:03:20 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// remove file from dstFiles if present
|
|
|
|
s.dstFilesMu.Lock()
|
|
|
|
delete(s.dstFiles, dst.Remote())
|
|
|
|
s.dstFilesMu.Unlock()
|
|
|
|
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(src, "Renamed from %q", dst.Remote())
|
2017-01-03 23:03:20 +00:00
|
|
|
return true
|
2016-12-18 10:03:56 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 19:59:53 +00:00
|
|
|
// Syncs fsrc into fdst
|
|
|
|
//
|
|
|
|
// If Delete is true then it deletes any files in fdst that aren't in fsrc
|
|
|
|
//
|
|
|
|
// If DoMove is true then files will be moved instead of copied
|
|
|
|
//
|
|
|
|
// dir is the start directory, "" for root
|
2017-06-13 13:35:51 +00:00
|
|
|
func (s *syncCopyMove) run() error {
|
2018-01-12 16:30:54 +00:00
|
|
|
if operations.Same(s.fdst, s.fsrc) {
|
|
|
|
fs.Errorf(s.fdst, "Nothing to do as source and destination are the same")
|
2017-01-25 19:59:53 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start background checking and transferring pipeline
|
|
|
|
s.startCheckers()
|
|
|
|
s.startRenamers()
|
|
|
|
s.startTransfers()
|
|
|
|
s.startDeleters()
|
2018-01-12 16:30:54 +00:00
|
|
|
s.dstFiles = make(map[string]fs.Object)
|
2017-01-25 19:59:53 +00:00
|
|
|
|
|
|
|
s.startTrackRenames()
|
|
|
|
|
2017-09-01 14:47:32 +00:00
|
|
|
// set up a march over fdst and fsrc
|
2018-01-12 16:30:54 +00:00
|
|
|
m := march.New(s.ctx, s.fdst, s.fsrc, s.dir, s)
|
|
|
|
m.Run()
|
2017-01-25 19:59:53 +00:00
|
|
|
|
|
|
|
s.stopTrackRenames()
|
|
|
|
if s.trackRenames {
|
|
|
|
// Build the map of the remaining dstFiles by hash
|
|
|
|
s.makeRenameMap()
|
|
|
|
// Attempt renames for all the files which don't have a matching dst
|
|
|
|
for _, src := range s.renameCheck {
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
break
|
|
|
|
case s.toBeRenamed <- fs.ObjectPair{Src: src, Dst: nil}:
|
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop background checking and transferring pipeline
|
|
|
|
s.stopCheckers()
|
|
|
|
s.stopRenamers()
|
|
|
|
s.stopTransfers()
|
|
|
|
s.stopDeleters()
|
|
|
|
|
|
|
|
// Delete files after
|
2018-01-12 16:30:54 +00:00
|
|
|
if s.deleteMode == fs.DeleteModeAfter {
|
2018-03-12 23:40:19 +00:00
|
|
|
if s.currentError() != nil && !fs.Config.IgnoreErrors {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)
|
2017-01-25 19:59:53 +00:00
|
|
|
} else {
|
|
|
|
s.processError(s.deleteFiles(false))
|
|
|
|
}
|
|
|
|
}
|
2017-08-09 20:06:39 +00:00
|
|
|
|
|
|
|
// Prune empty directories
|
2018-01-12 16:30:54 +00:00
|
|
|
if s.deleteMode != fs.DeleteModeOff {
|
2018-03-12 23:40:19 +00:00
|
|
|
if s.currentError() != nil && !fs.Config.IgnoreErrors {
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeletingDirs)
|
2017-08-09 20:06:39 +00:00
|
|
|
} else {
|
|
|
|
s.processError(deleteEmptyDirectories(s.fdst, s.dstEmptyDirs))
|
|
|
|
}
|
|
|
|
}
|
2017-11-06 16:18:13 +00:00
|
|
|
|
2017-11-27 11:42:02 +00:00
|
|
|
// Delete empty fsrc subdirectories
|
|
|
|
// if DoMove and --delete-empty-src-dirs flag is set
|
|
|
|
if s.DoMove && s.deleteEmptySrcDirs {
|
2017-11-23 05:22:01 +00:00
|
|
|
//delete empty subdirectories that were part of the move
|
2017-11-06 16:18:13 +00:00
|
|
|
s.processError(deleteEmptyDirectories(s.fsrc, s.srcEmptyDirs))
|
|
|
|
}
|
2018-04-21 21:01:27 +00:00
|
|
|
|
|
|
|
// cancel the context to free resources
|
|
|
|
s.cancel()
|
2017-01-25 19:59:53 +00:00
|
|
|
return s.currentError()
|
|
|
|
}
|
|
|
|
|
2017-09-01 14:21:46 +00:00
|
|
|
// DstOnly have an object which is in the destination only
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
|
|
|
|
if s.deleteMode == fs.DeleteModeOff {
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
switch x := dst.(type) {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Object:
|
2017-01-25 19:59:53 +00:00
|
|
|
switch s.deleteMode {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.DeleteModeAfter:
|
2017-01-25 19:59:53 +00:00
|
|
|
// record object as needs deleting
|
|
|
|
s.dstFilesMu.Lock()
|
|
|
|
s.dstFiles[x.Remote()] = x
|
|
|
|
s.dstFilesMu.Unlock()
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.DeleteModeDuring, fs.DeleteModeOnly:
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case s.deleteFilesCh <- x:
|
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
default:
|
|
|
|
panic(fmt.Sprintf("unexpected delete mode %d", s.deleteMode))
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Directory:
|
2017-01-25 19:59:53 +00:00
|
|
|
// Do the same thing to the entire contents of the directory
|
2017-08-09 20:06:39 +00:00
|
|
|
// Record directory as it is potentially empty and needs deleting
|
|
|
|
if s.fdst.Features().CanHaveEmptyDirectories {
|
|
|
|
s.dstEmptyDirsMu.Lock()
|
|
|
|
s.dstEmptyDirs = append(s.dstEmptyDirs, dst)
|
|
|
|
s.dstEmptyDirsMu.Unlock()
|
|
|
|
}
|
2017-09-01 14:21:46 +00:00
|
|
|
return true
|
2017-01-25 19:59:53 +00:00
|
|
|
default:
|
|
|
|
panic("Bad object in DirEntries")
|
|
|
|
|
|
|
|
}
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
|
2017-09-01 14:21:46 +00:00
|
|
|
// SrcOnly have an object which is in the source only
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
|
|
|
|
if s.deleteMode == fs.DeleteModeOnly {
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
switch x := src.(type) {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Object:
|
2017-01-25 19:59:53 +00:00
|
|
|
if s.trackRenames {
|
|
|
|
// Save object to check for a rename later
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case s.trackRenamesCh <- x:
|
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
} else {
|
|
|
|
// No need to check since doesn't exist
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case s.toBeUploaded <- fs.ObjectPair{Src: x, Dst: nil}:
|
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Directory:
|
2017-01-25 19:59:53 +00:00
|
|
|
// Do the same thing to the entire contents of the directory
|
2017-11-06 16:18:13 +00:00
|
|
|
// Record the directory for deletion
|
|
|
|
s.srcEmptyDirsMu.Lock()
|
|
|
|
s.srcEmptyDirs = append(s.srcEmptyDirs, src)
|
|
|
|
s.srcEmptyDirsMu.Unlock()
|
2017-09-01 14:21:46 +00:00
|
|
|
return true
|
2017-01-25 19:59:53 +00:00
|
|
|
default:
|
|
|
|
panic("Bad object in DirEntries")
|
|
|
|
}
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
|
2017-09-01 14:21:46 +00:00
|
|
|
// Match is called when src and dst are present, so sync src to dst
|
2018-01-12 16:30:54 +00:00
|
|
|
func (s *syncCopyMove) Match(dst, src fs.DirEntry) (recurse bool) {
|
2017-01-25 19:59:53 +00:00
|
|
|
switch srcX := src.(type) {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Object:
|
|
|
|
if s.deleteMode == fs.DeleteModeOnly {
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
dstX, ok := dst.(fs.Object)
|
2017-01-25 19:59:53 +00:00
|
|
|
if ok {
|
2018-04-21 21:01:27 +00:00
|
|
|
select {
|
|
|
|
case <-s.ctx.Done():
|
|
|
|
return
|
|
|
|
case s.toBeChecked <- fs.ObjectPair{Src: srcX, Dst: dstX}:
|
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
} else {
|
|
|
|
// FIXME src is file, dst is directory
|
|
|
|
err := errors.New("can't overwrite directory with file")
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(dst, "%v", err)
|
2017-01-25 19:59:53 +00:00
|
|
|
s.processError(err)
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.Directory:
|
2017-01-25 19:59:53 +00:00
|
|
|
// Do the same thing to the entire contents of the directory
|
2018-01-12 16:30:54 +00:00
|
|
|
_, ok := dst.(fs.Directory)
|
2017-01-25 19:59:53 +00:00
|
|
|
if ok {
|
2017-11-06 16:18:13 +00:00
|
|
|
// Record the src directory for deletion
|
|
|
|
s.srcEmptyDirsMu.Lock()
|
|
|
|
s.srcEmptyDirs = append(s.srcEmptyDirs, src)
|
|
|
|
s.srcEmptyDirsMu.Unlock()
|
2017-09-01 14:21:46 +00:00
|
|
|
return true
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
2017-09-01 14:21:46 +00:00
|
|
|
// FIXME src is dir, dst is file
|
|
|
|
err := errors.New("can't overwrite file with directory")
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Errorf(dst, "%v", err)
|
2017-09-01 14:21:46 +00:00
|
|
|
s.processError(err)
|
2017-01-25 19:59:53 +00:00
|
|
|
default:
|
|
|
|
panic("Bad object in DirEntries")
|
|
|
|
}
|
2017-09-01 14:21:46 +00:00
|
|
|
return false
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 19:35:14 +00:00
|
|
|
// Syncs fsrc into fdst
|
|
|
|
//
|
|
|
|
// If Delete is true then it deletes any files in fdst that aren't in fsrc
|
|
|
|
//
|
|
|
|
// If DoMove is true then files will be moved instead of copied
|
|
|
|
//
|
|
|
|
// dir is the start directory, "" for root
|
2018-01-12 16:30:54 +00:00
|
|
|
func runSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool) error {
|
|
|
|
if deleteMode != fs.DeleteModeOff && DoMove {
|
|
|
|
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
|
2017-01-25 19:35:14 +00:00
|
|
|
}
|
2017-01-25 19:59:53 +00:00
|
|
|
// Run an extra pass to delete only
|
2018-01-12 16:30:54 +00:00
|
|
|
if deleteMode == fs.DeleteModeBefore {
|
|
|
|
if fs.Config.TrackRenames {
|
|
|
|
return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
|
|
|
// only delete stuff during in this pass
|
2018-01-12 16:30:54 +00:00
|
|
|
do, err := newSyncCopyMove(fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs)
|
2017-01-25 19:59:53 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-06-13 13:35:51 +00:00
|
|
|
err = do.run()
|
2017-01-25 19:59:53 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Next pass does a copy only
|
2018-01-12 16:30:54 +00:00
|
|
|
deleteMode = fs.DeleteModeOff
|
2017-01-25 19:59:53 +00:00
|
|
|
}
|
2017-11-27 11:42:02 +00:00
|
|
|
do, err := newSyncCopyMove(fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs)
|
2017-01-10 21:47:03 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-06-13 13:35:51 +00:00
|
|
|
return do.run()
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 19:35:14 +00:00
|
|
|
// Sync fsrc into fdst
|
2018-01-12 16:30:54 +00:00
|
|
|
func Sync(fdst, fsrc fs.Fs) error {
|
|
|
|
return runSyncCopyMove(fdst, fsrc, fs.Config.DeleteMode, false, false)
|
2017-01-25 19:35:14 +00:00
|
|
|
}
|
|
|
|
|
2016-07-04 12:12:33 +00:00
|
|
|
// CopyDir copies fsrc into fdst
|
2018-01-12 16:30:54 +00:00
|
|
|
func CopyDir(fdst, fsrc fs.Fs) error {
|
|
|
|
return runSyncCopyMove(fdst, fsrc, fs.DeleteModeOff, false, false)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// moveDir moves fsrc into fdst
|
2018-01-12 16:30:54 +00:00
|
|
|
func moveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool) error {
|
|
|
|
return runSyncCopyMove(fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// MoveDir moves fsrc into fdst
|
2018-01-12 16:30:54 +00:00
|
|
|
func MoveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool) error {
|
|
|
|
if operations.Same(fdst, fsrc) {
|
|
|
|
fs.Errorf(fdst, "Nothing to do as source and destination are the same")
|
2016-07-04 12:12:33 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// First attempt to use DirMover if exists, same Fs and no filters are active
|
2018-01-12 16:30:54 +00:00
|
|
|
if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && filter.Active.InActive() {
|
|
|
|
if fs.Config.DryRun {
|
|
|
|
fs.Logf(fdst, "Not doing server side directory move as --dry-run")
|
2016-07-11 10:36:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Debugf(fdst, "Using server side directory move")
|
2017-02-05 21:20:56 +00:00
|
|
|
err := fdstDirMove(fsrc, "", "")
|
2016-07-04 12:12:33 +00:00
|
|
|
switch err {
|
2018-01-12 16:30:54 +00:00
|
|
|
case fs.ErrorCantDirMove, fs.ErrorDirExists:
|
|
|
|
fs.Infof(fdst, "Server side directory move failed - fallback to file moves: %v", err)
|
2016-07-04 12:12:33 +00:00
|
|
|
case nil:
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.Infof(fdst, "Server side directory move succeeded")
|
2016-07-04 12:12:33 +00:00
|
|
|
return nil
|
|
|
|
default:
|
2018-01-12 16:30:54 +00:00
|
|
|
fs.CountError(err)
|
|
|
|
fs.Errorf(fdst, "Server side directory move failed: %v", err)
|
2016-07-04 12:12:33 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-10-22 16:53:52 +00:00
|
|
|
// The two remotes mustn't overlap if we didn't do server side move
|
2018-01-12 16:30:54 +00:00
|
|
|
if operations.Overlapping(fdst, fsrc) {
|
|
|
|
err := fs.ErrorCantMoveOverlapping
|
|
|
|
fs.Errorf(fdst, "%v", err)
|
2016-10-22 16:53:52 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-07-11 10:36:46 +00:00
|
|
|
// Otherwise move the files one by one
|
2017-11-27 11:42:02 +00:00
|
|
|
return moveDir(fdst, fsrc, deleteEmptySrcDirs)
|
2016-07-04 12:12:33 +00:00
|
|
|
}
|