Implement --backup-dir - fixes #98

The parameter of backup-dir specifies a remote that all deleted or
overwritten files will be copied to.
This commit is contained in:
ncw 2017-01-10 21:47:03 +00:00 committed by Nick Craig-Wood
parent c123c702ab
commit 3745c526f1
6 changed files with 165 additions and 27 deletions

View file

@ -189,6 +189,24 @@ for bytes, `k` for kBytes, `M` for MBytes and `G` for GBytes may be
used. These are the binary units, eg 1, 2\*\*10, 2\*\*20, 2\*\*30
respectively.
### --backup-dir=DIR ###
When using `sync`, `copy` or `move` any files which would have been
overwritten or deleted are moved in their original hierarchy into this
directory.
The remote in use must support server side move or copy and you must
use the same remote as the destination of the sync. The backup
directory must not overlap the destination directory.
For example
rclone sync /path/to/local remote:current --backup-dir remote:old
will sync `/path/to/local` to `remote:current`, but for any files
which would have been updated or deleted will be stored in
`remote:old`.
### --bwlimit=BANDWIDTH_SPEC ###
This option controls the bandwidth limit. Limits can be specified

View file

@ -86,6 +86,7 @@ var (
ignoreSize = BoolP("ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
noTraverse = BoolP("no-traverse", "", false, "Don't traverse destination file system on copy.")
noUpdateModTime = BoolP("no-update-modtime", "", false, "Don't update destination mod-time if files identical.")
backupDir = StringP("backup-dir", "", "", "Make backups into hierarchy based in DIR.")
bwLimit BwTimetable
// Key to use for password en/decryption.
@ -209,6 +210,7 @@ type ConfigInfo struct {
NoTraverse bool
NoUpdateModTime bool
DataRateUnit string
BackupDir string
}
// Find the config directory
@ -260,6 +262,7 @@ func LoadConfig() {
Config.IgnoreSize = *ignoreSize
Config.NoTraverse = *noTraverse
Config.NoUpdateModTime = *noUpdateModTime
Config.BackupDir = *backupDir
ConfigPath = *configFile

View file

@ -396,26 +396,52 @@ func CanServerSideMove(fdst Fs) bool {
return canMove || canCopy
}
// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
func DeleteFile(dst Object) (err error) {
if Config.DryRun {
Log(dst, "Not deleting as --dry-run")
} else {
// deleteFileWithBackupDir deletes a single file respecting --dry-run
// and accumulating stats and errors.
//
// If backupDir is set then it moves the file to there instead of
// deleting
func deleteFileWithBackupDir(dst Object, backupDir Fs) (err error) {
Stats.Checking(dst.Remote())
action, actioned, actioning := "delete", "Deleted", "deleting"
if backupDir != nil {
action, actioned, actioning = "move into backup dir", "Moved into backup dir", "moving into backup dir"
}
if Config.DryRun {
Log(dst, "Not %s as --dry-run", actioning)
} else if backupDir != nil {
if !SameConfig(dst.Fs(), backupDir) {
err = errors.New("parameter to --backup-dir has to be on the same remote as destination")
} else {
err = Move(backupDir, nil, dst.Remote(), dst)
}
} else {
err = dst.Remove()
Stats.DoneChecking(dst.Remote())
}
if err != nil {
Stats.Error()
ErrorLog(dst, "Couldn't delete: %v", err)
ErrorLog(dst, "Couldn't %s: %v", action, err)
} else {
Debug(dst, "Deleted")
}
Debug(dst, actioned)
}
Stats.DoneChecking(dst.Remote())
return err
}
// DeleteFiles removes all the files passed in the channel
func DeleteFiles(toBeDeleted ObjectsChan) error {
// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
//
// If useBackupDir is set and --backup-dir is in effect then it moves
// the file to there instead of deleting
func DeleteFile(dst Object) (err error) {
return deleteFileWithBackupDir(dst, nil)
}
// deleteFilesWithBackupDir removes all the files passed in the
// channel
//
// If backupDir is set the files will be placed into that directory
// instead of being deleted.
func deleteFilesWithBackupDir(toBeDeleted ObjectsChan, backupDir Fs) error {
var wg sync.WaitGroup
wg.Add(Config.Transfers)
var errorCount int32
@ -423,7 +449,7 @@ func DeleteFiles(toBeDeleted ObjectsChan) error {
go func() {
defer wg.Done()
for dst := range toBeDeleted {
err := DeleteFile(dst)
err := deleteFileWithBackupDir(dst, backupDir)
if err != nil {
atomic.AddInt32(&errorCount, 1)
}
@ -438,6 +464,11 @@ func DeleteFiles(toBeDeleted ObjectsChan) error {
return nil
}
// DeleteFiles removes all the files passed in the channel
func DeleteFiles(toBeDeleted ObjectsChan) error {
return deleteFilesWithBackupDir(toBeDeleted, nil)
}
// Read a Objects into add() for the given Fs.
// dir is the start directory, "" for root
// If includeAll is specified all files will be added,

View file

@ -41,10 +41,10 @@ type syncCopyMove struct {
renameMap map[string][]Object // dst files by hash - only used by trackRenames
renamerWg sync.WaitGroup // wait for renamers
toBeRenamed ObjectPairChan // renamers channel
backupDir Fs // place to store overwrites/deletes
}
func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) *syncCopyMove {
canServerSideMove := CanServerSideMove(fdst)
func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) (*syncCopyMove, error) {
s := &syncCopyMove{
fdst: fdst,
fsrc: fsrc,
@ -69,7 +69,7 @@ func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) *syncCopyMove {
}
if s.trackRenames {
// Don't track renames for remotes without server-side move support.
if !canServerSideMove {
if !CanServerSideMove(fdst) {
ErrorLog(fdst, "Ignoring --track-renames as the destination does not support server-side move or copy")
s.trackRenames = false
}
@ -82,7 +82,27 @@ func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) *syncCopyMove {
Debug(s.fdst, "Ignoring --no-traverse with --track-renames")
s.noTraverse = false
}
return s
// Make Fs for --backup-dir if required
if Config.BackupDir != "" {
var err error
s.backupDir, err = NewFs(Config.BackupDir)
if err != nil {
return nil, FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", Config.BackupDir, err))
}
if !CanServerSideMove(s.backupDir) {
return nil, FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server side move or copy"))
}
if !SameConfig(fdst, s.backupDir) {
return nil, FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination"))
}
if Overlapping(fdst, s.backupDir) {
return nil, FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap"))
}
if Overlapping(fsrc, s.backupDir) {
return nil, FatalError(errors.New("source and parameter to --backup-dir mustn't overlap"))
}
}
return s, nil
}
// Check to see if have set the abort flag
@ -264,7 +284,19 @@ func (s *syncCopyMove) pairChecker(in ObjectPairChan, out ObjectPairChan, wg *sy
// Check to see if can store this
if src.Storable() {
if NeedTransfer(pair.dst, pair.src) {
// If destination already exists, then we must move it into --backup-dir if required
if pair.dst != nil && s.backupDir != nil {
err := Move(s.backupDir, nil, pair.dst.Remote(), pair.dst)
if err != nil {
s.processError(err)
} else {
// If successful zero out the dst as it is no longer there and copy the file
pair.dst = nil
out <- pair
}
} else {
out <- pair
}
} else {
// If moving need to delete the files we don't need to copy
if s.DoMove {
@ -411,7 +443,7 @@ func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
}
close(toDelete)
}()
return DeleteFiles(toDelete)
return deleteFilesWithBackupDir(toDelete, s.backupDir)
}
// renameHash makes a string with the size and the hash for rename detection
@ -655,17 +687,29 @@ func (s *syncCopyMove) run() error {
// Sync fsrc into fdst
func Sync(fdst, fsrc Fs) error {
return newSyncCopyMove(fdst, fsrc, true, false).run()
do, err := newSyncCopyMove(fdst, fsrc, true, false)
if err != nil {
return err
}
return do.run()
}
// CopyDir copies fsrc into fdst
func CopyDir(fdst, fsrc Fs) error {
return newSyncCopyMove(fdst, fsrc, false, false).run()
do, err := newSyncCopyMove(fdst, fsrc, false, false)
if err != nil {
return err
}
return do.run()
}
// moveDir moves fsrc into fdst
func moveDir(fdst, fsrc Fs) error {
return newSyncCopyMove(fdst, fsrc, false, true).run()
do, err := newSyncCopyMove(fdst, fsrc, false, true)
if err != nil {
return err
}
return do.run()
}
// MoveDir moves fsrc into fdst

View file

@ -742,3 +742,45 @@ func TestServerSideMoveOverlap(t *testing.T) {
err = fs.MoveDir(fremoteMove, r.fremote)
assert.EqualError(t, err, fs.ErrorCantMoveOverlapping.Error())
}
// Test with BackupDir set
func TestSyncBackupDir(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
if !fs.CanServerSideMove(r.fremote) {
t.Skip("Skipping test as remote does not support server side move")
}
r.Mkdir(r.fremote)
fs.Config.BackupDir = r.fremoteName + "/backup"
defer func() {
fs.Config.BackupDir = ""
}()
file1 := r.WriteObject("dst/one", "one", t1)
file2 := r.WriteObject("dst/two", "two", t2)
file3 := r.WriteObject("dst/three", "three", t3)
file2a := r.WriteFile("two", "two", t2)
file1a := r.WriteFile("one", "oneone", t2)
fstest.CheckItems(t, r.fremote, file1, file2, file3)
fstest.CheckItems(t, r.flocal, file1a, file2a)
fdst, err := fs.NewFs(r.fremoteName + "/dst")
require.NoError(t, err)
fs.Stats.ResetCounters()
err = fs.Sync(fdst, r.flocal)
require.NoError(t, err)
// file1 is overwritten and the old version moved to backup-dir
file1.Path = "backup/one"
file1a.Path = "dst/one"
// file 2 is unchanged
// file 3 is deleted (moved to backup dir)
file3.Path = "backup/three"
fstest.CheckItems(t, r.fremote, file1, file2, file3, file1a)
}

View file

@ -93,7 +93,7 @@ func (i *Item) CheckHashes(t *testing.T, obj fs.Object) {
// Check checks all the attributes of the object are correct
func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
i.CheckHashes(t, obj)
assert.Equal(t, i.Size, obj.Size())
assert.Equal(t, i.Size, obj.Size(), fmt.Sprintf("%s: size incorrect", i.Path))
i.CheckModTime(t, obj, obj.ModTime(), precision)
}