KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

Make --delete-after the default and refactor --delete-{before,during,after} parsing

Browse source
This commit is contained in:
Nick Craig-Wood 2017-01-25 19:35:14 +00:00
parent bcdd73369f
commit 7c6cd3a9e1
4 changed files with 90 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
docs/content/docs.md
View file

@ -507,6 +507,10 @@ to the console.
Note that `--track-renames` is incompatible with `--no-traverse` and
that it uses extra memory to keep track of all the rename candidates.
Note also that `--track-renames` is incompatible with
`--delete-before` and will select `--delete-after` instead of
`--delete-during`.
### --delete-(before,during,after) ###
This option allows you to specify when files on your destination are
@ -514,16 +518,21 @@ deleted when you sync folders.
Specifying the value `--delete-before` will delete all files present
on the destination, but not on the source *before* starting the
transfer of any new or updated files. This uses extra memory as it
has to store the source listing before proceeding.
transfer of any new or updated files. This uses two passes through the
file systems, one for the deletions and one for the copies.
Specifying `--delete-during` (default value) will delete files while
checking and uploading files. This is usually the fastest option.
Currently this works the same as `--delete-after` but it may change in
the future.
Specifying `--delete-during` will delete files while checking and
uploading files. This is the fastest option and uses the least memory.
Specifying `--delete-after` will delay deletion of files until all new/updated
files have been successfully transfered.
Specifying `--delete-after` (the default value) will delay deletion of
files until all new/updated files have been successfully transfered.
The files to be deleted are collected in the copy pass then deleted
after the copy pass has completed sucessfully. The files to be
deleted are held in memory so this mode may use more memory. This is
the safest mode as it will only delete files if there have been no
errors subsequent to that. If there have been errors before the
deletions start then you will get the message `not deleting files as
there were IO errors`.
### --timeout=TIME ###

33
fs/config.go
View file

@ -199,9 +199,7 @@ type ConfigInfo struct {
DumpAuth bool
Filter *Filter
InsecureSkipVerify bool // Skip server certificate verification
DeleteBefore bool // Delete before checking
DeleteDuring bool // Delete during checking/transfer
DeleteAfter bool // Delete after successful transfer.
DeleteMode DeleteMode
TrackRenames bool // Track file renames.
LowLevelRetries int
UpdateOlder bool // Skip files that are newer on the destination
@ -284,6 +282,19 @@ func makeConfigPath() string {
return hiddenConfigFileName
}
// DeleteMode describes the possible delete modes in the config
type DeleteMode byte
// DeleteMode constants
const (
DeleteModeOff DeleteMode = iota
DeleteModeBefore
DeleteModeDuring
DeleteModeAfter
DeleteModeOnly
DeleteModeDefault = DeleteModeAfter
)
// LoadConfig loads the config file
func LoadConfig() {
// Read some flags if set
@ -318,20 +329,20 @@ func LoadConfig() {
ConfigPath = *configFile
Config.DeleteBefore = *deleteBefore
Config.DeleteDuring = *deleteDuring
Config.DeleteAfter = *deleteAfter
Config.TrackRenames = *trackRenames
switch {
case *deleteBefore && (*deleteDuring || *deleteAfter),
*deleteDuring && *deleteAfter:
log.Fatalf(`Only one of --delete-before, --delete-during or --delete-after can be used.`)
// If none are specified, use "during".
case !*deleteBefore && !*deleteDuring && !*deleteAfter:
Config.DeleteDuring = true
case *deleteBefore:
Config.DeleteMode = DeleteModeBefore
case *deleteDuring:
Config.DeleteMode = DeleteModeDuring
case *deleteAfter:
Config.DeleteMode = DeleteModeAfter
default:
Config.DeleteMode = DeleteModeDefault
}
if Config.IgnoreSize && Config.SizeOnly {

60
fs/sync.go
View file

@ -14,12 +14,11 @@ type syncCopyMove struct {
// parameters
fdst Fs
fsrc Fs
Delete bool
deleteMode DeleteMode // how we are doing deletions
DoMove bool
dir string
// internal state
noTraverse bool // if set don't trafevers the dst
deleteBefore bool // set if we must delete objects before copying
trackRenames bool // set if we should do server side renames
dstFilesMu sync.Mutex // protect dstFiles
dstFiles map[string]Object // dst files, always filled
@ -48,11 +47,11 @@ type syncCopyMove struct {
suffix string // suffix to add to files placed in backupDir
}
func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) (*syncCopyMove, error) {
func newSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) (*syncCopyMove, error) {
s := &syncCopyMove{
fdst: fdst,
fsrc: fsrc,
Delete: Delete,
deleteMode: deleteMode,
DoMove: DoMove,
dir: "",
srcFilesChan: make(chan Object, Config.Checkers+Config.Transfers),
@ -62,13 +61,12 @@ func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) (*syncCopyMove, er
abort: make(chan struct{}),
toBeChecked: make(ObjectPairChan, Config.Transfers),
toBeUploaded: make(ObjectPairChan, Config.Transfers),
deleteBefore: Delete && Config.DeleteBefore,
trackRenames: Config.TrackRenames,
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
toBeRenamed: make(ObjectPairChan, Config.Transfers),
trackRenamesCh: make(chan Object, Config.Checkers),
}
if s.noTraverse && s.Delete {
if s.noTraverse && s.deleteMode != DeleteModeOff {
Debug(s.fdst, "Ignoring --no-traverse with sync")
s.noTraverse = false
}
@ -83,14 +81,16 @@ func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) (*syncCopyMove, er
s.trackRenames = false
}
}
if s.deleteBefore && s.trackRenames {
ErrorLog(fdst, "Ignoring --delete-before with --track-renames - using --delete-after")
s.deleteBefore = false
if s.trackRenames {
// track renames needs delete after
if s.deleteMode != DeleteModeOff {
s.deleteMode = DeleteModeAfter
}
if s.noTraverse && s.trackRenames {
if s.noTraverse {
Debug(s.fdst, "Ignoring --no-traverse with --track-renames")
s.noTraverse = false
}
}
// Make Fs for --backup-dir if required
if Config.BackupDir != "" {
var err error
@ -614,8 +614,8 @@ func (s *syncCopyMove) run() error {
go s.readDstFiles()
}
// If s.deleteBefore then we need to read the whole source map first
readSourceMap := s.deleteBefore
// If --delete-before then we need to read the whole source map first
readSourceMap := s.deleteMode == DeleteModeBefore
if readSourceMap {
// Read source files into the map
@ -636,7 +636,7 @@ func (s *syncCopyMove) run() error {
}
// Delete files first if required
if s.deleteBefore {
if s.deleteMode == DeleteModeBefore {
err = s.deleteFiles(true)
if err != nil {
return err
@ -710,7 +710,7 @@ func (s *syncCopyMove) run() error {
err = <-s.srcFilesResult
// Delete files during or after
if s.Delete && (Config.DeleteDuring || Config.DeleteAfter) {
if s.deleteMode == DeleteModeDuring || s.deleteMode == DeleteModeAfter {
if err != nil {
ErrorLog(s.fdst, "%v", ErrorNotDeleting)
} else {
@ -721,31 +721,37 @@ func (s *syncCopyMove) run() error {
return s.currentError()
}
// Sync fsrc into fdst
func Sync(fdst, fsrc Fs) error {
do, err := newSyncCopyMove(fdst, fsrc, true, false)
// Syncs fsrc into fdst
//
// If Delete is true then it deletes any files in fdst that aren't in fsrc
//
// If DoMove is true then files will be moved instead of copied
//
// dir is the start directory, "" for root
func runSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) error {
if deleteMode != DeleteModeOff && DoMove {
return errors.New("can't delete and move at the same time")
}
do, err := newSyncCopyMove(fdst, fsrc, deleteMode, DoMove)
if err != nil {
return err
}
return do.run()
}
// Sync fsrc into fdst
func Sync(fdst, fsrc Fs) error {
return runSyncCopyMove(fdst, fsrc, Config.DeleteMode, false)
}
// CopyDir copies fsrc into fdst
func CopyDir(fdst, fsrc Fs) error {
do, err := newSyncCopyMove(fdst, fsrc, false, false)
if err != nil {
return err
}
return do.run()
return runSyncCopyMove(fdst, fsrc, DeleteModeOff, false)
}
// moveDir moves fsrc into fdst
func moveDir(fdst, fsrc Fs) error {
do, err := newSyncCopyMove(fdst, fsrc, false, true)
if err != nil {
return err
}
return do.run()
return runSyncCopyMove(fdst, fsrc, DeleteModeOff, true)
}
// MoveDir moves fsrc into fdst

32
fs/sync_test.go
View file

@ -475,38 +475,28 @@ func TestSyncAfterRemovingAFileAndAddingAFileWithErrors(t *testing.T) {
fstest.CheckItems(t, r.fremote, file1, file2, file3)
}
// Sync test delete during
func TestSyncDeleteDuring(t *testing.T) {
// Sync test delete after
func TestSyncDeleteAfter(t *testing.T) {
// This is the default so we've checked this already
// check it is the default
if !(!fs.Config.DeleteBefore && fs.Config.DeleteDuring && !fs.Config.DeleteAfter) {
t.Fatalf("Didn't default to --delete-during")
}
require.Equal(t, fs.Config.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after")
}
// Sync test delete before
func TestSyncDeleteBefore(t *testing.T) {
fs.Config.DeleteBefore = true
fs.Config.DeleteDuring = false
fs.Config.DeleteAfter = false
// Sync test delete during
func TestSyncDeleteDuring(t *testing.T) {
fs.Config.DeleteMode = fs.DeleteModeDuring
defer func() {
fs.Config.DeleteBefore = false
fs.Config.DeleteDuring = true
fs.Config.DeleteAfter = false
fs.Config.DeleteMode = fs.DeleteModeDefault
}()
TestSyncAfterRemovingAFileAndAddingAFile(t)
}
// Sync test delete after
func TestSyncDeleteAfter(t *testing.T) {
fs.Config.DeleteBefore = false
fs.Config.DeleteDuring = false
fs.Config.DeleteAfter = true
// Sync test delete before
func TestSyncDeleteBefore(t *testing.T) {
fs.Config.DeleteMode = fs.DeleteModeBefore
defer func() {
fs.Config.DeleteBefore = false
fs.Config.DeleteDuring = true
fs.Config.DeleteAfter = false
fs.Config.DeleteMode = fs.DeleteModeDefault
}()
TestSyncAfterRemovingAFileAndAddingAFile(t)