fs: Use --cutoff-mode hard,soft,catious instead of 3 --max-transfer-mode flags

Fixes #2672

docs/content/docs.md

@@ -736,18 +736,18 @@ When the limit is reached all transfers will stop immediately.
 
 Rclone will exit with exit code 8 if the transfer limit is reached.
 
-### --max-transfer-(hard,soft,cautious) ###
+### --cutoff-mode=hard|soft|cautious ###
 
 This modifies the behavior of `--max-transfer`
-Defaults to `--max-transfer-hard`.
+Defaults to `--cutoff-mode=hard`.
 
-Specifiying `--max-transfer-hard` will stop transferring immediately
+Specifiying `--cutoff-mode=hard` will stop transferring immediately
 when Rclone reaches the limit.
 
-Specifiying `--max-transfer-soft` will stop starting new transfers
+Specifiying `--cutoff-mode=soft` will stop starting new transfers
 when Rclone reaches the limit.
 
-Specifiying `--max-transfer-cautious` will try to prevent Rclone
+Specifiying `--cutoff-mode=cautious` will try to prevent Rclone
 from reaching the limit.
 
 ### --modify-window=TIME ###

fs/accounting/accounting.go

@@ -63,7 +63,7 @@ func newAccountSizeName(stats *StatsInfo, in io.ReadCloser, size int64, name str
 		lpTime: time.Now(),
 		max:    -1,
 	}
-	if fs.Config.MaxTransferMode != fs.MaxTransferModeSoft {
+	if fs.Config.CutoffMode == fs.CutoffModeHard {
 		acc.max = int64((fs.Config.MaxTransfer))
 	}
 	go acc.averageLoop()

fs/accounting/accounting_test.go

@@ -197,12 +197,12 @@ func TestAccountAccounter(t *testing.T) {
 
 func TestAccountMaxTransfer(t *testing.T) {
 	old := fs.Config.MaxTransfer
-	oldMode := fs.Config.MaxTransferMode
+	oldMode := fs.Config.CutoffMode
 
 	fs.Config.MaxTransfer = 15
 	defer func() {
 		fs.Config.MaxTransfer = old
-		fs.Config.MaxTransferMode = oldMode
+		fs.Config.CutoffMode = oldMode
 	}()
 
 	in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
@@ -222,7 +222,7 @@ func TestAccountMaxTransfer(t *testing.T) {
 	assert.Equal(t, ErrorMaxTransferLimitReached, err)
 	assert.True(t, fserrors.IsFatalError(err))
 
-	fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
+	fs.Config.CutoffMode = fs.CutoffModeSoft
 	stats = NewStats()
 	acc = newAccountSizeName(stats, in, 1, "test")
 

fs/config.go

@@ -93,7 +93,7 @@ type ConfigInfo struct {
 	UseServerModTime       bool
 	MaxTransfer            SizeSuffix
 	MaxDuration            time.Duration
-	MaxTransferMode        MaxTransferMode
+	CutoffMode             CutoffMode
 	MaxBacklog             int
 	MaxStatsGroups         int
 	StatsOneLine           bool

fs/config/configflags/configflags.go

@@ -29,9 +29,6 @@ var (
 	deleteAfter     bool
 	bindAddr        string
 	disableFeatures string
-	maxTransferHard     bool
-	maxTransferSoft     bool
-	maxTransferCautious bool
 )
 
 // AddFlags adds the non filing system specific flags to the command
@@ -97,9 +94,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
 	flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
 	flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
-	flags.BoolVarP(flagSet, &maxTransferHard, "max-transfer-hard", "", false, "When transferring, stop immediately when --max-transfer is reached")
-	flags.BoolVarP(flagSet, &maxTransferSoft, "max-transfer-soft", "", false, "When transferring, stop starting new transfers when --max-transfer is reached")
-	flags.BoolVarP(flagSet, &maxTransferCautious, "max-transfer-cautious", "", false, "When transferring, try to avoid reaching --max-transfer")
+	flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
 	flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
 	flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
@@ -184,20 +179,6 @@ func SetFlags() {
 		fs.Config.DeleteMode = fs.DeleteModeDefault
 	}
 
-	switch {
-	case maxTransferHard && (maxTransferSoft || maxTransferCautious),
-		maxTransferSoft && maxTransferCautious:
-		log.Fatalf(`Only one of --max-trans fer-hard, --max-transfer-soft or --max-transfer-cautious can be used.`)
-	case maxTransferHard:
-		fs.Config.MaxTransferMode = fs.MaxTransferModeHard
-	case maxTransferSoft:
-		fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
-	case maxTransferCautious:
-		fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
-	default:
-		fs.Config.MaxTransferMode = fs.MaxTransferModeDefault
-	}
-
 	if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
 		log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
 	}

fs/cutoffmode.go

@@ -0,0 +1,49 @@
+package fs
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// CutoffMode describes the possible delete modes in the config
+type CutoffMode byte
+
+// MaxTransferMode constants
+const (
+	CutoffModeHard CutoffMode = iota
+	CutoffModeSoft
+	CutoffModeCautious
+	CutoffModeDefault = CutoffModeHard
+)
+
+var cutoffModeToString = []string{
+	CutoffModeHard:     "HARD",
+	CutoffModeSoft:     "SOFT",
+	CutoffModeCautious: "CAUTIOUS",
+}
+
+// String turns a LogLevel into a string
+func (m CutoffMode) String() string {
+	if m >= CutoffMode(len(cutoffModeToString)) {
+		return fmt.Sprintf("CutoffMode(%d)", m)
+	}
+	return cutoffModeToString[m]
+}
+
+// Set a LogLevel
+func (m *CutoffMode) Set(s string) error {
+	for n, name := range cutoffModeToString {
+		if s != "" && name == strings.ToUpper(s) {
+			*m = CutoffMode(n)
+			return nil
+		}
+	}
+	return errors.Errorf("Unknown cutoff mode %q", s)
+}
+
+// Type of the value
+func (m *CutoffMode) Type() string {
+	return "string"
+}

fs/cutoffmode_test.go

@@ -0,0 +1,6 @@
+package fs
+
+import "github.com/spf13/pflag"
+
+// Check it satisfies the interface
+var _ pflag.Value = (*CutoffMode)(nil)

fs/maxtransfermode.go

@@ -1,12 +0,0 @@
-package fs
-
-// MaxTransferMode describes the possible delete modes in the config
-type MaxTransferMode byte
-
-// MaxTransferMode constants
-const (
-	MaxTransferModeHard MaxTransferMode = iota
-	MaxTransferModeSoft
-	MaxTransferModeCautious
-	MaxTransferModeDefault = MaxTransferModeHard
-)

fs/operations/operations.go

@@ -362,12 +362,11 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		// Try server side copy first - if has optional interface and
 		// is same underlying remote
 		actionTaken = "Copied (server side copy)"
-		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
-			// Check transfer limit for server side copies
 		if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
-				(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
+			(fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
 			return nil, accounting.ErrorMaxTransferLimitReached
 		}
+		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
 			in := tr.Account(nil) // account the transfer
 			in.ServerSideCopyStart()
 			newDst, err = doCopy(ctx, src, remote)
@@ -386,10 +385,6 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		}
 		// If can't server side copy, do it manually
 		if err == fs.ErrorCantCopy {
-			if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
-				(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
-				return nil, accounting.ErrorMaxTransferLimitReached
-			}
 			if doMultiThreadCopy(f, src) {
 				// Number of streams proportional to size
 				streams := src.Size() / int64(fs.Config.MultiThreadCutoff)

fs/operations/operations_test.go

@@ -1533,11 +1533,11 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 	old := fs.Config.MaxTransfer
-	oldMode := fs.Config.MaxTransferMode
+	oldMode := fs.Config.CutoffMode
 
 	defer func() {
 		fs.Config.MaxTransfer = old
-		fs.Config.MaxTransferMode = oldMode
+		fs.Config.CutoffMode = oldMode
 		accounting.Stats(context.Background()).ResetCounters()
 	}()
 
@@ -1550,7 +1550,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 	rfile2.Path = "sub/file2"
 
 	fs.Config.MaxTransfer = 15
-	fs.Config.MaxTransferMode = fs.MaxTransferModeHard
+	fs.Config.CutoffMode = fs.CutoffModeHard
 	accounting.Stats(context.Background()).ResetCounters()
 
 	err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile1.Path, file1.Path)
@@ -1563,19 +1563,19 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
 	fstest.CheckItems(t, r.Flocal, file1, file2)
 	fstest.CheckItems(t, r.Fremote, rfile1)
-	assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
+	assert.Contains(t, err.Error(), "Max transfer limit reached")
 	assert.True(t, fserrors.IsFatalError(err))
 
-	fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
+	fs.Config.CutoffMode = fs.CutoffModeCautious
 	accounting.Stats(context.Background()).ResetCounters()
 
 	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
 	fstest.CheckItems(t, r.Flocal, file1, file2)
 	fstest.CheckItems(t, r.Fremote, rfile1)
-	assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
+	assert.Contains(t, err.Error(), "Max transfer limit reached")
 	assert.True(t, fserrors.IsFatalError(err))
 
-	fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
+	fs.Config.CutoffMode = fs.CutoffModeSoft
 	accounting.Stats(context.Background()).ResetCounters()
 
 	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)