accounting: show server side stats in own lines and not as bytes transferred

Before this change we showed both server side moves and server side
copies as bytes transferred.

This made a nice easy to use stats display, but also caused confusion
for users who saw unrealistic transfer times. It also caused a problem
with --max-transfer and chunker which renames each chunk after
uploading which was counted as a transfer byte.

This patch instead accounts the server side move and copy statistics
as a seperate lines in the stats display which will only appear if
there are any server side moves / copies. This is also output in the
rc.

This gives users something to look at when transfers are running which
was the point of the original change but it now means that transfer
bytes represents data transfers through this rclone instance only.

Fixes #7183

fs/accounting/accounting.go

@@ -272,10 +272,10 @@ func (acc *Account) checkReadAfter(bytesUntilLimit int64, n int, err error) (out
 	return n, err
 }
 
-// ServerSideCopyStart should be called at the start of a server-side copy
+// ServerSideTransferStart should be called at the start of a server-side transfer
 //
 // This pretends a transfer has started
-func (acc *Account) ServerSideCopyStart() {
+func (acc *Account) ServerSideTransferStart() {
 	acc.values.mu.Lock()
 	// Set start time.
 	if acc.values.start.IsZero() {
@@ -284,8 +284,9 @@ func (acc *Account) ServerSideCopyStart() {
 	acc.values.mu.Unlock()
 }
 
-// ServerSideCopyEnd accounts for a read of n bytes in a sever side copy
+// ServerSideTransferEnd accounts for a read of n bytes in a sever
-func (acc *Account) ServerSideCopyEnd(n int64) {
+// side transfer to be treated as a normal transfer.
+func (acc *Account) ServerSideTransferEnd(n int64) {
 	// Update Stats
 	acc.values.mu.Lock()
 	acc.values.bytes += n
@@ -294,10 +295,20 @@ func (acc *Account) ServerSideCopyEnd(n int64) {
 	acc.stats.Bytes(n)
 }
 
+// ServerSideCopyEnd accounts for a read of n bytes in a sever side copy
+func (acc *Account) ServerSideCopyEnd(n int64) {
+	acc.stats.AddServerSideCopy(n)
+}
+
+// ServerSideMoveEnd accounts for a read of n bytes in a sever side move
+func (acc *Account) ServerSideMoveEnd(n int64) {
+	acc.stats.AddServerSideMove(n)
+}
+
 // DryRun accounts for statistics without running the operation
 func (acc *Account) DryRun(n int64) {
-	acc.ServerSideCopyStart()
+	acc.ServerSideTransferStart()
-	acc.ServerSideCopyEnd(n)
+	acc.ServerSideTransferEnd(n)
 }
 
 // Account for n bytes from the current file bandwidth limit (if any)

fs/accounting/stats.go

@@ -28,36 +28,40 @@ var MaxCompletedTransfers = 100
 // N.B.: if this struct is modified, please remember to also update sum() function in stats_groups
 // to correctly count the updated fields
 type StatsInfo struct {
-	mu                sync.RWMutex
+	mu                  sync.RWMutex
-	ctx               context.Context
+	ctx                 context.Context
-	ci                *fs.ConfigInfo
+	ci                  *fs.ConfigInfo
-	bytes             int64
+	bytes               int64
-	errors            int64
+	errors              int64
-	lastError         error
+	lastError           error
-	fatalError        bool
+	fatalError          bool
-	retryError        bool
+	retryError          bool
-	retryAfter        time.Time
+	retryAfter          time.Time
-	checks            int64
+	checks              int64
-	checking          *transferMap
+	checking            *transferMap
-	checkQueue        int
+	checkQueue          int
-	checkQueueSize    int64
+	checkQueueSize      int64
-	transfers         int64
+	transfers           int64
-	transferring      *transferMap
+	transferring        *transferMap
-	transferQueue     int
+	transferQueue       int
-	transferQueueSize int64
+	transferQueueSize   int64
-	renames           int64
+	renames             int64
-	renameQueue       int
+	renameQueue         int
-	renameQueueSize   int64
+	renameQueueSize     int64
-	deletes           int64
+	deletes             int64
-	deletesSize       int64
+	deletesSize         int64
-	deletedDirs       int64
+	deletedDirs         int64
-	inProgress        *inProgress
+	inProgress          *inProgress
-	startedTransfers  []*Transfer   // currently active transfers
+	startedTransfers    []*Transfer   // currently active transfers
-	oldTimeRanges     timeRanges    // a merged list of time ranges for the transfers
+	oldTimeRanges       timeRanges    // a merged list of time ranges for the transfers
-	oldDuration       time.Duration // duration of transfers we have culled
+	oldDuration         time.Duration // duration of transfers we have culled
-	group             string
+	group               string
-	startTime         time.Time // the moment these stats were initialized or reset
+	startTime           time.Time // the moment these stats were initialized or reset
-	average           averageValues
+	average             averageValues
+	serverSideCopies    int64
+	serverSideCopyBytes int64
+	serverSideMoves     int64
+	serverSideMoveBytes int64
 }
 
 type averageValues struct {
@@ -110,6 +114,10 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 	out["deletedDirs"] = s.deletedDirs
 	out["renames"] = s.renames
 	out["elapsedTime"] = time.Since(s.startTime).Seconds()
+	out["serverSideCopies"] = s.serverSideCopies
+	out["serverSideCopyBytes"] = s.serverSideCopyBytes
+	out["serverSideMoves"] = s.serverSideMoves
+	out["serverSideMoveBytes"] = s.serverSideMoveBytes
 	eta, etaOK := eta(s.bytes, ts.totalBytes, ts.speed)
 	if etaOK {
 		out["eta"] = eta.Seconds()
@@ -449,6 +457,16 @@ func (s *StatsInfo) String() string {
 			_, _ = fmt.Fprintf(buf, "Transferred:   %10d / %d, %s\n",
 				s.transfers, ts.totalTransfers, percent(s.transfers, ts.totalTransfers))
 		}
+		if s.serverSideCopies != 0 || s.serverSideCopyBytes != 0 {
+			_, _ = fmt.Fprintf(buf, "Server Side Copies:%6d @ %s\n",
+				s.serverSideCopies, fs.SizeSuffix(s.serverSideCopyBytes).ByteUnit(),
+			)
+		}
+		if s.serverSideMoves != 0 || s.serverSideMoveBytes != 0 {
+			_, _ = fmt.Fprintf(buf, "Server Side Moves:%7d @ %s\n",
+				s.serverSideMoves, fs.SizeSuffix(s.serverSideMoveBytes).ByteUnit(),
+			)
+		}
 		_, _ = fmt.Fprintf(buf, "Elapsed time:  %10ss\n", strings.TrimRight(fs.Duration(elapsedTime.Truncate(time.Minute)).ReadableString(), "0s")+fmt.Sprintf("%.1f", elapsedTimeSecondsOnly.Seconds()))
 	}
 
@@ -856,3 +874,19 @@ func (s *StatsInfo) PruneTransfers() {
 	}
 	s.mu.Unlock()
 }
+
+// AddServerSideMove counts a server side move
+func (s *StatsInfo) AddServerSideMove(n int64) {
+	s.mu.Lock()
+	s.serverSideMoves += 1
+	s.serverSideMoveBytes += n
+	s.mu.Unlock()
+}
+
+// AddServerSideCopy counts a server side copy
+func (s *StatsInfo) AddServerSideCopy(n int64) {
+	s.mu.Lock()
+	s.serverSideCopies += 1
+	s.serverSideCopyBytes += n
+	s.mu.Unlock()
+}

fs/accounting/stats_groups.go

@@ -97,6 +97,10 @@ Returns the following values:
 	"lastError": last error string,
 	"renames" : number of files renamed,
 	"retryError": boolean showing whether there has been at least one non-NoRetryError,
+        "serverSideCopies": number of server side copies done,
+        "serverSideCopyBytes": number bytes server side copied,
+        "serverSideMoves": number of server side moves done,
+        "serverSideMoveBytes": number bytes server side moved,
 	"speed": average speed in bytes per second since start of the group,
 	"totalBytes": total number of bytes in the group,
 	"totalChecks": total number of checks in the group,

fs/operations/operations.go

@@ -392,7 +392,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		}
 		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && (f.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
 			in := tr.Account(ctx, nil) // account the transfer
-			in.ServerSideCopyStart()
+			in.ServerSideTransferStart()
 			newDst, err = doCopy(ctx, src, remote)
 			if err == nil {
 				dst = newDst
@@ -639,7 +639,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 		}
 		// Move dst <- src
 		in := tr.Account(ctx, nil) // account the transfer
-		in.ServerSideCopyStart()
+		in.ServerSideTransferStart()
 		newDst, err = doMove(ctx, src, remote)
 		switch err {
 		case nil:
@@ -648,7 +648,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 			} else {
 				fs.Infof(src, "Moved (server-side)")
 			}
-			in.ServerSideCopyEnd(newDst.Size()) // account the bytes for the server-side transfer
+			in.ServerSideMoveEnd(newDst.Size()) // account the bytes for the server-side transfer
 			_ = in.Close()
 			return newDst, nil
 		case fs.ErrorCantMove: