fs/accounting: fix stats display which was missing transferring data

Before this change the total stats were ignoring the in flight
checking, transferring and bytes.

fs/accounting/stats.go

@@ -179,6 +179,11 @@ func percent(a int64, b int64) string {
 
 // String convert the StatsInfo to a string for printing
 func (s *StatsInfo) String() string {
+	// checking and transferring have their own locking so read
+	// here before lock to prevent deadlock on GetBytes
+	transferring, checking := s.transferring.count(), s.checking.count()
+	transferringBytesDone, transferringBytesTotal := s.transferring.progress()
+
 	s.mu.RLock()
 
 	dt := time.Now().Sub(s.start)
@@ -188,33 +193,25 @@ func (s *StatsInfo) String() string {
 		speed = float64(s.bytes) / dtSeconds
 	}
 	dtRounded := dt - (dt % (time.Second / 10))
-	buf := &bytes.Buffer{}
 
 	if fs.Config.DataRateUnit == "bits" {
 		speed = speed * 8
 	}
 
-	percent := func(a int64, b int64) int {
+	var (
-		if b <= 0 {
+		totalChecks   = int64(s.checkQueue) + s.checks + int64(checking)
-			return 0
+		totalTransfer = int64(s.transferQueue) + s.transfers + int64(transferring)
-		}
+		// note that s.bytes already includes transferringBytesDone so
-		return int(float64(a)*100/float64(b) + 0.5)
+		// we take it off here to avoid double counting
-	}
+		totalSize    = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
+		currentSize  = s.bytes
+		buf          = &bytes.Buffer{}
+		xfrchkString = ""
+	)
 
-	totalChecks, totalTransfer, totalSize := int64(s.checkQueue)+s.checks, int64(s.transferQueue)+s.transfers, s.transferQueueSize+s.bytes
-	eta := time.Duration(0)
-	if speed > 0 {
-		eta = time.Second * time.Duration(float64(s.transferQueueSize)/float64(speed)+0.5)
-	}
-	etaString := "-"
-	if eta > 0 {
-		etaString = eta.String()
-	}
 	if !fs.Config.StatsOneLine {
 		_, _ = fmt.Fprintf(buf, "\nTransferred:   	")
-	}
+	} else {
-	xfrchkString := ""
-	if fs.Config.StatsOneLine {
 		xfrchk := []string{}
 		if totalTransfer > 0 && s.transferQueue > 0 {
 			xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
@@ -226,13 +223,21 @@ func (s *StatsInfo) String() string {
 			xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
 		}
 	}
-	_, _ = fmt.Fprintf(buf, "%10s / %s, %d%%, %s, ETA %s%s",
+
-		fs.SizeSuffix(s.bytes), fs.SizeSuffix(totalSize).Unit("Bytes"), percent(s.bytes, totalSize), fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"), etaString, xfrchkString)
+	_, _ = fmt.Fprintf(buf, "%10s / %s, %s, %s, ETA %s%s",
+		fs.SizeSuffix(s.bytes),
+		fs.SizeSuffix(totalSize).Unit("Bytes"),
+		percent(s.bytes, totalSize),
+		fs.SizeSuffix(speed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"),
+		etaString(currentSize, totalSize, speed),
+		xfrchkString,
+	)
+
 	if !fs.Config.StatsOneLine {
 		_, _ = fmt.Fprintf(buf, `
 Errors:        %10d
-Checks:        %10d / %d, %d%%
+Checks:        %10d / %d, %s
-Transferred:   %10d / %d, %d%%
+Transferred:   %10d / %d, %s
 Elapsed time:  %10v
 `,
 			s.errors,
@@ -245,6 +250,7 @@ Elapsed time:  %10v
 	// here to prevent deadlock on GetBytes
 	s.mu.RUnlock()
 
+	// Add per transfer stats if required
 	if !fs.Config.StatsOneLine {
 		if !s.checking.empty() {
 			_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
@@ -253,6 +259,7 @@ Elapsed time:  %10v
 			_, _ = fmt.Fprintf(buf, "Transferring:\n%s\n", s.transferring)
 		}
 	}
+
 	return buf.String()
 }
 

fs/accounting/stringset.go

@@ -40,6 +40,13 @@ func (ss *stringSet) empty() bool {
 	return len(ss.items) == 0
 }
 
+// count returns the number of items in the set
+func (ss *stringSet) count() int {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	return len(ss.items)
+}
+
 // Strings returns all the strings in the stringSet
 func (ss *stringSet) Strings() []string {
 	ss.mu.RLock()
@@ -63,3 +70,19 @@ func (ss *stringSet) Strings() []string {
 func (ss *stringSet) String() string {
 	return strings.Join(ss.Strings(), "\n")
 }
+
+// progress returns total bytes read as well as the size.
+func (ss *stringSet) progress() (totalBytes, totalSize int64) {
+	ss.mu.RLock()
+	defer ss.mu.RUnlock()
+	for name := range ss.items {
+		if acc := Stats.inProgress.get(name); acc != nil {
+			bytes, size := acc.progress()
+			if size >= 0 && bytes >= 0 {
+				totalBytes += bytes
+				totalSize += size
+			}
+		}
+	}
+	return totalBytes, totalSize
+}