From 268a7ff7b87d9d85e224492b575a6e847c3dd769 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Mon, 15 Mar 2021 15:50:04 +0000
Subject: [PATCH] rc: add a full set of stats to core/stats
This patch adds the missing stats to the output of core/stats
- totalChecks
- totalTransfers
- totalBytes
- eta
This now includes enough information to rebuild the normal stats
output from rclone including percentage completions and ETAs.
Fixes #5116
---
fs/accounting/accounting.go | 13 ++---
fs/accounting/prometheus.go | 2 +-
fs/accounting/stats.go | 106 ++++++++++++++++++++++++----------
fs/accounting/stats_groups.go | 24 ++++----
4 files changed, 96 insertions(+), 49 deletions(-)
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go
index 8777447d3..b82480068 100644
--- a/fs/accounting/accounting.go
+++ b/fs/accounting/accounting.go
@@ -527,14 +527,11 @@ func (acc *Account) rcStats() (out rc.Params) {
out["speed"] = spd
out["speedAvg"] = cur
- eta, etaok := acc.eta()
- out["eta"] = nil
- if etaok {
- if eta > 0 {
- out["eta"] = eta.Seconds()
- } else {
- out["eta"] = 0
- }
+ eta, etaOK := acc.eta()
+ if etaOK {
+ out["eta"] = eta.Seconds()
+ } else {
+ out["eta"] = nil
}
out["name"] = acc.name
diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go
index a4c2a625e..9138d9120 100644
--- a/fs/accounting/prometheus.go
+++ b/fs/accounting/prometheus.go
@@ -90,7 +90,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
s.mu.RLock()
ch <- prometheus.MustNewConstMetric(c.bytesTransferred, prometheus.CounterValue, float64(s.bytes))
- ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.Speed())
+ ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.speed())
ch <- prometheus.MustNewConstMetric(c.numOfErrors, prometheus.CounterValue, float64(s.errors))
ch <- prometheus.MustNewConstMetric(c.numOfCheckFiles, prometheus.CounterValue, float64(s.checks))
ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers))
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 742a4ef6c..be6a7b7e1 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -65,9 +65,19 @@ func NewStats(ctx context.Context) *StatsInfo {
// RemoteStats returns stats for rc
func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
+ // NB if adding values here - make sure you update the docs in
+ // stats_groups.go
+
out = make(rc.Params)
+
+ ts := s.calculateTransferStats()
+ out["totalChecks"] = ts.totalChecks
+ out["totalTransfers"] = ts.totalTransfers
+ out["totalBytes"] = ts.totalBytes
+ out["transferTime"] = ts.transferTime
+ out["speed"] = ts.speed
+
s.mu.RLock()
- out["speed"] = s.Speed()
out["bytes"] = s.bytes
out["errors"] = s.errors
out["fatalError"] = s.fatalError
@@ -77,9 +87,15 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
out["deletes"] = s.deletes
out["deletedDirs"] = s.deletedDirs
out["renames"] = s.renames
- out["transferTime"] = s.totalDuration().Seconds()
out["elapsedTime"] = time.Since(startTime).Seconds()
+ eta, etaOK := eta(s.bytes, ts.totalBytes, ts.speed)
+ if etaOK {
+ out["eta"] = eta.Seconds()
+ } else {
+ out["eta"] = nil
+ }
s.mu.RUnlock()
+
if !s.checking.empty() {
out["checking"] = s.checking.remotes()
}
@@ -89,11 +105,14 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
if s.errors > 0 {
out["lastError"] = s.lastError.Error()
}
+
return out, nil
}
// Speed returns the average speed of the transfer in bytes/second
-func (s *StatsInfo) Speed() float64 {
+//
+// Call with lock held
+func (s *StatsInfo) speed() float64 {
dt := s.totalDuration()
dtSeconds := dt.Seconds()
speed := 0.0
@@ -202,6 +221,9 @@ func eta(size, total int64, rate float64) (eta time.Duration, ok bool) {
return 0, false
}
seconds := float64(remaining) / rate
+ if seconds < 0 {
+ seconds = 0
+ }
return time.Second * time.Duration(seconds), true
}
@@ -227,36 +249,60 @@ func percent(a int64, b int64) string {
return fmt.Sprintf("%d%%", int(float64(a)*100/float64(b)+0.5))
}
-// String convert the StatsInfo to a string for printing
-func (s *StatsInfo) String() string {
+// returned from calculateTransferStats
+type transferStats struct {
+ totalChecks int64
+ totalTransfers int64
+ totalBytes int64
+ transferTime float64
+ speed float64
+}
+
+// calculateTransferStats calculates some addtional transfer stats not
+// stored directly in StatsInfo
+func (s *StatsInfo) calculateTransferStats() (ts transferStats) {
// checking and transferring have their own locking so read
// here before lock to prevent deadlock on GetBytes
transferring, checking := s.transferring.count(), s.checking.count()
transferringBytesDone, transferringBytesTotal := s.transferring.progress(s)
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+
+ ts.totalChecks = int64(s.checkQueue) + s.checks + int64(checking)
+ ts.totalTransfers = int64(s.transferQueue) + s.transfers + int64(transferring)
+ // note that s.bytes already includes transferringBytesDone so
+ // we take it off here to avoid double counting
+ ts.totalBytes = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
+
+ dt := s.totalDuration()
+ ts.transferTime = dt.Seconds()
+ ts.speed = 0.0
+ if dt > 0 {
+ ts.speed = float64(s.bytes) / ts.transferTime
+ }
+
+ return ts
+}
+
+// String convert the StatsInfo to a string for printing
+func (s *StatsInfo) String() string {
+ // NB if adding more stats in here, remember to add them into
+ // RemoteStats() too.
+
+ ts := s.calculateTransferStats()
+
s.mu.RLock()
elapsedTime := time.Since(startTime)
elapsedTimeSecondsOnly := elapsedTime.Truncate(time.Second/10) % time.Minute
- dt := s.totalDuration()
- dtSeconds := dt.Seconds()
- speed := 0.0
- if dt > 0 {
- speed = float64(s.bytes) / dtSeconds
- }
- displaySpeed := speed
+ displaySpeed := ts.speed
if s.ci.DataRateUnit == "bits" {
displaySpeed *= 8
}
var (
- totalChecks = int64(s.checkQueue) + s.checks + int64(checking)
- totalTransfer = int64(s.transferQueue) + s.transfers + int64(transferring)
- // note that s.bytes already includes transferringBytesDone so
- // we take it off here to avoid double counting
- totalSize = s.transferQueueSize + s.bytes + transferringBytesTotal - transferringBytesDone
- currentSize = s.bytes
buf = &bytes.Buffer{}
xfrchkString = ""
dateString = ""
@@ -266,11 +312,11 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "\nTransferred: ")
} else {
xfrchk := []string{}
- if totalTransfer > 0 && s.transferQueue > 0 {
- xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, totalTransfer))
+ if ts.totalTransfers > 0 && s.transferQueue > 0 {
+ xfrchk = append(xfrchk, fmt.Sprintf("xfr#%d/%d", s.transfers, ts.totalTransfers))
}
- if totalChecks > 0 && s.checkQueue > 0 {
- xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, totalChecks))
+ if ts.totalChecks > 0 && s.checkQueue > 0 {
+ xfrchk = append(xfrchk, fmt.Sprintf("chk#%d/%d", s.checks, ts.totalChecks))
}
if len(xfrchk) > 0 {
xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
@@ -284,16 +330,16 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "%s%10s / %s, %s, %s, ETA %s%s",
dateString,
fs.SizeSuffix(s.bytes),
- fs.SizeSuffix(totalSize).Unit("Bytes"),
- percent(s.bytes, totalSize),
+ fs.SizeSuffix(ts.totalBytes).Unit("Bytes"),
+ percent(s.bytes, ts.totalBytes),
fs.SizeSuffix(displaySpeed).Unit(strings.Title(s.ci.DataRateUnit)+"/s"),
- etaString(currentSize, totalSize, speed),
+ etaString(s.bytes, ts.totalBytes, ts.speed),
xfrchkString,
)
if s.ci.ProgressTerminalTitle {
// Writes ETA to the terminal title
- terminal.WriteTerminalTitle("ETA: " + etaString(currentSize, totalSize, speed))
+ terminal.WriteTerminalTitle("ETA: " + etaString(s.bytes, ts.totalBytes, ts.speed))
}
if !s.ci.StatsOneLine {
@@ -314,9 +360,9 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "Errors: %10d%s\n",
s.errors, errorDetails)
}
- if s.checks != 0 || totalChecks != 0 {
+ if s.checks != 0 || ts.totalChecks != 0 {
_, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n",
- s.checks, totalChecks, percent(s.checks, totalChecks))
+ s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))
}
if s.deletes != 0 || s.deletedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs)\n", s.deletes, s.deletedDirs)
@@ -324,9 +370,9 @@ func (s *StatsInfo) String() string {
if s.renames != 0 {
_, _ = fmt.Fprintf(buf, "Renamed: %10d\n", s.renames)
}
- if s.transfers != 0 || totalTransfer != 0 {
+ if s.transfers != 0 || ts.totalTransfers != 0 {
_, _ = fmt.Fprintf(buf, "Transferred: %10d / %d, %s\n",
- s.transfers, totalTransfer, percent(s.transfers, totalTransfer))
+ s.transfers, ts.totalTransfers, percent(s.transfers, ts.totalTransfers))
}
_, _ = fmt.Fprintf(buf, "Elapsed time: %10ss\n", strings.TrimRight(elapsedTime.Truncate(time.Minute).String(), "0s")+fmt.Sprintf("%.1f", elapsedTimeSecondsOnly.Seconds()))
}
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index 0aace47c3..49657fc83 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -86,18 +86,22 @@ Returns the following values:
` + "```" + `
{
- "speed": average speed in bytes/sec since start of the process,
- "bytes": total transferred bytes since the start of the process,
+ "bytes": total transferred bytes since the start of the group,
+ "checks": number of files checked,
+ "deletes" : number of files deleted,
+ "elapsedTime": time in floating point seconds since rclone was started,
"errors": number of errors,
- "fatalError": whether there has been at least one FatalError,
- "retryError": whether there has been at least one non-NoRetryError,
- "checks": number of checked files,
- "transfers": number of transferred files,
- "deletes" : number of deleted files,
- "renames" : number of renamed files,
+ "eta": estimated time in seconds until the group completes,
+ "fatalError": boolean whether there has been at least one fatal error,
+ "lastError": last error string,
+ "renames" : number of files renamed,
+ "retryError": boolean showing whether there has been at least one non-NoRetryError,
+ "speed": average speed in bytes/sec since start of the group,
+ "totalBytes": total number of bytes in the group,
+ "totalChecks": total number of checks in the group,
+ "totalTransfers": total number of transfers in the group,
"transferTime" : total time spent on running jobs,
- "elapsedTime": time in seconds since the start of the process,
- "lastError": last occurred error,
+ "transfers": number of transferred files,
"transferring": an array of currently active file transfers:
[
{