From 13e46c4b3f831fa20cfa9fa51f3fbf9f9ad0cbca Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Thu, 17 Oct 2019 11:43:32 +0100
Subject: [PATCH] accounting: cull the old time ranges when possible to save
memory
---
fs/accounting/stats.go | 39 +++++++++++++++--
fs/accounting/stats_test.go | 83 ++++++++++++++++++++++++++++++++++---
2 files changed, 113 insertions(+), 9 deletions(-)
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index e41273ee8..afa06cf46 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -37,8 +37,9 @@ type StatsInfo struct {
renameQueueSize int64
deletes int64
inProgress *inProgress
- startedTransfers []*Transfer // currently active transfers
- oldTimeRanges timeRanges // a merged list of time ranges for the transfers
+ startedTransfers []*Transfer // currently active transfers
+ oldTimeRanges timeRanges // a merged list of time ranges for the transfers
+ oldDuration time.Duration // duration of transfers we have culled
}
// NewStats creates an initialised StatsInfo
@@ -155,6 +156,21 @@ func (trs *timeRanges) merge() {
*trs = newTrs
}
+// cull remove any ranges whose start and end are before cutoff
+// returning their duration sum
+func (trs *timeRanges) cull(cutoff time.Time) (d time.Duration) {
+ var newTrs = (*trs)[:0]
+ for _, tr := range *trs {
+ if cutoff.Before(tr.start) || cutoff.Before(tr.end) {
+ newTrs = append(newTrs, tr)
+ } else {
+ d += tr.end.Sub(tr.start)
+ }
+ }
+ *trs = newTrs
+ return d
+}
+
// total the time out of the time ranges
func (trs timeRanges) total() (total time.Duration) {
for _, tr := range trs {
@@ -182,7 +198,7 @@ func (s *StatsInfo) totalDuration() time.Duration {
}
timeRanges.merge()
- return timeRanges.total()
+ return s.oldDuration + timeRanges.total()
}
// eta returns the ETA of the current operation,
@@ -436,6 +452,7 @@ func (s *StatsInfo) ResetCounters() {
s.transfers = 0
s.deletes = 0
s.startedTransfers = nil
+ s.oldDuration = 0
}
// ResetErrors sets the errors count to 0 and resets lastError, fatalError and retryError
@@ -568,16 +585,30 @@ func (s *StatsInfo) AddTransfer(transfer *Transfer) {
//
// Must be called with the lock held
func (s *StatsInfo) removeTransfer(transfer *Transfer, i int) {
+ now := time.Now()
+
// add finished transfer onto old time ranges
start, end := transfer.TimeRange()
if end.IsZero() {
- end = time.Now()
+ end = now
}
s.oldTimeRanges = append(s.oldTimeRanges, timeRange{start, end})
s.oldTimeRanges.merge()
// remove the found entry
s.startedTransfers = append(s.startedTransfers[:i], s.startedTransfers[i+1:]...)
+
+ // Find youngest active transfer
+ oldestStart := now
+ for i := range s.startedTransfers {
+ start, _ := s.startedTransfers[i].TimeRange()
+ if start.Before(oldestStart) {
+ oldestStart = start
+ }
+ }
+
+ // remove old entries older than that
+ s.oldDuration += s.oldTimeRanges.cull(oldestStart)
}
// RemoveTransfer removes a reference to the started transfer.
diff --git a/fs/accounting/stats_test.go b/fs/accounting/stats_test.go
index 5ae5ad956..ea6f90ac9 100644
--- a/fs/accounting/stats_test.go
+++ b/fs/accounting/stats_test.go
@@ -254,6 +254,14 @@ func makeTimeRanges(t *testing.T, in []string) timeRanges {
return trs
}
+func (trs timeRanges) toStrings() (out []string) {
+ out = []string{}
+ for _, tr := range trs {
+ out = append(out, fmt.Sprintf("%d-%d", tr.start.Unix(), tr.end.Unix()))
+ }
+ return out
+}
+
func TestTimeRangeMerge(t *testing.T) {
for _, test := range []struct {
in []string
@@ -293,15 +301,80 @@ func TestTimeRangeMerge(t *testing.T) {
in := makeTimeRanges(t, test.in)
in.merge()
- got := []string{}
- for _, tr := range in {
- got = append(got, fmt.Sprintf("%d-%d", tr.start.Unix(), tr.end.Unix()))
- }
-
+ got := in.toStrings()
assert.Equal(t, test.want, got)
}
}
+func TestTimeRangeCull(t *testing.T) {
+ for _, test := range []struct {
+ in []string
+ cutoff int64
+ want []string
+ wantDuration time.Duration
+ }{{
+ in: []string{},
+ cutoff: 1,
+ want: []string{},
+ wantDuration: 0 * time.Second,
+ }, {
+ in: []string{"1-2"},
+ cutoff: 1,
+ want: []string{"1-2"},
+ wantDuration: 0 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 1,
+ want: []string{"2-5", "7-9"},
+ wantDuration: 0 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 4,
+ want: []string{"2-5", "7-9"},
+ wantDuration: 0 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 5,
+ want: []string{"7-9"},
+ wantDuration: 3 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9", "2-5", "2-5"},
+ cutoff: 6,
+ want: []string{"7-9"},
+ wantDuration: 9 * time.Second,
+ }, {
+ in: []string{"7-9", "3-3", "2-5"},
+ cutoff: 7,
+ want: []string{"7-9"},
+ wantDuration: 3 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 8,
+ want: []string{"7-9"},
+ wantDuration: 3 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 9,
+ want: []string{},
+ wantDuration: 5 * time.Second,
+ }, {
+ in: []string{"2-5", "7-9"},
+ cutoff: 10,
+ want: []string{},
+ wantDuration: 5 * time.Second,
+ }} {
+
+ in := makeTimeRanges(t, test.in)
+ cutoff := time.Unix(test.cutoff, 0)
+ gotDuration := in.cull(cutoff)
+
+ what := fmt.Sprintf("in=%q, cutoff=%d", test.in, test.cutoff)
+ got := in.toStrings()
+ assert.Equal(t, test.want, got, what)
+ assert.Equal(t, test.wantDuration, gotDuration, what)
+ }
+}
+
func TestTimeRangeDuration(t *testing.T) {
assert.Equal(t, 0*time.Second, timeRanges{}.total())
assert.Equal(t, 1*time.Second, makeTimeRanges(t, []string{"1-2"}).total())