From e2183ad661fdddbb097fa502d95511cf1887f3ee Mon Sep 17 00:00:00 2001 From: Max Sum Date: Sun, 14 Jun 2020 02:37:42 +0800 Subject: [PATCH] fs/accounting: use transferMap instead of stringSet --- fs/accounting/stats.go | 42 ++++++------- fs/accounting/stringset.go | 112 ----------------------------------- fs/accounting/transfermap.go | 112 +++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 135 deletions(-) delete mode 100644 fs/accounting/stringset.go create mode 100644 fs/accounting/transfermap.go diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index 0ed50a322..cfa2bbba5 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -26,11 +26,11 @@ type StatsInfo struct { retryError bool retryAfter time.Time checks int64 - checking *stringSet + checking *transferMap checkQueue int checkQueueSize int64 transfers int64 - transferring *stringSet + transferring *transferMap transferQueue int transferQueueSize int64 renames int64 @@ -47,8 +47,8 @@ type StatsInfo struct { // NewStats creates an initialised StatsInfo func NewStats() *StatsInfo { return &StatsInfo{ - checking: newStringSet(fs.Config.Checkers, "checking"), - transferring: newStringSet(fs.Config.Transfers, "transferring"), + checking: newTransferMap(fs.Config.Checkers, "checking"), + transferring: newTransferMap(fs.Config.Transfers, "transferring"), inProgress: newInProgress(), } } @@ -81,11 +81,11 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) { s.transferring.mu.RLock() var t []rc.Params - for name := range s.transferring.items { + for name, tr := range s.transferring.items { if acc := s.inProgress.get(name); acc != nil { t = append(t, acc.RemoteStats()) } else { - t = append(t, s.transferRemoteStats(name)) + t = append(t, s.transferRemoteStats(tr)) } } out["transferring"] = t @@ -108,18 +108,11 @@ func (s *StatsInfo) Speed() float64 { return speed } -func (s *StatsInfo) transferRemoteStats(name string) rc.Params { - s.mu.RLock() - defer s.mu.RUnlock() - for _, tr := range s.startedTransfers { - if tr.remote == name { - return rc.Params{ - "name": name, - "size": tr.size, - } - } +func (s *StatsInfo) transferRemoteStats(tr *Transfer) rc.Params { + return rc.Params{ + "name": tr.remote, + "size": tr.size, } - return rc.Params{"name": name} } // timeRange is a start and end time of a transfer @@ -558,8 +551,9 @@ func (s *StatsInfo) RetryAfter() time.Time { // NewCheckingTransfer adds a checking transfer to the stats, from the object. func (s *StatsInfo) NewCheckingTransfer(obj fs.Object) *Transfer { - s.checking.add(obj.Remote()) - return newCheckingTransfer(s, obj) + tr := newCheckingTransfer(s, obj) + s.checking.add(tr) + return tr } // DoneChecking removes a check from the stats @@ -579,14 +573,16 @@ func (s *StatsInfo) GetTransfers() int64 { // NewTransfer adds a transfer to the stats from the object. func (s *StatsInfo) NewTransfer(obj fs.Object) *Transfer { - s.transferring.add(obj.Remote()) - return newTransfer(s, obj) + tr := newTransfer(s, obj) + s.transferring.add(tr) + return tr } // NewTransferRemoteSize adds a transfer to the stats based on remote and size. func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer { - s.transferring.add(remote) - return newTransferRemoteSize(s, remote, size, false) + tr := newTransferRemoteSize(s, remote, size, false) + s.transferring.add(tr) + return tr } // DoneTransferring removes a transfer from the stats diff --git a/fs/accounting/stringset.go b/fs/accounting/stringset.go deleted file mode 100644 index 52df0ab4b..000000000 --- a/fs/accounting/stringset.go +++ /dev/null @@ -1,112 +0,0 @@ -package accounting - -import ( - "fmt" - "sort" - "strings" - "sync" - - "github.com/rclone/rclone/fs" -) - -// stringSet holds a set of strings -type stringSet struct { - mu sync.RWMutex - items map[string]struct{} - name string -} - -// newStringSet creates a new empty string set of capacity size -func newStringSet(size int, name string) *stringSet { - return &stringSet{ - items: make(map[string]struct{}, size), - name: name, - } -} - -// add adds remote to the set -func (ss *stringSet) add(remote string) { - ss.mu.Lock() - ss.items[remote] = struct{}{} - ss.mu.Unlock() -} - -// del removes remote from the set -func (ss *stringSet) del(remote string) { - ss.mu.Lock() - delete(ss.items, remote) - ss.mu.Unlock() -} - -// merge adds items from another set -func (ss *stringSet) merge(m *stringSet) { - ss.mu.Lock() - m.mu.Lock() - for item := range m.items { - ss.items[item] = struct{}{} - } - m.mu.Unlock() - ss.mu.Unlock() -} - -// empty returns whether the set has any items -func (ss *stringSet) empty() bool { - ss.mu.RLock() - defer ss.mu.RUnlock() - return len(ss.items) == 0 -} - -// count returns the number of items in the set -func (ss *stringSet) count() int { - ss.mu.RLock() - defer ss.mu.RUnlock() - return len(ss.items) -} - -// String returns string representation of set items excluding any in -// exclude (if set). -func (ss *stringSet) String(progress *inProgress, exclude *stringSet) string { - ss.mu.RLock() - defer ss.mu.RUnlock() - strngs := make([]string, 0, len(ss.items)) - for name := range ss.items { - if exclude != nil { - exclude.mu.RLock() - _, found := exclude.items[name] - exclude.mu.RUnlock() - if found { - continue - } - } - var out string - if acc := progress.get(name); acc != nil { - out = acc.String() - } else { - out = fmt.Sprintf("%*s: %s", - fs.Config.StatsFileNameLength, - shortenName(name, fs.Config.StatsFileNameLength), - ss.name, - ) - } - strngs = append(strngs, " * "+out) - } - sorted := sort.StringSlice(strngs) - sorted.Sort() - return strings.Join(sorted, "\n") -} - -// progress returns total bytes read as well as the size. -func (ss *stringSet) progress(stats *StatsInfo) (totalBytes, totalSize int64) { - ss.mu.RLock() - defer ss.mu.RUnlock() - for name := range ss.items { - if acc := stats.inProgress.get(name); acc != nil { - bytes, size := acc.progress() - if size >= 0 && bytes >= 0 { - totalBytes += bytes - totalSize += size - } - } - } - return totalBytes, totalSize -} diff --git a/fs/accounting/transfermap.go b/fs/accounting/transfermap.go new file mode 100644 index 000000000..4663802af --- /dev/null +++ b/fs/accounting/transfermap.go @@ -0,0 +1,112 @@ +package accounting + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/rclone/rclone/fs" +) + +// transferMap holds name to transfer map +type transferMap struct { + mu sync.RWMutex + items map[string]*Transfer + name string +} + +// newTransferMap creates a new empty transfer map of capacity size +func newTransferMap(size int, name string) *transferMap { + return &transferMap{ + items: make(map[string]*Transfer, size), + name: name, + } +} + +// add adds a new transfer to the map +func (tm *transferMap) add(tr *Transfer) { + tm.mu.Lock() + tm.items[tr.remote] = tr + tm.mu.Unlock() +} + +// del removes a transfer from the map by name +func (tm *transferMap) del(remote string) { + tm.mu.Lock() + delete(tm.items, remote) + tm.mu.Unlock() +} + +// merge adds items from another map +func (tm *transferMap) merge(m *transferMap) { + tm.mu.Lock() + m.mu.Lock() + for name, tr := range m.items { + tm.items[name] = tr + } + m.mu.Unlock() + tm.mu.Unlock() +} + +// empty returns whether the map has any items +func (tm *transferMap) empty() bool { + tm.mu.RLock() + defer tm.mu.RUnlock() + return len(tm.items) == 0 +} + +// count returns the number of items in the map +func (tm *transferMap) count() int { + tm.mu.RLock() + defer tm.mu.RUnlock() + return len(tm.items) +} + +// String returns string representation of map items excluding any in +// exclude (if set). +func (tm *transferMap) String(progress *inProgress, exclude *transferMap) string { + tm.mu.RLock() + defer tm.mu.RUnlock() + strngs := make([]string, 0, len(tm.items)) + for name, _ := range tm.items { + if exclude != nil { + exclude.mu.RLock() + _, found := exclude.items[name] + exclude.mu.RUnlock() + if found { + continue + } + } + var out string + if acc := progress.get(name); acc != nil { + out = acc.String() + } else { + out = fmt.Sprintf("%*s: %s", + fs.Config.StatsFileNameLength, + shortenName(name, fs.Config.StatsFileNameLength), + tm.name, + ) + } + strngs = append(strngs, " * "+out) + } + sorted := sort.StringSlice(strngs) + sorted.Sort() + return strings.Join(sorted, "\n") +} + +// progress returns total bytes read as well as the size. +func (tm *transferMap) progress(stats *StatsInfo) (totalBytes, totalSize int64) { + tm.mu.RLock() + defer tm.mu.RUnlock() + for name := range tm.items { + if acc := stats.inProgress.get(name); acc != nil { + bytes, size := acc.progress() + if size >= 0 && bytes >= 0 { + totalBytes += bytes + totalSize += size + } + } + } + return totalBytes, totalSize +}