TrueCloudLab/rclone
16
0
Fork 4
Code Issues 1 Pull requests Projects Releases 4 Packages Wiki Activity Actions

accounting: isolate stats to groups

Browse source 
Introduce stats groups that will isolate accounting for logically
different transferring operations. That way multiple accounting
operations can be done in parallel without interfering with each other
stats.

Using groups is optional. There is dedicated global stats that will be
used by default if no group is specified. This is operating mode for CLI
usage which is just fire and forget operation.

For running rclone as rc http server each request will create it's own
group. Also there is an option to specify your own group.
This commit is contained in:
Aleksandar Jankovic 2019-07-18 12:13:54 +02:00 • committed by Nick Craig-Wood
parent be0464f5f1
commit 8243ff8bc8
25 changed files with 551 additions and 281 deletions
Download patch file Download diff file Expand all files Collapse all files

22
fs/sync/sync.go
View file

@ -82,12 +82,12 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
dstEmptyDirs: make(map[string]fs.DirEntry),
srcEmptyDirs: make(map[string]fs.DirEntry),
noTraverse: fs.Config.NoTraverse,
toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog),
toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog),
toBeChecked: newPipe(accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog),
toBeUploaded: newPipe(accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog),
deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
trackRenames: fs.Config.TrackRenames,
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
toBeRenamed: newPipe(accounting.Stats.SetRenameQueue, fs.Config.MaxBacklog),
toBeRenamed: newPipe(accounting.Stats(ctx).SetRenameQueue, fs.Config.MaxBacklog),
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
}
s.ctx, s.cancel = context.WithCancel(ctx)
@ -215,7 +215,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
return
}
src := pair.Src
accounting.Stats.Checking(src.Remote())
accounting.Stats(s.ctx).Checking(src.Remote())
// Check to see if can store this
if src.Storable() {
NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)
@ -256,7 +256,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
}
}
}
accounting.Stats.DoneChecking(src.Remote())
accounting.Stats(s.ctx).DoneChecking(src.Remote())
}
}
@ -401,7 +401,7 @@ func (s *syncCopyMove) stopDeleters() {
// checkSrcMap is clear then it assumes that the any source files that
// have been found have been removed from dstFiles already.
func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
if accounting.Stats.Errored() && !fs.Config.IgnoreErrors {
if accounting.Stats(s.ctx).Errored() && !fs.Config.IgnoreErrors {
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)
return fs.ErrorNotDeleting
}
@ -437,7 +437,7 @@ func deleteEmptyDirectories(ctx context.Context, f fs.Fs, entriesMap map[string]
if len(entriesMap) == 0 {
return nil
}
if accounting.Stats.Errored() && !fs.Config.IgnoreErrors {
if accounting.Stats(ctx).Errored() && !fs.Config.IgnoreErrors {
fs.Errorf(f, "%v", fs.ErrorNotDeletingDirs)
return fs.ErrorNotDeletingDirs
}
@ -497,8 +497,8 @@ func copyEmptyDirectories(ctx context.Context, f fs.Fs, entries map[string]fs.Di
}
}
if accounting.Stats.Errored() {
fs.Debugf(f, "failed to copy %d directories", accounting.Stats.GetErrors())
if accounting.Stats(ctx).Errored() {
fs.Debugf(f, "failed to copy %d directories", accounting.Stats(ctx).GetErrors())
}
if okCount > 0 {
@ -587,12 +587,12 @@ func (s *syncCopyMove) makeRenameMap() {
for obj := range in {
// only create hash for dst fs.Object if its size could match
if _, found := possibleSizes[obj.Size()]; found {
accounting.Stats.Checking(obj.Remote())
accounting.Stats(s.ctx).Checking(obj.Remote())
hash := s.renameHash(obj)
if hash != "" {
s.pushRenameMap(hash, obj)
}
accounting.Stats.DoneChecking(obj.Remote())
accounting.Stats(s.ctx).DoneChecking(obj.Remote())
}
}
}()