accounting: Make checkers show what they are doing
Before this change, all types of checkers showed "checking" after the file name despite the fact that not all of them were checking. After this change, they can show - checking - deleting - hashing - importing - listing - merging - moving - renaming See: https://forum.rclone.org/t/what-is-rclone-checking-during-a-purge/35931/
This commit is contained in:
parent
34ce11d2be
commit
019a486d5b
10 changed files with 28 additions and 24 deletions
|
@ -1221,7 +1221,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
|
||||||
fs.Errorf(object.Name, "Can't create object %v", err)
|
fs.Errorf(object.Name, "Can't create object %v", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
|
||||||
err = f.deleteByID(ctx, object.ID, object.Name)
|
err = f.deleteByID(ctx, object.ID, object.Name)
|
||||||
checkErr(err)
|
checkErr(err)
|
||||||
tr.Done(ctx, err)
|
tr.Done(ctx, err)
|
||||||
|
@ -1235,7 +1235,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fs.Errorf(object, "Can't create object %+v", err)
|
fs.Errorf(object, "Can't create object %+v", err)
|
||||||
}
|
}
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
|
||||||
if oldOnly && last != remote {
|
if oldOnly && last != remote {
|
||||||
// Check current version of the file
|
// Check current version of the file
|
||||||
if object.Action == "hide" {
|
if object.Action == "hide" {
|
||||||
|
|
|
@ -161,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
|
||||||
if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
|
if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
|
||||||
fs.Errorf(nil, "%s: failed to import: %v", remote, err)
|
fs.Errorf(nil, "%s: failed to import: %v", remote, err)
|
||||||
}
|
}
|
||||||
accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
|
accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
|
||||||
doneCount++
|
doneCount++
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
@ -4542,7 +4542,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
|
||||||
fs.Errorf(object, "Can't create object %+v", err)
|
fs.Errorf(object, "Can't create object %+v", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
|
||||||
// Work out whether the file is the current version or not
|
// Work out whether the file is the current version or not
|
||||||
isCurrentVersion := !versioned || !version.Match(remote)
|
isCurrentVersion := !versioned || !version.Match(remote)
|
||||||
fs.Debugf(nil, "%q version %v", remote, version.Match(remote))
|
fs.Debugf(nil, "%q version %v", remote, version.Match(remote))
|
||||||
|
|
|
@ -689,8 +689,8 @@ func (s *StatsInfo) RetryAfter() time.Time {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCheckingTransfer adds a checking transfer to the stats, from the object.
|
// NewCheckingTransfer adds a checking transfer to the stats, from the object.
|
||||||
func (s *StatsInfo) NewCheckingTransfer(obj fs.DirEntry) *Transfer {
|
func (s *StatsInfo) NewCheckingTransfer(obj fs.DirEntry, what string) *Transfer {
|
||||||
tr := newCheckingTransfer(s, obj)
|
tr := newCheckingTransfer(s, obj, what)
|
||||||
s.checking.add(tr)
|
s.checking.add(tr)
|
||||||
return tr
|
return tr
|
||||||
}
|
}
|
||||||
|
@ -720,7 +720,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry) *Transfer {
|
||||||
|
|
||||||
// NewTransferRemoteSize adds a transfer to the stats based on remote and size.
|
// NewTransferRemoteSize adds a transfer to the stats based on remote and size.
|
||||||
func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer {
|
func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer {
|
||||||
tr := newTransferRemoteSize(s, remote, size, false)
|
tr := newTransferRemoteSize(s, remote, size, false, "")
|
||||||
s.transferring.add(tr)
|
s.transferring.add(tr)
|
||||||
s.startAverageLoop()
|
s.startAverageLoop()
|
||||||
return tr
|
return tr
|
||||||
|
|
|
@ -50,6 +50,7 @@ type Transfer struct {
|
||||||
size int64
|
size int64
|
||||||
startedAt time.Time
|
startedAt time.Time
|
||||||
checking bool
|
checking bool
|
||||||
|
what string // what kind of transfer this is
|
||||||
|
|
||||||
// Protects all below
|
// Protects all below
|
||||||
//
|
//
|
||||||
|
@ -63,22 +64,23 @@ type Transfer struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// newCheckingTransfer instantiates new checking of the object.
|
// newCheckingTransfer instantiates new checking of the object.
|
||||||
func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
|
func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry, what string) *Transfer {
|
||||||
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true)
|
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true, what)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newTransfer instantiates new transfer.
|
// newTransfer instantiates new transfer.
|
||||||
func newTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
|
func newTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
|
||||||
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false)
|
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool) *Transfer {
|
func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool, what string) *Transfer {
|
||||||
tr := &Transfer{
|
tr := &Transfer{
|
||||||
stats: stats,
|
stats: stats,
|
||||||
remote: remote,
|
remote: remote,
|
||||||
size: size,
|
size: size,
|
||||||
startedAt: time.Now(),
|
startedAt: time.Now(),
|
||||||
checking: checking,
|
checking: checking,
|
||||||
|
what: what,
|
||||||
}
|
}
|
||||||
stats.AddTransfer(tr)
|
stats.AddTransfer(tr)
|
||||||
return tr
|
return tr
|
||||||
|
|
|
@ -110,10 +110,14 @@ func (tm *transferMap) String(ctx context.Context, progress *inProgress, exclude
|
||||||
if acc := progress.get(tr.remote); acc != nil {
|
if acc := progress.get(tr.remote); acc != nil {
|
||||||
out = acc.String()
|
out = acc.String()
|
||||||
} else {
|
} else {
|
||||||
|
what := tr.what
|
||||||
|
if what == "" {
|
||||||
|
what = tm.name
|
||||||
|
}
|
||||||
out = fmt.Sprintf("%*s: %s",
|
out = fmt.Sprintf("%*s: %s",
|
||||||
ci.StatsFileNameLength,
|
ci.StatsFileNameLength,
|
||||||
shortenName(tr.remote, ci.StatsFileNameLength),
|
shortenName(tr.remote, ci.StatsFileNameLength),
|
||||||
tm.name,
|
what,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
stringList = append(stringList, " * "+out)
|
stringList = append(stringList, " * "+out)
|
||||||
|
|
|
@ -120,7 +120,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||||
// check to see if two objects are identical using the check function
|
// check to see if two objects are identical using the check function
|
||||||
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
|
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
|
||||||
ci := fs.GetConfig(ctx)
|
ci := fs.GetConfig(ctx)
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(src, "checking")
|
||||||
defer func() {
|
defer func() {
|
||||||
tr.Done(ctx, err)
|
tr.Done(ctx, err)
|
||||||
}()
|
}()
|
||||||
|
@ -450,7 +450,7 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(obj)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(obj, "hashing")
|
||||||
defer tr.Done(ctx, err)
|
defer tr.Done(ctx, err)
|
||||||
|
|
||||||
if !sumFound {
|
if !sumFound {
|
||||||
|
|
|
@ -286,7 +286,7 @@ func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*d
|
||||||
ci := fs.GetConfig(ctx)
|
ci := fs.GetConfig(ctx)
|
||||||
err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
|
err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(entry)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(entry, "merging")
|
||||||
|
|
||||||
remote := entry.Remote()
|
remote := entry.Remote()
|
||||||
parentRemote := path.Dir(remote)
|
parentRemote := path.Dir(remote)
|
||||||
|
@ -438,7 +438,7 @@ func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool
|
||||||
files := map[string][]fs.Object{}
|
files := map[string][]fs.Object{}
|
||||||
err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||||
entries.ForObject(func(o fs.Object) {
|
entries.ForObject(func(o fs.Object) {
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "checking")
|
||||||
defer tr.Done(ctx, nil)
|
defer tr.Done(ctx, nil)
|
||||||
|
|
||||||
var remote string
|
var remote string
|
||||||
|
|
|
@ -544,7 +544,7 @@ func SameObject(src, dst fs.Object) bool {
|
||||||
// be nil.
|
// be nil.
|
||||||
func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
|
func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
|
||||||
ci := fs.GetConfig(ctx)
|
ci := fs.GetConfig(ctx)
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(src, "moving")
|
||||||
defer func() {
|
defer func() {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
accounting.Stats(ctx).Renames(1)
|
accounting.Stats(ctx).Renames(1)
|
||||||
|
@ -633,7 +633,7 @@ func SuffixName(ctx context.Context, remote string) string {
|
||||||
// deleting
|
// deleting
|
||||||
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
|
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
|
||||||
ci := fs.GetConfig(ctx)
|
ci := fs.GetConfig(ctx)
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(dst)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting")
|
||||||
defer func() {
|
defer func() {
|
||||||
tr.Done(ctx, err)
|
tr.Done(ctx, err)
|
||||||
}()
|
}()
|
||||||
|
@ -938,7 +938,7 @@ func List(ctx context.Context, f fs.Fs, w io.Writer) error {
|
||||||
func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
|
func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
|
||||||
ci := fs.GetConfig(ctx)
|
ci := fs.GetConfig(ctx)
|
||||||
return ListFn(ctx, f, func(o fs.Object) {
|
return ListFn(ctx, f, func(o fs.Object) {
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "listing")
|
||||||
defer func() {
|
defer func() {
|
||||||
tr.Done(ctx, nil)
|
tr.Done(ctx, nil)
|
||||||
}()
|
}()
|
||||||
|
@ -996,7 +996,7 @@ func hashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag
|
||||||
return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
|
return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
|
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "hashing")
|
||||||
defer func() {
|
defer func() {
|
||||||
tr.Done(ctx, err)
|
tr.Done(ctx, err)
|
||||||
}()
|
}()
|
||||||
|
@ -1929,7 +1929,6 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
|
||||||
|
|
||||||
_, err = Op(ctx, fdst, dstObj, dstFileName, srcObj)
|
_, err = Op(ctx, fdst, dstObj, dstFileName, srcObj)
|
||||||
} else {
|
} else {
|
||||||
tr := accounting.Stats(ctx).NewCheckingTransfer(srcObj)
|
|
||||||
if !cp {
|
if !cp {
|
||||||
if ci.IgnoreExisting {
|
if ci.IgnoreExisting {
|
||||||
fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")
|
fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")
|
||||||
|
@ -1937,7 +1936,6 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
|
||||||
err = DeleteFile(ctx, srcObj)
|
err = DeleteFile(ctx, srcObj)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tr.Done(ctx, err)
|
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -329,7 +329,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
|
||||||
}
|
}
|
||||||
src := pair.Src
|
src := pair.Src
|
||||||
var err error
|
var err error
|
||||||
tr := accounting.Stats(s.ctx).NewCheckingTransfer(src)
|
tr := accounting.Stats(s.ctx).NewCheckingTransfer(src, "checking")
|
||||||
// Check to see if can store this
|
// Check to see if can store this
|
||||||
if src.Storable() {
|
if src.Storable() {
|
||||||
needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src)
|
needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src)
|
||||||
|
@ -779,7 +779,7 @@ func (s *syncCopyMove) makeRenameMap() {
|
||||||
for obj := range in {
|
for obj := range in {
|
||||||
// only create hash for dst fs.Object if its size could match
|
// only create hash for dst fs.Object if its size could match
|
||||||
if _, found := possibleSizes[obj.Size()]; found {
|
if _, found := possibleSizes[obj.Size()]; found {
|
||||||
tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj)
|
tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj, "renaming")
|
||||||
hash := s.renameID(obj, s.trackRenamesStrategy, s.modifyWindow)
|
hash := s.renameID(obj, s.trackRenamesStrategy, s.modifyWindow)
|
||||||
|
|
||||||
if hash != "" {
|
if hash != "" {
|
||||||
|
|
Loading…
Reference in a new issue