alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

operations: speed up hash checking by aborting the other hash if first returns nothing

Browse source 
This speeds up hash checks when a Hash() function returns "" - this
means that the hash can be canceled for the other side.

In the common case of local hash vs remote hash empty this saves a lot
of time.

See: https://forum.rclone.org/t/rclone-s3-backend-copy-is-2x-slower-than-aws-s3-cp/27321/9
This commit is contained in:
Nick Craig-Wood 2021-11-09 09:45:36 +00:00
parent 5e4caa69ce
commit 115f1c2cc9
1 changed files with 30 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
fs/operations/operations.go
View file

@ -62,37 +62,51 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b
return equal, ht, err return equal, ht, err
} }
var errNoHash = errors.New("no hash available")
// checkHashes does the work of CheckHashes but takes a hash.Type and // checkHashes does the work of CheckHashes but takes a hash.Type and
// returns the effective hash type used. // returns the effective hash type used.
func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) { func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) {
// Calculate hashes in parallel // Calculate hashes in parallel
g, ctx := errgroup.WithContext(ctx) g, ctx := errgroup.WithContext(ctx)
var srcErr, dstErr error
g.Go(func() (err error) { g.Go(func() (err error) {
srcHash, err = src.Hash(ctx, ht) srcHash, srcErr = src.Hash(ctx, ht)
if err != nil { if srcErr != nil {
err = fs.CountError(err) return srcErr
fs.Errorf(src, "Failed to calculate src hash: %v", err)
} }
return err if srcHash == "" {
fs.Debugf(src, "Src hash empty - aborting Dst hash check")
return errNoHash
}
return nil
}) })
g.Go(func() (err error) { g.Go(func() (err error) {
dstHash, err = dst.Hash(ctx, ht) dstHash, dstErr = dst.Hash(ctx, ht)
if err != nil { if dstErr != nil {
err = fs.CountError(err) return dstErr
fs.Errorf(dst, "Failed to calculate dst hash: %v", err)
} }
return err if dstHash == "" {
fs.Debugf(src, "Dst hash empty - aborting Src hash check")
return errNoHash
}
return nil
}) })
err = g.Wait() err = g.Wait()
if err == errNoHash {
return true, hash.None, srcHash, dstHash, nil
}
if srcErr != nil {
err = fs.CountError(srcErr)
fs.Errorf(dst, "Failed to calculate src hash: %v", err)
}
if dstErr != nil {
err = fs.CountError(dstErr)
fs.Errorf(src, "Failed to calculate dst hash: %v", err)
}
if err != nil { if err != nil {
return false, ht, srcHash, dstHash, err return false, ht, srcHash, dstHash, err
} }
if srcHash == "" {
return true, hash.None, srcHash, dstHash, nil
}
if dstHash == "" {
return true, hash.None, srcHash, dstHash, nil
}
if srcHash != dstHash { if srcHash != dstHash {
fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs()) fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs())
fs.Debugf(dst, "%v = %s (%v)", ht, dstHash, dst.Fs()) fs.Debugf(dst, "%v = %s (%v)", ht, dstHash, dst.Fs())