TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

local: only calculate the required hashes for big speedup

Browse source 
Before this change we calculated all possible hashes for the file when
the `Hashes` method was called.

After we only calculate the Hash requested.

Almost all uses of `Hash` just need one checksum.  This will slow down
`rclone lsjson` with the `--hash` flag.  Perhaps lsjson should have a
`--hash-type` flag.

However it will speed up sync/copy/move/check/md5sum/sha1sum etc.

Before it took 12.4 seconds to md5sum a 1GB file, after it takes 3.1
seconds which is the same time the md5sum utility takes.
This commit is contained in:
Nick Craig-Wood 2019-05-11 22:21:37 +01:00
parent 822483aac5
commit c3af0a1eca
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
backend/local/local.go
View file

@ -709,9 +709,10 @@ func (o *Object) Hash(r hash.Type) (string, error) {
o.fs.objectHashesMu.Lock() o.fs.objectHashesMu.Lock()
hashes := o.hashes hashes := o.hashes
hashValue, hashFound := o.hashes[r]
o.fs.objectHashesMu.Unlock() o.fs.objectHashesMu.Unlock()
if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil { if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
var in io.ReadCloser var in io.ReadCloser
if !o.translatedLink { if !o.translatedLink {
@ -722,7 +723,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
if err != nil { if err != nil {
return "", errors.Wrap(err, "hash: failed to open") return "", errors.Wrap(err, "hash: failed to open")
} }
hashes, err = hash.Stream(in) hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
closeErr := in.Close() closeErr := in.Close()
if err != nil { if err != nil {
return "", errors.Wrap(err, "hash: failed to read") return "", errors.Wrap(err, "hash: failed to read")
@ -730,11 +731,16 @@ func (o *Object) Hash(r hash.Type) (string, error) {
if closeErr != nil { if closeErr != nil {
return "", errors.Wrap(closeErr, "hash: failed to close") return "", errors.Wrap(closeErr, "hash: failed to close")
} }
hashValue = hashes[r]
o.fs.objectHashesMu.Lock() o.fs.objectHashesMu.Lock()
if o.hashes == nil {
o.hashes = hashes o.hashes = hashes
} else {
o.hashes[r] = hashValue
}
o.fs.objectHashesMu.Unlock() o.fs.objectHashesMu.Unlock()
} }
return hashes[r], nil return hashValue, nil
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes