vfs: add --vfs-fast-fingerprint for less accurate but faster fingerprints

This commit is contained in:
Nick Craig-Wood 2022-03-21 16:15:12 +00:00
parent e4f5912294
commit d4da9b98d6
4 changed files with 36 additions and 2 deletions

View file

@ -184,6 +184,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
#### Fingerprinting
Various parts of the VFS use fingerprinting to see if a local file
copy has changed relative to a remote file. Fingerprints are made
from:
- size
- modification time
- hash
where available on an object.
On some backends some of these attributes are slow to read (they take
an extra API call per object, or extra work per object).
For example !hash! is slow with the !local! and !sftp! backends as
they have to read the entire file and hash it, and !modtime! is slow
with the !s3!, !swift!, !ftp! and !qinqstor! backends because they
need to do an extra API call to fetch it.
If you use the !--vfs-fast-fingerprint! flag then rclone will not
include the slow operations in the fingerprint. This makes the
fingerprinting less accurate but much faster and will improve the
opening time of cached files.
If you are running a vfs cache over !local!, !s3! or !swift! backends
then using this flag is recommended.
Note that if you change the value of this flag, the fingerprints of
the files in the cache may be invalidated and the files will need to
be downloaded again.
### VFS Chunked Reading
When rclone reads files from a remote it reads them in chunks. This

View file

@ -784,7 +784,7 @@ func (item *Item) _checkObject(o fs.Object) error {
// OK
}
} else {
remoteFingerprint := fs.Fingerprint(context.TODO(), o, false)
remoteFingerprint := fs.Fingerprint(context.TODO(), o, item.c.opt.FastFingerprint)
fs.Debugf(item.name, "vfs cache: checking remote fingerprint %q against cached fingerprint %q", remoteFingerprint, item.info.Fingerprint)
if item.info.Fingerprint != "" {
// remote object && local object
@ -1159,7 +1159,7 @@ func (item *Item) _updateFingerprint() {
return
}
oldFingerprint := item.info.Fingerprint
item.info.Fingerprint = fs.Fingerprint(context.TODO(), item.o, false)
item.info.Fingerprint = fs.Fingerprint(context.TODO(), item.o, item.c.opt.FastFingerprint)
if oldFingerprint != item.info.Fingerprint {
fs.Debugf(item.o, "vfs cache: fingerprint now %q", item.info.Fingerprint)
}

View file

@ -33,6 +33,7 @@ type Options struct {
WriteBack time.Duration // time to wait before writing back dirty files
ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full"
UsedIsSize bool // if true, use the `rclone size` algorithm for Used size
FastFingerprint bool // if set use fast fingerprints
}
// DefaultOpt is the default values uses for Opt

View file

@ -38,5 +38,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache")
flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full")
flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size")
flags.BoolVarP(flagSet, &Opt.FastFingerprint, "vfs-fast-fingerprint", "", Opt.FastFingerprint, "Use fast (less accurate) fingerprints for change detection")
platformFlags(flagSet)
}