diff --git a/vfs/help.go b/vfs/help.go index 444f400fc..7437aaabc 100644 --- a/vfs/help.go +++ b/vfs/help.go @@ -184,6 +184,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +#### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example !hash! is slow with the !local! and !sftp! backends as +they have to read the entire file and hash it, and !modtime! is slow +with the !s3!, !swift!, !ftp! and !qinqstor! backends because they +need to do an extra API call to fetch it. + +If you use the !--vfs-fast-fingerprint! flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over !local!, !s3! or !swift! backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ### VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index d2dc5d295..d61d6062a 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -784,7 +784,7 @@ func (item *Item) _checkObject(o fs.Object) error { // OK } } else { - remoteFingerprint := fs.Fingerprint(context.TODO(), o, false) + remoteFingerprint := fs.Fingerprint(context.TODO(), o, item.c.opt.FastFingerprint) fs.Debugf(item.name, "vfs cache: checking remote fingerprint %q against cached fingerprint %q", remoteFingerprint, item.info.Fingerprint) if item.info.Fingerprint != "" { // remote object && local object @@ -1159,7 +1159,7 @@ func (item *Item) _updateFingerprint() { return } oldFingerprint := item.info.Fingerprint - item.info.Fingerprint = fs.Fingerprint(context.TODO(), item.o, false) + item.info.Fingerprint = fs.Fingerprint(context.TODO(), item.o, item.c.opt.FastFingerprint) if oldFingerprint != item.info.Fingerprint { fs.Debugf(item.o, "vfs cache: fingerprint now %q", item.info.Fingerprint) } diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go index 36476d417..90cbe5bb7 100644 --- a/vfs/vfscommon/options.go +++ b/vfs/vfscommon/options.go @@ -33,6 +33,7 @@ type Options struct { WriteBack time.Duration // time to wait before writing back dirty files ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full" UsedIsSize bool // if true, use the `rclone size` algorithm for Used size + FastFingerprint bool // if set use fast fingerprints } // DefaultOpt is the default values uses for Opt diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index d1341a239..1dc26089d 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -38,5 +38,6 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache") flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full") flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size") + flags.BoolVarP(flagSet, &Opt.FastFingerprint, "vfs-fast-fingerprint", "", Opt.FastFingerprint, "Use fast (less accurate) fingerprints for change detection") platformFlags(flagSet) }