vfs: add --vfs-read-ahead parameter for use with --vfs-cache-mode full

This parameter causes extra read-ahead over --buffer-size which is not
buffered in memory but on disk.

vfs/help.go

@@ -159,6 +159,13 @@ the data that has been downloaded present in them.
 This mode should support all normal file system operations and is
 otherwise identical to --vfs-cache-mode writes.
 
+When reading a file rclone will read --buffer-size plus
+--vfs-read-ahead bytes ahead.  The --buffer-size is buffered in memory
+whereas the --vfs-read-ahead is buffered on disk.
+
+When using this mode it is recommended that --buffer-size is not set
+too big and --vfs-read-ahead is set large if required.
+
 ### VFS Performance
 
 These flags may be used to enable/disable features of the VFS for

vfs/vfscache/downloaders/downloaders.go

@@ -273,9 +273,14 @@ func (dls *Downloaders) _closeWaiters(err error) {
 //
 // call with lock held
 func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) {
-	// FIXME this window could be a different config var?
+	// The window includes potentially unread data in the buffer
 	window := int64(fs.Config.BufferSize)
 
+	// Increase the read range by the read ahead if set
+	if dls.opt.ReadAhead > 0 {
+		r.Size += int64(dls.opt.ReadAhead)
+	}
+
 	// We may be reopening a downloader after a failure here or
 	// doing a tentative prefetch so check to see that we haven't
 	// read some stuff already.
@@ -310,15 +315,15 @@ func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) {
 	// If there isn't one then start a new one
 	dls._removeClosed()
 	for _, dl = range dls.dls {
-		start, maxOffset := dl.getRange()
+		start, offset := dl.getRange()
 
 		// The downloader's offset to offset+window is the gap
 		// in which we would like to re-use this
 		// downloader. The downloader will never reach before
-		// start and maxOffset+windows is too far away - we'd
+		// start and offset+windows is too far away - we'd
 		// rather start another downloader.
-		// fs.Debugf(nil, "r=%v start=%d, maxOffset=%d, found=%v", r, start, maxOffset, r.Pos >= start && r.Pos < maxOffset+window)
-		if r.Pos >= start && r.Pos < maxOffset+window {
+		// fs.Debugf(nil, "r=%v start=%d, offset=%d, found=%v", r, start, offset, r.Pos >= start && r.Pos < offset+window)
+		if r.Pos >= start && r.Pos < offset+window {
 			// Found downloader which will soon have our data
 			dl.setRange(r)
 			return nil
@@ -598,8 +603,8 @@ func (dl *downloader) setRange(r ranges.Range) {
 }
 
 // get the current range this downloader is working on
-func (dl *downloader) getRange() (start, maxOffset int64) {
+func (dl *downloader) getRange() (start, offset int64) {
 	dl.mu.Lock()
 	defer dl.mu.Unlock()
-	return dl.start, dl.maxOffset
+	return dl.start, dl.offset
 }

vfs/vfscommon/options.go

@@ -31,6 +31,7 @@ type Options struct {
 	WriteWait         time.Duration // time to wait for in-sequence write
 	ReadWait          time.Duration // time to wait for in-sequence read
 	WriteBack         time.Duration // time to wait before writing back dirty files
+	ReadAhead         fs.SizeSuffix // bytes to read ahead in cache mode "full"
 }
 
 // DefaultOpt is the default values uses for Opt
@@ -56,4 +57,5 @@ var DefaultOpt = Options{
 	WriteWait:         1000 * time.Millisecond,
 	ReadWait:          20 * time.Millisecond,
 	WriteBack:         5 * time.Second,
+	ReadAhead:         0 * fs.MebiByte,
 }

vfs/vfsflags/vfsflags.go

@@ -36,5 +36,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.DurationVarP(flagSet, &Opt.WriteWait, "vfs-write-wait", "", Opt.WriteWait, "Time to wait for in-sequence write before giving error.")
 	flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
 	flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.")
+	flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.")
 	platformFlags(flagSet)
 }