vfs: stop virtual directory entries dropping out of the directory cache

Rclone adds virtual directory entries to the directory cache when it
creates a file or directory.

Before this change these dropped out of the directory cache when the
directory cache was reloaded. This meant that when the directory cache
expired:

- On bucket based backends, empty directories would disappear
- When using VFS writeback, files in the process of uploading would disappear

This is fixed by keeping track of the virtual entries in each
directory. The virtual entries are removed when they become real - ie
the object is read back from the listing.

This also keeps tracks of deletes in the same way so if a file is
deleted, it will not re-appear when the directory cache is reloaded if
the deletion hasn't finished yet.

vfs/dir.go

@@ -30,11 +30,23 @@ type Dir struct {
 	path    string
 	modTime time.Time
 	entry   fs.Directory
-	read    time.Time       // time directory entry last read
-	items   map[string]Node // directory entries - can be empty but not nil
-	sys     interface{}     // user defined info to be attached here
+	read    time.Time         // time directory entry last read
+	items   map[string]Node   // directory entries - can be empty but not nil
+	virtual map[string]vState // virtual directory entries - may be nil
+	sys     interface{}       // user defined info to be attached here
 }
 
+//go:generate stringer -type=vState
+
+// vState describes the state of the virtual directory entries
+type vState byte
+
+const (
+	vOK  vState = iota // Not virtual
+	vAdd               // added file or directory
+	vDel               // removed file or directory
+)
+
 func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
 	return &Dir{
 		vfs:     vfs,
@@ -48,7 +60,7 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
 	}
 }
 
-// String converts it to printablee
+// String converts it to printable
 func (d *Dir) String() string {
 	if d == nil {
 		return "<nil *Dir>"
@@ -126,7 +138,12 @@ func (d *Dir) forgetDirPath(relativePath string) {
 			// this is called with the mutex held
 			fs.Debugf(dir.path, "forgetting directory cache")
 			dir.read = time.Time{}
-			dir.items = make(map[string]Node)
+			// Don't clear directory entries if there are virtual
+			// items in there.
+			if len(dir.virtual) == 0 {
+				dir.items = make(map[string]Node)
+				dir.virtual = nil
+			}
 		})
 	}
 }
@@ -139,7 +156,7 @@ func (d *Dir) ForgetAll() {
 	d.forgetDirPath("")
 }
 
-// invalidateDir invalidates the directory cache for absPath relative to this dir
+// invalidateDir invalidates the directory cache for absPath relative to the root
 func (d *Dir) invalidateDir(absPath string) {
 	node := d.vfs.root.cachedNode(absPath)
 	if dir, ok := node.(*Dir); ok {
@@ -252,20 +269,77 @@ func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) {
 
 // addObject adds a new object or directory to the directory
 //
+// The name passed in is marked as virtual as it hasn't been read from a remote
+// directory listing.
+//
 // note that we add new objects rather than updating old ones
 func (d *Dir) addObject(node Node) {
 	d.mu.Lock()
-	d.items[node.Name()] = node
+	leaf := node.Name()
+	d.items[leaf] = node
+	if d.virtual == nil {
+		d.virtual = make(map[string]vState)
+	}
+	d.virtual[leaf] = vAdd
+	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vAdd, leaf)
 	d.mu.Unlock()
 }
 
+// AddVirtual adds a virtual object of name and size to the directory
+//
+// This will be replaced with a real object when it is read back from the
+// remote.
+//
+// This is used to add directory entries while things are uploading
+func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
+	var node Node
+	d.mu.RLock()
+	dPath := d.path
+	_, found := d.items[leaf]
+	d.mu.RUnlock()
+	if found {
+		// Don't overwrite existing objects
+		return
+	}
+	if isDir {
+		remote := path.Join(dPath, leaf)
+		entry := fs.NewDir(remote, time.Now())
+		node = newDir(d.vfs, d.f, d, entry)
+	} else {
+		f := newFile(d, dPath, nil, leaf)
+		f.setSize(size)
+		node = f
+	}
+	d.addObject(node)
+
+}
+
 // delObject removes an object from the directory
+//
+// The name passed in is marked as virtual as the delete it hasn't been read
+// from a remote directory listing.
 func (d *Dir) delObject(leaf string) {
 	d.mu.Lock()
 	delete(d.items, leaf)
+	if d.virtual == nil {
+		d.virtual = make(map[string]vState)
+	}
+	d.virtual[leaf] = vDel
+	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)
 	d.mu.Unlock()
 }
 
+// DelVirtual removes an object from the directory listing
+//
+// It marks it as removed until it has confirmed the object is missing when the
+// directory entries are re-read in which case the virtual mark is removed.
+//
+// This is used to remove directory entries after things have been deleted or
+// renamed but before we've had confirmation from the backend.
+func (d *Dir) DelVirtual(leaf string) {
+	d.delObject(leaf)
+}
+
 // read the directory and sets d.items - must be called with the lock held
 func (d *Dir) _readDir() error {
 	when := time.Now()
@@ -312,6 +386,21 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 		}
 		node := d.items[name]
 		found[name] = struct{}{}
+		virtualState := d.virtual[name]
+		switch virtualState {
+		case vAdd:
+			// item was added to the dir but since it is found in a
+			// listing is no longer virtual
+			delete(d.virtual, name)
+			if len(d.virtual) == 0 {
+				d.virtual = nil
+			}
+			fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
+		case vDel:
+			// item is deleted from the dir so skip it
+			continue
+		case vOK:
+		}
 		switch item := entry.(type) {
 		case fs.Object:
 			obj := item
@@ -349,10 +438,24 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 	}
 	// delete unused entries
 	for name := range d.items {
-		if _, ok := found[name]; !ok {
+		if _, ok := found[name]; !ok && d.virtual[name] != vAdd {
+			// item was added to the dir but wasn't found in the
+			// listing - remove it unless it was virtually added
 			delete(d.items, name)
 		}
 	}
+	// delete unused virtuals
+	for name, virtualState := range d.virtual {
+		if _, ok := found[name]; !ok && virtualState == vDel {
+			// We have a virtual delete but the item wasn't found in
+			// the listing so no longer needs a virtual delete.
+			delete(d.virtual, name)
+			fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
+		}
+	}
+	if len(d.virtual) == 0 {
+		d.virtual = nil
+	}
 	return nil
 }
 

vfs/dir_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -277,6 +278,45 @@ func TestDirReadDirAll(t *testing.T) {
 	dir = node.(*Dir)
 
 	checkListing(t, dir, []string{"file3,16,false"})
+
+	t.Run("Virtual", func(t *testing.T) {
+		node, err := vfs.Stat("dir")
+		require.NoError(t, err)
+		dir := node.(*Dir)
+
+		// Add some virtual entries and check what happens
+		dir.AddVirtual("virtualFile", 17, false)
+		dir.AddVirtual("virtualDir", 0, true)
+		// Remove some existing entries
+		dir.DelVirtual("file2")
+		dir.DelVirtual("subdir")
+
+		checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,17,false"})
+
+		// Force a directory reload...
+		dir.invalidateDir("dir")
+
+		checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,17,false"})
+
+		// Now action the deletes and uploads
+		_ = r.WriteObject(context.Background(), "dir/virtualFile", "virtualFile contents", t1)
+		_ = r.WriteObject(context.Background(), "dir/virtualDir/testFile", "testFile contents", t1)
+		o, err := r.Fremote.NewObject(context.Background(), "dir/file2")
+		require.NoError(t, err)
+		require.NoError(t, o.Remove(context.Background()))
+		require.NoError(t, operations.Purge(context.Background(), r.Fremote, "dir/subdir"))
+
+		// Force a directory reload...
+		dir.invalidateDir("dir")
+
+		checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,20,false"})
+
+		// check no virtuals left
+		dir.mu.Lock()
+		assert.Nil(t, dir.virtual)
+		dir.mu.Unlock()
+
+	})
 }
 
 func TestDirOpen(t *testing.T) {

vfs/vstate_string.go

@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=vState"; DO NOT EDIT.
+
+package vfs
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[vOK-0]
+	_ = x[vAdd-1]
+	_ = x[vDel-2]
+}
+
+const _vState_name = "vOKvAddvDel"
+
+var _vState_index = [...]uint8{0, 3, 7, 11}
+
+func (i vState) String() string {
+	if i >= vState(len(_vState_index)-1) {
+		return "vState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _vState_name[_vState_index[i]:_vState_index[i+1]]
+}