vfs: add the vfs/refresh rc command

vfs/refresh will walk the directory tree for the given paths and
freshen the directory cache. It will use the fast-list capability
of the remote when enabled.

vfs/dir.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/list"
+	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
 )
 
@@ -187,6 +188,21 @@ func (d *Dir) _readDir() error {
 		return err
 	}
 
+	err = d._readDirFromEntries(entries, nil, time.Time{})
+	if err != nil {
+		return err
+	}
+
+	d.read = when
+	return nil
+}
+
+func (d *Dir) _readDirFromDirTree(dirTree walk.DirTree, when time.Time) error {
+	return d._readDirFromEntries(dirTree[d.path], dirTree, when)
+}
+
+func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree walk.DirTree, when time.Time) error {
+	var err error
 	// Cache the items by name
 	found := make(map[string]struct{})
 	for _, entry := range entries {
@@ -206,10 +222,23 @@ func (d *Dir) _readDir() error {
 				node = newFile(d, obj, name)
 			}
 		case fs.Directory:
-			dir := item
 			// Reuse old dir value if it exists
 			if node == nil || !node.IsDir() {
-				node = newDir(d.vfs, d.f, d, dir)
+				node = newDir(d.vfs, d.f, d, item)
+			}
+			if dirTree != nil {
+				dir := node.(*Dir)
+				dir.mu.Lock()
+				err = dir._readDirFromDirTree(dirTree, when)
+				if err != nil {
+					dir.read = time.Time{}
+				} else {
+					dir.read = when
+				}
+				dir.mu.Unlock()
+				if err != nil {
+					return err
+				}
 			}
 		default:
 			err = errors.Errorf("unknown type %T", item)
@@ -224,6 +253,24 @@ func (d *Dir) _readDir() error {
 			delete(d.items, name)
 		}
 	}
+	return nil
+}
+
+func (d *Dir) readDirTree() error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	when := time.Now()
+	d.read = time.Time{}
+	fs.Debugf(d.path, "Reading directory tree")
+	dt, err := walk.NewDirTree(d.f, d.path, false, -1)
+	if err != nil {
+		return err
+	}
+	err = d._readDirFromDirTree(dt, when)
+	if err != nil {
+		return err
+	}
+	fs.Debugf(d.path, "Reading directory tree done in %s", time.Since(when))
 	d.read = when
 	return nil
 }

vfs/rc.go

@@ -59,6 +59,86 @@ starting with dir will forget that dir, eg
 
     rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
 
+`,
+	})
+	rc.Add(rc.Call{
+		Path: "vfs/refresh",
+		Fn: func(in rc.Params) (out rc.Params, err error) {
+			root, err := vfs.Root()
+			if err != nil {
+				return nil, err
+			}
+			getDir := func(path string) (*Dir, error) {
+				path = strings.Trim(path, "/")
+				segments := strings.Split(path, "/")
+				var node Node = root
+				for _, s := range segments {
+					if dir, ok := node.(*Dir); ok {
+						node, err = dir.stat(s)
+						if err != nil {
+							return nil, err
+						}
+					}
+				}
+				if dir, ok := node.(*Dir); ok {
+					return dir, nil
+				}
+				return nil, EINVAL
+			}
+
+			result := map[string]string{}
+			if len(in) == 0 {
+				err = root.readDirTree()
+				if err != nil {
+					result[""] = err.Error()
+				} else {
+					result[""] = "OK"
+				}
+			} else {
+				for k, v := range in {
+					path, ok := v.(string)
+					if !ok {
+						return out, errors.Errorf("value must be string %q=%v", k, v)
+					}
+					if strings.HasPrefix(k, "dir") {
+						dir, err := getDir(path)
+						if err != nil {
+							result[path] = err.Error()
+						} else {
+							err = dir.readDirTree()
+							if err != nil {
+								result[path] = err.Error()
+							} else {
+								result[path] = "OK"
+							}
+
+						}
+					} else {
+						return out, errors.Errorf("unknown key %q", k)
+					}
+				}
+			}
+			out = rc.Params{
+				"result": result,
+			}
+			return out, nil
+		},
+		Title: "Refresh the directory cache tree.",
+		Help: `
+This reads the full directory tree for the paths and freshens the
+directory cache.
+
+If no paths are passed in then it will refresh the root directory.
+
+    rclone rc vfs/refresh
+
+Otherwise pass directories in as dir=path. Any parameter key
+starting with dir will refresh that directory, eg
+
+    rclone rc vfs/refresh dir=home/junk dir2=data/misc
+
+This refresh will use --fast-list if enabled.
+
 `,
 	})
 }