diff --git a/vfs/dir.go b/vfs/dir.go index 26b706b62..a76d00cb1 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -459,7 +459,8 @@ func (d *Dir) addObject(node Node) { // This will be replaced with a real object when it is read back from the // remote. // -// This is used to add directory entries while things are uploading +// This is used by the vfs cache to insert objects that are uploading +// into the directory tree. func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) { var node Node d.mu.RLock() @@ -475,7 +476,16 @@ func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) { entry := fs.NewDir(remote, time.Now()) node = newDir(d.vfs, d.f, d, entry) } else { + isLink := false + if d.vfs.Opt.Links { + // since the path came from the cache it may have fs.LinkSuffix, + // so remove it and mark the *File accordingly + leaf, isLink = strings.CutSuffix(leaf, fs.LinkSuffix) + } f := newFile(d, dPath, nil, leaf) + if isLink { + f.setSymlink() + } f.setSize(size) node = f } @@ -628,7 +638,7 @@ func (d *Dir) _purgeVirtual() { // if writing in progress then leave virtual continue } - if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.Path()) { + if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.CachePath()) { // if object in use or dirty then leave virtual continue } @@ -718,6 +728,9 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree if name == "." || name == ".." { continue } + if d.vfs.Opt.Links { + name, _ = strings.CutSuffix(name, fs.LinkSuffix) + } node := d.items[name] if mv.add(d, name) { continue diff --git a/vfs/file.go b/vfs/file.go index 1c599ef01..b94e3dbb4 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "io" "os" "path" + "strings" "sync" "sync/atomic" "time" @@ -34,7 +36,7 @@ import ( // // File may **not** read any members of Dir directly. -// File represents a file +// File represents a file or a symlink type File struct { inode uint64 // inode number - read only size atomic.Int64 // size of file @@ -53,6 +55,7 @@ type File struct { sys atomic.Value // user defined info to be attached here nwriters atomic.Int32 // len(writers) appendMode bool // file was opened with O_APPEND + isLink bool // file represents a symlink } // newFile creates a new File @@ -69,9 +72,18 @@ func newFile(d *Dir, dPath string, o fs.Object, leaf string) *File { if o != nil { f.size.Store(o.Size()) } + f._setIsLink() return f } +// Set whether this is a link or not based on f.o +func (f *File) _setIsLink() { + if f.o == nil { + return + } + f.isLink = f.d.vfs.Opt.Links && strings.HasSuffix(f.o.Remote(), fs.LinkSuffix) +} + // String converts it to printable func (f *File) String() string { if f == nil { @@ -90,13 +102,31 @@ func (f *File) IsDir() bool { return false } +// IsSymlink returns true for symlinks when --links is enabled +func (f *File) IsSymlink() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.isLink +} + +// setSymlink marks this File as being a symlink +func (f *File) setSymlink() { + f.mu.RLock() + f.isLink = true + f.mu.RUnlock() +} + // Mode bits of the file or directory - satisfies Node interface func (f *File) Mode() (mode os.FileMode) { f.mu.RLock() defer f.mu.RUnlock() - mode = os.FileMode(f.d.vfs.Opt.FilePerms) - if f.appendMode { - mode |= os.ModeAppend + if f.isLink { + mode = os.FileMode(f.d.vfs.Opt.LinkPerms) + } else { + mode = os.FileMode(f.d.vfs.Opt.FilePerms) + if f.appendMode { + mode |= os.ModeAppend + } } return mode } @@ -122,6 +152,34 @@ func (f *File) Path() string { return path.Join(dPath, leaf) } +// _fixCachePath returns fullPath with the fs.LinkSuffix added if appropriate +// use when lock is held +func (f *File) _fixCachePath(fullPath string) string { + if !f.isLink { + return fullPath + } + return fullPath + fs.LinkSuffix +} + +// _cachePath returns the full path of the file with the fs.LinkSuffix if appropriate +// use when lock is held +func (f *File) _cachePath() string { + dPath, leaf := f.dPath, f.leaf + if f.isLink { + leaf += fs.LinkSuffix + } + return path.Join(dPath, leaf) +} + +// CachePath returns the full path of the file with the fs.LinkSuffix if appropriate +// +// We use this path when storing files in the cache. +func (f *File) CachePath() string { + f.mu.RLock() + defer f.mu.RUnlock() + return f._cachePath() +} + // Sys returns underlying data source (can be nil) - satisfies Node interface func (f *File) Sys() interface{} { return f.sys.Load() @@ -172,6 +230,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error { f.mu.RLock() d := f.d oldPendingRenameFun := f.pendingRenameFun + oldPath := f._cachePath() + newCacheName := f._fixCachePath(newName) f.mu.RUnlock() if features := d.Fs().Features(); features.Move == nil && features.Copy == nil { @@ -180,9 +240,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error { return err } - oldPath := f.Path() // File.mu is unlocked here to call Dir.Path() - newPath := path.Join(destDir.Path(), newName) + newPath := path.Join(destDir.Path(), newCacheName) renameCall := func(ctx context.Context) (err error) { // chain rename calls if any @@ -231,6 +290,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error { f.mu.Lock() if newObject != nil { f.o = newObject + f._setIsLink() } f.pendingRenameFun = nil f.mu.Unlock() @@ -334,7 +394,7 @@ func (f *File) ModTime() (modTime time.Time) { } // Read the modtime from a dirty item if it exists if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal { - if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil { + if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil { modTime, err := item.GetModTime() if err != nil { fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err) @@ -371,7 +431,7 @@ func (f *File) Size() int64 { // Read the size from a dirty item if it exists if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal { - if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil { + if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil { size, err := item.GetSize() if err != nil { fs.Errorf(f._path(), "Size: Item GetSize failed: %v", err) @@ -404,8 +464,8 @@ func (f *File) SetModTime(modTime time.Time) error { f.pendingModTime = modTime // set the time of the file in the cache - if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) { - f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime) + if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._cachePath()) { + f.d.vfs.cache.SetModTime(f._cachePath(), f.pendingModTime) } // Only update the ModTime when there are no writers, setObject will do it @@ -480,6 +540,7 @@ func (f *File) setSize(n int64) { func (f *File) setObject(o fs.Object) { f.mu.Lock() f.o = o + f._setIsLink() _ = f._applyPendingModTime() d := f.d f.mu.Unlock() @@ -493,6 +554,7 @@ func (f *File) setObject(o fs.Object) { func (f *File) setObjectNoUpdate(o fs.Object) { f.mu.Lock() f.o = o + f._setIsLink() f.virtualModTime = nil fs.Debugf(f._path(), "Reset virtual modtime") f.mu.Unlock() @@ -611,8 +673,8 @@ func (f *File) Remove() (err error) { // Remove the object from the cache wasWriting := false - if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) { - wasWriting = d.vfs.cache.Remove(f.Path()) + if d.vfs.cache != nil && d.vfs.cache.Exists(f.CachePath()) { + wasWriting = d.vfs.cache.Remove(f.CachePath()) } f.muRW.Lock() // muRW must be locked before mu to avoid @@ -673,6 +735,85 @@ func (f *File) Fs() fs.Fs { return f.d.Fs() } +// MaxSymlinkIterations is the largest number of symlink evaluations EvalSymlinks will do. +const MaxSymlinkIterations = 32 + +// If f is a symlink then it resolves it to a new Node. +// +// This is a simplistic symlink resolver - it only resolves direct +// symlinks, it will **not** resolve paths that point into a directory +// via a symlink. +// +// It returns the target node after the evaluation of all symbolic +// links. +// +// It returns an error if too many symlinks need to be resolved +// (ELOOP) or there is a loop. +func (f *File) resolveNode() (target Node, err error) { + defer log.Trace(f.Path(), "")("target=%v, err=%v", &target, &err) + seen := make(map[string]struct{}) + for tries := 0; tries < MaxSymlinkIterations; tries++ { + // If f isn't a symlink, we've arrived at the target + if !f.IsSymlink() { + return f, nil + } + + // Read the symlink + fd, err := f.Open(os.O_RDONLY | o_SYMLINK) + if err != nil { + return nil, err + } + b, err := io.ReadAll(fd) + closeErr := fd.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + targetPath := string(b) + + // Convert to a path relative to the root + // Symlinks are relative to their file node + if !path.IsAbs(targetPath) { + basePath := path.Dir(f.Path()) + targetPath = path.Join(basePath, targetPath) + } + + // Clean the path, rclone style + targetPath = path.Clean(targetPath) + if targetPath == "." { + targetPath = "" + } + + // Check if we've already seen this path + if _, ok := seen[targetPath]; ok { + return nil, ELOOP + } + seen[targetPath] = struct{}{} + + // Resolve the targetPath into a node + target, err = f.d.vfs.Stat(targetPath) + if err != nil { + return nil, err + } + + // Return node as it must be the destination if not a file + var ok bool + f, ok = target.(*File) + if !ok { + return target, nil + } + } + return nil, ELOOP +} + +// Open also also implements the internal flag o_SYMLINK which instead +// of opening the file a symlink points to, opens the symlink itself. +// This is used for reading and writing the symlink and shouldn't be +// used externally. +const o_SYMLINK = 0x4000_0000 //nolint:revive + // Open a file according to the flags provided // // O_RDONLY open the file read-only. @@ -694,6 +835,16 @@ func (f *File) Open(flags int) (fd Handle, err error) { rdwrMode = flags & accessModeMask ) + // If this is a symlink, then resolve it + if f.IsSymlink() && flags&o_SYMLINK == 0 { + target, err := f.resolveNode() + if err != nil { + return nil, err + } + return target.Open(flags) + } + flags &^= o_SYMLINK + // http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html // The result of using O_TRUNC with O_RDONLY is undefined. // Linux seems to truncate the file, but we prefer to return EINVAL @@ -738,7 +889,7 @@ func (f *File) Open(flags int) (fd Handle, err error) { d := f.d f.mu.RUnlock() CacheMode := d.vfs.Opt.CacheMode - if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.Path()) || d.vfs.cache.Exists(f.Path())) { + if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.CachePath()) || d.vfs.cache.Exists(f.CachePath())) { fd, err = f.openRW(flags) } else if read && write { if CacheMode >= vfscommon.CacheModeMinimal { diff --git a/vfs/read_write.go b/vfs/read_write.go index 4130e68b0..56b9c467d 100644 --- a/vfs/read_write.go +++ b/vfs/read_write.go @@ -43,7 +43,7 @@ func (fh *RWFileHandle) Unlock() error { func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) { defer log.Trace(f.Path(), "")("err=%v", &err) // get an item to represent this from the cache - item := d.vfs.cache.Item(f.Path()) + item := d.vfs.cache.Item(f.CachePath()) exists := f.exists() || (item.Exists() && !item.WrittenBack()) diff --git a/vfs/vfs.go b/vfs/vfs.go index 2e60c8fb6..aa6fe15eb 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -791,6 +791,9 @@ func (vfs *VFS) WriteFile(name string, data []byte, perm os.FileMode) (err error } // AddVirtual adds the object (file or dir) to the directory cache +// +// This is used by the vfs cache to insert objects that are uploading +// into the directory tree. func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) { remote = strings.TrimRight(remote, "/") var dir *Dir @@ -808,3 +811,85 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) { dir.AddVirtual(leaf, size, false) return nil } + +// Readlink returns the destination of the named symbolic link. +// If there is an error, it will be of type *PathError. +func (vfs *VFS) Readlink(name string) (s string, err error) { + if !vfs.Opt.Links { + fs.Errorf(nil, "symlinks not supported without the --links flag: %v", name) + return "", ENOSYS + } + node, err := vfs.Stat(name) + if err != nil { + return "", err + } + file, ok := node.(*File) + if !ok || !file.IsSymlink() { + return "", EINVAL // not a symlink + } + fd, err := file.Open(os.O_RDONLY | o_SYMLINK) + if err != nil { + return "", err + } + defer fs.CheckClose(fd, &err) + b, err := io.ReadAll(fd) + if err != nil { + return "", err + } + return string(b), nil +} + +// CreateSymlink creates newname as a symbolic link to oldname. +// On Windows, a symlink to a non-existent oldname creates a file symlink; +// if oldname is later created as a directory the symlink will not work. +// It returns the node created +func (vfs *VFS) CreateSymlink(oldname, newname string) (Node, error) { + if !vfs.Opt.Links { + fs.Errorf(newname, "symlinks not supported without the --links flag") + return nil, ENOSYS + } + + // Destination can't exist + _, err := vfs.Stat(newname) + if err == nil { + return nil, EEXIST + } else if err != ENOENT { + return nil, err + } + + // Find the parent + dir, leaf, err := vfs.StatParent(newname) + if err != nil { + return nil, err + } + + // Create the file node + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | o_SYMLINK + file, err := dir.Create(leaf, flags) + if err != nil { + return nil, err + } + + // Force the file to be a link + file.setSymlink() + + // Open the file + fh, err := file.Open(flags) + if err != nil { + return nil, err + } + defer fs.CheckClose(fh, &err) + + // Write the symlink data + _, err = fh.Write([]byte(strings.ReplaceAll(oldname, "\\", "/"))) + + return file, nil +} + +// Symlink creates newname as a symbolic link to oldname. +// On Windows, a symlink to a non-existent oldname creates a file symlink; +// if oldname is later created as a directory the symlink will not work. +func (vfs *VFS) Symlink(oldname, newname string) error { + _, err := vfs.CreateSymlink(oldname, newname) + return err +} diff --git a/vfs/write.go b/vfs/write.go index c799f98d5..8e74b45a8 100644 --- a/vfs/write.go +++ b/vfs/write.go @@ -37,6 +37,9 @@ var ( ) func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHandle, error) { + if f.IsSymlink() { + remote += fs.LinkSuffix + } fh := &WriteFileHandle{ remote: remote, flags: flags,