github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 13:26:11 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

local: always use UNC paths on Windows - fixes #124, fixes #130, fixes #90

Browse Source 
* Convert all paths to UNC paths on Windows.
  * Update local filesystem to always use UNC paths.
  * Change tests, so they can work with Windows character replacements.
  * Remove "/" suffix on paths.
  * Always use path/filepath
This commit is contained in:
klauspost 2015-09-11 11:37:12 +02:00 committed by Nick Craig-Wood
parent 00afe6cc96
commit f50f353b5d
4 changed files with 191 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
fstest/fstest.go
View File

@ -28,6 +28,7 @@ type Item struct {
Md5sum string
ModTime time.Time
Size int64
WinPath string
}
// Checks the times are equal within the precision, returns the delta and a flag
@ -67,19 +68,22 @@ func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
// Represents all items for checking
type Items struct {
byName map[string]*Item
items []Item
byName map[string]*Item
byNameAlt map[string]*Item
items []Item
}
// Make an Items
func NewItems(items []Item) *Items {
is := &Items{
byName: make(map[string]*Item),
items: items,
byName: make(map[string]*Item),
byNameAlt: make(map[string]*Item),
items: items,
}
// Fill up byName
for i := range items {
is.byName[items[i].Path] = &items[i]
is.byNameAlt[items[i].WinPath] = &items[i]
}
return is
}
@ -88,10 +92,14 @@ func NewItems(items []Item) *Items {
func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
i, ok := is.byName[obj.Remote()]
if !ok {
t.Errorf("Unexpected file %q", obj.Remote())
return
i, ok = is.byNameAlt[obj.Remote()]
if !ok {
t.Errorf("Unexpected file %q", obj.Remote())
return
}
}
delete(is.byName, obj.Remote())
delete(is.byName, i.Path)
delete(is.byName, i.WinPath)
i.Check(t, obj, precision)
}

15
fstest/fstests/fstests.go
View File

@ -34,6 +34,7 @@ var (
file2 = fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`,
WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ _ _/z.txt`,
}
)
@ -169,7 +170,7 @@ func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t)
found := false
for obj := range remote.ListDir() {
if obj.Name != `hello? sausage` {
if obj.Name != `hello? sausage` && obj.Name != `hello_ sausage` {
t.Errorf("Found unexpected item %q", obj.Name)
} else {
found = true
@ -205,17 +206,18 @@ func TestFsListRoot(t *testing.T) {
}
// Should either find file1 and file2 or nothing
found1 := false
file1 := subRemoteLeaf + "/" + file1.Path
f1 := subRemoteLeaf + "/" + file1.Path
found2 := false
file2 := subRemoteLeaf + "/" + file2.Path
f2 := subRemoteLeaf + "/" + file2.Path
f2Alt := subRemoteLeaf + "/" + file2.WinPath
count := 0
errors := fs.Stats.GetErrors()
for obj := range rootRemote.List() {
count++
if obj.Remote() == file1 {
if obj.Remote() == f1 {
found1 = true
}
if obj.Remote() == file2 {
if obj.Remote() == f2 || obj.Remote() == f2Alt {
found2 = true
}
}
@ -232,7 +234,7 @@ func TestFsListRoot(t *testing.T) {
}
return
}
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", file1, found1, file2, found2, count)
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count)
}
func TestFsListFile1(t *testing.T) {
@ -360,6 +362,7 @@ func TestFsDirMove(t *testing.T) {
}
// check remotes
// FIXME: Prints errors.
fstest.CheckListing(t, remote, []fstest.Item{})
fstest.CheckListing(t, newRemote, []fstest.Item{file2, file1})

136
local/local.go
View File

@ -1,11 +1,6 @@
// Local filesystem interface
package local
// Note that all rclone paths should be / separated. Anything coming
// from the filepath module will have \ separators on windows so
// should be converted using filepath.ToSlash. Windows is quite happy
// with / separators so there is no need to convert them back.
import (
"crypto/md5"
"encoding/hex"
@ -14,13 +9,15 @@ import (
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"sync"
"time"
"unicode/utf8"
"github.com/ncw/rclone/fs"
"regexp"
"runtime"
"strings"
)
// Register with Fs
@ -51,20 +48,22 @@ type FsObjectLocal struct {
// ------------------------------------------------------------
// NewFs contstructs an FsLocal from the path
// NewFs constructs an FsLocal from the path
func NewFs(name, root string) (fs.Fs, error) {
root = filepath.ToSlash(path.Clean(root))
var err error
f := &FsLocal{
name: name,
root: root,
warned: make(map[string]struct{}),
}
f.root = filterPath(f.cleanUtf8(root))
// Check to see if this points to a file
fi, err := os.Lstat(f.root)
if err == nil && fi.Mode().IsRegular() {
// It is a file, so use the parent as the root
remote := path.Base(root)
f.root = path.Dir(root)
var remote string
f.root, remote = getDirFile(f.root)
obj := f.NewFsObject(remote)
// return a Fs Limited to this object
return fs.NewLimited(f, obj), nil
@ -90,7 +89,7 @@ func (f *FsLocal) String() string {
// newFsObject makes a half completed FsObjectLocal
func (f *FsLocal) newFsObject(remote string) *FsObjectLocal {
remote = filepath.ToSlash(remote)
dstPath := path.Join(f.root, remote)
dstPath := filterPath(filepath.Join(f.root, f.cleanUtf8(remote)))
return &FsObjectLocal{local: f, remote: remote, path: dstPath}
}
@ -160,14 +159,40 @@ func (f *FsLocal) List() fs.ObjectsChan {
//
// Any invalid UTF-8 characters will be replaced with utf8.RuneError
func (f *FsLocal) cleanUtf8(name string) string {
if utf8.ValidString(name) {
return name
if !utf8.ValidString(name) {
if _, ok := f.warned[name]; !ok {
fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
f.warned[name] = struct{}{}
}
name = string([]rune(name))
}
if _, ok := f.warned[name]; !ok {
fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
f.warned[name] = struct{}{}
if runtime.GOOS == "windows" {
var name2 string
if strings.HasPrefix(name, `\\?\`) {
name2 = `\\?\`
strings.TrimPrefix(name, `\\?\`)
}
if strings.HasPrefix(name, `//?/`) {
name2 = `//?/`
strings.TrimPrefix(name, `//?/`)
}
name2 += strings.Map(func(r rune) rune {
switch r {
case '<', '>', '"', '|', '?', '*', '&':
return '_'
}
return r
}, name)
if name2 != name {
if _, ok := f.warned[name]; !ok {
fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
f.warned[name] = struct{}{}
}
name = name2
}
}
return string([]rune(name))
return name
}
// Walk the path returning a channel of FsObjects
@ -189,7 +214,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
Count: 0,
}
// Go down the tree to count the files and directories
dirpath := path.Join(f.root, item.Name())
dirpath := filterPath(filepath.Join(f.root, item.Name()))
err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
if err != nil {
fs.Stats.Error()
@ -226,6 +251,7 @@ func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64
// Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error {
// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
return os.MkdirAll(f.root, 0777)
}
@ -376,11 +402,22 @@ func (dstFs *FsLocal) DirMove(src fs.Fs) error {
fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
// Check if source exists
sstat, err := os.Lstat(srcFs.root)
if err != nil {
return err
}
// And is a directory
if !sstat.IsDir() {
return fs.ErrorCantDirMove
}
// Check if destination exists
_, err := os.Lstat(dstFs.root)
_, err = os.Lstat(dstFs.root)
if !os.IsNotExist(err) {
return fs.ErrorDirExists
}
// Do the move
return os.Rename(srcFs.root, dstFs.root)
}
@ -517,7 +554,7 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
// mkdirAll makes all the directories needed to store the object
func (o *FsObjectLocal) mkdirAll() error {
dir := path.Dir(o.path)
dir, _ := getDirFile(o.path)
return os.MkdirAll(dir, 0777)
}
@ -568,6 +605,63 @@ func (o *FsObjectLocal) Remove() error {
return os.Remove(o.path)
}
// Return the current directory and file from a path
// Assumes os.PathSeparator is used.
func getDirFile(s string) (string, string) {
i := strings.LastIndex(s, string(os.PathSeparator))
return s[:i], s[i+1:]
}
func filterPath(s string) string {
s = filepath.Clean(s)
if runtime.GOOS == "windows" {
s = strings.Replace(s, `/`, `\`, -1)
if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
s2, err := filepath.Abs(s)
if err == nil {
s = s2
}
}
// Convert to UNC
return uncPath(s)
}
if !filepath.IsAbs(s) {
s2, err := filepath.Abs(s)
if err == nil {
s = s2
}
}
return s
}
// Pattern to match a windows absolute path: c:\temp path.
var isAbsWinDrive = regexp.MustCompile(`[a-zA-Z]\:\\`)
// uncPath converts an absolute Windows path
// to a UNC long path.
func uncPath(s string) string {
// UNC can NOT use "/", so convert all to "\"
s = strings.Replace(s, `/`, `\`, -1)
// If prefix is "\\", we already have a UNC path or server.
if strings.HasPrefix(s, `\\`) {
// If already long path, just keep it
if strings.HasPrefix(s, `\\?\`) {
return s
}
// Trim "//" from path and add UNC prefix.
return `\\?\UNC\` + strings.TrimPrefix(s, `\\`)
}
if isAbsWinDrive.Match([]byte(s)) {
return `\\?\` + s
}
return s
}
// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.Purger = &FsLocal{}

52
local/tests_test.go Normal file
View File

@ -0,0 +1,52 @@
package local
import (
"testing"
)
var uncTestPaths = []string{
"C:\\Ba*d\\P|a?t<h>\\Windows\\Folder",
"C:/Ba*d/P|a?t<h>/Windows\\Folder",
"C:\\Windows\\Folder",
"\\\\?\\C:\\Windows\\Folder",
"//?/C:/Windows/Folder",
"\\\\?\\UNC\\server\\share\\Desktop",
"\\\\?\\unC\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
"\\\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
"C:\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
"C:\\AbsoluteToRoot\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
"\\\\server\\share\\Desktop",
"\\\\?\\UNC\\\\share\\folder\\Desktop",
"\\\\server\\share",
}
var uncTestPathsResults = []string{
`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
`\\?\C:\Windows\Folder`,
`\\?\C:\Windows\Folder`,
`\\?\C:\Windows\Folder`,
`\\?\UNC\server\share\Desktop`,
`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
`\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
`\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
`\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
`\\?\UNC\server\share\Desktop`,
`\\?\UNC\\share\folder\Desktop`,
`\\?\UNC\server\share`,
}
// Test that UNC paths are converted.
func TestUncPaths(t *testing.T) {
for i, p := range uncTestPaths {
unc := uncPath(p)
if unc != uncTestPathsResults[i] {
t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
}
// Test we don't add more.
unc = uncPath(unc)
if unc != uncTestPathsResults[i] {
t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
}
}
}