From 3e27ff1b9590f7356d9784fc89c6564ed8b8473e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 31 Jul 2014 21:24:52 +0100 Subject: [PATCH] Add Root List test and fs.Limited tests for single files --- drive/drive_test.go | 3 + dropbox/dropbox_test.go | 3 + fs/fs.go | 5 +- fstest/fstest.go | 61 ++++++++----- fstest/fstests/fstests.go | 85 +++++++++++++++++-- googlecloudstorage/googlecloudstorage_test.go | 3 + local/local_test.go | 3 + notes.txt | 16 ++-- s3/s3_test.go | 3 + swift/swift_test.go | 3 + 10 files changed, 147 insertions(+), 38 deletions(-) diff --git a/drive/drive_test.go b/drive/drive_test.go index d8ee07f3b..a9fda817a 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 99e816cfc..16d2cf000 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/fs/fs.go b/fs/fs.go index 489c4aeb7..30295db75 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -184,9 +184,10 @@ func Find(name string) (*FsInfo, error) { // NewFs makes a new Fs object from the path // -// The path is of the form service://path +// The path is of the form remote:path // -// Services are looked up in the config file +// Remotes are looked up in the config file. If the remote isn't +// found then NotFoundInConfigFile will be returned. func NewFs(path string) (Fs, error) { parts := matcher.FindStringSubmatch(path) fsName, configName, fsPath := "local", "local", path diff --git a/fstest/fstest.go b/fstest/fstest.go index c3c2e77cf..4d9f40d0d 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -125,29 +125,53 @@ func RandomString(n int) string { return string(out) } +// Creates a temporary directory name for local remotes +func LocalRemote() (path string, err error) { + path, err = ioutil.TempDir("", "rclone") + if err == nil { + // Now remove the directory + err = os.Remove(path) + } + return +} + +// Make a random bucket or subdirectory name +// +// Returns a random remote name plus the leaf name +func RandomRemoteName(remoteName string) (string, string, error) { + var err error + var leafName string + + // Make a directory if remote name is null + if remoteName == "" { + remoteName, err = LocalRemote() + if err != nil { + return "", "", err + } + } else { + if !strings.HasSuffix(remoteName, ":") { + remoteName += "/" + } + leafName = RandomString(32) + remoteName += leafName + } + return remoteName, leafName, nil +} + // Make a random bucket or subdirectory on the remote // // Call the finalise function returned to Purge the fs at the end (and // the parent if necessary) func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) { - // Make a directory if remote name is null - rmdir := "" var err error - if remoteName == "" { - remoteName, err = ioutil.TempDir("", "rclone") - if err != nil { - return nil, nil, err - } - rmdir = remoteName + var parentRemote fs.Fs + + remoteName, _, err = RandomRemoteName(remoteName) + if err != nil { + return nil, nil, err } - if !strings.HasSuffix(remoteName, ":") { - remoteName += "/" - } - remoteName += RandomString(32) - var parentRemote fs.Fs if subdir { - var err error parentRemote, err = fs.NewFs(remoteName) if err != nil { return nil, nil, err @@ -168,13 +192,6 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) { log.Printf("Failed to purge %v: %v", parentRemote, err) } } - // Delete directory if we made one above - if rmdir != "" { - err := os.RemoveAll(rmdir) - if err != nil { - Fatalf("Failed to remove %q: %v", rmdir, err) - } - } } return remote, finalise, nil @@ -191,7 +208,7 @@ func TestMkdir(remote fs.Fs) { func TestPurge(remote fs.Fs) { err := fs.Purge(remote) if err != nil { - log.Fatalf("Purge failed: %v", err) + Fatalf("Purge failed: %v", err) } CheckListing(remote, []Item{}) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 7e0968025..e58409ad6 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -7,8 +7,11 @@ import ( "bytes" "crypto/md5" "encoding/hex" + "fmt" "io" "log" + "os" + "strings" "testing" "time" @@ -17,11 +20,12 @@ import ( ) var ( - remote fs.Fs - RemoteName = "" - remoteFinalise func() - NilObject fs.Object - file1 = fstest.Item{ + remote fs.Fs + RemoteName = "" + subRemoteName = "" + subRemoteLeaf = "" + NilObject fs.Object + file1 = fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: "file name.txt", } @@ -32,11 +36,22 @@ var ( ) func TestInit(t *testing.T) { + var err error fs.LoadConfig() fs.Config.Verbose = false fs.Config.Quiet = true - var err error - remote, remoteFinalise, err = fstest.RandomRemote(RemoteName, false) + if RemoteName == "" { + RemoteName, err = fstest.LocalRemote() + if err != nil { + log.Fatalf("Failed to create tmp dir: %v", err) + } + } + subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName) + if err != nil { + t.Fatalf("Couldn't make remote name: %v", err) + } + + remote, err = fs.NewFs(subRemoteName) if err == fs.NotFoundInConfigFile { log.Printf("Didn't find %q in config file - skipping tests", RemoteName) return @@ -160,6 +175,24 @@ func TestFsListDirFile2(t *testing.T) { } } +func TestFsListDirRoot(t *testing.T) { + skipIfNotOk(t) + rootRemote, err := fs.NewFs(RemoteName) + if err != nil { + t.Fatal("Failed to make remote %q: %v", RemoteName, err) + } + found := false + for obj := range rootRemote.ListDir() { + fmt.Printf("obj = %q\n", obj.Name) + if obj.Name == subRemoteLeaf { + found = true + } + } + if !found { + t.Errorf("Didn't find %q", subRemoteLeaf) + } +} + func TestFsListFile1(t *testing.T) { skipIfNotOk(t) fstest.CheckListing(remote, []fstest.Item{file1, file2}) @@ -312,6 +345,36 @@ func TestObjectStorable(t *testing.T) { } } +func TestLimitedFs(t *testing.T) { + skipIfNotOk(t) + remoteName := subRemoteName + "/" + file2.Path + file2Copy := file2 + file2Copy.Path = "z.txt" + fileRemote, err := fs.NewFs(remoteName) + if err != nil { + t.Fatal("Failed to make remote %q: %v", remoteName, err) + } + fstest.CheckListing(fileRemote, []fstest.Item{file2Copy}) + _, ok := fileRemote.(*fs.Limited) + if !ok { + t.Errorf("%v is not a fs.Limited", fileRemote) + } +} + +func TestLimitedFsNotFound(t *testing.T) { + skipIfNotOk(t) + remoteName := subRemoteName + "/not found.txt" + fileRemote, err := fs.NewFs(remoteName) + if err != nil { + t.Fatal("Failed to make remote %q: %v", remoteName, err) + } + fstest.CheckListing(fileRemote, []fstest.Item{}) + _, ok := fileRemote.(*fs.Limited) + if ok { + t.Errorf("%v is is a fs.Limited", fileRemote) + } +} + func TestObjectRemove(t *testing.T) { skipIfNotOk(t) obj := findObject(t, file1.Path) @@ -333,7 +396,11 @@ func TestObjectPurge(t *testing.T) { func TestFinalise(t *testing.T) { skipIfNotOk(t) - if remoteFinalise != nil { - remoteFinalise() + if strings.HasPrefix(RemoteName, "/") { + // Remove temp directory + err := os.Remove(RemoteName) + if err != nil { + log.Printf("Failed to remove %q: %v\n", RemoteName, err) + } } } diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index 31487840a..135e379e9 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/local/local_test.go b/local/local_test.go index 8f67161e5..3abe16ff9 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/notes.txt b/notes.txt index 2159e3c26..7d8bf1524 100644 --- a/notes.txt +++ b/notes.txt @@ -1,11 +1,17 @@ Change lsd command so it doesn't show -1 * Make sure all Fses show -1 for objects Zero for dates etc + * Make test? -Make unit tests for the fses - * Make them try to load a TestFSName config - * Then do a proper unit test of each one - * If there is no TestFSName config it should exit quietly - * However the Local FS should default to a tmpdir +Make a subdir test on an existing file (file2 say) (List and ListDir) + +Put the TestRemote names into the Fs description +Make rclonetest use the TestRemote name automatically +Put rclonetest back into rclone as tests + +Run errcheck and go vet in the make file +.. Also race detector? + +Write developer manual Todo * FIXME: More -dry-run checks for object transfer diff --git a/s3/s3_test.go b/s3/s3_test.go index f63e794ff..a5196f17f 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/swift/swift_test.go b/swift/swift_test.go index d09f0bf46..43057282d 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -29,6 +29,7 @@ func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } @@ -44,6 +45,8 @@ func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }