From a5b0d8860859e3ea8d8490024ed4e18d6b0d9b3e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 24 Jul 2014 22:50:11 +0100 Subject: [PATCH] Make tests for each Fs Factor tests out of rclonetest --- drive/drive_test.go | 46 +++ dropbox/dropbox_test.go | 46 +++ fs/fs.go | 4 + fstest/fstest.go | 192 ++++++++++ fstest/fstests/fstests.go | 338 ++++++++++++++++++ googlecloudstorage/googlecloudstorage_test.go | 46 +++ local/local_test.go | 46 +++ rclonetest/rclonetest.go | 216 ++--------- rclonetest/test.sh | 10 +- s3/s3_test.go | 46 +++ swift/swift_test.go | 46 +++ 11 files changed, 851 insertions(+), 185 deletions(-) create mode 100644 drive/drive_test.go create mode 100644 dropbox/dropbox_test.go create mode 100644 fstest/fstest.go create mode 100644 fstest/fstests/fstests.go create mode 100644 googlecloudstorage/googlecloudstorage_test.go create mode 100644 local/local_test.go create mode 100644 s3/s3_test.go create mode 100644 swift/swift_test.go diff --git a/drive/drive_test.go b/drive/drive_test.go new file mode 100644 index 000000000..803c6cf4c --- /dev/null +++ b/drive/drive_test.go @@ -0,0 +1,46 @@ +// Test Drive filesystem interface +package drive_test + +import ( + "testing" + + "github.com/ncw/rclone/drive" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +func init() { + fstests.NilObject = fs.Object((*drive.FsObjectDrive)(nil)) + fstests.RemoteName = "TestDrive:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go new file mode 100644 index 000000000..9ba2d08d7 --- /dev/null +++ b/dropbox/dropbox_test.go @@ -0,0 +1,46 @@ +// Test Dropbox filesystem interface +package dropbox_test + +import ( + "testing" + + "github.com/ncw/rclone/dropbox" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" +) + +func init() { + fstests.NilObject = fs.Object((*dropbox.FsObjectDropbox)(nil)) + fstests.RemoteName = "TestDropbox:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/fs/fs.go b/fs/fs.go index 7c5fd839e..b53507f3d 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -79,9 +79,13 @@ type Fs interface { Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) // Make the directory (container, bucket) + // + // Shouldn't return an error if it already exists Mkdir() error // Remove the directory (container, bucket) if empty + // + // Return an error if it doesn't exists or isn't empty Rmdir() error // Precision of the ModTimes in this Fs diff --git a/fstest/fstest.go b/fstest/fstest.go new file mode 100644 index 000000000..7ffbcb4fd --- /dev/null +++ b/fstest/fstest.go @@ -0,0 +1,192 @@ +// Utilities for testing the fs + +package fstest + +// FIXME put name of test FS in Fs structure + +import ( + "io/ioutil" + "log" + "math/rand" + "os" + "strings" + "time" + + "github.com/ncw/rclone/fs" +) + +var Fatalf = log.Fatalf + +// Represents an item for checking +type Item struct { + Path string + Md5sum string + ModTime time.Time + Size int64 +} + +func (i *Item) Check(obj fs.Object) { + if obj == nil { + Fatalf("Object is nil") + } + // Check attributes + Md5sum, err := obj.Md5sum() + if err != nil { + Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err) + } + if i.Md5sum != Md5sum { + Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum) + } + if i.Size != obj.Size() { + Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size()) + } + // check the mod time to the given precision + modTime := obj.ModTime() + dt := modTime.Sub(i.ModTime) + if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow { + Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime) + } + +} + +// Represents all items for checking +type Items struct { + byName map[string]*Item + items []Item +} + +// Make an Items +func NewItems(items []Item) *Items { + is := &Items{ + byName: make(map[string]*Item), + items: items, + } + // Fill up byName + for i := range items { + is.byName[items[i].Path] = &items[i] + } + return is +} + +// Check off an item +func (is *Items) Find(obj fs.Object) { + i, ok := is.byName[obj.Remote()] + if !ok { + Fatalf("Unexpected file %q", obj.Remote()) + } + delete(is.byName, obj.Remote()) + i.Check(obj) +} + +// Check all done +func (is *Items) Done() { + if len(is.byName) != 0 { + for name := range is.byName { + log.Printf("Not found %q", name) + } + Fatalf("%d objects not found", len(is.byName)) + } +} + +// Checks the fs to see if it has the expected contents +func CheckListing(f fs.Fs, items []Item) { + is := NewItems(items) + for obj := range f.List() { + is.Find(obj) + } + is.Done() +} + +// Parse a time string or explode +func Time(timeString string) time.Time { + t, err := time.Parse(time.RFC3339Nano, timeString) + if err != nil { + Fatalf("Failed to parse time %q: %v", timeString, err) + } + return t +} + +// Create a random string +func RandomString(n int) string { + source := "abcdefghijklmnopqrstuvwxyz0123456789" + out := make([]byte, n) + for i := range out { + out[i] = source[rand.Intn(len(source))] + } + return string(out) +} + +// Make a random bucket or subdirectory on the remote +// +// Call the finalise function returned to Purge the fs at the end (and +// the parent if necessary) +func RandomRemote(remoteName string, subdir bool) (fs.Fs, func()) { + // Make a directory if remote name is null + rmdir := "" + var err error + if remoteName == "" { + remoteName, err = ioutil.TempDir("", "rclone") + if err != nil { + Fatalf("Failed to create temp dir: %v", err) + } + rmdir = remoteName + } + + if !strings.HasSuffix(remoteName, ":") { + remoteName += "/" + } + remoteName += RandomString(32) + var parentRemote fs.Fs + if subdir { + var err error + parentRemote, err = fs.NewFs(remoteName) + if err != nil { + log.Fatalf("Failed to make parent %q: %v", remoteName, err) + } + remoteName += "/" + RandomString(8) + } + + remote, err := fs.NewFs(remoteName) + if err != nil { + log.Fatalf("Failed to make %q: %v", remoteName, err) + } + + finalise := func() { + TestPurge(remote) + if parentRemote != nil { + TestPurge(parentRemote) + } + // Delete directory if we made one above + if rmdir != "" { + err := os.RemoveAll(rmdir) + if err != nil { + Fatalf("Failed to remove %q: %v", rmdir, err) + } + } + } + + return remote, finalise +} + +func TestMkdir(remote fs.Fs) { + err := fs.Mkdir(remote) + if err != nil { + Fatalf("Mkdir failed: %v", err) + } + CheckListing(remote, []Item{}) +} + +func TestPurge(remote fs.Fs) { + err := fs.Purge(remote) + if err != nil { + log.Fatalf("Purge failed: %v", err) + } + CheckListing(remote, []Item{}) +} + +func TestRmdir(remote fs.Fs) { + err := fs.Rmdir(remote) + if err != nil { + Fatalf("Rmdir failed: %v", err) + } +} diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go new file mode 100644 index 000000000..ad7ba2282 --- /dev/null +++ b/fstest/fstests/fstests.go @@ -0,0 +1,338 @@ +// Generic tests for testing the Fs and Object interfaces +package fstests + +// FIXME need to check the limited file system + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "io" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest" +) + +var ( + remote fs.Fs + RemoteName = "" + remoteFinalise func() + NilObject fs.Object + file1 = fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "file name.txt", + } + file2 = fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), + Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, + } +) + +func TestInit(t *testing.T) { + fs.LoadConfig() + fs.Config.Verbose = false + fs.Config.Quiet = true + remote, remoteFinalise = fstest.RandomRemote(RemoteName, false) + // if err != nil { + // if strings.Contains(err.Error(), "Didn't find section in config file") { + // return + // } + // t.Fatalf("Couldn't start FS: %v", err) + // } + fstest.Fatalf = t.Fatalf + fstest.TestMkdir(remote) +} + +func skipIfNotOk(t *testing.T) { + fstest.Fatalf = t.Fatalf + if remote == nil { + t.Skip("FS not configured") + } +} + +// String returns a description of the FS + +func TestFsString(t *testing.T) { + skipIfNotOk(t) + str := remote.String() + if str == "" { + t.Fatal("Bad fs.String()") + } +} + +type TestFile struct { + ModTime time.Time + Path string + Size int64 + Md5sum string +} + +func TestFsRmdirEmpty(t *testing.T) { + skipIfNotOk(t) + fstest.TestRmdir(remote) +} + +func TestFsRmdirNotFound(t *testing.T) { + skipIfNotOk(t) + err := remote.Rmdir() + if err == nil { + t.Fatalf("Expecting error on Rmdir non existent") + } +} + +func TestFsMkdir(t *testing.T) { + skipIfNotOk(t) + fstest.TestMkdir(remote) + fstest.TestMkdir(remote) +} + +func TestFsListEmpty(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(remote, []fstest.Item{}) +} + +func TestFsListDirEmpty(t *testing.T) { + skipIfNotOk(t) + for obj := range remote.ListDir() { + t.Error("Found unexpected item %q", obj.Name) + } +} + +func TestFsNewFsObjectNotFound(t *testing.T) { + skipIfNotOk(t) + if remote.NewFsObject("potato") != nil { + t.Fatal("Didn't expect to find object") + } +} + +func findObject(t *testing.T, Name string) fs.Object { + obj := remote.NewFsObject(Name) + if obj == nil { + t.Fatalf("nil object") + } + return obj +} + +func testPut(t *testing.T, file *fstest.Item) { + buf := bytes.NewBufferString(fstest.RandomString(100)) + hash := md5.New() + in := io.TeeReader(buf, hash) + + file.Size = int64(buf.Len()) + obj, err := remote.Put(in, file.Path, file.ModTime, file.Size) + if err != nil { + t.Fatal("Put error", err) + } + file.Md5sum = hex.EncodeToString(hash.Sum(nil)) + file.Check(obj) + // Re-read the object and check again + obj = findObject(t, file.Path) + file.Check(obj) +} + +func TestFsPutFile1(t *testing.T) { + skipIfNotOk(t) + testPut(t, &file1) +} + +func TestFsPutFile2(t *testing.T) { + skipIfNotOk(t) + testPut(t, &file2) +} + +func TestFsListDirFile2(t *testing.T) { + skipIfNotOk(t) + found := false + for obj := range remote.ListDir() { + if obj.Name != `hello? sausage` { + t.Errorf("Found unexpected item %q", obj.Name) + } else { + found = true + } + } + if !found { + t.Errorf("Didn't find %q", `hello? sausage`) + } +} + +func TestFsListFile1(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(remote, []fstest.Item{file1, file2}) +} + +func TestFsNewFsObject(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + file1.Check(obj) +} + +func TestFsListFile1and2(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(remote, []fstest.Item{file1, file2}) +} + +func TestFsRmdirFull(t *testing.T) { + skipIfNotOk(t) + err := remote.Rmdir() + if err == nil { + t.Fatalf("Expecting error on RMdir on non empty remote") + } +} + +func TestFsPrecision(t *testing.T) { + skipIfNotOk(t) + precision := remote.Precision() + if precision > time.Second || precision < 0 { + t.Fatalf("Precision out of range %v", precision) + } + // FIXME check expected precision +} + +func TestObjectString(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + s := obj.String() + if s != file1.Path { + t.Errorf("String() wrong %v != %v", s, file1.Path) + } + obj = NilObject + s = obj.String() + if s != "" { + t.Errorf("String() wrong %v != %v", s, "") + } +} + +func TestObjectFs(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + if obj.Fs() != remote { + t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote) + } +} + +func TestObjectRemote(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + if obj.Remote() != file1.Path { + t.Errorf("Remote is wrong %v != %v", obj.Remote(), file1.Path) + } +} + +func TestObjectMd5sum(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + Md5sum, err := obj.Md5sum() + if err != nil { + t.Errorf("Error in Md5sum: %v", err) + } + if Md5sum != file1.Md5sum { + t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum) + } +} + +func TestObjectModTime(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + if !obj.ModTime().Equal(file1.ModTime) { + t.Errorf("ModTime is wrong %v != %v", obj.ModTime(), file1.ModTime) + } +} + +func TestObjectSetModTime(t *testing.T) { + skipIfNotOk(t) + newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") + obj := findObject(t, file1.Path) + obj.SetModTime(newModTime) + // Check in this object + if !obj.ModTime().Equal(newModTime) { + t.Errorf("newModTime is wrong %v != %v", obj.ModTime(), newModTime) + } + file1.ModTime = newModTime + // And make a new object and read it from there too + TestObjectModTime(t) +} + +func TestObjectSize(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + if obj.Size() != file1.Size { + t.Errorf("Size is wrong %v != %v", obj.Size(), file1.Size) + } +} + +func TestObjectOpen(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + in, err := obj.Open() + if err != nil { + t.Fatalf("Open() return error: %v", err) + } + hash := md5.New() + n, err := io.Copy(hash, in) + if err != nil { + t.Fatalf("io.Copy() return error: %v", err) + } + if n != file1.Size { + t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size) + } + err = in.Close() + if err != nil { + t.Fatalf("in.Close() return error: %v", err) + } + Md5sum := hex.EncodeToString(hash.Sum(nil)) + if Md5sum != file1.Md5sum { + t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum) + } +} + +func TestObjectUpdate(t *testing.T) { + skipIfNotOk(t) + buf := bytes.NewBufferString(fstest.RandomString(200)) + hash := md5.New() + in := io.TeeReader(buf, hash) + + file1.Size = int64(buf.Len()) + obj := findObject(t, file1.Path) + err := obj.Update(in, file1.ModTime, file1.Size) + if err != nil { + t.Fatal("Update error", err) + } + file1.Md5sum = hex.EncodeToString(hash.Sum(nil)) + file1.Check(obj) + // Re-read the object and check again + obj = findObject(t, file1.Path) + file1.Check(obj) +} + +func TestObjectStorable(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + if !obj.Storable() { + t.Fatalf("Expecting %v to be storable", obj) + } +} + +func TestObjectRemove(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + err := obj.Remove() + if err != nil { + t.Fatal("Remove error", err) + } + fstest.CheckListing(remote, []fstest.Item{file2}) +} + +func TestObjectPurge(t *testing.T) { + skipIfNotOk(t) + fstest.TestPurge(remote) + fstest.TestPurge(remote) +} + +func TestFinalise(t *testing.T) { + skipIfNotOk(t) + if remoteFinalise != nil { + remoteFinalise() + } +} diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go new file mode 100644 index 000000000..b83a0b36e --- /dev/null +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -0,0 +1,46 @@ +// Test Google cloud storage filesystem interface +package googlecloudstorage_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/googlecloudstorage" +) + +func init() { + fstests.NilObject = fs.Object((*googlecloudstorage.FsObjectStorage)(nil)) + fstests.RemoteName = "TestGoogleCloudStorage:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/local/local_test.go b/local/local_test.go new file mode 100644 index 000000000..689be7fff --- /dev/null +++ b/local/local_test.go @@ -0,0 +1,46 @@ +// Test Local filesystem interface +package local_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/local" +) + +func init() { + fstests.NilObject = fs.Object((*local.FsObjectLocal)(nil)) + fstests.RemoteName = "" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/rclonetest/rclonetest.go b/rclonetest/rclonetest.go index 224b43da8..6c2827069 100644 --- a/rclonetest/rclonetest.go +++ b/rclonetest/rclonetest.go @@ -10,10 +10,10 @@ import ( "math/rand" "os" "path" - "strings" "time" "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest" "github.com/ogier/pflag" // Active file systems @@ -32,88 +32,6 @@ var ( subDir = pflag.BoolP("subdir", "S", false, "Test with a sub directory") ) -// Represents an item for checking -type Item struct { - Path string - Md5sum string - ModTime time.Time - Size int64 -} - -// Represents all items for checking -type Items struct { - byName map[string]*Item - items []Item -} - -// Make an Items -func NewItems(items []Item) *Items { - is := &Items{ - byName: make(map[string]*Item), - items: items, - } - // Fill up byName - for i := range items { - is.byName[items[i].Path] = &items[i] - } - return is -} - -// Check off an item -func (is *Items) Find(obj fs.Object) { - i, ok := is.byName[obj.Remote()] - if !ok { - log.Fatalf("Unexpected file %q", obj.Remote()) - } - delete(is.byName, obj.Remote()) - // Check attributes - Md5sum, err := obj.Md5sum() - if err != nil { - log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err) - } - if i.Md5sum != Md5sum { - log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum) - } - if i.Size != obj.Size() { - log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size()) - } - // check the mod time to the given precision - modTime := obj.ModTime() - dt := modTime.Sub(i.ModTime) - if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow { - log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime) - } - -} - -// Check all done -func (is *Items) Done() { - if len(is.byName) != 0 { - for name := range is.byName { - log.Printf("Not found %q", name) - } - log.Fatalf("%d objects not found", len(is.byName)) - } -} - -// Checks the fs to see if it has the expected contents -func CheckListing(f fs.Fs, items []Item) { - is := NewItems(items) - for obj := range f.List() { - is.Find(obj) - } - is.Done() -} - -// Parse a time string or explode -func Time(timeString string) time.Time { - t, err := time.Parse(time.RFC3339Nano, timeString) - if err != nil { - log.Fatalf("Failed to parse time %q: %v", timeString, err) - } - return t -} - // Write a file func WriteFile(filePath, content string, t time.Time) { // FIXME make directories? @@ -133,29 +51,9 @@ func WriteFile(filePath, content string, t time.Time) { } } -// Create a random string -func RandomString(n int) string { - source := "abcdefghijklmnopqrstuvwxyz0123456789" - out := make([]byte, n) - for i := range out { - out[i] = source[rand.Intn(len(source))] - } - return string(out) -} - -func TestMkdir(flocal, fremote fs.Fs) { - err := fs.Mkdir(fremote) - if err != nil { - log.Fatalf("Mkdir failed: %v", err) - } - items := []Item{} - CheckListing(flocal, items) - CheckListing(fremote, items) -} - -var t1 = Time("2001-02-03T04:05:06.499999999Z") -var t2 = Time("2011-12-25T12:59:59.123456789Z") -var t3 = Time("2011-12-30T12:59:59.000000000Z") +var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") +var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") +var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") func TestCopy(flocal, fremote fs.Fs) { WriteFile("sub dir/hello world", "hello world", t1) @@ -169,12 +67,12 @@ func TestCopy(flocal, fremote fs.Fs) { log.Fatalf("Copy failed: %v", err) } - items := []Item{ + items := []fstest.Item{ {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, } - CheckListing(flocal, items) - CheckListing(fremote, []Item{}) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, []fstest.Item{}) // Now without dry run @@ -184,8 +82,8 @@ func TestCopy(flocal, fremote fs.Fs) { log.Fatalf("Copy failed: %v", err) } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // Now delete the local file and download it @@ -194,8 +92,8 @@ func TestCopy(flocal, fremote fs.Fs) { log.Fatalf("Remove failed: %v", err) } - CheckListing(flocal, []Item{}) - CheckListing(fremote, items) + fstest.CheckListing(flocal, []fstest.Item{}) + fstest.CheckListing(fremote, items) log.Printf("Copy - redownload") err = fs.Sync(flocal, fremote, false) @@ -203,8 +101,8 @@ func TestCopy(flocal, fremote fs.Fs) { log.Fatalf("Copy failed: %v", err) } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // Clean the directory cleanTempDir() @@ -222,11 +120,11 @@ func TestSync(flocal, fremote fs.Fs) { if err != nil { log.Fatalf("Sync failed: %v", err) } - items := []Item{ + items := []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // ------------------------------------------------------------ @@ -236,12 +134,12 @@ func TestSync(flocal, fremote fs.Fs) { if err != nil { log.Fatalf("Sync failed: %v", err) } - items = []Item{ + items = []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, {Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // ------------------------------------------------------------ @@ -251,12 +149,12 @@ func TestSync(flocal, fremote fs.Fs) { if err != nil { log.Fatalf("Sync failed: %v", err) } - items = []Item{ + items = []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, {Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"}, } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // ------------------------------------------------------------ @@ -266,12 +164,12 @@ func TestSync(flocal, fremote fs.Fs) { if err != nil { log.Fatalf("Sync failed: %v", err) } - items = []Item{ + items = []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) // ------------------------------------------------------------ @@ -288,24 +186,24 @@ func TestSync(flocal, fremote fs.Fs) { log.Fatalf("Sync failed: %v", err) } - before := []Item{ + before := []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, } - items = []Item{ + items = []fstest.Item{ {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, {Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, } - CheckListing(flocal, items) - CheckListing(fremote, before) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, before) log.Printf("Sync after removing a file and adding a file") err = fs.Sync(fremote, flocal, true) if err != nil { log.Fatalf("Sync failed: %v", err) } - CheckListing(flocal, items) - CheckListing(fremote, items) + fstest.CheckListing(flocal, items) + fstest.CheckListing(fremote, items) } func TestLs(flocal, fremote fs.Fs) { @@ -322,28 +220,6 @@ func TestLsd(flocal, fremote fs.Fs) { func TestCheck(flocal, fremote fs.Fs) { } -func TestPurge(fremote fs.Fs) { - err := fs.Purge(fremote) - if err != nil { - log.Fatalf("Purge failed: %v", err) - } - unexpected := 0 - for obj := range fremote.List() { - unexpected++ - log.Printf("Found unexpected item %s", obj.Remote()) - } - if unexpected != 0 { - log.Fatalf("exiting as found %d unexpected items", unexpected) - } -} - -func TestRmdir(flocal, fremote fs.Fs) { - err := fs.Rmdir(fremote) - if err != nil { - log.Fatalf("Rmdir failed: %v", err) - } -} - func syntaxError() { fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s. @@ -383,32 +259,15 @@ func main() { os.Exit(1) } - remoteName = args[0] - if !strings.HasSuffix(remoteName, ":") { - remoteName += "/" - } - remoteName += RandomString(32) - var parentRemote fs.Fs - if *subDir { - var err error - parentRemote, err = fs.NewFs(remoteName) - if err != nil { - log.Fatalf("Failed to make parent %q: %v", remoteName, err) - } - remoteName += "/" + RandomString(8) - } - log.Printf("Testing with remote %q", remoteName) + fremote, finalise := fstest.RandomRemote(args[0], *subDir) + log.Printf("Testing with remote %v", fremote) + var err error localName, err = ioutil.TempDir("", "rclone") if err != nil { log.Fatalf("Failed to create temp dir: %v", err) } log.Printf("Testing with local %q", localName) - - fremote, err := fs.NewFs(remoteName) - if err != nil { - log.Fatalf("Failed to make %q: %v", remoteName, err) - } flocal, err := fs.NewFs(localName) if err != nil { log.Fatalf("Failed to make %q: %v", remoteName, err) @@ -416,18 +275,15 @@ func main() { fs.CalculateModifyWindow(fremote, flocal) - TestMkdir(flocal, fremote) + fstest.TestMkdir(fremote) TestCopy(flocal, fremote) TestSync(flocal, fremote) TestLs(flocal, fremote) TestLsd(flocal, fremote) TestCheck(flocal, fremote) - TestPurge(fremote) //TestRmdir(flocal, fremote) - if parentRemote != nil { - TestPurge(parentRemote) - } + finalise() cleanTempDir() log.Printf("Tests OK") diff --git a/rclonetest/test.sh b/rclonetest/test.sh index 4eb6eb606..740f3fec4 100755 --- a/rclonetest/test.sh +++ b/rclonetest/test.sh @@ -3,11 +3,11 @@ go install REMOTES=" -memstore: -s3: -drive2: -gcs: -dropbox: +TestSwift: +TestS3: +TestDrive: +TestGoogleCloudStorage: +TestDropbox: /tmp/z " diff --git a/s3/s3_test.go b/s3/s3_test.go new file mode 100644 index 000000000..a2ef7054a --- /dev/null +++ b/s3/s3_test.go @@ -0,0 +1,46 @@ +// Test S3 filesystem interface +package s3_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/s3" +) + +func init() { + fstests.NilObject = fs.Object((*s3.FsObjectS3)(nil)) + fstests.RemoteName = "TestS3:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/swift/swift_test.go b/swift/swift_test.go new file mode 100644 index 000000000..19f47d7d9 --- /dev/null +++ b/swift/swift_test.go @@ -0,0 +1,46 @@ +// Test Swift filesystem interface +package swift_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/swift" +) + +func init() { + fstests.NilObject = fs.Object((*swift.FsObjectSwift)(nil)) + fstests.RemoteName = "TestSwift:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }