// Integration tests - test rclone by doing real transactions to a // storage provider to and from the local disk. // // By default it will use a local fs, however you can provide a // -remote option to use a different remote. The test_all.go script // is a wrapper to call this for all the test remotes. // // FIXME not safe for concurrent running of tests until fs.Config is // no longer a global // // NB When writing tests // // Make sure every series of writes to the remote has a // fstest.CheckItems() before use. This make sure the directory // listing is now consistent and stops cascading errors. // // Call accounting.GlobalStats().ResetCounters() before every fs.Sync() as it // uses the error count internally. package operations_test import ( "bytes" "context" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "regexp" "strings" "testing" "time" _ "github.com/rclone/rclone/backend/all" // import all backends "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/text/cases" "golang.org/x/text/language" ) // Some times used in the tests var ( t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") ) // TestMain drives the tests func TestMain(m *testing.M) { fstest.TestMain(m) } func TestMkdir(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) err := operations.Mkdir(ctx, r.Fremote, "") require.NoError(t, err) fstest.CheckListing(t, r.Fremote, []fstest.Item{}) err = operations.Mkdir(ctx, r.Fremote, "") require.NoError(t, err) } func TestLsd(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) r.CheckRemoteItems(t, file1) var buf bytes.Buffer err := operations.ListDir(ctx, r.Fremote, &buf) require.NoError(t, err) res := buf.String() assert.Contains(t, res, "sub dir\n") } func TestLs(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) r.CheckRemoteItems(t, file1, file2) var buf bytes.Buffer err := operations.List(ctx, r.Fremote, &buf) require.NoError(t, err) res := buf.String() assert.Contains(t, res, " 1 empty space\n") assert.Contains(t, res, " 60 potato2\n") } func TestLsWithFilesFrom(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) r.CheckRemoteItems(t, file1, file2) // Set the --files-from equivalent f, err := filter.NewFilter(nil) require.NoError(t, err) require.NoError(t, f.AddFile("potato2")) require.NoError(t, f.AddFile("notfound")) // Change the active filter ctx = filter.ReplaceConfig(ctx, f) var buf bytes.Buffer err = operations.List(ctx, r.Fremote, &buf) require.NoError(t, err) assert.Equal(t, " 60 potato2\n", buf.String()) // Now try with --no-traverse ci.NoTraverse = true buf.Reset() err = operations.List(ctx, r.Fremote, &buf) require.NoError(t, err) assert.Equal(t, " 60 potato2\n", buf.String()) } func TestLsLong(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) r.CheckRemoteItems(t, file1, file2) var buf bytes.Buffer err := operations.ListLong(ctx, r.Fremote, &buf) require.NoError(t, err) res := buf.String() lines := strings.Split(strings.Trim(res, "\n"), "\n") assert.Equal(t, 2, len(lines)) timeFormat := "2006-01-02 15:04:05.000000000" precision := r.Fremote.Precision() location := time.Now().Location() checkTime := func(m, filename string, expected time.Time) { modTime, err := time.ParseInLocation(timeFormat, m, location) // parse as localtime if err != nil { t.Errorf("Error parsing %q: %v", m, err) } else { fstest.AssertTimeEqualWithPrecision(t, filename, expected, modTime, precision) } } m1 := regexp.MustCompile(`(?m)^ 1 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) empty space$`) if ms := m1.FindStringSubmatch(res); ms == nil { t.Errorf("empty space missing: %q", res) } else { checkTime(ms[1], "empty space", t2.Local()) } m2 := regexp.MustCompile(`(?m)^ 60 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) potato2$`) if ms := m2.FindStringSubmatch(res); ms == nil { t.Errorf("potato2 missing: %q", res) } else { checkTime(ms[1], "potato2", t1.Local()) } } func TestHashSums(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) r.CheckRemoteItems(t, file1, file2) hashes := r.Fremote.Hashes() var quickXorHash hash.Type err := quickXorHash.Set("QuickXorHash") require.NoError(t, err) for _, test := range []struct { name string download bool base64 bool ht hash.Type want []string }{ { ht: hash.MD5, want: []string{ "336d5ebc5436534e61d16e63ddfca327 empty space\n", "d6548b156ea68a4e003e786df99eee76 potato2\n", }, }, { ht: hash.MD5, download: true, want: []string{ "336d5ebc5436534e61d16e63ddfca327 empty space\n", "d6548b156ea68a4e003e786df99eee76 potato2\n", }, }, { ht: hash.SHA1, want: []string{ "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n", "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n", }, }, { ht: hash.SHA1, download: true, want: []string{ "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n", "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n", }, }, { ht: quickXorHash, want: []string{ "2d00000000000000000000000100000000000000 empty space\n", "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n", }, }, { ht: quickXorHash, download: true, want: []string{ "2d00000000000000000000000100000000000000 empty space\n", "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n", }, }, { ht: quickXorHash, base64: true, want: []string{ "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n", "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n", }, }, { ht: quickXorHash, base64: true, download: true, want: []string{ "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n", "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n", }, }, } { if !hashes.Contains(test.ht) { continue } name := cases.Title(language.Und, cases.NoLower).String(test.ht.String()) if test.download { name += "Download" } if test.base64 { name += "Base64" } t.Run(name, func(t *testing.T) { var buf bytes.Buffer err := operations.HashLister(ctx, test.ht, test.base64, test.download, r.Fremote, &buf) require.NoError(t, err) res := buf.String() for _, line := range test.want { assert.Contains(t, res, line) } }) } } func TestHashSumsWithErrors(t *testing.T) { ctx := context.Background() memFs, err := fs.NewFs(ctx, ":memory:") require.NoError(t, err) // Make a test file content := "-" item1 := fstest.NewItem("file1", content, t1) _ = fstests.PutTestContents(ctx, t, memFs, &item1, content, true) // MemoryFS supports MD5 buf := &bytes.Buffer{} err = operations.HashLister(ctx, hash.MD5, false, false, memFs, buf) require.NoError(t, err) assert.Contains(t, buf.String(), "336d5ebc5436534e61d16e63ddfca327 file1\n") // MemoryFS can't do SHA1, but UNSUPPORTED must not appear in the output buf.Reset() err = operations.HashLister(ctx, hash.SHA1, false, false, memFs, buf) require.NoError(t, err) assert.NotContains(t, buf.String(), " UNSUPPORTED ") // ERROR must not appear in the output either assert.NotContains(t, buf.String(), " ERROR ") // TODO mock an unreadable file } func TestHashStream(t *testing.T) { reader := strings.NewReader("") in := io.NopCloser(reader) out := &bytes.Buffer{} for _, test := range []struct { input string ht hash.Type wantHex string wantBase64 string }{ { input: "", ht: hash.MD5, wantHex: "d41d8cd98f00b204e9800998ecf8427e -\n", wantBase64: "1B2M2Y8AsgTpgAmY7PhCfg== -\n", }, { input: "", ht: hash.SHA1, wantHex: "da39a3ee5e6b4b0d3255bfef95601890afd80709 -\n", wantBase64: "2jmj7l5rSw0yVb_vlWAYkK_YBwk= -\n", }, { input: "Hello world!", ht: hash.MD5, wantHex: "86fb269d190d2c85f6e0468ceca42a20 -\n", wantBase64: "hvsmnRkNLIX24EaM7KQqIA== -\n", }, { input: "Hello world!", ht: hash.SHA1, wantHex: "d3486ae9136e7856bc42212385ea797094475802 -\n", wantBase64: "00hq6RNueFa8QiEjhep5cJRHWAI= -\n", }, } { reader.Reset(test.input) require.NoError(t, operations.HashSumStream(test.ht, false, in, out)) assert.Equal(t, test.wantHex, out.String()) _, _ = reader.Seek(0, io.SeekStart) out.Reset() require.NoError(t, operations.HashSumStream(test.ht, true, in, out)) assert.Equal(t, test.wantBase64, out.String()) out.Reset() } } func TestSuffixName(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) for _, test := range []struct { remote string suffix string keepExt bool want string }{ {"test.txt", "", false, "test.txt"}, {"test.txt", "", true, "test.txt"}, {"test.txt", "-suffix", false, "test.txt-suffix"}, {"test.txt", "-suffix", true, "test-suffix.txt"}, {"test.txt.csv", "-suffix", false, "test.txt.csv-suffix"}, {"test.txt.csv", "-suffix", true, "test-suffix.txt.csv"}, {"test", "-suffix", false, "test-suffix"}, {"test", "-suffix", true, "test-suffix"}, {"test.html", "-suffix", true, "test-suffix.html"}, {"test.html.txt", "-suffix", true, "test-suffix.html.txt"}, {"test.csv.html.txt", "-suffix", true, "test-suffix.csv.html.txt"}, {"test.badext.csv.html.txt", "-suffix", true, "test.badext-suffix.csv.html.txt"}, {"test.badext", "-suffix", true, "test-suffix.badext"}, } { ci.Suffix = test.suffix ci.SuffixKeepExtension = test.keepExt got := operations.SuffixName(ctx, test.remote) assert.Equal(t, test.want, got, fmt.Sprintf("%+v", test)) } } func TestCount(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2) r.CheckRemoteItems(t, file1, file2, file3) // Check the MaxDepth too ci.MaxDepth = 1 objects, size, sizeless, err := operations.Count(ctx, r.Fremote) require.NoError(t, err) assert.Equal(t, int64(2), objects) assert.Equal(t, int64(61), size) assert.Equal(t, int64(0), sizeless) } func TestDelete(t *testing.T) { ctx := context.Background() fi, err := filter.NewFilter(nil) require.NoError(t, err) fi.Opt.MaxSize = 60 ctx = filter.ReplaceConfig(ctx, fi) r := fstest.NewRun(t) file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes r.CheckRemoteItems(t, file1, file2, file3) err = operations.Delete(ctx, r.Fremote) require.NoError(t, err) r.CheckRemoteItems(t, file3) } func isChunker(f fs.Fs) bool { return strings.HasPrefix(f.Name(), "TestChunker") } func skipIfChunker(t *testing.T, f fs.Fs) { if isChunker(f) { t.Skip("Skipping test on chunker backend") } } func TestMaxDelete(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) accounting.GlobalStats().ResetCounters() ci.MaxDelete = 2 defer r.Finalise() skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes r.CheckRemoteItems(t, file1, file2, file3) err := operations.Delete(ctx, r.Fremote) require.Error(t, err) objects, _, _, err := operations.Count(ctx, r.Fremote) require.NoError(t, err) assert.Equal(t, int64(1), objects) } // TestMaxDeleteSizeLargeFile one of the files is larger than allowed func TestMaxDeleteSizeLargeFile(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) accounting.GlobalStats().ResetCounters() ci.MaxDeleteSize = 70 defer r.Finalise() skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes r.CheckRemoteItems(t, file1, file2, file3) err := operations.Delete(ctx, r.Fremote) require.Error(t, err) r.CheckRemoteItems(t, file3) } func TestMaxDeleteSize(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) accounting.GlobalStats().ResetCounters() ci.MaxDeleteSize = 160 defer r.Finalise() skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes r.CheckRemoteItems(t, file1, file2, file3) err := operations.Delete(ctx, r.Fremote) require.Error(t, err) objects, _, _, err := operations.Count(ctx, r.Fremote) require.NoError(t, err) assert.Equal(t, int64(1), objects) // 10 or 100 bytes } func TestRetry(t *testing.T) { ctx := context.Background() var i int var err error fn := func() error { i-- if i <= 0 { return nil } return err } i, err = 3, io.EOF assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 0, i) i, err = 10, io.EOF assert.Equal(t, io.EOF, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 5, i) i, err = 10, fs.ErrorObjectNotFound assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 9, i) } func TestCat(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteBoth(ctx, "file1", "ABCDEFGHIJ", t1) file2 := r.WriteBoth(ctx, "file2", "012345678", t2) r.CheckRemoteItems(t, file1, file2) for _, test := range []struct { offset int64 count int64 separator string a string b string }{ {0, -1, "", "ABCDEFGHIJ", "012345678"}, {0, 5, "", "ABCDE", "01234"}, {-3, -1, "", "HIJ", "678"}, {1, 3, "", "BCD", "123"}, {0, -1, "\n", "ABCDEFGHIJ", "012345678"}, } { var buf bytes.Buffer err := operations.Cat(ctx, r.Fremote, &buf, test.offset, test.count, []byte(test.separator)) require.NoError(t, err) res := buf.String() if res != test.a+test.separator+test.b+test.separator && res != test.b+test.separator+test.a+test.separator { t.Errorf("Incorrect output from Cat(%d,%d,%s): %q", test.offset, test.count, test.separator, res) } } } func TestPurge(t *testing.T) { ctx := context.Background() r := fstest.NewRunIndividual(t) // make new container (azureblob has delayed mkdir after rmdir) r.Mkdir(ctx, r.Fremote) // Make some files and dirs r.ForceMkdir(ctx, r.Fremote) file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1) //..and dirs we expect to delete require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4")) //..and one more file at the end file2 := r.WriteObject(ctx, "A1/two", "bbb", t2) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{ file1, file2, }, []string{ "A1", "A1/B1", "A1/B1/C1", "A2", "A1/B2", "A1/B2/C2", "A1/B1/C3", "A3", "A3/B3", "A3/B3/C4", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Purge(ctx, r.Fremote, "A1/B1")) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{ file2, }, []string{ "A1", "A2", "A1/B2", "A1/B2/C2", "A3", "A3/B3", "A3/B3/C4", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Purge(ctx, r.Fremote, "")) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote), ) } func TestRmdirsNoLeaveRoot(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) r.Mkdir(ctx, r.Fremote) // Make some files and dirs we expect to keep r.ForceMkdir(ctx, r.Fremote) file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1) //..and dirs we expect to delete require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4")) //..and one more file at the end file2 := r.WriteObject(ctx, "A1/two", "bbb", t2) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{ file1, file2, }, []string{ "A1", "A1/B1", "A1/B1/C1", "A2", "A1/B2", "A1/B2/C2", "A1/B1/C3", "A3", "A3/B3", "A3/B3/C4", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A3/B3/C4", false)) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{ file1, file2, }, []string{ "A1", "A1/B1", "A1/B1/C1", "A2", "A1/B2", "A1/B2/C2", "A1/B1/C3", "A3", "A3/B3", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{ file1, file2, }, []string{ "A1", "A1/B1", "A1/B1/C1", }, fs.GetModifyWindow(ctx, r.Fremote), ) // Delete the files so we can remove everything including the root for _, file := range []fstest.Item{file1, file2} { o, err := r.Fremote.NewObject(ctx, file.Path) require.NoError(t, err) require.NoError(t, o.Remove(ctx)) } require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote), ) } func TestRmdirsLeaveRoot(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) r.Mkdir(ctx, r.Fremote) r.ForceMkdir(ctx, r.Fremote) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1")) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{ "A1", "A1/B1", "A1/B1/C1", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A1", true)) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{ "A1", }, fs.GetModifyWindow(ctx, r.Fremote), ) } func TestRmdirsWithFilter(t *testing.T) { ctx := context.Background() ctx, fi := filter.AddConfig(ctx) require.NoError(t, fi.AddRule("+ /A1/B1/**")) require.NoError(t, fi.AddRule("- *")) r := fstest.NewRun(t) r.Mkdir(ctx, r.Fremote) r.ForceMkdir(ctx, r.Fremote) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1")) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{ "A1", "A1/B1", "A1/B1/C1", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) fstest.CheckListingWithPrecision( t, r.Fremote, []fstest.Item{}, []string{ "A1", }, fs.GetModifyWindow(ctx, r.Fremote), ) } func TestCopyURL(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) contents := "file contents\n" file1 := r.WriteFile("file1", contents, t1) file2 := r.WriteFile("file2", contents, t1) r.Mkdir(ctx, r.Fremote) r.CheckRemoteItems(t) // check when reading from regular HTTP server status := 0 nameHeader := false headerFilename := "headerfilename.txt" handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if status != 0 { http.Error(w, "an error occurred", status) } if nameHeader { w.Header().Set("Content-Disposition", `attachment; filename="folder\`+headerFilename+`"`) } _, err := w.Write([]byte(contents)) assert.NoError(t, err) }) ts := httptest.NewServer(handler) defer ts.Close() o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) // Check file clobbering _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, true) require.Error(t, err) // Check auto file naming status = 0 urlFileName := "filename.txt" o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL+"/"+urlFileName, true, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) assert.Equal(t, urlFileName, o.Remote()) // Check header file naming nameHeader = true o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL, true, true, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) assert.Equal(t, headerFilename, o.Remote()) // Check auto file naming when url without file name _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false, false) require.Error(t, err) // Check header file naming without header set nameHeader = false _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, true, false) require.Error(t, err) // Check an error is returned for a 404 status = http.StatusNotFound o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) require.Error(t, err) assert.Contains(t, err.Error(), "Not Found") assert.Nil(t, o) status = 0 // check when reading from unverified HTTPS server ci.InsecureSkipVerify = true fshttp.ResetTransport() defer fshttp.ResetTransport() tss := httptest.NewTLSServer(handler) defer tss.Close() o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported) } func TestCopyURLToWriter(t *testing.T) { ctx := context.Background() contents := "file contents\n" // check when reading from regular HTTP server status := 0 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if status != 0 { http.Error(w, "an error occurred", status) return } _, err := w.Write([]byte(contents)) assert.NoError(t, err) }) ts := httptest.NewServer(handler) defer ts.Close() // test normal fetch var buf bytes.Buffer err := operations.CopyURLToWriter(ctx, ts.URL, &buf) require.NoError(t, err) assert.Equal(t, contents, buf.String()) // test fetch with error status = http.StatusNotFound buf.Reset() err = operations.CopyURLToWriter(ctx, ts.URL, &buf) require.Error(t, err) assert.Contains(t, err.Error(), "Not Found") assert.Equal(t, 0, len(buf.String())) } func TestMoveFile(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file2 := file1 file2.Path = "sub/file2" err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2) r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2) err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2) } func TestMoveFileWithIgnoreExisting(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) ci.IgnoreExisting = true err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file1) // Recreate file with updated content file1b := r.WriteFile("file1", "file1 modified", t2) r.CheckLocalItems(t, file1b) // Ensure modified file did not transfer and was not deleted err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1b.Path) require.NoError(t, err) r.CheckLocalItems(t, file1b) r.CheckRemoteItems(t, file1) } func TestCaseInsensitiveMoveFile(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) if !r.Fremote.Features().CaseInsensitive { return } file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file2 := file1 file2.Path = "sub/file2" err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2) r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2) file2Capitalized := file2 file2Capitalized.Path = "sub/File2" err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2Capitalized.Path, file2.Path) require.NoError(t, err) r.CheckLocalItems(t) r.CheckRemoteItems(t, file2Capitalized) } func TestCaseInsensitiveMoveFileDryRun(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if !r.Fremote.Features().CaseInsensitive { return } file1 := r.WriteObject(ctx, "hello", "world", t1) r.CheckRemoteItems(t, file1) ci.DryRun = true err := operations.MoveFile(ctx, r.Fremote, r.Fremote, "HELLO", file1.Path) require.NoError(t, err) r.CheckRemoteItems(t, file1) } func TestMoveFileBackupDir(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if !operations.CanServerSideMove(r.Fremote) { t.Skip("Skipping test as remote does not support server-side move or copy") } ci.BackupDir = r.FremoteName + "/backup" file1 := r.WriteFile("dst/file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1) r.CheckRemoteItems(t, file1old) err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t) file1old.Path = "backup/dst/file1" r.CheckRemoteItems(t, file1old, file1) } // testFsInfo is for unit testing fs.Info type testFsInfo struct { name string root string stringVal string precision time.Duration hashes hash.Set features fs.Features } // Name of the remote (as passed into NewFs) func (i *testFsInfo) Name() string { return i.name } // Root of the remote (as passed into NewFs) func (i *testFsInfo) Root() string { return i.root } // String returns a description of the FS func (i *testFsInfo) String() string { return i.stringVal } // Precision of the ModTimes in this Fs func (i *testFsInfo) Precision() time.Duration { return i.precision } // Returns the supported hash types of the filesystem func (i *testFsInfo) Hashes() hash.Set { return i.hashes } // Returns the supported hash types of the filesystem func (i *testFsInfo) Features() *fs.Features { return &i.features } func TestSameConfig(t *testing.T) { a := &testFsInfo{name: "name", root: "root"} for _, test := range []struct { name string root string expected bool }{ {"name", "root", true}, {"name", "rooty", true}, {"namey", "root", false}, {"namey", "roott", false}, } { b := &testFsInfo{name: test.name, root: test.root} actual := operations.SameConfig(a, b) assert.Equal(t, test.expected, actual) actual = operations.SameConfig(b, a) assert.Equal(t, test.expected, actual) } } func TestSame(t *testing.T) { a := &testFsInfo{name: "name", root: "root"} for _, test := range []struct { name string root string expected bool }{ {"name", "root", true}, {"name", "rooty", false}, {"namey", "root", false}, {"namey", "roott", false}, } { b := &testFsInfo{name: test.name, root: test.root} actual := operations.Same(a, b) assert.Equal(t, test.expected, actual) actual = operations.Same(b, a) assert.Equal(t, test.expected, actual) } } // testFs is for unit testing fs.Fs type testFs struct { testFsInfo } func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { return nil, nil } func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil } func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { return nil, nil } func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil } func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil } // copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping // when no filters are set func TestOverlappingFilterCheckWithoutFilter(t *testing.T) { ctx := context.Background() src := &testFs{testFsInfo{name: "name", root: "root"}} slash := string(os.PathSeparator) // native path separator for _, test := range []struct { name string root string expected bool }{ {"name", "root", true}, {"name", "/root", true}, {"namey", "root", false}, {"name", "rooty", false}, {"namey", "rooty", false}, {"name", "roo", false}, {"name", "root/toot", true}, {"name", "root/toot/", true}, {"name", "root" + slash + "toot", true}, {"name", "root" + slash + "toot" + slash, true}, {"name", "", true}, {"name", "/", true}, } { dst := &testFs{testFsInfo{name: test.name, root: test.root}} what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root) actual := operations.OverlappingFilterCheck(ctx, src, dst) assert.Equal(t, test.expected, actual, what) actual = operations.OverlappingFilterCheck(ctx, dst, src) assert.Equal(t, test.expected, actual, what) } } func TestOverlappingFilterCheckWithFilter(t *testing.T) { ctx := context.Background() fi, err := filter.NewFilter(nil) require.NoError(t, err) require.NoError(t, fi.Add(false, "/exclude/")) require.NoError(t, fi.Add(false, "/Exclude2/")) require.NoError(t, fi.Add(true, "*")) ctx = filter.ReplaceConfig(ctx, fi) src := &testFs{testFsInfo{name: "name", root: "root"}} src.features.CaseInsensitive = true slash := string(os.PathSeparator) // native path separator for _, test := range []struct { name string root string expected bool }{ {"name", "root", true}, {"name", "ROOT", true}, // case insensitive is set {"name", "/root", true}, {"name", "root/", true}, {"name", "root" + slash, true}, {"name", "root/exclude", false}, {"name", "root/Exclude2", false}, {"name", "root/include", true}, {"name", "root/exclude/", false}, {"name", "root/Exclude2/", false}, {"name", "root/exclude/sub", false}, {"name", "root/Exclude2/sub", false}, {"name", "/root/exclude/", false}, {"name", "root" + slash + "exclude", false}, {"name", "root" + slash + "exclude" + slash, false}, {"namey", "root/include", false}, {"namey", "root/include/", false}, {"namey", "root" + slash + "include", false}, {"namey", "root" + slash + "include" + slash, false}, } { dst := &testFs{testFsInfo{name: test.name, root: test.root}} dst.features.CaseInsensitive = true what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root) actual := operations.OverlappingFilterCheck(ctx, dst, src) assert.Equal(t, test.expected, actual, what) actual = operations.OverlappingFilterCheck(ctx, src, dst) assert.Equal(t, test.expected, actual, what) } } func TestListFormat(t *testing.T) { item0 := &operations.ListJSONItem{ Path: "a", Name: "a", Encrypted: "encryptedFileName", Size: 1, MimeType: "application/octet-stream", ModTime: operations.Timestamp{ When: t1, Format: "2006-01-02T15:04:05.000000000Z07:00"}, IsDir: false, Hashes: map[string]string{ "md5": "0cc175b9c0f1b6a831c399e269772661", "sha1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "dropbox": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", "quickxor": "6100000000000000000000000100000000000000"}, ID: "fileID", OrigID: "fileOrigID", } item1 := &operations.ListJSONItem{ Path: "subdir", Name: "subdir", Encrypted: "encryptedDirName", Size: -1, MimeType: "inode/directory", ModTime: operations.Timestamp{ When: t2, Format: "2006-01-02T15:04:05.000000000Z07:00"}, IsDir: true, Hashes: map[string]string(nil), ID: "dirID", OrigID: "dirOrigID", } var list operations.ListFormat list.AddPath() list.SetDirSlash(false) assert.Equal(t, "subdir", list.Format(item1)) list.SetDirSlash(true) assert.Equal(t, "subdir/", list.Format(item1)) list.SetOutput(nil) assert.Equal(t, "", list.Format(item1)) list.AppendOutput(func(item *operations.ListJSONItem) string { return "a" }) list.AppendOutput(func(item *operations.ListJSONItem) string { return "b" }) assert.Equal(t, "ab", list.Format(item1)) list.SetSeparator(":::") assert.Equal(t, "a:::b", list.Format(item1)) list.SetOutput(nil) list.AddModTime("") assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) list.SetOutput(nil) list.SetSeparator("|") list.AddID() list.AddOrigID() assert.Equal(t, "fileID|fileOrigID", list.Format(item0)) assert.Equal(t, "dirID|dirOrigID", list.Format(item1)) list.SetOutput(nil) list.AddMimeType() assert.Contains(t, list.Format(item0), "/") assert.Equal(t, "inode/directory", list.Format(item1)) list.SetOutput(nil) list.AddMetadata() assert.Equal(t, "{}", list.Format(item0)) assert.Equal(t, "{}", list.Format(item1)) list.SetOutput(nil) list.AddPath() list.SetAbsolute(true) assert.Equal(t, "/a", list.Format(item0)) list.SetAbsolute(false) assert.Equal(t, "a", list.Format(item0)) list.SetOutput(nil) list.AddSize() assert.Equal(t, "1", list.Format(item0)) list.AddPath() list.AddModTime("") list.SetDirSlash(true) list.SetSeparator("__SEP__") assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) assert.Equal(t, "-1__SEP__subdir/__SEP__"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) for _, test := range []struct { ht hash.Type want string }{ {hash.MD5, "0cc175b9c0f1b6a831c399e269772661"}, {hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}, } { list.SetOutput(nil) list.AddHash(test.ht) assert.Equal(t, test.want, list.Format(item0)) } list.SetOutput(nil) list.SetSeparator("|") list.SetCSV(true) list.AddSize() list.AddPath() list.AddModTime("") list.SetDirSlash(true) assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) list.SetOutput(nil) list.SetSeparator("|") list.AddPath() list.AddEncrypted() assert.Equal(t, "a|encryptedFileName", list.Format(item0)) assert.Equal(t, "subdir/|encryptedDirName/", list.Format(item1)) } func TestDirMove(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) r.Mkdir(ctx, r.Fremote) // Make some files and dirs r.ForceMkdir(ctx, r.Fremote) files := []fstest.Item{ r.WriteObject(ctx, "A1/one", "one", t1), r.WriteObject(ctx, "A1/two", "two", t2), r.WriteObject(ctx, "A1/B1/three", "three", t3), r.WriteObject(ctx, "A1/B1/C1/four", "four", t1), r.WriteObject(ctx, "A1/B1/C2/five", "five", t2), } require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) fstest.CheckListingWithPrecision( t, r.Fremote, files, []string{ "A1", "A1/B1", "A1/B2", "A1/B1/C1", "A1/B1/C2", "A1/B1/C3", }, fs.GetModifyWindow(ctx, r.Fremote), ) require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2")) for i := range files { files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/") } fstest.CheckListingWithPrecision( t, r.Fremote, files, []string{ "A2", "A2/B1", "A2/B2", "A2/B1/C1", "A2/B1/C2", "A2/B1/C3", }, fs.GetModifyWindow(ctx, r.Fremote), ) // Disable DirMove features := r.Fremote.Features() features.DirMove = nil require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3")) for i := range files { files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/") } fstest.CheckListingWithPrecision( t, r.Fremote, files, []string{ "A3", "A3/B1", "A3/B2", "A3/B1/C1", "A3/B1/C2", "A3/B1/C3", }, fs.GetModifyWindow(ctx, r.Fremote), ) } func TestGetFsInfo(t *testing.T) { r := fstest.NewRun(t) f := r.Fremote info := operations.GetFsInfo(f) assert.Equal(t, f.Name(), info.Name) assert.Equal(t, f.Root(), info.Root) assert.Equal(t, f.String(), info.String) assert.Equal(t, f.Precision(), info.Precision) hashSet := hash.NewHashSet() for _, hashName := range info.Hashes { var ht hash.Type require.NoError(t, ht.Set(hashName)) hashSet.Add(ht) } assert.Equal(t, f.Hashes(), hashSet) assert.Equal(t, f.Features().Enabled(), info.Features) } func TestRcat(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) check := func(t *testing.T, withChecksum, ignoreChecksum bool) { ci.CheckSum, ci.IgnoreChecksum = withChecksum, ignoreChecksum var prefix string if withChecksum { prefix = "with_checksum_" } else { prefix = "no_checksum_" } if ignoreChecksum { prefix = "ignore_checksum_" } r := fstest.NewRun(t) if *fstest.SizeLimit > 0 && int64(ci.StreamingUploadCutoff) > *fstest.SizeLimit { savedCutoff := ci.StreamingUploadCutoff ci.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit) t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", ci.StreamingUploadCutoff, savedCutoff) } fstest.CheckListing(t, r.Fremote, []fstest.Item{}) data1 := "this is some really nice test data" path1 := prefix + "small_file_from_pipe" data2 := string(make([]byte, ci.StreamingUploadCutoff+1)) path2 := prefix + "big_file_from_pipe" in := io.NopCloser(strings.NewReader(data1)) _, err := operations.Rcat(ctx, r.Fremote, path1, in, t1, nil) require.NoError(t, err) in = io.NopCloser(strings.NewReader(data2)) _, err = operations.Rcat(ctx, r.Fremote, path2, in, t2, nil) require.NoError(t, err) file1 := fstest.NewItem(path1, data1, t1) file2 := fstest.NewItem(path2, data2, t2) r.CheckRemoteItems(t, file1, file2) } for i := 0; i < 4; i++ { withChecksum := (i & 1) != 0 ignoreChecksum := (i & 2) != 0 t.Run(fmt.Sprintf("withChecksum=%v,ignoreChecksum=%v", withChecksum, ignoreChecksum), func(t *testing.T) { check(t, withChecksum, ignoreChecksum) }) } } func TestRcatMetadata(t *testing.T) { r := fstest.NewRun(t) if !r.Fremote.Features().UserMetadata { t.Skip("Skipping as destination doesn't support user metadata") } test := func(disableUploadCutoff bool) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) ci.Metadata = true data := "this is some really nice test data with metadata" path := "rcat_metadata" meta := fs.Metadata{ "key": "value", "sausage": "potato", } if disableUploadCutoff { ci.StreamingUploadCutoff = 0 data += " uploadCutoff=0" path += "_uploadcutoff0" } fstest.CheckListing(t, r.Fremote, []fstest.Item{}) in := io.NopCloser(strings.NewReader(data)) _, err := operations.Rcat(ctx, r.Fremote, path, in, t1, meta) require.NoError(t, err) file := fstest.NewItem(path, data, t1) r.CheckRemoteItems(t, file) o, err := r.Fremote.NewObject(ctx, path) require.NoError(t, err) gotMeta, err := fs.GetMetadata(ctx, o) require.NoError(t, err) // Check the specific user data we set is set // Likely there will be other values assert.Equal(t, "value", gotMeta["key"]) assert.Equal(t, "potato", gotMeta["sausage"]) // Delete the test file require.NoError(t, o.Remove(ctx)) } t.Run("Normal", func(t *testing.T) { test(false) }) t.Run("ViaDisk", func(t *testing.T) { test(true) }) } func TestRcatSize(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) const body = "------------------------------------------------------------" file1 := r.WriteFile("potato1", body, t1) file2 := r.WriteFile("potato2", body, t2) // Test with known length bodyReader := io.NopCloser(strings.NewReader(body)) obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, nil) require.NoError(t, err) assert.Equal(t, int64(len(body)), obj.Size()) assert.Equal(t, file1.Path, obj.Remote()) // Test with unknown length bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader io.NopCloser(strings.NewReader(body)) obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, nil) require.NoError(t, err) assert.Equal(t, int64(len(body)), obj.Size()) assert.Equal(t, file2.Path, obj.Remote()) // Check files exist r.CheckRemoteItems(t, file1, file2) } func TestRcatSizeMetadata(t *testing.T) { r := fstest.NewRun(t) if !r.Fremote.Features().UserMetadata { t.Skip("Skipping as destination doesn't support user metadata") } ctx := context.Background() ctx, ci := fs.AddConfig(ctx) ci.Metadata = true meta := fs.Metadata{ "key": "value", "sausage": "potato", } const body = "------------------------------------------------------------" file1 := r.WriteFile("potato1", body, t1) file2 := r.WriteFile("potato2", body, t2) // Test with known length bodyReader := io.NopCloser(strings.NewReader(body)) obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, meta) require.NoError(t, err) assert.Equal(t, int64(len(body)), obj.Size()) assert.Equal(t, file1.Path, obj.Remote()) // Test with unknown length bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader io.NopCloser(strings.NewReader(body)) obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, meta) require.NoError(t, err) assert.Equal(t, int64(len(body)), obj.Size()) assert.Equal(t, file2.Path, obj.Remote()) // Check files exist r.CheckRemoteItems(t, file1, file2) // Check metadata OK for _, path := range []string{file1.Path, file2.Path} { o, err := r.Fremote.NewObject(ctx, path) require.NoError(t, err) gotMeta, err := fs.GetMetadata(ctx, o) require.NoError(t, err) // Check the specific user data we set is set // Likely there will be other values assert.Equal(t, "value", gotMeta["key"]) assert.Equal(t, "potato", gotMeta["sausage"]) } } func TestTouchDir(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) if r.Fremote.Precision() == fs.ModTimeNotSupported { t.Skip("Skipping test as remote does not support modtime") } file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) file2 := r.WriteBoth(ctx, "empty space", "-", t2) file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2) r.CheckRemoteItems(t, file1, file2, file3) accounting.GlobalStats().ResetCounters() timeValue := time.Date(2010, 9, 8, 7, 6, 5, 4, time.UTC) err := operations.TouchDir(ctx, r.Fremote, "", timeValue, true) require.NoError(t, err) if accounting.Stats(ctx).GetErrors() != 0 { err = accounting.Stats(ctx).GetLastError() require.True(t, errors.Is(err, fs.ErrorCantSetModTime) || errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete)) } else { file1.ModTime = timeValue file2.ModTime = timeValue file3.ModTime = timeValue r.CheckRemoteItems(t, file1, file2, file3) } } var testMetadata = fs.Metadata{ // System metadata supported by all backends "mtime": t1.Format(time.RFC3339Nano), // User metadata "potato": "jersey", } func TestMkdirMetadata(t *testing.T) { const name = "dir with metadata" ctx := context.Background() ctx, ci := fs.AddConfig(ctx) ci.Metadata = true r := fstest.NewRun(t) features := r.Fremote.Features() if features.MkdirMetadata == nil { t.Skip("Skipping test as remote does not support MkdirMetadata") } newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata) require.NoError(t, err) require.NotNil(t, newDst) require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported") // Check the returned directory and one read from the listing fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata) } func TestMkdirModTime(t *testing.T) { const name = "directory with modtime" ctx := context.Background() r := fstest.NewRun(t) if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata") } newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2) require.NoError(t, err) // Check the returned directory and one read from the listing // newDst may be nil here depending on how the modtime was set if newDst != nil { fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) } fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) } func TestCopyDirMetadata(t *testing.T) { const nameNonExistent = "non existent directory" const nameExistent = "existing directory" ctx := context.Background() ctx, ci := fs.AddConfig(ctx) ci.Metadata = true r := fstest.NewRun(t) if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil { t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata") } // Create a source local directory with metadata newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata) require.NoError(t, err) require.NotNil(t, newSrc) // First try with the directory not existing newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc) require.NoError(t, err) require.NotNil(t, newDst) // Check the returned directory and one read from the listing fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata) // Then try with the directory existing require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent)) require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent)) existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent) newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc) require.NoError(t, err) require.NotNil(t, newDst) // Check the returned directory and one read from the listing fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata) } func TestSetDirModTime(t *testing.T) { const name = "set modtime on existing directory" ctx := context.Background() r := fstest.NewRun(t) if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime { t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime") } // First try with the directory not existing - should return an error newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2) require.Error(t, err) require.Nil(t, newDst) // Then try with the directory existing require.NoError(t, r.Fremote.Mkdir(ctx, name)) existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name) newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2) require.NoError(t, err) require.NotNil(t, newDst) // Check the returned directory and one read from the listing fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) }