package b2 import ( "context" "crypto/sha1" "fmt" "path" "strings" "testing" "time" "github.com/rclone/rclone/backend/b2/api" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Test b2 string encoding // https://www.backblaze.com/docs/cloud-storage-native-api-string-encoding var encodeTest = []struct { fullyEncoded string minimallyEncoded string plainText string }{ {fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "}, {fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"}, {fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""}, {fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"}, {fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"}, {fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"}, {fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"}, {fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"}, {fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("}, {fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"}, {fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"}, {fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"}, {fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","}, {fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"}, {fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."}, {fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"}, {fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"}, {fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"}, {fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"}, {fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"}, {fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"}, {fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"}, {fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"}, {fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"}, {fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"}, {fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"}, {fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"}, {fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"}, {fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"}, {fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="}, {fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"}, {fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"}, {fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"}, {fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"}, {fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"}, {fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"}, {fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"}, {fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"}, {fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"}, {fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"}, {fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"}, {fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"}, {fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"}, {fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"}, {fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"}, {fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"}, {fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"}, {fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"}, {fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"}, {fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"}, {fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"}, {fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"}, {fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"}, {fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"}, {fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"}, {fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"}, {fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"}, {fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"}, {fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"}, {fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["}, {fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"}, {fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"}, {fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"}, {fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"}, {fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"}, {fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"}, {fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"}, {fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"}, {fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"}, {fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"}, {fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"}, {fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"}, {fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"}, {fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"}, {fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"}, {fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"}, {fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"}, {fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"}, {fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"}, {fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"}, {fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"}, {fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"}, {fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"}, {fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"}, {fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"}, {fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"}, {fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"}, {fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"}, {fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"}, {fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"}, {fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"}, {fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"}, {fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"}, {fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"}, {fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"}, {fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"}, {fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"}, {fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"}, } func TestUrlEncode(t *testing.T) { for _, test := range encodeTest { got := urlEncode(test.plainText) if got != test.minimallyEncoded && got != test.fullyEncoded { t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded) } } } func TestTimeString(t *testing.T) { for _, test := range []struct { in time.Time want string }{ {fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"}, {fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"}, {fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"}, } { got := timeString(test.in) if test.want != got { t.Logf("%v: want %v got %v", test.in, test.want, got) } } } func TestParseTimeString(t *testing.T) { for _, test := range []struct { in string want time.Time wantError string }{ {"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""}, {"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""}, {"", time.Time{}, ""}, {"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`}, } { o := Object{} err := o.parseTimeString(test.in) got := o.modTime var gotError string if err != nil { gotError = err.Error() } if test.want != got { t.Logf("%v: want %v got %v", test.in, test.want, got) } if test.wantError != gotError { t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError) } } } // This is adapted from the s3 equivalent. func (f *Fs) InternalTestMetadata(t *testing.T) { ctx := context.Background() original := random.String(1000) contents := fstest.Gz(t, original) mimeType := "text/html" item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z")) btime := time.Now() obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, mimeType, nil) defer func() { assert.NoError(t, obj.Remove(ctx)) }() o := obj.(*Object) gotMetadata, err := o.getMetaData(ctx) require.NoError(t, err) // We currently have a limited amount of metadata to test with B2 assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type") // Modification time from the x-bz-info-src_last_modified_millis header var mtime api.Timestamp err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey])) if err != nil { fs.Debugf(o, "Bad "+timeHeader+" header: %v", err) } assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time") // Upload time gotBtime := time.Time(gotMetadata.UploadTimestamp) dt := gotBtime.Sub(btime) assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt)) t.Run("GzipEncoding", func(t *testing.T) { // Test that the gzipped file we uploaded can be // downloaded checkDownload := func(wantContents string, wantSize int64, wantHash string) { gotContents := fstests.ReadObject(ctx, t, o, -1) assert.Equal(t, wantContents, gotContents) assert.Equal(t, wantSize, o.Size()) gotHash, err := o.Hash(ctx, hash.SHA1) require.NoError(t, err) assert.Equal(t, wantHash, gotHash) } t.Run("NoDecompress", func(t *testing.T) { checkDownload(contents, int64(len(contents)), sha1Sum(t, contents)) }) }) } func sha1Sum(t *testing.T, s string) string { hash := sha1.Sum([]byte(s)) return fmt.Sprintf("%x", hash) } // This is adapted from the s3 equivalent. func (f *Fs) InternalTestVersions(t *testing.T) { ctx := context.Background() // Small pause to make the LastModified different since AWS // only seems to track them to 1 second granularity time.Sleep(2 * time.Second) // Create an object const dirName = "versions" const fileName = dirName + "/" + "test-versions.txt" contents := random.String(100) item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) defer func() { assert.NoError(t, obj.Remove(ctx)) }() objMetadata, err := obj.(*Object).getMetaData(ctx) require.NoError(t, err) // Small pause time.Sleep(2 * time.Second) // Remove it assert.NoError(t, obj.Remove(ctx)) // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity time.Sleep(2 * time.Second) // And create it with different size and contents newContents := random.String(101) newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) newObjMetadata, err := newObj.(*Object).getMetaData(ctx) require.NoError(t, err) t.Run("Versions", func(t *testing.T) { // Set --b2-versions for this test f.opt.Versions = true defer func() { f.opt.Versions = false }() // Read the contents entries, err := f.List(ctx, dirName) require.NoError(t, err) tests := 0 var fileNameVersion string for _, entry := range entries { t.Log(entry) remote := entry.Remote() if remote == fileName { t.Run("ReadCurrent", func(t *testing.T) { assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) }) tests++ } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName { t.Run("ReadVersion", func(t *testing.T) { assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) }) assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time") fileNameVersion = remote tests++ } } assert.Equal(t, 2, tests, "object missing from listing") // Check we can read the object with a version suffix t.Run("NewObject", func(t *testing.T) { o, err := f.NewObject(ctx, fileNameVersion) require.NoError(t, err) require.NotNil(t, o) assert.Equal(t, int64(100), o.Size(), o.Remote()) }) // Check we can make a NewFs from that object with a version suffix t.Run("NewFs", func(t *testing.T) { newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion) // Make sure --b2-versions is set in the config of the new remote fs.Debugf(nil, "oldPath = %q", newPath) lastColon := strings.LastIndex(newPath, ":") require.True(t, lastColon >= 0) newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:] fs.Debugf(nil, "newPath = %q", newPath) fNew, err := cache.Get(ctx, newPath) // This should return pointing to a file require.Equal(t, fs.ErrorIsFile, err) require.NotNil(t, fNew) // With the directory above assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew))) }) }) t.Run("VersionAt", func(t *testing.T) { // We set --b2-version-at for this test so make sure we reset it at the end defer func() { f.opt.VersionAt = fs.Time{} }() var ( firstObjectTime = time.Time(objMetadata.UploadTimestamp) secondObjectTime = time.Time(newObjMetadata.UploadTimestamp) ) for _, test := range []struct { what string at time.Time want []fstest.Item wantErr error wantSize int64 }{ { what: "Before", at: firstObjectTime.Add(-time.Second), want: fstests.InternalTestFiles, wantErr: fs.ErrorObjectNotFound, }, { what: "AfterOne", at: firstObjectTime.Add(time.Second), want: append([]fstest.Item{item}, fstests.InternalTestFiles...), wantSize: 100, }, { what: "AfterDelete", at: secondObjectTime.Add(-time.Second), want: fstests.InternalTestFiles, wantErr: fs.ErrorObjectNotFound, }, { what: "AfterTwo", at: secondObjectTime.Add(time.Second), want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...), wantSize: 101, }, } { t.Run(test.what, func(t *testing.T) { f.opt.VersionAt = fs.Time(test.at) t.Run("List", func(t *testing.T) { fstest.CheckListing(t, f, test.want) }) // b2 NewObject doesn't work with VersionAt //t.Run("NewObject", func(t *testing.T) { // gotObj, gotErr := f.NewObject(ctx, fileName) // assert.Equal(t, test.wantErr, gotErr) // if gotErr == nil { // assert.Equal(t, test.wantSize, gotObj.Size()) // } //}) }) } }) t.Run("Cleanup", func(t *testing.T) { require.NoError(t, f.cleanUp(ctx, true, false, 0)) items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) fstest.CheckListing(t, f, items) // Set --b2-versions for this test f.opt.Versions = true defer func() { f.opt.Versions = false }() fstest.CheckListing(t, f, items) }) // Purge gets tested later } // -run TestIntegration/FsMkdir/FsPutFiles/Internal func (f *Fs) InternalTest(t *testing.T) { t.Run("Metadata", f.InternalTestMetadata) t.Run("Versions", f.InternalTestVersions) } var _ fstests.InternalTester = (*Fs)(nil)