From 93c960df59497269c93d4751b73dd3e8b89d8d18 Mon Sep 17 00:00:00 2001 From: Pat Patterson Date: Thu, 29 Feb 2024 16:49:34 -0800 Subject: [PATCH] b2: Add tests for new `cleanup` and `cleanup-hidden` backend commands. --- backend/b2/b2_internal_test.go | 241 ++++++++++++++++++++++++++++++++- fstest/fstest.go | 12 ++ 2 files changed, 252 insertions(+), 1 deletion(-) diff --git a/backend/b2/b2_internal_test.go b/backend/b2/b2_internal_test.go index 8253ca167..b922301b1 100644 --- a/backend/b2/b2_internal_test.go +++ b/backend/b2/b2_internal_test.go @@ -1,11 +1,25 @@ package b2 import ( + "context" + "crypto/sha1" + "fmt" + "path" + "strings" "testing" "time" + "github.com/rclone/rclone/backend/b2/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/bucket" + "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/version" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // Test b2 string encoding @@ -170,9 +184,234 @@ func TestParseTimeString(t *testing.T) { } +// This is adapted from the s3 equivalent. +func (f *Fs) InternalTestMetadata(t *testing.T) { + ctx := context.Background() + original := random.String(1000) + contents := fstest.Gz(t, original) + mimeType := "text/html" + + item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z")) + btime := time.Now() + obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, mimeType, nil) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + o := obj.(*Object) + gotMetadata, err := o.getMetaData(ctx) + require.NoError(t, err) + + // We currently have a limited amount of metadata to test with B2 + assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type") + + // Modification time from the x-bz-info-src_last_modified_millis header + var mtime api.Timestamp + err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey])) + if err != nil { + fs.Debugf(o, "Bad "+timeHeader+" header: %v", err) + } + assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time") + + // Upload time + gotBtime := time.Time(gotMetadata.UploadTimestamp) + dt := gotBtime.Sub(btime) + assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt)) + + t.Run("GzipEncoding", func(t *testing.T) { + // Test that the gzipped file we uploaded can be + // downloaded + checkDownload := func(wantContents string, wantSize int64, wantHash string) { + gotContents := fstests.ReadObject(ctx, t, o, -1) + assert.Equal(t, wantContents, gotContents) + assert.Equal(t, wantSize, o.Size()) + gotHash, err := o.Hash(ctx, hash.SHA1) + require.NoError(t, err) + assert.Equal(t, wantHash, gotHash) + } + + t.Run("NoDecompress", func(t *testing.T) { + checkDownload(contents, int64(len(contents)), sha1Sum(t, contents)) + }) + }) +} + +func sha1Sum(t *testing.T, s string) string { + hash := sha1.Sum([]byte(s)) + return fmt.Sprintf("%x", hash) +} + +// This is adapted from the s3 equivalent. +func (f *Fs) InternalTestVersions(t *testing.T) { + ctx := context.Background() + + // Small pause to make the LastModified different since AWS + // only seems to track them to 1 second granularity + time.Sleep(2 * time.Second) + + // Create an object + const dirName = "versions" + const fileName = dirName + "/" + "test-versions.txt" + contents := random.String(100) + item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + objMetadata, err := obj.(*Object).getMetaData(ctx) + require.NoError(t, err) + + // Small pause + time.Sleep(2 * time.Second) + + // Remove it + assert.NoError(t, obj.Remove(ctx)) + + // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity + time.Sleep(2 * time.Second) + + // And create it with different size and contents + newContents := random.String(101) + newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) + newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) + newObjMetadata, err := newObj.(*Object).getMetaData(ctx) + require.NoError(t, err) + + t.Run("Versions", func(t *testing.T) { + // Set --b2-versions for this test + f.opt.Versions = true + defer func() { + f.opt.Versions = false + }() + + // Read the contents + entries, err := f.List(ctx, dirName) + require.NoError(t, err) + tests := 0 + var fileNameVersion string + for _, entry := range entries { + t.Log(entry) + remote := entry.Remote() + if remote == fileName { + t.Run("ReadCurrent", func(t *testing.T) { + assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) + }) + tests++ + } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName { + t.Run("ReadVersion", func(t *testing.T) { + assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) + }) + assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time") + fileNameVersion = remote + tests++ + } + } + assert.Equal(t, 2, tests, "object missing from listing") + + // Check we can read the object with a version suffix + t.Run("NewObject", func(t *testing.T) { + o, err := f.NewObject(ctx, fileNameVersion) + require.NoError(t, err) + require.NotNil(t, o) + assert.Equal(t, int64(100), o.Size(), o.Remote()) + }) + + // Check we can make a NewFs from that object with a version suffix + t.Run("NewFs", func(t *testing.T) { + newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion) + // Make sure --b2-versions is set in the config of the new remote + fs.Debugf(nil, "oldPath = %q", newPath) + lastColon := strings.LastIndex(newPath, ":") + require.True(t, lastColon >= 0) + newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:] + fs.Debugf(nil, "newPath = %q", newPath) + fNew, err := cache.Get(ctx, newPath) + // This should return pointing to a file + require.Equal(t, fs.ErrorIsFile, err) + require.NotNil(t, fNew) + // With the directory above + assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew))) + }) + }) + + t.Run("VersionAt", func(t *testing.T) { + // We set --b2-version-at for this test so make sure we reset it at the end + defer func() { + f.opt.VersionAt = fs.Time{} + }() + + var ( + firstObjectTime = time.Time(objMetadata.UploadTimestamp) + secondObjectTime = time.Time(newObjMetadata.UploadTimestamp) + ) + + for _, test := range []struct { + what string + at time.Time + want []fstest.Item + wantErr error + wantSize int64 + }{ + { + what: "Before", + at: firstObjectTime.Add(-time.Second), + want: fstests.InternalTestFiles, + wantErr: fs.ErrorObjectNotFound, + }, + { + what: "AfterOne", + at: firstObjectTime.Add(time.Second), + want: append([]fstest.Item{item}, fstests.InternalTestFiles...), + wantSize: 100, + }, + { + what: "AfterDelete", + at: secondObjectTime.Add(-time.Second), + want: fstests.InternalTestFiles, + wantErr: fs.ErrorObjectNotFound, + }, + { + what: "AfterTwo", + at: secondObjectTime.Add(time.Second), + want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...), + wantSize: 101, + }, + } { + t.Run(test.what, func(t *testing.T) { + f.opt.VersionAt = fs.Time(test.at) + t.Run("List", func(t *testing.T) { + fstest.CheckListing(t, f, test.want) + }) + // b2 NewObject doesn't work with VersionAt + //t.Run("NewObject", func(t *testing.T) { + // gotObj, gotErr := f.NewObject(ctx, fileName) + // assert.Equal(t, test.wantErr, gotErr) + // if gotErr == nil { + // assert.Equal(t, test.wantSize, gotObj.Size()) + // } + //}) + }) + } + }) + + t.Run("Cleanup", func(t *testing.T) { + require.NoError(t, f.cleanUp(ctx, true, false, 0)) + items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) + fstest.CheckListing(t, f, items) + // Set --b2-versions for this test + f.opt.Versions = true + defer func() { + f.opt.Versions = false + }() + fstest.CheckListing(t, f, items) + }) + + // Purge gets tested later +} + // -run TestIntegration/FsMkdir/FsPutFiles/Internal func (f *Fs) InternalTest(t *testing.T) { - // Internal tests go here + t.Run("Metadata", f.InternalTestMetadata) + t.Run("Versions", f.InternalTestVersions) } var _ fstests.InternalTester = (*Fs)(nil) diff --git a/fstest/fstest.go b/fstest/fstest.go index ebea012c7..835a76d60 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -5,6 +5,7 @@ package fstest import ( "bytes" + "compress/gzip" "context" "flag" "fmt" @@ -609,3 +610,14 @@ func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Director gotT := dir.ModTime(ctx) AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision()) } + +// Gz returns a compressed version of its input string +func Gz(t *testing.T, s string) string { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write([]byte(s)) + require.NoError(t, err) + err = zw.Close() + require.NoError(t, err) + return buf.String() +}