From a1f52bcf50dd20b18249b189348bdbcb6d86f431 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 6 Aug 2024 11:50:21 +0100 Subject: [PATCH] fstest: implement method to skip ChunkedCopy tests --- backend/s3/s3.go | 7 +++++++ fstest/fstests/fstests.go | 12 ++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index f2d75a1ae..cb8baf134 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -3091,6 +3091,9 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { } func (f *Fs) setCopyCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { + if f.opt.CopyCutoff == math.MaxInt64 { + return f.opt.CopyCutoff, fmt.Errorf("--s3-copy-cutoff not supported: %w", fs.ErrorNotImplemented) + } err = checkUploadChunkSize(cs) if err == nil { old, f.opt.CopyCutoff = f.opt.CopyCutoff, cs @@ -3234,6 +3237,10 @@ func setQuirks(opt *Options) { useMultipartEtag = false useAlreadyExists = false // useMultipartUploads = false - set this manually + // rclone serve doesn't support multi-part server side copy: + // See: https://github.com/rclone/rclone/issues/7454 + // So make cutoff very large which it does support + opt.CopyCutoff = math.MaxInt64 case "Scaleway": // Scaleway can only have 1000 parts in an upload if opt.MaxUploadParts > 1000 { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 00e5372f5..06e6efb72 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -83,6 +83,7 @@ type SetUploadCutoffer interface { type SetCopyCutoffer interface { // Change the configured CopyCutoff. // Will only be called while no transfer is in progress. + // Return fs.ErrorNotImplemented if you can't implement this SetCopyCutoff(fs.SizeSuffix) (fs.SizeSuffix, error) } @@ -2345,10 +2346,6 @@ func Run(t *testing.T, opt *Opt) { t.Skip("skipping as ChunkedUpload.Skip is set") } - if strings.HasPrefix(f.Name(), "serves3") || strings.HasPrefix(f.Name(), "TestS3Rclone") { - t.Skip("FIXME skip test - see #7454") - } - do, _ := f.(SetCopyCutoffer) if do == nil { t.Skipf("%T does not implement SetCopyCutoff", f) @@ -2362,6 +2359,13 @@ func Run(t *testing.T, opt *Opt) { minChunkSize = opt.ChunkedUpload.CeilChunkSize(minChunkSize) } + // Test setting the copy cutoff before we get going + _, err := do.SetCopyCutoff(minChunkSize) + if errors.Is(err, fs.ErrorNotImplemented) { + t.Skipf("%T does not support SetCopyCutoff: %v", f, err) + } + require.NoError(t, err) + chunkSizes := fs.SizeSuffixList{ minChunkSize, minChunkSize + 1,