From f36ca0cd255bbcfe00ffd410f1eff86b16a1ff92 Mon Sep 17 00:00:00 2001 From: Vitor Gomes Date: Thu, 13 Jul 2023 21:01:10 +0200 Subject: [PATCH] features: add new interfaces OpenChunkWriter and ChunkWriter #7056 --- backend/chunker/chunker_test.go | 1 + backend/combine/combine_test.go | 2 +- backend/compress/compress_test.go | 1 + backend/crypt/crypt_test.go | 14 +++++----- backend/hasher/hasher_test.go | 1 + backend/union/union_test.go | 2 +- fs/features.go | 32 +++++++++++++++++++++ fs/open_options.go | 16 +++++++++++ fstest/fstests/fstests.go | 46 +++++++++++++++++++++++++++++++ 9 files changed, 106 insertions(+), 9 deletions(-) diff --git a/backend/chunker/chunker_test.go b/backend/chunker/chunker_test.go index 7f0c0310d..d5ce656dd 100644 --- a/backend/chunker/chunker_test.go +++ b/backend/chunker/chunker_test.go @@ -40,6 +40,7 @@ func TestIntegration(t *testing.T) { UnimplementableFsMethods: []string{ "PublicLink", "OpenWriterAt", + "OpenChunkWriter", "MergeDirs", "DirCacheFlush", "UserInfo", diff --git a/backend/combine/combine_test.go b/backend/combine/combine_test.go index 0a0d357dd..c132377b1 100644 --- a/backend/combine/combine_test.go +++ b/backend/combine/combine_test.go @@ -11,7 +11,7 @@ import ( ) var ( - unimplementableFsMethods = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect"} + unimplementableFsMethods = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "OpenChunkWriter"} unimplementableObjectMethods = []string{} ) diff --git a/backend/compress/compress_test.go b/backend/compress/compress_test.go index 356e9dd43..bb74f0aca 100644 --- a/backend/compress/compress_test.go +++ b/backend/compress/compress_test.go @@ -45,6 +45,7 @@ func TestRemoteGzip(t *testing.T) { NilObject: (*Object)(nil), UnimplementableFsMethods: []string{ "OpenWriterAt", + "OpenChunkWriter", "MergeDirs", "DirCacheFlush", "PutUnchecked", diff --git a/backend/crypt/crypt_test.go b/backend/crypt/crypt_test.go index d4b9dc1e4..49db07dc7 100644 --- a/backend/crypt/crypt_test.go +++ b/backend/crypt/crypt_test.go @@ -24,7 +24,7 @@ func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ RemoteName: *fstest.RemoteName, NilObject: (*crypt.Object)(nil), - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, }) } @@ -45,7 +45,7 @@ func TestStandardBase32(t *testing.T) { {Name: name, Key: "password", Value: obscure.MustObscure("potato")}, {Name: name, Key: "filename_encryption", Value: "standard"}, }, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) @@ -67,7 +67,7 @@ func TestStandardBase64(t *testing.T) { {Name: name, Key: "filename_encryption", Value: "standard"}, {Name: name, Key: "filename_encoding", Value: "base64"}, }, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) @@ -89,7 +89,7 @@ func TestStandardBase32768(t *testing.T) { {Name: name, Key: "filename_encryption", Value: "standard"}, {Name: name, Key: "filename_encoding", Value: "base32768"}, }, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) @@ -111,7 +111,7 @@ func TestOff(t *testing.T) { {Name: name, Key: "password", Value: obscure.MustObscure("potato2")}, {Name: name, Key: "filename_encryption", Value: "off"}, }, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) @@ -137,7 +137,7 @@ func TestObfuscate(t *testing.T) { {Name: name, Key: "filename_encryption", Value: "obfuscate"}, }, SkipBadWindowsCharacters: true, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) @@ -164,7 +164,7 @@ func TestNoDataObfuscate(t *testing.T) { {Name: name, Key: "no_data_encryption", Value: "true"}, }, SkipBadWindowsCharacters: true, - UnimplementableFsMethods: []string{"OpenWriterAt"}, + UnimplementableFsMethods: []string{"OpenWriterAt", "OpenChunkWriter"}, UnimplementableObjectMethods: []string{"MimeType"}, QuickTestOK: true, }) diff --git a/backend/hasher/hasher_test.go b/backend/hasher/hasher_test.go index 23feb9b96..f17303ecc 100644 --- a/backend/hasher/hasher_test.go +++ b/backend/hasher/hasher_test.go @@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) { NilObject: (*hasher.Object)(nil), UnimplementableFsMethods: []string{ "OpenWriterAt", + "OpenChunkWriter", }, UnimplementableObjectMethods: []string{}, } diff --git a/backend/union/union_test.go b/backend/union/union_test.go index c9482c56c..b9f4e24e2 100644 --- a/backend/union/union_test.go +++ b/backend/union/union_test.go @@ -12,7 +12,7 @@ import ( ) var ( - unimplementableFsMethods = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "PublicLink", "PutUnchecked", "MergeDirs", "OpenWriterAt"} + unimplementableFsMethods = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "PublicLink", "PutUnchecked", "MergeDirs", "OpenWriterAt", "OpenChunkWriter"} unimplementableObjectMethods = []string{} ) diff --git a/fs/features.go b/fs/features.go index 4c3612285..d722b4412 100644 --- a/fs/features.go +++ b/fs/features.go @@ -150,6 +150,13 @@ type Features struct { // It truncates any existing object OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error) + // OpenChunkWriter returns the chunk size and a ChunkWriter + // + // Pass in the remote and the src object + // You can also use options to hint at the desired chunk size + // + OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (chunkSize int64, writer ChunkWriter, err error) + // UserInfo returns info about the connected user UserInfo func(ctx context.Context) (map[string]string, error) @@ -301,6 +308,9 @@ func (ft *Features) Fill(ctx context.Context, f Fs) *Features { if do, ok := f.(OpenWriterAter); ok { ft.OpenWriterAt = do.OpenWriterAt } + if do, ok := f.(OpenChunkWriter); ok { + ft.OpenChunkWriter = do.OpenChunkWriter + } if do, ok := f.(UserInfoer); ok { ft.UserInfo = do.UserInfo } @@ -393,6 +403,9 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features { if mask.OpenWriterAt == nil { ft.OpenWriterAt = nil } + if mask.OpenChunkWriter == nil { + ft.OpenChunkWriter = nil + } if mask.UserInfo == nil { ft.UserInfo = nil } @@ -623,6 +636,25 @@ type OpenWriterAter interface { OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error) } +type OpenChunkWriter interface { + // OpenChunkWriter returns the chunk size and a ChunkWriter + // + // Pass in the remote and the src object + // You can also use options to hint at the desired chunk size + OpenChunkWriter(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (chunkSize int64, writer ChunkWriter, err error) +} + +type ChunkWriter interface { + // WriteChunk will write chunk number with reader bytes, where chunk number >= 0 + WriteChunk(chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) + + // Close complete chunked writer + Close() error + + // Abort chunk write + Abort() error +} + // UserInfoer is an optional interface for Fs type UserInfoer interface { // UserInfo returns info about the connected user diff --git a/fs/open_options.go b/fs/open_options.go index 5dc3e32d4..288a36303 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -276,6 +276,22 @@ func (o MetadataOption) Mandatory() bool { return false } +type ChunkOption struct { + ChunkSize int64 +} + +func (o *ChunkOption) Header() (key string, value string) { + return "chunkSize", fmt.Sprintf("%v", o.ChunkSize) +} + +func (o *ChunkOption) Mandatory() bool { + return false +} + +func (o *ChunkOption) String() string { + return fmt.Sprintf("ChunkOption(%v)", o.ChunkSize) +} + // OpenOptionAddHeaders adds each header found in options to the // headers map provided the key was non empty. func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 99ec177b8..5ac7978cf 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -785,6 +785,52 @@ func Run(t *testing.T, opt *Opt) { assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) }) + // TestFsOpenChunkWriter tests writing in chunks to fs + // then reads back the contents and check if they match + // go test -v -run 'TestIntegration/FsMkdir/FsOpenChunkWriter' + t.Run("FsOpenChunkWriter", func(t *testing.T) { + skipIfNotOk(t) + openChunkWriter := f.Features().OpenChunkWriter + if openChunkWriter == nil { + t.Skip("FS has no OpenChunkWriter interface") + } + size5MBs := 5 * 1024 * 1024 + contents1 := random.String(size5MBs) + contents2 := random.String(size5MBs) + + size1MB := 1 * 1024 * 1024 + contents3 := random.String(size1MB) + + path := "writer-at-subdir/writer-at-file" + objSrc := object.NewStaticObjectInfo(path, file1.ModTime, -1, true, nil, nil) + _, out, err := openChunkWriter(ctx, objSrc.Remote(), objSrc, &fs.ChunkOption{ + ChunkSize: int64(size5MBs), + }) + require.NoError(t, err) + + var n int64 + n, err = out.WriteChunk(1, strings.NewReader(contents2)) + assert.NoError(t, err) + assert.Equal(t, int64(size5MBs), n) + n, err = out.WriteChunk(2, strings.NewReader(contents3)) + assert.NoError(t, err) + assert.Equal(t, int64(size1MB), n) + n, err = out.WriteChunk(0, strings.NewReader(contents1)) + assert.NoError(t, err) + assert.Equal(t, int64(size5MBs), n) + + assert.NoError(t, out.Close()) + + obj := findObject(ctx, t, f, path) + originalContents := contents1 + contents2 + contents3 + fileContents := ReadObject(ctx, t, obj, -1) + isEqual := originalContents == fileContents + assert.True(t, isEqual, "contents of file differ") + + assert.NoError(t, obj.Remove(ctx)) + assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) + }) + // TestFsChangeNotify tests that changes are properly // propagated //