features: add new interfaces OpenChunkWriter and ChunkWriter #7056

backend/chunker/chunker_test.go

@@ -40,6 +40,7 @@ func TestIntegration(t *testing.T) {
 		UnimplementableFsMethods: []string{
 			"PublicLink",
 			"OpenWriterAt",
+			"OpenChunkWriter",
 			"MergeDirs",
 			"DirCacheFlush",
 			"UserInfo",

backend/combine/combine_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect"}
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "OpenChunkWriter"}
 	unimplementableObjectMethods = []string{}
 )
 

backend/compress/compress_test.go

@@ -45,6 +45,7 @@ func TestRemoteGzip(t *testing.T) {
 		NilObject:  (*Object)(nil),
 		UnimplementableFsMethods: []string{
 			"OpenWriterAt",
+			"OpenChunkWriter",
 			"MergeDirs",
 			"DirCacheFlush",
 			"PutUnchecked",

backend/crypt/crypt_test.go

@@ -24,7 +24,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
 		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -45,7 +45,7 @@ func TestStandardBase32(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -67,7 +67,7 @@ func TestStandardBase64(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base64"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -89,7 +89,7 @@ func TestStandardBase32768(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base32768"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -111,7 +111,7 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -137,7 +137,7 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -164,7 +164,7 @@ func TestNoDataObfuscate(t *testing.T) {
 			{Name: name, Key: "no_data_encryption", Value: "true"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})

backend/hasher/hasher_test.go

@@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*hasher.Object)(nil),
 		UnimplementableFsMethods: []string{
 			"OpenWriterAt",
+			"OpenChunkWriter",
 		},
 		UnimplementableObjectMethods: []string{},
 	}

backend/union/union_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 var (
-	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "PublicLink", "PutUnchecked", "MergeDirs", "OpenWriterAt"}
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "PublicLink", "PutUnchecked", "MergeDirs", "OpenWriterAt", "OpenChunkWriter"}
 	unimplementableObjectMethods = []string{}
 )
 

fs/features.go

@@ -150,6 +150,13 @@ type Features struct {
 	// It truncates any existing object
 	OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
 
+	// OpenChunkWriter returns the chunk size and a ChunkWriter
+	//
+	// Pass in the remote and the src object
+	// You can also use options to hint at the desired chunk size
+	//
+	OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (chunkSize int64, writer ChunkWriter, err error)
+
 	// UserInfo returns info about the connected user
 	UserInfo func(ctx context.Context) (map[string]string, error)
 
@@ -301,6 +308,9 @@ func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
 	if do, ok := f.(OpenWriterAter); ok {
 		ft.OpenWriterAt = do.OpenWriterAt
 	}
+	if do, ok := f.(OpenChunkWriter); ok {
+		ft.OpenChunkWriter = do.OpenChunkWriter
+	}
 	if do, ok := f.(UserInfoer); ok {
 		ft.UserInfo = do.UserInfo
 	}
@@ -393,6 +403,9 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
 	if mask.OpenWriterAt == nil {
 		ft.OpenWriterAt = nil
 	}
+	if mask.OpenChunkWriter == nil {
+		ft.OpenChunkWriter = nil
+	}
 	if mask.UserInfo == nil {
 		ft.UserInfo = nil
 	}
@@ -623,6 +636,25 @@ type OpenWriterAter interface {
 	OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
 }
 
+type OpenChunkWriter interface {
+	// OpenChunkWriter returns the chunk size and a ChunkWriter
+	//
+	// Pass in the remote and the src object
+	// You can also use options to hint at the desired chunk size
+	OpenChunkWriter(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (chunkSize int64, writer ChunkWriter, err error)
+}
+
+type ChunkWriter interface {
+	// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
+	WriteChunk(chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error)
+
+	// Close complete chunked writer
+	Close() error
+
+	// Abort chunk write
+	Abort() error
+}
+
 // UserInfoer is an optional interface for Fs
 type UserInfoer interface {
 	// UserInfo returns info about the connected user

fs/open_options.go

@@ -276,6 +276,22 @@ func (o MetadataOption) Mandatory() bool {
 	return false
 }
 
+type ChunkOption struct {
+	ChunkSize int64
+}
+
+func (o *ChunkOption) Header() (key string, value string) {
+	return "chunkSize", fmt.Sprintf("%v", o.ChunkSize)
+}
+
+func (o *ChunkOption) Mandatory() bool {
+	return false
+}
+
+func (o *ChunkOption) String() string {
+	return fmt.Sprintf("ChunkOption(%v)", o.ChunkSize)
+}
+
 // OpenOptionAddHeaders adds each header found in options to the
 // headers map provided the key was non empty.
 func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {

fstest/fstests/fstests.go

@@ -785,6 +785,52 @@ func Run(t *testing.T, opt *Opt) {
 			assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir"))
 		})
 
+		// TestFsOpenChunkWriter tests writing in chunks to fs
+		// then reads back the contents and check if they match
+		// go test -v -run 'TestIntegration/FsMkdir/FsOpenChunkWriter'
+		t.Run("FsOpenChunkWriter", func(t *testing.T) {
+			skipIfNotOk(t)
+			openChunkWriter := f.Features().OpenChunkWriter
+			if openChunkWriter == nil {
+				t.Skip("FS has no OpenChunkWriter interface")
+			}
+			size5MBs := 5 * 1024 * 1024
+			contents1 := random.String(size5MBs)
+			contents2 := random.String(size5MBs)
+
+			size1MB := 1 * 1024 * 1024
+			contents3 := random.String(size1MB)
+
+			path := "writer-at-subdir/writer-at-file"
+			objSrc := object.NewStaticObjectInfo(path, file1.ModTime, -1, true, nil, nil)
+			_, out, err := openChunkWriter(ctx, objSrc.Remote(), objSrc, &fs.ChunkOption{
+				ChunkSize: int64(size5MBs),
+			})
+			require.NoError(t, err)
+
+			var n int64
+			n, err = out.WriteChunk(1, strings.NewReader(contents2))
+			assert.NoError(t, err)
+			assert.Equal(t, int64(size5MBs), n)
+			n, err = out.WriteChunk(2, strings.NewReader(contents3))
+			assert.NoError(t, err)
+			assert.Equal(t, int64(size1MB), n)
+			n, err = out.WriteChunk(0, strings.NewReader(contents1))
+			assert.NoError(t, err)
+			assert.Equal(t, int64(size5MBs), n)
+
+			assert.NoError(t, out.Close())
+
+			obj := findObject(ctx, t, f, path)
+			originalContents := contents1 + contents2 + contents3
+			fileContents := ReadObject(ctx, t, obj, -1)
+			isEqual := originalContents == fileContents
+			assert.True(t, isEqual, "contents of file differ")
+
+			assert.NoError(t, obj.Remove(ctx))
+			assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir"))
+		})
+
 		// TestFsChangeNotify tests that changes are properly
 		// propagated
 		//