2014-07-25 05:50:11 +08:00
|
|
|
// Test Swift filesystem interface
|
2018-09-07 19:02:27 +08:00
|
|
|
package swift
|
2014-07-25 05:50:11 +08:00
|
|
|
|
|
|
|
import (
|
2019-08-08 00:21:49 +08:00
|
|
|
"bytes"
|
|
|
|
"context"
|
2021-01-29 01:09:41 +08:00
|
|
|
"errors"
|
2019-08-08 00:21:49 +08:00
|
|
|
"io"
|
2014-07-25 05:50:11 +08:00
|
|
|
"testing"
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
"github.com/ncw/swift/v2"
|
2019-07-29 01:47:38 +08:00
|
|
|
"github.com/rclone/rclone/fs"
|
2019-08-08 00:21:49 +08:00
|
|
|
"github.com/rclone/rclone/fs/hash"
|
|
|
|
"github.com/rclone/rclone/fs/object"
|
|
|
|
"github.com/rclone/rclone/fstest"
|
2019-07-29 01:47:38 +08:00
|
|
|
"github.com/rclone/rclone/fstest/fstests"
|
2019-08-08 00:21:49 +08:00
|
|
|
"github.com/rclone/rclone/lib/random"
|
2021-01-29 01:09:41 +08:00
|
|
|
"github.com/rclone/rclone/lib/readers"
|
2019-08-08 00:21:49 +08:00
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/require"
|
2014-07-25 05:50:11 +08:00
|
|
|
)
|
|
|
|
|
2018-04-08 01:48:11 +08:00
|
|
|
// TestIntegration runs integration tests against the remote
|
|
|
|
func TestIntegration(t *testing.T) {
|
|
|
|
fstests.Run(t, &fstests.Opt{
|
2019-10-04 23:51:07 +08:00
|
|
|
RemoteName: "TestSwiftAIO:",
|
2018-09-07 19:02:27 +08:00
|
|
|
NilObject: (*Object)(nil),
|
2018-04-08 01:48:11 +08:00
|
|
|
})
|
2014-07-25 05:50:11 +08:00
|
|
|
}
|
2018-09-07 19:02:27 +08:00
|
|
|
|
|
|
|
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
|
|
|
return f.setUploadChunkSize(cs)
|
|
|
|
}
|
|
|
|
|
|
|
|
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
2019-08-08 00:21:49 +08:00
|
|
|
|
|
|
|
// Check that PutStream works with NoChunk as it is the major code
|
|
|
|
// deviation
|
|
|
|
func (f *Fs) testNoChunk(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
f.opt.NoChunk = true
|
|
|
|
defer func() {
|
|
|
|
f.opt.NoChunk = false
|
|
|
|
}()
|
|
|
|
|
|
|
|
file := fstest.Item{
|
|
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
|
|
Path: "piped data no chunk.txt",
|
|
|
|
Size: -1, // use unknown size during upload
|
|
|
|
}
|
|
|
|
|
|
|
|
const contentSize = 100
|
|
|
|
|
|
|
|
contents := random.String(contentSize)
|
|
|
|
buf := bytes.NewBufferString(contents)
|
|
|
|
uploadHash := hash.NewMultiHasher()
|
|
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
|
|
|
|
file.Size = -1
|
|
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
|
|
obj, err := f.Features().PutStream(ctx, in, obji)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
file.Hashes = uploadHash.Sums()
|
|
|
|
file.Size = int64(contentSize) // use correct size when checking
|
|
|
|
file.Check(t, obj, f.Precision())
|
|
|
|
|
|
|
|
// Re-read the object and check again
|
|
|
|
obj, err = f.NewObject(ctx, file.Path)
|
|
|
|
require.NoError(t, err)
|
|
|
|
file.Check(t, obj, f.Precision())
|
|
|
|
|
|
|
|
// Delete the object
|
|
|
|
assert.NoError(t, obj.Remove(ctx))
|
|
|
|
}
|
|
|
|
|
|
|
|
// Additional tests that aren't in the framework
|
|
|
|
func (f *Fs) InternalTest(t *testing.T) {
|
|
|
|
t.Run("NoChunk", f.testNoChunk)
|
2021-01-29 01:09:41 +08:00
|
|
|
t.Run("WithChunk", f.testWithChunk)
|
|
|
|
t.Run("WithChunkFail", f.testWithChunkFail)
|
2021-03-02 16:58:14 +08:00
|
|
|
t.Run("CopyLargeObject", f.testCopyLargeObject)
|
2021-01-29 01:09:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Fs) testWithChunk(t *testing.T) {
|
|
|
|
preConfChunkSize := f.opt.ChunkSize
|
|
|
|
preConfChunk := f.opt.NoChunk
|
|
|
|
f.opt.NoChunk = false
|
2021-03-03 03:11:57 +08:00
|
|
|
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
2021-01-29 01:09:41 +08:00
|
|
|
defer func() {
|
|
|
|
//restore old config after test
|
|
|
|
f.opt.ChunkSize = preConfChunkSize
|
|
|
|
f.opt.NoChunk = preConfChunk
|
|
|
|
}()
|
|
|
|
|
|
|
|
file := fstest.Item{
|
|
|
|
ModTime: fstest.Time("2020-12-31T04:05:06.499999999Z"),
|
|
|
|
Path: "piped data chunk.txt",
|
|
|
|
Size: -1, // use unknown size during upload
|
|
|
|
}
|
|
|
|
const contentSize = 2048
|
|
|
|
contents := random.String(contentSize)
|
|
|
|
buf := bytes.NewBufferString(contents)
|
|
|
|
uploadHash := hash.NewMultiHasher()
|
|
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
|
|
|
|
file.Size = -1
|
|
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
|
|
ctx := context.TODO()
|
|
|
|
obj, err := f.Features().PutStream(ctx, in, obji)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.NotEmpty(t, obj)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Fs) testWithChunkFail(t *testing.T) {
|
|
|
|
preConfChunkSize := f.opt.ChunkSize
|
|
|
|
preConfChunk := f.opt.NoChunk
|
|
|
|
f.opt.NoChunk = false
|
2021-03-03 03:11:57 +08:00
|
|
|
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
2021-01-29 01:09:41 +08:00
|
|
|
segmentContainer := f.root + "_segments"
|
|
|
|
defer func() {
|
|
|
|
//restore config
|
|
|
|
f.opt.ChunkSize = preConfChunkSize
|
|
|
|
f.opt.NoChunk = preConfChunk
|
|
|
|
}()
|
|
|
|
path := "piped data chunk with error.txt"
|
|
|
|
file := fstest.Item{
|
|
|
|
ModTime: fstest.Time("2021-01-04T03:46:00.499999999Z"),
|
|
|
|
Path: path,
|
|
|
|
Size: -1, // use unknown size during upload
|
|
|
|
}
|
|
|
|
const contentSize = 4096
|
|
|
|
const errPosition = 3072
|
|
|
|
contents := random.String(contentSize)
|
|
|
|
buf := bytes.NewBufferString(contents[:errPosition])
|
|
|
|
errMessage := "potato"
|
|
|
|
er := &readers.ErrorReader{Err: errors.New(errMessage)}
|
2022-08-20 22:38:02 +08:00
|
|
|
in := io.NopCloser(io.MultiReader(buf, er))
|
2021-01-29 01:09:41 +08:00
|
|
|
|
|
|
|
file.Size = contentSize
|
|
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
|
|
ctx := context.TODO()
|
|
|
|
_, err := f.Features().PutStream(ctx, in, obji)
|
|
|
|
// error is potato
|
|
|
|
require.NotNil(t, err)
|
|
|
|
require.Equal(t, errMessage, err.Error())
|
2021-01-23 01:23:51 +08:00
|
|
|
_, _, err = f.c.Object(ctx, f.rootContainer, path)
|
2021-01-29 01:09:41 +08:00
|
|
|
assert.Equal(t, swift.ObjectNotFound, err)
|
|
|
|
prefix := path
|
2021-01-23 01:23:51 +08:00
|
|
|
objs, err := f.c.Objects(ctx, segmentContainer, &swift.ObjectsOpts{
|
2021-01-29 01:09:41 +08:00
|
|
|
Prefix: prefix,
|
|
|
|
})
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Empty(t, objs)
|
2019-08-08 00:21:49 +08:00
|
|
|
}
|
|
|
|
|
2021-03-02 16:58:14 +08:00
|
|
|
func (f *Fs) testCopyLargeObject(t *testing.T) {
|
|
|
|
preConfChunkSize := f.opt.ChunkSize
|
|
|
|
preConfChunk := f.opt.NoChunk
|
|
|
|
f.opt.NoChunk = false
|
2021-03-03 03:11:57 +08:00
|
|
|
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
2021-03-02 16:58:14 +08:00
|
|
|
defer func() {
|
|
|
|
//restore old config after test
|
|
|
|
f.opt.ChunkSize = preConfChunkSize
|
|
|
|
f.opt.NoChunk = preConfChunk
|
|
|
|
}()
|
|
|
|
|
|
|
|
file := fstest.Item{
|
|
|
|
ModTime: fstest.Time("2020-12-31T04:05:06.499999999Z"),
|
|
|
|
Path: "large.txt",
|
|
|
|
Size: -1, // use unknown size during upload
|
|
|
|
}
|
|
|
|
const contentSize = 2048
|
|
|
|
contents := random.String(contentSize)
|
|
|
|
buf := bytes.NewBufferString(contents)
|
|
|
|
uploadHash := hash.NewMultiHasher()
|
|
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
|
|
|
|
file.Size = -1
|
|
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
|
|
ctx := context.TODO()
|
|
|
|
obj, err := f.Features().PutStream(ctx, in, obji)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.NotEmpty(t, obj)
|
|
|
|
remoteTarget := "large.txt (copy)"
|
|
|
|
objTarget, err := f.Features().Copy(ctx, obj, remoteTarget)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.NotEmpty(t, objTarget)
|
|
|
|
require.Equal(t, obj.Size(), objTarget.Size())
|
|
|
|
}
|
|
|
|
|
2019-08-08 00:21:49 +08:00
|
|
|
var _ fstests.InternalTester = (*Fs)(nil)
|