github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 11:35:21 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

s3: update to using AWS SDK v2 - fixes #4989

Browse Source 
SDK v2 conversion

Changes

  - `--s3-sts-endpoint` is no longer supported
  - `--s3-use-unsigned-payload` to control use of trailer checksums (needed for non AWS)
This commit is contained in:
Nick Craig-Wood 2024-08-03 11:35:32 +01:00
parent a1f52bcf50
commit fd9049c83d
9 changed files with 558 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/s3/gen_setfrom.go
View File

@ -13,7 +13,8 @@ import (
"reflect" "reflect"
"strings" "strings"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
) )
// flags // flags
@ -82,15 +83,18 @@ func main() {
package s3 package s3
import "github.com/aws/aws-sdk-go/service/s3" import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
`) `)
genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input)) genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input))
genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput)) genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput))
genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input)) genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input))
genSetFrom(new(s3.ObjectVersion), new(s3.DeleteMarkerEntry)) genSetFrom(new(types.ObjectVersion), new(types.DeleteMarkerEntry))
genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput)) genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput))
genSetFrom(new(s3.Object), new(s3.ObjectVersion)) genSetFrom(new(types.Object), new(types.ObjectVersion))
genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput)) genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput))
genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput)) genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput))
genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput)) genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput))

863
backend/s3/s3.go
View File

File diff suppressed because it is too large Load Diff

73
backend/s3/s3_internal_test.go
View File

@ -5,15 +5,17 @@ import (
"compress/gzip" "compress/gzip"
"context" "context"
"crypto/md5" "crypto/md5"
"errors"
"fmt" "fmt"
"path" "path"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/hash"
@ -131,20 +133,20 @@ func TestVersionLess(t *testing.T) {
t1 := fstest.Time("2022-01-21T12:00:00+01:00") t1 := fstest.Time("2022-01-21T12:00:00+01:00")
t2 := fstest.Time("2022-01-21T12:00:01+01:00") t2 := fstest.Time("2022-01-21T12:00:01+01:00")
for n, test := range []struct { for n, test := range []struct {
a, b *s3.ObjectVersion a, b *types.ObjectVersion
want bool want bool
}{ }{
{a: nil, b: nil, want: true}, {a: nil, b: nil, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false},
{a: nil, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, {a: nil, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, want: false}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t2}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t2}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, want: true}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key2, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, {a: &types.ObjectVersion{Key: &key2, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false}, {a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false},
} { } {
got := versionLess(test.a, test.b) got := versionLess(test.a, test.b)
assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test)) assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
@ -157,24 +159,24 @@ func TestMergeDeleteMarkers(t *testing.T) {
t1 := fstest.Time("2022-01-21T12:00:00+01:00") t1 := fstest.Time("2022-01-21T12:00:00+01:00")
t2 := fstest.Time("2022-01-21T12:00:01+01:00") t2 := fstest.Time("2022-01-21T12:00:01+01:00")
for n, test := range []struct { for n, test := range []struct {
versions []*s3.ObjectVersion versions []types.ObjectVersion
markers []*s3.DeleteMarkerEntry markers []types.DeleteMarkerEntry
want []*s3.ObjectVersion want []types.ObjectVersion
}{ }{
{ {
versions: []*s3.ObjectVersion{}, versions: []types.ObjectVersion{},
markers: []*s3.DeleteMarkerEntry{}, markers: []types.DeleteMarkerEntry{},
want: []*s3.ObjectVersion{}, want: []types.ObjectVersion{},
}, },
{ {
versions: []*s3.ObjectVersion{ versions: []types.ObjectVersion{
{ {
Key: &key1, Key: &key1,
LastModified: &t1, LastModified: &t1,
}, },
}, },
markers: []*s3.DeleteMarkerEntry{}, markers: []types.DeleteMarkerEntry{},
want: []*s3.ObjectVersion{ want: []types.ObjectVersion{
{ {
Key: &key1, Key: &key1,
LastModified: &t1, LastModified: &t1,
@ -182,14 +184,14 @@ func TestMergeDeleteMarkers(t *testing.T) {
}, },
}, },
{ {
versions: []*s3.ObjectVersion{}, versions: []types.ObjectVersion{},
markers: []*s3.DeleteMarkerEntry{ markers: []types.DeleteMarkerEntry{
{ {
Key: &key1, Key: &key1,
LastModified: &t1, LastModified: &t1,
}, },
}, },
want: []*s3.ObjectVersion{ want: []types.ObjectVersion{
{ {
Key: &key1, Key: &key1,
LastModified: &t1, LastModified: &t1,
@ -198,7 +200,7 @@ func TestMergeDeleteMarkers(t *testing.T) {
}, },
}, },
{ {
versions: []*s3.ObjectVersion{ versions: []types.ObjectVersion{
{ {
Key: &key1, Key: &key1,
LastModified: &t2, LastModified: &t2,
@ -208,13 +210,13 @@ func TestMergeDeleteMarkers(t *testing.T) {
LastModified: &t2, LastModified: &t2,
}, },
}, },
markers: []*s3.DeleteMarkerEntry{ markers: []types.DeleteMarkerEntry{
{ {
Key: &key1, Key: &key1,
LastModified: &t1, LastModified: &t1,
}, },
}, },
want: []*s3.ObjectVersion{ want: []types.ObjectVersion{
{ {
Key: &key1, Key: &key1,
LastModified: &t2, LastModified: &t2,
@ -399,22 +401,23 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
// quirk is set correctly // quirk is set correctly
req := s3.CreateBucketInput{ req := s3.CreateBucketInput{
Bucket: &f.rootBucket, Bucket: &f.rootBucket,
ACL: stringPointerOrNil(f.opt.BucketACL), ACL: types.BucketCannedACL(f.opt.BucketACL),
} }
if f.opt.LocationConstraint != "" { if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{ req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
LocationConstraint: &f.opt.LocationConstraint, LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
} }
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
_, err := f.c.CreateBucketWithContext(ctx, &req) _, err := f.c.CreateBucket(ctx, &req)
return f.shouldRetry(ctx, err) return f.shouldRetry(ctx, err)
}) })
var errString string var errString string
var awsError smithy.APIError
if err == nil { if err == nil {
errString = "No Error" errString = "No Error"
} else if awsErr, ok := err.(awserr.Error); ok { } else if errors.As(err, &awsError) {
errString = awsErr.Code() errString = awsError.ErrorCode()
} else { } else {
assert.Fail(t, "Unknown error %T %v", err, err) assert.Fail(t, "Unknown error %T %v", err, err)
} }

20
backend/s3/s3_test.go
View File

@ -4,12 +4,14 @@ package s3
import ( import (
"context" "context"
"net/http" "net/http"
"strings"
"testing" "testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests" "github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) { func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) {
@ -54,20 +56,16 @@ func TestAWSDualStackOption(t *testing.T) {
// test enabled // test enabled
ctx, opt, client := SetupS3Test(t) ctx, opt, client := SetupS3Test(t)
opt.UseDualStack = true opt.UseDualStack = true
s3Conn, _, _ := s3Connection(ctx, opt, client) s3Conn, err := s3Connection(ctx, opt, client)
if !strings.Contains(s3Conn.Endpoint, "dualstack") { require.NoError(t, err)
t.Errorf("dualstack failed got: %s, wanted: dualstack", s3Conn.Endpoint) assert.Equal(t, aws.DualStackEndpointStateEnabled, s3Conn.Options().EndpointOptions.UseDualStackEndpoint)
t.Fail()
}
} }
{ {
// test default case // test default case
ctx, opt, client := SetupS3Test(t) ctx, opt, client := SetupS3Test(t)
s3Conn, _, _ := s3Connection(ctx, opt, client) s3Conn, err := s3Connection(ctx, opt, client)
if strings.Contains(s3Conn.Endpoint, "dualstack") { require.NoError(t, err)
t.Errorf("dualstack failed got: %s, NOT wanted: dualstack", s3Conn.Endpoint) assert.Equal(t, aws.DualStackEndpointStateDisabled, s3Conn.Options().EndpointOptions.UseDualStackEndpoint)
t.Fail()
}
} }
} }

34
backend/s3/setfrom.go
View File

@ -2,7 +2,10 @@
package s3 package s3
import "github.com/aws/aws-sdk-go/service/s3" import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b // setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b
func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) { func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) {
@ -27,6 +30,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output
a.Name = b.Name a.Name = b.Name
a.Prefix = b.Prefix a.Prefix = b.Prefix
a.RequestCharged = b.RequestCharged a.RequestCharged = b.RequestCharged
a.ResultMetadata = b.ResultMetadata
} }
// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b // setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b
@ -41,8 +45,8 @@ func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVers
a.RequestPayer = b.RequestPayer a.RequestPayer = b.RequestPayer
} }
// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b // setFrom_typesObjectVersion_typesDeleteMarkerEntry copies matching elements from a to b
func setFrom_s3ObjectVersion_s3DeleteMarkerEntry(a *s3.ObjectVersion, b *s3.DeleteMarkerEntry) { func setFrom_typesObjectVersion_typesDeleteMarkerEntry(a *types.ObjectVersion, b *types.DeleteMarkerEntry) {
a.IsLatest = b.IsLatest a.IsLatest = b.IsLatest
a.Key = b.Key a.Key = b.Key
a.LastModified = b.LastModified a.LastModified = b.LastModified
@ -60,10 +64,11 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV
a.Name = b.Name a.Name = b.Name
a.Prefix = b.Prefix a.Prefix = b.Prefix
a.RequestCharged = b.RequestCharged a.RequestCharged = b.RequestCharged
a.ResultMetadata = b.ResultMetadata
} }
// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b // setFrom_typesObject_typesObjectVersion copies matching elements from a to b
func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) { func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersion) {
a.ChecksumAlgorithm = b.ChecksumAlgorithm a.ChecksumAlgorithm = b.ChecksumAlgorithm
a.ETag = b.ETag a.ETag = b.ETag
a.Key = b.Key a.Key = b.Key
@ -71,7 +76,6 @@ func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {
a.Owner = b.Owner a.Owner = b.Owner
a.RestoreStatus = b.RestoreStatus a.RestoreStatus = b.RestoreStatus
a.Size = b.Size a.Size = b.Size
a.StorageClass = b.StorageClass
} }
// setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b // setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b
@ -82,6 +86,7 @@ func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipa
a.ContentEncoding = b.ContentEncoding a.ContentEncoding = b.ContentEncoding
a.ContentLanguage = b.ContentLanguage a.ContentLanguage = b.ContentLanguage
a.ContentType = b.ContentType a.ContentType = b.ContentType
a.Expires = b.Expires
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode a.ObjectLockMode = b.ObjectLockMode
@ -96,8 +101,9 @@ func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipa
// setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b // setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b
func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) { func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm a.ChecksumAlgorithm = b.ChecksumAlgorithm
@ -111,7 +117,6 @@ func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipar
a.GrantRead = b.GrantRead a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode a.ObjectLockMode = b.ObjectLockMode
@ -132,6 +137,7 @@ func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipar
func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) { func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) {
a.Bucket = b.Bucket a.Bucket = b.Bucket
a.CopySource = b.CopySource a.CopySource = b.CopySource
a.Key = b.Key
a.CopySourceIfMatch = b.CopySourceIfMatch a.CopySourceIfMatch = b.CopySourceIfMatch
a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince
a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch
@ -141,7 +147,6 @@ func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput,
a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5 a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5
a.ExpectedBucketOwner = b.ExpectedBucketOwner a.ExpectedBucketOwner = b.ExpectedBucketOwner
a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner
a.Key = b.Key
a.RequestPayer = b.RequestPayer a.RequestPayer = b.RequestPayer
a.SSECustomerAlgorithm = b.SSECustomerAlgorithm a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
a.SSECustomerKey = b.SSECustomerKey a.SSECustomerKey = b.SSECustomerKey
@ -166,6 +171,7 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.ETag = b.ETag a.ETag = b.ETag
a.Expiration = b.Expiration a.Expiration = b.Expiration
a.Expires = b.Expires a.Expires = b.Expires
a.ExpiresString = b.ExpiresString
a.LastModified = b.LastModified a.LastModified = b.LastModified
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.MissingMeta = b.MissingMeta a.MissingMeta = b.MissingMeta
@ -183,12 +189,14 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.StorageClass = b.StorageClass a.StorageClass = b.StorageClass
a.VersionId = b.VersionId a.VersionId = b.VersionId
a.WebsiteRedirectLocation = b.WebsiteRedirectLocation a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
a.ResultMetadata = b.ResultMetadata
} }
// setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b // setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b
func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) { func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm a.ChecksumAlgorithm = b.ChecksumAlgorithm
@ -202,7 +210,6 @@ func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipart
a.GrantRead = b.GrantRead a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode a.ObjectLockMode = b.ObjectLockMode
@ -232,6 +239,7 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
a.ContentLanguage = b.ContentLanguage a.ContentLanguage = b.ContentLanguage
a.ContentLength = b.ContentLength a.ContentLength = b.ContentLength
a.ContentType = b.ContentType a.ContentType = b.ContentType
a.Expires = b.Expires
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode a.ObjectLockMode = b.ObjectLockMode
@ -246,8 +254,9 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
// setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b // setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b
func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) { func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm a.ChecksumAlgorithm = b.ChecksumAlgorithm
@ -261,7 +270,6 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Put
a.GrantRead = b.GrantRead a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode a.ObjectLockMode = b.ObjectLockMode

22
backend/s3/v2sign.go
View File

@ -3,6 +3,7 @@
package s3 package s3
import ( import (
"context"
"crypto/hmac" "crypto/hmac"
"crypto/sha1" "crypto/sha1"
"encoding/base64" "encoding/base64"
@ -10,6 +11,9 @@ import (
"sort" "sort"
"strings" "strings"
"time" "time"
"github.com/aws/aws-sdk-go-v2/aws"
v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
) )
// URL parameters that need to be added to the signature // URL parameters that need to be added to the signature
@ -36,10 +40,17 @@ var s3ParamsToSign = map[string]struct{}{
"response-content-encoding": {}, "response-content-encoding": {},
} }
// sign signs requests using v2 auth // Implement HTTPSignerV4 interface
type v2Signer struct {
opt *Options
}
// SignHTTP signs requests using v2 auth.
// //
// Cobbled together from goamz and aws-sdk-go // Cobbled together from goamz and aws-sdk-go.
func sign(AccessKey, SecretKey string, req *http.Request) { //
// Bodged up to compile with AWS SDK v2
func (v2 *v2Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error {
// Set date // Set date
date := time.Now().UTC().Format(time.RFC1123) date := time.Now().UTC().Format(time.RFC1123)
req.Header.Set("Date", date) req.Header.Set("Date", date)
@ -107,11 +118,12 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
// Make signature // Make signature
payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
hash := hmac.New(sha1.New, []byte(SecretKey)) hash := hmac.New(sha1.New, []byte(v2.opt.SecretAccessKey))
_, _ = hash.Write([]byte(payload)) _, _ = hash.Write([]byte(payload))
signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size())) signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
base64.StdEncoding.Encode(signature, hash.Sum(nil)) base64.StdEncoding.Encode(signature, hash.Sum(nil))
// Set signature in request // Set signature in request
req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature)) req.Header.Set("Authorization", "AWS "+v2.opt.AccessKeyID+":"+string(signature))
return nil
} }

29
docs/content/s3.md
View File

@ -571,15 +571,21 @@ The different authentication methods are tried in this order:
- Session Token: `AWS_SESSION_TOKEN` (optional) - Session Token: `AWS_SESSION_TOKEN` (optional)
- Or, use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html): - Or, use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html):
- Profile files are standard files used by AWS CLI tools - Profile files are standard files used by AWS CLI tools
- By default it will use the profile in your home directory (e.g. `~/.aws/credentials` on unix based systems) file and the "default" profile, to change set these environment variables: - By default it will use the profile in your home directory (e.g. `~/.aws/credentials` on unix based systems) file and the "default" profile, to change set these environment variables or config keys:
- `AWS_SHARED_CREDENTIALS_FILE` to control which file. - `AWS_SHARED_CREDENTIALS_FILE` to control which file or the `shared_credentials_file` config key.
- `AWS_PROFILE` to control which profile to use. - `AWS_PROFILE` to control which profile to use or the `profile` config key.
- Or, run `rclone` in an ECS task with an IAM role (AWS only). - Or, run `rclone` in an ECS task with an IAM role (AWS only).
- Or, run `rclone` on an EC2 instance with an IAM role (AWS only). - Or, run `rclone` on an EC2 instance with an IAM role (AWS only).
- Or, run `rclone` in an EKS pod with an IAM role that is associated with a service account (AWS only). - Or, run `rclone` in an EKS pod with an IAM role that is associated with a service account (AWS only).
- Or, use [process credentials](https://docs.aws.amazon.com/sdkref/latest/guide/feature-process-credentials.html) to read config from an external program.
With `env_auth = true` rclone (which uses the SDK for Go v2) should support
[all authentication methods](https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html)
that the `aws` CLI tool does and the other AWS SDKs.
If none of these option actually end up providing `rclone` with AWS If none of these option actually end up providing `rclone` with AWS
credentials then S3 interaction will be non-authenticated (see below). credentials then S3 interaction will be non-authenticated (see the
[anonymous access](#anonymous-access) section for more info).
### S3 Permissions ### S3 Permissions
@ -2171,7 +2177,7 @@ It doesn't return anything.
{{< rem autogenerated options stop >}} {{< rem autogenerated options stop >}}
### Anonymous access to public buckets ### Anonymous access to public buckets {#anonymous-access}
If you want to use rclone to access a public bucket, configure with a If you want to use rclone to access a public bucket, configure with a
blank `access_key_id` and `secret_access_key`. Your config should end blank `access_key_id` and `secret_access_key`. Your config should end
@ -2181,15 +2187,6 @@ up looking like this:
[anons3] [anons3]
type = s3 type = s3
provider = AWS provider = AWS
env_auth = false
access_key_id =
secret_access_key =
region = us-east-1
endpoint =
location_constraint =
acl = private
server_side_encryption =
storage_class =
``` ```
Then use it as normal with the name of the public bucket, e.g. Then use it as normal with the name of the public bucket, e.g.
@ -2198,6 +2195,10 @@ Then use it as normal with the name of the public bucket, e.g.
You will be able to list and copy data but not upload it. You will be able to list and copy data but not upload it.
You can also do this entirely on the command line
rclone lsd :s3,provider=AWS:1000genomes
## Providers ## Providers
### AWS S3 ### AWS S3

19
go.mod
View File

@ -18,6 +18,12 @@ require (
github.com/anacrolix/log v0.15.2 github.com/anacrolix/log v0.15.2
github.com/atotto/clipboard v0.1.4 github.com/atotto/clipboard v0.1.4
github.com/aws/aws-sdk-go v1.54.19 github.com/aws/aws-sdk-go v1.54.19
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/config v1.27.27
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3
github.com/aws/smithy-go v1.20.3
github.com/buengese/sgzip v0.1.1 github.com/buengese/sgzip v0.1.1
github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891
github.com/colinmarc/hdfs/v2 v2.4.0 github.com/colinmarc/hdfs/v2 v2.4.0
@ -99,6 +105,19 @@ require (
github.com/akavel/rsrc v0.10.2 // indirect github.com/akavel/rsrc v0.10.2 // indirect
github.com/anacrolix/generics v0.0.1 // indirect github.com/anacrolix/generics v0.0.1 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bradenaw/juniper v0.15.2 // indirect github.com/bradenaw/juniper v0.15.2 // indirect
github.com/calebcase/tmpfile v1.0.3 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect

38
go.sum
View File

@ -104,6 +104,44 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 h1:zeN9UtUlA6FTx0vFSayxSX32HDw73Yb6Hh2izDSFxXY=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10/go.mod h1:3HKuexPDcwLWPaqpW2UR/9n8N/u/3CKcGAzSs8p8u8g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U=