2015-09-23 01:47:16 +08:00
|
|
|
// Package swift provides an interface to the Swift object storage system
|
2013-06-28 03:13:07 +08:00
|
|
|
package swift
|
2012-12-26 20:23:58 +08:00
|
|
|
|
|
|
|
import (
|
2017-09-14 13:42:16 +08:00
|
|
|
"bufio"
|
2015-09-23 14:57:48 +08:00
|
|
|
"bytes"
|
2019-06-17 16:34:30 +08:00
|
|
|
"context"
|
2021-11-04 18:12:57 +08:00
|
|
|
"errors"
|
2012-12-26 20:23:58 +08:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2019-11-21 21:25:02 +08:00
|
|
|
"net/url"
|
2014-05-06 02:52:52 +08:00
|
|
|
"path"
|
2015-10-03 22:38:35 +08:00
|
|
|
"strconv"
|
2012-12-26 20:23:58 +08:00
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2021-03-02 16:58:14 +08:00
|
|
|
"github.com/google/uuid"
|
2021-01-23 01:23:51 +08:00
|
|
|
"github.com/ncw/swift/v2"
|
2019-07-29 01:47:38 +08:00
|
|
|
"github.com/rclone/rclone/fs"
|
2020-01-15 01:33:35 +08:00
|
|
|
"github.com/rclone/rclone/fs/config"
|
2019-07-29 01:47:38 +08:00
|
|
|
"github.com/rclone/rclone/fs/config/configmap"
|
|
|
|
"github.com/rclone/rclone/fs/config/configstruct"
|
|
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
|
|
"github.com/rclone/rclone/fs/fshttp"
|
|
|
|
"github.com/rclone/rclone/fs/hash"
|
|
|
|
"github.com/rclone/rclone/fs/operations"
|
|
|
|
"github.com/rclone/rclone/fs/walk"
|
2021-01-29 01:09:41 +08:00
|
|
|
"github.com/rclone/rclone/lib/atexit"
|
2019-08-07 05:17:40 +08:00
|
|
|
"github.com/rclone/rclone/lib/bucket"
|
2020-01-15 01:33:35 +08:00
|
|
|
"github.com/rclone/rclone/lib/encoder"
|
2019-07-29 01:47:38 +08:00
|
|
|
"github.com/rclone/rclone/lib/pacer"
|
2019-08-08 00:21:49 +08:00
|
|
|
"github.com/rclone/rclone/lib/readers"
|
2015-09-23 14:57:48 +08:00
|
|
|
)
|
|
|
|
|
2015-11-07 23:17:46 +08:00
|
|
|
// Constants
|
|
|
|
const (
|
|
|
|
directoryMarkerContentType = "application/directory" // content type of directory marker objects
|
2017-06-06 22:03:52 +08:00
|
|
|
listChunks = 1000 // chunk size to read directory listings
|
2021-03-03 03:11:57 +08:00
|
|
|
defaultChunkSize = 5 * fs.Gibi
|
2018-11-23 06:15:52 +08:00
|
|
|
minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep.
|
2015-11-07 23:17:46 +08:00
|
|
|
)
|
|
|
|
|
2022-10-05 00:10:00 +08:00
|
|
|
// SharedOptions are shared between swift and backends which depend on swift
|
2018-09-05 03:27:48 +08:00
|
|
|
var SharedOptions = []fs.Option{{
|
2018-10-02 01:36:15 +08:00
|
|
|
Name: "chunk_size",
|
|
|
|
Help: `Above this size files will be chunked into a _segments container.
|
|
|
|
|
|
|
|
Above this size files will be chunked into a _segments container. The
|
2021-03-03 03:11:57 +08:00
|
|
|
default for this is 5 GiB which is its maximum value.`,
|
2018-09-07 19:02:27 +08:00
|
|
|
Default: defaultChunkSize,
|
2018-09-05 03:27:48 +08:00
|
|
|
Advanced: true,
|
2018-11-28 23:51:18 +08:00
|
|
|
}, {
|
|
|
|
Name: "no_chunk",
|
|
|
|
Help: `Don't chunk files during streaming upload.
|
|
|
|
|
2020-10-14 05:49:58 +08:00
|
|
|
When doing streaming uploads (e.g. using rcat or mount) setting this
|
2018-11-28 23:51:18 +08:00
|
|
|
flag will cause the swift backend to not upload chunked files.
|
|
|
|
|
2021-03-03 03:11:57 +08:00
|
|
|
This will limit the maximum upload size to 5 GiB. However non chunked
|
2018-11-28 23:51:18 +08:00
|
|
|
files are easier to deal with and have an MD5SUM.
|
|
|
|
|
|
|
|
Rclone will still chunk files bigger than chunk_size when doing normal
|
|
|
|
copy operations.`,
|
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2022-10-06 00:39:24 +08:00
|
|
|
}, {
|
|
|
|
Name: "no_large_objects",
|
|
|
|
Help: strings.ReplaceAll(`Disable support for static and dynamic large objects
|
|
|
|
|
|
|
|
Swift cannot transparently store files bigger than 5 GiB. There are
|
|
|
|
two schemes for doing that, static or dynamic large objects, and the
|
|
|
|
API does not allow rclone to determine whether a file is a static or
|
|
|
|
dynamic large object without doing a HEAD on the object. Since these
|
|
|
|
need to be treated differently, this means rclone has to issue HEAD
|
|
|
|
requests for objects for example when reading checksums.
|
|
|
|
|
|
|
|
When |no_large_objects| is set, rclone will assume that there are no
|
|
|
|
static or dynamic large objects stored. This means it can stop doing
|
|
|
|
the extra HEAD calls which in turn increases performance greatly
|
|
|
|
especially when doing a swift to swift transfer with |--checksum| set.
|
|
|
|
|
|
|
|
Setting this option implies |no_chunk| and also that no files will be
|
|
|
|
uploaded in chunks, so files bigger than 5 GiB will just fail on
|
|
|
|
upload.
|
|
|
|
|
|
|
|
If you set this option and there *are* static or dynamic large objects,
|
|
|
|
then this will give incorrect hashes for them. Downloads will succeed,
|
|
|
|
but other operations such as Remove and Copy will fail.
|
|
|
|
`, "|", "`"),
|
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2020-01-15 01:33:35 +08:00
|
|
|
}, {
|
|
|
|
Name: config.ConfigEncoding,
|
|
|
|
Help: config.ConfigEncodingHelp,
|
|
|
|
Advanced: true,
|
2020-01-15 05:51:49 +08:00
|
|
|
Default: (encoder.EncodeInvalidUtf8 |
|
|
|
|
encoder.EncodeSlash),
|
2018-09-05 03:27:48 +08:00
|
|
|
}}
|
|
|
|
|
2013-06-28 03:13:07 +08:00
|
|
|
// Register with Fs
|
|
|
|
func init() {
|
2016-02-18 19:35:25 +08:00
|
|
|
fs.Register(&fs.RegInfo{
|
2016-02-16 02:11:53 +08:00
|
|
|
Name: "swift",
|
2020-05-20 18:54:33 +08:00
|
|
|
Description: "OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
|
2016-02-16 02:11:53 +08:00
|
|
|
NewFs: NewFs,
|
2018-09-05 03:27:48 +08:00
|
|
|
Options: append([]fs.Option{{
|
2018-05-15 01:06:57 +08:00
|
|
|
Name: "env_auth",
|
|
|
|
Help: "Get swift credentials from environment variables in standard OpenStack form.",
|
|
|
|
Default: false,
|
2017-08-11 04:38:45 +08:00
|
|
|
Examples: []fs.OptionExample{
|
|
|
|
{
|
|
|
|
Value: "false",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Enter swift credentials in the next step.",
|
2017-08-11 04:38:45 +08:00
|
|
|
}, {
|
|
|
|
Value: "true",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Get swift credentials from environment vars.\nLeave other fields blank if using this.",
|
2017-08-11 04:38:45 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}, {
|
2014-03-16 00:06:11 +08:00
|
|
|
Name: "user",
|
2017-11-02 05:26:04 +08:00
|
|
|
Help: "User name to log in (OS_USERNAME).",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Name: "key",
|
2017-11-02 05:26:04 +08:00
|
|
|
Help: "API key or password (OS_PASSWORD).",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Name: "auth",
|
2017-11-02 05:26:04 +08:00
|
|
|
Help: "Authentication URL for server (OS_AUTH_URL).",
|
2014-03-16 00:06:11 +08:00
|
|
|
Examples: []fs.OptionExample{{
|
|
|
|
Value: "https://auth.api.rackspacecloud.com/v1.0",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Rackspace US",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Rackspace UK",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Value: "https://identity.api.rackspacecloud.com/v2.0",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Rackspace v2",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Value: "https://auth.storage.memset.com/v1.0",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Memset Memstore UK",
|
2014-03-16 00:06:11 +08:00
|
|
|
}, {
|
|
|
|
Value: "https://auth.storage.memset.com/v2.0",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Memset Memstore UK v2",
|
2016-01-07 21:15:10 +08:00
|
|
|
}, {
|
2020-01-25 01:43:00 +08:00
|
|
|
Value: "https://auth.cloud.ovh.net/v3",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "OVH",
|
2014-03-16 00:06:11 +08:00
|
|
|
}},
|
2017-11-02 05:26:04 +08:00
|
|
|
}, {
|
|
|
|
Name: "user_id",
|
|
|
|
Help: "User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).",
|
2016-04-21 07:00:49 +08:00
|
|
|
}, {
|
|
|
|
Name: "domain",
|
2017-11-02 05:26:04 +08:00
|
|
|
Help: "User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)",
|
2014-11-24 20:05:10 +08:00
|
|
|
}, {
|
|
|
|
Name: "tenant",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME).",
|
2017-11-02 22:49:07 +08:00
|
|
|
}, {
|
|
|
|
Name: "tenant_id",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID).",
|
2016-07-06 00:34:22 +08:00
|
|
|
}, {
|
|
|
|
Name: "tenant_domain",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME).",
|
2015-03-05 01:09:53 +08:00
|
|
|
}, {
|
|
|
|
Name: "region",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Region name - optional (OS_REGION_NAME).",
|
2016-02-03 03:45:05 +08:00
|
|
|
}, {
|
|
|
|
Name: "storage_url",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Storage URL - optional (OS_STORAGE_URL).",
|
2017-12-08 05:30:58 +08:00
|
|
|
}, {
|
|
|
|
Name: "auth_token",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN).",
|
2019-01-29 00:55:17 +08:00
|
|
|
}, {
|
|
|
|
Name: "application_credential_id",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID).",
|
2019-01-29 00:55:17 +08:00
|
|
|
}, {
|
|
|
|
Name: "application_credential_name",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME).",
|
2019-01-29 00:55:17 +08:00
|
|
|
}, {
|
|
|
|
Name: "application_credential_secret",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET).",
|
2016-06-04 00:52:02 +08:00
|
|
|
}, {
|
2018-05-15 01:06:57 +08:00
|
|
|
Name: "auth_version",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION).",
|
2018-05-15 01:06:57 +08:00
|
|
|
Default: 0,
|
2017-08-11 04:38:45 +08:00
|
|
|
}, {
|
2018-05-15 01:06:57 +08:00
|
|
|
Name: "endpoint_type",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE).",
|
2018-05-15 01:06:57 +08:00
|
|
|
Default: "public",
|
2017-08-11 04:38:45 +08:00
|
|
|
Examples: []fs.OptionExample{{
|
|
|
|
Value: "public",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Public (default, choose this if not sure)",
|
2017-08-11 04:38:45 +08:00
|
|
|
}, {
|
|
|
|
Value: "internal",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Internal (use internal service net)",
|
2017-08-11 04:38:45 +08:00
|
|
|
}, {
|
|
|
|
Value: "admin",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Admin",
|
2017-08-11 04:38:45 +08:00
|
|
|
}},
|
2021-01-29 01:09:41 +08:00
|
|
|
}, {
|
2021-08-16 17:30:01 +08:00
|
|
|
Name: "leave_parts_on_error",
|
|
|
|
Help: `If true avoid calling abort upload on a failure.
|
|
|
|
|
|
|
|
It should be set to true for resuming uploads across different sessions.`,
|
2021-01-29 01:09:41 +08:00
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2018-08-05 04:09:17 +08:00
|
|
|
}, {
|
2018-10-02 01:36:15 +08:00
|
|
|
Name: "storage_policy",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: `The storage policy to use when creating a new container.
|
2018-10-02 01:36:15 +08:00
|
|
|
|
|
|
|
This applies the specified storage policy when creating a new
|
|
|
|
container. The policy cannot be changed afterwards. The allowed
|
|
|
|
configuration values and their meaning depend on your Swift storage
|
|
|
|
provider.`,
|
2018-08-05 04:09:17 +08:00
|
|
|
Default: "",
|
|
|
|
Examples: []fs.OptionExample{{
|
|
|
|
Value: "",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "Default",
|
2018-08-05 04:09:17 +08:00
|
|
|
}, {
|
|
|
|
Value: "pcs",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "OVH Public Cloud Storage",
|
2018-08-05 04:09:17 +08:00
|
|
|
}, {
|
|
|
|
Value: "pca",
|
2021-08-16 17:30:01 +08:00
|
|
|
Help: "OVH Public Cloud Archive",
|
2018-08-05 04:09:17 +08:00
|
|
|
}},
|
2018-09-05 03:27:48 +08:00
|
|
|
}}, SharedOptions...),
|
2014-03-16 00:06:11 +08:00
|
|
|
})
|
2018-05-15 01:06:57 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Options defines the configuration for this backend
|
|
|
|
type Options struct {
|
2020-01-15 01:33:35 +08:00
|
|
|
EnvAuth bool `config:"env_auth"`
|
|
|
|
User string `config:"user"`
|
|
|
|
Key string `config:"key"`
|
|
|
|
Auth string `config:"auth"`
|
|
|
|
UserID string `config:"user_id"`
|
|
|
|
Domain string `config:"domain"`
|
|
|
|
Tenant string `config:"tenant"`
|
|
|
|
TenantID string `config:"tenant_id"`
|
|
|
|
TenantDomain string `config:"tenant_domain"`
|
|
|
|
Region string `config:"region"`
|
|
|
|
StorageURL string `config:"storage_url"`
|
|
|
|
AuthToken string `config:"auth_token"`
|
|
|
|
AuthVersion int `config:"auth_version"`
|
|
|
|
ApplicationCredentialID string `config:"application_credential_id"`
|
|
|
|
ApplicationCredentialName string `config:"application_credential_name"`
|
|
|
|
ApplicationCredentialSecret string `config:"application_credential_secret"`
|
2021-01-29 01:09:41 +08:00
|
|
|
LeavePartsOnError bool `config:"leave_parts_on_error"`
|
2020-01-15 01:33:35 +08:00
|
|
|
StoragePolicy string `config:"storage_policy"`
|
|
|
|
EndpointType string `config:"endpoint_type"`
|
|
|
|
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
|
|
|
NoChunk bool `config:"no_chunk"`
|
2022-10-06 00:39:24 +08:00
|
|
|
NoLargeObjects bool `config:"no_large_objects"`
|
2020-01-15 01:33:35 +08:00
|
|
|
Enc encoder.MultiEncoder `config:"encoding"`
|
2013-06-28 03:13:07 +08:00
|
|
|
}
|
|
|
|
|
2015-11-07 19:14:46 +08:00
|
|
|
// Fs represents a remote swift server
|
|
|
|
type Fs struct {
|
2019-08-07 05:17:40 +08:00
|
|
|
name string // name of this remote
|
|
|
|
root string // the path we are working on if any
|
|
|
|
features *fs.Features // optional features
|
|
|
|
opt Options // options for this backend
|
2020-11-05 19:33:32 +08:00
|
|
|
ci *fs.ConfigInfo // global config
|
2019-08-07 05:17:40 +08:00
|
|
|
c *swift.Connection // the connection to the swift server
|
|
|
|
rootContainer string // container part of root (if any)
|
|
|
|
rootDirectory string // directory part of root (if any)
|
|
|
|
cache *bucket.Cache // cache of container status
|
|
|
|
noCheckContainer bool // don't check the container before creating it
|
|
|
|
pacer *fs.Pacer // To pace the API calls
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2015-11-07 19:14:46 +08:00
|
|
|
// Object describes a swift object
|
2012-12-26 20:23:58 +08:00
|
|
|
//
|
|
|
|
// Will definitely have info but maybe not meta
|
2015-11-07 19:14:46 +08:00
|
|
|
type Object struct {
|
2018-12-02 18:23:55 +08:00
|
|
|
fs *Fs // what this object is part of
|
|
|
|
remote string // The remote path
|
|
|
|
size int64
|
|
|
|
lastModified time.Time
|
|
|
|
contentType string
|
|
|
|
md5 string
|
|
|
|
headers swift.Headers // The object headers if known
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Name of the remote (as passed into NewFs)
|
2015-11-07 19:14:46 +08:00
|
|
|
func (f *Fs) Name() string {
|
2015-08-22 23:53:11 +08:00
|
|
|
return f.name
|
|
|
|
}
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Root of the remote (as passed into NewFs)
|
2015-11-07 19:14:46 +08:00
|
|
|
func (f *Fs) Root() string {
|
2019-08-07 05:17:40 +08:00
|
|
|
return f.root
|
2015-09-02 03:45:27 +08:00
|
|
|
}
|
|
|
|
|
2015-11-07 19:14:46 +08:00
|
|
|
// String converts this Fs to a string
|
|
|
|
func (f *Fs) String() string {
|
2019-08-07 05:17:40 +08:00
|
|
|
if f.rootContainer == "" {
|
2022-06-09 04:25:17 +08:00
|
|
|
return "Swift root"
|
2019-08-07 05:17:40 +08:00
|
|
|
}
|
|
|
|
if f.rootDirectory == "" {
|
|
|
|
return fmt.Sprintf("Swift container %s", f.rootContainer)
|
2014-05-06 00:30:55 +08:00
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
return fmt.Sprintf("Swift container %s path %s", f.rootContainer, f.rootDirectory)
|
2013-01-01 00:40:34 +08:00
|
|
|
}
|
|
|
|
|
2017-01-14 01:21:47 +08:00
|
|
|
// Features returns the optional features of this Fs
|
|
|
|
func (f *Fs) Features() *fs.Features {
|
|
|
|
return f.features
|
|
|
|
}
|
|
|
|
|
2018-11-23 06:15:52 +08:00
|
|
|
// retryErrorCodes is a slice of error codes that we will retry
|
|
|
|
var retryErrorCodes = []int{
|
2020-10-14 05:49:58 +08:00
|
|
|
401, // Unauthorized (e.g. "Token has expired")
|
2018-11-23 06:15:52 +08:00
|
|
|
408, // Request Timeout
|
|
|
|
409, // Conflict - various states that could be resolved on a retry
|
|
|
|
429, // Rate exceeded.
|
|
|
|
500, // Get occasional 500 Internal Server Error
|
|
|
|
503, // Service Unavailable/Slow Down - "Reduce your request rate"
|
|
|
|
504, // Gateway Time-out
|
|
|
|
}
|
|
|
|
|
|
|
|
// shouldRetry returns a boolean as to whether this err deserves to be
|
|
|
|
// retried. It returns the err as a convenience
|
2021-03-16 23:50:02 +08:00
|
|
|
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
|
|
|
if fserrors.ContextError(ctx, &err) {
|
|
|
|
return false, err
|
|
|
|
}
|
2020-05-20 18:39:20 +08:00
|
|
|
// If this is a swift.Error object extract the HTTP error code
|
2018-11-23 06:15:52 +08:00
|
|
|
if swiftError, ok := err.(*swift.Error); ok {
|
|
|
|
for _, e := range retryErrorCodes {
|
|
|
|
if swiftError.StatusCode == e {
|
|
|
|
return true, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Check for generic failure conditions
|
|
|
|
return fserrors.ShouldRetry(err), err
|
|
|
|
}
|
|
|
|
|
2019-03-17 19:38:25 +08:00
|
|
|
// shouldRetryHeaders returns a boolean as to whether this err
|
|
|
|
// deserves to be retried. It reads the headers passed in looking for
|
|
|
|
// `Retry-After`. It returns the err as a convenience
|
2021-03-16 23:50:02 +08:00
|
|
|
func shouldRetryHeaders(ctx context.Context, headers swift.Headers, err error) (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
|
|
|
|
if value := headers["Retry-After"]; value != "" {
|
|
|
|
retryAfter, parseErr := strconv.Atoi(value)
|
|
|
|
if parseErr != nil {
|
|
|
|
fs.Errorf(nil, "Failed to parse Retry-After: %q: %v", value, parseErr)
|
|
|
|
} else {
|
|
|
|
duration := time.Second * time.Duration(retryAfter)
|
|
|
|
if duration <= 60*time.Second {
|
|
|
|
// Do a short sleep immediately
|
|
|
|
fs.Debugf(nil, "Sleeping for %v to obey Retry-After", duration)
|
|
|
|
time.Sleep(duration)
|
|
|
|
return true, err
|
|
|
|
}
|
|
|
|
// Delay a long sleep for a retry
|
|
|
|
return false, fserrors.NewErrorRetryAfter(duration)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2019-03-17 19:38:25 +08:00
|
|
|
}
|
|
|
|
|
2019-08-07 05:17:40 +08:00
|
|
|
// parsePath parses a remote 'url'
|
|
|
|
func parsePath(path string) (root string) {
|
|
|
|
root = strings.Trim(path, "/")
|
2012-12-29 19:35:41 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-08-07 05:17:40 +08:00
|
|
|
// split returns container and containerPath from the rootRelativePath
|
|
|
|
// relative to f.root
|
|
|
|
func (f *Fs) split(rootRelativePath string) (container, containerPath string) {
|
2019-09-21 18:47:59 +08:00
|
|
|
container, containerPath = bucket.Split(path.Join(f.root, rootRelativePath))
|
2020-01-15 01:33:35 +08:00
|
|
|
return f.opt.Enc.FromStandardName(container), f.opt.Enc.FromStandardPath(containerPath)
|
2019-08-07 05:17:40 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// split returns container and containerPath from the object
|
|
|
|
func (o *Object) split() (container, containerPath string) {
|
|
|
|
return o.fs.split(o.remote)
|
|
|
|
}
|
|
|
|
|
2012-12-29 19:35:41 +08:00
|
|
|
// swiftConnection makes a connection to swift
|
2020-11-05 19:33:32 +08:00
|
|
|
func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {
|
|
|
|
ci := fs.GetConfig(ctx)
|
2012-12-29 19:35:41 +08:00
|
|
|
c := &swift.Connection{
|
2017-11-02 05:26:04 +08:00
|
|
|
// Keep these in the same order as the Config for ease of checking
|
2019-01-29 00:55:17 +08:00
|
|
|
UserName: opt.User,
|
|
|
|
ApiKey: opt.Key,
|
|
|
|
AuthUrl: opt.Auth,
|
|
|
|
UserId: opt.UserID,
|
|
|
|
Domain: opt.Domain,
|
|
|
|
Tenant: opt.Tenant,
|
|
|
|
TenantId: opt.TenantID,
|
|
|
|
TenantDomain: opt.TenantDomain,
|
|
|
|
Region: opt.Region,
|
|
|
|
StorageUrl: opt.StorageURL,
|
|
|
|
AuthToken: opt.AuthToken,
|
|
|
|
AuthVersion: opt.AuthVersion,
|
2019-01-12 00:57:05 +08:00
|
|
|
ApplicationCredentialId: opt.ApplicationCredentialID,
|
2019-01-29 00:55:17 +08:00
|
|
|
ApplicationCredentialName: opt.ApplicationCredentialName,
|
|
|
|
ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
|
|
|
|
EndpointType: swift.EndpointType(opt.EndpointType),
|
2020-11-05 19:33:32 +08:00
|
|
|
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
|
|
|
|
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
|
2020-11-13 23:24:43 +08:00
|
|
|
Transport: fshttp.NewTransport(ctx),
|
2012-12-29 19:35:41 +08:00
|
|
|
}
|
2018-05-15 01:06:57 +08:00
|
|
|
if opt.EnvAuth {
|
2017-08-11 04:38:45 +08:00
|
|
|
err := c.ApplyEnvironment()
|
|
|
|
if err != nil {
|
2021-11-04 18:12:57 +08:00
|
|
|
return nil, fmt.Errorf("failed to read environment variables: %w", err)
|
2017-08-11 04:38:45 +08:00
|
|
|
}
|
|
|
|
}
|
2018-05-05 18:49:03 +08:00
|
|
|
StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
|
2017-12-08 05:30:58 +08:00
|
|
|
if !c.Authenticated() {
|
2019-01-29 00:55:17 +08:00
|
|
|
if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
|
|
|
|
if c.UserName == "" && c.UserId == "" {
|
|
|
|
return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
|
|
|
|
}
|
|
|
|
if c.ApiKey == "" {
|
|
|
|
return nil, errors.New("key not found")
|
|
|
|
}
|
2017-12-08 05:30:58 +08:00
|
|
|
}
|
|
|
|
if c.AuthUrl == "" {
|
|
|
|
return nil, errors.New("auth not found")
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
err := c.Authenticate(ctx) // fills in c.StorageUrl and c.AuthToken
|
2017-12-08 05:30:58 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2012-12-29 19:35:41 +08:00
|
|
|
}
|
2017-12-14 22:23:55 +08:00
|
|
|
// Make sure we re-auth with the AuthToken and StorageUrl
|
2018-02-16 03:18:11 +08:00
|
|
|
// provided by wrapping the existing auth, so we can just
|
|
|
|
// override one or the other or both.
|
|
|
|
if StorageUrl != "" || AuthToken != "" {
|
2018-08-21 23:12:42 +08:00
|
|
|
// Re-write StorageURL and AuthToken if they are being
|
|
|
|
// overridden as c.Authenticate above will have
|
|
|
|
// overwritten them.
|
|
|
|
if StorageUrl != "" {
|
|
|
|
c.StorageUrl = StorageUrl
|
|
|
|
}
|
|
|
|
if AuthToken != "" {
|
|
|
|
c.AuthToken = AuthToken
|
|
|
|
}
|
2018-02-16 03:18:11 +08:00
|
|
|
c.Auth = newAuth(c.Auth, StorageUrl, AuthToken)
|
2017-12-14 22:23:55 +08:00
|
|
|
}
|
2012-12-29 19:35:41 +08:00
|
|
|
return c, nil
|
|
|
|
}
|
|
|
|
|
2018-09-07 19:02:27 +08:00
|
|
|
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
2021-03-03 03:11:57 +08:00
|
|
|
const minChunkSize = fs.SizeSuffixBase
|
2018-09-07 19:02:27 +08:00
|
|
|
if cs < minChunkSize {
|
2021-11-04 18:12:57 +08:00
|
|
|
return fmt.Errorf("%s is less than %s", cs, minChunkSize)
|
2018-09-07 19:02:27 +08:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
|
|
|
err = checkUploadChunkSize(cs)
|
|
|
|
if err == nil {
|
|
|
|
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-08-07 05:17:40 +08:00
|
|
|
// setRoot changes the root of the Fs
|
|
|
|
func (f *Fs) setRoot(root string) {
|
|
|
|
f.root = parsePath(root)
|
|
|
|
f.rootContainer, f.rootDirectory = bucket.Split(f.root)
|
|
|
|
}
|
|
|
|
|
2018-09-05 03:27:48 +08:00
|
|
|
// NewFsWithConnection constructs an Fs from the path, container:path
|
2017-08-30 22:54:49 +08:00
|
|
|
// and authenticated connection.
|
|
|
|
//
|
|
|
|
// if noCheckContainer is set then the Fs won't check the container
|
|
|
|
// exists before creating it.
|
2020-11-05 23:18:51 +08:00
|
|
|
func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) {
|
2020-11-05 19:33:32 +08:00
|
|
|
ci := fs.GetConfig(ctx)
|
2015-11-07 19:14:46 +08:00
|
|
|
f := &Fs{
|
2019-08-07 05:17:40 +08:00
|
|
|
name: name,
|
|
|
|
opt: *opt,
|
2020-11-05 19:33:32 +08:00
|
|
|
ci: ci,
|
2019-08-07 05:17:40 +08:00
|
|
|
c: c,
|
|
|
|
noCheckContainer: noCheckContainer,
|
2020-11-05 19:33:32 +08:00
|
|
|
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
|
2019-08-07 05:17:40 +08:00
|
|
|
cache: bucket.NewCache(),
|
|
|
|
}
|
|
|
|
f.setRoot(root)
|
2017-08-09 22:27:43 +08:00
|
|
|
f.features = (&fs.Features{
|
2019-08-07 05:17:40 +08:00
|
|
|
ReadMimeType: true,
|
|
|
|
WriteMimeType: true,
|
|
|
|
BucketBased: true,
|
|
|
|
BucketBasedRootOK: true,
|
2020-06-19 17:28:34 +08:00
|
|
|
SlowModTime: true,
|
2020-11-06 00:00:40 +08:00
|
|
|
}).Fill(ctx, f)
|
2019-08-07 05:17:40 +08:00
|
|
|
if f.rootContainer != "" && f.rootDirectory != "" {
|
2015-11-12 01:19:57 +08:00
|
|
|
// Check to see if the object exists - ignoring directory markers
|
2018-11-23 06:15:52 +08:00
|
|
|
var info swift.Object
|
2019-08-07 05:17:40 +08:00
|
|
|
var err error
|
2020-01-15 01:33:35 +08:00
|
|
|
encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
|
2018-11-23 06:15:52 +08:00
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
info, rxHeaders, err = f.c.Object(ctx, f.rootContainer, encodedDirectory)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2015-11-12 01:19:57 +08:00
|
|
|
if err == nil && info.ContentType != directoryMarkerContentType {
|
2019-08-07 05:17:40 +08:00
|
|
|
newRoot := path.Dir(f.root)
|
|
|
|
if newRoot == "." {
|
|
|
|
newRoot = ""
|
2014-05-06 02:52:52 +08:00
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
f.setRoot(newRoot)
|
2016-06-22 01:01:53 +08:00
|
|
|
// return an error with an fs which points to the parent
|
|
|
|
return f, fs.ErrorIsFile
|
2014-05-06 02:52:52 +08:00
|
|
|
}
|
|
|
|
}
|
2012-12-29 19:35:41 +08:00
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
|
2019-02-08 01:41:17 +08:00
|
|
|
// NewFs constructs an Fs from the path, container:path
|
2020-11-05 23:18:51 +08:00
|
|
|
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
2018-05-15 01:06:57 +08:00
|
|
|
// Parse config into Options struct
|
|
|
|
opt := new(Options)
|
|
|
|
err := configstruct.Set(m, opt)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-09-07 19:02:27 +08:00
|
|
|
err = checkUploadChunkSize(opt.ChunkSize)
|
|
|
|
if err != nil {
|
2021-11-04 18:12:57 +08:00
|
|
|
return nil, fmt.Errorf("swift: chunk size: %w", err)
|
2018-09-07 19:02:27 +08:00
|
|
|
}
|
2018-05-15 01:06:57 +08:00
|
|
|
|
2020-11-05 19:33:32 +08:00
|
|
|
c, err := swiftConnection(ctx, opt, name)
|
2015-11-08 23:29:58 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-11-05 23:18:51 +08:00
|
|
|
return NewFsWithConnection(ctx, opt, name, root, c, false)
|
2015-11-08 23:29:58 +08:00
|
|
|
}
|
|
|
|
|
2016-06-26 04:58:34 +08:00
|
|
|
// Return an Object from a path
|
2012-12-26 20:23:58 +08:00
|
|
|
//
|
2016-06-26 04:23:20 +08:00
|
|
|
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
2021-01-23 01:23:51 +08:00
|
|
|
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *swift.Object) (fs.Object, error) {
|
2015-11-07 19:14:46 +08:00
|
|
|
o := &Object{
|
|
|
|
fs: f,
|
2012-12-26 20:23:58 +08:00
|
|
|
remote: remote,
|
|
|
|
}
|
2016-03-18 01:36:20 +08:00
|
|
|
// Note that due to a quirk of swift, dynamic large objects are
|
2015-12-03 05:23:56 +08:00
|
|
|
// returned as 0 bytes in the listing. Correct this here by
|
|
|
|
// making sure we read the full metadata for all 0 byte files.
|
2016-09-08 23:44:11 +08:00
|
|
|
// We don't read the metadata for directory marker objects.
|
|
|
|
if info != nil && info.Bytes == 0 && info.ContentType != "application/directory" {
|
2021-01-23 01:23:51 +08:00
|
|
|
err := o.readMetaData(ctx) // reads info and headers, returning an error
|
2020-07-10 18:03:08 +08:00
|
|
|
if err == fs.ErrorObjectNotFound {
|
|
|
|
// We have a dangling large object here so just return the original metadata
|
|
|
|
fs.Errorf(o, "dangling large object with no contents")
|
|
|
|
} else if err != nil {
|
|
|
|
return nil, err
|
|
|
|
} else {
|
|
|
|
return o, nil
|
|
|
|
}
|
2015-12-03 05:23:56 +08:00
|
|
|
}
|
2012-12-26 20:23:58 +08:00
|
|
|
if info != nil {
|
2015-09-25 00:41:16 +08:00
|
|
|
// Set info but not headers
|
2018-12-02 18:23:55 +08:00
|
|
|
err := o.decodeMetaData(info)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2012-12-26 20:23:58 +08:00
|
|
|
} else {
|
2021-01-23 01:23:51 +08:00
|
|
|
err := o.readMetaData(ctx) // reads info and headers, returning an error
|
2012-12-26 20:23:58 +08:00
|
|
|
if err != nil {
|
2016-06-26 04:23:20 +08:00
|
|
|
return nil, err
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
}
|
2016-06-26 04:23:20 +08:00
|
|
|
return o, nil
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-06-26 04:23:20 +08:00
|
|
|
// NewObject finds the Object at remote. If it can't be found it
|
|
|
|
// returns the error fs.ErrorObjectNotFound.
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
return f.newObjectWithInfo(ctx, remote, nil)
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-04-22 03:06:21 +08:00
|
|
|
// listFn is called from list and listContainerRoot to handle an object.
|
|
|
|
type listFn func(remote string, object *swift.Object, isDirectory bool) error
|
2015-09-26 19:51:05 +08:00
|
|
|
|
|
|
|
// listContainerRoot lists the objects into the function supplied from
|
2019-08-07 05:17:40 +08:00
|
|
|
// the container and directory supplied. The remote has prefix
|
|
|
|
// removed from it and if addContainer is set then it adds the
|
|
|
|
// container to the start.
|
2014-05-06 00:30:55 +08:00
|
|
|
//
|
2017-06-12 05:43:31 +08:00
|
|
|
// Set recurse to read sub directories
|
2021-01-23 01:23:51 +08:00
|
|
|
func (f *Fs) listContainerRoot(ctx context.Context, container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn listFn) error {
|
2019-11-19 00:26:10 +08:00
|
|
|
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
2019-08-07 05:17:40 +08:00
|
|
|
prefix += "/"
|
|
|
|
}
|
2019-11-19 00:26:10 +08:00
|
|
|
if directory != "" && !strings.HasSuffix(directory, "/") {
|
2019-08-07 05:17:40 +08:00
|
|
|
directory += "/"
|
2016-04-24 04:46:52 +08:00
|
|
|
}
|
2014-05-06 00:30:55 +08:00
|
|
|
// Options for ObjectsWalk
|
|
|
|
opts := swift.ObjectsOpts{
|
2019-08-07 05:17:40 +08:00
|
|
|
Prefix: directory,
|
2017-06-06 22:03:52 +08:00
|
|
|
Limit: listChunks,
|
2014-05-06 00:30:55 +08:00
|
|
|
}
|
2017-06-12 05:43:31 +08:00
|
|
|
if !recurse {
|
2014-05-06 00:30:55 +08:00
|
|
|
opts.Delimiter = '/'
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
return f.c.ObjectsWalk(ctx, container, &opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) {
|
2018-11-23 06:15:52 +08:00
|
|
|
var objects []swift.Object
|
|
|
|
var err error
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
objects, err = f.c.Objects(ctx, container, opts)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2014-05-06 00:30:55 +08:00
|
|
|
if err == nil {
|
|
|
|
for i := range objects {
|
|
|
|
object := &objects[i]
|
2016-04-22 03:06:21 +08:00
|
|
|
isDirectory := false
|
2017-06-12 05:43:31 +08:00
|
|
|
if !recurse {
|
2017-07-06 18:31:09 +08:00
|
|
|
isDirectory = strings.HasSuffix(object.Name, "/")
|
2014-05-06 00:30:55 +08:00
|
|
|
}
|
2020-01-15 01:33:35 +08:00
|
|
|
remote := f.opt.Enc.ToStandardPath(object.Name)
|
2019-09-21 18:47:59 +08:00
|
|
|
if !strings.HasPrefix(remote, prefix) {
|
|
|
|
fs.Logf(f, "Odd name received %q", remote)
|
2014-05-06 00:30:55 +08:00
|
|
|
continue
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
2020-07-10 22:16:11 +08:00
|
|
|
if !includeDirMarkers && remote == prefix {
|
2017-07-19 06:38:48 +08:00
|
|
|
// If we have zero length directory markers ending in / then swift
|
|
|
|
// will return them in the listing for the directory which causes
|
|
|
|
// duplicate directories. Ignore them here.
|
|
|
|
continue
|
|
|
|
}
|
2019-09-21 18:47:59 +08:00
|
|
|
remote = remote[len(prefix):]
|
2019-08-07 05:17:40 +08:00
|
|
|
if addContainer {
|
|
|
|
remote = path.Join(container, remote)
|
|
|
|
}
|
2016-04-22 03:06:21 +08:00
|
|
|
err = fn(remote, object, isDirectory)
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
|
|
|
break
|
|
|
|
}
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
}
|
2014-05-06 00:30:55 +08:00
|
|
|
return objects, err
|
|
|
|
})
|
2015-09-26 19:51:05 +08:00
|
|
|
}
|
|
|
|
|
2017-06-30 17:54:14 +08:00
|
|
|
type addEntryFn func(fs.DirEntry) error
|
2014-05-06 00:30:55 +08:00
|
|
|
|
2017-06-12 05:43:31 +08:00
|
|
|
// list the objects into the function supplied
|
2021-01-23 01:23:51 +08:00
|
|
|
func (f *Fs) list(ctx context.Context, container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn addEntryFn) error {
|
|
|
|
err := f.listContainerRoot(ctx, container, directory, prefix, addContainer, recurse, includeDirMarkers, func(remote string, object *swift.Object, isDirectory bool) (err error) {
|
2016-04-22 03:06:21 +08:00
|
|
|
if isDirectory {
|
2017-07-06 18:31:09 +08:00
|
|
|
remote = strings.TrimRight(remote, "/")
|
2017-06-30 20:37:29 +08:00
|
|
|
d := fs.NewDir(remote, time.Time{}).SetSize(object.Bytes)
|
2017-06-12 05:43:31 +08:00
|
|
|
err = fn(d)
|
2016-04-22 03:06:21 +08:00
|
|
|
} else {
|
2017-11-02 22:49:07 +08:00
|
|
|
// newObjectWithInfo does a full metadata read on 0 size objects which might be dynamic large objects
|
2018-05-04 22:19:50 +08:00
|
|
|
var o fs.Object
|
2021-01-23 01:23:51 +08:00
|
|
|
o, err = f.newObjectWithInfo(ctx, remote, object)
|
2016-06-26 04:23:20 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-07-10 22:16:11 +08:00
|
|
|
if includeDirMarkers || o.Storable() {
|
2017-06-12 05:43:31 +08:00
|
|
|
err = fn(o)
|
2016-04-22 03:06:21 +08:00
|
|
|
}
|
|
|
|
}
|
2017-06-12 05:43:31 +08:00
|
|
|
return err
|
|
|
|
})
|
2017-09-15 01:00:00 +08:00
|
|
|
if err == swift.ContainerNotFound {
|
|
|
|
err = fs.ErrorDirNotFound
|
|
|
|
}
|
|
|
|
return err
|
2017-06-12 05:43:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// listDir lists a single directory
|
2021-01-23 01:23:51 +08:00
|
|
|
func (f *Fs) listDir(ctx context.Context, container, directory, prefix string, addContainer bool) (entries fs.DirEntries, err error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
if container == "" {
|
2017-06-12 05:43:31 +08:00
|
|
|
return nil, fs.ErrorListBucketRequired
|
|
|
|
}
|
|
|
|
// List the objects
|
2021-01-23 01:23:51 +08:00
|
|
|
err = f.list(ctx, container, directory, prefix, addContainer, false, false, func(entry fs.DirEntry) error {
|
2017-06-12 05:43:31 +08:00
|
|
|
entries = append(entries, entry)
|
2016-04-22 03:06:21 +08:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
2017-06-12 05:43:31 +08:00
|
|
|
return nil, err
|
2014-05-06 00:30:55 +08:00
|
|
|
}
|
2018-03-01 20:11:34 +08:00
|
|
|
// container must be present if listing succeeded
|
2019-08-07 05:17:40 +08:00
|
|
|
f.cache.MarkOK(container)
|
2017-06-12 05:43:31 +08:00
|
|
|
return entries, nil
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-04-22 03:06:21 +08:00
|
|
|
// listContainers lists the containers
|
2019-08-23 04:30:55 +08:00
|
|
|
func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err error) {
|
2018-11-23 06:15:52 +08:00
|
|
|
var containers []swift.Container
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
containers, err = f.c.ContainersAll(ctx, nil)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2016-04-22 03:06:21 +08:00
|
|
|
if err != nil {
|
2021-11-04 18:12:57 +08:00
|
|
|
return nil, fmt.Errorf("container listing failed: %w", err)
|
2016-04-22 03:06:21 +08:00
|
|
|
}
|
|
|
|
for _, container := range containers {
|
2019-08-07 05:17:40 +08:00
|
|
|
f.cache.MarkOK(container.Name)
|
2020-01-15 01:33:35 +08:00
|
|
|
d := fs.NewDir(f.opt.Enc.ToStandardName(container.Name), time.Time{}).SetSize(container.Bytes).SetItems(container.Count)
|
2017-06-12 05:43:31 +08:00
|
|
|
entries = append(entries, d)
|
2016-04-22 03:06:21 +08:00
|
|
|
}
|
2017-06-12 05:43:31 +08:00
|
|
|
return entries, nil
|
2015-11-07 23:17:46 +08:00
|
|
|
}
|
|
|
|
|
2017-06-12 05:43:31 +08:00
|
|
|
// List the objects and directories in dir into entries. The
|
|
|
|
// entries can be returned in any order but should be for a
|
|
|
|
// complete directory.
|
|
|
|
//
|
|
|
|
// dir should be "" to list the root, and should not have
|
|
|
|
// trailing slashes.
|
|
|
|
//
|
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, directory := f.split(dir)
|
|
|
|
if container == "" {
|
2019-08-23 04:30:55 +08:00
|
|
|
if directory != "" {
|
|
|
|
return nil, fs.ErrorListBucketRequired
|
|
|
|
}
|
|
|
|
return f.listContainers(ctx)
|
2014-05-06 00:30:55 +08:00
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
return f.listDir(ctx, container, directory, f.rootDirectory, f.rootContainer == "")
|
2013-01-24 06:43:20 +08:00
|
|
|
}
|
|
|
|
|
2017-06-05 23:14:24 +08:00
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
|
|
// from dir recursively into out.
|
2017-06-12 05:43:31 +08:00
|
|
|
//
|
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
//
|
|
|
|
// It should call callback for each tranche of entries read.
|
|
|
|
// These need not be returned in any particular order. If
|
|
|
|
// callback returns an error then the listing will stop
|
|
|
|
// immediately.
|
|
|
|
//
|
|
|
|
// Don't implement this unless you have a more efficient way
|
2019-08-07 05:17:40 +08:00
|
|
|
// of listing recursively than doing a directory traversal.
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, directory := f.split(dir)
|
2018-01-13 00:30:54 +08:00
|
|
|
list := walk.NewListRHelper(callback)
|
2019-08-07 05:17:40 +08:00
|
|
|
listR := func(container, directory, prefix string, addContainer bool) error {
|
2021-01-23 01:23:51 +08:00
|
|
|
return f.list(ctx, container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {
|
2019-08-07 05:17:40 +08:00
|
|
|
return list.Add(entry)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
if container == "" {
|
2019-08-23 04:30:55 +08:00
|
|
|
entries, err := f.listContainers(ctx)
|
2019-08-07 05:17:40 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
|
|
err = list.Add(entry)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
container := entry.Remote()
|
|
|
|
err = listR(container, "", f.rootDirectory, true)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-08-23 04:30:55 +08:00
|
|
|
// container must be present if listing succeeded
|
|
|
|
f.cache.MarkOK(container)
|
2019-08-07 05:17:40 +08:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
err = listR(container, directory, f.rootDirectory, f.rootContainer == "")
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-08-23 04:30:55 +08:00
|
|
|
// container must be present if listing succeeded
|
|
|
|
f.cache.MarkOK(container)
|
2017-06-12 05:43:31 +08:00
|
|
|
}
|
|
|
|
return list.Flush()
|
2017-06-05 23:14:24 +08:00
|
|
|
}
|
|
|
|
|
2018-04-17 05:19:25 +08:00
|
|
|
// About gets quota information
|
2022-02-17 21:47:05 +08:00
|
|
|
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
2018-04-17 05:19:25 +08:00
|
|
|
var total, objects int64
|
2022-02-17 21:47:05 +08:00
|
|
|
if f.rootContainer != "" {
|
|
|
|
var container swift.Container
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
container, _, err = f.c.Container(ctx, f.rootContainer)
|
|
|
|
return shouldRetry(ctx, err)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("container info failed: %w", err)
|
|
|
|
}
|
|
|
|
total = container.Bytes
|
|
|
|
objects = container.Count
|
|
|
|
} else {
|
|
|
|
var containers []swift.Container
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
containers, err = f.c.ContainersAll(ctx, nil)
|
|
|
|
return shouldRetry(ctx, err)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("container listing failed: %w", err)
|
|
|
|
}
|
|
|
|
for _, c := range containers {
|
|
|
|
total += c.Bytes
|
|
|
|
objects += c.Count
|
|
|
|
}
|
2018-04-17 05:19:25 +08:00
|
|
|
}
|
2022-02-17 21:47:05 +08:00
|
|
|
usage = &fs.Usage{
|
2018-04-17 05:19:25 +08:00
|
|
|
Used: fs.NewUsageValue(total), // bytes in use
|
|
|
|
Objects: fs.NewUsageValue(objects), // objects in use
|
|
|
|
}
|
|
|
|
return usage, nil
|
|
|
|
}
|
|
|
|
|
2014-04-19 00:04:21 +08:00
|
|
|
// Put the object into the container
|
2013-01-11 05:58:46 +08:00
|
|
|
//
|
2022-08-05 23:35:41 +08:00
|
|
|
// Copy the reader in to the new object which is returned.
|
2013-01-11 05:58:46 +08:00
|
|
|
//
|
2014-04-19 00:04:21 +08:00
|
|
|
// The new object may have been created if an error is returned
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
2015-11-07 19:14:46 +08:00
|
|
|
// Temporary Object under construction
|
|
|
|
fs := &Object{
|
2018-02-06 22:39:55 +08:00
|
|
|
fs: f,
|
|
|
|
remote: src.Remote(),
|
|
|
|
headers: swift.Headers{}, // Empty object headers to stop readMetaData being called
|
2015-11-07 19:14:46 +08:00
|
|
|
}
|
2019-06-17 16:34:30 +08:00
|
|
|
return fs, fs.Update(ctx, in, src, options...)
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2017-09-14 13:42:16 +08:00
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
|
|
return f.Put(ctx, in, src, options...)
|
2017-09-14 13:42:16 +08:00
|
|
|
}
|
|
|
|
|
2012-12-26 20:23:58 +08:00
|
|
|
// Mkdir creates the container if it doesn't exist
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, _ := f.split(dir)
|
2019-08-23 04:30:55 +08:00
|
|
|
return f.makeContainer(ctx, container)
|
|
|
|
}
|
|
|
|
|
|
|
|
// makeContainer creates the container if it doesn't exist
|
|
|
|
func (f *Fs) makeContainer(ctx context.Context, container string) error {
|
2019-08-07 05:17:40 +08:00
|
|
|
return f.cache.Create(container, func() error {
|
|
|
|
// Check to see if container exists first
|
|
|
|
var err error = swift.ContainerNotFound
|
|
|
|
if !f.noCheckContainer {
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
_, rxHeaders, err = f.c.Container(ctx, container)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2019-08-07 05:17:40 +08:00
|
|
|
})
|
2018-08-05 04:09:17 +08:00
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
if err == swift.ContainerNotFound {
|
|
|
|
headers := swift.Headers{}
|
|
|
|
if f.opt.StoragePolicy != "" {
|
|
|
|
headers["X-Storage-Policy"] = f.opt.StoragePolicy
|
|
|
|
}
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
err = f.c.ContainerCreate(ctx, container, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2019-08-07 05:17:40 +08:00
|
|
|
})
|
|
|
|
if err == nil {
|
|
|
|
fs.Infof(f, "Container %q created", container)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}, nil)
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2015-11-07 23:31:04 +08:00
|
|
|
// Rmdir deletes the container if the fs is at the root
|
2012-12-26 20:23:58 +08:00
|
|
|
//
|
|
|
|
// Returns an error if it isn't empty
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, directory := f.split(dir)
|
|
|
|
if container == "" || directory != "" {
|
2015-11-07 23:31:04 +08:00
|
|
|
return nil
|
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
err := f.cache.Remove(container, func() error {
|
|
|
|
err := f.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
err := f.c.ContainerDelete(ctx, container)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2019-08-07 05:17:40 +08:00
|
|
|
})
|
|
|
|
if err == nil {
|
|
|
|
fs.Infof(f, "Container %q removed", container)
|
|
|
|
}
|
|
|
|
return err
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2017-06-07 21:16:50 +08:00
|
|
|
return err
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Precision of the remote
|
2015-11-07 19:14:46 +08:00
|
|
|
func (f *Fs) Precision() time.Duration {
|
2013-01-19 07:21:02 +08:00
|
|
|
return time.Nanosecond
|
|
|
|
}
|
|
|
|
|
2020-06-05 05:25:14 +08:00
|
|
|
// Purge deletes all the files in the directory
|
2015-11-07 23:17:46 +08:00
|
|
|
//
|
|
|
|
// Implemented here so we can make sure we delete directory markers
|
2020-06-05 05:25:14 +08:00
|
|
|
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
|
|
|
container, directory := f.split(dir)
|
|
|
|
if container == "" {
|
|
|
|
return fs.ErrorListBucketRequired
|
|
|
|
}
|
2016-04-22 03:06:21 +08:00
|
|
|
// Delete all the files including the directory markers
|
2020-11-05 19:33:32 +08:00
|
|
|
toBeDeleted := make(chan fs.Object, f.ci.Transfers)
|
2016-06-25 21:27:44 +08:00
|
|
|
delErr := make(chan error, 1)
|
2016-04-22 03:06:21 +08:00
|
|
|
go func() {
|
2019-06-17 16:34:30 +08:00
|
|
|
delErr <- operations.DeleteFiles(ctx, toBeDeleted)
|
2016-04-22 03:06:21 +08:00
|
|
|
}()
|
2021-01-23 01:23:51 +08:00
|
|
|
err := f.list(ctx, container, directory, f.rootDirectory, false, true, true, func(entry fs.DirEntry) error {
|
2017-06-12 05:43:31 +08:00
|
|
|
if o, ok := entry.(*Object); ok {
|
2016-06-25 21:27:44 +08:00
|
|
|
toBeDeleted <- o
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
close(toBeDeleted)
|
|
|
|
delError := <-delErr
|
|
|
|
if err == nil {
|
|
|
|
err = delError
|
|
|
|
}
|
2016-04-22 03:06:21 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-06-05 05:25:14 +08:00
|
|
|
return f.Rmdir(ctx, dir)
|
2015-11-07 23:17:46 +08:00
|
|
|
}
|
|
|
|
|
2020-10-14 05:43:40 +08:00
|
|
|
// Copy src to this remote using server-side copy operations.
|
2015-02-15 02:48:08 +08:00
|
|
|
//
|
2022-08-05 23:35:41 +08:00
|
|
|
// This is stored with the remote path given.
|
2015-02-15 02:48:08 +08:00
|
|
|
//
|
2022-08-05 23:35:41 +08:00
|
|
|
// It returns the destination Object and a possible error.
|
2015-02-15 02:48:08 +08:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
2019-06-17 16:34:30 +08:00
|
|
|
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
dstContainer, dstPath := f.split(remote)
|
2019-08-23 04:30:55 +08:00
|
|
|
err := f.makeContainer(ctx, dstContainer)
|
2017-06-29 04:14:53 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-11-07 19:14:46 +08:00
|
|
|
srcObj, ok := src.(*Object)
|
2015-02-15 02:48:08 +08:00
|
|
|
if !ok {
|
2017-02-09 19:01:20 +08:00
|
|
|
fs.Debugf(src, "Can't copy - not same remote type")
|
2015-02-15 02:48:08 +08:00
|
|
|
return nil, fs.ErrorCantCopy
|
|
|
|
}
|
2021-03-02 16:58:14 +08:00
|
|
|
isLargeObject, err := srcObj.isLargeObject(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if isLargeObject {
|
|
|
|
/*handle large object*/
|
|
|
|
err = copyLargeObject(ctx, f, srcObj, dstContainer, dstPath)
|
|
|
|
} else {
|
|
|
|
srcContainer, srcPath := srcObj.split()
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
var rxHeaders swift.Headers
|
|
|
|
rxHeaders, err = f.c.ObjectCopy(ctx, srcContainer, srcPath, dstContainer, dstPath, nil)
|
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return f.NewObject(ctx, remote)
|
|
|
|
}
|
|
|
|
|
|
|
|
func copyLargeObject(ctx context.Context, f *Fs, src *Object, dstContainer string, dstPath string) error {
|
|
|
|
segmentsContainer := dstContainer + "_segments"
|
|
|
|
err := f.makeContainer(ctx, segmentsContainer)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
segments, err := src.getSegmentsLargeObject(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if len(segments) == 0 {
|
|
|
|
return errors.New("could not copy object, list segments are empty")
|
|
|
|
}
|
|
|
|
nanoSeconds := time.Now().Nanosecond()
|
|
|
|
prefixSegment := fmt.Sprintf("%v/%v/%s", nanoSeconds, src.size, strings.ReplaceAll(uuid.New().String(), "-", ""))
|
|
|
|
copiedSegmentsLen := 10
|
|
|
|
for _, value := range segments {
|
|
|
|
if len(value) <= 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
fragment := value[0]
|
|
|
|
if len(fragment) <= 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
copiedSegmentsLen = len(value)
|
|
|
|
firstIndex := strings.Index(fragment, "/")
|
|
|
|
if firstIndex < 0 {
|
|
|
|
firstIndex = 0
|
|
|
|
} else {
|
|
|
|
firstIndex = firstIndex + 1
|
|
|
|
}
|
|
|
|
lastIndex := strings.LastIndex(fragment, "/")
|
|
|
|
if lastIndex < 0 {
|
|
|
|
lastIndex = len(fragment)
|
|
|
|
} else {
|
|
|
|
lastIndex = lastIndex - 1
|
|
|
|
}
|
|
|
|
prefixSegment = fragment[firstIndex:lastIndex]
|
|
|
|
break
|
|
|
|
}
|
|
|
|
copiedSegments := make([]string, copiedSegmentsLen)
|
|
|
|
defer handleCopyFail(ctx, f, segmentsContainer, copiedSegments, err)
|
|
|
|
for c, ss := range segments {
|
|
|
|
if len(ss) <= 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, s := range ss {
|
|
|
|
lastIndex := strings.LastIndex(s, "/")
|
|
|
|
if lastIndex <= 0 {
|
|
|
|
lastIndex = 0
|
|
|
|
} else {
|
|
|
|
lastIndex = lastIndex + 1
|
|
|
|
}
|
|
|
|
segmentName := dstPath + "/" + prefixSegment + "/" + s[lastIndex:]
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
var rxHeaders swift.Headers
|
|
|
|
rxHeaders, err = f.c.ObjectCopy(ctx, c, s, segmentsContainer, segmentName, nil)
|
|
|
|
copiedSegments = append(copiedSegments, segmentName)
|
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
m := swift.Metadata{}
|
|
|
|
headers := m.ObjectHeaders()
|
|
|
|
headers["X-Object-Manifest"] = urlEncode(fmt.Sprintf("%s/%s/%s", segmentsContainer, dstPath, prefixSegment))
|
|
|
|
headers["Content-Length"] = "0"
|
|
|
|
emptyReader := bytes.NewReader(nil)
|
2018-11-23 06:15:52 +08:00
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-03-02 16:58:14 +08:00
|
|
|
rxHeaders, err = f.c.ObjectPut(ctx, dstContainer, dstPath, emptyReader, true, "", src.contentType, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2021-03-02 16:58:14 +08:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-08-05 23:35:41 +08:00
|
|
|
// remove copied segments when copy process failed
|
2021-03-02 16:58:14 +08:00
|
|
|
func handleCopyFail(ctx context.Context, f *Fs, segmentsContainer string, segments []string, err error) {
|
|
|
|
fs.Debugf(f, "handle copy segment fail")
|
|
|
|
if err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if len(segmentsContainer) == 0 {
|
|
|
|
fs.Debugf(f, "invalid segments container")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if len(segments) == 0 {
|
|
|
|
fs.Debugf(f, "segments is empty")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fs.Debugf(f, "action delete segments what copied")
|
|
|
|
for _, v := range segments {
|
|
|
|
_ = f.c.ObjectDelete(ctx, segmentsContainer, v)
|
2015-02-15 02:48:08 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-11 20:39:33 +08:00
|
|
|
// Hashes returns the supported hash sets.
|
2018-01-13 00:30:54 +08:00
|
|
|
func (f *Fs) Hashes() hash.Set {
|
2018-01-19 04:27:52 +08:00
|
|
|
return hash.Set(hash.MD5)
|
2016-01-11 20:39:33 +08:00
|
|
|
}
|
|
|
|
|
2012-12-26 20:23:58 +08:00
|
|
|
// ------------------------------------------------------------
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Fs returns the parent Fs
|
2016-02-18 19:35:25 +08:00
|
|
|
func (o *Object) Fs() fs.Info {
|
2015-11-07 19:14:46 +08:00
|
|
|
return o.fs
|
2014-03-29 01:56:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return a string version
|
2015-11-07 19:14:46 +08:00
|
|
|
func (o *Object) String() string {
|
2014-03-29 01:56:04 +08:00
|
|
|
if o == nil {
|
|
|
|
return "<nil>"
|
|
|
|
}
|
|
|
|
return o.remote
|
|
|
|
}
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Remote returns the remote path
|
2015-11-07 19:14:46 +08:00
|
|
|
func (o *Object) Remote() string {
|
2013-06-28 03:13:07 +08:00
|
|
|
return o.remote
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-01-11 20:39:33 +08:00
|
|
|
// Hash returns the Md5sum of an object returning a lowercase hex string
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
2018-01-19 04:27:52 +08:00
|
|
|
if t != hash.MD5 {
|
|
|
|
return "", hash.ErrUnsupported
|
2016-01-11 20:39:33 +08:00
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
isDynamicLargeObject, err := o.isDynamicLargeObject(ctx)
|
2015-09-23 14:57:48 +08:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
isStaticLargeObject, err := o.isStaticLargeObject(ctx)
|
2016-03-18 01:36:20 +08:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if isDynamicLargeObject || isStaticLargeObject {
|
2017-02-09 19:01:20 +08:00
|
|
|
fs.Debugf(o, "Returning empty Md5sum for swift large object")
|
2015-09-23 14:57:48 +08:00
|
|
|
return "", nil
|
|
|
|
}
|
2018-12-02 18:23:55 +08:00
|
|
|
return strings.ToLower(o.md5), nil
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-03-18 01:36:20 +08:00
|
|
|
// hasHeader checks for the header passed in returning false if the
|
|
|
|
// object isn't found.
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) hasHeader(ctx context.Context, header string) (bool, error) {
|
|
|
|
err := o.readMetaData(ctx)
|
2015-09-23 14:57:48 +08:00
|
|
|
if err != nil {
|
2016-06-26 04:23:20 +08:00
|
|
|
if err == fs.ErrorObjectNotFound {
|
2015-09-26 19:51:05 +08:00
|
|
|
return false, nil
|
|
|
|
}
|
2015-09-23 14:57:48 +08:00
|
|
|
return false, err
|
|
|
|
}
|
2018-02-06 22:39:55 +08:00
|
|
|
_, isDynamicLargeObject := o.headers[header]
|
2016-03-18 01:36:20 +08:00
|
|
|
return isDynamicLargeObject, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// isDynamicLargeObject checks for X-Object-Manifest header
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) isDynamicLargeObject(ctx context.Context) (bool, error) {
|
2022-10-06 00:39:24 +08:00
|
|
|
if o.fs.opt.NoLargeObjects {
|
|
|
|
return false, nil
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
return o.hasHeader(ctx, "X-Object-Manifest")
|
2016-03-18 01:36:20 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// isStaticLargeObjectFile checks for the X-Static-Large-Object header
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) isStaticLargeObject(ctx context.Context) (bool, error) {
|
2022-10-06 00:39:24 +08:00
|
|
|
if o.fs.opt.NoLargeObjects {
|
|
|
|
return false, nil
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
return o.hasHeader(ctx, "X-Static-Large-Object")
|
2015-09-23 14:57:48 +08:00
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) isLargeObject(ctx context.Context) (result bool, err error) {
|
2022-10-06 00:39:24 +08:00
|
|
|
if o.fs.opt.NoLargeObjects {
|
|
|
|
return false, nil
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
result, err = o.hasHeader(ctx, "X-Static-Large-Object")
|
2020-12-28 21:21:11 +08:00
|
|
|
if result {
|
|
|
|
return
|
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
result, err = o.hasHeader(ctx, "X-Object-Manifest")
|
2020-12-28 21:21:11 +08:00
|
|
|
if result {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) isInContainerVersioning(ctx context.Context, container string) (bool, error) {
|
|
|
|
_, headers, err := o.fs.c.Container(ctx, container)
|
2019-11-19 00:26:10 +08:00
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
xHistoryLocation := headers["X-History-Location"]
|
|
|
|
if len(xHistoryLocation) > 0 {
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
2012-12-26 20:23:58 +08:00
|
|
|
// Size returns the size of an object in bytes
|
2015-11-07 19:14:46 +08:00
|
|
|
func (o *Object) Size() int64 {
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.size
|
|
|
|
}
|
|
|
|
|
|
|
|
// decodeMetaData sets the metadata in the object from a swift.Object
|
|
|
|
//
|
|
|
|
// Sets
|
2022-08-05 23:35:41 +08:00
|
|
|
//
|
|
|
|
// o.lastModified
|
|
|
|
// o.size
|
|
|
|
// o.md5
|
|
|
|
// o.contentType
|
2018-12-02 18:23:55 +08:00
|
|
|
func (o *Object) decodeMetaData(info *swift.Object) (err error) {
|
|
|
|
o.lastModified = info.LastModified
|
|
|
|
o.size = info.Bytes
|
|
|
|
o.md5 = info.Hash
|
|
|
|
o.contentType = info.ContentType
|
|
|
|
return nil
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// readMetaData gets the metadata if it hasn't already been fetched
|
|
|
|
//
|
|
|
|
// it also sets the info
|
2016-06-26 04:23:20 +08:00
|
|
|
//
|
|
|
|
// it returns fs.ErrorObjectNotFound if the object isn't found
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) readMetaData(ctx context.Context) (err error) {
|
2015-09-25 00:41:16 +08:00
|
|
|
if o.headers != nil {
|
2012-12-26 20:23:58 +08:00
|
|
|
return nil
|
|
|
|
}
|
2018-11-23 06:15:52 +08:00
|
|
|
var info swift.Object
|
|
|
|
var h swift.Headers
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
info, h, err = o.fs.c.Object(ctx, container, containerPath)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, h, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2012-12-26 20:23:58 +08:00
|
|
|
if err != nil {
|
2016-06-26 04:23:20 +08:00
|
|
|
if err == swift.ObjectNotFound {
|
|
|
|
return fs.ErrorObjectNotFound
|
|
|
|
}
|
2012-12-26 20:23:58 +08:00
|
|
|
return err
|
|
|
|
}
|
2018-02-06 22:39:55 +08:00
|
|
|
o.headers = h
|
2018-12-02 18:23:55 +08:00
|
|
|
err = o.decodeMetaData(&info)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2012-12-26 20:23:58 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ModTime returns the modification time of the object
|
2013-01-02 23:21:55 +08:00
|
|
|
//
|
|
|
|
// It attempts to read the objects mtime and if that isn't present the
|
|
|
|
// LastModified returned in the http headers
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) ModTime(ctx context.Context) time.Time {
|
2020-11-05 19:33:32 +08:00
|
|
|
if o.fs.ci.UseServerModTime {
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.lastModified
|
2018-04-13 20:32:17 +08:00
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
err := o.readMetaData(ctx)
|
2012-12-26 20:23:58 +08:00
|
|
|
if err != nil {
|
2017-02-09 19:01:20 +08:00
|
|
|
fs.Debugf(o, "Failed to read metadata: %s", err)
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.lastModified
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
2015-09-25 00:41:16 +08:00
|
|
|
modTime, err := o.headers.ObjectMetadata().GetModTime()
|
2012-12-26 20:23:58 +08:00
|
|
|
if err != nil {
|
2017-02-09 19:01:20 +08:00
|
|
|
// fs.Logf(o, "Failed to read mtime from object: %v", err)
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.lastModified
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
2013-01-02 23:21:55 +08:00
|
|
|
return modTime
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// SetModTime sets the modification time of the local fs object
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
2021-01-23 01:23:51 +08:00
|
|
|
err := o.readMetaData(ctx)
|
2012-12-26 20:23:58 +08:00
|
|
|
if err != nil {
|
2016-03-22 23:07:10 +08:00
|
|
|
return err
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
2015-09-25 00:41:16 +08:00
|
|
|
meta := o.headers.ObjectMetadata()
|
|
|
|
meta.SetModTime(modTime)
|
|
|
|
newHeaders := meta.ObjectHeaders()
|
|
|
|
for k, v := range newHeaders {
|
2018-02-06 22:39:55 +08:00
|
|
|
o.headers[k] = v
|
2015-09-25 00:41:16 +08:00
|
|
|
}
|
2015-11-09 04:45:42 +08:00
|
|
|
// Include any other metadata from request
|
2018-02-06 22:39:55 +08:00
|
|
|
for k, v := range o.headers {
|
2015-11-09 04:45:42 +08:00
|
|
|
if strings.HasPrefix(k, "X-Object-") {
|
|
|
|
newHeaders[k] = v
|
|
|
|
}
|
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
2018-11-23 06:15:52 +08:00
|
|
|
return o.fs.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
err = o.fs.c.ObjectUpdate(ctx, container, containerPath, newHeaders)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2015-09-23 01:47:16 +08:00
|
|
|
// Storable returns if this object is storable
|
2015-11-07 23:17:46 +08:00
|
|
|
//
|
2015-12-03 05:23:56 +08:00
|
|
|
// It compares the Content-Type to directoryMarkerContentType - that
|
|
|
|
// makes it a directory marker which is not storable.
|
2015-11-07 19:14:46 +08:00
|
|
|
func (o *Object) Storable() bool {
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.contentType != directoryMarkerContentType
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Open an object for read
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
2019-08-04 00:45:14 +08:00
|
|
|
fs.FixRangeOption(options, o.size)
|
2016-09-10 18:29:57 +08:00
|
|
|
headers := fs.OpenOptionHeaders(options)
|
|
|
|
_, isRanging := headers["Range"]
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
in, rxHeaders, err = o.fs.c.ObjectOpen(ctx, container, containerPath, !isRanging, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2012-12-26 20:23:58 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-09-23 14:57:48 +08:00
|
|
|
// min returns the smallest of x, y
|
|
|
|
func min(x, y int64) int64 {
|
|
|
|
if x < y {
|
|
|
|
return x
|
|
|
|
}
|
|
|
|
return y
|
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) getSegmentsLargeObject(ctx context.Context) (map[string][]string, error) {
|
2021-01-29 01:09:41 +08:00
|
|
|
container, objectName := o.split()
|
2021-01-23 01:23:51 +08:00
|
|
|
segmentContainer, segmentObjects, err := o.fs.c.LargeObjectGetSegments(ctx, container, objectName)
|
2020-08-03 21:45:03 +08:00
|
|
|
if err != nil {
|
2021-01-29 01:09:41 +08:00
|
|
|
fs.Debugf(o, "Failed to get list segments of object: %v", err)
|
|
|
|
return nil, err
|
2020-08-03 21:45:03 +08:00
|
|
|
}
|
2021-01-29 01:09:41 +08:00
|
|
|
var containerSegments = make(map[string][]string)
|
|
|
|
for _, segment := range segmentObjects {
|
|
|
|
if _, ok := containerSegments[segmentContainer]; !ok {
|
|
|
|
containerSegments[segmentContainer] = make([]string, 0, len(segmentObjects))
|
2015-09-26 19:51:05 +08:00
|
|
|
}
|
2022-06-09 04:25:17 +08:00
|
|
|
segments := containerSegments[segmentContainer]
|
2021-01-29 01:09:41 +08:00
|
|
|
segments = append(segments, segment.Name)
|
|
|
|
containerSegments[segmentContainer] = segments
|
2015-09-26 19:51:05 +08:00
|
|
|
}
|
2021-01-29 01:09:41 +08:00
|
|
|
return containerSegments, nil
|
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) removeSegmentsLargeObject(ctx context.Context, containerSegments map[string][]string) error {
|
2021-01-29 01:09:41 +08:00
|
|
|
if containerSegments == nil || len(containerSegments) <= 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
for container, segments := range containerSegments {
|
2021-01-23 01:23:51 +08:00
|
|
|
_, err := o.fs.c.BulkDelete(ctx, container, segments)
|
2021-01-29 01:09:41 +08:00
|
|
|
if err != nil {
|
|
|
|
fs.Debugf(o, "Failed to delete bulk segments %v", err)
|
|
|
|
return err
|
2020-06-23 23:41:46 +08:00
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) getSegmentsDlo(ctx context.Context) (segmentsContainer string, prefix string, err error) {
|
|
|
|
if err = o.readMetaData(ctx); err != nil {
|
2019-11-19 00:26:10 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
dirManifest := o.headers["X-Object-Manifest"]
|
2019-11-21 21:25:02 +08:00
|
|
|
dirManifest, err = url.PathUnescape(dirManifest)
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
2019-11-19 00:26:10 +08:00
|
|
|
delimiter := strings.Index(dirManifest, "/")
|
|
|
|
if len(dirManifest) == 0 || delimiter < 0 {
|
2022-06-09 04:54:39 +08:00
|
|
|
err = errors.New("missing or wrong structure of manifest of Dynamic large object")
|
2019-11-19 00:26:10 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
return dirManifest[:delimiter], dirManifest[delimiter+1:], nil
|
|
|
|
}
|
|
|
|
|
2016-02-28 02:55:21 +08:00
|
|
|
// urlEncode encodes a string so that it is a valid URL
|
|
|
|
//
|
|
|
|
// We don't use any of Go's standard methods as we need `/` not
|
|
|
|
// encoded but we need '&' encoded.
|
|
|
|
func urlEncode(str string) string {
|
|
|
|
var buf bytes.Buffer
|
|
|
|
for i := 0; i < len(str); i++ {
|
|
|
|
c := str[i]
|
2021-01-29 01:09:41 +08:00
|
|
|
if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '/' || c == '.' || c == '_' || c == '-' {
|
2016-02-28 02:55:21 +08:00
|
|
|
_ = buf.WriteByte(c)
|
|
|
|
} else {
|
|
|
|
_, _ = buf.WriteString(fmt.Sprintf("%%%02X", c))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
|
2015-09-26 19:51:05 +08:00
|
|
|
// updateChunks updates the existing object using chunks to a separate
|
|
|
|
// container. It returns a string which prefixes current segments.
|
2021-01-23 01:23:51 +08:00
|
|
|
func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.Headers, size int64, contentType string) (string, error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
|
|
|
segmentsContainer := container + "_segments"
|
2015-09-26 19:51:05 +08:00
|
|
|
// Create the segmentsContainer if it doesn't exist
|
2018-05-04 22:19:50 +08:00
|
|
|
var err error
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
_, rxHeaders, err = o.fs.c.Container(ctx, segmentsContainer)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2017-10-25 07:17:13 +08:00
|
|
|
if err == swift.ContainerNotFound {
|
2018-08-05 04:09:17 +08:00
|
|
|
headers := swift.Headers{}
|
|
|
|
if o.fs.opt.StoragePolicy != "" {
|
|
|
|
headers["X-Storage-Policy"] = o.fs.opt.StoragePolicy
|
|
|
|
}
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
err = o.fs.c.ContainerCreate(ctx, segmentsContainer, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2017-10-25 07:17:13 +08:00
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
2015-09-23 14:57:48 +08:00
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
// Upload the chunks
|
|
|
|
left := size
|
|
|
|
i := 0
|
|
|
|
uniquePrefix := fmt.Sprintf("%s/%d", swift.TimeToFloatString(time.Now()), size)
|
2019-08-07 05:17:40 +08:00
|
|
|
segmentsPath := path.Join(containerPath, uniquePrefix)
|
2017-09-14 13:42:16 +08:00
|
|
|
in := bufio.NewReader(in0)
|
2021-01-29 01:09:41 +08:00
|
|
|
segmentInfos := make([]string, 0, (size/int64(o.fs.opt.ChunkSize))+1)
|
|
|
|
defer atexit.OnError(&err, func() {
|
|
|
|
if o.fs.opt.LeavePartsOnError {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fs.Debugf(o, "Delete segments when err raise %v", err)
|
2022-06-09 04:25:17 +08:00
|
|
|
if len(segmentInfos) == 0 {
|
2021-01-29 01:09:41 +08:00
|
|
|
return
|
|
|
|
}
|
2021-03-02 16:58:14 +08:00
|
|
|
_ctx := context.Background()
|
|
|
|
deleteChunks(_ctx, o, segmentsContainer, segmentInfos)
|
2021-01-29 01:09:41 +08:00
|
|
|
})()
|
2017-09-14 13:42:16 +08:00
|
|
|
for {
|
|
|
|
// can we read at least one byte?
|
2021-01-29 01:09:41 +08:00
|
|
|
if _, err = in.Peek(1); err != nil {
|
2017-09-14 13:42:16 +08:00
|
|
|
if left > 0 {
|
|
|
|
return "", err // read less than expected
|
|
|
|
}
|
2019-08-07 05:17:40 +08:00
|
|
|
fs.Debugf(o, "Uploading segments into %q seems done (%v)", segmentsContainer, err)
|
2017-09-14 13:42:16 +08:00
|
|
|
break
|
|
|
|
}
|
2018-05-15 01:06:57 +08:00
|
|
|
n := int64(o.fs.opt.ChunkSize)
|
2017-09-14 13:42:16 +08:00
|
|
|
if size != -1 {
|
|
|
|
n = min(left, n)
|
|
|
|
headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it
|
|
|
|
left -= n
|
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
segmentReader := io.LimitReader(in, n)
|
|
|
|
segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
|
2019-08-07 05:17:40 +08:00
|
|
|
fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, segmentsContainer)
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
rxHeaders, err = o.fs.c.ObjectPut(ctx, segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
|
2019-06-26 17:54:27 +08:00
|
|
|
if err == nil {
|
|
|
|
segmentInfos = append(segmentInfos, segmentPath)
|
|
|
|
}
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
i++
|
2015-09-23 14:57:48 +08:00
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
// Upload the manifest
|
2019-08-07 05:17:40 +08:00
|
|
|
headers["X-Object-Manifest"] = urlEncode(fmt.Sprintf("%s/%s", segmentsContainer, segmentsPath))
|
2015-10-03 22:38:35 +08:00
|
|
|
headers["Content-Length"] = "0" // set Content-Length as we know it
|
2015-09-26 19:51:05 +08:00
|
|
|
emptyReader := bytes.NewReader(nil)
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2019-03-17 19:38:25 +08:00
|
|
|
var rxHeaders swift.Headers
|
2021-01-23 01:23:51 +08:00
|
|
|
rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, emptyReader, true, "", contentType, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2021-01-29 01:09:41 +08:00
|
|
|
|
|
|
|
if err == nil {
|
|
|
|
//reset data
|
2019-06-27 11:17:42 +08:00
|
|
|
segmentInfos = nil
|
|
|
|
}
|
|
|
|
return uniquePrefix + "/", err
|
|
|
|
}
|
|
|
|
|
2021-01-23 01:23:51 +08:00
|
|
|
func deleteChunks(ctx context.Context, o *Object, segmentsContainer string, segmentInfos []string) {
|
2022-06-09 04:25:17 +08:00
|
|
|
if len(segmentInfos) == 0 {
|
2021-01-29 01:09:41 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
for _, v := range segmentInfos {
|
|
|
|
fs.Debugf(o, "Delete segment file %q on %q", v, segmentsContainer)
|
2021-01-23 01:23:51 +08:00
|
|
|
e := o.fs.c.ObjectDelete(ctx, segmentsContainer, v)
|
2021-01-29 01:09:41 +08:00
|
|
|
if e != nil {
|
|
|
|
fs.Errorf(o, "Error occurred in delete segment file %q on %q, error: %q", v, segmentsContainer, e)
|
2019-06-26 17:54:27 +08:00
|
|
|
}
|
|
|
|
}
|
2015-09-23 14:57:48 +08:00
|
|
|
}
|
|
|
|
|
2014-04-19 00:04:21 +08:00
|
|
|
// Update the object with the contents of the io.Reader, modTime and size
|
|
|
|
//
|
|
|
|
// The new object may have been created if an error is returned
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
|
|
|
if container == "" {
|
|
|
|
return fserrors.FatalError(errors.New("can't upload files to the root"))
|
2017-06-10 21:44:56 +08:00
|
|
|
}
|
2019-08-23 04:30:55 +08:00
|
|
|
err := o.fs.makeContainer(ctx, container)
|
2017-06-07 21:16:50 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-02-18 19:35:25 +08:00
|
|
|
size := src.Size()
|
2019-06-17 16:34:30 +08:00
|
|
|
modTime := src.ModTime(ctx)
|
2016-02-18 19:35:25 +08:00
|
|
|
|
2016-03-18 01:36:20 +08:00
|
|
|
// Note whether this is a dynamic large object before starting
|
2021-01-23 01:23:51 +08:00
|
|
|
isLargeObject, err := o.isLargeObject(ctx)
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-01-29 01:09:41 +08:00
|
|
|
//capture segments before upload
|
|
|
|
var segmentsContainer map[string][]string
|
|
|
|
if isLargeObject {
|
2021-01-23 01:23:51 +08:00
|
|
|
segmentsContainer, _ = o.getSegmentsLargeObject(ctx)
|
2021-01-29 01:09:41 +08:00
|
|
|
}
|
|
|
|
|
2014-04-19 00:04:21 +08:00
|
|
|
// Set the mtime
|
|
|
|
m := swift.Metadata{}
|
|
|
|
m.SetModTime(modTime)
|
2019-06-17 16:34:30 +08:00
|
|
|
contentType := fs.MimeType(ctx, src)
|
2015-09-26 19:51:05 +08:00
|
|
|
headers := m.ObjectHeaders()
|
2020-04-23 18:33:30 +08:00
|
|
|
fs.OpenOptionAddHeaders(options, headers)
|
2021-01-29 01:09:41 +08:00
|
|
|
|
2022-10-06 00:39:24 +08:00
|
|
|
if (size > int64(o.fs.opt.ChunkSize) || (size == -1 && !o.fs.opt.NoChunk)) && !o.fs.opt.NoLargeObjects {
|
2021-01-23 01:23:51 +08:00
|
|
|
_, err = o.updateChunks(ctx, in, headers, size, contentType)
|
2015-09-23 14:57:48 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-12-02 18:23:55 +08:00
|
|
|
o.headers = nil // wipe old metadata
|
2015-09-23 14:57:48 +08:00
|
|
|
} else {
|
2019-08-08 00:21:49 +08:00
|
|
|
var inCount *readers.CountingReader
|
2018-11-28 23:51:18 +08:00
|
|
|
if size >= 0 {
|
|
|
|
headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length if we know it
|
2019-08-08 00:21:49 +08:00
|
|
|
} else {
|
|
|
|
// otherwise count the size for later
|
|
|
|
inCount = readers.NewCountingReader(in)
|
|
|
|
in = inCount
|
2018-11-28 23:51:18 +08:00
|
|
|
}
|
2018-12-02 18:23:55 +08:00
|
|
|
var rxHeaders swift.Headers
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, in, true, "", contentType, headers)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetryHeaders(ctx, rxHeaders, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2015-09-23 14:57:48 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-12-02 18:23:55 +08:00
|
|
|
// set Metadata since ObjectPut checked the hash and length so we know the
|
|
|
|
// object has been safely uploaded
|
|
|
|
o.lastModified = modTime
|
|
|
|
o.size = size
|
2020-09-01 23:04:32 +08:00
|
|
|
o.md5 = rxHeaders["Etag"]
|
2018-12-02 18:23:55 +08:00
|
|
|
o.contentType = contentType
|
|
|
|
o.headers = headers
|
2019-08-08 00:21:49 +08:00
|
|
|
if inCount != nil {
|
|
|
|
// update the size if streaming from the reader
|
|
|
|
o.size = int64(inCount.BytesRead())
|
|
|
|
}
|
2014-07-19 19:36:56 +08:00
|
|
|
}
|
2021-01-23 01:23:51 +08:00
|
|
|
isInContainerVersioning, _ := o.isInContainerVersioning(ctx, container)
|
2021-01-29 01:09:41 +08:00
|
|
|
// If file was a large object and the container is not enable versioning then remove old/all segments
|
|
|
|
if isLargeObject && len(segmentsContainer) > 0 && !isInContainerVersioning {
|
2021-01-23 01:23:51 +08:00
|
|
|
err := o.removeSegmentsLargeObject(ctx, segmentsContainer)
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
2017-02-09 19:01:20 +08:00
|
|
|
fs.Logf(o, "Failed to remove old segments - carrying on with upload: %v", err)
|
2015-09-26 19:51:05 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-12-02 18:23:55 +08:00
|
|
|
// Read the metadata from the newly created object if necessary
|
2021-01-23 01:23:51 +08:00
|
|
|
return o.readMetaData(ctx)
|
2014-04-19 00:04:21 +08:00
|
|
|
}
|
|
|
|
|
2012-12-26 20:23:58 +08:00
|
|
|
// Remove an object
|
2019-11-19 00:26:10 +08:00
|
|
|
func (o *Object) Remove(ctx context.Context) (err error) {
|
2019-08-07 05:17:40 +08:00
|
|
|
container, containerPath := o.split()
|
2019-11-19 00:26:10 +08:00
|
|
|
|
2020-12-28 21:21:11 +08:00
|
|
|
//check object is large object
|
2021-01-23 01:23:51 +08:00
|
|
|
isLargeObject, err := o.isLargeObject(ctx)
|
2020-12-28 21:21:11 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
//check container has enabled version to reserve segment when delete
|
|
|
|
isInContainerVersioning := false
|
|
|
|
if isLargeObject {
|
2021-01-23 01:23:51 +08:00
|
|
|
isInContainerVersioning, err = o.isInContainerVersioning(ctx, container)
|
2020-12-28 21:21:11 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2021-01-29 01:09:41 +08:00
|
|
|
//capture segments object if this object is large object
|
|
|
|
var containerSegments map[string][]string
|
|
|
|
if isLargeObject {
|
2021-01-23 01:23:51 +08:00
|
|
|
containerSegments, err = o.getSegmentsLargeObject(ctx)
|
2020-12-28 21:21:11 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
// Remove file/manifest first
|
2018-11-23 06:15:52 +08:00
|
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
2021-01-23 01:23:51 +08:00
|
|
|
err = o.fs.c.ObjectDelete(ctx, container, containerPath)
|
2021-03-16 23:50:02 +08:00
|
|
|
return shouldRetry(ctx, err)
|
2018-11-23 06:15:52 +08:00
|
|
|
})
|
2015-09-26 19:51:05 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-12-28 21:21:11 +08:00
|
|
|
|
|
|
|
if !isLargeObject || isInContainerVersioning {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-01-29 01:09:41 +08:00
|
|
|
if isLargeObject {
|
2021-01-23 01:23:51 +08:00
|
|
|
return o.removeSegmentsLargeObject(ctx, containerSegments)
|
2015-09-23 14:57:48 +08:00
|
|
|
}
|
2015-09-26 19:51:05 +08:00
|
|
|
return nil
|
2012-12-26 20:23:58 +08:00
|
|
|
}
|
|
|
|
|
2016-09-22 05:13:24 +08:00
|
|
|
// MimeType of an Object if known, "" otherwise
|
2019-06-17 16:34:30 +08:00
|
|
|
func (o *Object) MimeType(ctx context.Context) string {
|
2018-12-02 18:23:55 +08:00
|
|
|
return o.contentType
|
2016-09-22 05:13:24 +08:00
|
|
|
}
|
|
|
|
|
2012-12-26 20:23:58 +08:00
|
|
|
// Check the interfaces are satisfied
|
2015-11-07 19:14:46 +08:00
|
|
|
var (
|
2017-09-14 13:42:16 +08:00
|
|
|
_ fs.Fs = &Fs{}
|
|
|
|
_ fs.Purger = &Fs{}
|
|
|
|
_ fs.PutStreamer = &Fs{}
|
|
|
|
_ fs.Copier = &Fs{}
|
|
|
|
_ fs.ListRer = &Fs{}
|
|
|
|
_ fs.Object = &Object{}
|
|
|
|
_ fs.MimeTyper = &Object{}
|
2015-11-07 19:14:46 +08:00
|
|
|
)
|