diff --git a/backend/all/all.go b/backend/all/all.go index 1abd6770f..df6a6e45b 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -10,6 +10,7 @@ import ( _ "github.com/rclone/rclone/backend/box" _ "github.com/rclone/rclone/backend/cache" _ "github.com/rclone/rclone/backend/chunker" + _ "github.com/rclone/rclone/backend/cloudinary" _ "github.com/rclone/rclone/backend/combine" _ "github.com/rclone/rclone/backend/compress" _ "github.com/rclone/rclone/backend/crypt" diff --git a/backend/cloudinary/api/types.go b/backend/cloudinary/api/types.go new file mode 100644 index 000000000..0a635a56b --- /dev/null +++ b/backend/cloudinary/api/types.go @@ -0,0 +1,48 @@ +// Package api has type definitions for cloudinary +package api + +import ( + "fmt" +) + +// CloudinaryEncoder extends the built-in encoder +type CloudinaryEncoder interface { + // FromStandardPath takes a / separated path in Standard encoding + // and converts it to a / separated path in this encoding. + FromStandardPath(string) string + // FromStandardName takes name in Standard encoding and converts + // it in this encoding. + FromStandardName(string) string + // ToStandardPath takes a / separated path in this encoding + // and converts it to a / separated path in Standard encoding. + ToStandardPath(string) string + // ToStandardName takes name in this encoding and converts + // it in Standard encoding. + ToStandardName(string) string + // Encoded root of the remote (as passed into NewFs) + FromStandardFullPath(string) string +} + +// UpdateOptions was created to pass options from Update to Put +type UpdateOptions struct { + PublicID string + ResourceType string + DeliveryType string + AssetFolder string + DisplayName string +} + +// Header formats the option as a string +func (o *UpdateOptions) Header() (string, string) { + return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o *UpdateOptions) Mandatory() bool { + return false +} + +// String formats the option into human-readable form +func (o *UpdateOptions) String() string { + return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID) +} diff --git a/backend/cloudinary/cloudinary.go b/backend/cloudinary/cloudinary.go new file mode 100644 index 000000000..6af5bd370 --- /dev/null +++ b/backend/cloudinary/cloudinary.go @@ -0,0 +1,711 @@ +// Package cloudinary provides an interface to the Cloudinary DAM +package cloudinary + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "time" + + "github.com/cloudinary/cloudinary-go/v2" + SDKApi "github.com/cloudinary/cloudinary-go/v2/api" + "github.com/cloudinary/cloudinary-go/v2/api/admin" + "github.com/cloudinary/cloudinary-go/v2/api/admin/search" + "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/rclone/rclone/backend/cloudinary/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" + "github.com/zeebo/blake3" +) + +// Cloudinary shouldn't have a trailing dot if there is no path +func cldPathDir(somePath string) string { + if somePath == "" || somePath == "." { + return somePath + } + dir := path.Dir(somePath) + if dir == "." { + return "" + } + return dir +} + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "cloudinary", + Description: "Cloudinary", + NewFs: NewFs, + Options: []fs.Option{ + { + Name: "cloud_name", + Help: "Cloudinary Environment Name", + Required: true, + Sensitive: true, + }, + { + Name: "api_key", + Help: "Cloudinary API Key", + Required: true, + Sensitive: true, + }, + { + Name: "api_secret", + Help: "Cloudinary API Secret", + Required: true, + Sensitive: true, + }, + { + Name: "upload_prefix", + Help: "Specify the API endpoint for environments out of the US", + }, + { + Name: "upload_preset", + Help: "Upload Preset to select asset manipulation on upload", + }, + { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: (encoder.Base | // Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot + encoder.EncodeSlash | + encoder.EncodeLtGt | + encoder.EncodeDoubleQuote | + encoder.EncodeQuestion | + encoder.EncodeAsterisk | + encoder.EncodePipe | + encoder.EncodeHash | + encoder.EncodePercent | + encoder.EncodeBackSlash | + encoder.EncodeDel | + encoder.EncodeCtl | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8 | + encoder.EncodeDot), + }, + { + Name: "eventually_consistent_delay", + Default: fs.Duration(0), + Advanced: true, + Help: "Wait N seconds for eventual consistency of the databases that support the backend operation", + }, + }, + }) +} + +// Options defines the configuration for this backend +type Options struct { + CloudName string `config:"cloud_name"` + APIKey string `config:"api_key"` + APISecret string `config:"api_secret"` + UploadPrefix string `config:"upload_prefix"` + UploadPreset string `config:"upload_preset"` + Enc encoder.MultiEncoder `config:"encoding"` + EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"` +} + +// Fs represents a remote cloudinary server +type Fs struct { + name string + root string + opt Options + features *fs.Features + pacer *fs.Pacer + srv *rest.Client // For downloading assets via the Cloudinary CDN + cld *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK + lastCRUD time.Time +} + +// Object describes a cloudinary object +type Object struct { + fs *Fs + remote string + size int64 + modTime time.Time + url string + md5sum string + publicID string + resourceType string + deliveryType string +} + +// NewFs constructs an Fs from the path, bucket:path +func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) { + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + // Initialize the Cloudinary client + cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret) + if err != nil { + return nil, fmt.Errorf("failed to create Cloudinary client: %w", err) + } + cld.Admin.Client = *fshttp.NewClient(ctx) + cld.Upload.Client = *fshttp.NewClient(ctx) + if opt.UploadPrefix != "" { + cld.Config.API.UploadPrefix = opt.UploadPrefix + } + client := fshttp.NewClient(ctx) + f := &Fs{ + name: name, + root: root, + opt: *opt, + cld: cld, + pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))), + srv: rest.NewClient(client), + } + + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + }).Fill(ctx, f) + + if root != "" { + // Check to see if the root actually an existing file + remote := path.Base(root) + f.root = cldPathDir(root) + _, err := f.NewObject(ctx, remote) + if err != nil { + if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) { + // File doesn't exist so return the previous root + f.root = root + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// ------------------------------------------------------------ + +// FromStandardPath implementation of the api.CloudinaryEncoder +func (f *Fs) FromStandardPath(s string) string { + return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06") +} + +// FromStandardName implementation of the api.CloudinaryEncoder +func (f *Fs) FromStandardName(s string) string { + return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06") +} + +// ToStandardPath implementation of the api.CloudinaryEncoder +func (f *Fs) ToStandardPath(s string) string { + return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&") +} + +// ToStandardName implementation of the api.CloudinaryEncoder +func (f *Fs) ToStandardName(s string) string { + return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") +} + +// FromStandardFullPath encodes a full path to Cloudinary standard +func (f *Fs) FromStandardFullPath(dir string) string { + return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir)) +} + +// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK +func (f *Fs) ToAssetFolderAPI(dir string) string { + return strings.ReplaceAll(dir, "%", "%25") +} + +// ToDisplayNameElastic encodes a special case of elasticsearch +func (f *Fs) ToDisplayNameElastic(dir string) string { + return strings.ReplaceAll(dir, "!", "\\!") +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// WaitEventuallyConsistent waits till the FS is eventually consistent +func (f *Fs) WaitEventuallyConsistent() { + if f.opt.EventuallyConsistentDelay == fs.Duration(0) { + return + } + delay := time.Duration(f.opt.EventuallyConsistentDelay) + timeSinceLastCRUD := time.Since(f.lastCRUD) + if timeSinceLastCRUD < delay { + time.Sleep(delay - timeSinceLastCRUD) + } +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Cloudinary root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// List the objects and directories in dir into entries +func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) { + remotePrefix := f.FromStandardFullPath(dir) + if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") { + remotePrefix += "/" + } + + var entries fs.DirEntries + dirs := make(map[string]struct{}) + nextCursor := "" + f.WaitEventuallyConsistent() + for { + // user the folders api to list folders. + folderParams := admin.SubFoldersParams{ + Folder: f.ToAssetFolderAPI(remotePrefix), + MaxResults: 500, + } + if nextCursor != "" { + folderParams.NextCursor = nextCursor + } + + results, err := f.cld.Admin.SubFolders(ctx, folderParams) + if err != nil { + return nil, fmt.Errorf("failed to list sub-folders: %w", err) + } + if results.Error.Message != "" { + if strings.HasPrefix(results.Error.Message, "Can't find folder with path") { + return nil, fs.ErrorDirNotFound + } + + return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message) + } + + for _, folder := range results.Folders { + relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix)) + parts := strings.Split(relativePath, "/") + + // It's a directory + dirName := parts[len(parts)-1] + if _, found := dirs[dirName]; !found { + d := fs.NewDir(path.Join(dir, dirName), time.Time{}) + entries = append(entries, d) + dirs[dirName] = struct{}{} + } + } + // Break if there are no more results + if results.NextCursor == "" { + break + } + nextCursor = results.NextCursor + } + + for { + // Use the assets.AssetsByAssetFolder API to list assets + assetsParams := admin.AssetsByAssetFolderParams{ + AssetFolder: remotePrefix, + MaxResults: 500, + } + if nextCursor != "" { + assetsParams.NextCursor = nextCursor + } + + results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams) + if err != nil { + return nil, fmt.Errorf("failed to list assets: %w", err) + } + + for _, asset := range results.Assets { + remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName) + if dir != "" { + remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)) + } + o := &Object{ + fs: f, + remote: remote, + size: int64(asset.Bytes), + modTime: asset.CreatedAt, + url: asset.SecureURL, + publicID: asset.PublicID, + resourceType: asset.AssetType, + deliveryType: asset.Type, + } + entries = append(entries, o) + } + + // Break if there are no more results + if results.NextCursor == "" { + break + } + nextCursor = results.NextCursor + } + + return entries, nil +} + +// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + searchParams := search.Query{ + Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"", + f.FromStandardFullPath(cldPathDir(remote)), + f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))), + SortBy: []search.SortByField{{"uploaded_at": "desc"}}, + MaxResults: 2, + } + var results *admin.SearchResult + f.WaitEventuallyConsistent() + err := f.pacer.Call(func() (bool, error) { + var err1 error + results, err1 = f.cld.Admin.Search(ctx, searchParams) + if err1 == nil && results.TotalCount != len(results.Assets) { + err1 = errors.New("partial response so waiting for eventual consistency") + } + return shouldRetry(ctx, nil, err1) + }) + if err != nil { + return nil, fs.ErrorObjectNotFound + } + if results.TotalCount == 0 || len(results.Assets) == 0 { + return nil, fs.ErrorObjectNotFound + } + asset := results.Assets[0] + + o := &Object{ + fs: f, + remote: remote, + size: int64(asset.Bytes), + modTime: asset.UploadedAt, + url: asset.SecureURL, + md5sum: asset.Etag, + publicID: asset.PublicID, + resourceType: asset.ResourceType, + deliveryType: asset.Type, + } + + return o, nil +} + +func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string { + payload := []byte(path.Join(assetFolder, displayName)) + hash := blake3.Sum256(payload) + return hex.EncodeToString(hash[:]) +} + +// Put uploads content to Cloudinary +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + if src.Size() == 0 { + return nil, fs.ErrorCantUploadEmptyFiles + } + + params := uploader.UploadParams{ + UploadPreset: f.opt.UploadPreset, + } + + updateObject := false + var modTime time.Time + for _, option := range options { + if updateOptions, ok := option.(*api.UpdateOptions); ok { + if updateOptions.PublicID != "" { + updateObject = true + params.Overwrite = SDKApi.Bool(true) + params.Invalidate = SDKApi.Bool(true) + params.PublicID = updateOptions.PublicID + params.ResourceType = updateOptions.ResourceType + params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType) + params.AssetFolder = updateOptions.AssetFolder + params.DisplayName = updateOptions.DisplayName + modTime = src.ModTime(ctx) + } + } + } + if !updateObject { + params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote())) + params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote())) + // We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified). + // We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case. + // Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support. + params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx)) + } + uploadResult, err := f.cld.Upload.Upload(ctx, in, params) + f.lastCRUD = time.Now() + if err != nil { + return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err) + } + if !updateObject { + modTime = uploadResult.CreatedAt + } + if uploadResult.Error.Message != "" { + return nil, errors.New(uploadResult.Error.Message) + } + + o := &Object{ + fs: f, + remote: src.Remote(), + size: int64(uploadResult.Bytes), + modTime: modTime, + url: uploadResult.SecureURL, + md5sum: uploadResult.Etag, + publicID: uploadResult.PublicID, + resourceType: uploadResult.ResourceType, + deliveryType: uploadResult.Type, + } + return o, nil +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Hashes returns the supported hash sets +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// Mkdir creates empty folders +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))} + res, err := f.cld.Admin.CreateFolder(ctx, params) + f.lastCRUD = time.Now() + if err != nil { + return err + } + if res.Error.Message != "" { + return errors.New(res.Error.Message) + } + + return nil +} + +// Rmdir deletes empty folders +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + // Additional test because Cloudinary will delete folders without + // assets, regardless of empty sub-folders + folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir)) + folderParams := admin.SubFoldersParams{ + Folder: folder, + MaxResults: 1, + } + results, err := f.cld.Admin.SubFolders(ctx, folderParams) + if err != nil { + return err + } + if results.TotalCount > 0 { + return fs.ErrorDirectoryNotEmpty + } + + params := admin.DeleteFolderParams{Folder: folder} + res, err := f.cld.Admin.DeleteFolder(ctx, params) + f.lastCRUD = time.Now() + if err != nil { + return err + } + if res.Error.Message != "" { + if strings.HasPrefix(res.Error.Message, "Can't find folder with path") { + return fs.ErrorDirNotFound + } + + return errors.New(res.Error.Message) + } + + return nil +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 420, // Too Many Requests (legacy) + 429, // Too Many Requests + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + if err != nil { + tryAgain := "Try again on " + if idx := strings.Index(err.Error(), tryAgain); idx != -1 { + layout := "2006-01-02 15:04:05 UTC" + dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)] + timestamp, err2 := time.Parse(layout, dateStr) + if err2 == nil { + return true, fserrors.NewErrorRetryAfter(time.Until(timestamp)) + } + } + + fs.Debugf(nil, "Retrying API error %v", err) + return true, err + } + + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// ------------------------------------------------------------ + +// Hash returns the MD5 of an object +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + if ty != hash.MD5 { + return "", hash.ErrUnsupported + } + return o.md5sum, nil +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime returns the modification time of the object +func (o *Object) ModTime(ctx context.Context) time.Time { + return o.modTime +} + +// Size of object in bytes +func (o *Object) Size() int64 { + return o.size +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: o.url, + Options: options, + } + var offset int64 + var count int64 + var key string + var value string + fs.FixRangeOption(options, o.size) + for _, option := range options { + switch x := option.(type) { + case *fs.RangeOption: + offset, count = x.Decode(o.size) + if count < 0 { + count = o.size - offset + } + key, value = option.Header() + case *fs.SeekOption: + offset = x.Offset + count = o.size - offset + key, value = option.Header() + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + if key != "" && value != "" { + opts.ExtraHeaders = make(map[string]string) + opts.ExtraHeaders[key] = value + } + // Make sure that the asset is fully available + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + if err == nil { + cl, clErr := strconv.Atoi(resp.Header.Get("content-length")) + if clErr == nil && count == int64(cl) { + return false, nil + } + } + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err) + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + options = append(options, &api.UpdateOptions{ + PublicID: o.publicID, + ResourceType: o.resourceType, + DeliveryType: o.deliveryType, + DisplayName: api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())), + AssetFolder: o.fs.FromStandardFullPath(cldPathDir(o.Remote())), + }) + updatedObj, err := o.fs.Put(ctx, in, src, options...) + if err != nil { + return err + } + if uo, ok := updatedObj.(*Object); ok { + o.size = uo.size + o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time + o.url = uo.url + o.md5sum = uo.md5sum + o.publicID = uo.publicID + o.resourceType = uo.resourceType + o.deliveryType = uo.deliveryType + } + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + params := uploader.DestroyParams{ + PublicID: o.publicID, + ResourceType: o.resourceType, + Type: o.deliveryType, + } + res, dErr := o.fs.cld.Upload.Destroy(ctx, params) + o.fs.lastCRUD = time.Now() + if dErr != nil { + return dErr + } + + if res.Error.Message != "" { + return errors.New(res.Error.Message) + } + + if res.Result != "ok" { + return errors.New(res.Result) + } + + return nil +} diff --git a/backend/cloudinary/cloudinary_test.go b/backend/cloudinary/cloudinary_test.go new file mode 100644 index 000000000..009771eaa --- /dev/null +++ b/backend/cloudinary/cloudinary_test.go @@ -0,0 +1,23 @@ +// Test Cloudinary filesystem interface + +package cloudinary_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/cloudinary" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + name := "TestCloudinary" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*cloudinary.Object)(nil), + SkipInvalidUTF8: true, + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "eventually_consistent_delay", Value: "7"}, + }, + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index 990ab7130..d2e756457 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -35,6 +35,7 @@ docs = [ "box.md", "cache.md", "chunker.md", + "cloudinary.md", "sharefile.md", "crypt.md", "compress.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index 711b55041..1a1c7fc79 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.ir/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}} {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}} {{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}} +{{< provider name="Cloudinary" home="https://cloudinary.com/" config="/cloudinary/" >}} {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} diff --git a/docs/content/cloudinary.md b/docs/content/cloudinary.md new file mode 100644 index 000000000..6b822d372 --- /dev/null +++ b/docs/content/cloudinary.md @@ -0,0 +1,222 @@ +--- +title: "Cloudinary" +description: "Rclone docs for Cloudinary backend" +versionIntroduced: "v1.69" + +--- +# {{< icon "fa fa-cloud" >}} Cloudinary + +This is a backend for the [Cloudinary](https://cloudinary.com/) platform + +## About Cloudinary + +[Cloudinary](https://cloudinary.com/) is an image and video API platform. +Trusted by 1.5 million developers and 10,000 enterprise and hyper-growth companies as a critical part of their tech stack to deliver visually engaging experiences. + +## Accounts & Pricing + +To use this backend, you need to [create a free account](https://cloudinary.com/users/register_free) on Cloudinary. Start with a free plan with generous usage limits. Then, as your requirements grow, upgrade to a plan that best fits your needs. See [the pricing details](https://cloudinary.com/pricing). + +## Securing Your Credentials + +Please refer to the [docs](/docs/#configuration-encryption-cheatsheet) + +## Configuration + +Here is an example of making a Cloudinary configuration. + +First, create a [cloudinary.com](https://cloudinary.com/users/register_free) account and choose a plan. + +You will need to log in and get the `API Key` and `API Secret` for your account from the developer section. + +Now run + +`rclone config` + +Follow the interactive setup process: + +```text +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter the name for the new remote. +name> cloudinary-media-library + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / cloudinary.com +\ (cloudinary) +[snip] +Storage> cloudinary + +Option cloud_name. +You can find your cloudinary.com cloud_name in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard) +Enter a value. +cloud_name> **************************** + +Option api_key. +You can find your cloudinary.com api key in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard) +Enter a value. +api_key> **************************** + +Option api_secret. +You can find your cloudinary.com api secret in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard) +This value must be a single character, one of the following: y, g. +y/g> y +Enter a value. +api_secret> **************************** + +Option upload_prefix. +[Upload prefix](https://cloudinary.com/documentation/cloudinary_sdks#configuration_parameters) to specify alternative data center +Enter a value. +upload_prefix> + +Option upload_preset. +[Upload presets](https://cloudinary.com/documentation/upload_presets) can be defined for different upload profiles +Enter a value. +upload_preset> + +Edit advanced config? +y) Yes +n) No (default) +y/n> n + +Configuration complete. +Options: +- type: cloudinary +- api_key: **************************** +- api_secret: **************************** +- cloud_name: **************************** +- upload_prefix: +- upload_preset: + +Keep this "cloudinary-media-library" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +List directories in the top level of your Media Library + +`rclone lsd cloudinary-media-library:` + +Make a new directory. + +`rclone mkdir cloudinary-media-library:directory` + +List the contents of a directory. + +`rclone ls cloudinary-media-library:directory` + +### Modified time and hashes + +Cloudinary stores md5 and timestamps for any successful Put automatically and read-only. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/cloudinary/cloudinary.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to cloudinary (Cloudinary). + +#### --cloudinary-cloud-name + +Cloudinary Environment Name + +Properties: + +- Config: cloud_name +- Env Var: RCLONE_CLOUDINARY_CLOUD_NAME +- Type: string +- Required: true + +#### --cloudinary-api-key + +Cloudinary API Key + +Properties: + +- Config: api_key +- Env Var: RCLONE_CLOUDINARY_API_KEY +- Type: string +- Required: true + +#### --cloudinary-api-secret + +Cloudinary API Secret + +**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). + +Properties: + +- Config: api_secret +- Env Var: RCLONE_CLOUDINARY_API_SECRET +- Type: string +- Required: true + +#### --cloudinary-upload-prefix + +Specify the API endpoint for environments out of the US + +Properties: + +- Config: upload_prefix +- Env Var: RCLONE_CLOUDINARY_UPLOAD_PREFIX +- Type: string +- Required: false + +#### --cloudinary-upload-preset + +Upload Preset to select asset manipulation on upload + +Properties: + +- Config: upload_preset +- Env Var: RCLONE_CLOUDINARY_UPLOAD_PRESET +- Type: string +- Required: false + +### Advanced options + +Here are the Advanced options specific to cloudinary (Cloudinary). + +#### --cloudinary-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_CLOUDINARY_ENCODING +- Type: Encoding +- Default: Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot + +#### --cloudinary-eventually-consistent-delay + +Wait N seconds for eventual consistency of the databases that support the backend operation + +Properties: + +- Config: eventually_consistent_delay +- Env Var: RCLONE_CLOUDINARY_EVENTUALLY_CONSISTENT_DELAY +- Type: Duration +- Default: 0s + +#### --cloudinary-description + +Description of the remote. + +Properties: + +- Config: description +- Env Var: RCLONE_CLOUDINARY_DESCRIPTION +- Type: string +- Required: false + +{{< rem autogenerated options stop >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index cb65ed746..dccb5c5f7 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -36,6 +36,7 @@ See the following for detailed instructions for * [Chunker](/chunker/) - transparently splits large files for other remotes * [Citrix ShareFile](/sharefile/) * [Compress](/compress/) + * [Cloudinary](/cloudinary/) * [Combine](/combine/) * [Crypt](/crypt/) - to encrypt other remotes * [DigitalOcean Spaces](/s3/#digitalocean-spaces) diff --git a/docs/content/overview.md b/docs/content/overview.md index 6376dc253..9cf84899f 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -22,6 +22,7 @@ Here is an overview of the major features of each cloud storage system. | Backblaze B2 | SHA1 | R/W | No | No | R/W | - | | Box | SHA1 | R/W | Yes | No | - | - | | Citrix ShareFile | MD5 | R/W | Yes | No | - | - | +| Cloudinary | MD5 | R | No | Yes | - | - | | Dropbox | DBHASH ยน | R | Yes | No | - | - | | Enterprise File Fabric | - | R/W | Yes | No | R/W | - | | Files.com | MD5, CRC32 | DR/W | Yes | No | R | - | @@ -502,6 +503,7 @@ upon backend-specific capabilities. | Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | | Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | | Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | +| Cloudinary | No | No | No | No | No | No | Yes | No | No | No | No | | Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | Files.com | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes | | FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes | diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index bc7bb0ce3..dd180a0d2 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -58,6 +58,7 @@ Backblaze B2 Box Chunker (splits large files) + Cloudinary Compress (transparent gzip compression) Combine (remotes into a directory tree) Citrix ShareFile diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 1565c7e8e..24fed9290 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -102,6 +102,21 @@ backends: fastlist: true maxfile: 1k ## end chunker + - backend: "cloudinary" + remote: "TestCloudinary:" + fastlist: false + ignore: + # fs/operations + - TestCheckSum + - TestCheckSumDownload + - TestHashSums/Md5 + - TestReadFile + - TestCopyURL + - TestMoveFileWithIgnoreExisting + #vfs + - TestFileSetModTime/cache=off,open=false,write=false + - TestFileSetModTime/cache=off,open=true,write=false + - TestRWFileHandleWriteNoWrite - backend: "combine" remote: "TestCombine:dir1" fastlist: false diff --git a/go.mod b/go.mod index b9b03e310..c1691b78f 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/smithy-go v1.22.1 github.com/buengese/sgzip v0.1.1 + github.com/cloudinary/cloudinary-go/v2 v2.9.0 github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 github.com/colinmarc/hdfs/v2 v2.4.0 github.com/coreos/go-semver v0.3.1 @@ -76,6 +77,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 github.com/yunify/qingstor-sdk-go/v3 v3.2.0 + github.com/zeebo/blake3 v0.2.3 go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 golang.org/x/crypto v0.31.0 @@ -128,6 +130,7 @@ require ( github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -151,6 +154,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect + github.com/gorilla/schema v1.4.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -197,7 +201,6 @@ require ( github.com/tklauser/numcpus v0.7.0 // indirect github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect diff --git a/go.sum b/go.sum index 70ad47b58..6a3820d35 100644 --- a/go.sum +++ b/go.sum @@ -175,6 +175,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudinary/cloudinary-go/v2 v2.9.0 h1:8C76QklmuV4qmKAC7cUnu9D68X9kCkFMuLspPikECCo= +github.com/cloudinary/cloudinary-go/v2 v2.9.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo= github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 h1:nPP4suUiNage0vvyEBgfAnhTPwwXhNqtHmSuiCIQwKU= github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= @@ -190,6 +192,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo= github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -345,6 +349,8 @@ github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= +github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=