rclone/backend/ulozto/ulozto.go

// Package ulozto provides an interface to the Uloz.to storage system.
package ulozto

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/gob"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/backend/ulozto/api"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config"
	"github.com/rclone/rclone/fs/config/configmap"
	"github.com/rclone/rclone/fs/config/configstruct"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/fs/fshttp"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/lib/dircache"
	"github.com/rclone/rclone/lib/encoder"
	"github.com/rclone/rclone/lib/pacer"
	"github.com/rclone/rclone/lib/rest"
)

// TODO Uloz.to only supports file names of 255 characters or less and silently truncates names that are longer.

const (
	minSleep      = 10 * time.Millisecond
	maxSleep      = 2 * time.Second
	decayConstant = 2 // bigger for slower decay, exponential
	rootURL       = "https://apis.uloz.to"
	// TODO temporary limitation, remove with chunked upload impl
	maxFileSizeBytes = 2500 * 1024 * 1024
)

// Options defines the configuration for this backend
type Options struct {
	AppToken       string               `config:"app_token"`
	Username       string               `config:"username"`
	Password       string               `config:"password"`
	RootFolderSlug string               `config:"root_folder_slug"`
	Enc            encoder.MultiEncoder `config:"encoding"`
	ListPageSize   int                  `config:"list_page_size"`
}

func init() {
	fs.Register(&fs.RegInfo{
		Name:        "ulozto",
		Description: "Uloz.to",
		NewFs:       NewFs,
		Options: []fs.Option{
			{
				Name:    "app_token",
				Default: "",
				Help: "The application token identifying the app. An app API key can be either found in the API " +
					"doc https://uloz.to/upload-resumable-api-beta or obtained from customer service.",
				Sensitive: true,
			},
			{
				Name:      "username",
				Default:   "",
				Help:      "The username of the principal to operate as.",
				Sensitive: true,
			},
			{
				Name:      "password",
				Default:   "",
				Help:      "The password for the user.",
				Sensitive: true,
			},
			{
				Name: "root_folder_slug",
				Help: "If set, rclone will use this folder as the root folder for all operations. For example, " +
					"if the slug identifies 'foo/bar/', 'ulozto:baz' is equivalent to 'ulozto:foo/bar/baz' without " +
					"any root slug set.",
				Default:   "",
				Advanced:  true,
				Sensitive: true,
			},
			{
				Name:     "list_page_size",
				Default:  500,
				Help:     "The size of a single page for list commands. 1-500",
				Advanced: true,
			},
			{
				Name:     config.ConfigEncoding,
				Help:     config.ConfigEncodingHelp,
				Advanced: true,
				Default:  encoder.Display | encoder.EncodeInvalidUtf8 | encoder.EncodeBackSlash,
			},
		}})
}

// Fs represents a remote uloz.to storage
type Fs struct {
	name     string             // name of this remote
	root     string             // the path we are working on
	opt      Options            // parsed options
	features *fs.Features       // optional features
	rest     *rest.Client       // REST client with authentication headers set, used to communicate with API endpoints
	cdn      *rest.Client       // REST client without authentication headers set, used for CDN payload upload/download
	dirCache *dircache.DirCache // Map of directory path to directory id
	pacer    *fs.Pacer          // pacer for API calls
}

// NewFs constructs a Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
	// Parse config into Options struct
	opt := new(Options)
	err := configstruct.Set(m, opt)
	if err != nil {
		return nil, err
	}

	client := fshttp.NewClient(ctx)

	f := &Fs{
		name:  name,
		root:  root,
		opt:   *opt,
		cdn:   rest.NewClient(client),
		rest:  rest.NewClient(client).SetRoot(rootURL),
		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
	}
	f.features = (&fs.Features{
		DuplicateFiles:          true,
		CanHaveEmptyDirectories: true,
	}).Fill(ctx, f)
	f.rest.SetErrorHandler(errorHandler)

	f.rest.SetHeader("X-Auth-Token", f.opt.AppToken)

	auth, err := f.authenticate(ctx)

	if err != nil {
		return f, err
	}

	var rootSlug string
	if opt.RootFolderSlug == "" {
		rootSlug = auth.Session.User.RootFolderSlug
	} else {
		rootSlug = opt.RootFolderSlug
	}

	f.dirCache = dircache.New(root, rootSlug, f)

	err = f.dirCache.FindRoot(ctx, false)

	if errors.Is(err, fs.ErrorDirNotFound) {
		// All good, we'll create the folder later on.
		return f, nil
	}

	if errors.Is(err, fs.ErrorIsFile) {
		rootFolder, _ := dircache.SplitPath(root)
		f.root = rootFolder
		f.dirCache = dircache.New(rootFolder, rootSlug, f)
		err = f.dirCache.FindRoot(ctx, false)
		if err != nil {
			return f, err
		}
		return f, fs.ErrorIsFile
	}

	return f, err
}

// errorHandler parses a non 2xx error response into an error
func errorHandler(resp *http.Response) error {
	// Decode error response
	errResponse := new(api.Error)
	err := rest.DecodeJSON(resp, &errResponse)
	if err != nil {
		fs.Debugf(nil, "Couldn't decode error response: %v", err)
	}
	if errResponse.StatusCode == 0 {
		errResponse.StatusCode = resp.StatusCode
	}

	return errors.WithStack(errResponse)
}

// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
	429, // Too Many Requests.
	// TODO: random 500s should be retried but the error code corresponds to a known issue with uploading large files,
	//   leading to numerous (slow & resource consuming) retries. Don't retry them until the root cause is addressed.
	// 500, // Internal Server Error
	502, // Bad Gateway
	503, // Service Unavailable
	504, // Gateway Timeout
}

// shouldRetry returns a boolean whether this resp and err should be retried.
// It also returns the err for convenience.
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, reauth bool) (bool, error) {
	if err == nil {
		return false, nil
	}

	if fserrors.ContextError(ctx, &err) {
		return false, err
	}

	var apiErr *api.Error
	if resp != nil && resp.StatusCode == 401 && errors.As(err, &apiErr) && apiErr.ErrorCode == 70001 {
		fs.Debugf(nil, "Should retry: %v", err)

		if reauth {
			_, err = f.authenticate(ctx)
			if err != nil {
				return false, err
			}
		}

		return true, err
	}

	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}

func (f *Fs) authenticate(ctx context.Context) (response *api.AuthenticateResponse, err error) {
	// TODO only reauth once if the token expires

	// Remove the old user token
	f.rest.RemoveHeader("X-User-Token")

	opts := rest.Opts{
		Method: "PUT",
		Path:   "/v6/session",
	}

	authRequest := api.AuthenticateRequest{
		Login:    f.opt.Username,
		Password: f.opt.Password,
	}

	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := f.rest.CallJSON(ctx, &opts, &authRequest, &response)
		return f.shouldRetry(ctx, httpResp, err, false)
	})

	if err != nil {
		return nil, err
	}

	f.rest.SetHeader("X-User-Token", response.TokenID)

	return response, nil
}

// UploadSession represents a single Uloz.to upload session.
//
// Uloz.to supports uploading multiple files at once and committing them atomically. This functionality isn't being used
// by the backend implementation and for simplicity, each session corresponds to a single file being uploaded.
type UploadSession struct {
	Filesystem  *Fs
	URL         string
	PrivateSlug string
	ValidUntil  time.Time
}

func (f *Fs) createUploadSession(ctx context.Context) (session *UploadSession, err error) {
	session = &UploadSession{
		Filesystem: f,
	}

	err = session.renewUploadSession(ctx)
	if err != nil {
		return nil, err
	}

	return session, nil
}

func (session *UploadSession) renewUploadSession(ctx context.Context) error {
	opts := rest.Opts{
		Method:     "POST",
		Path:       "/v5/upload/link",
		Parameters: url.Values{},
	}

	createUploadURLReq := api.CreateUploadURLRequest{
		UserLogin: session.Filesystem.opt.Username,
		Realm:     "ulozto",
	}

	if session.PrivateSlug != "" {
		createUploadURLReq.ExistingSessionSlug = session.PrivateSlug
	}

	var err error
	var response api.CreateUploadURLResponse

	err = session.Filesystem.pacer.Call(func() (bool, error) {
		httpResp, err := session.Filesystem.rest.CallJSON(ctx, &opts, &createUploadURLReq, &response)
		return session.Filesystem.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return err
	}

	session.PrivateSlug = response.PrivateSlug
	session.URL = response.UploadURL
	session.ValidUntil = response.ValidUntil

	return nil
}

func (f *Fs) uploadUnchecked(ctx context.Context, name, parentSlug string, info fs.ObjectInfo, payload io.Reader) (fs.Object, error) {
	session, err := f.createUploadSession(ctx)

	if err != nil {
		return nil, err
	}

	hashes := hash.NewHashSet(hash.MD5, hash.SHA256)
	hasher, err := hash.NewMultiHasherTypes(hashes)

	if err != nil {
		return nil, err
	}

	payload = io.TeeReader(payload, hasher)

	encodedName := f.opt.Enc.FromStandardName(name)

	opts := rest.Opts{
		Method: "POST",
		Body:   payload,
		// Not using Parameters as the session URL has parameters itself
		RootURL:              session.URL + "&batch_file_id=1&is_porn=false",
		MultipartContentName: "file",
		MultipartFileName:    encodedName,
		Parameters:           url.Values{},
	}
	if info.Size() > 0 {
		size := info.Size()
		opts.ContentLength = &size
	}

	var uploadResponse api.SendFilePayloadResponse

	err = f.pacer.CallNoRetry(func() (bool, error) {
		httpResp, err := f.cdn.CallJSON(ctx, &opts, nil, &uploadResponse)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return nil, err
	}

	sha256digest, err := hasher.Sum(hash.SHA256)
	if err != nil {
		return nil, err
	}

	md5digest, err := hasher.Sum(hash.MD5)
	if err != nil {
		return nil, err
	}

	if hex.EncodeToString(md5digest) != uploadResponse.Md5 {
		return nil, errors.New("MD5 digest mismatch")
	}

	metadata := DescriptionEncodedMetadata{
		Md5Hash:            md5digest,
		Sha256Hash:         sha256digest,
		ModTimeEpochMicros: info.ModTime(ctx).UnixMicro(),
	}

	encodedMetadata, err := metadata.encode()

	if err != nil {
		return nil, err
	}

	// Successfully uploaded, now move the file where it belongs and commit it
	updateReq := api.BatchUpdateFilePropertiesRequest{
		Name:         encodedName,
		FolderSlug:   parentSlug,
		Description:  encodedMetadata,
		Slugs:        []string{uploadResponse.Slug},
		UploadTokens: map[string]string{uploadResponse.Slug: session.PrivateSlug + ":1"},
	}

	var updateResponse []api.File

	opts = rest.Opts{
		Method:     "PATCH",
		Path:       "/v8/file-list/private",
		Parameters: url.Values{},
	}

	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := session.Filesystem.rest.CallJSON(ctx, &opts, &updateReq, &updateResponse)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return nil, err
	}

	if len(updateResponse) != 1 {
		return nil, errors.New("unexpected number of files in the response")
	}

	opts = rest.Opts{
		Method:     "PATCH",
		Path:       "/v8/upload-batch/private/" + session.PrivateSlug,
		Parameters: url.Values{},
	}

	commitRequest := api.CommitUploadBatchRequest{
		Status:     "confirmed",
		OwnerLogin: f.opt.Username,
	}

	var commitResponse api.CommitUploadBatchResponse

	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := session.Filesystem.rest.CallJSON(ctx, &opts, &commitRequest, &commitResponse)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return nil, err
	}

	file, err := f.newObjectWithInfo(ctx, info.Remote(), &updateResponse[0])

	return file, err
}

// Put implements the mandatory method fs.Fs.Put.
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	// TODO: workaround for uloz.to's bug. Remove when chunked upload support is implemented.
	if src.Size() > maxFileSizeBytes {
		return nil, errors.New("file size over the supported max threshold")
	}
	existingObj, err := f.NewObject(ctx, src.Remote())

	switch {
	case err == nil:
		return existingObj, existingObj.Update(ctx, in, src, options...)
	case errors.Is(err, fs.ErrorObjectNotFound):
		// Not found so create it
		return f.PutUnchecked(ctx, in, src, options...)
	default:
		return nil, err
	}
}

// PutUnchecked implements the optional interface fs.PutUncheckeder.
//
// Uloz.to allows to have multiple files of the same name in the same folder.
func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	filename, folderSlug, err := f.dirCache.FindPath(ctx, src.Remote(), true)

	if err != nil {
		return nil, err
	}

	return f.uploadUnchecked(ctx, filename, folderSlug, src, in)
}

// Mkdir implements the mandatory method fs.Fs.Mkdir.
func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
	_, err = f.dirCache.FindDir(ctx, dir, true)
	return err
}

func (f *Fs) isDirEmpty(ctx context.Context, slug string) (empty bool, err error) {
	folders, err := f.fetchListFolderPage(ctx, slug, "", 1, 0)

	if err != nil {
		return false, err
	}

	if len(folders) > 0 {
		return false, nil
	}

	files, err := f.fetchListFilePage(ctx, slug, "", 1, 0)

	if err != nil {
		return false, err
	}

	if len(files) > 0 {
		return false, nil
	}

	return true, nil
}

// Rmdir implements the mandatory method fs.Fs.Rmdir.
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
	slug, err := f.dirCache.FindDir(ctx, dir, false)

	if err != nil {
		return err
	}

	empty, err := f.isDirEmpty(ctx, slug)

	if err != nil {
		return err
	}

	if !empty {
		return fs.ErrorDirectoryNotEmpty
	}

	opts := rest.Opts{
		Method: "DELETE",
		Path:   "/v5/user/" + f.opt.Username + "/folder-list",
	}

	req := api.DeleteFoldersRequest{Slugs: []string{slug}}
	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := f.rest.CallJSON(ctx, &opts, req, nil)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return err
	}

	f.dirCache.FlushDir(dir)

	return nil
}

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("uloz.to root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// Precision return the precision of this Fs
func (f *Fs) Precision() time.Duration {
	return time.Microsecond
}

// Hashes implements fs.Fs.Hashes by returning the supported hash types of the filesystem.
func (f *Fs) Hashes() hash.Set {
	return hash.NewHashSet(hash.SHA256, hash.MD5)
}

// DescriptionEncodedMetadata represents a set of metadata encoded as Uloz.to description.
//
// Uloz.to doesn't support setting metadata such as mtime but allows the user to set an arbitrary description field.
// The content of this structure will be serialized and stored in the backend.
//
// The files themselves are immutable so there's no danger that the file changes, and we'll forget to update the hashes.
// It is theoretically possible to rewrite the description to provide incorrect information for a file. However, in case
// it's a real attack vector, a nefarious person already has write access to the repo, and the situation is above
// rclone's pay grade already.
type DescriptionEncodedMetadata struct {
	Md5Hash            []byte // The MD5 hash of the file
	Sha256Hash         []byte // The SHA256 hash of the file
	ModTimeEpochMicros int64  // The mtime of the file, as set by rclone
}

func (md *DescriptionEncodedMetadata) encode() (string, error) {
	b := bytes.Buffer{}
	e := gob.NewEncoder(&b)
	err := e.Encode(md)
	if err != nil {
		return "", err
	}
	// Version the encoded string from the beginning even though we don't need it yet.
	return "1;" + base64.StdEncoding.EncodeToString(b.Bytes()), nil
}

func decodeDescriptionMetadata(str string) (*DescriptionEncodedMetadata, error) {
	// The encoded data starts with a version number which is not a part iof the serialized object
	spl := strings.SplitN(str, ";", 2)

	if len(spl) < 2 || spl[0] != "1" {
		return nil, errors.New("can't decode, unknown encoded metadata version")
	}

	m := DescriptionEncodedMetadata{}
	by, err := base64.StdEncoding.DecodeString(spl[1])
	if err != nil {
		return nil, err
	}
	b := bytes.Buffer{}
	b.Write(by)
	d := gob.NewDecoder(&b)
	err = d.Decode(&m)
	if err != nil {
		return nil, err
	}
	return &m, nil
}

// Object describes an uloz.to object.
//
// Valid objects will always have all fields but encodedMetadata set.
type Object struct {
	fs            *Fs       // what this object is part of
	remote        string    // The remote path
	name          string    // The file name
	size          int64     // size of the object
	slug          string    // ID of the object
	remoteFsMtime time.Time // The time the object was last modified in the remote fs.
	// Metadata not available natively and encoded in the description field. May not be present if the encoded metadata
	// is not present (e.g. if file wasn't uploaded by rclone) or invalid.
	encodedMetadata *DescriptionEncodedMetadata
}

// Storable implements the mandatory method fs.ObjectInfo.Storable
func (o *Object) Storable() bool {
	return true
}

func (o *Object) updateFileProperties(ctx context.Context, req interface{}) (err error) {
	var resp *api.File

	opts := rest.Opts{
		Method: "PATCH",
		Path:   "/v8/file/" + o.slug + "/private",
	}

	err = o.fs.pacer.Call(func() (bool, error) {
		httpResp, err := o.fs.rest.CallJSON(ctx, &opts, &req, &resp)
		return o.fs.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return err
	}

	return o.setMetaData(resp)
}

// SetModTime implements the mandatory method fs.Object.SetModTime
func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
	var newMetadata DescriptionEncodedMetadata
	if o.encodedMetadata == nil {
		newMetadata = DescriptionEncodedMetadata{}
	} else {
		newMetadata = *o.encodedMetadata
	}

	newMetadata.ModTimeEpochMicros = t.UnixMicro()
	encoded, err := newMetadata.encode()
	if err != nil {
		return err
	}
	return o.updateFileProperties(ctx, api.UpdateDescriptionRequest{
		Description: encoded,
	})
}

// Open implements the mandatory method fs.Object.Open
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
	opts := rest.Opts{
		Method: "POST",
		Path:   "/v5/file/download-link/vipdata",
	}

	req := &api.GetDownloadLinkRequest{
		Slug:      o.slug,
		UserLogin: o.fs.opt.Username,
		// Has to be set but doesn't seem to be used server side.
		DeviceID: "foobar",
	}

	var resp *api.GetDownloadLinkResponse

	err = o.fs.pacer.Call(func() (bool, error) {
		httpResp, err := o.fs.rest.CallJSON(ctx, &opts, &req, &resp)
		return o.fs.shouldRetry(ctx, httpResp, err, true)
	})
	if err != nil {
		return nil, err
	}

	opts = rest.Opts{
		Method:  "GET",
		RootURL: resp.Link,
		Options: options,
	}

	var httpResp *http.Response

	err = o.fs.pacer.Call(func() (bool, error) {
		httpResp, err = o.fs.cdn.Call(ctx, &opts)
		return o.fs.shouldRetry(ctx, httpResp, err, true)
	})
	if err != nil {
		return nil, err
	}
	return httpResp.Body, err
}

func (o *Object) copyFrom(other *Object) {
	o.fs = other.fs
	o.remote = other.remote
	o.size = other.size
	o.slug = other.slug
	o.remoteFsMtime = other.remoteFsMtime
	o.encodedMetadata = other.encodedMetadata
}

// RenamingObjectInfoProxy is a delegating proxy for fs.ObjectInfo
// with the option of specifying a different remote path.
type RenamingObjectInfoProxy struct {
	delegate fs.ObjectInfo
	remote   string
}

// Remote implements fs.ObjectInfo.Remote by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) String() string {
	return s.delegate.String()
}

// Remote implements fs.ObjectInfo.Remote by returning the specified remote path.
func (s *RenamingObjectInfoProxy) Remote() string {
	return s.remote
}

// ModTime implements fs.ObjectInfo.ModTime by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) ModTime(ctx context.Context) time.Time {
	return s.delegate.ModTime(ctx)
}

// Size implements fs.ObjectInfo.Size by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) Size() int64 {
	return s.delegate.Size()
}

// Fs implements fs.ObjectInfo.Fs by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) Fs() fs.Info {
	return s.delegate.Fs()
}

// Hash implements fs.ObjectInfo.Hash by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) Hash(ctx context.Context, ty hash.Type) (string, error) {
	return s.delegate.Hash(ctx, ty)
}

// Storable implements fs.ObjectInfo.Storable by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) Storable() bool {
	return s.delegate.Storable()
}

// Update implements the mandatory method fs.Object.Update
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
	// The backend allows to store multiple files with the same name, so simply upload the new file and remove the old
	// one afterwards.
	info := &RenamingObjectInfoProxy{
		delegate: src,
		remote:   o.Remote(),
	}
	newo, err := o.fs.PutUnchecked(ctx, in, info, options...)

	if err != nil {
		return err
	}

	err = o.Remove(ctx)
	if err != nil {
		return err
	}

	o.copyFrom(newo.(*Object))

	return nil
}

// Remove implements the mandatory method fs.Object.Remove
func (o *Object) Remove(ctx context.Context) error {
	for i := 0; i < 2; i++ {
		// First call moves the item to recycle bin, second deletes it for good
		var err error
		opts := rest.Opts{
			Method: "DELETE",
			Path:   "/v6/file/" + o.slug + "/private",
		}
		err = o.fs.pacer.Call(func() (bool, error) {
			httpResp, err := o.fs.rest.CallJSON(ctx, &opts, nil, nil)
			return o.fs.shouldRetry(ctx, httpResp, err, true)
		})
		if err != nil {
			return err
		}
	}

	return nil
}

// ModTime implements the mandatory method fs.Object.ModTime
func (o *Object) ModTime(ctx context.Context) time.Time {
	if o.encodedMetadata != nil {
		return time.UnixMicro(o.encodedMetadata.ModTimeEpochMicros)
	}

	// The time the object was last modified on the server - a handwavy guess, but we don't have any better
	return o.remoteFsMtime

}

// Fs implements the mandatory method fs.Object.Fs
func (o *Object) Fs() fs.Info {
	return o.fs
}

// String returns the string representation of the remote object reference.
func (o *Object) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Remote returns the remote path
func (o *Object) Remote() string {
	return o.remote
}

// Size returns the size of an object in bytes
func (o *Object) Size() int64 {
	return o.size
}

// Hash implements the mandatory method fs.Object.Hash.
//
// Supports SHA256 and MD5 hashes.
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
	if t != hash.MD5 && t != hash.SHA256 {
		return "", hash.ErrUnsupported
	}

	if o.encodedMetadata == nil {
		return "", nil
	}

	switch t {
	case hash.MD5:
		return hex.EncodeToString(o.encodedMetadata.Md5Hash), nil
	case hash.SHA256:
		return hex.EncodeToString(o.encodedMetadata.Sha256Hash), nil
	}

	panic("Should never get here")
}

// FindLeaf implements dircache.DirCacher.FindLeaf by successively walking through the folder hierarchy until
// the desired folder is found, or there's nowhere to continue.
func (f *Fs) FindLeaf(ctx context.Context, folderSlug, leaf string) (leafSlug string, found bool, err error) {
	folders, err := f.listFolders(ctx, folderSlug, leaf)
	if err != nil {
		if errors.Is(err, fs.ErrorDirNotFound) {
			return "", false, nil
		}
		return "", false, err
	}

	for _, folder := range folders {
		if folder.Name == leaf {
			return folder.Slug, true, nil
		}
	}

	// Uloz.to allows creation of multiple files / folders with the same name in the same parent folder. rclone always
	// expects folder paths to be unique (no other file or folder with the same name should exist). As a result we also
	// need to look at the files to return the correct error if necessary.
	files, err := f.listFiles(ctx, folderSlug, leaf)
	if err != nil {
		return "", false, err
	}

	for _, file := range files {
		if file.Name == leaf {
			return "", false, fs.ErrorIsFile
		}
	}

	// The parent folder exists but no file or folder with the given name was found in it.
	return "", false, nil
}

// CreateDir implements dircache.DirCacher.CreateDir by creating a folder with the given name under a folder identified
// by parentSlug.
func (f *Fs) CreateDir(ctx context.Context, parentSlug, leaf string) (newID string, err error) {
	var folder *api.Folder
	opts := rest.Opts{
		Method:     "POST",
		Path:       "/v6/user/" + f.opt.Username + "/folder",
		Parameters: url.Values{},
	}
	mkdir := api.CreateFolderRequest{
		Name:             f.opt.Enc.FromStandardName(leaf),
		ParentFolderSlug: parentSlug,
	}
	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := f.rest.CallJSON(ctx, &opts, &mkdir, &folder)
		return f.shouldRetry(ctx, httpResp, err, true)
	})
	if err != nil {
		return "", err
	}
	return folder.Slug, nil
}

func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.File) (*Object, error) {
	o := &Object{
		fs:     f,
		remote: remote,
	}
	var err error

	if info == nil {
		info, err = f.readMetaDataForPath(ctx, remote)
	}

	if err != nil {
		return nil, err
	}

	err = o.setMetaData(info)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) {
	filename, folderSlug, err := f.dirCache.FindPath(ctx, path, false)
	if err != nil {
		if errors.Is(err, fs.ErrorDirNotFound) {
			return nil, fs.ErrorObjectNotFound
		}
		return nil, err
	}

	files, err := f.listFiles(ctx, folderSlug, filename)

	if err != nil {
		return nil, err
	}

	for _, file := range files {
		if file.Name == filename {
			return &file, nil
		}
	}

	folders, err := f.listFolders(ctx, folderSlug, filename)

	if err != nil {
		return nil, err
	}

	for _, file := range folders {
		if file.Name == filename {
			return nil, fs.ErrorIsDir
		}
	}

	return nil, fs.ErrorObjectNotFound
}

func (o *Object) setMetaData(info *api.File) (err error) {
	o.name = info.Name
	o.size = info.Filesize
	o.remoteFsMtime = info.LastUserModified
	o.encodedMetadata, err = decodeDescriptionMetadata(info.Description)
	if err != nil {
		fs.Debugf(o, "Couldn't decode metadata: %v", err)
	}
	o.slug = info.Slug
	return nil
}

// NewObject implements fs.Fs.NewObject.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
	return f.newObjectWithInfo(ctx, remote, nil)
}

// List implements fs.Fs.List by listing all files and folders in the given folder.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
	folderSlug, err := f.dirCache.FindDir(ctx, dir, false)
	if err != nil {
		return nil, err
	}

	folders, err := f.listFolders(ctx, folderSlug, "")
	if err != nil {
		return nil, err
	}

	for _, folder := range folders {
		remote := path.Join(dir, folder.Name)
		f.dirCache.Put(remote, folder.Slug)
		entries = append(entries, fs.NewDir(remote, folder.LastUserModified))
	}

	files, err := f.listFiles(ctx, folderSlug, "")
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		remote := path.Join(dir, file.Name)
		remoteFile, err := f.newObjectWithInfo(ctx, remote, &file)
		if err != nil {
			return nil, err
		}
		entries = append(entries, remoteFile)
	}

	return entries, nil
}

func (f *Fs) fetchListFolderPage(
	ctx context.Context,
	folderSlug string,
	searchQuery string,
	limit int,
	offset int) (folders []api.Folder, err error) {

	opts := rest.Opts{
		Method:     "GET",
		Path:       "/v9/user/" + f.opt.Username + "/folder/" + folderSlug + "/folder-list",
		Parameters: url.Values{},
	}

	opts.Parameters.Set("status", "ok")
	opts.Parameters.Set("limit", strconv.Itoa(limit))
	if offset > 0 {
		opts.Parameters.Set("offset", strconv.Itoa(offset))
	}

	if searchQuery != "" {
		opts.Parameters.Set("search_query", f.opt.Enc.FromStandardName(searchQuery))
	}

	var respBody *api.ListFoldersResponse

	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := f.rest.CallJSON(ctx, &opts, nil, &respBody)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return nil, err
	}

	for i := range respBody.Subfolders {
		respBody.Subfolders[i].Name = f.opt.Enc.ToStandardName(respBody.Subfolders[i].Name)
	}

	return respBody.Subfolders, nil
}

func (f *Fs) listFolders(
	ctx context.Context,
	folderSlug string,
	searchQuery string) (folders []api.Folder, err error) {

	targetPageSize := f.opt.ListPageSize
	lastPageSize := targetPageSize
	offset := 0

	for targetPageSize == lastPageSize {
		page, err := f.fetchListFolderPage(ctx, folderSlug, searchQuery, targetPageSize, offset)
		if err != nil {
			var apiErr *api.Error
			casted := errors.As(err, &apiErr)
			if casted && apiErr.ErrorCode == 30001 {
				return nil, fs.ErrorDirNotFound
			}
			return nil, err
		}
		lastPageSize = len(page)
		offset += lastPageSize
		folders = append(folders, page...)
	}

	return folders, nil
}

func (f *Fs) fetchListFilePage(
	ctx context.Context,
	folderSlug string,
	searchQuery string,
	limit int,
	offset int) (folders []api.File, err error) {

	opts := rest.Opts{
		Method:     "GET",
		Path:       "/v8/user/" + f.opt.Username + "/folder/" + folderSlug + "/file-list",
		Parameters: url.Values{},
	}
	opts.Parameters.Set("status", "ok")
	opts.Parameters.Set("limit", strconv.Itoa(limit))
	if offset > 0 {
		opts.Parameters.Set("offset", strconv.Itoa(offset))
	}

	if searchQuery != "" {
		opts.Parameters.Set("search_query", f.opt.Enc.FromStandardName(searchQuery))
	}

	var respBody *api.ListFilesResponse

	err = f.pacer.Call(func() (bool, error) {
		httpResp, err := f.rest.CallJSON(ctx, &opts, nil, &respBody)
		return f.shouldRetry(ctx, httpResp, err, true)
	})

	if err != nil {
		return nil, fmt.Errorf("couldn't list files: %w", err)
	}

	for i := range respBody.Items {
		respBody.Items[i].Name = f.opt.Enc.ToStandardName(respBody.Items[i].Name)
	}

	return respBody.Items, nil
}

func (f *Fs) listFiles(
	ctx context.Context,
	folderSlug string,
	searchQuery string) (folders []api.File, err error) {

	targetPageSize := f.opt.ListPageSize
	lastPageSize := targetPageSize
	offset := 0

	for targetPageSize == lastPageSize {
		page, err := f.fetchListFilePage(ctx, folderSlug, searchQuery, targetPageSize, offset)
		if err != nil {
			return nil, err
		}
		lastPageSize = len(page)
		offset += lastPageSize
		folders = append(folders, page...)
	}

	return folders, nil
}

// DirCacheFlush implements the optional fs.DirCacheFlusher interface.
func (f *Fs) DirCacheFlush() {
	f.dirCache.ResetRoot()
}

// Check the interfaces are satisfied
var (
	_ fs.Fs              = (*Fs)(nil)
	_ dircache.DirCacher = (*Fs)(nil)
	_ fs.DirCacheFlusher = (*Fs)(nil)
	_ fs.PutUncheckeder  = (*Fs)(nil)
	_ fs.Object          = (*Object)(nil)
	_ fs.ObjectInfo      = (*RenamingObjectInfoProxy)(nil)
)