azurefiles: finish docs and implementation and add optional interfaces

- use rclone's http Transport
- fix handling of 0 length files
- combine into one file and remove uneeded abstraction
- make `chunk_size` and `upload_concurrency` settable
- make auth the same as azureblob
- set the Features correctly
- implement `--azurefiles-max-stream-size`
- remove arbitrary sleep on Mkdir
- implement `--header-upload`
- implement read and write MimeType for objects
- implement optional methods
    - About
    - Copy
    - DirMove
    - Move
    - OpenWriterAt
    - PutStream
- finish documentation
- disable build on plan9 and js

Fixes #365
Fixes #7378
This commit is contained in:
Nick Craig-Wood 2023-11-15 09:37:57 +00:00
parent b5301e03a6
commit ddaf01ece9
15 changed files with 1994 additions and 723 deletions

View File

@ -59,6 +59,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* Mega [:page_facing_up:](https://rclone.org/mega/) * Mega [:page_facing_up:](https://rclone.org/mega/)
* Memory [:page_facing_up:](https://rclone.org/memory/) * Memory [:page_facing_up:](https://rclone.org/memory/)
* Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/) * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
* Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
* Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/) * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
* Minio [:page_facing_up:](https://rclone.org/s3/#minio) * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
* Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud) * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)

File diff suppressed because it is too large Load Diff

View File

@ -1,3 +1,6 @@
//go:build !plan9 && !js
// +build !plan9,!js
package azurefiles package azurefiles
import ( import (
@ -41,7 +44,7 @@ func (f *Fs) InternalTestAuth(t *testing.T) {
name: "SASUrl", name: "SASUrl",
options: &Options{ options: &Options{
ShareName: shareName, ShareName: shareName,
SASUrl: "", SASURL: "",
}}, }},
} }

View File

@ -1,3 +1,6 @@
//go:build !plan9 && !js
// +build !plan9,!js
package azurefiles package azurefiles
import ( import (

View File

@ -0,0 +1,7 @@
// Build for azurefiles for unsupported platforms to stop go complaining
// about "no buildable Go source files "
//go:build plan9 || js
// +build plan9 js
package azurefiles

View File

@ -1,44 +0,0 @@
package azurefiles
import (
"context"
"time"
)
// Directory is a filesystem like directory provided by an Fs
type Directory struct {
common
}
// Items returns the count of items in this directory or this
// directory and subdirectories if known, -1 for unknown
//
// It is unknown since getting the count of items results in a
// network request
func (d *Directory) Items() int64 {
return -1
}
// ID returns empty string. Can be implemented as part of IDer
func (d *Directory) ID() string {
return ""
}
// Size is returns the size of the file.
// This method is implemented because it is part of the [fs.DirEntry] interface
func (d *Directory) Size() int64 {
return 0
}
// ModTime returns the modification time of the object
//
// TODO: check whether FileLastWriteTime is what the clients of this API want. Maybe
// FileLastWriteTime does not get changed when directory contents are updated but consumers
// of this API expect d.ModTime to do so
func (d *Directory) ModTime(ctx context.Context) time.Time {
props, err := d.f.dirClient(d.remote).GetProperties(ctx, nil)
if err != nil {
return time.Now()
}
return *props.FileLastWriteTime
}

View File

@ -1,292 +0,0 @@
package azurefiles
import (
"context"
"errors"
"fmt"
"io"
"log"
"path"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
)
const sleepDurationBetweenRecursiveMkdirPutCalls = time.Millisecond * 500
const fourTbInBytes = 4398046511104
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
//
// Does not return ErrorIsDir when a directory exists instead of file. since the documentation
// for [rclone.fs.Fs.NewObject] rqeuires no extra work to determine whether it is directory
//
// Inspired by azureblob store, this initiates a network request and returns an error if object is not found.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
fileClient := f.fileClient(remote)
resp, err := fileClient.GetProperties(ctx, nil)
if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) {
return nil, fs.ErrorObjectNotFound
} else if err != nil {
return nil, fmt.Errorf("unable to find object remote=%s : %w", remote, err)
}
ob := objectInstance(f, remote, *resp.ContentLength, resp.ContentMD5, *resp.FileLastWriteTime)
return &ob, nil
}
// Mkdir creates nested directories as indicated by test FsMkdirRmdirSubdir
// TODO: write custom test case where parent directories are created
// Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, remote string) error {
return f.mkdirRelativeToRootOfShare(ctx, f.decodedFullPath(remote))
}
// rclone completes commands such as rclone copy localdir remote:parentcontainer/childcontainer
// where localdir is a tree of files and directories. The above command is expected to complete even
// when parentcontainer and childcontainer directors do not exist on the remote. The following
// code with emphasis on fullPathRelativeToShareRoot is written to handle such cases by recursiely creating
// parent directories all the way to the root of the share
//
// When path argument is an empty string, windows and linux return and error. However, this
// implementation does not return an error
func (f *Fs) mkdirRelativeToRootOfShare(ctx context.Context, fullPathRelativeToShareRoot string) error {
fp := fullPathRelativeToShareRoot
if fp == "" {
return nil
}
dirClient := f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(f.encodePath(fp))
// now := time.Now()
// smbProps := &file.SMBProperties{
// LastWriteTime: &now,
// }
// dirCreateOptions := &directory.CreateOptions{
// FileSMBProperties: smbProps,
// }
_, createDirErr := dirClient.Create(ctx, nil)
if fileerror.HasCode(createDirErr, fileerror.ParentNotFound) {
parentDir := path.Dir(fp)
if parentDir == fp {
log.Fatal("This will lead to infinite recursion since parent and remote are equal")
}
makeParentErr := f.mkdirRelativeToRootOfShare(ctx, parentDir)
if makeParentErr != nil {
return fmt.Errorf("could not make parent of %s : %w", fp, makeParentErr)
}
log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir)
time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls)
return f.mkdirRelativeToRootOfShare(ctx, fp)
} else if fileerror.HasCode(createDirErr, fileerror.ResourceAlreadyExists) {
return nil
} else if createDirErr != nil {
return fmt.Errorf("unable to MkDir: %w", createDirErr)
}
return nil
}
// Rmdir deletes the root folder
//
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, remote string) error {
dirClient := f.dirClient(remote)
_, err := dirClient.Delete(ctx, nil)
if err != nil {
if fileerror.HasCode(err, fileerror.DirectoryNotEmpty) {
return fs.ErrorDirectoryNotEmpty
} else if fileerror.HasCode(err, fileerror.ResourceNotFound) {
return fs.ErrorDirNotFound
}
return fmt.Errorf("could not rmdir dir=\"%s\" : %w", remote, err)
}
return nil
}
// Put the object
//
// Copies the reader in to the new object. This new object is returned.
//
// The new object may have been created if an error is returned
// TODO: when file.CLient.Creat is being used, provide HTTP headesr such as content type and content MD5
// TODO: maybe replace PUT with NewObject + Update
// TODO: in case file is created but there is a problem on upload, what happens
// TODO: what happens when file already exists at the location
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
if src.Size() > fourTbInBytes {
return nil, fmt.Errorf("max supported file size is 4TB. provided size is %d", src.Size())
} else if src.Size() < 0 {
// TODO: what should happened when src.Size == 0
return nil, fmt.Errorf("src.Size is a required to be a whole number : %d", src.Size())
}
fc := f.fileClient(src.Remote())
_, createErr := fc.Create(ctx, src.Size(), nil)
if fileerror.HasCode(createErr, fileerror.ParentNotFound) {
parentDir := path.Dir(src.Remote())
if mkDirErr := f.Mkdir(ctx, parentDir); mkDirErr != nil {
return nil, fmt.Errorf("unable to make parent directories : %w", mkDirErr)
}
log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir)
time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls)
return f.Put(ctx, in, src, options...)
} else if createErr != nil {
return nil, fmt.Errorf("unable to create file : %w", createErr)
}
obj := &Object{
common: common{
f: f,
remote: src.Remote(),
},
}
if updateErr := obj.upload(ctx, in, src, true, options...); updateErr != nil {
err := fmt.Errorf("while executing update after creating file as part of fs.Put : %w", updateErr)
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
return nil, errors.Join(delErr, updateErr)
}
return obj, err
}
return obj, nil
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
return fmt.Sprintf("azurefiles root '%s'", f.root)
}
// Precision return the precision of this Fs
//
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
func (f *Fs) Precision() time.Duration {
return time.Second
}
// Hashes returns the supported hash sets.
//
// MD5: since it is listed as header in the response for file properties
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/get-file-properties
func (f *Fs) Hashes() hash.Set {
return hash.NewHashSet(hash.MD5)
}
// Features returns the optional features of this Fs
//
// TODO: add features:- public link, SlowModTime, SlowHash,
// ReadMetadata, WriteMetadata,UserMetadata,PutUnchecked, PutStream
// PartialUploads: Maybe????
// FileID and DirectoryID can be implemented. They are atleast returned as part of listing response
func (f *Fs) Features() *fs.Features {
return &fs.Features{
CanHaveEmptyDirectories: true,
// Copy: func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
// return f.CopyFile(ctx, src, remote)
// },
}
}
// List the objects and directories in dir into entries. The entries can be
// returned in any order but should be for a complete directory.
//
// dir should be "" to list the root, and should not have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't found.
//
// TODO: handle case regariding "" and "/". I remember reading about them somewhere
func (f *Fs) List(ctx context.Context, remote string) (fs.DirEntries, error) {
var entries fs.DirEntries
subDirClient := f.dirClient(remote)
// Checking whether directory exists
_, err := subDirClient.GetProperties(ctx, nil)
if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) {
return entries, fs.ErrorDirNotFound
} else if err != nil {
return entries, err
}
pager := subDirClient.NewListFilesAndDirectoriesPager(listFilesAndDirectoriesOptions)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return entries, err
}
for _, dir := range resp.Segment.Directories {
de := &Directory{
common{f: f,
remote: path.Join(remote, f.decodePath(*dir.Name)),
properties: properties{
lastWriteTime: *dir.Properties.LastWriteTime,
}},
}
entries = append(entries, de)
}
for _, file := range resp.Segment.Files {
de := &Object{
common{f: f,
remote: path.Join(remote, f.decodePath(*file.Name)),
properties: properties{
contentLength: *file.Properties.ContentLength,
lastWriteTime: *file.Properties.LastWriteTime,
}},
}
entries = append(entries, de)
}
}
return entries, nil
}
type encodedPath string
func (f *Fs) decodedFullPath(decodedRemote string) string {
return path.Join(f.root, decodedRemote)
}
func (f *Fs) dirClient(decodedRemote string) *directory.Client {
fullPathDecoded := f.decodedFullPath(decodedRemote)
fullPathEncoded := f.encodePath(fullPathDecoded)
return f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(fullPathEncoded)
}
func (f *Fs) newSubdirectoryClientFromEncodedPathRelativeToShareRoot(p encodedPath) *directory.Client {
return f.shareRootDirClient.NewSubdirectoryClient(string(p))
}
func (f *Fs) fileClient(decodedRemote string) *file.Client {
fullPathDecoded := f.decodedFullPath(decodedRemote)
fullPathEncoded := f.encodePath(fullPathDecoded)
return f.fileClientFromEncodedPathRelativeToShareRoot(fullPathEncoded)
}
func (f *Fs) fileClientFromEncodedPathRelativeToShareRoot(p encodedPath) *file.Client {
return f.shareRootDirClient.NewFileClient(string(p))
}
func (f *Fs) encodePath(p string) encodedPath {
return encodedPath(f.opt.Enc.FromStandardPath(p))
}
func (f *Fs) decodePath(p string) string {
return f.opt.Enc.ToStandardPath(p)
}
// on 20231019 at 1324 work to be continued at trying to fix FAIL: TestIntegration/FsMkdir/FsPutFiles/FromRoot

View File

@ -1,279 +0,0 @@
package azurefiles
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"log/slog"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
)
// TODO: maybe use this in the result of list. or replace all instances where object instances are created
func objectInstance(f *Fs, remote string, contentLength int64, md5Hash []byte, lwt time.Time) Object {
return Object{common: common{
f: f,
remote: remote,
properties: properties{
contentLength: contentLength,
md5Hash: md5Hash,
lastWriteTime: lwt,
},
}}
}
// Size of object in bytes
func (o *Object) Size() int64 {
return o.properties.contentLength
}
// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
return o.f
}
// Hash returns the MD5 of an object returning a lowercase hex string
//
// May make a network request becaue the [fs.List] method does not
// return MD5 hashes for DirEntry
func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
if ty != hash.MD5 {
return "", hash.ErrUnsupported
}
if len(o.common.properties.md5Hash) == 0 {
props, err := o.fileClient().GetProperties(ctx, nil)
if err != nil {
return "", fmt.Errorf("unable to fetch properties to determine hash")
}
o.common.properties.md5Hash = props.ContentMD5
}
return hex.EncodeToString(o.common.properties.md5Hash), nil
}
// Storable returns a boolean showing whether this object storable
func (o *Object) Storable() bool {
return true
}
// Object describes a Azure File Share File not a Directory
type Object struct {
common
}
// These fields have pointer types because it seems to
// TODO: descide whether these could be pointer or not
type properties struct {
contentLength int64
md5Hash []byte
lastWriteTime time.Time
}
func (o *Object) fileClient() *file.Client {
decodedFullPath := o.f.decodedFullPath(o.remote)
fullEncodedPath := o.f.encodePath(decodedFullPath)
return o.f.fileClientFromEncodedPathRelativeToShareRoot(fullEncodedPath)
}
// SetModTime sets the modification time
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
smbProps := file.SMBProperties{
LastWriteTime: &t,
}
setHeadersOptions := file.SetHTTPHeadersOptions{
SMBProperties: &smbProps,
}
_, err := o.fileClient().SetHTTPHeaders(ctx, &setHeadersOptions)
if err != nil {
return fmt.Errorf("unable to set modTime : %w", err)
}
o.lastWriteTime = t
return nil
}
// ModTime returns the modification time of the object
//
// Returns time.Now() if not present
// TODO: convert o.lastWriteTime to *time.Time so that one can know when it has
// been explicitly set
func (o *Object) ModTime(ctx context.Context) time.Time {
if o.lastWriteTime.Unix() <= 1 {
return time.Now()
}
return o.lastWriteTime
}
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
// TODO: should the options for delete not be nil. Depends on behaviour expected by consumers
if _, err := o.fileClient().Delete(ctx, nil); err != nil {
return fmt.Errorf("unable to delete remote=\"%s\" : %w", o.remote, err)
}
return nil
}
// Open an object for read
//
// TODO: check for mandatory options and the other options
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
downloadStreamOptions := file.DownloadStreamOptions{}
for _, opt := range options {
switch v := opt.(type) {
case *fs.SeekOption:
httpRange := file.HTTPRange{
Offset: v.Offset,
}
downloadStreamOptions.Range = httpRange
case *fs.RangeOption:
var start *int64
var end *int64
if v.Start >= 0 {
start = &v.Start
}
if v.End >= 0 {
end = &v.End
}
fhr := file.HTTPRange{}
if start != nil && end != nil {
fhr.Offset = *start
fhr.Count = *end - *start + 1
} else if start != nil && end == nil {
fhr.Offset = *start
} else if start == nil && end != nil {
fhr.Offset = o.contentLength - *end
}
downloadStreamOptions.Range = fhr
}
}
resp, err := o.fileClient().DownloadStream(ctx, &downloadStreamOptions)
if err != nil {
return nil, fmt.Errorf("could not open remote=\"%s\" : %w", o.remote, err)
}
return resp.Body, nil
}
func (o *Object) upload(ctx context.Context, in io.Reader, src fs.ObjectInfo, isDestNewlyCreated bool, options ...fs.OpenOption) error {
if src.Size() > fourTbInBytes {
return fmt.Errorf("max supported file size is 4TB. provided size is %d", src.Size())
} else if src.Size() < 0 {
return fmt.Errorf("files with unknown sizes are not supported")
}
fc := o.fileClient()
if !isDestNewlyCreated {
if src.Size() != o.Size() {
if _, resizeErr := fc.Resize(ctx, src.Size(), nil); resizeErr != nil {
return fmt.Errorf("unable to resize while trying to update. %w ", resizeErr)
}
}
}
var md5Hash []byte
hashToBeComputed := false
if hashStr, err := src.Hash(ctx, hash.MD5); err != nil || hashStr == "" {
hashToBeComputed = true
} else {
var decodeErr error
md5Hash, decodeErr = hex.DecodeString(hashStr)
if decodeErr != nil {
hashToBeComputed = true
msg := fmt.Sprintf("should not happen. Error while decoding hex encoded md5 '%s'. Error is %s",
hashStr, decodeErr.Error())
slog.Error(msg)
}
}
var uploadErr error
if hashToBeComputed {
md5Hash, uploadErr = uploadStreamAndComputeHash(ctx, fc, in, src, options...)
} else {
uploadErr = uploadStream(ctx, fc, in, src, options...)
}
if uploadErr != nil {
return fmt.Errorf("while uploading %s : %w", src.Remote(), uploadErr)
}
modTime := src.ModTime(ctx)
if err := uploadSizeHashLWT(ctx, fc, src.Size(), md5Hash, modTime); err != nil {
return fmt.Errorf("while setting size hash and last write time for %s : %w", src.Remote(), err)
}
o.properties.contentLength = src.Size()
o.properties.md5Hash = md5Hash
o.properties.lastWriteTime = modTime
return nil
}
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
// Does not create a new object
//
// TODO: implement options. understand purpose of options
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
return o.upload(ctx, in, src, false, options...)
}
// cannot set modTime header here because setHTTPHeaders does not allow setting metadata
func uploadStream(ctx context.Context, fc *file.Client, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
// TODO: set concurrency level
uploadStreamOptions := file.UploadStreamOptions{
ChunkSize: chunkSize(options...),
}
if err := fc.UploadStream(ctx, in, &uploadStreamOptions); err != nil {
return fmt.Errorf("unable to upload. cannot upload stream : %w", err)
}
return nil
}
func uploadStreamAndComputeHash(ctx context.Context, fc *file.Client, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) ([]byte, error) {
hasher := md5.New()
teeReader := io.TeeReader(in, hasher)
err := uploadStream(ctx, fc, teeReader, src, options...)
if err != nil {
return []byte{}, err
}
return hasher.Sum(nil), nil
}
// the function is named with prefix 'upload' since it indicates that things will be modified on the server
func uploadSizeHashLWT(ctx context.Context, fc *file.Client, size int64, hash []byte, lwt time.Time) error {
smbProps := file.SMBProperties{
LastWriteTime: &lwt,
}
httpHeaders := &file.HTTPHeaders{
ContentMD5: hash,
}
_, err := fc.SetHTTPHeaders(ctx, &file.SetHTTPHeadersOptions{
FileContentLength: &size,
SMBProperties: &smbProps,
HTTPHeaders: httpHeaders,
})
if err != nil {
return fmt.Errorf("while setting size, hash, lastWriteTime : %w", err)
}
return nil
}
func chunkSize(options ...fs.OpenOption) int64 {
for _, option := range options {
if chunkOpt, ok := option.(*fs.ChunkOption); ok {
return chunkOpt.ChunkSize
}
}
return 1048576
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.common.String()
}

View File

@ -58,6 +58,7 @@ docs = [
"memory.md", "memory.md",
"netstorage.md", "netstorage.md",
"azureblob.md", "azureblob.md",
"azurefiles.md",
"onedrive.md", "onedrive.md",
"opendrive.md", "opendrive.md",
"oracleobjectstorage.md", "oracleobjectstorage.md",

View File

@ -144,6 +144,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}} {{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}}
{{< provider name="Memory" home="/memory/" config="/memory/" >}} {{< provider name="Memory" home="/memory/" config="/memory/" >}}
{{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}} {{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}}
{{< provider name="Microsoft Azure Files Storage" home="https://azure.microsoft.com/en-us/services/storage/files/" config="/azurefiles/" >}}
{{< provider name="Microsoft OneDrive" home="https://onedrive.live.com/" config="/onedrive/" >}} {{< provider name="Microsoft OneDrive" home="https://onedrive.live.com/" config="/onedrive/" >}}
{{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
{{< provider name="Nextcloud" home="https://nextcloud.com/" config="/webdav/#nextcloud" >}} {{< provider name="Nextcloud" home="https://nextcloud.com/" config="/webdav/#nextcloud" >}}

View File

@ -1,21 +1,707 @@
--- ---
title: "Microsoft Azure Files Storage" title: "Microsoft Azure Files Storage"
description: "Rclone docs for Microsoft Azure Files Storage" description: "Rclone docs for Microsoft Azure Files Storage"
versionIntroduced: "v1.65"
--- ---
# Microsoft Azure File Storage # {{< icon "fab fa-windows" >}} Microsoft Azure Files Storage
Paths are specified as `remote:` You may put subdirectories in too,
e.g. `remote:path/to/dir`.
## Configuration
Here is an example of making a Microsoft Azure Files Storage
configuration. For a remote called `remote`. First run:
rclone config
This will guide you through an interactive setup process:
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> remote
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Microsoft Azure Files Storage
\ "azurefiles"
[snip]
Option account.
Azure Storage Account Name.
Set this to the Azure Storage Account Name in use.
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
If this is blank and if env_auth is set it will be read from the
environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible.
Enter a value. Press Enter to leave empty.
account> account_name
Option share_name.
Azure Files Share Name.
This is required and is the name of the share to access.
Enter a value. Press Enter to leave empty.
share_name> share_name
Option env_auth.
Read credentials from runtime (environment variables, CLI or MSI).
See the [authentication docs](/azurefiles#authentication) for full info.
Enter a boolean value (true or false). Press Enter for the default (false).
env_auth>
Option key.
Storage Account Shared Key.
Leave blank to use SAS URL or connection string.
Enter a value. Press Enter to leave empty.
key> base64encodedkey==
Option sas_url.
SAS URL.
Leave blank if using account/key or connection string.
Enter a value. Press Enter to leave empty.
sas_url>
Option connection_string.
Azure Files Connection String.
Enter a value. Press Enter to leave empty.
connection_string>
[snip]
Configuration complete.
Options:
- type: azurefiles
- account: account_name
- share_name: share_name
- key: base64encodedkey==
Keep this "remote" remote?
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d>
```
Once configured you can use rclone.
See all files in the top level:
rclone lsf remote:
Make a new directory in the root:
rclone mkdir remote:dir
Recursively List the contents:
rclone ls remote:
Sync `/home/local/directory` to the remote directory, deleting any
excess files in the directory.
rclone sync --interactive /home/local/directory remote:dir
### Modified time ### Modified time
Stored as azure standard `LastModified` time stored on files The modified time is stored as Azure standard `LastModified` time on
files
### Performance
When uploading large files, increasing the value of
`--azurefiles-upload-concurrency` will increase performance at the cost
of using more memory. The default of 16 is set quite conservatively to
use less memory. It maybe be necessary raise it to 64 or higher to
fully utilize a 1 GBit/s link with a single file transfer.
### Restricted filename characters
In addition to the [default restricted characters set](/overview/#restricted-characters)
the following characters are also replaced:
| Character | Value | Replacement |
| --------- |:-----:|:-----------:|
| " | 0x22 | |
| * | 0x2A | |
| : | 0x3A | |
| < | 0x3C | |
| > | 0x3E | |
| ? | 0x3F | |
| \ | 0x5C | |
| \| | 0x7C | |
File names can also not end with the following characters.
These only get replaced if they are the last character in the name:
| Character | Value | Replacement |
| --------- |:-----:|:-----------:|
| . | 0x2E | |
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
as they can't be used in JSON strings.
### Hashes ### Hashes
MD5 hashes are stored with files. MD5 hashes are stored with files. Not all files will have MD5 hashes
as these have to be uploaded with the file.
### Authentication {#authentication} ### Authentication {#authentication}
1. ConnectionString There are a number of ways of supplying credentials for Azure Files
2. Accout and Key Storage. Rclone tries them in the order of the sections below.
3. SAS URL
#### Env Auth
If the `env_auth` config parameter is `true` then rclone will pull
credentials from the environment or runtime.
It tries these authentication methods in this order:
1. Environment Variables
2. Managed Service Identity Credentials
3. Azure CLI credentials (as used by the az tool)
These are described in the following sections
##### Env Auth: 1. Environment Variables
If `env_auth` is set and environment variables are present rclone
authenticates a service principal with a secret or certificate, or a
user with a password, depending on which environment variable are set.
It reads configuration from these variables, in the following order:
1. Service principal with client secret
- `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
- `AZURE_CLIENT_ID`: the service principal's client ID
- `AZURE_CLIENT_SECRET`: one of the service principal's client secrets
2. Service principal with certificate
- `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
- `AZURE_CLIENT_ID`: the service principal's client ID
- `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key.
- `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file.
- `AZURE_CLIENT_SEND_CERTIFICATE_CHAIN`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
3. User with username and password
- `AZURE_TENANT_ID`: (optional) tenant to authenticate in. Defaults to "organizations".
- `AZURE_CLIENT_ID`: client ID of the application the user will authenticate to
- `AZURE_USERNAME`: a username (usually an email address)
- `AZURE_PASSWORD`: the user's password
4. Workload Identity
- `AZURE_TENANT_ID`: Tenant to authenticate in.
- `AZURE_CLIENT_ID`: Client ID of the application the user will authenticate to.
- `AZURE_FEDERATED_TOKEN_FILE`: Path to projected service account token file.
- `AZURE_AUTHORITY_HOST`: Authority of an Azure Active Directory endpoint (default: login.microsoftonline.com).
##### Env Auth: 2. Managed Service Identity Credentials
When using Managed Service Identity if the VM(SS) on which this
program is running has a system-assigned identity, it will be used by
default. If the resource has no system-assigned but exactly one
user-assigned identity, the user-assigned identity will be used by
default.
If the resource has multiple user-assigned identities you will need to
unset `env_auth` and set `use_msi` instead. See the [`use_msi`
section](#use_msi).
##### Env Auth: 3. Azure CLI credentials (as used by the az tool)
Credentials created with the `az` tool can be picked up using `env_auth`.
For example if you were to login with a service principal like this:
az login --service-principal -u XXX -p XXX --tenant XXX
Then you could access rclone resources like this:
rclone lsf :azurefiles,env_auth,account=ACCOUNT:
Or
rclone lsf --azurefiles-env-auth --azurefiles-account=ACCOUNT :azurefiles:
#### Account and Shared Key
This is the most straight forward and least flexible way. Just fill
in the `account` and `key` lines and leave the rest blank.
#### SAS URL
To use it leave `account`, `key` and `connection_string` blank and fill in `sas_url`.
#### Connection String
To use it leave `account`, `key` and "sas_url" blank and fill in `connection_string`.
#### Service principal with client secret
If these variables are set, rclone will authenticate with a service principal with a client secret.
- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
- `client_id`: the service principal's client ID
- `client_secret`: one of the service principal's client secrets
The credentials can also be placed in a file using the
`service_principal_file` configuration option.
#### Service principal with certificate
If these variables are set, rclone will authenticate with a service principal with certificate.
- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
- `client_id`: the service principal's client ID
- `client_certificate_path`: path to a PEM or PKCS12 certificate file including the private key.
- `client_certificate_password`: (optional) password for the certificate file.
- `client_send_certificate_chain`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
**NB** `client_certificate_password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
#### User with username and password
If these variables are set, rclone will authenticate with username and password.
- `tenant`: (optional) tenant to authenticate in. Defaults to "organizations".
- `client_id`: client ID of the application the user will authenticate to
- `username`: a username (usually an email address)
- `password`: the user's password
Microsoft doesn't recommend this kind of authentication, because it's
less secure than other authentication flows. This method is not
interactive, so it isn't compatible with any form of multi-factor
authentication, and the application must already have user or admin
consent. This credential can only authenticate work and school
accounts; it can't authenticate Microsoft accounts.
**NB** `password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
#### Managed Service Identity Credentials {#use_msi}
If `use_msi` is set then managed service identity credentials are
used. This authentication only works when running in an Azure service.
`env_auth` needs to be unset to use this.
However if you have multiple user identities to choose from these must
be explicitly specified using exactly one of the `msi_object_id`,
`msi_client_id`, or `msi_mi_res_id` parameters.
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
set, this is is equivalent to using `env_auth`.
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/azurefiles/azurefiles.go then run make backenddocs" >}}
### Standard options
Here are the Standard options specific to azurefiles (Microsoft Azure Files).
#### --azurefiles-account
Azure Storage Account Name.
Set this to the Azure Storage Account Name in use.
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
If this is blank and if env_auth is set it will be read from the
environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible.
Properties:
- Config: account
- Env Var: RCLONE_AZUREFILES_ACCOUNT
- Type: string
- Required: false
#### --azurefiles-share-name
Azure Files Share Name.
This is required and is the name of the share to access.
Properties:
- Config: share_name
- Env Var: RCLONE_AZUREFILES_SHARE_NAME
- Type: string
- Required: false
#### --azurefiles-env-auth
Read credentials from runtime (environment variables, CLI or MSI).
See the [authentication docs](/azurefiles#authentication) for full info.
Properties:
- Config: env_auth
- Env Var: RCLONE_AZUREFILES_ENV_AUTH
- Type: bool
- Default: false
#### --azurefiles-key
Storage Account Shared Key.
Leave blank to use SAS URL or connection string.
Properties:
- Config: key
- Env Var: RCLONE_AZUREFILES_KEY
- Type: string
- Required: false
#### --azurefiles-sas-url
SAS URL.
Leave blank if using account/key or connection string.
Properties:
- Config: sas_url
- Env Var: RCLONE_AZUREFILES_SAS_URL
- Type: string
- Required: false
#### --azurefiles-connection-string
Azure Files Connection String.
Properties:
- Config: connection_string
- Env Var: RCLONE_AZUREFILES_CONNECTION_STRING
- Type: string
- Required: false
#### --azurefiles-tenant
ID of the service principal's tenant. Also called its directory ID.
Set this if using
- Service principal with client secret
- Service principal with certificate
- User with username and password
Properties:
- Config: tenant
- Env Var: RCLONE_AZUREFILES_TENANT
- Type: string
- Required: false
#### --azurefiles-client-id
The ID of the client in use.
Set this if using
- Service principal with client secret
- Service principal with certificate
- User with username and password
Properties:
- Config: client_id
- Env Var: RCLONE_AZUREFILES_CLIENT_ID
- Type: string
- Required: false
#### --azurefiles-client-secret
One of the service principal's client secrets
Set this if using
- Service principal with client secret
Properties:
- Config: client_secret
- Env Var: RCLONE_AZUREFILES_CLIENT_SECRET
- Type: string
- Required: false
#### --azurefiles-client-certificate-path
Path to a PEM or PKCS12 certificate file including the private key.
Set this if using
- Service principal with certificate
Properties:
- Config: client_certificate_path
- Env Var: RCLONE_AZUREFILES_CLIENT_CERTIFICATE_PATH
- Type: string
- Required: false
#### --azurefiles-client-certificate-password
Password for the certificate file (optional).
Optionally set this if using
- Service principal with certificate
And the certificate has a password.
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
Properties:
- Config: client_certificate_password
- Env Var: RCLONE_AZUREFILES_CLIENT_CERTIFICATE_PASSWORD
- Type: string
- Required: false
### Advanced options
Here are the Advanced options specific to azurefiles (Microsoft Azure Files).
#### --azurefiles-client-send-certificate-chain
Send the certificate chain when using certificate auth.
Specifies whether an authentication request will include an x5c header
to support subject name / issuer based authentication. When set to
true, authentication requests include the x5c header.
Optionally set this if using
- Service principal with certificate
Properties:
- Config: client_send_certificate_chain
- Env Var: RCLONE_AZUREFILES_CLIENT_SEND_CERTIFICATE_CHAIN
- Type: bool
- Default: false
#### --azurefiles-username
User name (usually an email address)
Set this if using
- User with username and password
Properties:
- Config: username
- Env Var: RCLONE_AZUREFILES_USERNAME
- Type: string
- Required: false
#### --azurefiles-password
The user's password
Set this if using
- User with username and password
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
Properties:
- Config: password
- Env Var: RCLONE_AZUREFILES_PASSWORD
- Type: string
- Required: false
#### --azurefiles-service-principal-file
Path to file containing credentials for use with a service principal.
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
$ az ad sp create-for-rbac --name "<name>" \
--role "Storage Files Data Owner" \
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
> azure-principal.json
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to files data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
**NB** this section needs updating for Azure Files - pull requests appreciated!
It may be more convenient to put the credentials directly into the
rclone config file under the `client_id`, `tenant` and `client_secret`
keys instead of setting `service_principal_file`.
Properties:
- Config: service_principal_file
- Env Var: RCLONE_AZUREFILES_SERVICE_PRINCIPAL_FILE
- Type: string
- Required: false
#### --azurefiles-use-msi
Use a managed service identity to authenticate (only works in Azure).
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
to authenticate to Azure Storage instead of a SAS token or account key.
If the VM(SS) on which this program is running has a system-assigned identity, it will
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
the user-assigned identity will be used by default. If the resource has multiple user-assigned
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
msi_client_id, or msi_mi_res_id parameters.
Properties:
- Config: use_msi
- Env Var: RCLONE_AZUREFILES_USE_MSI
- Type: bool
- Default: false
#### --azurefiles-msi-object-id
Object ID of the user-assigned MSI to use, if any.
Leave blank if msi_client_id or msi_mi_res_id specified.
Properties:
- Config: msi_object_id
- Env Var: RCLONE_AZUREFILES_MSI_OBJECT_ID
- Type: string
- Required: false
#### --azurefiles-msi-client-id
Object ID of the user-assigned MSI to use, if any.
Leave blank if msi_object_id or msi_mi_res_id specified.
Properties:
- Config: msi_client_id
- Env Var: RCLONE_AZUREFILES_MSI_CLIENT_ID
- Type: string
- Required: false
#### --azurefiles-msi-mi-res-id
Azure resource ID of the user-assigned MSI to use, if any.
Leave blank if msi_client_id or msi_object_id specified.
Properties:
- Config: msi_mi_res_id
- Env Var: RCLONE_AZUREFILES_MSI_MI_RES_ID
- Type: string
- Required: false
#### --azurefiles-endpoint
Endpoint for the service.
Leave blank normally.
Properties:
- Config: endpoint
- Env Var: RCLONE_AZUREFILES_ENDPOINT
- Type: string
- Required: false
#### --azurefiles-chunk-size
Upload chunk size.
Note that this is stored in memory and there may be up to
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.
Properties:
- Config: chunk_size
- Env Var: RCLONE_AZUREFILES_CHUNK_SIZE
- Type: SizeSuffix
- Default: 4Mi
#### --azurefiles-upload-concurrency
Concurrency for multipart uploads.
This is the number of chunks of the same file that are uploaded
concurrently.
If you are uploading small numbers of large files over high-speed
links and these uploads do not fully utilize your bandwidth, then
increasing this may help to speed up the transfers.
Note that chunks are stored in memory and there may be up to
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.
Properties:
- Config: upload_concurrency
- Env Var: RCLONE_AZUREFILES_UPLOAD_CONCURRENCY
- Type: int
- Default: 16
#### --azurefiles-max-stream-size
Max size for streamed files.
Azure files needs to know in advance how big the file will be. When
rclone doesn't know it uses this value instead.
This will be used when rclone is streaming data, the most common uses are:
- Uploading files with `--vfs-cache-mode off` with `rclone mount`
- Using `rclone rcat`
- Copying files with unknown length
You will need this much free space in the share as the file will be this size temporarily.
Properties:
- Config: max_stream_size
- Env Var: RCLONE_AZUREFILES_MAX_STREAM_SIZE
- Type: SizeSuffix
- Default: 10Gi
#### --azurefiles-encoding
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Properties:
- Config: encoding
- Env Var: RCLONE_AZUREFILES_ENCODING
- Type: Encoding
- Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8,Dot
{{< rem autogenerated options stop >}}
### Custom upload headers
You can set custom upload headers with the `--header-upload` flag.
- Cache-Control
- Content-Disposition
- Content-Encoding
- Content-Language
- Content-Type
Eg `--header-upload "Content-Type: text/potato"`
## Limitations
MD5 sums are only uploaded with chunked files if the source has an MD5
sum. This will always be the case for a local to azure copy.

View File

@ -58,6 +58,7 @@ See the following for detailed instructions for
* [Mega](/mega/) * [Mega](/mega/)
* [Memory](/memory/) * [Memory](/memory/)
* [Microsoft Azure Blob Storage](/azureblob/) * [Microsoft Azure Blob Storage](/azureblob/)
* [Microsoft Azure Files Storage](/azurefiles/)
* [Microsoft OneDrive](/onedrive/) * [Microsoft OneDrive](/onedrive/)
* [OpenStack Swift / Rackspace Cloudfiles / Blomp Cloud Storage / Memset Memstore](/swift/) * [OpenStack Swift / Rackspace Cloudfiles / Blomp Cloud Storage / Memset Memstore](/swift/)
* [OpenDrive](/opendrive/) * [OpenDrive](/opendrive/)

View File

@ -39,6 +39,7 @@ Here is an overview of the major features of each cloud storage system.
| Mega | - | - | No | Yes | - | - | | Mega | - | - | No | Yes | - | - |
| Memory | MD5 | R/W | No | No | - | - | | Memory | MD5 | R/W | No | No | - | - |
| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - | | Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - |
| Microsoft Azure Files Storage | MD5 | R/W | Yes | No | R/W | - |
| Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - | | Microsoft OneDrive | QuickXorHash ⁵ | R/W | Yes | No | R | - |
| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - | | OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - |
| OpenStack Swift | MD5 | R/W | No | No | R/W | - | | OpenStack Swift | MD5 | R/W | No | No | R/W | - |
@ -490,6 +491,7 @@ upon backend-specific capabilities.
| Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes | | Mega | Yes | No | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |
| Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No | | Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | No |
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No | | Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | Yes | No | No | No |
| Microsoft Azure Files Storage | No | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes |
| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | | OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
| OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No | | OpenStack Swift | Yes ¹ | Yes | No | No | No | Yes | Yes | No | No | Yes | No |

View File

@ -81,6 +81,7 @@
<a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a> <a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a>
<a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a> <a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a>
<a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a> <a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a>
<a class="dropdown-item" href="/azurefiles/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Files Storage</a>
<a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows fa-fw"></i> Microsoft OneDrive</a> <a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows fa-fw"></i> Microsoft OneDrive</a>
<a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle fa-fw"></i> OpenDrive</a> <a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle fa-fw"></i> OpenDrive</a>
<a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd fa-fw"></i> QingStor</a> <a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd fa-fw"></i> QingStor</a>

View File

@ -320,6 +320,8 @@ backends:
- backend: "azureblob" - backend: "azureblob"
remote: "TestAzureBlob,directory_markers:" remote: "TestAzureBlob,directory_markers:"
fastlist: true fastlist: true
- backend: "azurefiles"
remote: "TestAzureFiles:"
- backend: "pcloud" - backend: "pcloud"
remote: "TestPcloud:" remote: "TestPcloud:"
fastlist: true fastlist: true