github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-25 09:41:44 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

onedrive: add support for china region operated by 21vianet #4963 (#4963)

Browse Source 
fixes #3804
obsoletes #3973
obsoletes #4072
This commit is contained in:
NyaMisty 2021-01-30 04:04:21 +08:00 committed by GitHub
parent 2bddba118e
commit 15da53696e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 215 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

299
backend/onedrive/onedrive.go
View File

@ -11,6 +11,7 @@ import (
"io"
"log"
"net/http"
"net/url"
"path"
"regexp"
"strconv"
@ -46,7 +47,6 @@ const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
graphURL = "https://graph.microsoft.com/v1.0"
configDriveID = "drive_id"
configDriveType = "drive_type"
driveTypePersonal = "personal"
@ -54,22 +54,40 @@ const (
driveTypeSharepoint = "documentLibrary"
defaultChunkSize = 10 * fs.MebiByte
chunkSizeMultiple = 320 * fs.KibiByte
regionGlobal = "global"
regionUS = "us"
regionDE = "de"
regionCN = "cn"
)
// Globals
var (
authPath = "/common/oauth2/v2.0/authorize"
tokenPath = "/common/oauth2/v2.0/token"
// Description of how to auth for this app for a business account
oauthConfig = &oauth2.Config{
Endpoint: oauth2.Endpoint{
AuthURL: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
},
Scopes: []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access", "Sites.Read.All"},
ClientID: rcloneClientID,
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
RedirectURL: oauthutil.RedirectLocalhostURL,
}
graphAPIEndpoint = map[string]string{
"global": "https://graph.microsoft.com",
"us": "https://graph.microsoft.us",
"de": "https://graph.microsoft.de",
"cn": "https://microsoftgraph.chinacloudapi.cn",
}
authEndpoint = map[string]string{
"global": "https://login.microsoftonline.com",
"us": "https://login.microsoftonline.us",
"de": "https://login.microsoftonline.de",
"cn": "https://login.chinacloudapi.cn",
}
// QuickXorHashType is the hash.Type for OneDrive
QuickXorHashType hash.Type
)
@ -82,6 +100,12 @@ func init() {
Description: "Microsoft OneDrive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) {
region, _ := m.Get("region")
graphURL := graphAPIEndpoint[region] + "/v1.0"
oauthConfig.Endpoint = oauth2.Endpoint{
AuthURL: authEndpoint[region] + authPath,
TokenURL: authEndpoint[region] + tokenPath,
}
ci := fs.GetConfig(ctx)
err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
if err != nil {
@ -281,6 +305,25 @@ func init() {
config.SaveConfig()
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "region",
Help: "Choose national cloud region for OneDrive.",
Default: "global",
Examples: []fs.OptionExample{
{
Value: regionGlobal,
Help: "Microsoft Cloud Global",
}, {
Value: regionUS,
Help: "Microsoft Cloud for US Government",
}, {
Value: regionDE,
Help: "Microsoft Cloud Germany",
}, {
Value: regionCN,
Help: "Azure and Office 365 operated by 21Vianet in China",
},
},
}, {
Name: "chunk_size",
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@ -420,6 +463,7 @@ At the time of writing this only works with OneDrive personal paid accounts.
// Options defines the configuration for this backend
type Options struct {
Region string `config:"region"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
DriveID string `config:"drive_id"`
DriveType string `config:"drive_type"`
@ -549,10 +593,8 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
//
// If `relPath` == '', do not append the slash (See #3664)
func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
if relPath != "" {
relPath = "/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(relPath)))
}
opts := newOptsCall(normalizedID, "GET", ":"+relPath)
opts, _ := f.newOptsCallWithIDPath(normalizedID, relPath, true, "GET", "")
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err)
@ -567,17 +609,8 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
if f.driveType != driveTypePersonal || firstSlashIndex == -1 {
var opts rest.Opts
if len(path) == 0 {
opts = rest.Opts{
Method: "GET",
Path: "/root",
}
} else {
opts = rest.Opts{
Method: "GET",
Path: "/root:/" + rest.URLPathEscape(f.opt.Enc.FromStandardPath(path)),
}
}
opts = f.newOptsCallWithPath(ctx, path, "GET", "")
opts.Path = strings.TrimSuffix(opts.Path, ":")
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err)
@ -688,6 +721,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
}
rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
oauthConfig.Endpoint = oauth2.Endpoint{
AuthURL: authEndpoint[opt.Region] + authPath,
TokenURL: authEndpoint[opt.Region] + tokenPath,
}
root = parsePath(root)
oAuthClient, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
@ -702,7 +741,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
driveID: opt.DriveID,
driveType: opt.DriveType,
srv: rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
@ -823,7 +862,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
// fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf)
var resp *http.Response
var info *api.Item
opts := newOptsCall(dirID, "POST", "/children")
opts := f.newOptsCall(dirID, "POST", "/children")
mkdir := api.CreateItemRequest{
Name: f.opt.Enc.FromStandardName(leaf),
ConflictBehavior: "fail",
@ -855,7 +894,7 @@ type listAllFn func(*api.Item) bool
func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
// Top parameter asks for bigger pages of data
// https://dev.onedrive.com/odata/optional-query-parameters.htm
opts := newOptsCall(dirID, "GET", "/children?$top=1000")
opts := f.newOptsCall(dirID, "GET", "/children?$top=1000")
OUTER:
for {
var result api.ListChildrenResponse
@ -994,7 +1033,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// deleteObject removes an object by ID
func (f *Fs) deleteObject(ctx context.Context, id string) error {
opts := newOptsCall(id, "DELETE", "")
opts := f.newOptsCall(id, "DELETE", "")
opts.NoResponse = true
return f.pacer.Call(func() (bool, error) {
@ -1138,11 +1177,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
// Copy the object
// The query param is a workaround for OneDrive Business for #4590
opts := newOptsCall(srcObj.id, "POST", "/copy?@microsoft.graph.conflictBehavior=replace")
opts := f.newOptsCall(srcObj.id, "POST", "/copy?@microsoft.graph.conflictBehavior=replace")
opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"}
opts.NoResponse = true
id, dstDriveID, _ := parseNormalizedID(directoryID)
id, dstDriveID, _ := f.parseNormalizedID(directoryID)
replacedLeaf := f.opt.Enc.FromStandardName(leaf)
copyReq := api.CopyItemRequest{
@ -1219,8 +1258,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, err
}
id, dstDriveID, _ := parseNormalizedID(directoryID)
_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
id, dstDriveID, _ := f.parseNormalizedID(directoryID)
_, srcObjDriveID, _ := f.parseNormalizedID(srcObj.id)
if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) {
// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
@ -1230,7 +1269,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
}
// Move the object
opts := newOptsCall(srcObj.id, "PATCH", "")
opts := f.newOptsCall(srcObj.id, "PATCH", "")
move := api.MoveItemRequest{
Name: f.opt.Enc.FromStandardName(leaf),
@ -1281,8 +1320,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return err
}
parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
_, srcDriveID, _ := parseNormalizedID(srcID)
parsedDstDirID, dstDriveID, _ := f.parseNormalizedID(dstDirectoryID)
_, srcDriveID, _ := f.parseNormalizedID(srcID)
if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) {
// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
@ -1298,7 +1337,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// Do the move
opts := newOptsCall(srcID, "PATCH", "")
opts := f.newOptsCall(srcID, "PATCH", "")
move := api.MoveItemRequest{
Name: f.opt.Enc.FromStandardName(dstLeaf),
ParentReference: &api.ItemReference{
@ -1374,7 +1413,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
if err != nil {
return "", err
}
opts := newOptsCall(info.GetID(), "POST", "/createLink")
opts := f.newOptsCall(info.GetID(), "POST", "/createLink")
share := api.CreateShareLinkRequest{
Type: f.opt.LinkType,
@ -1432,7 +1471,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
// Finds and removes any old versions for o
func (o *Object) deleteVersions(ctx context.Context) error {
opts := newOptsCall(o.id, "GET", "/versions")
opts := o.fs.newOptsCall(o.id, "GET", "/versions")
var versions api.VersionsResponse
err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
@ -1459,7 +1498,7 @@ func (o *Object) deleteVersion(ctx context.Context, ID string) error {
return nil
}
fs.Infof(o, "removing version %q", ID)
opts := newOptsCall(o.id, "DELETE", "/versions/"+ID)
opts := o.fs.newOptsCall(o.id, "DELETE", "/versions/"+ID)
opts.NoResponse = true
return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.Call(ctx, &opts)
@ -1604,21 +1643,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
// setModTime sets the modification time of the local fs object
func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, error) {
var opts rest.Opts
leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false)
trueDirID, drive, rootURL := parseNormalizedID(directoryID)
if drive != "" {
opts = rest.Opts{
Method: "PATCH",
RootURL: rootURL,
Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
}
} else {
opts = rest.Opts{
Method: "PATCH",
Path: "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
}
}
opts := o.fs.newOptsCallWithPath(ctx, o.remote, "PATCH", "")
update := api.SetFileSystemInfo{
FileSystemInfo: api.FileSystemInfoFacet{
CreatedDateTime: api.Timestamp(modTime),
@ -1665,7 +1690,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
fs.FixRangeOption(options, o.size)
var resp *http.Response
opts := newOptsCall(o.id, "GET", "/content")
opts := o.fs.newOptsCall(o.id, "GET", "/content")
opts.Options = options
err = o.fs.pacer.Call(func() (bool, error) {
@ -1685,22 +1710,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// createUploadSession creates an upload session for the object
func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (response *api.CreateUploadResponse, err error) {
leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false)
id, drive, rootURL := parseNormalizedID(directoryID)
var opts rest.Opts
if drive != "" {
opts = rest.Opts{
Method: "POST",
RootURL: rootURL,
Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession",
drive, id, rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
}
} else {
opts = rest.Opts{
Method: "POST",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/createUploadSession",
}
}
opts := o.fs.newOptsCallWithPath(ctx, o.remote, "POST", "/createUploadSession")
createRequest := api.CreateUploadRequest{}
createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime)
createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime)
@ -1873,27 +1883,10 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
fs.Debugf(o, "Starting singlepart upload")
var resp *http.Response
var opts rest.Opts
leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false)
trueDirID, drive, rootURL := parseNormalizedID(directoryID)
if drive != "" {
opts = rest.Opts{
Method: "PUT",
RootURL: rootURL,
Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf)) + ":/content",
ContentLength: &size,
Body: in,
Options: options,
}
} else {
opts = rest.Opts{
Method: "PUT",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
ContentLength: &size,
Body: in,
Options: options,
}
}
opts := o.fs.newOptsCallWithPath(ctx, o.remote, "PUT", "/content")
opts.ContentLength = &size
opts.Body = in
opts.Options = options
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
@ -1969,8 +1962,42 @@ func (o *Object) ID() string {
return o.id
}
func newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) {
id, drive, rootURL := parseNormalizedID(normalizedID)
/*
* URL Build routine area start
* 1. In this area, region-related URL rewrites are applied. As the API is blackbox,
* we cannot thoroughly test this part. Please be extremely careful while changing them.
* 2. If possible, please don't introduce region related code in other region, but patch these helper functions.
* 3. To avoid region-related issues, please don't manually build rest.Opts from scratch.
* Instead, use these helper function, and customize the URL afterwards if needed.
*
* currently, the 21ViaNet's API differs in the following places:
* - https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route}
* - this API doesn't work (gives invalid request)
* - can be replaced with the following API:
* - https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route}
* - however, this API does NOT support multi-level leaf like a/b/c
* - https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'")
* - this API does support multi-level leaf like a/b/c
* - https://{Endpoint}/drives/{driveID}/root/children('@a1')/{route}?@a1=URLEncode({path})
* - Same as above
*/
// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`)
// and returns itemID, driveID, rootURL.
// Such a normalized ID can come from (*Item).GetID()
func (f *Fs) parseNormalizedID(ID string) (string, string, string) {
rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
if strings.Index(ID, "#") >= 0 {
s := strings.Split(ID, "#")
return s[1], s[0], rootURL
}
return ID, "", ""
}
// newOptsCall build the rest.Opts structure with *a normalizedID(driveID#fileID, or simply fileID)*
// using url template https://{Endpoint}/drives/{driveID}/items/{itemID}/{route}
func (f *Fs) newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) {
id, drive, rootURL := f.parseNormalizedID(normalizedID)
if drive != "" {
return rest.Opts{
@ -1985,17 +2012,91 @@ func newOptsCall(normalizedID string, method string, route string) (opts rest.Op
}
}
// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`)
// and returns itemID, driveID, rootURL.
// Such a normalized ID can come from (*Item).GetID()
func parseNormalizedID(ID string) (string, string, string) {
if strings.Index(ID, "#") >= 0 {
s := strings.Split(ID, "#")
return s[1], s[0], graphURL + "/drives"
}
return ID, "", ""
func escapeSingleQuote(str string) string {
return strings.ReplaceAll(str, "'", "''")
}
// newOptsCallWithIDPath build the rest.Opts structure with *a normalizedID (driveID#fileID, or simply fileID) and leaf*
// using url template https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} (for international OneDrive)
// or https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route}
// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for 21ViaNet)
// if isPath is false, this function will only work when the leaf is "" or a child name (i.e. it doesn't accept multi-level leaf)
// if isPath is true, multi-level leaf like a/b/c can be passed
func (f *Fs) newOptsCallWithIDPath(normalizedID string, leaf string, isPath bool, method string, route string) (opts rest.Opts, ok bool) {
encoder := f.opt.Enc.FromStandardName
if isPath {
encoder = f.opt.Enc.FromStandardPath
}
trueDirID, drive, rootURL := f.parseNormalizedID(normalizedID)
if drive == "" {
trueDirID = normalizedID
}
entity := "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(encoder(leaf))) + route
if f.opt.Region == regionCN {
if isPath {
entity = "/items/" + trueDirID + "/children('@a1')" + route + "?@a1=" + url.QueryEscape("'"+encoder(escapeSingleQuote(leaf))+"'")
} else {
entity = "/items/" + trueDirID + "/children('" + rest.URLPathEscape(encoder(escapeSingleQuote(leaf))) + "')" + route
}
}
if drive == "" {
ok = false
opts = rest.Opts{
Method: method,
Path: entity,
}
return
}
ok = true
opts = rest.Opts{
Method: method,
RootURL: rootURL,
Path: "/" + drive + entity,
}
return
}
// newOptsCallWithIDPath build the rest.Opts structure with an *absolute path start from root*
// using url template https://{Endpoint}/drives/{driveID}/root:/{path}:/{route}
// or https://{Endpoint}/drives/{driveID}/root/children('@a1')/{route}?@a1=URLEncode({path})
func (f *Fs) newOptsCallWithRootPath(path string, method string, route string) (opts rest.Opts) {
path = strings.TrimSuffix(path, "/")
newURL := "/root:/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(path))) + route
if f.opt.Region == regionCN {
newURL = "/root/children('@a1')" + route + "?@a1=" + url.QueryEscape("'"+escapeSingleQuote(f.opt.Enc.FromStandardPath(path))+"'")
}
return rest.Opts{
Method: method,
Path: newURL,
}
}
// newOptsCallWithPath build the rest.Opt intelligently.
// It will first try to resolve the path using dircache, which enables support for "Share with me" files.
// If present in cache, then use ID + Path variant, else fallback into RootPath variant
func (f *Fs) newOptsCallWithPath(ctx context.Context, path string, method string, route string) (opts rest.Opts) {
if path == "" {
url := "/root" + route
return rest.Opts{
Method: method,
Path: url,
}
}
// find dircache
leaf, directoryID, _ := f.dirCache.FindPath(ctx, path, false)
// try to use IDPath variant first
if opts, ok := f.newOptsCallWithIDPath(directoryID, leaf, false, method, route); ok {
return opts
}
// fallback to use RootPath variant first
return f.newOptsCallWithRootPath(path, method, route)
}
/*
* URL Build routine area end
*/
// Returns the canonical form of the driveID
func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
if driveID == "" {

15
backend/onedrive/onedrive_test.go
View File

@ -5,6 +5,7 @@ import (
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
)
@ -19,6 +20,20 @@ func TestIntegration(t *testing.T) {
})
}
// TestIntegrationCn runs integration tests against the remote
func TestIntegrationCn(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("skipping as -remote is set")
}
fstests.Run(t, &fstests.Opt{
RemoteName: "TestOneDriveCn:",
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
CeilChunkSize: fstests.NextMultipleOf(chunkSizeMultiple),
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}