From 15da53696e73e603ec60dee95d2364a0d0f105e5 Mon Sep 17 00:00:00 2001 From: NyaMisty Date: Sat, 30 Jan 2021 04:04:21 +0800 Subject: [PATCH] onedrive: add support for china region operated by 21vianet #4963 (#4963) fixes #3804 obsoletes #3973 obsoletes #4072 --- backend/onedrive/onedrive.go | 299 ++++++++++++++++++++---------- backend/onedrive/onedrive_test.go | 15 ++ 2 files changed, 215 insertions(+), 99 deletions(-) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 83705b7bb..e0523b454 100755 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -11,6 +11,7 @@ import ( "io" "log" "net/http" + "net/url" "path" "regexp" "strconv" @@ -46,7 +47,6 @@ const ( minSleep = 10 * time.Millisecond maxSleep = 2 * time.Second decayConstant = 2 // bigger for slower decay, exponential - graphURL = "https://graph.microsoft.com/v1.0" configDriveID = "drive_id" configDriveType = "drive_type" driveTypePersonal = "personal" @@ -54,22 +54,40 @@ const ( driveTypeSharepoint = "documentLibrary" defaultChunkSize = 10 * fs.MebiByte chunkSizeMultiple = 320 * fs.KibiByte + + regionGlobal = "global" + regionUS = "us" + regionDE = "de" + regionCN = "cn" ) // Globals var ( + authPath = "/common/oauth2/v2.0/authorize" + tokenPath = "/common/oauth2/v2.0/token" + // Description of how to auth for this app for a business account oauthConfig = &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", - TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token", - }, Scopes: []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access", "Sites.Read.All"}, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, } + graphAPIEndpoint = map[string]string{ + "global": "https://graph.microsoft.com", + "us": "https://graph.microsoft.us", + "de": "https://graph.microsoft.de", + "cn": "https://microsoftgraph.chinacloudapi.cn", + } + + authEndpoint = map[string]string{ + "global": "https://login.microsoftonline.com", + "us": "https://login.microsoftonline.us", + "de": "https://login.microsoftonline.de", + "cn": "https://login.chinacloudapi.cn", + } + // QuickXorHashType is the hash.Type for OneDrive QuickXorHashType hash.Type ) @@ -82,6 +100,12 @@ func init() { Description: "Microsoft OneDrive", NewFs: NewFs, Config: func(ctx context.Context, name string, m configmap.Mapper) { + region, _ := m.Get("region") + graphURL := graphAPIEndpoint[region] + "/v1.0" + oauthConfig.Endpoint = oauth2.Endpoint{ + AuthURL: authEndpoint[region] + authPath, + TokenURL: authEndpoint[region] + tokenPath, + } ci := fs.GetConfig(ctx) err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil) if err != nil { @@ -281,6 +305,25 @@ func init() { config.SaveConfig() }, Options: append(oauthutil.SharedOptions, []fs.Option{{ + Name: "region", + Help: "Choose national cloud region for OneDrive.", + Default: "global", + Examples: []fs.OptionExample{ + { + Value: regionGlobal, + Help: "Microsoft Cloud Global", + }, { + Value: regionUS, + Help: "Microsoft Cloud for US Government", + }, { + Value: regionDE, + Help: "Microsoft Cloud Germany", + }, { + Value: regionCN, + Help: "Azure and Office 365 operated by 21Vianet in China", + }, + }, + }, { Name: "chunk_size", Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes). @@ -420,6 +463,7 @@ At the time of writing this only works with OneDrive personal paid accounts. // Options defines the configuration for this backend type Options struct { + Region string `config:"region"` ChunkSize fs.SizeSuffix `config:"chunk_size"` DriveID string `config:"drive_id"` DriveType string `config:"drive_type"` @@ -549,10 +593,8 @@ func shouldRetry(resp *http.Response, err error) (bool, error) { // // If `relPath` == '', do not append the slash (See #3664) func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { - if relPath != "" { - relPath = "/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(relPath))) - } - opts := newOptsCall(normalizedID, "GET", ":"+relPath) + opts, _ := f.newOptsCallWithIDPath(normalizedID, relPath, true, "GET", "") + err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) @@ -567,17 +609,8 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It if f.driveType != driveTypePersonal || firstSlashIndex == -1 { var opts rest.Opts - if len(path) == 0 { - opts = rest.Opts{ - Method: "GET", - Path: "/root", - } - } else { - opts = rest.Opts{ - Method: "GET", - Path: "/root:/" + rest.URLPathEscape(f.opt.Enc.FromStandardPath(path)), - } - } + opts = f.newOptsCallWithPath(ctx, path, "GET", "") + opts.Path = strings.TrimSuffix(opts.Path, ":") err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) @@ -688,6 +721,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend") } + rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID + oauthConfig.Endpoint = oauth2.Endpoint{ + AuthURL: authEndpoint[opt.Region] + authPath, + TokenURL: authEndpoint[opt.Region] + tokenPath, + } + root = parsePath(root) oAuthClient, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig) if err != nil { @@ -702,7 +741,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ci: ci, driveID: opt.DriveID, driveType: opt.DriveType, - srv: rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID), + srv: rest.NewClient(oAuthClient).SetRoot(rootURL), pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } f.features = (&fs.Features{ @@ -823,7 +862,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e // fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf) var resp *http.Response var info *api.Item - opts := newOptsCall(dirID, "POST", "/children") + opts := f.newOptsCall(dirID, "POST", "/children") mkdir := api.CreateItemRequest{ Name: f.opt.Enc.FromStandardName(leaf), ConflictBehavior: "fail", @@ -855,7 +894,7 @@ type listAllFn func(*api.Item) bool func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { // Top parameter asks for bigger pages of data // https://dev.onedrive.com/odata/optional-query-parameters.htm - opts := newOptsCall(dirID, "GET", "/children?$top=1000") + opts := f.newOptsCall(dirID, "GET", "/children?$top=1000") OUTER: for { var result api.ListChildrenResponse @@ -994,7 +1033,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // deleteObject removes an object by ID func (f *Fs) deleteObject(ctx context.Context, id string) error { - opts := newOptsCall(id, "DELETE", "") + opts := f.newOptsCall(id, "DELETE", "") opts.NoResponse = true return f.pacer.Call(func() (bool, error) { @@ -1138,11 +1177,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Copy the object // The query param is a workaround for OneDrive Business for #4590 - opts := newOptsCall(srcObj.id, "POST", "/copy?@microsoft.graph.conflictBehavior=replace") + opts := f.newOptsCall(srcObj.id, "POST", "/copy?@microsoft.graph.conflictBehavior=replace") opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"} opts.NoResponse = true - id, dstDriveID, _ := parseNormalizedID(directoryID) + id, dstDriveID, _ := f.parseNormalizedID(directoryID) replacedLeaf := f.opt.Enc.FromStandardName(leaf) copyReq := api.CopyItemRequest{ @@ -1219,8 +1258,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, return nil, err } - id, dstDriveID, _ := parseNormalizedID(directoryID) - _, srcObjDriveID, _ := parseNormalizedID(srcObj.id) + id, dstDriveID, _ := f.parseNormalizedID(directoryID) + _, srcObjDriveID, _ := f.parseNormalizedID(srcObj.id) if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) { // https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0 @@ -1230,7 +1269,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } // Move the object - opts := newOptsCall(srcObj.id, "PATCH", "") + opts := f.newOptsCall(srcObj.id, "PATCH", "") move := api.MoveItemRequest{ Name: f.opt.Enc.FromStandardName(leaf), @@ -1281,8 +1320,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return err } - parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID) - _, srcDriveID, _ := parseNormalizedID(srcID) + parsedDstDirID, dstDriveID, _ := f.parseNormalizedID(dstDirectoryID) + _, srcDriveID, _ := f.parseNormalizedID(srcID) if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) { // https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0 @@ -1298,7 +1337,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } // Do the move - opts := newOptsCall(srcID, "PATCH", "") + opts := f.newOptsCall(srcID, "PATCH", "") move := api.MoveItemRequest{ Name: f.opt.Enc.FromStandardName(dstLeaf), ParentReference: &api.ItemReference{ @@ -1374,7 +1413,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, if err != nil { return "", err } - opts := newOptsCall(info.GetID(), "POST", "/createLink") + opts := f.newOptsCall(info.GetID(), "POST", "/createLink") share := api.CreateShareLinkRequest{ Type: f.opt.LinkType, @@ -1432,7 +1471,7 @@ func (f *Fs) CleanUp(ctx context.Context) error { // Finds and removes any old versions for o func (o *Object) deleteVersions(ctx context.Context) error { - opts := newOptsCall(o.id, "GET", "/versions") + opts := o.fs.newOptsCall(o.id, "GET", "/versions") var versions api.VersionsResponse err := o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions) @@ -1459,7 +1498,7 @@ func (o *Object) deleteVersion(ctx context.Context, ID string) error { return nil } fs.Infof(o, "removing version %q", ID) - opts := newOptsCall(o.id, "DELETE", "/versions/"+ID) + opts := o.fs.newOptsCall(o.id, "DELETE", "/versions/"+ID) opts.NoResponse = true return o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.Call(ctx, &opts) @@ -1604,21 +1643,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time { // setModTime sets the modification time of the local fs object func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, error) { - var opts rest.Opts - leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false) - trueDirID, drive, rootURL := parseNormalizedID(directoryID) - if drive != "" { - opts = rest.Opts{ - Method: "PATCH", - RootURL: rootURL, - Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))), - } - } else { - opts = rest.Opts{ - Method: "PATCH", - Path: "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())), - } - } + opts := o.fs.newOptsCallWithPath(ctx, o.remote, "PATCH", "") update := api.SetFileSystemInfo{ FileSystemInfo: api.FileSystemInfoFacet{ CreatedDateTime: api.Timestamp(modTime), @@ -1665,7 +1690,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read fs.FixRangeOption(options, o.size) var resp *http.Response - opts := newOptsCall(o.id, "GET", "/content") + opts := o.fs.newOptsCall(o.id, "GET", "/content") opts.Options = options err = o.fs.pacer.Call(func() (bool, error) { @@ -1685,22 +1710,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // createUploadSession creates an upload session for the object func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (response *api.CreateUploadResponse, err error) { - leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false) - id, drive, rootURL := parseNormalizedID(directoryID) - var opts rest.Opts - if drive != "" { - opts = rest.Opts{ - Method: "POST", - RootURL: rootURL, - Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession", - drive, id, rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))), - } - } else { - opts = rest.Opts{ - Method: "POST", - Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/createUploadSession", - } - } + opts := o.fs.newOptsCallWithPath(ctx, o.remote, "POST", "/createUploadSession") createRequest := api.CreateUploadRequest{} createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime) createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime) @@ -1873,27 +1883,10 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, fs.Debugf(o, "Starting singlepart upload") var resp *http.Response - var opts rest.Opts - leaf, directoryID, _ := o.fs.dirCache.FindPath(ctx, o.remote, false) - trueDirID, drive, rootURL := parseNormalizedID(directoryID) - if drive != "" { - opts = rest.Opts{ - Method: "PUT", - RootURL: rootURL, - Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf)) + ":/content", - ContentLength: &size, - Body: in, - Options: options, - } - } else { - opts = rest.Opts{ - Method: "PUT", - Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content", - ContentLength: &size, - Body: in, - Options: options, - } - } + opts := o.fs.newOptsCallWithPath(ctx, o.remote, "PUT", "/content") + opts.ContentLength = &size + opts.Body = in + opts.Options = options err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) @@ -1969,8 +1962,42 @@ func (o *Object) ID() string { return o.id } -func newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) { - id, drive, rootURL := parseNormalizedID(normalizedID) +/* + * URL Build routine area start + * 1. In this area, region-related URL rewrites are applied. As the API is blackbox, + * we cannot thoroughly test this part. Please be extremely careful while changing them. + * 2. If possible, please don't introduce region related code in other region, but patch these helper functions. + * 3. To avoid region-related issues, please don't manually build rest.Opts from scratch. + * Instead, use these helper function, and customize the URL afterwards if needed. + * + * currently, the 21ViaNet's API differs in the following places: + * - https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} + * - this API doesn't work (gives invalid request) + * - can be replaced with the following API: + * - https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route} + * - however, this API does NOT support multi-level leaf like a/b/c + * - https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") + * - this API does support multi-level leaf like a/b/c + * - https://{Endpoint}/drives/{driveID}/root/children('@a1')/{route}?@a1=URLEncode({path}) + * - Same as above + */ + +// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`) +// and returns itemID, driveID, rootURL. +// Such a normalized ID can come from (*Item).GetID() +func (f *Fs) parseNormalizedID(ID string) (string, string, string) { + rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives" + if strings.Index(ID, "#") >= 0 { + s := strings.Split(ID, "#") + return s[1], s[0], rootURL + } + return ID, "", "" +} + +// newOptsCall build the rest.Opts structure with *a normalizedID(driveID#fileID, or simply fileID)* +// using url template https://{Endpoint}/drives/{driveID}/items/{itemID}/{route} +func (f *Fs) newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) { + id, drive, rootURL := f.parseNormalizedID(normalizedID) if drive != "" { return rest.Opts{ @@ -1985,17 +2012,91 @@ func newOptsCall(normalizedID string, method string, route string) (opts rest.Op } } -// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`) -// and returns itemID, driveID, rootURL. -// Such a normalized ID can come from (*Item).GetID() -func parseNormalizedID(ID string) (string, string, string) { - if strings.Index(ID, "#") >= 0 { - s := strings.Split(ID, "#") - return s[1], s[0], graphURL + "/drives" - } - return ID, "", "" +func escapeSingleQuote(str string) string { + return strings.ReplaceAll(str, "'", "''") } +// newOptsCallWithIDPath build the rest.Opts structure with *a normalizedID (driveID#fileID, or simply fileID) and leaf* +// using url template https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} (for international OneDrive) +// or https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route} +// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for 21ViaNet) +// if isPath is false, this function will only work when the leaf is "" or a child name (i.e. it doesn't accept multi-level leaf) +// if isPath is true, multi-level leaf like a/b/c can be passed +func (f *Fs) newOptsCallWithIDPath(normalizedID string, leaf string, isPath bool, method string, route string) (opts rest.Opts, ok bool) { + encoder := f.opt.Enc.FromStandardName + if isPath { + encoder = f.opt.Enc.FromStandardPath + } + trueDirID, drive, rootURL := f.parseNormalizedID(normalizedID) + if drive == "" { + trueDirID = normalizedID + } + entity := "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(encoder(leaf))) + route + if f.opt.Region == regionCN { + if isPath { + entity = "/items/" + trueDirID + "/children('@a1')" + route + "?@a1=" + url.QueryEscape("'"+encoder(escapeSingleQuote(leaf))+"'") + } else { + entity = "/items/" + trueDirID + "/children('" + rest.URLPathEscape(encoder(escapeSingleQuote(leaf))) + "')" + route + } + } + if drive == "" { + ok = false + opts = rest.Opts{ + Method: method, + Path: entity, + } + return + } + ok = true + opts = rest.Opts{ + Method: method, + RootURL: rootURL, + Path: "/" + drive + entity, + } + return +} + +// newOptsCallWithIDPath build the rest.Opts structure with an *absolute path start from root* +// using url template https://{Endpoint}/drives/{driveID}/root:/{path}:/{route} +// or https://{Endpoint}/drives/{driveID}/root/children('@a1')/{route}?@a1=URLEncode({path}) +func (f *Fs) newOptsCallWithRootPath(path string, method string, route string) (opts rest.Opts) { + path = strings.TrimSuffix(path, "/") + newURL := "/root:/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(path))) + route + if f.opt.Region == regionCN { + newURL = "/root/children('@a1')" + route + "?@a1=" + url.QueryEscape("'"+escapeSingleQuote(f.opt.Enc.FromStandardPath(path))+"'") + } + return rest.Opts{ + Method: method, + Path: newURL, + } +} + +// newOptsCallWithPath build the rest.Opt intelligently. +// It will first try to resolve the path using dircache, which enables support for "Share with me" files. +// If present in cache, then use ID + Path variant, else fallback into RootPath variant +func (f *Fs) newOptsCallWithPath(ctx context.Context, path string, method string, route string) (opts rest.Opts) { + if path == "" { + url := "/root" + route + return rest.Opts{ + Method: method, + Path: url, + } + } + + // find dircache + leaf, directoryID, _ := f.dirCache.FindPath(ctx, path, false) + // try to use IDPath variant first + if opts, ok := f.newOptsCallWithIDPath(directoryID, leaf, false, method, route); ok { + return opts + } + // fallback to use RootPath variant first + return f.newOptsCallWithRootPath(path, method, route) +} + +/* + * URL Build routine area end + */ + // Returns the canonical form of the driveID func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) { if driveID == "" { diff --git a/backend/onedrive/onedrive_test.go b/backend/onedrive/onedrive_test.go index 93d6d41b8..c0c4340e4 100644 --- a/backend/onedrive/onedrive_test.go +++ b/backend/onedrive/onedrive_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" ) @@ -19,6 +20,20 @@ func TestIntegration(t *testing.T) { }) } +// TestIntegrationCn runs integration tests against the remote +func TestIntegrationCn(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("skipping as -remote is set") + } + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestOneDriveCn:", + NilObject: (*Object)(nil), + ChunkedUpload: fstests.ChunkedUploadConfig{ + CeilChunkSize: fstests.NextMultipleOf(chunkSizeMultiple), + }, + }) +} + func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) { return f.setUploadChunkSize(cs) }