From 65012beea4b93b7faf20692079e2b688c5164716 Mon Sep 17 00:00:00 2001 From: Martin Hassack Date: Tue, 26 Jul 2022 07:28:37 +0100 Subject: [PATCH] lib/oauthutil: add support for OAuth client credential flow This commit reorganises the oauth code to use our own config struct which has all the info for the normal oauth method and also the client credentials flow method. It updates all backends which use lib/oauthutil to use the new config struct which shouldn't change any functionality. It also adds code for dealing with the client credential flow config which doesn't require the use of a browser and doesn't have or need a refresh token. Co-authored-by: Nick Craig-Wood --- backend/box/box.go | 11 +- backend/drive/drive.go | 5 +- backend/dropbox/dropbox.go | 7 +- .../googlecloudstorage/googlecloudstorage.go | 19 +- backend/googlephotos/googlephotos.go | 6 +- backend/hidrive/hidrive.go | 9 +- backend/jottacloud/jottacloud.go | 48 ++--- backend/mailru/mailru.go | 14 +- backend/onedrive/onedrive.go | 16 +- backend/pcloud/pcloud.go | 16 +- backend/pikpak/pikpak.go | 12 +- backend/premiumizeme/premiumizeme.go | 11 +- backend/putio/putio.go | 11 +- backend/sharefile/sharefile.go | 17 +- backend/yandex/yandex.go | 9 +- backend/zoho/zoho.go | 15 +- fs/config/config.go | 3 + lib/oauthutil/oauthutil.go | 204 +++++++++++++++--- 18 files changed, 272 insertions(+), 161 deletions(-) diff --git a/backend/box/box.go b/backend/box/box.go index 7183f35e5..6f73441cc 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -46,7 +46,6 @@ import ( "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/rest" "github.com/youmark/pkcs8" - "golang.org/x/oauth2" ) const ( @@ -65,12 +64,10 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Scopes: nil, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://app.box.com/api/oauth2/authorize", - TokenURL: "https://app.box.com/api/oauth2/token", - }, + oauthConfig = &oauthutil.Config{ + Scopes: nil, + AuthURL: "https://app.box.com/api/oauth2/authorize", + TokenURL: "https://app.box.com/api/oauth2/token", ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectURL, diff --git a/backend/drive/drive.go b/backend/drive/drive.go index f49603437..8e878751a 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -80,9 +80,10 @@ const ( // Globals var ( // Description of how to auth for this app - driveConfig = &oauth2.Config{ + driveConfig = &oauthutil.Config{ Scopes: []string{scopePrefix + "drive"}, - Endpoint: google.Endpoint, + AuthURL: google.Endpoint.AuthURL, + TokenURL: google.Endpoint.TokenURL, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectURL, diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 520c5dbc2..c1762a506 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -94,7 +94,7 @@ const ( var ( // Description of how to auth for this app - dropboxConfig = &oauth2.Config{ + dropboxConfig = &oauthutil.Config{ Scopes: []string{ "files.metadata.write", "files.content.write", @@ -109,7 +109,8 @@ var ( // AuthURL: "https://www.dropbox.com/1/oauth2/authorize", // TokenURL: "https://api.dropboxapi.com/1/oauth2/token", // }, - Endpoint: dropbox.OAuthEndpoint(""), + AuthURL: dropbox.OAuthEndpoint("").AuthURL, + TokenURL: dropbox.OAuthEndpoint("").TokenURL, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, @@ -134,7 +135,7 @@ var ( ) // Gets an oauth config with the right scopes -func getOauthConfig(m configmap.Mapper) *oauth2.Config { +func getOauthConfig(m configmap.Mapper) *oauthutil.Config { // If not impersonating, use standard scopes if impersonate, _ := m.Get("impersonate"); impersonate == "" { return dropboxConfig diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 5953e24b8..cd57edd53 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -60,14 +60,17 @@ const ( minSleep = 10 * time.Millisecond ) -// Description of how to auth for this app -var storageConfig = &oauth2.Config{ - Scopes: []string{storage.DevstorageReadWriteScope}, - Endpoint: google.Endpoint, - ClientID: rcloneClientID, - ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), - RedirectURL: oauthutil.RedirectURL, -} +var ( + // Description of how to auth for this app + storageConfig = &oauthutil.Config{ + Scopes: []string{storage.DevstorageReadWriteScope}, + AuthURL: google.Endpoint.AuthURL, + TokenURL: google.Endpoint.TokenURL, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectURL, + } +) // Register with Fs func init() { diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index 263a5610b..33ee41a2e 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -33,7 +33,6 @@ import ( "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" "golang.org/x/oauth2/google" ) @@ -60,13 +59,14 @@ const ( var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ + oauthConfig = &oauthutil.Config{ Scopes: []string{ "openid", "profile", scopeReadWrite, // this must be at position scopeAccess }, - Endpoint: google.Endpoint, + AuthURL: google.Endpoint.AuthURL, + TokenURL: google.Endpoint.TokenURL, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectURL, diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.go index 6f690f629..33b2fb09c 100644 --- a/backend/hidrive/hidrive.go +++ b/backend/hidrive/hidrive.go @@ -31,7 +31,6 @@ import ( "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" ) const ( @@ -48,11 +47,9 @@ const ( // Globals var ( // Description of how to auth for this app. - oauthConfig = &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: "https://my.hidrive.com/client/authorize", - TokenURL: "https://my.hidrive.com/oauth2/token", - }, + oauthConfig = &oauthutil.Config{ + AuthURL: "https://my.hidrive.com/client/authorize", + TokenURL: "https://my.hidrive.com/oauth2/token", ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.TitleBarRedirectURL, diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 8ad457c74..6482070ab 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -277,11 +277,9 @@ machines.`) m.Set(configClientID, teliaseCloudClientID) m.Set(configTokenURL, teliaseCloudTokenURL) return oauthutil.ConfigOut("choose_device", &oauthutil.Options{ - OAuth2Config: &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: teliaseCloudAuthURL, - TokenURL: teliaseCloudTokenURL, - }, + OAuth2Config: &oauthutil.Config{ + AuthURL: teliaseCloudAuthURL, + TokenURL: teliaseCloudTokenURL, ClientID: teliaseCloudClientID, Scopes: []string{"openid", "jotta-default", "offline_access"}, RedirectURL: oauthutil.RedirectLocalhostURL, @@ -292,11 +290,9 @@ machines.`) m.Set(configClientID, telianoCloudClientID) m.Set(configTokenURL, telianoCloudTokenURL) return oauthutil.ConfigOut("choose_device", &oauthutil.Options{ - OAuth2Config: &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: telianoCloudAuthURL, - TokenURL: telianoCloudTokenURL, - }, + OAuth2Config: &oauthutil.Config{ + AuthURL: telianoCloudAuthURL, + TokenURL: telianoCloudTokenURL, ClientID: telianoCloudClientID, Scopes: []string{"openid", "jotta-default", "offline_access"}, RedirectURL: oauthutil.RedirectLocalhostURL, @@ -307,11 +303,9 @@ machines.`) m.Set(configClientID, tele2CloudClientID) m.Set(configTokenURL, tele2CloudTokenURL) return oauthutil.ConfigOut("choose_device", &oauthutil.Options{ - OAuth2Config: &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: tele2CloudAuthURL, - TokenURL: tele2CloudTokenURL, - }, + OAuth2Config: &oauthutil.Config{ + AuthURL: tele2CloudAuthURL, + TokenURL: tele2CloudTokenURL, ClientID: tele2CloudClientID, Scopes: []string{"openid", "jotta-default", "offline_access"}, RedirectURL: oauthutil.RedirectLocalhostURL, @@ -322,11 +316,9 @@ machines.`) m.Set(configClientID, onlimeCloudClientID) m.Set(configTokenURL, onlimeCloudTokenURL) return oauthutil.ConfigOut("choose_device", &oauthutil.Options{ - OAuth2Config: &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: onlimeCloudAuthURL, - TokenURL: onlimeCloudTokenURL, - }, + OAuth2Config: &oauthutil.Config{ + AuthURL: onlimeCloudAuthURL, + TokenURL: onlimeCloudTokenURL, ClientID: onlimeCloudClientID, Scopes: []string{"openid", "jotta-default", "offline_access"}, RedirectURL: oauthutil.RedirectLocalhostURL, @@ -924,19 +916,17 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth } baseClient := fshttp.NewClient(ctx) - oauthConfig := &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: defaultTokenURL, - TokenURL: defaultTokenURL, - }, + oauthConfig := &oauthutil.Config{ + AuthURL: defaultTokenURL, + TokenURL: defaultTokenURL, } if ver == configVersion { oauthConfig.ClientID = defaultClientID // if custom endpoints are set use them else stick with defaults if tokenURL, ok := m.Get(configTokenURL); ok { - oauthConfig.Endpoint.TokenURL = tokenURL + oauthConfig.TokenURL = tokenURL // jottacloud is weird. we need to use the tokenURL as authURL - oauthConfig.Endpoint.AuthURL = tokenURL + oauthConfig.AuthURL = tokenURL } } else if ver == legacyConfigVersion { clientID, ok := m.Get(configClientID) @@ -950,8 +940,8 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth oauthConfig.ClientID = clientID oauthConfig.ClientSecret = obscure.MustReveal(clientSecret) - oauthConfig.Endpoint.TokenURL = legacyTokenURL - oauthConfig.Endpoint.AuthURL = legacyTokenURL + oauthConfig.TokenURL = legacyTokenURL + oauthConfig.AuthURL = legacyTokenURL // add the request filter to fix token refresh if do, ok := baseClient.Transport.(interface { diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index d92a5290c..c0a89939a 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -68,14 +68,12 @@ var ( ) // Description of how to authorize -var oauthConfig = &oauth2.Config{ +var oauthConfig = &oauthutil.Config{ ClientID: api.OAuthClientID, ClientSecret: "", - Endpoint: oauth2.Endpoint{ - AuthURL: api.OAuthURL, - TokenURL: api.OAuthURL, - AuthStyle: oauth2.AuthStyleInParams, - }, + AuthURL: api.OAuthURL, + TokenURL: api.OAuthURL, + AuthStyle: oauth2.AuthStyleInParams, } // Register with Fs @@ -438,7 +436,9 @@ func (f *Fs) authorize(ctx context.Context, force bool) (err error) { if err != nil || !tokenIsValid(t) { fs.Infof(f, "Valid token not found, authorizing.") ctx := oauthutil.Context(ctx, f.cli) - t, err = oauthConfig.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password) + + oauth2Conf := oauthConfig.MakeOauth2Config() + t, err = oauth2Conf.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password) } if err == nil && !tokenIsValid(t) { err = errors.New("invalid token") diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 97dc8245f..29eb270ff 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -40,7 +40,6 @@ import ( "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" ) const ( @@ -72,7 +71,7 @@ var ( scopeAccessWithoutSites = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"} // Description of how to auth for this app for a business account - oauthConfig = &oauth2.Config{ + oauthConfig = &oauthutil.Config{ Scopes: scopeAccess, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), @@ -543,10 +542,9 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf if disableSitePermission == "true" { oauthConfig.Scopes = scopeAccessWithoutSites } - oauthConfig.Endpoint = oauth2.Endpoint{ - AuthURL: authEndpoint[region] + authPath, - TokenURL: authEndpoint[region] + tokenPath, - } + oauthConfig.TokenURL = authEndpoint[region] + tokenPath + oauthConfig.AuthURL = authEndpoint[region] + authPath + return oauthutil.ConfigOut("choose_type", &oauthutil.Options{ OAuth2Config: oauthConfig, }) @@ -994,10 +992,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.DisableSitePermission { oauthConfig.Scopes = scopeAccessWithoutSites } - oauthConfig.Endpoint = oauth2.Endpoint{ - AuthURL: authEndpoint[opt.Region] + authPath, - TokenURL: authEndpoint[opt.Region] + tokenPath, - } + oauthConfig.AuthURL = authEndpoint[opt.Region] + authPath + oauthConfig.TokenURL = authEndpoint[opt.Region] + tokenPath client := fshttp.NewClient(ctx) root = parsePath(root) diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 6d1524390..f198c5563 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -48,12 +48,10 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Scopes: nil, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://my.pcloud.com/oauth2/authorize", - // TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL - }, + oauthConfig = &oauthutil.Config{ + Scopes: nil, + AuthURL: "https://my.pcloud.com/oauth2/authorize", + // TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, @@ -61,8 +59,8 @@ var ( ) // Update the TokenURL with the actual hostname -func updateTokenURL(oauthConfig *oauth2.Config, hostname string) { - oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token" +func updateTokenURL(oauthConfig *oauthutil.Config, hostname string) { + oauthConfig.TokenURL = "https://" + hostname + "/oauth2_token" } // Register with Fs @@ -79,7 +77,7 @@ func init() { fs.Errorf(nil, "Failed to read config: %v", err) } updateTokenURL(oauthConfig, optc.Hostname) - checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error { + checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error { if auth == nil || auth.Form == nil { return errors.New("form not found in response") } diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go index f2a1390f1..c1376949a 100644 --- a/backend/pikpak/pikpak.go +++ b/backend/pikpak/pikpak.go @@ -82,13 +82,11 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Scopes: nil, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.com/v1/auth/signin", - TokenURL: "https://user.mypikpak.com/v1/auth/token", - AuthStyle: oauth2.AuthStyleInParams, - }, + oauthConfig = &oauthutil.Config{ + Scopes: nil, + AuthURL: "https://user.mypikpak.com/v1/auth/signin", + TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthStyle: oauth2.AuthStyleInParams, ClientID: clientID, RedirectURL: oauthutil.RedirectURL, } diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index ef54aac9f..5bf87badf 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -43,7 +43,6 @@ import ( "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" ) const ( @@ -59,12 +58,10 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Scopes: nil, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://www.premiumize.me/authorize", - TokenURL: "https://www.premiumize.me/token", - }, + oauthConfig = &oauthutil.Config{ + Scopes: nil, + AuthURL: "https://www.premiumize.me/authorize", + TokenURL: "https://www.premiumize.me/token", ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectURL, diff --git a/backend/putio/putio.go b/backend/putio/putio.go index a12be5f9b..e003b89dc 100644 --- a/backend/putio/putio.go +++ b/backend/putio/putio.go @@ -13,7 +13,6 @@ import ( "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" - "golang.org/x/oauth2" ) /* @@ -41,12 +40,10 @@ const ( var ( // Description of how to auth for this app - putioConfig = &oauth2.Config{ - Scopes: []string{}, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://api.put.io/v2/oauth2/authenticate", - TokenURL: "https://api.put.io/v2/oauth2/access_token", - }, + putioConfig = &oauthutil.Config{ + Scopes: []string{}, + AuthURL: "https://api.put.io/v2/oauth2/authenticate", + TokenURL: "https://api.put.io/v2/oauth2/access_token", ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneObscuredClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index a135f8705..d35468e0c 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -97,7 +97,6 @@ import ( "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" ) const ( @@ -115,13 +114,11 @@ const ( ) // Generate a new oauth2 config which we will update when we know the TokenURL -func newOauthConfig(tokenURL string) *oauth2.Config { - return &oauth2.Config{ - Scopes: nil, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://secure.sharefile.com/oauth/authorize", - TokenURL: tokenURL, - }, +func newOauthConfig(tokenURL string) *oauthutil.Config { + return &oauthutil.Config{ + Scopes: nil, + AuthURL: "https://secure.sharefile.com/oauth/authorize", + TokenURL: tokenURL, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectPublicSecureURL, @@ -136,7 +133,7 @@ func init() { NewFs: NewFs, Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { oauthConfig := newOauthConfig("") - checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error { + checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error { if auth == nil || auth.Form == nil { return errors.New("endpoint not found in response") } @@ -147,7 +144,7 @@ func init() { } endpoint := "https://" + subdomain + "." + apicp m.Set("endpoint", endpoint) - oauthConfig.Endpoint.TokenURL = endpoint + tokenPath + oauthConfig.TokenURL = endpoint + tokenPath return nil } return oauthutil.ConfigOut("", &oauthutil.Options{ diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 0d5f18e18..9d201c268 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -29,7 +29,6 @@ import ( "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" - "golang.org/x/oauth2" ) // oAuth @@ -47,11 +46,9 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Endpoint: oauth2.Endpoint{ - AuthURL: "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize - TokenURL: "https://oauth.yandex.com/token", //same as https://oauth.yandex.ru/token - }, + oauthConfig = &oauthutil.Config{ + AuthURL: "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize + TokenURL: "https://oauth.yandex.com/token", //same as https://oauth.yandex.ru/token ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectURL, diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 9561f56c0..30adb413e 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -47,7 +47,7 @@ const ( // Globals var ( // Description of how to auth for this app - oauthConfig = &oauth2.Config{ + oauthConfig = &oauthutil.Config{ Scopes: []string{ "aaaserver.profile.read", "WorkDrive.team.READ", @@ -55,11 +55,10 @@ var ( "WorkDrive.files.ALL", "ZohoFiles.files.ALL", }, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://accounts.zoho.eu/oauth/v2/auth", - TokenURL: "https://accounts.zoho.eu/oauth/v2/token", - AuthStyle: oauth2.AuthStyleInParams, - }, + + AuthURL: "https://accounts.zoho.eu/oauth/v2/auth", + TokenURL: "https://accounts.zoho.eu/oauth/v2/token", + AuthStyle: oauth2.AuthStyleInParams, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, @@ -276,8 +275,8 @@ func setupRegion(m configmap.Mapper) error { downloadURL = fmt.Sprintf("https://download.zoho.%s/v1/workdrive", region) uploadURL = fmt.Sprintf("https://upload.zoho.%s/workdrive-api/v1", region) accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region) - oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region) - oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region) + oauthConfig.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region) + oauthConfig.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region) return nil } diff --git a/fs/config/config.go b/fs/config/config.go index 0e078e131..ca8d370da 100644 --- a/fs/config/config.go +++ b/fs/config/config.go @@ -46,6 +46,9 @@ const ( // ConfigTokenURL is the config key used to store the token server endpoint ConfigTokenURL = "token_url" + // ConfigClientCredentials - use OAUTH2 client credentials + ConfigClientCredentials = "client_credentials" + // ConfigEncoding is the config key to change the encoding for a backend ConfigEncoding = "encoding" diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 50b2d2079..323737bcf 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( "github.com/rclone/rclone/lib/random" "github.com/skratchdot/open-golang/open" "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" ) var ( @@ -85,6 +87,49 @@ All done. Please go back to rclone. // should work for most uses, but may be overridden. var OpenURL = open.Start +// Config - structure that we will use to store the OAuth configuration +// settings. This is based on the union of the configuration structures for the two +// OAuth modules that we are using (oauth2 and oauth2.clientcrentials), along with a +// flag indicating if we are going to use the client credential flow +type Config struct { + ClientID string + ClientSecret string + TokenURL string + AuthURL string + Scopes []string + EndpointParams url.Values + RedirectURL string + ClientCredentialFlow bool + AuthStyle oauth2.AuthStyle +} + +// MakeOauth2Config makes an oauth2.Config from our config +func (conf *Config) MakeOauth2Config() *oauth2.Config { + return &oauth2.Config{ + ClientID: conf.ClientID, + ClientSecret: conf.ClientSecret, + RedirectURL: RedirectLocalhostURL, + Scopes: conf.Scopes, + Endpoint: oauth2.Endpoint{ + AuthURL: conf.AuthURL, + TokenURL: conf.TokenURL, + AuthStyle: conf.AuthStyle, + }, + } +} + +// MakeClientCredentialsConfig makes a clientcredentials.Config from our config +func (conf *Config) MakeClientCredentialsConfig() *clientcredentials.Config { + return &clientcredentials.Config{ + ClientID: conf.ClientID, + ClientSecret: conf.ClientSecret, + Scopes: conf.Scopes, + TokenURL: conf.TokenURL, + AuthStyle: conf.AuthStyle, + // EndpointParams url.Values + } +} + // SharedOptions are shared between backends the utilize an OAuth flow var SharedOptions = []fs.Option{{ Name: config.ConfigClientID, @@ -107,6 +152,11 @@ var SharedOptions = []fs.Option{{ Name: config.ConfigTokenURL, Help: "Token server url.\n\nLeave blank to use the provider defaults.", Advanced: true, +}, { + Name: config.ConfigClientCredentials, + Default: false, + Help: "Use client credentials OAuth flow.\n\nThis will use the OAUTH2 client Credentials Flow as described in RFC 6749.", + Advanced: true, }} // oldToken contains an end-user's tokens. @@ -178,7 +228,7 @@ type TokenSource struct { m configmap.Mapper tokenSource oauth2.TokenSource token *oauth2.Token - config *oauth2.Config + config *Config ctx context.Context expiryTimer *time.Timer // signals whenever the token expires } @@ -264,6 +314,11 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) { ) const maxTries = 5 + // If we have a cached valid token, use that + if ts.token.Valid() { + return ts.token, nil + } + // Try getting the token a few times for i := 1; i <= maxTries; i++ { // Try reading the token from the config file in case it has @@ -271,7 +326,7 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) { if !ts.token.Valid() { if ts.reReadToken() { changed = true - } else if ts.token.RefreshToken == "" { + } else if !ts.config.ClientCredentialFlow && ts.token.RefreshToken == "" { return nil, fserrors.FatalError( fmt.Errorf("token expired and there's no refresh token - manually refresh with \"rclone config reconnect %s:\"", ts.name), ) @@ -280,7 +335,11 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) { // Make a new token source if required if ts.tokenSource == nil { - ts.tokenSource = ts.config.TokenSource(ts.ctx, ts.token) + if ts.config.ClientCredentialFlow { + ts.tokenSource = ts.config.MakeClientCredentialsConfig().TokenSource(ts.ctx) + } else { + ts.tokenSource = ts.config.MakeOauth2Config().TokenSource(ts.ctx, ts.token) + } } token, err = ts.tokenSource.Token() @@ -297,7 +356,7 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) { if err != nil { return nil, fmt.Errorf("couldn't fetch token: %w", err) } - changed = changed || token.AccessToken != ts.token.AccessToken || token.RefreshToken != ts.token.RefreshToken || token.Expiry != ts.token.Expiry + changed = changed || ts.token == nil || token.AccessToken != ts.token.AccessToken || token.RefreshToken != ts.token.RefreshToken || token.Expiry != ts.token.Expiry ts.token = token if changed { // Bump on the expiry timer if it is set @@ -370,12 +429,12 @@ func Context(ctx context.Context, client *http.Client) context.Context { return context.WithValue(ctx, oauth2.HTTPClient, client) } -// overrideCredentials sets the ClientID and ClientSecret from the +// OverrideCredentials sets the ClientID and ClientSecret from the // config file if they are not blank. // If any value is overridden, true is returned. // the origConfig is copied -func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) { - newConfig = new(oauth2.Config) +func OverrideCredentials(name string, m configmap.Mapper, origConfig *Config) (newConfig *Config, changed bool) { + newConfig = new(Config) *newConfig = *origConfig changed = false ClientID, ok := m.Get(config.ConfigClientID) @@ -393,12 +452,22 @@ func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Con } AuthURL, ok := m.Get(config.ConfigAuthURL) if ok && AuthURL != "" { - newConfig.Endpoint.AuthURL = AuthURL + newConfig.AuthURL = AuthURL changed = true } TokenURL, ok := m.Get(config.ConfigTokenURL) if ok && TokenURL != "" { - newConfig.Endpoint.TokenURL = TokenURL + newConfig.TokenURL = TokenURL + changed = true + } + ClientCredentialStr, ok := m.Get(config.ConfigClientCredentials) + if ok && ClientCredentialStr != "" { + ClientCredential, err := strconv.ParseBool(ClientCredentialStr) + if err != nil { + fs.Errorf(nil, "Invalid setting for %q: %v", config.ConfigClientCredentials, err) + } else { + newConfig.ClientCredentialFlow = ClientCredential + } changed = true } return newConfig, changed @@ -408,8 +477,8 @@ func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Con // configures a Client with it. It returns the client and a // TokenSource which Invalidate may need to be called on. It uses the // httpClient passed in as the base client. -func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mapper, config *oauth2.Config, baseClient *http.Client) (*http.Client, *TokenSource, error) { - config, _ = overrideCredentials(name, m, config) +func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mapper, config *Config, baseClient *http.Client) (*http.Client, *TokenSource, error) { + config, _ = OverrideCredentials(name, m, config) token, err := GetToken(name, m) if err != nil { return nil, nil, err @@ -428,12 +497,39 @@ func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mappe ctx: ctx, } return oauth2.NewClient(ctx, ts), ts, nil +} +// NewClientCredentialsClient creates a new OAuth module using the +// ClientCredential flow +func NewClientCredentialsClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, baseClient *http.Client) (*http.Client, *TokenSource, error) { + oauthConfig, _ = OverrideCredentials(name, m, oauthConfig) + token, _ := GetToken(name, m) + // If the token doesn't exist then we will fetch one in the next step as we don't need a refresh token + + // Set our own http client in the context + ctx = Context(ctx, baseClient) + + // Wrap the TokenSource in our TokenSource which saves changed + // tokens in the config file + ts := &TokenSource{ + name: name, + m: m, + token: token, + config: oauthConfig, + ctx: ctx, + } + return oauth2.NewClient(ctx, ts), ts, nil } // NewClient gets a token from the config file and configures a Client -// with it. It returns the client and a TokenSource which Invalidate may need to be called on -func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) { +// with it. It returns the client and a TokenSource which Invalidate +// may need to be called on +func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config) (*http.Client, *TokenSource, error) { + // Check whether we are using the client credentials flow + if oauthConfig.ClientCredentialFlow { + + return NewClientCredentialsClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx)) + } return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx)) } @@ -460,11 +556,11 @@ func (ar *AuthResult) Error() string { } // CheckAuthFn is called when a good Auth has been received -type CheckAuthFn func(*oauth2.Config, *AuthResult) error +type CheckAuthFn func(*Config, *AuthResult) error // Options for the oauth config type Options struct { - OAuth2Config *oauth2.Config // Basic config for oauth2 + OAuth2Config *Config // Basic config for oauth2 NoOffline bool // If set then "access_type=offline" parameter is not passed CheckAuth CheckAuthFn // When the AuthResult is known the checkAuth function is called if set OAuth2Opts []oauth2.AuthCodeOption // extra oauth2 options @@ -532,6 +628,15 @@ func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.Re if in.Result == "false" { return fs.ConfigGoto(newState("*oauth-done")) } + opt, err := getOAuth() + if err != nil { + return nil, err + } + oauthConfig, _ := OverrideCredentials(name, m, opt.OAuth2Config) + if oauthConfig.ClientCredentialFlow { + // If using client credential flow, skip straight to getting the token since we don't need a browser + return fs.ConfigGoto(newState("*oauth-do")) + } return fs.ConfigConfirm(newState("*oauth-islocal"), true, "config_is_local", "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n") case "*oauth-islocal": if in.Result == "true" { @@ -626,20 +731,27 @@ version recommended): if err != nil { return nil, err } - oauthConfig, changed := overrideCredentials(name, m, opt.OAuth2Config) + oauthConfig, changed := OverrideCredentials(name, m, opt.OAuth2Config) if changed { fs.Logf(nil, "Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL) } - if code == "" { - oauthConfig = fixRedirect(oauthConfig) - code, err = configSetup(ctx, ri.Name, name, m, oauthConfig, opt) + if oauthConfig.ClientCredentialFlow { + err = clientCredentialsFlowGetToken(ctx, name, m, oauthConfig, opt) if err != nil { - return nil, fmt.Errorf("config failed to refresh token: %w", err) + return nil, err + } + } else { + if code == "" { + oauthConfig = fixRedirect(oauthConfig) + code, err = configSetup(ctx, ri.Name, name, m, oauthConfig, opt) + if err != nil { + return nil, fmt.Errorf("config failed to refresh token: %w", err) + } + } + err = configExchange(ctx, name, m, oauthConfig, code) + if err != nil { + return nil, err } - } - err = configExchange(ctx, name, m, oauthConfig, code) - if err != nil { - return nil, err } return fs.ConfigGoto(newState("*oauth-done")) case "*oauth-done": @@ -656,13 +768,13 @@ func init() { } // Return true if can run without a webserver and just entering a code -func noWebserverNeeded(oauthConfig *oauth2.Config) bool { +func noWebserverNeeded(oauthConfig *Config) bool { return oauthConfig.RedirectURL == TitleBarRedirectURL } // get the URL we need to send the user to -func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (authURL string, state string, err error) { - oauthConfig, _ = overrideCredentials(name, m, oauthConfig) +func getAuthURL(name string, m configmap.Mapper, oauthConfig *Config, opt *Options) (authURL string, state string, err error) { + oauthConfig, _ = OverrideCredentials(name, m, oauthConfig) // Make random state state, err = random.Password(128) @@ -670,18 +782,21 @@ func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt return "", "", err } + // Create the configuration required for the OAuth flow + oauth2Conf := oauthConfig.MakeOauth2Config() + // Generate oauth URL opts := opt.OAuth2Opts if !opt.NoOffline { opts = append(opts, oauth2.AccessTypeOffline) } - authURL = oauthConfig.AuthCodeURL(state, opts...) + authURL = oauth2Conf.AuthCodeURL(state, opts...) return authURL, state, nil } // If TitleBarRedirect is set but we are doing a real oauth, then // override our redirect URL -func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config { +func fixRedirect(oauthConfig *Config) *Config { switch oauthConfig.RedirectURL { case TitleBarRedirectURL: // copy the config and set to use the internal webserver @@ -692,12 +807,33 @@ func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config { return oauthConfig } +// configSetup does the initial creation of the token for the client credentials flow +// +// If opt is nil it will use the default Options. +func clientCredentialsFlowGetToken(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, opt *Options) error { + if opt == nil { + opt = &Options{} + } + _ = opt // not currently using the Options + fs.Debugf(nil, "Getting token for client credentials flow") + _, tokenSource, err := NewClientCredentialsClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx)) + if err != nil { + return fmt.Errorf("client credentials flow: failed to make client: %w", err) + } + // Get the token and save it in the config file + _, err = tokenSource.Token() + if err != nil { + return fmt.Errorf("client credentials flow: failed to get token: %w", err) + } + return nil +} + // configSetup does the initial creation of the token // // If opt is nil it will use the default Options. // // It will run an internal webserver to receive the results -func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (string, error) { +func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *Config, opt *Options) (string, error) { if opt == nil { opt = &Options{} } @@ -749,9 +885,13 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth } // Exchange the code for a token -func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config, code string) error { +func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, code string) error { ctx = Context(ctx, fshttp.NewClient(ctx)) - token, err := oauthConfig.Exchange(ctx, code) + + // Create the configuration required for the OAuth flow + oauth2Conf := oauthConfig.MakeOauth2Config() + + token, err := oauth2Conf.Exchange(ctx, code) if err != nil { return fmt.Errorf("failed to get token: %w", err) }