From 7f776c64f011ddc9b5f7115ae2c0b21b29e59a1f Mon Sep 17 00:00:00 2001 From: buengese Date: Fri, 24 Apr 2020 03:15:52 +0200 Subject: [PATCH] fichier: implement custom pacer to deal with the new rate limiting --- backend/fichier/api.go | 1 + backend/fichier/fichier.go | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/fichier/api.go b/backend/fichier/api.go index 4454e0421..d58f58626 100644 --- a/backend/fichier/api.go +++ b/backend/fichier/api.go @@ -17,6 +17,7 @@ import ( // retryErrorCodes is a slice of error codes that we will retry var retryErrorCodes = []int{ 429, // Too Many Requests. + 403, // Forbidden (may happen when request limit is exceeded) 500, // Internal Server Error 502, // Bad Gateway 503, // Service Unavailable diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index 3f0da19a6..0349a2198 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -23,11 +23,12 @@ import ( ) const ( - rootID = "0" - apiBaseURL = "https://api.1fichier.com/v1" - minSleep = 334 * time.Millisecond // 3 API calls per second is recommended - maxSleep = 5 * time.Second - decayConstant = 2 // bigger for slower decay, exponential + rootID = "0" + apiBaseURL = "https://api.1fichier.com/v1" + minSleep = 400 * time.Millisecond // api is extremely rate limited now + maxSleep = 5 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + attackConstant = 0 // start with max sleep ) func init() { @@ -185,7 +186,7 @@ func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) { name: name, root: root, opt: *opt, - pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))), baseClient: &http.Client{}, }