diff --git a/backend/putio/error.go b/backend/putio/error.go index 34eecabb9..27476b838 100644 --- a/backend/putio/error.go +++ b/backend/putio/error.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "net/http" + "strconv" + "time" "github.com/putdotio/go-putio/putio" "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/lib/pacer" ) func checkStatusCode(resp *http.Response, expected int) error { @@ -24,8 +27,10 @@ func (e *statusCodeError) Error() string { return fmt.Sprintf("unexpected status code (%d) response while doing %s to %s", e.response.StatusCode, e.response.Request.Method, e.response.Request.URL.String()) } +// This method is called from fserrors.ShouldRetry() to determine if an error should be retried. +// Some errors (e.g. 429 Too Many Requests) are handled before this step, so they are not included here. func (e *statusCodeError) Temporary() bool { - return e.response.StatusCode == 429 || e.response.StatusCode >= 500 + return e.response.StatusCode >= 500 } // shouldRetry returns a boolean as to whether this err deserves to be @@ -40,6 +45,16 @@ func shouldRetry(ctx context.Context, err error) (bool, error) { if perr, ok := err.(*putio.ErrorResponse); ok { err = &statusCodeError{response: perr.Response} } + if scerr, ok := err.(*statusCodeError); ok && scerr.response.StatusCode == 429 { + delay := defaultRateLimitSleep + header := scerr.response.Header.Get("x-ratelimit-reset") + if header != "" { + if resetTime, cerr := strconv.ParseInt(header, 10, 64); cerr == nil { + delay = time.Until(time.Unix(resetTime+1, 0)) + } + } + return true, pacer.RetryAfterError(scerr, delay) + } if fserrors.ShouldRetry(err) { return true, err } diff --git a/backend/putio/fs.go b/backend/putio/fs.go index 073f86a44..29cc2f840 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -302,8 +302,8 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID if err != nil { return false, err } - if resp.StatusCode != 201 { - return false, fmt.Errorf("unexpected status code from upload create: %d", resp.StatusCode) + if err := checkStatusCode(resp, 201); err != nil { + return shouldRetry(ctx, err) } location = resp.Header.Get("location") if location == "" { diff --git a/backend/putio/object.go b/backend/putio/object.go index 0383a355e..4b5f72015 100644 --- a/backend/putio/object.go +++ b/backend/putio/object.go @@ -241,7 +241,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } // fs.Debugf(o, "opening file: id=%d", o.file.ID) resp, err = o.fs.httpClient.Do(req) - return shouldRetry(ctx, err) + if err != nil { + return shouldRetry(ctx, err) + } + if err := checkStatusCode(resp, 200); err != nil { + return shouldRetry(ctx, err) + } + return false, nil }) if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 { _ = resp.Body.Close() diff --git a/backend/putio/putio.go b/backend/putio/putio.go index 0019bc089..766a99b77 100644 --- a/backend/putio/putio.go +++ b/backend/putio/putio.go @@ -33,8 +33,9 @@ const ( rcloneObscuredClientSecret = "cMwrjWVmrHZp3gf1ZpCrlyGAmPpB-YY5BbVnO1fj-G9evcd8" minSleep = 10 * time.Millisecond maxSleep = 2 * time.Second - decayConstant = 2 // bigger for slower decay, exponential + decayConstant = 1 // bigger for slower decay, exponential defaultChunkSize = 48 * fs.Mebi + defaultRateLimitSleep = 60 * time.Second ) var ( diff --git a/docs/content/putio.md b/docs/content/putio.md index 069b21fd3..f80a1f3bd 100644 --- a/docs/content/putio.md +++ b/docs/content/putio.md @@ -127,3 +127,12 @@ Properties: - Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot {{< rem autogenerated options stop >}} + +## Limitations + +put.io has rate limiting. When you hit a limit, rclone automatically +retries after waiting the amount of time requested by the server. + +If you want to avoid ever hitting these limits, you may use the +`--tpslimit` flag with a low number. Note that the imposed limits +may be different for different operations, and may change over time.