github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-23 20:59:01 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

acd: Simplify the wait options into a single --acd-upload-wait-per-gb - fixes #262

Browse Source 
This means the feature can be disabled by setting the time to 0.

This also logs the HTTP status for analysis purposes.

Thanks Felix Bünemann for extensive testing and data collection.
This commit is contained in:
Nick Craig-Wood 2016-11-05 13:57:03 +00:00
parent d1080d5456
commit e162377ca3
2 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
amazonclouddrive/amazonclouddrive.go
View File

@ -49,9 +49,7 @@ const (
var ( var (
// Flags // Flags
tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
uploadWaitLimit = pflag.DurationP("acd-upload-wait-limit", "", 60*time.Second, "Don't wait for completed uploads to appear if they took less than this time.") uploadWaitPerGB = pflag.DurationP("acd-upload-wait-per-gb", "", 180*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
uploadWaitTime = pflag.DurationP("acd-upload-wait-time", "", 2*60*time.Second, "Time to wait after a failed complete upload to see if it appears.")
uploadWaitPerGB = pflag.DurationP("acd-upload-wait-per-gb", "", 30*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
// Description of how to auth for this app // Description of how to auth for this app
acdConfig = &oauth2.Config{ acdConfig = &oauth2.Config{
Scopes: []string{"clouddrive:read_all", "clouddrive:write"}, Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
@ -491,28 +489,35 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
// if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 { // if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 {
// return false, inInfo, inErr // return false, inInfo, inErr
// } // }
// The HTTP status
httpStatus := "HTTP status UNKNOWN"
if resp != nil {
httpStatus = resp.Status
}
// check to see if we read to the end // check to see if we read to the end
buf := make([]byte, 1) buf := make([]byte, 1)
n, err := in.Read(buf) n, err := in.Read(buf)
if !(n == 0 && err == io.EOF) { if !(n == 0 && err == io.EOF) {
fs.Debug(src, "Upload error detected but didn't finish upload: %v", inErr) fs.Debug(src, "Upload error detected but didn't finish upload: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr return false, inInfo, inErr
} }
// Only wait for items which have been in transit for > uploadWaitLimit // Don't wait for uploads - assume they will appear later
if uploadTime < *uploadWaitLimit { if *uploadWaitPerGB <= 0 {
fs.Debug(src, "Upload error detected but not waiting since it only took %v to upload: %v", uploadTime, inErr) fs.Debug(src, "Upload error detected but waiting disabled: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr return false, inInfo, inErr
} }
// Time we should wait for the upload // Time we should wait for the upload
uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024 uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024
timeToWait := time.Duration(uploadWaitPerByte*float64(src.Size())) + *uploadWaitTime timeToWait := time.Duration(uploadWaitPerByte * float64(src.Size()))
const sleepTime = 5 * time.Second // sleep between tries const sleepTime = 5 * time.Second // sleep between tries
retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up
fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v", inErr) fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v (%q)", inErr, httpStatus)
remote := src.Remote() remote := src.Remote()
for i := 1; i <= retries; i++ { for i := 1; i <= retries; i++ {
o, err := f.NewObject(remote) o, err := f.NewObject(remote)
@ -532,7 +537,7 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
} }
time.Sleep(sleepTime) time.Sleep(sleepTime)
} }
fs.Debug(src, "Giving up waiting for object - returning original error: %v", inErr) fs.Debug(src, "Giving up waiting for object - returning original error: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr return false, inInfo, inErr
} }

15
docs/content/amazonclouddrive.md
View File

@ -133,20 +133,21 @@ To download files above this threshold, rclone requests a `tempLink`
which downloads the file through a temporary URL directly from the which downloads the file through a temporary URL directly from the
underlying S3 storage. underlying S3 storage.
#### --acd-upload-wait-time=TIME, --acd-upload-wait-per-gb=TIME, --acd-upload-wait-limit=TIME #### #### --acd-upload-wait-per-gb=TIME ####
Sometimes Amazon Drive gives an error when a file has been fully Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This uploaded but the file appears anyway after a little while. This
happens sometimes for files over 1GB in size and nearly every time for happens sometimes for files over 1GB in size and nearly every time for
files bigger than 10GB. These parameters control the time rclone waits files bigger than 10GB. This parameter controls the time rclone waits
for the file to appear. for the file to appear.
If the upload took less than `--acd-upload-wait-limit` (default 60s), The default value for this parameter is 3 minutes per GB, so by
then we go ahead an upload it again as that will be quicker. default it will wait 3 minutes for every GB uploaded to see if the
file appears.
We wait `--acd-upload-wait-time` (default 2m) for the file to appear, You can disable this feature by setting it to 0. This may cause
with an additional `--acd-upload-wait-per-gb` (default 30s) per GB of conflict errors as rclone retries the failed upload but the file will
the uploaded file. most likely appear correctly eventually.
These values were determined empirically by observing lots of uploads These values were determined empirically by observing lots of uploads
of big files for a range of file sizes. of big files for a range of file sizes.