github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 22:22:21 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

dropbox: add --dropbox-batch-commit-timeout to control batch timeout

Browse Source 
This also adds an Debug message log showing how long each batch took

See: #5491
This commit is contained in:
Nick Craig-Wood 2021-08-14 13:22:54 +01:00
parent f4c5f1f185
commit 9ce0df3242
2 changed files with 30 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
backend/dropbox/batcher.go
View File

@ -147,8 +147,14 @@ func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *f
} }
var batchStatus *files.UploadSessionFinishBatchJobStatus var batchStatus *files.UploadSessionFinishBatchJobStatus
sleepTime := 100 * time.Millisecond sleepTime := 100 * time.Millisecond
const maxTries = 120 const maxSleepTime = 1 * time.Second
for try := 1; try <= maxTries; try++ { startTime := time.Now()
try := 1
for {
remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
if remaining < 0 {
break
}
err = b.f.pacer.Call(func() (bool, error) { err = b.f.pacer.Call(func() (bool, error) {
batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{ batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
AsyncJobId: launchBatchStatus.AsyncJobId, AsyncJobId: launchBatchStatus.AsyncJobId,
@ -156,23 +162,25 @@ func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *f
return shouldRetry(ctx, err) return shouldRetry(ctx, err)
}) })
if err != nil { if err != nil {
fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d/%d", sleepTime, err, try, maxTries) fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
} else { } else {
if batchStatus.Tag == "complete" { if batchStatus.Tag == "complete" {
fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
return batchStatus.Complete, nil return batchStatus.Complete, nil
} }
fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d/%d", sleepTime, batchStatus.Tag, try, maxTries) fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
} }
time.Sleep(sleepTime) time.Sleep(sleepTime)
sleepTime *= 2 sleepTime *= 2
if sleepTime > time.Second { if sleepTime > maxSleepTime {
sleepTime = time.Second sleepTime = maxSleepTime
} }
try++
} }
if err == nil { if err == nil {
err = errors.New("batch didn't complete") err = errors.New("batch didn't complete")
} }
return nil, errors.Wrapf(err, "wait for batch failed after %d tries", maxTries) return nil, errors.Wrapf(err, "wait for batch failed after %d tries in %v", try, time.Since(startTime))
} }
// commit a batch // commit a batch

24
backend/dropbox/dropbox.go
View File

@ -266,6 +266,11 @@ default based on the batch_mode in use.
`, `,
Default: fs.Duration(0), Default: fs.Duration(0),
Advanced: true, Advanced: true,
}, {
Name: "batch_commit_timeout",
Help: `Max time to wait for a batch to finish comitting`,
Default: fs.Duration(10 * time.Minute),
Advanced: true,
}, { }, {
Name: config.ConfigEncoding, Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp, Help: config.ConfigEncodingHelp,
@ -285,15 +290,16 @@ default based on the batch_mode in use.
// Options defines the configuration for this backend // Options defines the configuration for this backend
type Options struct { type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"` ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"` Impersonate string `config:"impersonate"`
SharedFiles bool `config:"shared_files"` SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"` SharedFolders bool `config:"shared_folders"`
BatchMode string `config:"batch_mode"` BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"` BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"` BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"` BatchCommitTimeout fs.Duration `config:"batch_commit_timeout"`
Enc encoder.MultiEncoder `config:"encoding"` AsyncBatch bool `config:"async_batch"`
Enc encoder.MultiEncoder `config:"encoding"`
} }
// Fs represents a remote dropbox server // Fs represents a remote dropbox server