mirror of
https://github.com/rclone/rclone.git
synced 2025-03-11 05:25:14 +08:00
401cf81034
This commit modernizes Go usage. This was done with:
go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -fix -test ./...
Then files needed to be `go fmt`ed and a few comments needed to be
restored.
The modernizations include replacing
- if/else conditional assignment by a call to the built-in min or max functions added in go1.21
- sort.Slice(x, func(i, j int) bool) { return s[i] < s[j] } by a call to slices.Sort(s), added in go1.21
- interface{} by the 'any' type added in go1.18
- append([]T(nil), s...) by slices.Clone(s) or slices.Concat(s), added in go1.21
- loop around an m[k]=v map update by a call to one of the Collect, Copy, Clone, or Insert functions from the maps package, added in go1.21
- []byte(fmt.Sprintf...) by fmt.Appendf(nil, ...), added in go1.19
- append(s[:i], s[i+1]...) by slices.Delete(s, i, i+1), added in go1.21
- a 3-clause for i := 0; i < n; i++ {} loop by for i := range n {}, added in go1.22
105 lines
2.3 KiB
Go
105 lines
2.3 KiB
Go
package quatrix
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
)
|
|
|
|
// UploadMemoryManager dynamically calculates every chunk size for the transfer and increases or decreases it
|
|
// depending on the upload speed. This makes general upload time smaller, because transfers that are faster
|
|
// does not have to wait for the slower ones until they finish upload.
|
|
type UploadMemoryManager struct {
|
|
m sync.Mutex
|
|
useDynamicSize bool
|
|
shared int64
|
|
reserved int64
|
|
effectiveTime time.Duration
|
|
fileUsage map[string]int64
|
|
}
|
|
|
|
// NewUploadMemoryManager is a constructor for UploadMemoryManager
|
|
func NewUploadMemoryManager(ci *fs.ConfigInfo, opt *Options) *UploadMemoryManager {
|
|
useDynamicSize := true
|
|
|
|
sharedMemory := int64(opt.MaximalSummaryChunkSize) - int64(opt.MinimalChunkSize)*int64(ci.Transfers)
|
|
if sharedMemory <= 0 {
|
|
sharedMemory = 0
|
|
useDynamicSize = false
|
|
}
|
|
|
|
return &UploadMemoryManager{
|
|
useDynamicSize: useDynamicSize,
|
|
shared: sharedMemory,
|
|
reserved: int64(opt.MinimalChunkSize),
|
|
effectiveTime: time.Duration(opt.EffectiveUploadTime),
|
|
fileUsage: map[string]int64{},
|
|
}
|
|
}
|
|
|
|
// Consume -- decide amount of memory to consume
|
|
func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed float64) int64 {
|
|
if !u.useDynamicSize {
|
|
if neededMemory < u.reserved {
|
|
return neededMemory
|
|
}
|
|
|
|
return u.reserved
|
|
}
|
|
|
|
u.m.Lock()
|
|
defer u.m.Unlock()
|
|
|
|
borrowed, found := u.fileUsage[fileID]
|
|
if found {
|
|
u.shared += borrowed
|
|
borrowed = 0
|
|
}
|
|
|
|
defer func() { u.fileUsage[fileID] = borrowed }()
|
|
|
|
effectiveChunkSize := max(int64(speed*u.effectiveTime.Seconds()), u.reserved)
|
|
|
|
if neededMemory < effectiveChunkSize {
|
|
effectiveChunkSize = neededMemory
|
|
}
|
|
|
|
if effectiveChunkSize <= u.reserved {
|
|
return effectiveChunkSize
|
|
}
|
|
|
|
toBorrow := effectiveChunkSize - u.reserved
|
|
|
|
if toBorrow <= u.shared {
|
|
u.shared -= toBorrow
|
|
borrowed = toBorrow
|
|
|
|
return effectiveChunkSize
|
|
}
|
|
|
|
borrowed = u.shared
|
|
u.shared = 0
|
|
|
|
return borrowed + u.reserved
|
|
}
|
|
|
|
// Return returns consumed memory for the previous chunk upload to the memory pool
|
|
func (u *UploadMemoryManager) Return(fileID string) {
|
|
if !u.useDynamicSize {
|
|
return
|
|
}
|
|
|
|
u.m.Lock()
|
|
defer u.m.Unlock()
|
|
|
|
borrowed, found := u.fileUsage[fileID]
|
|
if !found {
|
|
return
|
|
}
|
|
|
|
u.shared += borrowed
|
|
|
|
delete(u.fileUsage, fileID)
|
|
}
|