vfs: fix download loop when file size shrunk

Before this change, if a file shrunk in size on the remote then rclone
could get into an loop trying to download the file forever.

The symptom was repeating errors like this:

    vfs cache: restart download failed: failed to start downloader: failed to open downloader: vfs reader: failed to open source file: invalid seek position

The fix was to check that file size in various places and makes sure
that we weren't trying to download too much data.

This was a problems with backends (like s3) which update the size of
the object on Open to the actual size of the object.
This commit is contained in:
Nick Craig-Wood 2024-03-04 09:47:48 +00:00
parent ac6ba11d22
commit f3f743c3f9
2 changed files with 17 additions and 1 deletions

View File

@ -359,6 +359,10 @@ func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) {
if !startNew { if !startNew {
return nil return nil
} }
// Size can be 0 here if file shrinks - no need to download
if r.Size == 0 {
return nil
}
// Downloader not found so start a new one // Downloader not found so start a new one
_, err = dls._newDownloader(r) _, err = dls._newDownloader(r)
if err != nil { if err != nil {
@ -389,7 +393,10 @@ func (dls *Downloaders) _dispatchWaiters() {
newWaiters := dls.waiters[:0] newWaiters := dls.waiters[:0]
for _, waiter := range dls.waiters { for _, waiter := range dls.waiters {
if dls.item.HasRange(waiter.r) { // Clip the size against the actual size in case it has shrunk
r := waiter.r
r.Clip(dls.src.Size())
if dls.item.HasRange(r) {
waiter.errChan <- nil waiter.errChan <- nil
} else { } else {
newWaiters = append(newWaiters, waiter) newWaiters = append(newWaiters, waiter)

View File

@ -1279,6 +1279,15 @@ func (item *Item) readAt(b []byte, off int64) (n int, err error) {
return 0, err return 0, err
} }
// Check to see if object has shrunk - if so don't read too much.
if item.o != nil && !item.info.Dirty && item.o.Size() != item.info.Size {
fs.Debugf(item.o, "Size has changed from %d to %d", item.info.Size, item.o.Size())
err = item._truncate(item.o.Size())
if err != nil {
return 0, err
}
}
item.info.ATime = time.Now() item.info.ATime = time.Now()
// Do the reading with Item.mu unlocked and cache protected by preAccess // Do the reading with Item.mu unlocked and cache protected by preAccess
n, err = item.fd.ReadAt(b, off) n, err = item.fd.ReadAt(b, off)