github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-21 02:44:02 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity
704217b698
rclone/vfs/vfscache/cache_test.go

754 lines
18 KiB
Go
Raw Normal View History

vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
package vfscache
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"testing"
"time"
_ "github.com/rclone/rclone/backend/local" // import the local backend
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
"github.com/rclone/rclone/fstest"
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
"github.com/rclone/rclone/lib/diskusage"
rc: add vfs/queue to show the status of the upload queue
2024-06-20 22:34:26 +08:00
"github.com/rclone/rclone/vfs/vfscache/writeback"
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestMain drives the tests
func TestMain(m *testing.M) {
fstest.TestMain(m)
}
// convert c.item to a string
func itemAsString(c *Cache) []string {
c.mu.Lock()
defer c.mu.Unlock()
var out []string
for name, item := range c.item {
out = append(out, fmt.Sprintf("name=%q opens=%d size=%d", filepath.ToSlash(name), item.opens, item.info.Size))
}
sort.Strings(out)
return out
}
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
// convert c.item to a string
func itemSpaceAsString(c *Cache) []string {
c.mu.Lock()
defer c.mu.Unlock()
var out []string
for name, item := range c.item {
space := item.info.Rs.Size()
out = append(out, fmt.Sprintf("name=%q opens=%d size=%d space=%d", filepath.ToSlash(name), item.opens, item.info.Size, space))
}
sort.Strings(out)
return out
}
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
// open an item and write to it
func itemWrite(t *testing.T, item *Item, contents string) {
require.NoError(t, item.Open(nil))
_, err := item.WriteAt([]byte(contents), 0)
require.NoError(t, err)
}
func assertPathNotExist(t *testing.T, path string) {
_, err := os.Stat(path)
assert.True(t, os.IsNotExist(err))
}
func assertPathExist(t *testing.T, path string) os.FileInfo {
fi, err := os.Stat(path)
assert.NoError(t, err)
return fi
}
vfs: Add recovered items on cache reload to directory listings Before this change, if we restarted an upload after a restart then the file would get uploaded but never added to the directory listings. This change makes sure we add virtual items to the directory cache when reloading the cache so that they show up properly.
2020-06-23 22:18:58 +08:00
type avInfo struct {
Remote string
Size int64
IsDir bool
}
var avInfos []avInfo
func addVirtual(remote string, size int64, isDir bool) error {
avInfos = append(avInfos, avInfo{
Remote: remote,
Size: size,
IsDir: isDir,
})
return nil
}
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cache) {
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
r = fstest.NewRun(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
ctx, cancel := context.WithCancel(context.Background())
vfs: Add recovered items on cache reload to directory listings Before this change, if we restarted an upload after a restart then the file would get uploaded but never added to the directory listings. This change makes sure we add virtual items to the directory cache when reloading the cache so that they show up properly.
2020-06-23 22:18:58 +08:00
avInfos = nil
c, err := New(ctx, r.Fremote, &opt, addVirtual)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
require.NoError(t, err)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
t.Cleanup(func() {
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
err := c.CleanUp()
require.NoError(t, err)
assertPathNotExist(t, c.root)
cancel()
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
})
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
return r, c
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
}
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
func newTestCache(t *testing.T) (r *fstest.Run, c *Cache) {
vfs: convert vfs options to new style This also - move in use options (Opt) from vfsflags to vfscommon - change os.FileMode to vfscommon.FileMode in parameters - rework vfscommon.FileMode and add tests
2024-07-03 18:34:29 +08:00
opt := vfscommon.Opt
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
// Disable the cache cleaner as it interferes with these tests
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
opt.CachePollInterval = 0
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
vfs: Make tests run reliably On file Remove - cancel any writebacks in progress - ignore error message deleting non existent file if file was in the process of being uploaded Writeback - Don't transfer the file if it has disappeared in the meantime - Take our own copy of the file name to avoid deadlocks - Fix delayed retry logic - Wait for upload to finish when cancelling upload Fix race condition in item saving Fix race condition in vfscache test Make sure we delete the file on the error path - this makes cascading failures much less likely
2020-06-03 22:49:41 +08:00
// Disable synchronous write
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
opt.WriteBack = 0
return newTestCacheOpt(t, opt)
}
func TestCacheNew(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
r, c := newTestCache(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Contains(t, c.root, "vfs")
assert.Contains(t, c.fcache.Root(), filepath.Base(r.Fremote.Root()))
assert.Equal(t, []string(nil), itemAsString(c))
vfs: ensure names used in cache path are legal on current os Fixes #5360
2021-05-28 21:11:19 +08:00
// createItemDir
p, err := c.createItemDir("potato")
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string(nil), itemAsString(c))
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
fi := assertPathExist(t, filepath.Dir(p))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.True(t, fi.IsDir())
// get
item, _ := c.get("potato")
item2, _ := c.get("potato")
assert.Equal(t, item, item2)
assert.WithinDuration(t, time.Now(), item.info.ATime, time.Second)
// open
assert.Equal(t, []string{
`name="potato" opens=0 size=0`,
}, itemAsString(c))
potato := c.Item("/potato")
require.NoError(t, potato.Open(nil))
assert.Equal(t, []string{
`name="potato" opens=1 size=0`,
}, itemAsString(c))
assert.WithinDuration(t, time.Now(), potato.info.ATime, time.Second)
assert.Equal(t, 1, potato.opens)
// write the file
require.NoError(t, potato.Truncate(5))
atime := time.Now()
potato.info.ATime = atime
assert.Equal(t, []string{
`name="potato" opens=1 size=5`,
}, itemAsString(c))
assert.True(t, atime.Equal(potato.info.ATime), fmt.Sprintf("%v != %v", atime, potato.info.ATime))
// try purging with file open
c.purgeOld(10 * time.Second)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assertPathExist(t, p)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// close
assert.Equal(t, []string{
`name="potato" opens=1 size=5`,
}, itemAsString(c))
require.NoError(t, potato.Truncate(6))
assert.Equal(t, []string{
`name="potato" opens=1 size=6`,
}, itemAsString(c))
require.NoError(t, potato.Close(nil))
assert.Equal(t, []string{
`name="potato" opens=0 size=6`,
}, itemAsString(c))
item, _ = c.get("potato")
assert.WithinDuration(t, time.Now(), item.info.ATime, time.Second)
assert.Equal(t, 0, item.opens)
// try purging with file closed
c.purgeOld(10 * time.Second)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assertPathExist(t, p)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
//.. purge again with -ve age
c.purgeOld(-10 * time.Second)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assertPathNotExist(t, p)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// clean - have tested the internals already
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.clean(false)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
}
func TestCacheOpens(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string(nil), itemAsString(c))
potato := c.Item("potato")
require.NoError(t, potato.Open(nil))
assert.Equal(t, []string{
`name="potato" opens=1 size=0`,
}, itemAsString(c))
require.NoError(t, potato.Open(nil))
assert.Equal(t, []string{
`name="potato" opens=2 size=0`,
}, itemAsString(c))
require.NoError(t, potato.Close(nil))
assert.Equal(t, []string{
`name="potato" opens=1 size=0`,
}, itemAsString(c))
require.NoError(t, potato.Close(nil))
assert.Equal(t, []string{
`name="potato" opens=0 size=0`,
}, itemAsString(c))
require.NoError(t, potato.Open(nil))
a1 := c.Item("a//b/c/d/one")
a2 := c.Item("a/b/c/d/e/two")
a3 := c.Item("a/b/c/d/e/f/three")
require.NoError(t, a1.Open(nil))
require.NoError(t, a2.Open(nil))
require.NoError(t, a3.Open(nil))
assert.Equal(t, []string{
`name="a/b/c/d/e/f/three" opens=1 size=0`,
`name="a/b/c/d/e/two" opens=1 size=0`,
`name="a/b/c/d/one" opens=1 size=0`,
`name="potato" opens=1 size=0`,
}, itemAsString(c))
require.NoError(t, potato.Close(nil))
require.NoError(t, a1.Close(nil))
require.NoError(t, a2.Close(nil))
require.NoError(t, a3.Close(nil))
assert.Equal(t, []string{
`name="a/b/c/d/e/f/three" opens=0 size=0`,
`name="a/b/c/d/e/two" opens=0 size=0`,
`name="a/b/c/d/one" opens=0 size=0`,
`name="potato" opens=0 size=0`,
}, itemAsString(c))
}
vfs: ensure names used in cache path are legal on current os Fixes #5360
2021-05-28 21:11:19 +08:00
// test the open, createItemDir, purge, close, purge sequence
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
func TestCacheOpenMkdir(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// open
potato := c.Item("sub/potato")
require.NoError(t, potato.Open(nil))
assert.Equal(t, []string{
`name="sub/potato" opens=1 size=0`,
}, itemAsString(c))
vfs: ensure names used in cache path are legal on current os Fixes #5360
2021-05-28 21:11:19 +08:00
// createItemDir
p, err := c.createItemDir("sub/potato")
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string{
`name="sub/potato" opens=1 size=0`,
}, itemAsString(c))
// test directory exists
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
fi := assertPathExist(t, filepath.Dir(p))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.True(t, fi.IsDir())
// clean the cache
c.purgeOld(-10 * time.Second)
// test directory still exists
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
fi = assertPathExist(t, filepath.Dir(p))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.True(t, fi.IsDir())
// close
require.NoError(t, potato.Close(nil))
assert.Equal(t, []string{
`name="sub/potato" opens=0 size=0`,
}, itemAsString(c))
// clean the cache
c.purgeOld(-10 * time.Second)
vfs: rename files in cache and cancel uploads on directory rename Before this change rclone did not cancel an uploads or rename the cached files in the directory cache when a directory was renamed. This caused issues with uploads arriving in the wrong place on bucket based file systems. See: https://forum.rclone.org/t/after-a-directory-renmane-using-mv-files-are-not-visible-any-longer/22797
2021-03-17 17:48:25 +08:00
c.purgeEmptyDirs("", true)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string(nil), itemAsString(c))
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
// test directory does not exist
assertPathNotExist(t, filepath.Dir(p))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
}
func TestCachePurgeOld(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// Test funcs
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.purgeOld(-10 * time.Second)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
potato2 := c.Item("sub/dir2/potato2")
require.NoError(t, potato2.Open(nil))
potato := c.Item("sub/dir/potato")
require.NoError(t, potato.Open(nil))
require.NoError(t, potato2.Close(nil))
require.NoError(t, potato.Open(nil))
assert.Equal(t, []string{
`name="sub/dir/potato" opens=2 size=0`,
`name="sub/dir2/potato2" opens=0 size=0`,
}, itemAsString(c))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.purgeOld(-10 * time.Second)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string{
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
`name="sub/dir/potato" opens=2 size=0`,
}, itemAsString(c))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
require.NoError(t, potato.Close(nil))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
assert.Equal(t, []string{
`name="sub/dir/potato" opens=1 size=0`,
}, itemAsString(c))
c.purgeOld(-10 * time.Second)
assert.Equal(t, []string{
`name="sub/dir/potato" opens=1 size=0`,
}, itemAsString(c))
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
require.NoError(t, potato.Close(nil))
assert.Equal(t, []string{
`name="sub/dir/potato" opens=0 size=0`,
}, itemAsString(c))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.purgeOld(10 * time.Second)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string{
`name="sub/dir/potato" opens=0 size=0`,
}, itemAsString(c))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.purgeOld(-10 * time.Second)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string(nil), itemAsString(c))
}
func TestCachePurgeOverQuota(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// Test funcs
// Make some test files
potato := c.Item("sub/dir/potato")
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
itemWrite(t, potato, "hello")
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
potato2 := c.Item("sub/dir2/potato2")
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
itemWrite(t, potato2, "hello2")
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string{
`name="sub/dir/potato" opens=1 size=5`,
`name="sub/dir2/potato2" opens=1 size=6`,
}, itemAsString(c))
// Check nothing removed
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 1
c.purgeOverQuota()
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
// Close the files
require.NoError(t, potato.Close(nil))
require.NoError(t, potato2.Close(nil))
assert.Equal(t, []string{
`name="sub/dir/potato" opens=0 size=5`,
`name="sub/dir2/potato2" opens=0 size=6`,
}, itemAsString(c))
// Update the stats to read the total size
c.updateUsed()
// make potato2 definitely after potato
t1 := time.Now().Add(10 * time.Second)
potato2.info.ATime = t1
// Check only potato removed to get below quota
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 10
c.purgeOverQuota()
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, int64(6), c.used)
assert.Equal(t, []string{
`name="sub/dir2/potato2" opens=0 size=6`,
}, itemAsString(c))
// Put potato back
potato = c.Item("sub/dir/potato")
require.NoError(t, potato.Open(nil))
require.NoError(t, potato.Truncate(5))
require.NoError(t, potato.Close(nil))
// Update the stats to read the total size
c.updateUsed()
assert.Equal(t, []string{
`name="sub/dir/potato" opens=0 size=5`,
`name="sub/dir2/potato2" opens=0 size=6`,
}, itemAsString(c))
// make potato definitely after potato2
t2 := t1.Add(20 * time.Second)
potato.info.ATime = t2
// Check only potato2 removed to get below quota
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 10
c.purgeOverQuota()
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, int64(5), c.used)
vfs: rename files in cache and cancel uploads on directory rename Before this change rclone did not cancel an uploads or rename the cached files in the directory cache when a directory was renamed. This caused issues with uploads arriving in the wrong place on bucket based file systems. See: https://forum.rclone.org/t/after-a-directory-renmane-using-mv-files-are-not-visible-any-longer/22797
2021-03-17 17:48:25 +08:00
c.purgeEmptyDirs("", true)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string{
`name="sub/dir/potato" opens=0 size=5`,
}, itemAsString(c))
// Now purge everything
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 1
c.purgeOverQuota()
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, int64(0), c.used)
vfs: rename files in cache and cancel uploads on directory rename Before this change rclone did not cancel an uploads or rename the cached files in the directory cache when a directory was renamed. This caused issues with uploads arriving in the wrong place on bucket based file systems. See: https://forum.rclone.org/t/after-a-directory-renmane-using-mv-files-are-not-visible-any-longer/22797
2021-03-17 17:48:25 +08:00
c.purgeEmptyDirs("", true)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, []string(nil), itemAsString(c))
// Check nothing left behind
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
c.clean(false)
vfs: implement partial reads in --vfs-cache-mode full This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
2020-03-01 02:08:22 +08:00
assert.Equal(t, int64(0), c.used)
assert.Equal(t, []string(nil), itemAsString(c))
}
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
func TestCachePurgeMinFreeSpace(t *testing.T) {
du, err := diskusage.New(config.GetCacheDir())
if err == diskusage.ErrUnsupported {
t.Skip(err)
}
// We've tested the quota mechanism already, so just test the
// min free space quota is working.
_, c := newTestCache(t)
// First set free space quota very small and check it is OK
c.opt.CacheMinFreeSpace = 1
assert.True(t, c.minFreeSpaceQuotaOK())
assert.True(t, c.quotasOK())
// Now set it a bit larger than the current disk available and check it is BAD
c.opt.CacheMinFreeSpace = fs.SizeSuffix(du.Available) + fs.Gibi
assert.False(t, c.minFreeSpaceQuotaOK())
assert.False(t, c.quotasOK())
}
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
// test reset clean files
func TestCachePurgeClean(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
r, c := newItemTestCache(t)
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
contents, obj, potato1 := newFile(t, r, c, "existing")
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
_ = contents
// Open the object to create metadata for it
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
require.NoError(t, potato1.Open(obj))
require.NoError(t, potato1.Open(obj))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
size, err := potato1.GetSize()
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
require.NoError(t, err)
assert.Equal(t, int64(100), size)
// Read something to instantiate the cache file
buf := make([]byte, 10)
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
_, err = potato1.ReadAt(buf, 10)
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
require.NoError(t, err)
// Test cache file present
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
_, err = os.Stat(potato1.c.toOSPath(potato1.name))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
require.NoError(t, err)
all: fix spelling across the project * abcdefghijklmnopqrstuvwxyz * accounting * additional * allowed * almost * already * appropriately * arise * bandwidth * behave * bidirectional * brackets * cached * characters * cloud * committing * concatenating * configured * constructs * current * cutoff * deferred * different * directory * disposition * dropbox * either way * error * excess * experiments * explicitly * externally * files * github * gzipped * hierarchies * huffman * hyphen * implicitly * independent * insensitive * integrity * libraries * literally * metadata * mimics * missing * modification * multipart * multiple * nightmare * nonexistent * number * obscure * ourselves * overridden * potatoes * preexisting * priority * received * remote * replacement * represents * reproducibility * response * satisfies * sensitive * separately * separator * specifying * string * successful * synchronization * syncing * šenfeld * take * temporarily * testcontents * that * the * themselves * throttling * timeout * transaction * transferred * unnecessary * using * webbrowser * which * with * workspace Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2022-08-14 10:56:32 +08:00
// Add some potatoes
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
potato2 := c.Item("sub/dir/potato2")
require.NoError(t, potato2.Open(nil))
require.NoError(t, potato2.Truncate(5))
potato3 := c.Item("sub/dir/potato3")
require.NoError(t, potato3.Open(nil))
require.NoError(t, potato3.Truncate(6))
c.updateUsed()
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 1
c.purgeClean()
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
assert.Equal(t, []string{
`name="existing" opens=2 size=100 space=0`,
`name="sub/dir/potato2" opens=1 size=5 space=5`,
`name="sub/dir/potato3" opens=1 size=6 space=6`,
}, itemSpaceAsString(c))
assert.Equal(t, int64(11), c.used)
require.NoError(t, potato2.Close(nil))
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 1
c.purgeClean()
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
assert.Equal(t, []string{
`name="existing" opens=2 size=100 space=0`,
`name="sub/dir/potato3" opens=1 size=6 space=6`,
}, itemSpaceAsString(c))
assert.Equal(t, int64(6), c.used)
Spelling fixes Fix spelling of: above, already, anonymous, associated, authentication, bandwidth, because, between, blocks, calculate, candidates, cautious, changelog, cleaner, clipboard, command, completely, concurrently, considered, constructs, corrupt, current, daemon, dependencies, deprecated, directory, dispatcher, download, eligible, ellipsis, encrypter, endpoint, entrieslist, essentially, existing writers, existing, expires, filesystem, flushing, frequently, hierarchy, however, implementation, implements, inaccurate, individually, insensitive, longer, maximum, metadata, modified, multipart, namedirfirst, nextcloud, obscured, opened, optional, owncloud, pacific, passphrase, password, permanently, persimmon, positive, potato, protocol, quota, receiving, recommends, referring, requires, revisited, satisfied, satisfies, satisfy, semver, serialized, session, storage, strategies, stringlist, successful, supported, surprise, temporarily, temporary, transactions, unneeded, update, uploads, wrapped Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 08:17:24 +08:00
require.NoError(t, potato1.Close(nil))
require.NoError(t, potato1.Close(nil))
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
require.NoError(t, potato3.Close(nil))
// Remove all files now. The are all not in use.
// purgeClean does not remove empty cache files. purgeOverQuota does.
// So we use purgeOverQuota here for the cleanup.
vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/
2023-09-05 19:38:19 +08:00
c.opt.CacheMaxSize = 1
c.purgeOverQuota()
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
vfs: rename files in cache and cancel uploads on directory rename Before this change rclone did not cancel an uploads or rename the cached files in the directory cache when a directory was renamed. This caused issues with uploads arriving in the wrong place on bucket based file systems. See: https://forum.rclone.org/t/after-a-directory-renmane-using-mv-files-are-not-visible-any-longer/22797
2021-03-17 17:48:25 +08:00
c.purgeEmptyDirs("", true)
vfs: support synchronous cache space recovery upon ENOSPC This patch provides the support of synchronous cache space recovery to allow read threads to recover from ENOSPC errors when cache space can be recovered from cache items that are not in use or safe to be reset/emptied . The patch complements the existing cache cleaning process in two ways. Firstly, the existing cache cleaning process is time-driven that runs periodically. The cache space can run out while the cache cleaner thread is still waiting for its next scheduled run. The io threads encountering ENOSPC return an internal error to the applications in this case even when cache space can be recovered to avoid this error. This patch addresses this problem by having the read threads kick the cache cleaner thread in this condition to recover cache space preventing unnecessary ENOSPC errors from being seen by the applications. Secondly, this patch enhances the cache cleaner to support cache item reset. Currently the cache purge process removes cache items that are not in use. This may not be sufficient when the total size of the working set exceeds the cache directory's capacity. Like in the current code, this patch starts the purge process by removing cache files that are not in use. Cache items whose access times are older than vfs-cache-max-age are removed first. After that, other not-in-use items are removed in LRU order until vfs-cache-max-size is reached. If the vfs-cache-max-size (the quota) is still not reached at this time, this patch adds a cache reset step to reset/empty cache files that are still in use but not dirtied. This enables application processes to continue without seeing an error even when the working set depletes the cache space as long as there is not a large write working set hoarding the entire cache space. By design this patch does not add ENOSPC error recovery for write IOs. Rclone does not empty a write cache item until the file data is written back to the backend upon close. Allowing more cache space to be consumed by dirty cache items when the cache space is already running low would increase the risk of exhausting the cache space in a way that the vfs mount becomes unreadable.
2020-08-25 23:20:29 +08:00
assert.Equal(t, []string(nil), itemAsString(c))
}
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
func TestCacheInUse(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.False(t, c.InUse("potato"))
potato := c.Item("potato")
assert.False(t, c.InUse("potato"))
require.NoError(t, potato.Open(nil))
assert.True(t, c.InUse("potato"))
require.NoError(t, potato.Close(nil))
assert.False(t, c.InUse("potato"))
}
func TestCacheDirtyItem(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.Nil(t, c.DirtyItem("potato"))
potato := c.Item("potato")
assert.Nil(t, c.DirtyItem("potato"))
require.NoError(t, potato.Open(nil))
require.NoError(t, potato.Truncate(5))
assert.Equal(t, potato, c.DirtyItem("potato"))
require.NoError(t, potato.Close(nil))
assert.Nil(t, c.DirtyItem("potato"))
}
func TestCacheExistsAndRemove(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.False(t, c.Exists("potato"))
potato := c.Item("potato")
assert.False(t, c.Exists("potato"))
require.NoError(t, potato.Open(nil))
assert.True(t, c.Exists("potato"))
require.NoError(t, potato.Close(nil))
assert.True(t, c.Exists("potato"))
c.Remove("potato")
assert.False(t, c.Exists("potato"))
}
func TestCacheRename(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
// setup
assert.False(t, c.Exists("potato"))
potato := c.Item("potato")
require.NoError(t, potato.Open(nil))
require.NoError(t, potato.Close(nil))
assert.True(t, c.Exists("potato"))
osPath := c.toOSPath("potato")
osPathMeta := c.toOSPathMeta("potato")
assertPathExist(t, osPath)
assertPathExist(t, osPathMeta)
// rename potato -> newPotato
require.NoError(t, c.Rename("potato", "newPotato", nil))
assertPathNotExist(t, osPath)
assertPathNotExist(t, osPathMeta)
assert.False(t, c.Exists("potato"))
osPath = c.toOSPath("newPotato")
osPathMeta = c.toOSPathMeta("newPotato")
assertPathExist(t, osPath)
assertPathExist(t, osPathMeta)
assert.True(t, c.Exists("newPotato"))
// rename newPotato -> sub/newPotato
require.NoError(t, c.Rename("newPotato", "sub/newPotato", nil))
assertPathNotExist(t, osPath)
assertPathNotExist(t, osPathMeta)
assert.False(t, c.Exists("potato"))
osPath = c.toOSPath("sub/newPotato")
osPathMeta = c.toOSPathMeta("sub/newPotato")
assertPathExist(t, osPath)
assertPathExist(t, osPathMeta)
assert.True(t, c.Exists("sub/newPotato"))
// remove
c.Remove("sub/newPotato")
assertPathNotExist(t, osPath)
assertPathNotExist(t, osPathMeta)
assert.False(t, c.Exists("sub/newPotato"))
all: fix spelling across the project * abcdefghijklmnopqrstuvwxyz * accounting * additional * allowed * almost * already * appropriately * arise * bandwidth * behave * bidirectional * brackets * cached * characters * cloud * committing * concatenating * configured * constructs * current * cutoff * deferred * different * directory * disposition * dropbox * either way * error * excess * experiments * explicitly * externally * files * github * gzipped * hierarchies * huffman * hyphen * implicitly * independent * insensitive * integrity * libraries * literally * metadata * mimics * missing * modification * multipart * multiple * nightmare * nonexistent * number * obscure * ourselves * overridden * potatoes * preexisting * priority * received * remote * replacement * represents * reproducibility * response * satisfies * sensitive * separately * separator * specifying * string * successful * synchronization * syncing * šenfeld * take * temporarily * testcontents * that * the * themselves * throttling * timeout * transaction * transferred * unnecessary * using * webbrowser * which * with * workspace Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2022-08-14 10:56:32 +08:00
// nonexistent file - is ignored
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.NoError(t, c.Rename("nonexist", "nonexist2", nil))
}
func TestCacheCleaner(t *testing.T) {
vfs: convert vfs options to new style This also - move in use options (Opt) from vfsflags to vfscommon - change os.FileMode to vfscommon.FileMode in parameters - rework vfscommon.FileMode and add tests
2024-07-03 18:34:29 +08:00
opt := vfscommon.Opt
vfs: convert time.Duration option to fs.Duration
2024-07-03 00:31:51 +08:00
opt.CachePollInterval = fs.Duration(10 * time.Millisecond)
opt.CacheMaxAge = fs.Duration(20 * time.Millisecond)
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCacheOpt(t, opt)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
vfs: convert time.Duration option to fs.Duration
2024-07-03 00:31:51 +08:00
time.Sleep(time.Duration(2 * opt.CachePollInterval))
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
potato := c.Item("potato")
potato2, found := c.get("potato")
vfs: Make tests run reliably On file Remove - cancel any writebacks in progress - ignore error message deleting non existent file if file was in the process of being uploaded Writeback - Don't transfer the file if it has disappeared in the meantime - Take our own copy of the file name to avoid deadlocks - Fix delayed retry logic - Wait for upload to finish when cancelling upload Fix race condition in item saving Fix race condition in vfscache test Make sure we delete the file on the error path - this makes cascading failures much less likely
2020-06-03 22:49:41 +08:00
assert.Equal(t, fmt.Sprintf("%p", potato), fmt.Sprintf("%p", potato2))
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.True(t, found)
vfs: vfscache make TestCacheCleaner test more reliable
2020-06-23 21:41:59 +08:00
for i := 0; i < 100; i++ {
vfs: convert time.Duration option to fs.Duration
2024-07-03 00:31:51 +08:00
time.Sleep(time.Duration(10 * opt.CachePollInterval))
vfs: vfscache make TestCacheCleaner test more reliable
2020-06-23 21:41:59 +08:00
potato2, found = c.get("potato")
if !found {
break
}
}
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
vfs: Make tests run reliably On file Remove - cancel any writebacks in progress - ignore error message deleting non existent file if file was in the process of being uploaded Writeback - Don't transfer the file if it has disappeared in the meantime - Take our own copy of the file name to avoid deadlocks - Fix delayed retry logic - Wait for upload to finish when cancelling upload Fix race condition in item saving Fix race condition in vfscache test Make sure we delete the file on the error path - this makes cascading failures much less likely
2020-06-03 22:49:41 +08:00
assert.NotEqual(t, fmt.Sprintf("%p", potato), fmt.Sprintf("%p", potato2))
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.False(t, found)
}
func TestCacheSetModTime(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
t1 := time.Date(2010, 1, 2, 3, 4, 5, 9, time.UTC)
potato := c.Item("potato")
require.NoError(t, potato.Open(nil))
require.NoError(t, potato.Truncate(5))
require.NoError(t, potato.Close(nil))
c.SetModTime("potato", t1)
osPath := potato.c.toOSPath("potato")
fi, err := os.Stat(osPath)
require.NoError(t, err)
fstest.AssertTimeEqualWithPrecision(t, "potato", t1, fi.ModTime(), time.Second)
}
func TestCacheTotaInUse(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
assert.Equal(t, int(0), c.TotalInUse())
potato := c.Item("potato")
assert.Equal(t, int(0), c.TotalInUse())
require.NoError(t, potato.Open(nil))
assert.Equal(t, int(1), c.TotalInUse())
require.NoError(t, potato.Truncate(5))
assert.Equal(t, int(1), c.TotalInUse())
potato2 := c.Item("potato2")
assert.Equal(t, int(1), c.TotalInUse())
require.NoError(t, potato2.Open(nil))
assert.Equal(t, int(2), c.TotalInUse())
require.NoError(t, potato2.Close(nil))
assert.Equal(t, int(1), c.TotalInUse())
require.NoError(t, potato.Close(nil))
assert.Equal(t, int(0), c.TotalInUse())
}
func TestCacheDump(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add --vfs-writeback option to delay writes back to cloud storage This is enabled by default and can be disabled with --vfs-writeback 0
2020-04-17 18:18:58 +08:00
out := (*Cache)(nil).Dump()
assert.Equal(t, "Cache: <nil>\n", out)
out = c.Dump()
assert.Equal(t, "Cache{\n}\n", out)
c.Item("potato")
out = c.Dump()
want := "Cache{\n\t\"potato\": "
assert.Equal(t, want, out[:len(want)])
c.Remove("potato")
out = c.Dump()
assert.Equal(t, "Cache{\n}\n", out)
}
vfs: add vfs/stats remote control to show statistics - fixes #5816
2021-11-18 00:11:08 +08:00
func TestCacheStats(t *testing.T) {
test: replace defer cleanup with `t.Cleanup` Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-12-08 20:43:53 +08:00
_, c := newTestCache(t)
vfs: add vfs/stats remote control to show statistics - fixes #5816
2021-11-18 00:11:08 +08:00
out := c.Stats()
assert.Equal(t, int64(0), out["bytesUsed"])
assert.Equal(t, 0, out["erroredFiles"])
assert.Equal(t, 0, out["files"])
assert.Equal(t, 0, out["uploadsInProgress"])
assert.Equal(t, 0, out["uploadsQueued"])
}
rc: add vfs/queue to show the status of the upload queue
2024-06-20 22:34:26 +08:00
func TestCacheQueue(t *testing.T) {
_, c := newTestCache(t)
out := c.Queue()
// We've checked the contents of queue in the writeback tests
// Just check it is present here
queue, found := out["queue"]
require.True(t, found)
_, ok := queue.([]writeback.QueueInfo)
require.True(t, ok)
}
rc: add vfs/queue-set-expiry to adjust expiry of items in the VFS queue
2024-06-21 00:00:41 +08:00
func TestCacheQueueSetExpiry(t *testing.T) {
_, c := newTestCache(t)
// Check this returns the correct error when called so we know
// it is plumbed in correctly. The actual tests are done in
// writeback.
err := c.QueueSetExpiry(123123, time.Now())
assert.Equal(t, writeback.ErrorIDNotFound, err)
}
Reference in New Issue Copy Permalink