From 63ab2508173130d992a081f731919a6ca861bb17 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 5 Sep 2023 12:38:19 +0100 Subject: [PATCH] vfs: add --vfs-cache-min-free-space to control minimum free space on the disk containing the cache See: https://forum.rclone.org/t/rclone-fails-to-control-disk-usage-and-its-filling-the-disk-to-100/41494/ --- vfs/help.go | 30 +++++++++-------- vfs/vfscache/cache.go | 67 ++++++++++++++++++++++++++++++-------- vfs/vfscache/cache_test.go | 44 +++++++++++++++++++++---- vfs/vfscommon/options.go | 2 ++ vfs/vfsflags/vfsflags.go | 1 + 5 files changed, 109 insertions(+), 35 deletions(-) diff --git a/vfs/help.go b/vfs/help.go index fe3c5f2a9..c509696ad 100644 --- a/vfs/help.go +++ b/vfs/help.go @@ -84,12 +84,13 @@ write simultaneously to a file. See below for more details. Note that the VFS cache is separate from the cache backend and you may find that you need one or the other or both. - --cache-dir string Directory rclone will use for caching. - --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-max-age duration Max time since last access of objects in the cache (default 1h0m0s) - --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) + --cache-dir string Directory rclone will use for caching. + --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) + --vfs-cache-max-age duration Max time since last access of objects in the cache (default 1h0m0s) + --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) + --vfs-cache-min-free-space SizeSuffix Target minimum free space on the disk containing the cache (default off) + --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) If run with !-vv! rclone will print the location of the file cache. The files are stored in the user cache file area which is OS dependent but @@ -106,14 +107,15 @@ seconds. If rclone is quit or dies with files that haven't been uploaded, these will be uploaded next time rclone is run with the same flags. -If using !--vfs-cache-max-size! note that the cache may exceed this size -for two reasons. Firstly because it is only checked every -!--vfs-cache-poll-interval!. Secondly because open files cannot be -evicted from the cache. When !--vfs-cache-max-size! -is exceeded, rclone will attempt to evict the least accessed files -from the cache first. rclone will start with files that haven't -been accessed for the longest. This cache flushing strategy is -efficient and more relevant files are likely to remain cached. +If using !--vfs-cache-max-size! or !--vfs-cache-min-free-size! note +that the cache may exceed these quotas for two reasons. Firstly +because it is only checked every !--vfs-cache-poll-interval!. Secondly +because open files cannot be evicted from the cache. When +!--vfs-cache-max-size! or !--vfs-cache-min-free-size! is exceeded, +rclone will attempt to evict the least accessed files from the cache +first. rclone will start with files that haven't been accessed for the +longest. This cache flushing strategy is efficient and more relevant +files are likely to remain cached. The !--vfs-cache-max-age! will evict files from the cache after the set time since last access has passed. The default value of diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 20c9593c1..e5bb19c79 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -22,6 +22,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/lib/diskusage" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/file" "github.com/rclone/rclone/vfs/vfscache/writeback" @@ -607,16 +608,17 @@ func (c *Cache) retryFailedResets() { } } -func (c *Cache) purgeClean(quota int64) { +// Remove cache files that are not dirty until the quota is satisfied +func (c *Cache) purgeClean() { c.mu.Lock() defer c.mu.Unlock() - var items Items - - if quota <= 0 || c.used < quota { + if c.quotasOK() { return } + var items Items + // Make a slice of clean cache files for _, item := range c.item { if !item.IsDirty() { @@ -628,7 +630,7 @@ func (c *Cache) purgeClean(quota int64) { // Reset items until the quota is OK for _, item := range items { - if c.used < quota { + if c.quotasOK() { break } resetResult, spaceFreed, err := item.Reset() @@ -661,7 +663,7 @@ func (c *Cache) purgeOld(maxAge time.Duration) { for _, item := range c.item { c.removeNotInUse(item, maxAge, false) } - if c.used < int64(c.opt.CacheMaxSize) { + if c.quotasOK() { c.outOfSpace = false c.cond.Broadcast() } @@ -693,16 +695,53 @@ func (c *Cache) updateUsed() (used int64) { return newUsed } +// Check the available space for a disk is in limits. +func (c *Cache) minFreeSpaceQuotaOK() bool { + if c.opt.CacheMinFreeSpace <= 0 { + return true + } + du, err := diskusage.New(config.GetCacheDir()) + if err == diskusage.ErrUnsupported { + return true + } + if err != nil { + fs.Errorf(nil, "disk usage returned error: %v", err) + return true + } + return du.Available >= uint64(c.opt.CacheMinFreeSpace) +} + +// Check the available quota for a disk is in limits. +// +// must be called with mu held. +func (c *Cache) maxSizeQuotaOK() bool { + if c.opt.CacheMaxSize <= 0 { + return true + } + return c.used <= int64(c.opt.CacheMaxSize) +} + +// Check the available quotas for a disk is in limits. +// +// must be called with mu held. +func (c *Cache) quotasOK() bool { + return c.maxSizeQuotaOK() && c.minFreeSpaceQuotaOK() +} + +// Return true if any quotas set +func (c *Cache) haveQuotas() bool { + return c.opt.CacheMaxSize > 0 || c.opt.CacheMinFreeSpace > 0 +} + // Remove clean cache files that are not open until the total space // is reduced below quota starting from the oldest first -func (c *Cache) purgeOverQuota(quota int64) { +func (c *Cache) purgeOverQuota() { c.updateUsed() c.mu.Lock() defer c.mu.Unlock() - if quota <= 0 || c.used < quota { - + if c.quotasOK() { return } @@ -719,9 +758,9 @@ func (c *Cache) purgeOverQuota(quota int64) { // Remove items until the quota is OK for _, item := range items { - c.removeNotInUse(item, 0, c.used <= quota) + c.removeNotInUse(item, 0, c.quotasOK()) } - if c.used < quota { + if c.quotasOK() { c.outOfSpace = false c.cond.Broadcast() } @@ -743,12 +782,12 @@ func (c *Cache) clean(kicked bool) { c.purgeOld(c.opt.CacheMaxAge) // If have a maximum cache size... - if int64(c.opt.CacheMaxSize) > 0 { + if c.haveQuotas() { // Remove files not in use until cache size is below quota starting from the oldest first - c.purgeOverQuota(int64(c.opt.CacheMaxSize)) + c.purgeOverQuota() // Remove cache files that are not dirty if we are still above the max cache size - c.purgeClean(int64(c.opt.CacheMaxSize)) + c.purgeClean() c.retryFailedResets() } diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go index 5ed346a13..a76d99c2c 100644 --- a/vfs/vfscache/cache_test.go +++ b/vfs/vfscache/cache_test.go @@ -10,7 +10,10 @@ import ( "time" _ "github.com/rclone/rclone/backend/local" // import the local backend + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/diskusage" "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -354,7 +357,8 @@ func TestCachePurgeOverQuota(t *testing.T) { }, itemAsString(c)) // Check nothing removed - c.purgeOverQuota(1) + c.opt.CacheMaxSize = 1 + c.purgeOverQuota() // Close the files require.NoError(t, potato.Close(nil)) @@ -373,7 +377,8 @@ func TestCachePurgeOverQuota(t *testing.T) { potato2.info.ATime = t1 // Check only potato removed to get below quota - c.purgeOverQuota(10) + c.opt.CacheMaxSize = 10 + c.purgeOverQuota() assert.Equal(t, int64(6), c.used) assert.Equal(t, []string{ @@ -399,7 +404,8 @@ func TestCachePurgeOverQuota(t *testing.T) { potato.info.ATime = t2 // Check only potato2 removed to get below quota - c.purgeOverQuota(10) + c.opt.CacheMaxSize = 10 + c.purgeOverQuota() assert.Equal(t, int64(5), c.used) c.purgeEmptyDirs("", true) @@ -408,7 +414,8 @@ func TestCachePurgeOverQuota(t *testing.T) { }, itemAsString(c)) // Now purge everything - c.purgeOverQuota(1) + c.opt.CacheMaxSize = 1 + c.purgeOverQuota() assert.Equal(t, int64(0), c.used) c.purgeEmptyDirs("", true) @@ -420,6 +427,26 @@ func TestCachePurgeOverQuota(t *testing.T) { assert.Equal(t, []string(nil), itemAsString(c)) } +func TestCachePurgeMinFreeSpace(t *testing.T) { + du, err := diskusage.New(config.GetCacheDir()) + if err == diskusage.ErrUnsupported { + t.Skip(err) + } + // We've tested the quota mechanism already, so just test the + // min free space quota is working. + _, c := newTestCache(t) + + // First set free space quota very small and check it is OK + c.opt.CacheMinFreeSpace = 1 + assert.True(t, c.minFreeSpaceQuotaOK()) + assert.True(t, c.quotasOK()) + + // Now set it a bit larger than the current disk available and check it is BAD + c.opt.CacheMinFreeSpace = fs.SizeSuffix(du.Available) + fs.Gibi + assert.False(t, c.minFreeSpaceQuotaOK()) + assert.False(t, c.quotasOK()) +} + // test reset clean files func TestCachePurgeClean(t *testing.T) { r, c := newItemTestCache(t) @@ -453,7 +480,8 @@ func TestCachePurgeClean(t *testing.T) { require.NoError(t, potato3.Truncate(6)) c.updateUsed() - c.purgeClean(1) + c.opt.CacheMaxSize = 1 + c.purgeClean() assert.Equal(t, []string{ `name="existing" opens=2 size=100 space=0`, `name="sub/dir/potato2" opens=1 size=5 space=5`, @@ -462,7 +490,8 @@ func TestCachePurgeClean(t *testing.T) { assert.Equal(t, int64(11), c.used) require.NoError(t, potato2.Close(nil)) - c.purgeClean(1) + c.opt.CacheMaxSize = 1 + c.purgeClean() assert.Equal(t, []string{ `name="existing" opens=2 size=100 space=0`, `name="sub/dir/potato3" opens=1 size=6 space=6`, @@ -476,7 +505,8 @@ func TestCachePurgeClean(t *testing.T) { // Remove all files now. The are all not in use. // purgeClean does not remove empty cache files. purgeOverQuota does. // So we use purgeOverQuota here for the cleanup. - c.purgeOverQuota(1) + c.opt.CacheMaxSize = 1 + c.purgeOverQuota() c.purgeEmptyDirs("", true) diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go index a5295c4ee..06135593e 100644 --- a/vfs/vfscommon/options.go +++ b/vfs/vfscommon/options.go @@ -26,6 +26,7 @@ type Options struct { CacheMode CacheMode CacheMaxAge time.Duration CacheMaxSize fs.SizeSuffix + CacheMinFreeSpace fs.SizeSuffix CachePollInterval time.Duration CaseInsensitive bool WriteWait time.Duration // time to wait for in-sequence write @@ -56,6 +57,7 @@ var DefaultOpt = Options{ ChunkSize: 128 * fs.Mebi, ChunkSizeLimit: -1, CacheMaxSize: -1, + CacheMinFreeSpace: -1, CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise WriteWait: 1000 * time.Millisecond, ReadWait: 20 * time.Millisecond, diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index 46fde2d65..b74c76d89 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -28,6 +28,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.DurationVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", Opt.CachePollInterval, "Interval to poll the cache for stale objects", "VFS") flags.DurationVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", Opt.CacheMaxAge, "Max time since last access of objects in the cache", "VFS") flags.FVarP(flagSet, &Opt.CacheMaxSize, "vfs-cache-max-size", "", "Max total size of objects in the cache", "VFS") + flags.FVarP(flagSet, &Opt.CacheMinFreeSpace, "vfs-cache-min-free-space", "", "Target minimum free space on the disk containing the cache", "VFS") flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks", "VFS") flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited)", "VFS") flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions", "VFS")