From c0cf54067ad9dde419ceb1329740e5ef67de4b32 Mon Sep 17 00:00:00 2001
From: tYYGH <tYYGH@users.noreply.github.com>
Date: Wed, 17 Feb 2021 21:36:13 +0100
Subject: [PATCH] vfs: --vfs-used-is-size to report used space using recursive
 scan (#4043)

Some backends, most notably S3, do not report the amount of bytes used.
This patch introduces a new flag that allows instead of relying on the
backend, use recursive scan similar to `rclone size` to compute the total
used space. However, this is ineffective and should be used as a last resort.

Co-authored-by: Yves G <theYinYeti@yalis.fr>
---
 vfs/help.go              | 13 +++++++++++++
 vfs/vfs.go               | 21 +++++++++++++++++++--
 vfs/vfscommon/options.go |  2 ++
 vfs/vfsflags/vfsflags.go |  1 +
 4 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/vfs/help.go b/vfs/help.go
index b068f88e8..b016d6880 100644
--- a/vfs/help.go
+++ b/vfs/help.go
@@ -259,4 +259,17 @@ The flag controls whether "fixup" is performed to satisfy the target.
 If the flag is not provided on the command line, then its default value depends
 on the operating system where rclone runs: "true" on Windows and macOS, "false"
 otherwise. If the flag is provided without a value, then it is "true".
+
+### Alternate report of used bytes
+
+Some backends, most notably S3, do not report the amount of bytes used.
+If you need this information to be available when running !df! on the
+filesystem, then pass the flag !--vfs-used-is-size! to rclone.
+With this flag set, instead of relying on the backend to report this
+information, rclone will scan the whole remote similar to !rclone size!
+and compute the total used space itself.
+
+_WARNING._ Contrary to !rclone size!, this flag ignores filters so that the
+result is accurate. However, this is very inefficient and may cost lots of API
+calls resulting in extra charges. Use it as a last resort and only with caching.
 `, "!", "`")
diff --git a/vfs/vfs.go b/vfs/vfs.go
index b0f50a11e..e4533fa35 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -36,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/vfs/vfscache"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -556,9 +557,25 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 	defer vfs.usageMu.Unlock()
 	total, used, free = -1, -1, -1
 	doAbout := vfs.f.Features().About
-	if doAbout != nil && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
+	if (doAbout != nil || vfs.Opt.UsedIsSize) && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
 		var err error
-		vfs.usage, err = doAbout(context.TODO())
+		ctx := context.TODO()
+		if doAbout == nil {
+			vfs.usage = &fs.Usage{}
+		} else {
+			vfs.usage, err = doAbout(ctx)
+		}
+		if vfs.Opt.UsedIsSize {
+			var usedBySizeAlgorithm int64 = 0
+			// Algorithm from `rclone size`
+			err = walk.ListR(ctx, vfs.f, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+				entries.ForObject(func(o fs.Object) {
+					usedBySizeAlgorithm += o.Size()
+				})
+				return nil
+			})
+			vfs.usage.Used = &usedBySizeAlgorithm
+		}
 		vfs.usageTime = time.Now()
 		if err != nil {
 			fs.Errorf(vfs.f, "Statfs failed: %v", err)
diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go
index 948c66d21..22ea117d6 100644
--- a/vfs/vfscommon/options.go
+++ b/vfs/vfscommon/options.go
@@ -32,6 +32,7 @@ type Options struct {
 	ReadWait          time.Duration // time to wait for in-sequence read
 	WriteBack         time.Duration // time to wait before writing back dirty files
 	ReadAhead         fs.SizeSuffix // bytes to read ahead in cache mode "full"
+	UsedIsSize        bool          // if true, use the `rclone size` algorithm for Used size
 }
 
 // DefaultOpt is the default values uses for Opt
@@ -58,4 +59,5 @@ var DefaultOpt = Options{
 	ReadWait:          20 * time.Millisecond,
 	WriteBack:         5 * time.Second,
 	ReadAhead:         0 * fs.MebiByte,
+	UsedIsSize:        false,
 }
diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go
index ebc1ead16..d08370399 100644
--- a/vfs/vfsflags/vfsflags.go
+++ b/vfs/vfsflags/vfsflags.go
@@ -37,5 +37,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
 	flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.")
 	flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.")
+	flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size.")
 	platformFlags(flagSet)
 }