vfs: --vfs-used-is-size to report used space using recursive scan (#4043)

Some backends, most notably S3, do not report the amount of bytes used.
This patch introduces a new flag that allows instead of relying on the
backend, use recursive scan similar to `rclone size` to compute the total
used space. However, this is ineffective and should be used as a last resort.

Co-authored-by: Yves G <theYinYeti@yalis.fr>

vfs/help.go

@@ -259,4 +259,17 @@ The flag controls whether "fixup" is performed to satisfy the target.
 If the flag is not provided on the command line, then its default value depends
 on the operating system where rclone runs: "true" on Windows and macOS, "false"
 otherwise. If the flag is provided without a value, then it is "true".
+
+### Alternate report of used bytes
+
+Some backends, most notably S3, do not report the amount of bytes used.
+If you need this information to be available when running !df! on the
+filesystem, then pass the flag !--vfs-used-is-size! to rclone.
+With this flag set, instead of relying on the backend to report this
+information, rclone will scan the whole remote similar to !rclone size!
+and compute the total used space itself.
+
+_WARNING._ Contrary to !rclone size!, this flag ignores filters so that the
+result is accurate. However, this is very inefficient and may cost lots of API
+calls resulting in extra charges. Use it as a last resort and only with caching.
 `, "!", "`")

vfs/vfs.go

@@ -36,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/vfs/vfscache"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -556,9 +557,25 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 	defer vfs.usageMu.Unlock()
 	total, used, free = -1, -1, -1
 	doAbout := vfs.f.Features().About
-	if doAbout != nil && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
+	if (doAbout != nil || vfs.Opt.UsedIsSize) && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
 		var err error
-		vfs.usage, err = doAbout(context.TODO())
+		ctx := context.TODO()
+		if doAbout == nil {
+			vfs.usage = &fs.Usage{}
+		} else {
+			vfs.usage, err = doAbout(ctx)
+		}
+		if vfs.Opt.UsedIsSize {
+			var usedBySizeAlgorithm int64 = 0
+			// Algorithm from `rclone size`
+			err = walk.ListR(ctx, vfs.f, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+				entries.ForObject(func(o fs.Object) {
+					usedBySizeAlgorithm += o.Size()
+				})
+				return nil
+			})
+			vfs.usage.Used = &usedBySizeAlgorithm
+		}
 		vfs.usageTime = time.Now()
 		if err != nil {
 			fs.Errorf(vfs.f, "Statfs failed: %v", err)

vfs/vfscommon/options.go

@@ -32,6 +32,7 @@ type Options struct {
 	ReadWait          time.Duration // time to wait for in-sequence read
 	WriteBack         time.Duration // time to wait before writing back dirty files
 	ReadAhead         fs.SizeSuffix // bytes to read ahead in cache mode "full"
+	UsedIsSize        bool          // if true, use the `rclone size` algorithm for Used size
 }
 
 // DefaultOpt is the default values uses for Opt
@@ -58,4 +59,5 @@ var DefaultOpt = Options{
 	ReadWait:          20 * time.Millisecond,
 	WriteBack:         5 * time.Second,
 	ReadAhead:         0 * fs.MebiByte,
+	UsedIsSize:        false,
 }

vfs/vfsflags/vfsflags.go

@@ -37,5 +37,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
 	flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.")
 	flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.")
+	flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size.")
 	platformFlags(flagSet)
 }