vfs: --vfs-used-is-size to report used space using recursive scan (#4043)
Some backends, most notably S3, do not report the amount of bytes used. This patch introduces a new flag that allows instead of relying on the backend, use recursive scan similar to `rclone size` to compute the total used space. However, this is ineffective and should be used as a last resort. Co-authored-by: Yves G <theYinYeti@yalis.fr>
This commit is contained in:
parent
297ca23abd
commit
c0cf54067a
4 changed files with 35 additions and 2 deletions
13
vfs/help.go
13
vfs/help.go
|
@ -259,4 +259,17 @@ The flag controls whether "fixup" is performed to satisfy the target.
|
||||||
If the flag is not provided on the command line, then its default value depends
|
If the flag is not provided on the command line, then its default value depends
|
||||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||||
otherwise. If the flag is provided without a value, then it is "true".
|
otherwise. If the flag is provided without a value, then it is "true".
|
||||||
|
|
||||||
|
### Alternate report of used bytes
|
||||||
|
|
||||||
|
Some backends, most notably S3, do not report the amount of bytes used.
|
||||||
|
If you need this information to be available when running !df! on the
|
||||||
|
filesystem, then pass the flag !--vfs-used-is-size! to rclone.
|
||||||
|
With this flag set, instead of relying on the backend to report this
|
||||||
|
information, rclone will scan the whole remote similar to !rclone size!
|
||||||
|
and compute the total used space itself.
|
||||||
|
|
||||||
|
_WARNING._ Contrary to !rclone size!, this flag ignores filters so that the
|
||||||
|
result is accurate. However, this is very inefficient and may cost lots of API
|
||||||
|
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||||
`, "!", "`")
|
`, "!", "`")
|
||||||
|
|
21
vfs/vfs.go
21
vfs/vfs.go
|
@ -36,6 +36,7 @@ import (
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
"github.com/rclone/rclone/fs/cache"
|
"github.com/rclone/rclone/fs/cache"
|
||||||
"github.com/rclone/rclone/fs/log"
|
"github.com/rclone/rclone/fs/log"
|
||||||
|
"github.com/rclone/rclone/fs/walk"
|
||||||
"github.com/rclone/rclone/vfs/vfscache"
|
"github.com/rclone/rclone/vfs/vfscache"
|
||||||
"github.com/rclone/rclone/vfs/vfscommon"
|
"github.com/rclone/rclone/vfs/vfscommon"
|
||||||
)
|
)
|
||||||
|
@ -556,9 +557,25 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
|
||||||
defer vfs.usageMu.Unlock()
|
defer vfs.usageMu.Unlock()
|
||||||
total, used, free = -1, -1, -1
|
total, used, free = -1, -1, -1
|
||||||
doAbout := vfs.f.Features().About
|
doAbout := vfs.f.Features().About
|
||||||
if doAbout != nil && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
|
if (doAbout != nil || vfs.Opt.UsedIsSize) && (vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime) {
|
||||||
var err error
|
var err error
|
||||||
vfs.usage, err = doAbout(context.TODO())
|
ctx := context.TODO()
|
||||||
|
if doAbout == nil {
|
||||||
|
vfs.usage = &fs.Usage{}
|
||||||
|
} else {
|
||||||
|
vfs.usage, err = doAbout(ctx)
|
||||||
|
}
|
||||||
|
if vfs.Opt.UsedIsSize {
|
||||||
|
var usedBySizeAlgorithm int64 = 0
|
||||||
|
// Algorithm from `rclone size`
|
||||||
|
err = walk.ListR(ctx, vfs.f, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||||
|
entries.ForObject(func(o fs.Object) {
|
||||||
|
usedBySizeAlgorithm += o.Size()
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
vfs.usage.Used = &usedBySizeAlgorithm
|
||||||
|
}
|
||||||
vfs.usageTime = time.Now()
|
vfs.usageTime = time.Now()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fs.Errorf(vfs.f, "Statfs failed: %v", err)
|
fs.Errorf(vfs.f, "Statfs failed: %v", err)
|
||||||
|
|
|
@ -32,6 +32,7 @@ type Options struct {
|
||||||
ReadWait time.Duration // time to wait for in-sequence read
|
ReadWait time.Duration // time to wait for in-sequence read
|
||||||
WriteBack time.Duration // time to wait before writing back dirty files
|
WriteBack time.Duration // time to wait before writing back dirty files
|
||||||
ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full"
|
ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full"
|
||||||
|
UsedIsSize bool // if true, use the `rclone size` algorithm for Used size
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultOpt is the default values uses for Opt
|
// DefaultOpt is the default values uses for Opt
|
||||||
|
@ -58,4 +59,5 @@ var DefaultOpt = Options{
|
||||||
ReadWait: 20 * time.Millisecond,
|
ReadWait: 20 * time.Millisecond,
|
||||||
WriteBack: 5 * time.Second,
|
WriteBack: 5 * time.Second,
|
||||||
ReadAhead: 0 * fs.MebiByte,
|
ReadAhead: 0 * fs.MebiByte,
|
||||||
|
UsedIsSize: false,
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,5 +37,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
|
||||||
flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
|
flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
|
||||||
flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.")
|
flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.")
|
||||||
flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.")
|
flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full.")
|
||||||
|
flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size.")
|
||||||
platformFlags(flagSet)
|
platformFlags(flagSet)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue