From fc32fee4add93e7a3bda180c7c1088f93c759d13 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 2 Mar 2018 16:39:42 +0000 Subject: [PATCH] mount, cmount: add --attr-timeout to control attribute caching in kernel This flag allows the attribute caching in the kernel to be controlled. The default is 0s - no caching - which is recommended for filesystems which can change outside the control of the kernel. Previously this was at the default meaning it was 60s for mount and 1s for cmount. This showed strange effects when files changed on the remote not via the kernel. For instance Caddy would serve corrupted files for a while when serving from an rclone mount when a file changed on the remote. --- cmd/cmount/mount.go | 1 + cmd/mount/dir.go | 3 +++ cmd/mount/file.go | 2 ++ cmd/mountlib/mount.go | 17 +++++++++++++++++ 4 files changed, 23 insertions(+) diff --git a/cmd/cmount/mount.go b/cmd/cmount/mount.go index c87a5c515..325dfe175 100644 --- a/cmd/cmount/mount.go +++ b/cmd/cmount/mount.go @@ -40,6 +40,7 @@ func mountOptions(device string, mountpoint string) (options []string) { "-o", "fsname=" + device, "-o", "subtype=rclone", "-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead), + "-o", fmt.Sprintf("attr_timeout=%g", mountlib.AttrTimeout.Seconds()), // This causes FUSE to supply O_TRUNC with the Open // call which is more efficient for cmount. However // it does not work with cgofuse on Windows with diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index 5f3600717..939935678 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -8,6 +8,7 @@ import ( "bazil.org/fuse" fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/cmd/mountlib" "github.com/ncw/rclone/fs/log" "github.com/ncw/rclone/vfs" "github.com/pkg/errors" @@ -25,6 +26,7 @@ var _ fusefs.Node = (*Dir)(nil) // Attr updates the attributes of a directory func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) { defer log.Trace(d, "")("attr=%+v, err=%v", a, &err) + a.Valid = mountlib.AttrTimeout a.Gid = d.VFS().Opt.GID a.Uid = d.VFS().Opt.UID a.Mode = os.ModeDir | d.VFS().Opt.DirPerms @@ -72,6 +74,7 @@ func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.Lo if err != nil { return nil, translateError(err) } + resp.EntryValid = mountlib.AttrTimeout switch x := mnode.(type) { case *vfs.File: return &File{x}, nil diff --git a/cmd/mount/file.go b/cmd/mount/file.go index f4a10f789..7632203fe 100644 --- a/cmd/mount/file.go +++ b/cmd/mount/file.go @@ -8,6 +8,7 @@ import ( "bazil.org/fuse" fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/cmd/mountlib" "github.com/ncw/rclone/fs/log" "github.com/ncw/rclone/vfs" "golang.org/x/net/context" @@ -24,6 +25,7 @@ var _ fusefs.Node = (*File)(nil) // Attr fills out the attributes for the file func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) { defer log.Trace(f, "")("a=%+v, err=%v", a, &err) + a.Valid = mountlib.AttrTimeout modTime := f.File.ModTime() Size := uint64(f.File.Size()) Blocks := (Size + 511) / 512 diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go index 980106ce7..7327022b8 100644 --- a/cmd/mountlib/mount.go +++ b/cmd/mountlib/mount.go @@ -5,6 +5,7 @@ import ( "log" "os" "runtime" + "time" "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/fs" @@ -27,6 +28,7 @@ var ( MaxReadAhead fs.SizeSuffix = 128 * 1024 ExtraOptions []string ExtraFlags []string + AttrTimeout = 0 * time.Second // how long the kernel caches attribute for ) // Check is folder is empty @@ -143,6 +145,20 @@ can't use retries in the same way without making local copies of the uploads. Look at the **EXPERIMENTAL** [file caching](#file-caching) for solutions to make ` + commandName + ` mount more reliable. +### Attribute caching + +You can use the flag --attr-timeout to set the time the kernel caches +the attributes (size, modification time etc) for directory entries. + +The default is 0s - no caching - which is recommended for filesystems +which can change outside the control of the kernel. + +If you set it higher ('1s' or '1m' say) then the kernel will call back +to rclone less often making it more efficient, however there may be +strange effects when files change on the remote. + +This is the same as setting the attr_timeout option in mount.fuse. + ### Filters Note that all the rclone filters can be used to select a subset of the @@ -203,6 +219,7 @@ will see all files and folders immediately in this mode. flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.") flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.") flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.") + flags.DurationVarP(flagSet, &AttrTimeout, "attr-timeout", "", AttrTimeout, "Time for which file/directory attributes are cached.") flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.") flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.") flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")