From 58a82cd5780a91cf3adfb043d476a752688d4f32 Mon Sep 17 00:00:00 2001 From: Stefan Breunig Date: Thu, 4 May 2017 22:49:06 +0200 Subject: [PATCH] allow the fuse directory cached to be cleaned manually (fixes #803) --- cmd/mount/dir.go | 47 +++++++++++++++++++++++++++++- cmd/mount/dir_test.go | 66 +++++++++++++++++++++++++++++++++++++++++++ cmd/mount/fs.go | 45 +++++++++++++++++++++-------- cmd/mount/fs_test.go | 3 +- cmd/mount/mount.go | 17 ++++++++++- 5 files changed, 163 insertions(+), 15 deletions(-) diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index 0860e4fa9..4c7b02340 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -5,6 +5,7 @@ package mount import ( "os" "path" + "strings" "sync" "time" @@ -43,14 +44,58 @@ func newDir(f fs.Fs, fsDir *fs.Dir) *Dir { } } +// ForgetAll ensures the directory and all its children are purged +// from the cache. +func (d *Dir) ForgetAll() { + d.ForgetPath("") +} + +// ForgetPath clears the cache for itself and all subdirectories if +// they match the given path. The path is specified relative from the +// directory it is called from. +// It is not possible to traverse the directory tree upwards, i.e. +// you cannot clear the cache for the Dir's ancestors or siblings. +func (d *Dir) ForgetPath(relativePath string) { + absPath := path.Join(d.path, relativePath) + if absPath == "." { + absPath = "" + } + + d.walk(absPath, func(dir *Dir) { + fs.Debugf(dir.path, "forgetting directory cache") + dir.read = time.Time{} + dir.items = nil + }) +} + +// walk runs a function on all directories whose path matches +// the given absolute one. It will be called on a directory's +// children first. It will not apply the function to parent +// nodes, regardless of the given path. +func (d *Dir) walk(absPath string, fun func(*Dir)) { + if d.items != nil { + for _, entry := range d.items { + if dir, ok := entry.node.(*Dir); ok { + dir.walk(absPath, fun) + } + } + } + + if d.path == absPath || absPath == "" || strings.HasPrefix(d.path, absPath+"/") { + d.mu.Lock() + defer d.mu.Unlock() + fun(d) + } +} + // rename should be called after the directory is renamed // // Reset the directory to new state, discarding all the objects and // reading everything again func (d *Dir) rename(newParent *Dir, fsDir *fs.Dir) { + d.ForgetAll() d.path = fsDir.Name d.modTime = fsDir.When - d.items = nil d.read = time.Time{} } diff --git a/cmd/mount/dir_test.go b/cmd/mount/dir_test.go index 3833438e3..50fe33447 100644 --- a/cmd/mount/dir_test.go +++ b/cmd/mount/dir_test.go @@ -151,3 +151,69 @@ func TestDirModTime(t *testing.T) { run.rmdir(t, "dir") } + +func TestDirCacheFlush(t *testing.T) { + run.skipIfNoFUSE(t) + + run.checkDir(t, "") + + run.mkdir(t, "dir") + run.mkdir(t, "otherdir") + run.createFile(t, "dir/file", "1") + run.createFile(t, "otherdir/file", "1") + + dm := newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1") + localDm := make(dirMap) + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + err := run.fremote.Mkdir("dir/subdir") + require.NoError(t, err) + + // expect newly created "subdir" on remote to not show up + run.mountFS.rootDir.ForgetPath("otherdir") + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + run.mountFS.rootDir.ForgetPath("dir") + dm = newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1|dir/subdir/") + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + run.rm(t, "otherdir/file") + run.rmdir(t, "otherdir") + run.rm(t, "dir/file") + run.rmdir(t, "dir/subdir") + run.rmdir(t, "dir") + run.checkDir(t, "") +} + +func TestDirCacheFlushOnDirRename(t *testing.T) { + run.skipIfNoFUSE(t) + run.mkdir(t, "dir") + run.createFile(t, "dir/file", "1") + + dm := newDirMap("dir/|dir/file 1") + localDm := make(dirMap) + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + // expect remotely created directory to not show up + err := run.fremote.Mkdir("dir/subdir") + require.NoError(t, err) + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + err = os.Rename(run.path("dir"), run.path("rid")) + require.NoError(t, err) + + dm = newDirMap("rid/|rid/subdir/|rid/file 1") + localDm = make(dirMap) + run.readLocal(t, localDm, "") + assert.Equal(t, dm, localDm, "expected vs fuse mount") + + run.rm(t, "rid/file") + run.rmdir(t, "rid/subdir") + run.rmdir(t, "rid") + run.checkDir(t, "") +} diff --git a/cmd/mount/fs.go b/cmd/mount/fs.go index 5b75c1972..e018a3c2a 100644 --- a/cmd/mount/fs.go +++ b/cmd/mount/fs.go @@ -5,6 +5,9 @@ package mount import ( + "os" + "os/signal" + "syscall" "time" "bazil.org/fuse" @@ -15,7 +18,8 @@ import ( // FS represents the top level filing system type FS struct { - f fs.Fs + f fs.Fs + rootDir *Dir } // Check interface satistfied @@ -24,11 +28,14 @@ var _ fusefs.FS = (*FS)(nil) // Root returns the root node func (f *FS) Root() (fusefs.Node, error) { fs.Debugf(f.f, "Root()") - fsDir := &fs.Dir{ - Name: "", - When: time.Now(), + if f.rootDir == nil { + fsDir := &fs.Dir{ + Name: "", + When: time.Now(), + } + f.rootDir = newDir(f.f, fsDir) } - return newDir(f.f, fsDir), nil + return f.rootDir, nil } // mountOptions configures the options from the command line flags @@ -74,17 +81,17 @@ func mountOptions(device string) (options []fuse.MountOption) { // // returns an error, and an error channel for the serve process to // report an error when fusermount is called. -func mount(f fs.Fs, mountpoint string) (<-chan error, error) { +func mount(f fs.Fs, mountpoint string) (*FS, <-chan error, error) { fs.Debugf(f, "Mounting on %q", mountpoint) - c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...) - if err != nil { - return nil, err - } filesys := &FS{ f: f, } + c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...) + if err != nil { + return filesys, nil, err + } server := fusefs.New(c, nil) // Serve the mount point in the background returning error to errChan @@ -101,10 +108,11 @@ func mount(f fs.Fs, mountpoint string) (<-chan error, error) { // check if the mount process has an error to report <-c.Ready if err := c.MountError; err != nil { - return nil, err + return filesys, nil, err } - return errChan, nil + filesys.startSignalHandler() + return filesys, errChan, nil } // Check interface satsified @@ -125,3 +133,16 @@ func (f *FS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.Sta resp.Frsize = blockSize // Fragment size, smallest addressable data size in the file system. return nil } + +func (f *FS) startSignalHandler() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGHUP) + go func() { + for { + <-sigChan + if f.rootDir != nil { + f.rootDir.ForgetAll() + } + } + }() +} diff --git a/cmd/mount/fs_test.go b/cmd/mount/fs_test.go index a58d4d7b3..53a722d2f 100644 --- a/cmd/mount/fs_test.go +++ b/cmd/mount/fs_test.go @@ -49,6 +49,7 @@ type Run struct { fremoteName string cleanRemote func() umountResult <-chan error + mountFS *FS skip bool } @@ -102,7 +103,7 @@ func newRun() *Run { func (r *Run) mount() { log.Printf("mount %q %q", r.fremote, r.mountPath) var err error - r.umountResult, err = mount(r.fremote, r.mountPath) + r.mountFS, r.umountResult, err = mount(r.fremote, r.mountPath) if err != nil { log.Printf("mount failed: %v", err) r.skip = true diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go index d21a177b6..6a6212cdb 100644 --- a/cmd/mount/mount.go +++ b/cmd/mount/mount.go @@ -122,6 +122,21 @@ mount won't do that, so will be less reliable than the rclone command. Note that all the rclone filters can be used to select a subset of the files to be visible in the mount. +### Directory Cache ### + +Using the ` + "`--dir-cache-time`" + ` flag, you can set how long a +directory should be considered up to date and not refreshed from the +backend. Changes made locally in the mount may appear immediately or +invalidate the cache. However, changes done on the remote will only +be picked up once the cache expires. + +Alternatively, you can send a ` + "`SIGHUP`" + ` signal to rclone for +it to flush all directory caches, regardless of how old they are. +Assuming only one rlcone instance is running, you can reset the cache +like this: + + kill -SIGHUP $(pidof rclone) + ### Bugs ### * All the remotes should work for read, but some may not for write @@ -160,7 +175,7 @@ func Mount(f fs.Fs, mountpoint string) error { } // Mount it - errChan, err := mount(f, mountpoint) + _, errChan, err := mount(f, mountpoint) if err != nil { return errors.Wrap(err, "failed to mount FUSE fs") }