allow the fuse directory cached to be cleaned manually (fixes #803)

cmd/mount/dir.go

@@ -5,6 +5,7 @@ package mount
 import (
 	"os"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
@@ -43,14 +44,58 @@ func newDir(f fs.Fs, fsDir *fs.Dir) *Dir {
 	}
 }
 
+// ForgetAll ensures the directory and all its children are purged
+// from the cache.
+func (d *Dir) ForgetAll() {
+	d.ForgetPath("")
+}
+
+// ForgetPath clears the cache for itself and all subdirectories if
+// they match the given path. The path is specified relative from the
+// directory it is called from.
+// It is not possible to traverse the directory tree upwards, i.e.
+// you cannot clear the cache for the Dir's ancestors or siblings.
+func (d *Dir) ForgetPath(relativePath string) {
+	absPath := path.Join(d.path, relativePath)
+	if absPath == "." {
+		absPath = ""
+	}
+
+	d.walk(absPath, func(dir *Dir) {
+		fs.Debugf(dir.path, "forgetting directory cache")
+		dir.read = time.Time{}
+		dir.items = nil
+	})
+}
+
+// walk runs a function on all directories whose path matches
+// the given absolute one. It will be called on a directory's
+// children first. It will not apply the function to parent
+// nodes, regardless of the given path.
+func (d *Dir) walk(absPath string, fun func(*Dir)) {
+	if d.items != nil {
+		for _, entry := range d.items {
+			if dir, ok := entry.node.(*Dir); ok {
+				dir.walk(absPath, fun)
+			}
+		}
+	}
+
+	if d.path == absPath || absPath == "" || strings.HasPrefix(d.path, absPath+"/") {
+		d.mu.Lock()
+		defer d.mu.Unlock()
+		fun(d)
+	}
+}
+
 // rename should be called after the directory is renamed
 //
 // Reset the directory to new state, discarding all the objects and
 // reading everything again
 func (d *Dir) rename(newParent *Dir, fsDir *fs.Dir) {
+	d.ForgetAll()
 	d.path = fsDir.Name
 	d.modTime = fsDir.When
-	d.items = nil
 	d.read = time.Time{}
 }
 

cmd/mount/dir_test.go

@@ -151,3 +151,69 @@ func TestDirModTime(t *testing.T) {
 
 	run.rmdir(t, "dir")
 }
+
+func TestDirCacheFlush(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.checkDir(t, "")
+
+	run.mkdir(t, "dir")
+	run.mkdir(t, "otherdir")
+	run.createFile(t, "dir/file", "1")
+	run.createFile(t, "otherdir/file", "1")
+
+	dm := newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1")
+	localDm := make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	err := run.fremote.Mkdir("dir/subdir")
+	require.NoError(t, err)
+
+	// expect newly created "subdir" on remote to not show up
+	run.mountFS.rootDir.ForgetPath("otherdir")
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	run.mountFS.rootDir.ForgetPath("dir")
+	dm = newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1|dir/subdir/")
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	run.rm(t, "otherdir/file")
+	run.rmdir(t, "otherdir")
+	run.rm(t, "dir/file")
+	run.rmdir(t, "dir/subdir")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+func TestDirCacheFlushOnDirRename(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	run.mkdir(t, "dir")
+	run.createFile(t, "dir/file", "1")
+
+	dm := newDirMap("dir/|dir/file 1")
+	localDm := make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	// expect remotely created directory to not show up
+	err := run.fremote.Mkdir("dir/subdir")
+	require.NoError(t, err)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	err = os.Rename(run.path("dir"), run.path("rid"))
+	require.NoError(t, err)
+
+	dm = newDirMap("rid/|rid/subdir/|rid/file 1")
+	localDm = make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	run.rm(t, "rid/file")
+	run.rmdir(t, "rid/subdir")
+	run.rmdir(t, "rid")
+	run.checkDir(t, "")
+}

cmd/mount/fs.go

@@ -5,6 +5,9 @@
 package mount
 
 import (
+	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"bazil.org/fuse"
@@ -15,7 +18,8 @@ import (
 
 // FS represents the top level filing system
 type FS struct {
-	f fs.Fs
+	f       fs.Fs
+	rootDir *Dir
 }
 
 // Check interface satistfied
@@ -24,11 +28,14 @@ var _ fusefs.FS = (*FS)(nil)
 // Root returns the root node
 func (f *FS) Root() (fusefs.Node, error) {
 	fs.Debugf(f.f, "Root()")
-	fsDir := &fs.Dir{
-		Name: "",
-		When: time.Now(),
+	if f.rootDir == nil {
+		fsDir := &fs.Dir{
+			Name: "",
+			When: time.Now(),
+		}
+		f.rootDir = newDir(f.f, fsDir)
 	}
-	return newDir(f.f, fsDir), nil
+	return f.rootDir, nil
 }
 
 // mountOptions configures the options from the command line flags
@@ -74,17 +81,17 @@ func mountOptions(device string) (options []fuse.MountOption) {
 //
 // returns an error, and an error channel for the serve process to
 // report an error when fusermount is called.
-func mount(f fs.Fs, mountpoint string) (<-chan error, error) {
+func mount(f fs.Fs, mountpoint string) (*FS, <-chan error, error) {
 	fs.Debugf(f, "Mounting on %q", mountpoint)
-	c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
-	if err != nil {
-		return nil, err
-	}
 
 	filesys := &FS{
 		f: f,
 	}
 
+	c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
+	if err != nil {
+		return filesys, nil, err
+	}
 	server := fusefs.New(c, nil)
 
 	// Serve the mount point in the background returning error to errChan
@@ -101,10 +108,11 @@ func mount(f fs.Fs, mountpoint string) (<-chan error, error) {
 	// check if the mount process has an error to report
 	<-c.Ready
 	if err := c.MountError; err != nil {
-		return nil, err
+		return filesys, nil, err
 	}
 
-	return errChan, nil
+	filesys.startSignalHandler()
+	return filesys, errChan, nil
 }
 
 // Check interface satsified
@@ -125,3 +133,16 @@ func (f *FS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.Sta
 	resp.Frsize = blockSize // Fragment size, smallest addressable data size in the file system.
 	return nil
 }
+
+func (f *FS) startSignalHandler() {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGHUP)
+	go func() {
+		for {
+			<-sigChan
+			if f.rootDir != nil {
+				f.rootDir.ForgetAll()
+			}
+		}
+	}()
+}

cmd/mount/fs_test.go

@@ -49,6 +49,7 @@ type Run struct {
 	fremoteName  string
 	cleanRemote  func()
 	umountResult <-chan error
+	mountFS      *FS
 	skip         bool
 }
 
@@ -102,7 +103,7 @@ func newRun() *Run {
 func (r *Run) mount() {
 	log.Printf("mount %q %q", r.fremote, r.mountPath)
 	var err error
-	r.umountResult, err = mount(r.fremote, r.mountPath)
+	r.mountFS, r.umountResult, err = mount(r.fremote, r.mountPath)
 	if err != nil {
 		log.Printf("mount failed: %v", err)
 		r.skip = true

cmd/mount/mount.go

@@ -122,6 +122,21 @@ mount won't do that, so will be less reliable than the rclone command.
 Note that all the rclone filters can be used to select a subset of the
 files to be visible in the mount.
 
+### Directory Cache ###
+
+Using the ` + "`--dir-cache-time`" + ` flag, you can set how long a
+directory should be considered up to date and not refreshed from the
+backend. Changes made locally in the mount may appear immediately or
+invalidate the cache. However, changes done on the remote will only
+be picked up once the cache expires.
+
+Alternatively, you can send a ` + "`SIGHUP`" + ` signal to rclone for
+it to flush all directory caches, regardless of how old they are.
+Assuming only one rlcone instance is running, you can reset the cache
+like this:
+
+    kill -SIGHUP $(pidof rclone)
+
 ### Bugs ###
 
   * All the remotes should work for read, but some may not for write
@@ -160,7 +175,7 @@ func Mount(f fs.Fs, mountpoint string) error {
 	}
 
 	// Mount it
-	errChan, err := mount(f, mountpoint)
+	_, errChan, err := mount(f, mountpoint)
 	if err != nil {
 		return errors.Wrap(err, "failed to mount FUSE fs")
 	}