fuse: forget fs.Node instances on request by the kernel

Forget fs.Node instances once the kernel frees the corresponding nodeId.
This ensures that restic does not run out of memory on large snapshots.

internal/fuse/dir.go

@@ -20,12 +20,14 @@ import (
 
 // Statically ensure that *dir implement those interface
 var _ = fs.HandleReadDirAller(&dir{})
+var _ = fs.NodeForgetter(&dir{})
 var _ = fs.NodeGetxattrer(&dir{})
 var _ = fs.NodeListxattrer(&dir{})
 var _ = fs.NodeStringLookuper(&dir{})
 
 type dir struct {
 	root        *Root
+	forget      forgetFn
 	items       map[string]*restic.Node
 	inode       uint64
 	parentInode uint64
@@ -38,11 +40,12 @@ func cleanupNodeName(name string) string {
 	return filepath.Base(name)
 }
 
-func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
+func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *restic.Node) (*dir, error) {
 	debug.Log("new dir for %v (%v)", node.Name, node.Subtree)
 
 	return &dir{
 		root:        root,
+		forget:      forget,
 		node:        node,
 		inode:       inode,
 		parentInode: parentInode,
@@ -79,10 +82,11 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *rest
 	return tree.Nodes, nil
 }
 
-func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
+func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
 	debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree)
 	return &dir{
-		root: root,
+		root:   root,
+		forget: forget,
 		node: &restic.Node{
 			AccessTime: snapshot.Time,
 			ModTime:    snapshot.Time,
@@ -213,7 +217,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, err
 	}
 
-	return d.cache.lookupOrCreate(name, func() (fs.Node, error) {
+	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
 		node, ok := d.items[name]
 		if !ok {
 			debug.Log("  Lookup(%v) -> not found", name)
@@ -222,13 +226,13 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		inode := inodeFromNode(d.inode, node)
 		switch node.Type {
 		case restic.NodeTypeDir:
-			return newDir(d.root, inode, d.inode, node)
+			return newDir(d.root, forget, inode, d.inode, node)
 		case restic.NodeTypeFile:
-			return newFile(d.root, inode, node)
+			return newFile(d.root, forget, inode, node)
 		case restic.NodeTypeSymlink:
-			return newLink(d.root, inode, node)
+			return newLink(d.root, forget, inode, node)
 		case restic.NodeTypeDev, restic.NodeTypeCharDev, restic.NodeTypeFifo, restic.NodeTypeSocket:
-			return newOther(d.root, inode, node)
+			return newOther(d.root, forget, inode, node)
 		default:
 			debug.Log("  node %v has unknown type %v", name, node.Type)
 			return nil, syscall.ENOENT
@@ -244,3 +248,7 @@ func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fus
 func (d *dir) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(d.node, req, resp)
 }
+
+func (d *dir) Forget() {
+	d.forget()
+}

internal/fuse/file.go

@@ -20,14 +20,16 @@ const blockSize = 512
 
 // Statically ensure that *file and *openFile implement the given interfaces
 var _ = fs.HandleReader(&openFile{})
-var _ = fs.NodeListxattrer(&file{})
+var _ = fs.NodeForgetter(&file{})
 var _ = fs.NodeGetxattrer(&file{})
+var _ = fs.NodeListxattrer(&file{})
 var _ = fs.NodeOpener(&file{})
 
 type file struct {
-	root  *Root
+	root   *Root
-	node  *restic.Node
+	forget forgetFn
-	inode uint64
+	node   *restic.Node
+	inode  uint64
 }
 
 type openFile struct {
@@ -36,12 +38,13 @@ type openFile struct {
 	cumsize []uint64
 }
 
-func newFile(root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
+func newFile(root *Root, forget forgetFn, inode uint64, node *restic.Node) (fusefile *file, err error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	return &file{
-		inode: inode,
+		inode:  inode,
-		root:  root,
+		forget: forget,
-		node:  node,
+		root:   root,
+		node:   node,
 	}, nil
 }
 
@@ -172,3 +175,7 @@ func (f *file) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
 func (f *file) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(f.node, req, resp)
 }
+
+func (f *file) Forget() {
+	f.forget()
+}

internal/fuse/fuse_test.go

@@ -119,7 +119,7 @@ func TestFuseFile(t *testing.T) {
 	root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
 
 	inode := inodeFromNode(1, node)
-	f, err := newFile(root, inode, node)
+	f, err := newFile(root, func() {}, inode, node)
 	rtest.OK(t, err)
 	of, err := f.Open(context.TODO(), nil, nil)
 	rtest.OK(t, err)
@@ -162,7 +162,7 @@ func TestFuseDir(t *testing.T) {
 	}
 	parentInode := inodeFromName(0, "parent")
 	inode := inodeFromName(1, "foo")
-	d, err := newDir(root, inode, parentInode, node)
+	d, err := newDir(root, func() {}, inode, parentInode, node)
 	rtest.OK(t, err)
 
 	// don't open the directory as that would require setting up a proper tree blob
@@ -276,7 +276,7 @@ func TestLink(t *testing.T) {
 		{Name: "foo", Value: []byte("bar")},
 	}}
 
-	lnk, err := newLink(&Root{}, 42, node)
+	lnk, err := newLink(&Root{}, func() {}, 42, node)
 	rtest.OK(t, err)
 	target, err := lnk.Readlink(context.TODO(), nil)
 	rtest.OK(t, err)

internal/fuse/link.go

@@ -12,18 +12,20 @@ import (
 )
 
 // Statically ensure that *link implements the given interface
+var _ = fs.NodeForgetter(&link{})
 var _ = fs.NodeGetxattrer(&link{})
 var _ = fs.NodeListxattrer(&link{})
 var _ = fs.NodeReadlinker(&link{})
 
 type link struct {
-	root  *Root
+	root   *Root
-	node  *restic.Node
+	forget forgetFn
-	inode uint64
+	node   *restic.Node
+	inode  uint64
 }
 
-func newLink(root *Root, inode uint64, node *restic.Node) (*link, error) {
+func newLink(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*link, error) {
-	return &link{root: root, inode: inode, node: node}, nil
+	return &link{root: root, forget: forget, inode: inode, node: node}, nil
 }
 
 func (l *link) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@@ -57,3 +59,7 @@ func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
 func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(l.node, req, resp)
 }
+
+func (l *link) Forget() {
+	l.forget()
+}

internal/fuse/other.go

@@ -12,16 +12,18 @@ import (
 )
 
 // Statically ensure that *other implements the given interface
+var _ = fs.NodeForgetter(&other{})
 var _ = fs.NodeReadlinker(&other{})
 
 type other struct {
-	root  *Root
+	root   *Root
-	node  *restic.Node
+	forget forgetFn
-	inode uint64
+	node   *restic.Node
+	inode  uint64
 }
 
-func newOther(root *Root, inode uint64, node *restic.Node) (*other, error) {
+func newOther(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*other, error) {
-	return &other{root: root, inode: inode, node: node}, nil
+	return &other{root: root, forget: forget, inode: inode, node: node}, nil
 }
 
 func (l *other) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@@ -44,3 +46,7 @@ func (l *other) Attr(_ context.Context, a *fuse.Attr) error {
 
 	return nil
 }
+
+func (l *other) Forget() {
+	l.forget()
+}

internal/fuse/root.go

@@ -66,7 +66,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root {
 		}
 	}
 
-	root.SnapshotsDir = NewSnapshotsDir(root, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
+	root.SnapshotsDir = NewSnapshotsDir(root, func() {}, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
 
 	return root
 }

internal/fuse/snapshots_dir.go

@@ -19,6 +19,7 @@ import (
 // It uses the saved prefix to select the corresponding MetaDirData.
 type SnapshotsDir struct {
 	root        *Root
+	forget      forgetFn
 	inode       uint64
 	parentInode uint64
 	dirStruct   *SnapshotsDirStructure
@@ -28,13 +29,15 @@ type SnapshotsDir struct {
 
 // ensure that *SnapshotsDir implements these interfaces
 var _ = fs.HandleReadDirAller(&SnapshotsDir{})
+var _ = fs.NodeForgetter(&SnapshotsDir{})
 var _ = fs.NodeStringLookuper(&SnapshotsDir{})
 
 // NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links
-func NewSnapshotsDir(root *Root, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
+func NewSnapshotsDir(root *Root, forget forgetFn, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
 	debug.Log("create snapshots dir, inode %d", inode)
 	return &SnapshotsDir{
 		root:        root,
+		forget:      forget,
 		inode:       inode,
 		parentInode: parentInode,
 		dirStruct:   dirStruct,
@@ -109,7 +112,7 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 		return nil, syscall.ENOENT
 	}
 
-	return d.cache.lookupOrCreate(name, func() (fs.Node, error) {
+	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
 		entry := meta.names[name]
 		if entry == nil {
 			return nil, syscall.ENOENT
@@ -117,27 +120,33 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 
 		inode := inodeFromName(d.inode, name)
 		if entry.linkTarget != "" {
-			return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
+			return newSnapshotLink(d.root, forget, inode, entry.linkTarget, entry.snapshot)
 		} else if entry.snapshot != nil {
-			return newDirFromSnapshot(d.root, inode, entry.snapshot)
+			return newDirFromSnapshot(d.root, forget, inode, entry.snapshot)
 		}
-		return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
+		return NewSnapshotsDir(d.root, forget, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 	})
 }
 
+func (d *SnapshotsDir) Forget() {
+	d.forget()
+}
+
 // SnapshotLink
 type snapshotLink struct {
 	root     *Root
+	forget   forgetFn
 	inode    uint64
 	target   string
 	snapshot *restic.Snapshot
 }
 
+var _ = fs.NodeForgetter(&snapshotLink{})
 var _ = fs.NodeReadlinker(&snapshotLink{})
 
 // newSnapshotLink
-func newSnapshotLink(root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
+func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
-	return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
+	return &snapshotLink{root: root, forget: forget, inode: inode, target: target, snapshot: snapshot}, nil
 }
 
 // Readlink
@@ -161,3 +170,7 @@ func (l *snapshotLink) Attr(_ context.Context, a *fuse.Attr) error {
 
 	return nil
 }
+
+func (l *snapshotLink) Forget() {
+	l.forget()
+}

internal/fuse/tree_cache.go

@@ -14,13 +14,15 @@ type treeCache struct {
 	m     sync.Mutex
 }
 
+type forgetFn func()
+
 func newTreeCache() *treeCache {
 	return &treeCache{
 		nodes: map[string]fs.Node{},
 	}
 }
 
-func (t *treeCache) lookupOrCreate(name string, create func() (fs.Node, error)) (fs.Node, error) {
+func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
 	t.m.Lock()
 	defer t.m.Unlock()
 
@@ -28,7 +30,12 @@ func (t *treeCache) lookupOrCreate(name string, create func() (fs.Node, error))
 		return node, nil
 	}
 
-	node, err := create()
+	node, err := create(func() {
+		t.m.Lock()
+		defer t.m.Unlock()
+
+		delete(t.nodes, name)
+	})
 	if err != nil {
 		return nil, err
 	}