KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

fuse: forget fs.Node instances on request by the kernel

Browse source 
Forget fs.Node instances once the kernel frees the corresponding nodeId.
This ensures that restic does not run out of memory on large snapshots.
This commit is contained in:
Michael Eischer 2024-09-09 22:37:51 +02:00
parent e9940f39dc
commit 51173c5003
8 changed files with 86 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
internal/fuse/dir.go
View file

@ -20,12 +20,14 @@ import (
// Statically ensure that *dir implement those interface
var _ = fs.HandleReadDirAller(&dir{})
var _ = fs.NodeForgetter(&dir{})
var _ = fs.NodeGetxattrer(&dir{})
var _ = fs.NodeListxattrer(&dir{})
var _ = fs.NodeStringLookuper(&dir{})
type dir struct {
root *Root
forget forgetFn
items map[string]*restic.Node
inode uint64
parentInode uint64
@ -38,11 +40,12 @@ func cleanupNodeName(name string) string {
return filepath.Base(name)
}
func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *restic.Node) (*dir, error) {
debug.Log("new dir for %v (%v)", node.Name, node.Subtree)
return &dir{
root: root,
forget: forget,
node: node,
inode: inode,
parentInode: parentInode,
@ -79,10 +82,11 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *rest
return tree.Nodes, nil
}
func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree)
return &dir{
root: root,
forget: forget,
node: &restic.Node{
AccessTime: snapshot.Time,
ModTime: snapshot.Time,
@ -213,7 +217,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, err
}
return d.cache.lookupOrCreate(name, func() (fs.Node, error) {
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
node, ok := d.items[name]
if !ok {
debug.Log(" Lookup(%v) -> not found", name)
@ -222,13 +226,13 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
inode := inodeFromNode(d.inode, node)
switch node.Type {
case restic.NodeTypeDir:
return newDir(d.root, inode, d.inode, node)
return newDir(d.root, forget, inode, d.inode, node)
case restic.NodeTypeFile:
return newFile(d.root, inode, node)
return newFile(d.root, forget, inode, node)
case restic.NodeTypeSymlink:
return newLink(d.root, inode, node)
return newLink(d.root, forget, inode, node)
case restic.NodeTypeDev, restic.NodeTypeCharDev, restic.NodeTypeFifo, restic.NodeTypeSocket:
return newOther(d.root, inode, node)
return newOther(d.root, forget, inode, node)
default:
debug.Log(" node %v has unknown type %v", name, node.Type)
return nil, syscall.ENOENT
@ -244,3 +248,7 @@ func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fus
func (d *dir) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(d.node, req, resp)
}
func (d *dir) Forget() {
d.forget()
}

11
internal/fuse/file.go
View file

@ -20,12 +20,14 @@ const blockSize = 512
// Statically ensure that *file and *openFile implement the given interfaces
var _ = fs.HandleReader(&openFile{})
var _ = fs.NodeListxattrer(&file{})
var _ = fs.NodeForgetter(&file{})
var _ = fs.NodeGetxattrer(&file{})
var _ = fs.NodeListxattrer(&file{})
var _ = fs.NodeOpener(&file{})
type file struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
@ -36,10 +38,11 @@ type openFile struct {
cumsize []uint64
}
func newFile(root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
func newFile(root *Root, forget forgetFn, inode uint64, node *restic.Node) (fusefile *file, err error) {
debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
return &file{
inode: inode,
forget: forget,
root: root,
node: node,
}, nil
@ -172,3 +175,7 @@ func (f *file) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
func (f *file) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(f.node, req, resp)
}
func (f *file) Forget() {
f.forget()
}

6
internal/fuse/fuse_test.go
View file

@ -119,7 +119,7 @@ func TestFuseFile(t *testing.T) {
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
inode := inodeFromNode(1, node)
f, err := newFile(root, inode, node)
f, err := newFile(root, func() {}, inode, node)
rtest.OK(t, err)
of, err := f.Open(context.TODO(), nil, nil)
rtest.OK(t, err)
@ -162,7 +162,7 @@ func TestFuseDir(t *testing.T) {
}
parentInode := inodeFromName(0, "parent")
inode := inodeFromName(1, "foo")
d, err := newDir(root, inode, parentInode, node)
d, err := newDir(root, func() {}, inode, parentInode, node)
rtest.OK(t, err)
// don't open the directory as that would require setting up a proper tree blob
@ -276,7 +276,7 @@ func TestLink(t *testing.T) {
{Name: "foo", Value: []byte("bar")},
}}
lnk, err := newLink(&Root{}, 42, node)
lnk, err := newLink(&Root{}, func() {}, 42, node)
rtest.OK(t, err)
target, err := lnk.Readlink(context.TODO(), nil)
rtest.OK(t, err)

10
internal/fuse/link.go
View file

@ -12,18 +12,20 @@ import (
)
// Statically ensure that *link implements the given interface
var _ = fs.NodeForgetter(&link{})
var _ = fs.NodeGetxattrer(&link{})
var _ = fs.NodeListxattrer(&link{})
var _ = fs.NodeReadlinker(&link{})
type link struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
func newLink(root *Root, inode uint64, node *restic.Node) (*link, error) {
return &link{root: root, inode: inode, node: node}, nil
func newLink(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*link, error) {
return &link{root: root, forget: forget, inode: inode, node: node}, nil
}
func (l *link) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@ -57,3 +59,7 @@ func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
return nodeGetXattr(l.node, req, resp)
}
func (l *link) Forget() {
l.forget()
}

10
internal/fuse/other.go
View file

@ -12,16 +12,18 @@ import (
)
// Statically ensure that *other implements the given interface
var _ = fs.NodeForgetter(&other{})
var _ = fs.NodeReadlinker(&other{})
type other struct {
root *Root
forget forgetFn
node *restic.Node
inode uint64
}
func newOther(root *Root, inode uint64, node *restic.Node) (*other, error) {
return &other{root: root, inode: inode, node: node}, nil
func newOther(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*other, error) {
return &other{root: root, forget: forget, inode: inode, node: node}, nil
}
func (l *other) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@ -44,3 +46,7 @@ func (l *other) Attr(_ context.Context, a *fuse.Attr) error {
return nil
}
func (l *other) Forget() {
l.forget()
}

2
internal/fuse/root.go
View file

@ -66,7 +66,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root {
}
}
root.SnapshotsDir = NewSnapshotsDir(root, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
root.SnapshotsDir = NewSnapshotsDir(root, func() {}, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
return root
}

27
internal/fuse/snapshots_dir.go
View file

@ -19,6 +19,7 @@ import (
// It uses the saved prefix to select the corresponding MetaDirData.
type SnapshotsDir struct {
root *Root
forget forgetFn
inode uint64
parentInode uint64
dirStruct *SnapshotsDirStructure
@ -28,13 +29,15 @@ type SnapshotsDir struct {
// ensure that *SnapshotsDir implements these interfaces
var _ = fs.HandleReadDirAller(&SnapshotsDir{})
var _ = fs.NodeForgetter(&SnapshotsDir{})
var _ = fs.NodeStringLookuper(&SnapshotsDir{})
// NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links
func NewSnapshotsDir(root *Root, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
func NewSnapshotsDir(root *Root, forget forgetFn, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
debug.Log("create snapshots dir, inode %d", inode)
return &SnapshotsDir{
root: root,
forget: forget,
inode: inode,
parentInode: parentInode,
dirStruct: dirStruct,
@ -109,7 +112,7 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
return nil, syscall.ENOENT
}
return d.cache.lookupOrCreate(name, func() (fs.Node, error) {
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
entry := meta.names[name]
if entry == nil {
return nil, syscall.ENOENT
@ -117,27 +120,33 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
inode := inodeFromName(d.inode, name)
if entry.linkTarget != "" {
return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
return newSnapshotLink(d.root, forget, inode, entry.linkTarget, entry.snapshot)
} else if entry.snapshot != nil {
return newDirFromSnapshot(d.root, inode, entry.snapshot)
return newDirFromSnapshot(d.root, forget, inode, entry.snapshot)
}
return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
return NewSnapshotsDir(d.root, forget, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
})
}
func (d *SnapshotsDir) Forget() {
d.forget()
}
// SnapshotLink
type snapshotLink struct {
root *Root
forget forgetFn
inode uint64
target string
snapshot *restic.Snapshot
}
var _ = fs.NodeForgetter(&snapshotLink{})
var _ = fs.NodeReadlinker(&snapshotLink{})
// newSnapshotLink
func newSnapshotLink(root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, forget: forget, inode: inode, target: target, snapshot: snapshot}, nil
}
// Readlink
@ -161,3 +170,7 @@ func (l *snapshotLink) Attr(_ context.Context, a *fuse.Attr) error {
return nil
}
func (l *snapshotLink) Forget() {
l.forget()
}

11
internal/fuse/tree_cache.go
View file

@ -14,13 +14,15 @@ type treeCache struct {
m sync.Mutex
}
type forgetFn func()
func newTreeCache() *treeCache {
return &treeCache{
nodes: map[string]fs.Node{},
}
}
func (t *treeCache) lookupOrCreate(name string, create func() (fs.Node, error)) (fs.Node, error) {
func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
t.m.Lock()
defer t.m.Unlock()
@ -28,7 +30,12 @@ func (t *treeCache) lookupOrCreate(name string, create func() (fs.Node, error))
return node, nil
}
node, err := create()
node, err := create(func() {
t.m.Lock()
defer t.m.Unlock()
delete(t.nodes, name)
})
if err != nil {
return nil, err
}