KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

fuse: Redesign snapshot dirstruct

Browse source 
Cleanly separate the directory presentation and the snapshot directory
structure. SnapshotsDir now translates the dirStruct into a format
usable by the fuse library and contains only minimal special case rules.
All decisions have moved into SnapshotsDirStructure which now creates a
fully preassembled tree data structure.
This commit is contained in:
Michael Eischer 2022-07-30 20:45:51 +02:00
parent 2db7733ee3
commit caa17988a3
3 changed files with 327 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
internal/fuse/snapshots_dir.go
View file

@ -6,7 +6,6 @@ package fuse
import (
"context"
"os"
"strings"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@ -16,8 +15,7 @@ import (
)
// SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure
// It uses the saved prefix to filter out the relevant subtrees or entries
// from SnapshotsDirStructure.names and .latest, respectively.
// It uses the saved prefix to select the corresponding MetaDirData.
type SnapshotsDir struct {
root *Root
inode uint64
@ -56,9 +54,11 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
// update snapshots
err := d.dirStruct.updateSnapshots(ctx)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, err
} else if meta == nil {
return nil, fuse.ENOENT
}
items := []fuse.Dirent{
@ -74,35 +74,16 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
},
}
// map to ensure that all names are only listed once
hasName := make(map[string]struct{})
for name := range d.dirStruct.names {
if !strings.HasPrefix(name, d.prefix) {
continue
}
shortname := strings.Split(name[len(d.prefix):], "/")[0]
if shortname == "" {
continue
}
if _, ok := hasName[shortname]; ok {
continue
}
hasName[shortname] = struct{}{}
items = append(items, fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, shortname),
Name: shortname,
for name, entry := range meta.names {
d := fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, name),
Name: name,
Type: fuse.DT_Dir,
})
}
// Latest
if _, ok := d.dirStruct.latest[d.prefix]; ok {
items = append(items, fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, "latest"),
Name: "latest",
Type: fuse.DT_Link,
})
}
if entry.linkTarget != "" {
d.Type = fuse.DT_Link
}
items = append(items, d)
}
return items, nil
@ -112,33 +93,21 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
err := d.dirStruct.updateSnapshots(ctx)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, err
} else if meta == nil {
return nil, fuse.ENOENT
}
fullname := d.prefix + name
// check if this is already a complete snapshot path
sn := d.dirStruct.names[fullname]
if sn != nil {
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
}
// handle latest case
if name == "latest" {
link := d.dirStruct.latest[d.prefix]
sn := d.dirStruct.names[d.prefix+link]
if sn != nil {
return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), link, sn)
}
}
// check if this is a valid subdir
fullname = fullname + "/"
for name := range d.dirStruct.names {
if strings.HasPrefix(name, fullname) {
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, fullname), nil
entry := meta.names[name]
if entry != nil {
if entry.linkTarget != "" {
return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.linkTarget, entry.snapshot)
} else if entry.snapshot != nil {
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.snapshot)
} else {
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, d.prefix+"/"+name), nil
}
}

153
internal/fuse/snapshots_dirstruct.go
View file

@ -9,12 +9,21 @@ import (
"path"
"sort"
"strings"
"sync"
"time"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
type MetaDirData struct {
// set if this is a symlink or a snapshot mount point
linkTarget string
snapshot *restic.Snapshot
// names is set if this is a pseudo directory
names map[string]*MetaDirData
}
// SnapshotsDirStructure contains the directory structure for snapshots.
// It uses a paths and time template to generate a map of pathnames
// pointing to the actual snapshots. For templates that end with a time,
@ -24,8 +33,12 @@ type SnapshotsDirStructure struct {
pathTemplates []string
timeTemplate string
names map[string]*restic.Snapshot
latest map[string]string
mutex sync.Mutex
// "" is the root path, subdirectory paths are assembled as parent+"/"+childFn
// thus all subdirectories are prefixed with a slash as the root is ""
// that way we don't need path processing special cases when using the entries tree
entries map[string]*MetaDirData
snCount int
lastCheck time.Time
}
@ -40,19 +53,6 @@ func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate s
}
}
// uniqueName returns a unique name to be used for prefix+name.
// It appends -number to make the name unique.
func (d *SnapshotsDirStructure) uniqueName(prefix, name string) (newname string) {
newname = name
for i := 1; ; i++ {
if _, ok := d.names[prefix+newname]; !ok {
break
}
newname = fmt.Sprintf("%s-%d", name, i)
}
return newname
}
// pathsFromSn generates the paths from pathTemplate and timeTemplate
// where the variables are replaced by the snapshot data.
// The time is given as suffix if the pathTemplate ends with "%T".
@ -90,6 +90,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot)
continue
case 't':
if len(sn.Tags) == 0 {
return nil, ""
}
if len(sn.Tags) != 1 {
// needs special treatment: Rebuild the string builders
newout := make([]strings.Builder, len(out)*len(sn.Tags))
@ -114,6 +117,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot)
case 'h':
repl = sn.Hostname
default:
repl = string(c)
}
// write replacement string to all string builders
@ -133,20 +139,102 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot)
return paths, timeSuffix
}
// determine static path prefix
func staticPrefix(pathTemplate string) (prefix string) {
inVerb := false
patternStart := -1
outer:
for i, c := range pathTemplate {
if !inVerb {
if c == '%' {
inVerb = true
}
continue
}
inVerb = false
switch c {
case 'i', 'I', 'u', 'h', 't', 'T':
patternStart = i
break outer
}
}
if patternStart < 0 {
// ignore patterns without template variable
return ""
}
p := pathTemplate[:patternStart]
idx := strings.LastIndex(p, "/")
if idx < 0 {
return ""
}
return p[:idx]
}
// uniqueName returns a unique name to be used for prefix+name.
// It appends -number to make the name unique.
func uniqueName(entries map[string]*MetaDirData, prefix, name string) string {
newname := name
for i := 1; ; i++ {
if _, ok := entries[prefix+newname]; !ok {
break
}
newname = fmt.Sprintf("%s-%d", name, i)
}
return newname
}
// makeDirs inserts all paths generated from pathTemplates and
// TimeTemplate for all given snapshots into d.names.
// Also adds d.latest links if "%T" is at end of a path template
func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) {
d.names = make(map[string]*restic.Snapshot)
d.latest = make(map[string]string)
entries := make(map[string]*MetaDirData)
type mountData struct {
sn *restic.Snapshot
linkTarget string // if linkTarget!= "", this is a symlink
childFn string
child *MetaDirData
}
// recursively build tree structure
var mount func(path string, data mountData)
mount = func(path string, data mountData) {
e := entries[path]
if e == nil {
e = &MetaDirData{}
}
if data.sn != nil {
e.snapshot = data.sn
e.linkTarget = data.linkTarget
} else {
// intermediate directory, register as a child directory
if e.names == nil {
e.names = make(map[string]*MetaDirData)
}
if data.child != nil {
e.names[data.childFn] = data.child
}
}
entries[path] = e
slashIdx := strings.LastIndex(path, "/")
if slashIdx >= 0 {
// add to parent dir, but without snapshot
mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e})
}
}
// root directory
mount("", mountData{})
// insert pure directories; needed to get empty structure even if there
// are no snapshots in these dirs
for _, p := range d.pathTemplates {
for _, pattern := range []string{"%i", "%I", "%u", "%h", "%t", "%T"} {
p = strings.ReplaceAll(p, pattern, "")
p = staticPrefix(p)
if p != "" {
mount(path.Clean("/"+p), mountData{})
}
d.names[path.Clean(p)+"/"] = nil
}
latestTime := make(map[string]time.Time)
@ -154,25 +242,33 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) {
for _, templ := range d.pathTemplates {
paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn)
for _, p := range paths {
suffix := d.uniqueName(p, timeSuffix)
d.names[path.Clean(p+suffix)] = sn
if p != "" {
p = "/" + p
}
suffix := uniqueName(entries, p, timeSuffix)
mount(path.Clean(p+suffix), mountData{sn: sn})
if timeSuffix != "" {
lt, ok := latestTime[p]
if !ok || !sn.Time.Before(lt) {
debug.Log("link (update) %v -> %v\n", p, suffix)
d.latest[p] = suffix
// inject symlink
mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix})
latestTime[p] = sn.Time
}
}
}
}
}
d.entries = entries
}
const minSnapshotsReloadTime = 60 * time.Second
// update snapshots if repository has changed
func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
d.mutex.Lock()
defer d.mutex.Unlock()
if time.Since(d.lastCheck) < minSnapshotsReloadTime {
return nil
}
@ -200,3 +296,14 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
d.makeDirs(snapshots)
return nil
}
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
err := d.updateSnapshots(ctx)
if err != nil {
return nil, err
}
d.mutex.Lock()
defer d.mutex.Unlock()
return d.entries[prefix], nil
}

228
internal/fuse/snapshots_dirstruct_test.go
View file

@ -4,6 +4,7 @@
package fuse
import (
"strings"
"testing"
"time"
@ -89,69 +90,186 @@ func TestMakeDirs(t *testing.T) {
expNames := make(map[string]*restic.Snapshot)
expLatest := make(map[string]string)
// empty entries for dir structure
expNames["ids/"] = nil
expNames["snapshots/"] = nil
expNames["hosts/"] = nil
expNames["tags/"] = nil
expNames["users/"] = nil
expNames["longids/"] = nil
expNames["//"] = nil
// entries for sn0
expNames["ids/00000000"] = sn0
expNames["snapshots/2020/12/31"] = sn0
expNames["hosts/host/2020/12/31"] = sn0
expNames["tags/tag1/2020/12/31"] = sn0
expNames["tags/tag2/2020/12/31"] = sn0
expNames["users/user/2020/12/31"] = sn0
expNames["longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0
expNames["2020/12/31/host"] = sn0
expNames["2020/12/31/00000000"] = sn0
expNames["/ids/00000000"] = sn0
expNames["/snapshots/2020/12/31"] = sn0
expNames["/hosts/host/2020/12/31"] = sn0
expNames["/tags/tag1/2020/12/31"] = sn0
expNames["/tags/tag2/2020/12/31"] = sn0
expNames["/users/user/2020/12/31"] = sn0
expNames["/longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0
expNames["/2020/12/31/host"] = sn0
expNames["/2020/12/31/00000000"] = sn0
// entries for sn1
expNames["ids/12345678"] = sn1
expNames["snapshots/2021/01/01"] = sn1
expNames["hosts/host/2021/01/01"] = sn1
expNames["tags/tag1/2021/01/01"] = sn1
expNames["tags/tag2/2021/01/01"] = sn1
expNames["users/user/2021/01/01"] = sn1
expNames["longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1
expNames["2021/01/01/host"] = sn1
expNames["2021/01/01/12345678"] = sn1
expNames["/ids/12345678"] = sn1
expNames["/snapshots/2021/01/01"] = sn1
expNames["/hosts/host/2021/01/01"] = sn1
expNames["/tags/tag1/2021/01/01"] = sn1
expNames["/tags/tag2/2021/01/01"] = sn1
expNames["/users/user/2021/01/01"] = sn1
expNames["/longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1
expNames["/2021/01/01/host"] = sn1
expNames["/2021/01/01/12345678"] = sn1
// entries for sn2
expNames["ids/87654321"] = sn2
expNames["snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string
expNames["hosts/host2/2021/01/01"] = sn2
expNames["tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string
expNames["tags/tag3/2021/01/01"] = sn2
expNames["tags/tag4/2021/01/01"] = sn2
expNames["users/user2/2021/01/01"] = sn2
expNames["longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2
expNames["2021/01/01/host2"] = sn2
expNames["2021/01/01/87654321"] = sn2
expNames["/ids/87654321"] = sn2
expNames["/snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string
expNames["/hosts/host2/2021/01/01"] = sn2
expNames["/tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string
expNames["/tags/tag3/2021/01/01"] = sn2
expNames["/tags/tag4/2021/01/01"] = sn2
expNames["/users/user2/2021/01/01"] = sn2
expNames["/longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2
expNames["/2021/01/01/host2"] = sn2
expNames["/2021/01/01/87654321"] = sn2
// entries for sn3
expNames["ids/aaaaaaaa"] = sn3
expNames["snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string
expNames["hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string
expNames["users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string
expNames["longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3
expNames["2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host
expNames["2021/01/01/aaaaaaaa"] = sn3
expNames["/ids/aaaaaaaa"] = sn3
expNames["/snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string
expNames["/hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string
expNames["/users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string
expNames["/longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3
expNames["/2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host
expNames["/2021/01/01/aaaaaaaa"] = sn3
// intermediate directories
// sn0
expNames["/ids"] = nil
expNames[""] = nil
expNames["/snapshots/2020/12"] = nil
expNames["/snapshots/2020"] = nil
expNames["/snapshots"] = nil
expNames["/hosts/host/2020/12"] = nil
expNames["/hosts/host/2020"] = nil
expNames["/hosts/host"] = nil
expNames["/hosts"] = nil
expNames["/tags/tag1/2020/12"] = nil
expNames["/tags/tag1/2020"] = nil
expNames["/tags/tag1"] = nil
expNames["/tags"] = nil
expNames["/tags/tag2/2020/12"] = nil
expNames["/tags/tag2/2020"] = nil
expNames["/tags/tag2"] = nil
expNames["/users/user/2020/12"] = nil
expNames["/users/user/2020"] = nil
expNames["/users/user"] = nil
expNames["/users"] = nil
expNames["/longids"] = nil
expNames["/2020/12/31"] = nil
expNames["/2020/12"] = nil
expNames["/2020"] = nil
// sn1
expNames["/snapshots/2021/01"] = nil
expNames["/snapshots/2021"] = nil
expNames["/hosts/host/2021/01"] = nil
expNames["/hosts/host/2021"] = nil
expNames["/tags/tag1/2021/01"] = nil
expNames["/tags/tag1/2021"] = nil
expNames["/tags/tag2/2021/01"] = nil
expNames["/tags/tag2/2021"] = nil
expNames["/users/user/2021/01"] = nil
expNames["/users/user/2021"] = nil
expNames["/2021/01/01"] = nil
expNames["/2021/01"] = nil
expNames["/2021"] = nil
// sn2
expNames["/hosts/host2/2021/01"] = nil
expNames["/hosts/host2/2021"] = nil
expNames["/hosts/host2"] = nil
expNames["/tags/tag3/2021/01"] = nil
expNames["/tags/tag3/2021"] = nil
expNames["/tags/tag3"] = nil
expNames["/tags/tag4/2021/01"] = nil
expNames["/tags/tag4/2021"] = nil
expNames["/tags/tag4"] = nil
expNames["/users/user2/2021/01"] = nil
expNames["/users/user2/2021"] = nil
expNames["/users/user2"] = nil
// target snapshots for links
expNames["/snapshots/latest"] = sn3 // sn1 - sn3 have same time string
expNames["/hosts/host/latest"] = sn3
expNames["/hosts/host2/latest"] = sn2
expNames["/tags/tag1/latest"] = sn1
expNames["/tags/tag2/latest"] = sn2 // sn1 and sn2 have same time string
expNames["/tags/tag3/latest"] = sn2
expNames["/tags/tag4/latest"] = sn2
expNames["/users/user/latest"] = sn1
expNames["/users/user2/latest"] = sn3 // sn2 and sn3 have same time string
// latest links
expLatest["snapshots/"] = "2021/01/01-2" // sn1 - sn3 have same time string
expLatest["hosts/host/"] = "2021/01/01-1"
expLatest["hosts/host2/"] = "2021/01/01"
expLatest["tags/tag1/"] = "2021/01/01"
expLatest["tags/tag2/"] = "2021/01/01-1" // sn1 and sn2 have same time string
expLatest["tags/tag3/"] = "2021/01/01"
expLatest["tags/tag4/"] = "2021/01/01"
expLatest["users/user/"] = "2021/01/01"
expLatest["users/user2/"] = "2021/01/01-1" // sn2 and sn3 have same time string
expLatest["/snapshots/latest"] = "2021/01/01-2" // sn1 - sn3 have same time string
expLatest["/hosts/host/latest"] = "2021/01/01-1"
expLatest["/hosts/host2/latest"] = "2021/01/01"
expLatest["/tags/tag1/latest"] = "2021/01/01"
expLatest["/tags/tag2/latest"] = "2021/01/01-1" // sn1 and sn2 have same time string
expLatest["/tags/tag3/latest"] = "2021/01/01"
expLatest["/tags/tag4/latest"] = "2021/01/01"
expLatest["/users/user/latest"] = "2021/01/01"
expLatest["/users/user2/latest"] = "2021/01/01-1" // sn2 and sn3 have same time string
test.Equals(t, expNames, sds.names)
test.Equals(t, expLatest, sds.latest)
verifyEntries(t, expNames, expLatest, sds.entries)
}
func verifyEntries(t *testing.T, expNames map[string]*restic.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) {
actNames := make(map[string]*restic.Snapshot)
actLatest := make(map[string]string)
for path, entry := range entries {
actNames[path] = entry.snapshot
if entry.linkTarget != "" {
actLatest[path] = entry.linkTarget
}
}
test.Equals(t, expNames, actNames)
test.Equals(t, expLatest, actLatest)
// verify tree integrity
for path, entry := range entries {
// check that all children are actually contained in entry.names
for otherPath := range entries {
if strings.HasPrefix(otherPath, path+"/") {
sub := otherPath[len(path)+1:]
// remaining path does not contain a directory
test.Assert(t, strings.Contains(sub, "/") || (entry.names != nil && entry.names[sub] != nil), "missing entry %v in %v", sub, path)
}
}
if entry.names == nil {
continue
}
// child entries reference the correct MetaDirData
for elem, subentry := range entry.names {
test.Equals(t, entries[path+"/"+elem], subentry)
}
}
}
func TestMakeEmptyDirs(t *testing.T) {
pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T",
"tags/%t/%T", "users/%u/%T", "longids/id-%I", "%T/%h", "%T/%i", "id-%i",
}
timeTemplate := "2006/01/02"
sds := &SnapshotsDirStructure{
pathTemplates: pathTemplates,
timeTemplate: timeTemplate,
}
sds.makeDirs(restic.Snapshots{})
expNames := make(map[string]*restic.Snapshot)
expLatest := make(map[string]string)
// empty entries for dir structure
expNames["/ids"] = nil
expNames["/snapshots"] = nil
expNames["/hosts"] = nil
expNames["/tags"] = nil
expNames["/users"] = nil
expNames["/longids"] = nil
expNames[""] = nil
verifyEntries(t, expNames, expLatest, sds.entries)
}