From bd3022c5043df887a4224c978f8626127f300d9d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 25 Feb 2024 17:26:55 +0100 Subject: [PATCH] wip --- cmd/restic/cmd_webdav.go | 30 ++++- internal/server/rofs/dir.go | 92 ++++++++++++++++ internal/server/rofs/file_info.go | 23 ++++ internal/server/rofs/fs.go | 152 ++++++++++++++++++++++++++ internal/server/rofs/fs_test.go | 35 ++++++ internal/server/rofs/mem_file.go | 117 ++++++++++++++++++++ internal/server/rofs/snapshots_dir.go | 117 ++++++++++++++++++++ 7 files changed, 560 insertions(+), 6 deletions(-) create mode 100644 internal/server/rofs/dir.go create mode 100644 internal/server/rofs/file_info.go create mode 100644 internal/server/rofs/fs.go create mode 100644 internal/server/rofs/fs_test.go create mode 100644 internal/server/rofs/mem_file.go create mode 100644 internal/server/rofs/snapshots_dir.go diff --git a/cmd/restic/cmd_webdav.go b/cmd/restic/cmd_webdav.go index 38c6ff504..0d52b19e9 100644 --- a/cmd/restic/cmd_webdav.go +++ b/cmd/restic/cmd_webdav.go @@ -10,9 +10,8 @@ import ( "github.com/spf13/cobra" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fuse" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/webdav" + "github.com/restic/restic/internal/server/rofs" ) var cmdWebDAV = &cobra.Command{ @@ -74,23 +73,42 @@ func runWebDAV(ctx context.Context, opts WebDAVOptions, gopts GlobalOptions, arg errorLogger := log.New(os.Stderr, "error log: ", log.Flags()) - cfg := fuse.Config{ + cfg := rofs.Config{ Filter: opts.SnapshotFilter, TimeTemplate: opts.TimeTemplate, PathTemplates: opts.PathTemplates, } - root := fuse.NewRoot(repo, cfg) - h, err := webdav.NewWebDAV(ctx, root) + root, err := rofs.New(ctx, repo, cfg) if err != nil { return err } + // root := os.DirFS(".") + + // h, err := webdav.NewWebDAV(ctx, root) + // if err != nil { + // return err + // } + + // root := fstest.MapFS{ + // "foobar": &fstest.MapFile{ + // Data: []byte("foobar test content"), + // Mode: 0644, + // ModTime: time.Now(), + // }, + // "test.txt": &fstest.MapFile{ + // Data: []byte("other file"), + // Mode: 0640, + // ModTime: time.Now(), + // }, + // } + srv := &http.Server{ ReadTimeout: 60 * time.Second, WriteTimeout: 60 * time.Second, Addr: opts.Listen, - Handler: h, + Handler: http.FileServer(http.FS(root)), ErrorLog: errorLogger, } diff --git a/internal/server/rofs/dir.go b/internal/server/rofs/dir.go new file mode 100644 index 000000000..2e504aa56 --- /dev/null +++ b/internal/server/rofs/dir.go @@ -0,0 +1,92 @@ +package rofs + +import ( + "io" + "io/fs" + "slices" + + "github.com/restic/restic/internal/debug" +) + +// dirEntry holds data for a file or directory. +type dirEntry struct { + fileInfo fs.FileInfo +} + +var _ fs.DirEntry = dirEntry{} + +func (d dirEntry) Name() string { return d.fileInfo.Name() } +func (d dirEntry) IsDir() bool { return d.fileInfo.IsDir() } +func (d dirEntry) Type() fs.FileMode { return d.fileInfo.Mode().Type() } +func (d dirEntry) Info() (fs.FileInfo, error) { return d.fileInfo, nil } + +// openDir represents a directory opened for reading. +type openDir struct { + path string + fileInfo FileInfo + entries []fs.DirEntry + offset int +} + +var _ fs.ReadDirFile = &openDir{} + +func (d *openDir) Close() error { + debug.Log("Close(%v)", d.path) + return nil +} + +func (d *openDir) Stat() (fs.FileInfo, error) { + debug.Log("Stat(%v)", d.path) + return d.fileInfo, nil +} + +func (d *openDir) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.path, Err: fs.ErrInvalid} +} + +func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) { + n := len(d.entries) - d.offset + + if n == 0 && count > 0 { + debug.Log("ReadDir(%v, %v) -> EOF", d.path, count) + + return nil, io.EOF + } + + if count > 0 && n > count { + n = count + } + + list := make([]fs.DirEntry, 0, n) + for i := 0; i < n; i++ { + list = append(list, d.entries[d.offset+i]) + } + + d.offset += n + + debug.Log("ReadDir(%v, %v) -> %v entries", d.path, count, len(list)) + + return list, nil +} + +func dirMap2DirEntry(m map[string]rofsEntry) []fs.DirEntry { + list := make([]fs.DirEntry, 0, len(m)) + + for _, entry := range m { + list = append(list, entry.DirEntry()) + } + + slices.SortFunc(list, func(a, b fs.DirEntry) int { + if a.Name() == b.Name() { + return 0 + } + + if a.Name() < b.Name() { + return -1 + } + + return 1 + }) + + return list +} diff --git a/internal/server/rofs/file_info.go b/internal/server/rofs/file_info.go new file mode 100644 index 000000000..963756506 --- /dev/null +++ b/internal/server/rofs/file_info.go @@ -0,0 +1,23 @@ +package rofs + +import ( + "io/fs" + "time" +) + +// FileInfo provides information about a file or directory. +type FileInfo struct { + name string + mode fs.FileMode + modtime time.Time + size int64 +} + +func (fi FileInfo) Name() string { return fi.name } +func (fi FileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi FileInfo) ModTime() time.Time { return fi.modtime } +func (fi FileInfo) Mode() fs.FileMode { return fi.mode } +func (fi FileInfo) Size() int64 { return fi.size } +func (fi FileInfo) Sys() any { return nil } + +var _ fs.FileInfo = FileInfo{} diff --git a/internal/server/rofs/fs.go b/internal/server/rofs/fs.go new file mode 100644 index 000000000..bb97c12e2 --- /dev/null +++ b/internal/server/rofs/fs.go @@ -0,0 +1,152 @@ +// Package rofs implements a read-only file system on top of a restic repository. +package rofs + +import ( + "context" + "fmt" + "io/fs" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +// ROFS implements a read-only filesystem on top of a repo. +type ROFS struct { + repo restic.Repository + cfg Config + entries map[string]rofsEntry + fileInfo FileInfo +} + +type rofsEntry interface { + Open() (fs.File, error) + DirEntry() fs.DirEntry +} + +// statically ensure that *FS implements fs.FS +var _ fs.FS = &ROFS{} + +// Config holds settings for a filesystem. +type Config struct { + Filter restic.SnapshotFilter + TimeTemplate string + PathTemplates []string +} + +// New returns a new filesystem for the repo. +func New(ctx context.Context, repo restic.Repository, cfg Config) (*ROFS, error) { + // set defaults, if PathTemplates is not set + if len(cfg.PathTemplates) == 0 { + cfg.PathTemplates = []string{ + "ids/%i", + "snapshots/%T", + "hosts/%h/%T", + "tags/%t/%T", + } + } + + rofs := &ROFS{ + repo: repo, + cfg: cfg, + entries: make(map[string]rofsEntry), + fileInfo: FileInfo{ + name: ".", + mode: 0755, + modtime: time.Now(), + }, + } + + err := rofs.updateSnapshots(ctx) + if err != nil { + return nil, err + } + + return rofs, nil +} + +// Open opens the named file. +// +// When Open returns an error, it should be of type *PathError +// with the Op field set to "open", the Path field set to name, +// and the Err field describing the problem. +// +// Open should reject attempts to open names that do not satisfy +// ValidPath(name), returning a *PathError with Err set to +// ErrInvalid or ErrNotExist. +func (rofs *ROFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + debug.Log("Open(%v), invalid path name", name) + + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + + if name == "." { + debug.Log("Open(%v) (root)", name) + + d := &openDir{ + path: ".", + fileInfo: FileInfo{ + name: ".", + mode: fs.ModeDir | 0555, + modtime: time.Now(), + }, + entries: dirMap2DirEntry(rofs.entries), + } + + return d, nil + } + + entry, ok := rofs.entries[name] + if !ok { + debug.Log("Open(%v) -> does not exist", name) + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrNotExist, + } + } + + debug.Log("Open(%v)", name) + + return entry.Open() +} + +func buildSnapshotEntries(ctx context.Context, repo restic.Repository, cfg Config) (map[string]rofsEntry, error) { + var snapshots restic.Snapshots + err := cfg.Filter.FindAll(ctx, repo, repo, nil, func(_ string, sn *restic.Snapshot, _ error) error { + if sn != nil { + snapshots = append(snapshots, sn) + } + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("filter snapshots: %w", err) + } + + debug.Log("found %d snapshots", len(snapshots)) + + list := make(map[string]rofsEntry) + list["snapshots"] = dirEntry{} + //FIXME CONTINUE + + return list, nil +} + +func (rofs *ROFS) updateSnapshots(ctx context.Context) error { + + entries, err := buildSnapshotEntries(ctx, rofs.repo, rofs.cfg) + if err != nil { + return err + } + + rofs.entries = entries + + return nil +} diff --git a/internal/server/rofs/fs_test.go b/internal/server/rofs/fs_test.go new file mode 100644 index 000000000..7263676a0 --- /dev/null +++ b/internal/server/rofs/fs_test.go @@ -0,0 +1,35 @@ +package rofs + +import ( + "context" + "testing" + "testing/fstest" + "time" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" +) + +func TestROFs(t *testing.T) { + repo := repository.TestRepository(t) + + timestamp, err := time.Parse(time.RFC3339, "2024-02-25T17:21:56+01:00") + if err != nil { + t.Fatal(err) + } + + restic.TestCreateSnapshot(t, repo, timestamp, 2) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + root, err := New(ctx, repo, Config{}) + if err != nil { + t.Fatal(err) + } + + err = fstest.TestFS(root, "snapshots") + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/server/rofs/mem_file.go b/internal/server/rofs/mem_file.go new file mode 100644 index 000000000..0bb98ffbe --- /dev/null +++ b/internal/server/rofs/mem_file.go @@ -0,0 +1,117 @@ +package rofs + +import ( + "io" + "io/fs" + "path" + "time" + + "github.com/restic/restic/internal/debug" +) + +type MemFile struct { + Path string + FileInfo FileInfo + Data []byte +} + +// NewMemFile returns a new file. +func NewMemFile(filename string, data []byte, modTime time.Time) MemFile { + return MemFile{ + Path: filename, + Data: data, + FileInfo: FileInfo{ + name: path.Base(filename), + size: int64(len(data)), + mode: 0644, + modtime: modTime, + }, + } +} + +func (f MemFile) Open() (fs.File, error) { + return &openMemFile{ + path: f.Path, + fileInfo: f.FileInfo, + data: f.Data, + }, nil +} + +func (f MemFile) DirEntry() fs.DirEntry { + return dirEntry{ + fileInfo: f.FileInfo, + } +} + +// openMemFile is a file that is currently open. +type openMemFile struct { + path string + + fileInfo FileInfo + data []byte + + offset int64 +} + +// make sure it implements all the necessary interfaces +var _ fs.File = &openMemFile{} +var _ io.Seeker = &openMemFile{} + +func (f *openMemFile) Close() error { + debug.Log("Close(%v)", f.path) + + return nil +} + +func (f *openMemFile) Stat() (fs.FileInfo, error) { + debug.Log("Stat(%v)", f.path) + + return f.fileInfo, nil +} + +func (f *openMemFile) Read(p []byte) (int, error) { + if f.offset >= int64(len(f.data)) { + debug.Log("Read(%v, %v) -> EOF", f.path, len(p)) + + return 0, io.EOF + } + + if f.offset < 0 { + debug.Log("Read(%v, %v) -> offset negative", f.path, len(p)) + + return 0, &fs.PathError{ + Op: "read", + Path: f.path, + Err: fs.ErrInvalid, + } + } + + n := copy(p, f.data[f.offset:]) + f.offset += int64(n) + + debug.Log("Read(%v, %v) -> %v bytes", f.path, len(p), n) + + return n, nil +} + +func (f *openMemFile) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + offset += f.offset + case io.SeekEnd: + offset += int64(len(f.data)) + } + + if offset < 0 || offset > int64(len(f.data)) { + debug.Log("Seek(%v, %v, %v) -> error invalid offset %v", f.path, offset, whence, offset) + + return 0, &fs.PathError{Op: "seek", Path: f.path, Err: fs.ErrInvalid} + } + + debug.Log("Seek(%v, %v, %v), new offset %v", f.path, offset, whence, offset) + + f.offset = offset + + return offset, nil +} diff --git a/internal/server/rofs/snapshots_dir.go b/internal/server/rofs/snapshots_dir.go new file mode 100644 index 000000000..da5f5d9fb --- /dev/null +++ b/internal/server/rofs/snapshots_dir.go @@ -0,0 +1,117 @@ +package rofs + +import ( + "io" + "io/fs" + "slices" + "time" + + "github.com/restic/restic/internal/debug" +) + +// SnapshotsDir implements a tree of snapshots in repo as a file system in various sub-directories. +type SnapshotsDir struct { + modTime time.Time + + pathTemplates []string + timeTemplate string + + // prepare the list of top-level directories + entries []fs.DirEntry + + // used by ReadDir() with positive number of entries to return + entriesRemaining []fs.DirEntry +} + +// NewSnapshotsDir initializes a new top-level snapshots directory. +func NewSnapshotsDir(pathTemplates []string, timeTemplate string) *SnapshotsDir { + dir := &SnapshotsDir{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + modTime: time.Now(), + } + + testnames := []string{"foo", "bar", "baz", "snapshots"} + for _, name := range testnames { + dir.entries = append(dir.entries, + fs.FileInfoToDirEntry(FileInfo{ + name: name, + mode: 0644, + modtime: time.Now(), + })) + } + + slices.SortFunc(dir.entries, func(a, b fs.DirEntry) int { + if a.Name() == b.Name() { + return 0 + } + + if a.Name() < b.Name() { + return 1 + } + + return -1 + }) + + // prepare for readdir with positive n + dir.entriesRemaining = dir.entries + + return dir +} + +// ensure that it implements all necessary interfaces. +var _ fs.ReadDirFile = &SnapshotsDir{} + +// Close closes the snapshots dir. +func (dir *SnapshotsDir) Close() error { + debug.Log("Close()") + + // reset readdir list + dir.entriesRemaining = dir.entries + + return nil +} + +// Read is not implemented for a dir. +func (dir *SnapshotsDir) Read([]byte) (int, error) { + return 0, &fs.PathError{ + Op: "read", + Err: fs.ErrInvalid, + } +} + +// Stat returns information about the dir. +func (dir *SnapshotsDir) Stat() (fs.FileInfo, error) { + debug.Log("Stat(root)") + + fi := FileInfo{ + name: "root", // use special name, this is the root node + size: 0, + modtime: dir.modTime, + mode: 0755, + } + + return fi, nil +} + +// ReadDir returns a list of entries. +func (dir *SnapshotsDir) ReadDir(n int) ([]fs.DirEntry, error) { + if n < 0 { + debug.Log("Readdir(root, %v), return %v entries", n, len(dir.entries)) + return dir.entries, nil + } + + // complicated pointer handling + if n > len(dir.entriesRemaining) { + n = len(dir.entriesRemaining) + } + + if n == 0 { + return nil, io.EOF + } + + list := dir.entriesRemaining[:n] + dir.entriesRemaining = dir.entriesRemaining[n:] + + return list, nil +}