From b9b32e5647a26ceb3a3ebe8a927ba35be6153323 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 23:03:25 +0200 Subject: [PATCH] restic: extract Node filesystem code to fs package --- internal/archiver/archiver.go | 2 +- internal/archiver/archiver_test.go | 4 +- internal/archiver/archiver_unix_test.go | 2 +- internal/archiver/file_saver_test.go | 2 +- internal/{restic => fs}/mknod_unix.go | 2 +- internal/fs/node.go | 334 ++++++++++++++++++ internal/{restic => fs}/node_aix.go | 14 +- internal/{restic => fs}/node_darwin.go | 2 +- internal/{restic => fs}/node_freebsd.go | 2 +- internal/{restic => fs}/node_linux.go | 5 +- internal/{restic => fs}/node_netbsd.go | 14 +- internal/{restic => fs}/node_openbsd.go | 14 +- internal/{restic => fs}/node_solaris.go | 2 +- internal/fs/node_test.go | 324 +++++++++++++++++ internal/{restic => fs}/node_unix.go | 2 +- internal/{restic => fs}/node_unix_test.go | 7 +- internal/{restic => fs}/node_windows.go | 66 ++-- internal/{restic => fs}/node_windows_test.go | 56 +-- internal/{restic => fs}/node_xattr.go | 17 +- .../{restic => fs}/node_xattr_all_test.go | 13 +- internal/{restic => fs}/node_xattr_test.go | 2 +- internal/restic/node.go | 323 ----------------- internal/restic/node_test.go | 314 ---------------- internal/restic/tree_test.go | 5 +- internal/restorer/restorer.go | 4 +- internal/restorer/restorer_windows_test.go | 3 +- 26 files changed, 783 insertions(+), 752 deletions(-) rename internal/{restic => fs}/mknod_unix.go (93%) create mode 100644 internal/fs/node.go rename internal/{restic => fs}/node_aix.go (68%) rename internal/{restic => fs}/node_darwin.go (95%) rename internal/{restic => fs}/node_freebsd.go (96%) rename internal/{restic => fs}/node_linux.go (88%) rename internal/{restic => fs}/node_netbsd.go (58%) rename internal/{restic => fs}/node_openbsd.go (57%) rename internal/{restic => fs}/node_solaris.go (95%) create mode 100644 internal/fs/node_test.go rename internal/{restic => fs}/node_unix.go (97%) rename internal/{restic => fs}/node_unix_test.go (94%) rename internal/{restic => fs}/node_windows.go (88%) rename internal/{restic => fs}/node_windows_test.go (90%) rename internal/{restic => fs}/node_xattr.go (80%) rename internal/{restic => fs}/node_xattr_all_test.go (77%) rename internal/{restic => fs}/node_xattr_test.go (98%) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e7c346d3a..c576d047c 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -248,7 +248,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I // nodeFromFileInfo returns the restic node from an os.FileInfo. func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + node, err := fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) if !arch.WithAtime { node.AccessTime = node.ModTime } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c54f9ea33..18151eb02 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -557,7 +557,7 @@ func rename(t testing.TB, oldname, newname string) { } func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node { - node, err := restic.NodeFromFileInfo(filename, fi, false) + node, err := fs.NodeFromFileInfo(filename, fi, false) if err != nil { t.Fatal(err) } @@ -2291,7 +2291,7 @@ func TestMetadataChanged(t *testing.T) { // get metadata fi := lstat(t, "testfile") - want, err := restic.NodeFromFileInfo("testfile", fi, false) + want, err := fs.NodeFromFileInfo("testfile", fi, false) if err != nil { t.Fatal(err) } diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index 4a380dff8..d91d993dd 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -48,7 +48,7 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo { func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) { fi := lstat(t, name) - want, err := restic.NodeFromFileInfo(name, fi, false) + want, err := fs.NodeFromFileInfo(name, fi, false) rtest.OK(t, err) _, node := snapshot(t, repo, fs.Local{}, nil, name) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 409bdedd0..4a4327572 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -50,7 +50,7 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + return fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) } return s, ctx, wg diff --git a/internal/restic/mknod_unix.go b/internal/fs/mknod_unix.go similarity index 93% rename from internal/restic/mknod_unix.go rename to internal/fs/mknod_unix.go index 7dd6c60d0..6127599f7 100644 --- a/internal/restic/mknod_unix.go +++ b/internal/fs/mknod_unix.go @@ -1,7 +1,7 @@ //go:build !freebsd && !windows // +build !freebsd,!windows -package restic +package fs import "golang.org/x/sys/unix" diff --git a/internal/fs/node.go b/internal/fs/node.go new file mode 100644 index 000000000..9bd507ba5 --- /dev/null +++ b/internal/fs/node.go @@ -0,0 +1,334 @@ +package fs + +import ( + "os" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// NodeFromFileInfo returns a new node from the given path and FileInfo. It +// returns the first error that is encountered, together with a node. +func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { + mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky + node := &restic.Node{ + Path: path, + Name: fi.Name(), + Mode: fi.Mode() & mask, + ModTime: fi.ModTime(), + } + + node.Type = nodeTypeFromFileInfo(fi) + if node.Type == "file" { + node.Size = uint64(fi.Size()) + } + + err := nodeFillExtra(node, path, fi, ignoreXattrListError) + return node, err +} + +func nodeTypeFromFileInfo(fi os.FileInfo) string { + switch fi.Mode() & os.ModeType { + case 0: + return "file" + case os.ModeDir: + return "dir" + case os.ModeSymlink: + return "symlink" + case os.ModeDevice | os.ModeCharDevice: + return "chardev" + case os.ModeDevice: + return "dev" + case os.ModeNamedPipe: + return "fifo" + case os.ModeSocket: + return "socket" + case os.ModeIrregular: + return "irregular" + } + + return "" +} + +func nodeFillExtra(node *restic.Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { + stat, ok := toStatT(fi.Sys()) + if !ok { + // fill minimal info with current values for uid, gid + node.UID = uint32(os.Getuid()) + node.GID = uint32(os.Getgid()) + node.ChangeTime = node.ModTime + return nil + } + + node.Inode = uint64(stat.ino()) + node.DeviceID = uint64(stat.dev()) + + nodeFillTimes(node, stat) + + nodeFillUser(node, stat) + + switch node.Type { + case "file": + node.Size = uint64(stat.size()) + node.Links = uint64(stat.nlink()) + case "dir": + case "symlink": + var err error + node.LinkTarget, err = Readlink(path) + node.Links = uint64(stat.nlink()) + if err != nil { + return errors.WithStack(err) + } + case "dev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "chardev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "fifo": + case "socket": + default: + return errors.Errorf("unsupported file type %q", node.Type) + } + + allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) + } + return err +} + +func nodeFillTimes(node *restic.Node, stat *statT) { + ctim := stat.ctim() + atim := stat.atim() + node.ChangeTime = time.Unix(ctim.Unix()) + node.AccessTime = time.Unix(atim.Unix()) +} + +func nodeFillUser(node *restic.Node, stat *statT) { + uid, gid := stat.uid(), stat.gid() + node.UID, node.GID = uid, gid + node.User = lookupUsername(uid) + node.Group = lookupGroup(gid) +} + +var ( + uidLookupCache = make(map[uint32]string) + uidLookupCacheMutex = sync.RWMutex{} +) + +// Cached user name lookup by uid. Returns "" when no name can be found. +func lookupUsername(uid uint32) string { + uidLookupCacheMutex.RLock() + username, ok := uidLookupCache[uid] + uidLookupCacheMutex.RUnlock() + + if ok { + return username + } + + u, err := user.LookupId(strconv.Itoa(int(uid))) + if err == nil { + username = u.Username + } + + uidLookupCacheMutex.Lock() + uidLookupCache[uid] = username + uidLookupCacheMutex.Unlock() + + return username +} + +var ( + gidLookupCache = make(map[uint32]string) + gidLookupCacheMutex = sync.RWMutex{} +) + +// Cached group name lookup by gid. Returns "" when no name can be found. +func lookupGroup(gid uint32) string { + gidLookupCacheMutex.RLock() + group, ok := gidLookupCache[gid] + gidLookupCacheMutex.RUnlock() + + if ok { + return group + } + + g, err := user.LookupGroupId(strconv.Itoa(int(gid))) + if err == nil { + group = g.Name + } + + gidLookupCacheMutex.Lock() + gidLookupCache[gid] = group + gidLookupCacheMutex.Unlock() + + return group +} + +// NodeCreateAt creates the node at the given path but does NOT restore node meta data. +func NodeCreateAt(node *restic.Node, path string) error { + debug.Log("create node %v at %v", node.Name, path) + + switch node.Type { + case "dir": + if err := nodeCreateDirAt(node, path); err != nil { + return err + } + case "file": + if err := nodeCreateFileAt(path); err != nil { + return err + } + case "symlink": + if err := nodeCreateSymlinkAt(node, path); err != nil { + return err + } + case "dev": + if err := nodeCreateDevAt(node, path); err != nil { + return err + } + case "chardev": + if err := nodeCreateCharDevAt(node, path); err != nil { + return err + } + case "fifo": + if err := nodeCreateFifoAt(path); err != nil { + return err + } + case "socket": + return nil + default: + return errors.Errorf("filetype %q not implemented", node.Type) + } + + return nil +} + +func nodeCreateDirAt(node *restic.Node, path string) error { + err := Mkdir(path, node.Mode) + if err != nil && !os.IsExist(err) { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateFileAt(path string) error { + f, err := OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return errors.WithStack(err) + } + + if err := f.Close(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateSymlinkAt(node *restic.Node, path string) error { + if err := Symlink(node.LinkTarget, path); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFBLK|0600, node.Device) +} + +func nodeCreateCharDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFCHR|0600, node.Device) +} + +func nodeCreateFifoAt(path string) error { + return mkfifo(path, 0600) +} + +func mkfifo(path string, mode uint32) (err error) { + return mknod(path, mode|syscall.S_IFIFO, 0) +} + +// NodeRestoreMetadata restores node metadata +func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + err := nodeRestoreMetadata(node, path, warn) + if err != nil { + // It is common to have permission errors for folders like /home + // unless you're running as root, so ignore those. + if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { + debug.Log("not running as root, ignoring permission error for %v: %v", + path, err) + return nil + } + debug.Log("restoreMetadata(%s) error %v", path, err) + } + + return err +} + +func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + var firsterr error + + if err := lchown(path, int(node.UID), int(node.GID)); err != nil { + firsterr = errors.WithStack(err) + } + + if err := nodeRestoreExtendedAttributes(node, path); err != nil { + debug.Log("error restoring extended attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := NodeRestoreTimestamps(node, path); err != nil { + debug.Log("error restoring timestamps for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := Chmod(path, node.Mode); err != nil { + if firsterr == nil { + firsterr = errors.WithStack(err) + } + } + } + + return firsterr +} + +func NodeRestoreTimestamps(node *restic.Node, path string) error { + var utimes = [...]syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + + if node.Type == "symlink" { + return nodeRestoreSymlinkTimestamps(path, utimes) + } + + if err := syscall.UtimesNano(path, utimes[:]); err != nil { + return errors.Wrap(err, "UtimesNano") + } + + return nil +} diff --git a/internal/restic/node_aix.go b/internal/fs/node_aix.go similarity index 68% rename from internal/restic/node_aix.go rename to internal/fs/node_aix.go index 4cd279973..4e6944425 100644 --- a/internal/restic/node_aix.go +++ b/internal/fs/node_aix.go @@ -1,11 +1,13 @@ //go:build aix // +build aix -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -24,12 +26,12 @@ func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } // nodeRestoreExtendedAttributes is a no-op on AIX. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on AIX. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -39,11 +41,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on AIX. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op on AIX. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_darwin.go b/internal/fs/node_darwin.go similarity index 95% rename from internal/restic/node_darwin.go rename to internal/fs/node_darwin.go index 099007e07..1ca7ce480 100644 --- a/internal/restic/node_darwin.go +++ b/internal/fs/node_darwin.go @@ -1,4 +1,4 @@ -package restic +package fs import "syscall" diff --git a/internal/restic/node_freebsd.go b/internal/fs/node_freebsd.go similarity index 96% rename from internal/restic/node_freebsd.go rename to internal/fs/node_freebsd.go index 6d2dd1d98..8796358b0 100644 --- a/internal/restic/node_freebsd.go +++ b/internal/fs/node_freebsd.go @@ -1,7 +1,7 @@ //go:build freebsd // +build freebsd -package restic +package fs import "syscall" diff --git a/internal/restic/node_linux.go b/internal/fs/node_linux.go similarity index 88% rename from internal/restic/node_linux.go rename to internal/fs/node_linux.go index 6311a224b..1cb4ee1ae 100644 --- a/internal/restic/node_linux.go +++ b/internal/fs/node_linux.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "path/filepath" @@ -7,11 +7,10 @@ import ( "golang.org/x/sys/unix" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" ) func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { - dir, err := fs.Open(filepath.Dir(path)) + dir, err := Open(filepath.Dir(path)) if err != nil { return errors.WithStack(err) } diff --git a/internal/restic/node_netbsd.go b/internal/fs/node_netbsd.go similarity index 58% rename from internal/restic/node_netbsd.go rename to internal/fs/node_netbsd.go index a53412afb..c71e4bdf5 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/fs/node_netbsd.go @@ -1,8 +1,10 @@ -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -14,12 +16,12 @@ func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } // nodeRestoreExtendedAttributes is a no-op on netbsd. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on netbsd. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -29,11 +31,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on netbsd. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op on netbsd. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_openbsd.go b/internal/fs/node_openbsd.go similarity index 57% rename from internal/restic/node_openbsd.go rename to internal/fs/node_openbsd.go index bbba89f2c..f74f2ae00 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/fs/node_openbsd.go @@ -1,8 +1,10 @@ -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -14,12 +16,12 @@ func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } // nodeRestoreExtendedAttributes is a no-op on openbsd. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on openbsd. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -29,11 +31,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on openbsd. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // fillGenericAttributes is a no-op on openbsd. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_solaris.go b/internal/fs/node_solaris.go similarity index 95% rename from internal/restic/node_solaris.go rename to internal/fs/node_solaris.go index 114d11766..3f025b334 100644 --- a/internal/restic/node_solaris.go +++ b/internal/fs/node_solaris.go @@ -1,4 +1,4 @@ -package restic +package fs import "syscall" diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go new file mode 100644 index 000000000..e7f608352 --- /dev/null +++ b/internal/fs/node_test.go @@ -0,0 +1,324 @@ +package fs + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" +) + +func BenchmarkNodeFillUser(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + rtest.OK(t, err) + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func BenchmarkNodeFromFileInfo(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + if err != nil { + t.Fatal(err) + } + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func parseTime(s string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05.999", s) + if err != nil { + panic(err) + } + + return t.Local() +} + +var nodeTests = []restic.Node{ + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetuid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile2", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetgid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSticky", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSticky, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSymlink", + Type: "symlink", + LinkTarget: "invalid", + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0777 | os.ModeSymlink, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + + // include "testFile" and "testDir" again with slightly different + // metadata, so we can test if CreateAt works with pre-existing files. + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testXattrFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrFileMacOSResourceFork", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "com.apple.ResourceFork", Value: []byte("bar")}, + }, + }, +} + +func TestNodeRestoreAt(t *testing.T) { + tempdir := t.TempDir() + + for _, test := range nodeTests { + t.Run("", func(t *testing.T) { + var nodePath string + if test.ExtendedAttributes != nil { + if runtime.GOOS == "windows" { + // In windows extended attributes are case insensitive and windows returns + // the extended attributes in UPPER case. + // Update the tests to use UPPER case xattr names for windows. + extAttrArr := test.ExtendedAttributes + // Iterate through the array using pointers + for i := 0; i < len(extAttrArr); i++ { + extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) + } + } + for _, attr := range test.ExtendedAttributes { + if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { + t.Skipf("attr %v only relevant on macOS", attr.Name) + } + } + + // tempdir might be backed by a filesystem that does not support + // extended attributes + nodePath = test.Name + defer func() { + _ = os.Remove(nodePath) + }() + } else { + nodePath = filepath.Join(tempdir, test.Name) + } + rtest.OK(t, NodeCreateAt(&test, nodePath)) + rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) + + fi, err := os.Lstat(nodePath) + rtest.OK(t, err) + + n2, err := NodeFromFileInfo(nodePath, fi, false) + rtest.OK(t, err) + n3, err := NodeFromFileInfo(nodePath, fi, true) + rtest.OK(t, err) + rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) + + rtest.Assert(t, test.Name == n2.Name, + "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) + rtest.Assert(t, test.Type == n2.Type, + "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) + rtest.Assert(t, test.Size == n2.Size, + "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) + + if runtime.GOOS != "windows" { + rtest.Assert(t, test.UID == n2.UID, + "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) + rtest.Assert(t, test.GID == n2.GID, + "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) + if test.Type != "symlink" { + // On OpenBSD only root can set sticky bit (see sticky(8)). + if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { + rtest.Assert(t, test.Mode == n2.Mode, + "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) + } + } + } + + AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) + AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) + if len(n2.ExtendedAttributes) == 0 { + n2.ExtendedAttributes = nil + } + rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), + "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) + }) + } +} + +func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { + var equal bool + + // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd + if nodeType == "symlink" { + switch runtime.GOOS { + case "darwin", "freebsd", "openbsd", "netbsd", "solaris": + return + } + } + + switch runtime.GOOS { + case "darwin": + // HFS+ timestamps don't support sub-second precision, + // see https://en.wikipedia.org/wiki/Comparison_of_file_systems + diff := int(t1.Sub(t2).Seconds()) + equal = diff == 0 + default: + equal = t1.Equal(t2) + } + + rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) +} + +func TestNodeRestoreMetadataError(t *testing.T) { + tempdir := t.TempDir() + + node := &nodeTests[0] + nodePath := filepath.Join(tempdir, node.Name) + + // This will fail because the target file does not exist + err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) + test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") +} diff --git a/internal/restic/node_unix.go b/internal/fs/node_unix.go similarity index 97% rename from internal/restic/node_unix.go rename to internal/fs/node_unix.go index 976cd7b03..fb247ac99 100644 --- a/internal/restic/node_unix.go +++ b/internal/fs/node_unix.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" diff --git a/internal/restic/node_unix_test.go b/internal/fs/node_unix_test.go similarity index 94% rename from internal/restic/node_unix_test.go rename to internal/fs/node_unix_test.go index 9ea7b1725..b505357f2 100644 --- a/internal/restic/node_unix_test.go +++ b/internal/fs/node_unix_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { return fi, true } -func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { t.Helper() if uint32(node.Mode.Perm()) != uint32(stat.Mode&0777) { t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) @@ -80,7 +81,7 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { } -func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { if node.Device != uint64(stat.Rdev) { t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) } diff --git a/internal/restic/node_windows.go b/internal/fs/node_windows.go similarity index 88% rename from internal/restic/node_windows.go rename to internal/fs/node_windows.go index 3f836ae61..90fa3462c 100644 --- a/internal/restic/node_windows.go +++ b/internal/fs/node_windows.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "encoding/json" @@ -14,7 +14,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "golang.org/x/sys/windows" ) @@ -82,12 +82,12 @@ func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error } // restore extended attributes for windows -func nodeRestoreExtendedAttributes(node *Node, path string) (err error) { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) { count := len(node.ExtendedAttributes) if count > 0 { - eas := make([]fs.ExtendedAttribute, count) + eas := make([]ExtendedAttribute, count) for i, attr := range node.ExtendedAttributes { - eas[i] = fs.ExtendedAttribute{Name: attr.Name, Value: attr.Value} + eas[i] = ExtendedAttribute{Name: attr.Name, Value: attr.Value} } if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { return errExt @@ -97,9 +97,9 @@ func nodeRestoreExtendedAttributes(node *Node, path string) (err error) { } // fill extended attributes in the node. This also includes the Generic attributes for windows. -func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { +func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(node.Type, path, false); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(node.Type, path, false); fileHandle == 0 { return nil } if err != nil { @@ -107,8 +107,8 @@ func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { } defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call //Get the windows Extended Attributes using the file handle - var extAtts []fs.ExtendedAttribute - extAtts, err = fs.GetFileEA(fileHandle) + var extAtts []ExtendedAttribute + extAtts, err = GetFileEA(fileHandle) debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) if err != nil { return errors.Errorf("get EA failed for path %v, with: %v", path, err) @@ -119,7 +119,7 @@ func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA for _, attr := range extAtts { - extendedAttr := ExtendedAttribute{ + extendedAttr := restic.ExtendedAttribute{ Name: attr.Name, Value: attr.Value, } @@ -139,9 +139,9 @@ func closeFileHandle(fileHandle windows.Handle, path string) { // restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. // The Windows API requires setting of all the Extended Attributes in one call. -func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { +func restoreExtendedAttributes(nodeType, path string, eas []ExtendedAttribute) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(nodeType, path, true); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(nodeType, path, true); fileHandle == 0 { return nil } if err != nil { @@ -150,7 +150,7 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call // clear old unexpected xattrs by setting them to an empty value - oldEAs, err := fs.GetFileEA(fileHandle) + oldEAs, err := GetFileEA(fileHandle) if err != nil { return err } @@ -165,11 +165,11 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute } if !found { - eas = append(eas, fs.ExtendedAttribute{Name: oldEA.Name, Value: nil}) + eas = append(eas, ExtendedAttribute{Name: oldEA.Name, Value: nil}) } } - if err = fs.SetFileEA(fileHandle, eas); err != nil { + if err = SetFileEA(fileHandle, eas); err != nil { return errors.Errorf("set EA failed for path %v, with: %v", path, err) } return nil @@ -210,7 +210,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func nodeRestoreGenericAttributes(node *Node, path string, warn func(msg string)) (err error) { +func nodeRestoreGenericAttributes(node *restic.Node, path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -230,19 +230,19 @@ func nodeRestoreGenericAttributes(node *Node, path string, warn func(msg string) } } if windowsAttributes.SecurityDescriptor != nil { - if err := fs.SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { + if err := SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { errs = append(errs, fmt.Errorf("error restoring security descriptor for: %s : %v", path, err)) } } - HandleUnknownGenericAttributesFound(unknownAttribs, warn) + restic.HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.CombineErrors(errs...) } // genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. -func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { +func genericAttributesToWindowsAttrs(attrs map[restic.GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []restic.GenericAttributeType, err error) { waValue := reflect.ValueOf(&windowsAttributes).Elem() - unknownAttribs, err = GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + unknownAttribs, err = restic.GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") return windowsAttributes, unknownAttribs, err } @@ -289,14 +289,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should be encrypted. err = encryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, encrypt file call fails. // The readonly and system flags will be set again at the end of this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) } @@ -317,14 +317,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should not be encrypted, but its already encrypted. Decrypt it. err = decryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, decrypt file call fails. // The readonly and system flags will be set again after this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) } @@ -365,7 +365,7 @@ func decryptFile(pathPointer *uint16) error { // Created time and Security Descriptors. // It also checks if the volume supports extended attributes and stores the result in a map // so that it does not have to be checked again for subsequent calls for paths in the same volume. -func nodeFillGenericAttributes(node *Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(node *restic.Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows // Also do not allow processing of extended attributes for ADS. @@ -392,7 +392,7 @@ func nodeFillGenericAttributes(node *Node, path string, fi os.FileInfo, stat *st if err != nil { return false, err } - if sd, err = fs.GetSecurityDescriptor(path); err != nil { + if sd, err = GetSecurityDescriptor(path); err != nil { return allowExtended, err } } @@ -422,7 +422,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If not found, check if EA is supported with manually prepared volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeName + `\`) // If the prepared volume name is not valid, we will fetch the actual volume name next. if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { debug.Log("Error checking if extended attributes are supported for prepared volume name %s: %v", volumeName, err) @@ -432,7 +432,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { } } // If an entry is not found, get the actual volume name using the GetVolumePathName function - volumeNameActual, err := fs.GetVolumePathName(path) + volumeNameActual, err := GetVolumePathName(path) if err != nil { debug.Log("Error getting actual volume name %s for path %s: %v", volumeName, path, err) // There can be multiple errors like path does not exist, bad network path, etc. @@ -447,7 +447,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeNameActual + `\`) // Debug log for cases where the prepared volume name is not valid if err != nil { debug.Log("Error checking if extended attributes are supported for actual volume name %s: %v", volumeNameActual, err) @@ -496,10 +496,10 @@ func prepareVolumeName(path string) (volumeName string, err error) { } // windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection -func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[restic.GenericAttributeType]json.RawMessage, err error) { // Get the value of the WindowsAttributes windowsAttributesValue := reflect.ValueOf(windowsAttributes) - return OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) + return restic.OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) } // getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. diff --git a/internal/restic/node_windows_test.go b/internal/fs/node_windows_test.go similarity index 90% rename from internal/restic/node_windows_test.go rename to internal/fs/node_windows_test.go index e78c8cb96..046c1984c 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -1,7 +1,7 @@ //go:build windows // +build windows -package restic +package fs import ( "encoding/base64" @@ -15,7 +15,7 @@ import ( "time" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -23,10 +23,10 @@ import ( func TestRestoreSecurityDescriptors(t *testing.T) { t.Parallel() tempDir := t.TempDir() - for i, sd := range fs.TestFileSDs { + for i, sd := range TestFileSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "file", fmt.Sprintf("testfile%d", i)) } - for i, sd := range fs.TestDirSDs { + for i, sd := range TestDirSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "dir", fmt.Sprintf("testdir%d", i)) } } @@ -47,17 +47,17 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir, fileType, f sdByteFromRestoredNode := getWindowsAttr(t, testPath, node).SecurityDescriptor // Get the security descriptor for the test path after the restore. - sdBytesFromRestoredPath, err := fs.GetSecurityDescriptor(testPath) + sdBytesFromRestoredPath, err := GetSecurityDescriptor(testPath) test.OK(t, errors.Wrapf(err, "Error while getting the security descriptor for: %s", testPath)) // Compare the input SD and the SD got from the restored file. - fs.CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) // Compare the SD got from node constructed from the restored file info and the SD got directly from the restored file. - fs.CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) } -func getNode(name string, fileType string, genericAttributes map[GenericAttributeType]json.RawMessage) Node { - return Node{ +func getNode(name string, fileType string, genericAttributes map[restic.GenericAttributeType]json.RawMessage) restic.Node { + return restic.Node{ Name: name, Type: fileType, Mode: 0644, @@ -68,7 +68,7 @@ func getNode(name string, fileType string, genericAttributes map[GenericAttribut } } -func getWindowsAttr(t *testing.T, testPath string, node *Node) WindowsAttributes { +func getWindowsAttr(t *testing.T, testPath string, node *restic.Node) WindowsAttributes { windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, errors.Wrapf(err, "Error getting windows attr from generic attr: %s", testPath)) test.Assert(t, len(unknownAttribs) == 0, "Unknown attribs found: %s for: %s", unknownAttribs, testPath) @@ -83,12 +83,12 @@ func TestRestoreCreationTime(t *testing.T) { creationTimeAttribute := getCreationTime(fi, path) test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) //Using the temp dir creation time as the test creation time for the test file and folder - runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) + runGenericAttributesTest(t, path, restic.TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) } func TestRestoreFileAttributes(t *testing.T) { t.Parallel() - genericAttributeName := TypeFileAttributes + genericAttributeName := restic.TypeFileAttributes tempDir := t.TempDir() normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) @@ -110,7 +110,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, fileAttr := range fileAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testfile%d", i), Type: "file", @@ -143,7 +143,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, folderAttr := range folderAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testdirectory%d", i), Type: "dir", @@ -158,10 +158,10 @@ func TestRestoreFileAttributes(t *testing.T) { } } -func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -183,7 +183,7 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName } runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) } -func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []restic.Node, tempDir string, genericAttr restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { for _, testNode := range expectedNodes { testPath, node := restoreAndGetNode(t, tempDir, &testNode, warningExpected) @@ -195,7 +195,7 @@ func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDi } } -func restoreAndGetNode(t *testing.T, tempDir string, testNode *Node, warningExpected bool) (string, *Node) { +func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warningExpected bool) (string, *restic.Node) { testPath := filepath.Join(tempDir, "001", testNode.Name) err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) @@ -230,16 +230,16 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *Node, warningExpe return testPath, nodeFromFileInfo } -const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" +const TypeSomeNewAttribute restic.GenericAttributeType = "MockAttributes.SomeNewAttribute" func TestNewGenericAttributeType(t *testing.T) { t.Parallel() - newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute := map[restic.GenericAttributeType]json.RawMessage{} newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -271,7 +271,7 @@ func TestNewGenericAttributeType(t *testing.T) { func TestRestoreExtendedAttributes(t *testing.T) { t.Parallel() tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -279,7 +279,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -290,7 +290,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -312,12 +312,12 @@ func TestRestoreExtendedAttributes(t *testing.T) { test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) }() - extAttr, err := fs.GetFileEA(handle) + extAttr, err := GetFileEA(handle) test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) for _, expectedExtAttr := range node.ExtendedAttributes { - var foundExtAttr *fs.ExtendedAttribute + var foundExtAttr *ExtendedAttribute for _, ea := range extAttr { if strings.EqualFold(ea.Name, expectedExtAttr.Name) { foundExtAttr = &ea @@ -491,13 +491,13 @@ func TestPrepareVolumeName(t *testing.T) { test.Equals(t, tc.expectedVolume, volume) if tc.isRealPath { - isEASupportedVolume, err := fs.PathSupportsExtendedAttributes(volume + `\`) + isEASupportedVolume, err := PathSupportsExtendedAttributes(volume + `\`) // If the prepared volume name is not valid, we will next fetch the actual volume name. test.OK(t, err) test.Equals(t, tc.expectedEASupported, isEASupportedVolume) - actualVolume, err := fs.GetVolumePathName(tc.path) + actualVolume, err := GetVolumePathName(tc.path) test.OK(t, err) test.Equals(t, tc.expectedVolume, actualVolume) } diff --git a/internal/restic/node_xattr.go b/internal/fs/node_xattr.go similarity index 80% rename from internal/restic/node_xattr.go rename to internal/fs/node_xattr.go index 062ef4345..11bdf382b 100644 --- a/internal/restic/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/pkg/xattr" ) @@ -65,16 +66,16 @@ func handleXattrErr(err error) error { } // nodeRestoreGenericAttributes is no-op. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } -func nodeRestoreExtendedAttributes(node *Node, path string) error { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { err := setxattr(path, attr.Name, attr.Value) @@ -101,7 +102,7 @@ func nodeRestoreExtendedAttributes(node *Node, path string) error { return nil } -func nodeFillExtendedAttributes(node *Node, path string, ignoreListError bool) error { +func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError bool) error { xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { @@ -111,14 +112,14 @@ func nodeFillExtendedAttributes(node *Node, path string, ignoreListError bool) e return err } - node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) + node.ExtendedAttributes = make([]restic.ExtendedAttribute, 0, len(xattrs)) for _, attr := range xattrs { attrVal, err := getxattr(path, attr) if err != nil { fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) continue } - attr := ExtendedAttribute{ + attr := restic.ExtendedAttribute{ Name: attr, Value: attrVal, } diff --git a/internal/restic/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go similarity index 77% rename from internal/restic/node_xattr_all_test.go rename to internal/fs/node_xattr_all_test.go index 30d29a6ed..39670d6e1 100644 --- a/internal/restic/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris || windows // +build darwin freebsd linux solaris windows -package restic +package fs import ( "os" @@ -10,10 +10,11 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { +func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribute) { if runtime.GOOS == "windows" { // windows seems to convert the xattr name to upper case for i := range attrs { @@ -21,13 +22,13 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { } } - node := &Node{ + node := &restic.Node{ Type: "file", ExtendedAttributes: attrs, } rtest.OK(t, nodeRestoreExtendedAttributes(node, file)) - nodeActual := &Node{ + nodeActual := &restic.Node{ Type: "file", } rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false)) @@ -40,14 +41,14 @@ func TestOverwriteXattr(t *testing.T) { file := filepath.Join(dir, "file") rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.foo", Value: []byte("bar"), }, }) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.other", Value: []byte("some"), diff --git a/internal/restic/node_xattr_test.go b/internal/fs/node_xattr_test.go similarity index 98% rename from internal/restic/node_xattr_test.go rename to internal/fs/node_xattr_test.go index 5ce77bd28..d948e3b31 100644 --- a/internal/restic/node_xattr_test.go +++ b/internal/fs/node_xattr_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "os" diff --git a/internal/restic/node.go b/internal/restic/node.go index e23d39f1b..8bf97e59c 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -4,12 +4,10 @@ import ( "encoding/json" "fmt" "os" - "os/user" "reflect" "strconv" "strings" "sync" - "syscall" "time" "unicode/utf8" @@ -18,7 +16,6 @@ import ( "bytes" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/fs" ) // ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. @@ -133,49 +130,6 @@ func (node Node) String() string { mode|node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name) } -// NodeFromFileInfo returns a new node from the given path and FileInfo. It -// returns the first error that is encountered, together with a node. -func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*Node, error) { - mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky - node := &Node{ - Path: path, - Name: fi.Name(), - Mode: fi.Mode() & mask, - ModTime: fi.ModTime(), - } - - node.Type = nodeTypeFromFileInfo(fi) - if node.Type == "file" { - node.Size = uint64(fi.Size()) - } - - err := nodeFillExtra(node, path, fi, ignoreXattrListError) - return node, err -} - -func nodeTypeFromFileInfo(fi os.FileInfo) string { - switch fi.Mode() & os.ModeType { - case 0: - return "file" - case os.ModeDir: - return "dir" - case os.ModeSymlink: - return "symlink" - case os.ModeDevice | os.ModeCharDevice: - return "chardev" - case os.ModeDevice: - return "dev" - case os.ModeNamedPipe: - return "fifo" - case os.ModeSocket: - return "socket" - case os.ModeIrregular: - return "irregular" - } - - return "" -} - // GetExtendedAttribute gets the extended attribute. func (node Node) GetExtendedAttribute(a string) []byte { for _, attr := range node.ExtendedAttributes { @@ -186,162 +140,6 @@ func (node Node) GetExtendedAttribute(a string) []byte { return nil } -// NodeCreateAt creates the node at the given path but does NOT restore node meta data. -func NodeCreateAt(node *Node, path string) error { - debug.Log("create node %v at %v", node.Name, path) - - switch node.Type { - case "dir": - if err := nodeCreateDirAt(node, path); err != nil { - return err - } - case "file": - if err := nodeCreateFileAt(path); err != nil { - return err - } - case "symlink": - if err := nodeCreateSymlinkAt(node, path); err != nil { - return err - } - case "dev": - if err := nodeCreateDevAt(node, path); err != nil { - return err - } - case "chardev": - if err := nodeCreateCharDevAt(node, path); err != nil { - return err - } - case "fifo": - if err := nodeCreateFifoAt(path); err != nil { - return err - } - case "socket": - return nil - default: - return errors.Errorf("filetype %q not implemented", node.Type) - } - - return nil -} - -// NodeRestoreMetadata restores node metadata -func NodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { - err := nodeRestoreMetadata(node, path, warn) - if err != nil { - // It is common to have permission errors for folders like /home - // unless you're running as root, so ignore those. - if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { - debug.Log("not running as root, ignoring permission error for %v: %v", - path, err) - return nil - } - debug.Log("restoreMetadata(%s) error %v", path, err) - } - - return err -} - -func nodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { - var firsterr error - - if err := lchown(path, int(node.UID), int(node.GID)); err != nil { - firsterr = errors.WithStack(err) - } - - if err := nodeRestoreExtendedAttributes(node, path); err != nil { - debug.Log("error restoring extended attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { - debug.Log("error restoring generic attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := NodeRestoreTimestamps(node, path); err != nil { - debug.Log("error restoring timestamps for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows - // calling Chmod below will no longer allow any modifications to be made on the file and the - // calls above would fail. - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr == nil { - firsterr = errors.WithStack(err) - } - } - } - - return firsterr -} - -func NodeRestoreTimestamps(node *Node, path string) error { - var utimes = [...]syscall.Timespec{ - syscall.NsecToTimespec(node.AccessTime.UnixNano()), - syscall.NsecToTimespec(node.ModTime.UnixNano()), - } - - if node.Type == "symlink" { - return nodeRestoreSymlinkTimestamps(path, utimes) - } - - if err := syscall.UtimesNano(path, utimes[:]); err != nil { - return errors.Wrap(err, "UtimesNano") - } - - return nil -} - -func nodeCreateDirAt(node *Node, path string) error { - err := fs.Mkdir(path, node.Mode) - if err != nil && !os.IsExist(err) { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateFileAt(path string) error { - f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return errors.WithStack(err) - } - - if err := f.Close(); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateSymlinkAt(node *Node, path string) error { - if err := fs.Symlink(node.LinkTarget, path); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateDevAt(node *Node, path string) error { - return mknod(path, syscall.S_IFBLK|0600, node.Device) -} - -func nodeCreateCharDevAt(node *Node, path string) error { - return mknod(path, syscall.S_IFCHR|0600, node.Device) -} - -func nodeCreateFifoAt(path string) error { - return mkfifo(path, 0600) -} - // FixTime returns a time.Time which can safely be used to marshal as JSON. If // the timestamp is earlier than year zero, the year is set to zero. In the same // way, if the year is larger than 9999, the year is set to 9999. Other than @@ -576,127 +374,6 @@ func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { return true } -func nodeFillUser(node *Node, stat *statT) { - uid, gid := stat.uid(), stat.gid() - node.UID, node.GID = uid, gid - node.User = lookupUsername(uid) - node.Group = lookupGroup(gid) -} - -var ( - uidLookupCache = make(map[uint32]string) - uidLookupCacheMutex = sync.RWMutex{} -) - -// Cached user name lookup by uid. Returns "" when no name can be found. -func lookupUsername(uid uint32) string { - uidLookupCacheMutex.RLock() - username, ok := uidLookupCache[uid] - uidLookupCacheMutex.RUnlock() - - if ok { - return username - } - - u, err := user.LookupId(strconv.Itoa(int(uid))) - if err == nil { - username = u.Username - } - - uidLookupCacheMutex.Lock() - uidLookupCache[uid] = username - uidLookupCacheMutex.Unlock() - - return username -} - -var ( - gidLookupCache = make(map[uint32]string) - gidLookupCacheMutex = sync.RWMutex{} -) - -// Cached group name lookup by gid. Returns "" when no name can be found. -func lookupGroup(gid uint32) string { - gidLookupCacheMutex.RLock() - group, ok := gidLookupCache[gid] - gidLookupCacheMutex.RUnlock() - - if ok { - return group - } - - g, err := user.LookupGroupId(strconv.Itoa(int(gid))) - if err == nil { - group = g.Name - } - - gidLookupCacheMutex.Lock() - gidLookupCache[gid] = group - gidLookupCacheMutex.Unlock() - - return group -} - -func nodeFillExtra(node *Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { - stat, ok := toStatT(fi.Sys()) - if !ok { - // fill minimal info with current values for uid, gid - node.UID = uint32(os.Getuid()) - node.GID = uint32(os.Getgid()) - node.ChangeTime = node.ModTime - return nil - } - - node.Inode = uint64(stat.ino()) - node.DeviceID = uint64(stat.dev()) - - nodeFillTimes(node, stat) - - nodeFillUser(node, stat) - - switch node.Type { - case "file": - node.Size = uint64(stat.size()) - node.Links = uint64(stat.nlink()) - case "dir": - case "symlink": - var err error - node.LinkTarget, err = fs.Readlink(path) - node.Links = uint64(stat.nlink()) - if err != nil { - return errors.WithStack(err) - } - case "dev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "chardev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "fifo": - case "socket": - default: - return errors.Errorf("unsupported file type %q", node.Type) - } - - allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) - if allowExtended { - // Skip processing ExtendedAttributes if allowExtended is false. - err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) - } - return err -} - -func mkfifo(path string, mode uint32) (err error) { - return mknod(path, mode|syscall.S_IFIFO, 0) -} - -func nodeFillTimes(node *Node, stat *statT) { - ctim := stat.ctim() - atim := stat.atim() - node.ChangeTime = time.Unix(ctim.Unix()) - node.AccessTime = time.Unix(atim.Unix()) -} - // HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { for _, unknownAttrib := range unknownAttribs { diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 075dd5cc5..38a17cb09 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -3,315 +3,12 @@ package restic import ( "encoding/json" "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" - rtest "github.com/restic/restic/internal/test" ) -func BenchmarkNodeFillUser(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - rtest.OK(t, err) - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func BenchmarkNodeFromFileInfo(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - if err != nil { - t.Fatal(err) - } - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func parseTime(s string) time.Time { - t, err := time.Parse("2006-01-02 15:04:05.999", s) - if err != nil { - panic(err) - } - - return t.Local() -} - -var nodeTests = []Node{ - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetuid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile2", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetgid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSticky", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSticky, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSymlink", - Type: "symlink", - LinkTarget: "invalid", - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0777 | os.ModeSymlink, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - - // include "testFile" and "testDir" again with slightly different - // metadata, so we can test if CreateAt works with pre-existing files. - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testXattrFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrFileMacOSResourceFork", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"com.apple.ResourceFork", []byte("bar")}, - }, - }, -} - -func TestNodeRestoreAt(t *testing.T) { - tempdir := t.TempDir() - - for _, test := range nodeTests { - t.Run("", func(t *testing.T) { - var nodePath string - if test.ExtendedAttributes != nil { - if runtime.GOOS == "windows" { - // In windows extended attributes are case insensitive and windows returns - // the extended attributes in UPPER case. - // Update the tests to use UPPER case xattr names for windows. - extAttrArr := test.ExtendedAttributes - // Iterate through the array using pointers - for i := 0; i < len(extAttrArr); i++ { - extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) - } - } - for _, attr := range test.ExtendedAttributes { - if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { - t.Skipf("attr %v only relevant on macOS", attr.Name) - } - } - - // tempdir might be backed by a filesystem that does not support - // extended attributes - nodePath = test.Name - defer func() { - _ = os.Remove(nodePath) - }() - } else { - nodePath = filepath.Join(tempdir, test.Name) - } - rtest.OK(t, NodeCreateAt(&test, nodePath)) - rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) - - fi, err := os.Lstat(nodePath) - rtest.OK(t, err) - - n2, err := NodeFromFileInfo(nodePath, fi, false) - rtest.OK(t, err) - n3, err := NodeFromFileInfo(nodePath, fi, true) - rtest.OK(t, err) - rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) - - rtest.Assert(t, test.Name == n2.Name, - "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) - rtest.Assert(t, test.Type == n2.Type, - "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) - rtest.Assert(t, test.Size == n2.Size, - "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - - if runtime.GOOS != "windows" { - rtest.Assert(t, test.UID == n2.UID, - "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) - rtest.Assert(t, test.GID == n2.GID, - "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != "symlink" { - // On OpenBSD only root can set sticky bit (see sticky(8)). - if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { - rtest.Assert(t, test.Mode == n2.Mode, - "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) - } - } - } - - AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) - AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) - if len(n2.ExtendedAttributes) == 0 { - n2.ExtendedAttributes = nil - } - rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), - "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) - }) - } -} - -func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { - var equal bool - - // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd - if nodeType == "symlink" { - switch runtime.GOOS { - case "darwin", "freebsd", "openbsd", "netbsd", "solaris": - return - } - } - - switch runtime.GOOS { - case "darwin": - // HFS+ timestamps don't support sub-second precision, - // see https://en.wikipedia.org/wiki/Comparison_of_file_systems - diff := int(t1.Sub(t2).Seconds()) - equal = diff == 0 - default: - equal = t1.Equal(t2) - } - - rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) -} - func parseTimeNano(t testing.TB, s string) time.Time { // 2006-01-02T15:04:05.999999999Z07:00 ts, err := time.Parse(time.RFC3339Nano, s) @@ -397,14 +94,3 @@ func TestSymlinkSerializationFormat(t *testing.T) { test.Assert(t, n2.LinkTargetRaw == nil, "quoted link target is just a helper field and must be unset after decoding") } } - -func TestNodeRestoreMetadataError(t *testing.T) { - tempdir := t.TempDir() - - node := &nodeTests[0] - nodePath := filepath.Join(tempdir, node.Name) - - // This will fail because the target file does not exist - err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) - test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") -} diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index 8e0b3587a..cdd6b3c18 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -86,7 +87,7 @@ func TestNodeComparison(t *testing.T) { fi, err := os.Lstat("tree_test.go") rtest.OK(t, err) - node, err := restic.NodeFromFileInfo("tree_test.go", fi, false) + node, err := fs.NodeFromFileInfo("tree_test.go", fi, false) rtest.OK(t, err) n2 := *node @@ -127,7 +128,7 @@ func TestTreeEqualSerialization(t *testing.T) { for _, fn := range files[:i] { fi, err := os.Lstat(fn) rtest.OK(t, err) - node, err := restic.NodeFromFileInfo(fn, fi, false) + node, err := fs.NodeFromFileInfo(fn, fi, false) rtest.OK(t, err) rtest.OK(t, tree.Insert(node)) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 83644c7ac..26b6f3474 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -272,7 +272,7 @@ func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) e return errors.Wrap(err, "RemoveNode") } - err := restic.NodeCreateAt(node, target) + err := fs.NodeCreateAt(node, target) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) return err @@ -288,7 +288,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return nil } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := restic.NodeRestoreMetadata(node, target, res.Warn) + err := fs.NodeRestoreMetadata(node, target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 4764bed2d..9fcdfc48d 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -16,6 +16,7 @@ import ( "unsafe" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -263,7 +264,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer { //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY } - attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + attrs, err := fs.WindowsAttrsToGenericAttributes(fs.WindowsAttributes{FileAttributes: &fileattr}) test.OK(t, err) return attrs }