diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e7c346d3a..4f0990843 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -248,7 +248,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I // nodeFromFileInfo returns the restic node from an os.FileInfo. func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + node, err := fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) if !arch.WithAtime { node.AccessTime = node.ModTime } @@ -446,7 +446,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous } switch { - case fs.IsRegularFile(fi): + case fi.Mode().IsRegular(): debug.Log(" %v regular file", target) // check if the file has not changed before performing a fopen operation (more expensive, specially @@ -505,7 +505,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous } // make sure it's still a file - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { err = errors.Errorf("file %v changed type, refusing to archive", fi.Name()) _ = file.Close() err = arch.error(abstarget, err) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c54f9ea33..d67b5b06a 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -557,7 +557,7 @@ func rename(t testing.TB, oldname, newname string) { } func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node { - node, err := restic.NodeFromFileInfo(filename, fi, false) + node, err := fs.NodeFromFileInfo(filename, fi, false) if err != nil { t.Fatal(err) } @@ -1664,15 +1664,6 @@ type MockFS struct { bytesRead map[string]int // tracks bytes read from all opened files } -func (m *MockFS) Open(name string) (fs.File, error) { - f, err := m.FS.Open(name) - if err != nil { - return f, err - } - - return MockFile{File: f, fs: m, filename: name}, nil -} - func (m *MockFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) { f, err := m.FS.OpenFile(name, flag, perm) if err != nil { @@ -2061,14 +2052,6 @@ type TrackFS struct { m sync.Mutex } -func (m *TrackFS) Open(name string) (fs.File, error) { - m.m.Lock() - m.opened[name]++ - m.m.Unlock() - - return m.FS.Open(name) -} - func (m *TrackFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) { m.m.Lock() m.opened[name]++ @@ -2291,7 +2274,7 @@ func TestMetadataChanged(t *testing.T) { // get metadata fi := lstat(t, "testfile") - want, err := restic.NodeFromFileInfo("testfile", fi, false) + want, err := fs.NodeFromFileInfo("testfile", fi, false) if err != nil { t.Fatal(err) } diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index 4a380dff8..d91d993dd 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -48,7 +48,7 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo { func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) { fi := lstat(t, name) - want, err := restic.NodeFromFileInfo(name, fi, false) + want, err := fs.NodeFromFileInfo(name, fi, false) rtest.OK(t, err) _, node := snapshot(t, repo, fs.Local{}, nil, name) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 409bdedd0..948d7ce3c 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -50,7 +50,7 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + return fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) } return s, ctx, wg @@ -72,7 +72,7 @@ func TestFileSaver(t *testing.T) { var results []FutureNode for _, filename := range files { - f, err := testFs.Open(filename) + f, err := testFs.OpenFile(filename, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index 106e68445..8bd854904 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -169,7 +169,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } return nil case TestFile: - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { t.Errorf("is not a regular file: %v", path) return nil } @@ -208,7 +208,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { }) // then, traverse the directory again, looking for additional files - err := fs.Walk(target, func(path string, fi os.FileInfo, err error) error { + err := filepath.Walk(target, func(path string, fi os.FileInfo, err error) error { if err != nil { return err } diff --git a/internal/archiver/testing_test.go b/internal/archiver/testing_test.go index ff3bd3668..bb4b63a82 100644 --- a/internal/archiver/testing_test.go +++ b/internal/archiver/testing_test.go @@ -122,7 +122,7 @@ func TestTestCreateFiles(t *testing.T) { switch node := item.(type) { case TestFile: - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { t.Errorf("is not regular file: %v", name) continue } diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index 74afd7aa5..e474a3735 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -142,7 +142,7 @@ func TestSetGetFileEA(t *testing.T) { testFilePath, testFile := setupTestFile(t) testEAs := generateTestEAs(t, 3, testFilePath) fileHandle := openFile(t, testFilePath, windows.FILE_ATTRIBUTE_NORMAL) - defer closeFileHandle(t, testFilePath, testFile, fileHandle) + defer testCloseFileHandle(t, testFilePath, testFile, fileHandle) testSetGetEA(t, testFilePath, fileHandle, testEAs) } @@ -154,7 +154,7 @@ func TestSetGetFolderEA(t *testing.T) { testEAs := generateTestEAs(t, 3, testFolderPath) fileHandle := openFile(t, testFolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) - defer closeFileHandle(t, testFolderPath, nil, fileHandle) + defer testCloseFileHandle(t, testFolderPath, nil, fileHandle) testSetGetEA(t, testFolderPath, fileHandle, testEAs) } @@ -212,7 +212,7 @@ func openFile(t *testing.T, path string, attributes uint32) windows.Handle { return fileHandle } -func closeFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { +func testCloseFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { if testFile != nil { err := testFile.Close() if err != nil { diff --git a/internal/fs/file.go b/internal/fs/file.go index 85b202dc8..356b466c3 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -3,7 +3,6 @@ package fs import ( "fmt" "os" - "path/filepath" "time" ) @@ -75,15 +74,6 @@ func Lstat(name string) (os.FileInfo, error) { return os.Lstat(fixpath(name)) } -// Create creates the named file with mode 0666 (before umask), truncating -// it if it already exists. If successful, methods on the returned -// File can be used for I/O; the associated file descriptor has mode -// O_RDWR. -// If there is an error, it will be of type *PathError. -func Create(name string) (*os.File, error) { - return os.Create(fixpath(name)) -} - // Open opens a file for reading. func Open(name string) (File, error) { return os.Open(fixpath(name)) @@ -98,25 +88,6 @@ func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { return os.OpenFile(fixpath(name), flag, perm) } -// Walk walks the file tree rooted at root, calling walkFn for each file or -// directory in the tree, including root. All errors that arise visiting files -// and directories are filtered by walkFn. The files are walked in lexical -// order, which makes the output deterministic but means that for very -// large directories Walk can be inefficient. -// Walk does not follow symbolic links. -func Walk(root string, walkFn filepath.WalkFunc) error { - return filepath.Walk(fixpath(root), walkFn) -} - -// RemoveIfExists removes a file, returning no error if it does not exist. -func RemoveIfExists(filename string) error { - err := os.Remove(filename) - if err != nil && os.IsNotExist(err) { - err = nil - } - return err -} - // Chtimes changes the access and modification times of the named file, // similar to the Unix utime() or utimes() functions. // diff --git a/internal/fs/fs_local.go b/internal/fs/fs_local.go index 48c40dc90..0bcbf7f3a 100644 --- a/internal/fs/fs_local.go +++ b/internal/fs/fs_local.go @@ -18,16 +18,6 @@ func (fs Local) VolumeName(path string) string { return filepath.VolumeName(path) } -// Open opens a file for reading. -func (fs Local) Open(name string) (File, error) { - f, err := os.Open(fixpath(name)) - if err != nil { - return nil, err - } - _ = setFlags(f) - return f, nil -} - // OpenFile is the generalized open call; most users will use Open // or Create instead. It opens the named file with specified flag // (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful, diff --git a/internal/fs/fs_local_vss.go b/internal/fs/fs_local_vss.go index 718dfc46d..46b40d013 100644 --- a/internal/fs/fs_local_vss.go +++ b/internal/fs/fs_local_vss.go @@ -125,11 +125,6 @@ func (fs *LocalVss) DeleteSnapshots() { fs.snapshots = activeSnapshots } -// Open wraps the Open method of the underlying file system. -func (fs *LocalVss) Open(name string) (File, error) { - return os.Open(fs.snapshotPath(name)) -} - // OpenFile wraps the Open method of the underlying file system. func (fs *LocalVss) OpenFile(name string, flag int, perm os.FileMode) (File, error) { return os.OpenFile(fs.snapshotPath(name), flag, perm) diff --git a/internal/fs/fs_reader.go b/internal/fs/fs_reader.go index 47af74245..93a42f9eb 100644 --- a/internal/fs/fs_reader.go +++ b/internal/fs/fs_reader.go @@ -39,29 +39,6 @@ func (fs *Reader) VolumeName(_ string) string { return "" } -// Open opens a file for reading. -func (fs *Reader) Open(name string) (f File, err error) { - switch name { - case fs.Name: - fs.open.Do(func() { - f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) - }) - - if f == nil { - return nil, pathError("open", name, syscall.EIO) - } - - return f, nil - case "/", ".": - f = fakeDir{ - entries: []os.FileInfo{fs.fi()}, - } - return f, nil - } - - return nil, pathError("open", name, syscall.ENOENT) -} - func (fs *Reader) fi() os.FileInfo { return fakeFileInfo{ name: fs.Name, @@ -82,15 +59,25 @@ func (fs *Reader) OpenFile(name string, flag int, _ os.FileMode) (f File, err er fmt.Errorf("invalid combination of flags 0x%x", flag)) } - fs.open.Do(func() { - f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) - }) + switch name { + case fs.Name: + fs.open.Do(func() { + f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) + }) - if f == nil { - return nil, pathError("open", name, syscall.EIO) + if f == nil { + return nil, pathError("open", name, syscall.EIO) + } + + return f, nil + case "/", ".": + f = fakeDir{ + entries: []os.FileInfo{fs.fi()}, + } + return f, nil } - return f, nil + return nil, pathError("open", name, syscall.ENOENT) } // Stat returns a FileInfo describing the named file. If there is an error, it diff --git a/internal/fs/fs_reader_test.go b/internal/fs/fs_reader_test.go index d3ef5608a..9fa67b5ac 100644 --- a/internal/fs/fs_reader_test.go +++ b/internal/fs/fs_reader_test.go @@ -15,27 +15,6 @@ import ( "github.com/restic/restic/internal/test" ) -func verifyFileContentOpen(t testing.TB, fs FS, filename string, want []byte) { - f, err := fs.Open(filename) - if err != nil { - t.Fatal(err) - } - - buf, err := io.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - err = f.Close() - if err != nil { - t.Fatal(err) - } - - if !cmp.Equal(want, buf) { - t.Error(cmp.Diff(want, buf)) - } -} - func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte) { f, err := fs.OpenFile(filename, O_RDONLY, 0) if err != nil { @@ -58,7 +37,7 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte } func verifyDirectoryContents(t testing.TB, fs FS, dir string, want []string) { - f, err := fs.Open(dir) + f, err := fs.OpenFile(dir, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -96,7 +75,7 @@ func (s fiSlice) Swap(i, j int) { } func verifyDirectoryContentsFI(t testing.TB, fs FS, dir string, want []os.FileInfo) { - f, err := fs.Open(dir) + f, err := fs.OpenFile(dir, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -219,12 +198,6 @@ func TestFSReader(t *testing.T) { verifyDirectoryContentsFI(t, fs, ".", []os.FileInfo{fi}) }, }, - { - name: "file/Open", - f: func(t *testing.T, fs FS) { - verifyFileContentOpen(t, fs, filename, data) - }, - }, { name: "file/OpenFile", f: func(t *testing.T, fs FS) { @@ -245,7 +218,7 @@ func TestFSReader(t *testing.T) { { name: "file/Stat", f: func(t *testing.T, fs FS) { - f, err := fs.Open(filename) + f, err := fs.OpenFile(filename, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -417,7 +390,7 @@ func TestFSReaderMinFileSize(t *testing.T) { AllowEmptyFile: test.allowEmpty, } - f, err := fs.Open("testfile") + f, err := fs.OpenFile("testfile", os.O_RDONLY, 0) if err != nil { t.Fatal(err) } diff --git a/internal/fs/fs_track.go b/internal/fs/fs_track.go index 0c65a8564..366bbee76 100644 --- a/internal/fs/fs_track.go +++ b/internal/fs/fs_track.go @@ -15,16 +15,6 @@ type Track struct { FS } -// Open wraps the Open method of the underlying file system. -func (fs Track) Open(name string) (File, error) { - f, err := fs.FS.Open(fixpath(name)) - if err != nil { - return nil, err - } - - return newTrackFile(debug.Stack(), name, f), nil -} - // OpenFile wraps the OpenFile method of the underlying file system. func (fs Track) OpenFile(name string, flag int, perm os.FileMode) (File, error) { f, err := fs.FS.OpenFile(fixpath(name), flag, perm) diff --git a/internal/fs/helpers.go b/internal/fs/helpers.go deleted file mode 100644 index 4dd1e0e73..000000000 --- a/internal/fs/helpers.go +++ /dev/null @@ -1,13 +0,0 @@ -package fs - -import "os" - -// IsRegularFile returns true if fi belongs to a normal file. If fi is nil, -// false is returned. -func IsRegularFile(fi os.FileInfo) bool { - if fi == nil { - return false - } - - return fi.Mode()&os.ModeType == 0 -} diff --git a/internal/fs/interface.go b/internal/fs/interface.go index b26c56944..e1f4ef2d9 100644 --- a/internal/fs/interface.go +++ b/internal/fs/interface.go @@ -7,7 +7,6 @@ import ( // FS bundles all methods needed for a file system. type FS interface { - Open(name string) (File, error) OpenFile(name string, flag int, perm os.FileMode) (File, error) Stat(name string) (os.FileInfo, error) Lstat(name string) (os.FileInfo, error) diff --git a/internal/restic/mknod_unix.go b/internal/fs/mknod_unix.go similarity index 93% rename from internal/restic/mknod_unix.go rename to internal/fs/mknod_unix.go index 7dd6c60d0..6127599f7 100644 --- a/internal/restic/mknod_unix.go +++ b/internal/fs/mknod_unix.go @@ -1,7 +1,7 @@ //go:build !freebsd && !windows // +build !freebsd,!windows -package restic +package fs import "golang.org/x/sys/unix" diff --git a/internal/fs/node.go b/internal/fs/node.go new file mode 100644 index 000000000..9bd507ba5 --- /dev/null +++ b/internal/fs/node.go @@ -0,0 +1,334 @@ +package fs + +import ( + "os" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// NodeFromFileInfo returns a new node from the given path and FileInfo. It +// returns the first error that is encountered, together with a node. +func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { + mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky + node := &restic.Node{ + Path: path, + Name: fi.Name(), + Mode: fi.Mode() & mask, + ModTime: fi.ModTime(), + } + + node.Type = nodeTypeFromFileInfo(fi) + if node.Type == "file" { + node.Size = uint64(fi.Size()) + } + + err := nodeFillExtra(node, path, fi, ignoreXattrListError) + return node, err +} + +func nodeTypeFromFileInfo(fi os.FileInfo) string { + switch fi.Mode() & os.ModeType { + case 0: + return "file" + case os.ModeDir: + return "dir" + case os.ModeSymlink: + return "symlink" + case os.ModeDevice | os.ModeCharDevice: + return "chardev" + case os.ModeDevice: + return "dev" + case os.ModeNamedPipe: + return "fifo" + case os.ModeSocket: + return "socket" + case os.ModeIrregular: + return "irregular" + } + + return "" +} + +func nodeFillExtra(node *restic.Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { + stat, ok := toStatT(fi.Sys()) + if !ok { + // fill minimal info with current values for uid, gid + node.UID = uint32(os.Getuid()) + node.GID = uint32(os.Getgid()) + node.ChangeTime = node.ModTime + return nil + } + + node.Inode = uint64(stat.ino()) + node.DeviceID = uint64(stat.dev()) + + nodeFillTimes(node, stat) + + nodeFillUser(node, stat) + + switch node.Type { + case "file": + node.Size = uint64(stat.size()) + node.Links = uint64(stat.nlink()) + case "dir": + case "symlink": + var err error + node.LinkTarget, err = Readlink(path) + node.Links = uint64(stat.nlink()) + if err != nil { + return errors.WithStack(err) + } + case "dev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "chardev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "fifo": + case "socket": + default: + return errors.Errorf("unsupported file type %q", node.Type) + } + + allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) + } + return err +} + +func nodeFillTimes(node *restic.Node, stat *statT) { + ctim := stat.ctim() + atim := stat.atim() + node.ChangeTime = time.Unix(ctim.Unix()) + node.AccessTime = time.Unix(atim.Unix()) +} + +func nodeFillUser(node *restic.Node, stat *statT) { + uid, gid := stat.uid(), stat.gid() + node.UID, node.GID = uid, gid + node.User = lookupUsername(uid) + node.Group = lookupGroup(gid) +} + +var ( + uidLookupCache = make(map[uint32]string) + uidLookupCacheMutex = sync.RWMutex{} +) + +// Cached user name lookup by uid. Returns "" when no name can be found. +func lookupUsername(uid uint32) string { + uidLookupCacheMutex.RLock() + username, ok := uidLookupCache[uid] + uidLookupCacheMutex.RUnlock() + + if ok { + return username + } + + u, err := user.LookupId(strconv.Itoa(int(uid))) + if err == nil { + username = u.Username + } + + uidLookupCacheMutex.Lock() + uidLookupCache[uid] = username + uidLookupCacheMutex.Unlock() + + return username +} + +var ( + gidLookupCache = make(map[uint32]string) + gidLookupCacheMutex = sync.RWMutex{} +) + +// Cached group name lookup by gid. Returns "" when no name can be found. +func lookupGroup(gid uint32) string { + gidLookupCacheMutex.RLock() + group, ok := gidLookupCache[gid] + gidLookupCacheMutex.RUnlock() + + if ok { + return group + } + + g, err := user.LookupGroupId(strconv.Itoa(int(gid))) + if err == nil { + group = g.Name + } + + gidLookupCacheMutex.Lock() + gidLookupCache[gid] = group + gidLookupCacheMutex.Unlock() + + return group +} + +// NodeCreateAt creates the node at the given path but does NOT restore node meta data. +func NodeCreateAt(node *restic.Node, path string) error { + debug.Log("create node %v at %v", node.Name, path) + + switch node.Type { + case "dir": + if err := nodeCreateDirAt(node, path); err != nil { + return err + } + case "file": + if err := nodeCreateFileAt(path); err != nil { + return err + } + case "symlink": + if err := nodeCreateSymlinkAt(node, path); err != nil { + return err + } + case "dev": + if err := nodeCreateDevAt(node, path); err != nil { + return err + } + case "chardev": + if err := nodeCreateCharDevAt(node, path); err != nil { + return err + } + case "fifo": + if err := nodeCreateFifoAt(path); err != nil { + return err + } + case "socket": + return nil + default: + return errors.Errorf("filetype %q not implemented", node.Type) + } + + return nil +} + +func nodeCreateDirAt(node *restic.Node, path string) error { + err := Mkdir(path, node.Mode) + if err != nil && !os.IsExist(err) { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateFileAt(path string) error { + f, err := OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return errors.WithStack(err) + } + + if err := f.Close(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateSymlinkAt(node *restic.Node, path string) error { + if err := Symlink(node.LinkTarget, path); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFBLK|0600, node.Device) +} + +func nodeCreateCharDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFCHR|0600, node.Device) +} + +func nodeCreateFifoAt(path string) error { + return mkfifo(path, 0600) +} + +func mkfifo(path string, mode uint32) (err error) { + return mknod(path, mode|syscall.S_IFIFO, 0) +} + +// NodeRestoreMetadata restores node metadata +func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + err := nodeRestoreMetadata(node, path, warn) + if err != nil { + // It is common to have permission errors for folders like /home + // unless you're running as root, so ignore those. + if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { + debug.Log("not running as root, ignoring permission error for %v: %v", + path, err) + return nil + } + debug.Log("restoreMetadata(%s) error %v", path, err) + } + + return err +} + +func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + var firsterr error + + if err := lchown(path, int(node.UID), int(node.GID)); err != nil { + firsterr = errors.WithStack(err) + } + + if err := nodeRestoreExtendedAttributes(node, path); err != nil { + debug.Log("error restoring extended attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := NodeRestoreTimestamps(node, path); err != nil { + debug.Log("error restoring timestamps for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := Chmod(path, node.Mode); err != nil { + if firsterr == nil { + firsterr = errors.WithStack(err) + } + } + } + + return firsterr +} + +func NodeRestoreTimestamps(node *restic.Node, path string) error { + var utimes = [...]syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + + if node.Type == "symlink" { + return nodeRestoreSymlinkTimestamps(path, utimes) + } + + if err := syscall.UtimesNano(path, utimes[:]); err != nil { + return errors.Wrap(err, "UtimesNano") + } + + return nil +} diff --git a/internal/fs/node_aix.go b/internal/fs/node_aix.go new file mode 100644 index 000000000..123985c2d --- /dev/null +++ b/internal/fs/node_aix.go @@ -0,0 +1,51 @@ +//go:build aix +// +build aix + +package fs + +import ( + "os" + "syscall" + + "github.com/restic/restic/internal/restic" +) + +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { + return nil +} + +// AIX has a funny timespec type in syscall, with 32-bit nanoseconds. +// golang.org/x/sys/unix handles this cleanly, but we're stuck with syscall +// because os.Stat returns a syscall type in its os.FileInfo.Sys(). +func toTimespec(t syscall.StTimespec_t) syscall.Timespec { + return syscall.Timespec{Sec: t.Sec, Nsec: int64(t.Nsec)} +} + +func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) } +func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } +func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } + +// nodeRestoreExtendedAttributes is a no-op on AIX. +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { + return nil +} + +// nodeFillExtendedAttributes is a no-op on AIX. +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { + return nil +} + +// isListxattrPermissionError is a no-op on AIX. +func isListxattrPermissionError(_ error) bool { + return false +} + +// nodeRestoreGenericAttributes is no-op on AIX. +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +} + +// nodeFillGenericAttributes is a no-op on AIX. +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_darwin.go b/internal/fs/node_darwin.go similarity index 67% rename from internal/restic/node_darwin.go rename to internal/fs/node_darwin.go index 803aa68e5..1ca7ce480 100644 --- a/internal/restic/node_darwin.go +++ b/internal/fs/node_darwin.go @@ -1,8 +1,8 @@ -package restic +package fs import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/restic/node_freebsd.go b/internal/fs/node_freebsd.go similarity index 77% rename from internal/restic/node_freebsd.go rename to internal/fs/node_freebsd.go index 34d5b272c..8796358b0 100644 --- a/internal/restic/node_freebsd.go +++ b/internal/fs/node_freebsd.go @@ -1,11 +1,11 @@ //go:build freebsd // +build freebsd -package restic +package fs import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/restic/node_linux.go b/internal/fs/node_linux.go similarity index 78% rename from internal/restic/node_linux.go rename to internal/fs/node_linux.go index 85a363830..1cb4ee1ae 100644 --- a/internal/restic/node_linux.go +++ b/internal/fs/node_linux.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "path/filepath" @@ -7,11 +7,10 @@ import ( "golang.org/x/sys/unix" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" ) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { - dir, err := fs.Open(filepath.Dir(path)) +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { + dir, err := Open(filepath.Dir(path)) if err != nil { return errors.WithStack(err) } diff --git a/internal/fs/node_netbsd.go b/internal/fs/node_netbsd.go new file mode 100644 index 000000000..996125851 --- /dev/null +++ b/internal/fs/node_netbsd.go @@ -0,0 +1,41 @@ +package fs + +import ( + "os" + "syscall" + + "github.com/restic/restic/internal/restic" +) + +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { + return nil +} + +func (s statT) atim() syscall.Timespec { return s.Atimespec } +func (s statT) mtim() syscall.Timespec { return s.Mtimespec } +func (s statT) ctim() syscall.Timespec { return s.Ctimespec } + +// nodeRestoreExtendedAttributes is a no-op on netbsd. +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { + return nil +} + +// nodeFillExtendedAttributes is a no-op on netbsd. +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { + return nil +} + +// isListxattrPermissionError is a no-op on netbsd. +func isListxattrPermissionError(_ error) bool { + return false +} + +// nodeRestoreGenericAttributes is no-op on netbsd. +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +} + +// nodeFillGenericAttributes is a no-op on netbsd. +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/fs/node_openbsd.go b/internal/fs/node_openbsd.go new file mode 100644 index 000000000..62eb78618 --- /dev/null +++ b/internal/fs/node_openbsd.go @@ -0,0 +1,41 @@ +package fs + +import ( + "os" + "syscall" + + "github.com/restic/restic/internal/restic" +) + +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { + return nil +} + +func (s statT) atim() syscall.Timespec { return s.Atim } +func (s statT) mtim() syscall.Timespec { return s.Mtim } +func (s statT) ctim() syscall.Timespec { return s.Ctim } + +// nodeRestoreExtendedAttributes is a no-op on openbsd. +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { + return nil +} + +// nodeFillExtendedAttributes is a no-op on openbsd. +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { + return nil +} + +// isListxattrPermissionError is a no-op on openbsd. +func isListxattrPermissionError(_ error) bool { + return false +} + +// nodeRestoreGenericAttributes is no-op on openbsd. +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +} + +// fillGenericAttributes is a no-op on openbsd. +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_solaris.go b/internal/fs/node_solaris.go similarity index 65% rename from internal/restic/node_solaris.go rename to internal/fs/node_solaris.go index c9d03f9c2..3f025b334 100644 --- a/internal/restic/node_solaris.go +++ b/internal/fs/node_solaris.go @@ -1,8 +1,8 @@ -package restic +package fs import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go new file mode 100644 index 000000000..e7f608352 --- /dev/null +++ b/internal/fs/node_test.go @@ -0,0 +1,324 @@ +package fs + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" +) + +func BenchmarkNodeFillUser(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + rtest.OK(t, err) + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func BenchmarkNodeFromFileInfo(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + if err != nil { + t.Fatal(err) + } + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func parseTime(s string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05.999", s) + if err != nil { + panic(err) + } + + return t.Local() +} + +var nodeTests = []restic.Node{ + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetuid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile2", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetgid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSticky", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSticky, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSymlink", + Type: "symlink", + LinkTarget: "invalid", + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0777 | os.ModeSymlink, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + + // include "testFile" and "testDir" again with slightly different + // metadata, so we can test if CreateAt works with pre-existing files. + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testXattrFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrFileMacOSResourceFork", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "com.apple.ResourceFork", Value: []byte("bar")}, + }, + }, +} + +func TestNodeRestoreAt(t *testing.T) { + tempdir := t.TempDir() + + for _, test := range nodeTests { + t.Run("", func(t *testing.T) { + var nodePath string + if test.ExtendedAttributes != nil { + if runtime.GOOS == "windows" { + // In windows extended attributes are case insensitive and windows returns + // the extended attributes in UPPER case. + // Update the tests to use UPPER case xattr names for windows. + extAttrArr := test.ExtendedAttributes + // Iterate through the array using pointers + for i := 0; i < len(extAttrArr); i++ { + extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) + } + } + for _, attr := range test.ExtendedAttributes { + if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { + t.Skipf("attr %v only relevant on macOS", attr.Name) + } + } + + // tempdir might be backed by a filesystem that does not support + // extended attributes + nodePath = test.Name + defer func() { + _ = os.Remove(nodePath) + }() + } else { + nodePath = filepath.Join(tempdir, test.Name) + } + rtest.OK(t, NodeCreateAt(&test, nodePath)) + rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) + + fi, err := os.Lstat(nodePath) + rtest.OK(t, err) + + n2, err := NodeFromFileInfo(nodePath, fi, false) + rtest.OK(t, err) + n3, err := NodeFromFileInfo(nodePath, fi, true) + rtest.OK(t, err) + rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) + + rtest.Assert(t, test.Name == n2.Name, + "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) + rtest.Assert(t, test.Type == n2.Type, + "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) + rtest.Assert(t, test.Size == n2.Size, + "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) + + if runtime.GOOS != "windows" { + rtest.Assert(t, test.UID == n2.UID, + "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) + rtest.Assert(t, test.GID == n2.GID, + "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) + if test.Type != "symlink" { + // On OpenBSD only root can set sticky bit (see sticky(8)). + if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { + rtest.Assert(t, test.Mode == n2.Mode, + "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) + } + } + } + + AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) + AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) + if len(n2.ExtendedAttributes) == 0 { + n2.ExtendedAttributes = nil + } + rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), + "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) + }) + } +} + +func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { + var equal bool + + // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd + if nodeType == "symlink" { + switch runtime.GOOS { + case "darwin", "freebsd", "openbsd", "netbsd", "solaris": + return + } + } + + switch runtime.GOOS { + case "darwin": + // HFS+ timestamps don't support sub-second precision, + // see https://en.wikipedia.org/wiki/Comparison_of_file_systems + diff := int(t1.Sub(t2).Seconds()) + equal = diff == 0 + default: + equal = t1.Equal(t2) + } + + rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) +} + +func TestNodeRestoreMetadataError(t *testing.T) { + tempdir := t.TempDir() + + node := &nodeTests[0] + nodePath := filepath.Join(tempdir, node.Name) + + // This will fail because the target file does not exist + err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) + test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") +} diff --git a/internal/restic/node_unix.go b/internal/fs/node_unix.go similarity index 97% rename from internal/restic/node_unix.go rename to internal/fs/node_unix.go index 976cd7b03..fb247ac99 100644 --- a/internal/restic/node_unix.go +++ b/internal/fs/node_unix.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" diff --git a/internal/restic/node_unix_test.go b/internal/fs/node_unix_test.go similarity index 94% rename from internal/restic/node_unix_test.go rename to internal/fs/node_unix_test.go index 9ea7b1725..b505357f2 100644 --- a/internal/restic/node_unix_test.go +++ b/internal/fs/node_unix_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { return fi, true } -func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { t.Helper() if uint32(node.Mode.Perm()) != uint32(stat.Mode&0777) { t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) @@ -80,7 +81,7 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { } -func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { if node.Device != uint64(stat.Rdev) { t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) } diff --git a/internal/restic/node_windows.go b/internal/fs/node_windows.go similarity index 87% rename from internal/restic/node_windows.go rename to internal/fs/node_windows.go index bce01ccad..90fa3462c 100644 --- a/internal/restic/node_windows.go +++ b/internal/fs/node_windows.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "encoding/json" @@ -14,7 +14,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "golang.org/x/sys/windows" ) @@ -56,7 +56,7 @@ func lchown(_ string, _ int, _ int) (err error) { } // restoreSymlinkTimestamps restores timestamps for symlinks -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) if e != nil { @@ -82,12 +82,12 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe } // restore extended attributes for windows -func (node Node) restoreExtendedAttributes(path string) (err error) { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) { count := len(node.ExtendedAttributes) if count > 0 { - eas := make([]fs.ExtendedAttribute, count) + eas := make([]ExtendedAttribute, count) for i, attr := range node.ExtendedAttributes { - eas[i] = fs.ExtendedAttribute{Name: attr.Name, Value: attr.Value} + eas[i] = ExtendedAttribute{Name: attr.Name, Value: attr.Value} } if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { return errExt @@ -97,9 +97,9 @@ func (node Node) restoreExtendedAttributes(path string) (err error) { } // fill extended attributes in the node. This also includes the Generic attributes for windows. -func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { +func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(node.Type, path, false); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(node.Type, path, false); fileHandle == 0 { return nil } if err != nil { @@ -107,8 +107,8 @@ func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { } defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call //Get the windows Extended Attributes using the file handle - var extAtts []fs.ExtendedAttribute - extAtts, err = fs.GetFileEA(fileHandle) + var extAtts []ExtendedAttribute + extAtts, err = GetFileEA(fileHandle) debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) if err != nil { return errors.Errorf("get EA failed for path %v, with: %v", path, err) @@ -119,7 +119,7 @@ func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA for _, attr := range extAtts { - extendedAttr := ExtendedAttribute{ + extendedAttr := restic.ExtendedAttribute{ Name: attr.Name, Value: attr.Value, } @@ -139,9 +139,9 @@ func closeFileHandle(fileHandle windows.Handle, path string) { // restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. // The Windows API requires setting of all the Extended Attributes in one call. -func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { +func restoreExtendedAttributes(nodeType, path string, eas []ExtendedAttribute) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(nodeType, path, true); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(nodeType, path, true); fileHandle == 0 { return nil } if err != nil { @@ -150,7 +150,7 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call // clear old unexpected xattrs by setting them to an empty value - oldEAs, err := fs.GetFileEA(fileHandle) + oldEAs, err := GetFileEA(fileHandle) if err != nil { return err } @@ -165,11 +165,11 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute } if !found { - eas = append(eas, fs.ExtendedAttribute{Name: oldEA.Name, Value: nil}) + eas = append(eas, ExtendedAttribute{Name: oldEA.Name, Value: nil}) } } - if err = fs.SetFileEA(fileHandle, eas); err != nil { + if err = SetFileEA(fileHandle, eas); err != nil { return errors.Errorf("set EA failed for path %v, with: %v", path, err) } return nil @@ -210,7 +210,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { +func nodeRestoreGenericAttributes(node *restic.Node, path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -230,19 +230,19 @@ func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (e } } if windowsAttributes.SecurityDescriptor != nil { - if err := fs.SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { + if err := SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { errs = append(errs, fmt.Errorf("error restoring security descriptor for: %s : %v", path, err)) } } - HandleUnknownGenericAttributesFound(unknownAttribs, warn) + restic.HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.CombineErrors(errs...) } // genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. -func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { +func genericAttributesToWindowsAttrs(attrs map[restic.GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []restic.GenericAttributeType, err error) { waValue := reflect.ValueOf(&windowsAttributes).Elem() - unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + unknownAttribs, err = restic.GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") return windowsAttributes, unknownAttribs, err } @@ -289,14 +289,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should be encrypted. err = encryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, encrypt file call fails. // The readonly and system flags will be set again at the end of this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) } @@ -317,14 +317,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should not be encrypted, but its already encrypted. Decrypt it. err = decryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, decrypt file call fails. // The readonly and system flags will be set again after this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) } @@ -361,11 +361,11 @@ func decryptFile(pathPointer *uint16) error { return nil } -// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// nodeFillGenericAttributes fills in the generic attributes for windows like File Attributes, // Created time and Security Descriptors. // It also checks if the volume supports extended attributes and stores the result in a map // so that it does not have to be checked again for subsequent calls for paths in the same volume. -func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(node *restic.Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows // Also do not allow processing of extended attributes for ADS. @@ -392,7 +392,7 @@ func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT if err != nil { return false, err } - if sd, err = fs.GetSecurityDescriptor(path); err != nil { + if sd, err = GetSecurityDescriptor(path); err != nil { return allowExtended, err } } @@ -422,7 +422,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If not found, check if EA is supported with manually prepared volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeName + `\`) // If the prepared volume name is not valid, we will fetch the actual volume name next. if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { debug.Log("Error checking if extended attributes are supported for prepared volume name %s: %v", volumeName, err) @@ -432,7 +432,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { } } // If an entry is not found, get the actual volume name using the GetVolumePathName function - volumeNameActual, err := fs.GetVolumePathName(path) + volumeNameActual, err := GetVolumePathName(path) if err != nil { debug.Log("Error getting actual volume name %s for path %s: %v", volumeName, path, err) // There can be multiple errors like path does not exist, bad network path, etc. @@ -447,7 +447,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeNameActual + `\`) // Debug log for cases where the prepared volume name is not valid if err != nil { debug.Log("Error checking if extended attributes are supported for actual volume name %s: %v", volumeNameActual, err) @@ -496,10 +496,10 @@ func prepareVolumeName(path string) (volumeName string, err error) { } // windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection -func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[restic.GenericAttributeType]json.RawMessage, err error) { // Get the value of the WindowsAttributes windowsAttributesValue := reflect.ValueOf(windowsAttributes) - return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) + return restic.OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) } // getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. diff --git a/internal/restic/node_windows_test.go b/internal/fs/node_windows_test.go similarity index 88% rename from internal/restic/node_windows_test.go rename to internal/fs/node_windows_test.go index 6ba25559b..046c1984c 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -1,7 +1,7 @@ //go:build windows // +build windows -package restic +package fs import ( "encoding/base64" @@ -15,7 +15,7 @@ import ( "time" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -23,10 +23,10 @@ import ( func TestRestoreSecurityDescriptors(t *testing.T) { t.Parallel() tempDir := t.TempDir() - for i, sd := range fs.TestFileSDs { + for i, sd := range TestFileSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "file", fmt.Sprintf("testfile%d", i)) } - for i, sd := range fs.TestDirSDs { + for i, sd := range TestDirSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "dir", fmt.Sprintf("testdir%d", i)) } } @@ -42,22 +42,22 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir, fileType, f expectedNode := getNode(fileName, fileType, genericAttributes) // Restore the file/dir and restore the meta data including the security descriptors. - testPath, node := restoreAndGetNode(t, tempDir, expectedNode, false) + testPath, node := restoreAndGetNode(t, tempDir, &expectedNode, false) // Get the security descriptor from the node constructed from the file info of the restored path. sdByteFromRestoredNode := getWindowsAttr(t, testPath, node).SecurityDescriptor // Get the security descriptor for the test path after the restore. - sdBytesFromRestoredPath, err := fs.GetSecurityDescriptor(testPath) + sdBytesFromRestoredPath, err := GetSecurityDescriptor(testPath) test.OK(t, errors.Wrapf(err, "Error while getting the security descriptor for: %s", testPath)) // Compare the input SD and the SD got from the restored file. - fs.CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) // Compare the SD got from node constructed from the restored file info and the SD got directly from the restored file. - fs.CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) } -func getNode(name string, fileType string, genericAttributes map[GenericAttributeType]json.RawMessage) Node { - return Node{ +func getNode(name string, fileType string, genericAttributes map[restic.GenericAttributeType]json.RawMessage) restic.Node { + return restic.Node{ Name: name, Type: fileType, Mode: 0644, @@ -68,7 +68,7 @@ func getNode(name string, fileType string, genericAttributes map[GenericAttribut } } -func getWindowsAttr(t *testing.T, testPath string, node *Node) WindowsAttributes { +func getWindowsAttr(t *testing.T, testPath string, node *restic.Node) WindowsAttributes { windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, errors.Wrapf(err, "Error getting windows attr from generic attr: %s", testPath)) test.Assert(t, len(unknownAttribs) == 0, "Unknown attribs found: %s for: %s", unknownAttribs, testPath) @@ -83,12 +83,12 @@ func TestRestoreCreationTime(t *testing.T) { creationTimeAttribute := getCreationTime(fi, path) test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) //Using the temp dir creation time as the test creation time for the test file and folder - runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) + runGenericAttributesTest(t, path, restic.TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) } func TestRestoreFileAttributes(t *testing.T) { t.Parallel() - genericAttributeName := TypeFileAttributes + genericAttributeName := restic.TypeFileAttributes tempDir := t.TempDir() normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) @@ -110,7 +110,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, fileAttr := range fileAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testfile%d", i), Type: "file", @@ -143,7 +143,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, folderAttr := range folderAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testdirectory%d", i), Type: "dir", @@ -158,10 +158,10 @@ func TestRestoreFileAttributes(t *testing.T) { } } -func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -183,10 +183,10 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName } runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) } -func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []restic.Node, tempDir string, genericAttr restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, warningExpected) rawMessage := node.GenericAttributes[genericAttr] genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) @@ -195,7 +195,7 @@ func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDi } } -func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { +func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warningExpected bool) (string, *restic.Node) { testPath := filepath.Join(tempDir, "001", testNode.Name) err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) @@ -211,7 +211,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpec test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) } - err = testNode.RestoreMetadata(testPath, func(msg string) { + err = NodeRestoreMetadata(testNode, testPath, func(msg string) { if warningExpected { test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) } else { @@ -230,16 +230,16 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpec return testPath, nodeFromFileInfo } -const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" +const TypeSomeNewAttribute restic.GenericAttributeType = "MockAttributes.SomeNewAttribute" func TestNewGenericAttributeType(t *testing.T) { t.Parallel() - newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute := map[restic.GenericAttributeType]json.RawMessage{} newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -260,7 +260,7 @@ func TestNewGenericAttributeType(t *testing.T) { }, } for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, true) _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, err) // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. @@ -271,7 +271,7 @@ func TestNewGenericAttributeType(t *testing.T) { func TestRestoreExtendedAttributes(t *testing.T) { t.Parallel() tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -279,7 +279,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -290,13 +290,13 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, } for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, false) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, false) var handle windows.Handle var err error @@ -312,12 +312,12 @@ func TestRestoreExtendedAttributes(t *testing.T) { test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) }() - extAttr, err := fs.GetFileEA(handle) + extAttr, err := GetFileEA(handle) test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) for _, expectedExtAttr := range node.ExtendedAttributes { - var foundExtAttr *fs.ExtendedAttribute + var foundExtAttr *ExtendedAttribute for _, ea := range extAttr { if strings.EqualFold(ea.Name, expectedExtAttr.Name) { foundExtAttr = &ea @@ -491,13 +491,13 @@ func TestPrepareVolumeName(t *testing.T) { test.Equals(t, tc.expectedVolume, volume) if tc.isRealPath { - isEASupportedVolume, err := fs.PathSupportsExtendedAttributes(volume + `\`) + isEASupportedVolume, err := PathSupportsExtendedAttributes(volume + `\`) // If the prepared volume name is not valid, we will next fetch the actual volume name. test.OK(t, err) test.Equals(t, tc.expectedEASupported, isEASupportedVolume) - actualVolume, err := fs.GetVolumePathName(tc.path) + actualVolume, err := GetVolumePathName(tc.path) test.OK(t, err) test.Equals(t, tc.expectedVolume, actualVolume) } diff --git a/internal/restic/node_xattr.go b/internal/fs/node_xattr.go similarity index 74% rename from internal/restic/node_xattr.go rename to internal/fs/node_xattr.go index 5a5a253d9..55376ba58 100644 --- a/internal/restic/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/pkg/xattr" ) @@ -27,7 +28,7 @@ func listxattr(path string) ([]string, error) { return l, handleXattrErr(err) } -func IsListxattrPermissionError(err error) bool { +func isListxattrPermissionError(err error) bool { var xerr *xattr.Error if errors.As(err, &xerr) { return xerr.Op == "xattr.list" && errors.Is(xerr.Err, os.ErrPermission) @@ -64,17 +65,17 @@ func handleXattrErr(err error) error { } } -// restoreGenericAttributes is no-op. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) +// nodeRestoreGenericAttributes is no-op. +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } -// fillGenericAttributes is a no-op. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +// nodeFillGenericAttributes is a no-op. +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } -func (node Node) restoreExtendedAttributes(path string) error { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { err := setxattr(path, attr.Name, attr.Value) @@ -101,24 +102,24 @@ func (node Node) restoreExtendedAttributes(path string) error { return nil } -func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error { +func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError bool) error { xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { - if ignoreListError && IsListxattrPermissionError(err) { + if ignoreListError && isListxattrPermissionError(err) { return nil } return err } - node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) + node.ExtendedAttributes = make([]restic.ExtendedAttribute, 0, len(xattrs)) for _, attr := range xattrs { attrVal, err := getxattr(path, attr) if err != nil { fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) continue } - attr := ExtendedAttribute{ + attr := restic.ExtendedAttribute{ Name: attr, Value: attrVal, } diff --git a/internal/restic/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go similarity index 57% rename from internal/restic/node_xattr_all_test.go rename to internal/fs/node_xattr_all_test.go index 56ce5e286..39670d6e1 100644 --- a/internal/restic/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris || windows // +build darwin freebsd linux solaris windows -package restic +package fs import ( "os" @@ -10,10 +10,11 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { +func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribute) { if runtime.GOOS == "windows" { // windows seems to convert the xattr name to upper case for i := range attrs { @@ -21,18 +22,18 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { } } - node := Node{ + node := &restic.Node{ Type: "file", ExtendedAttributes: attrs, } - rtest.OK(t, node.restoreExtendedAttributes(file)) + rtest.OK(t, nodeRestoreExtendedAttributes(node, file)) - nodeActual := Node{ + nodeActual := &restic.Node{ Type: "file", } - rtest.OK(t, nodeActual.fillExtendedAttributes(file, false)) + rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false)) - rtest.Assert(t, nodeActual.sameExtendedAttributes(node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) + rtest.Assert(t, nodeActual.Equals(*node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) } func TestOverwriteXattr(t *testing.T) { @@ -40,14 +41,14 @@ func TestOverwriteXattr(t *testing.T) { file := filepath.Join(dir, "file") rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.foo", Value: []byte("bar"), }, }) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.other", Value: []byte("some"), diff --git a/internal/restic/node_xattr_test.go b/internal/fs/node_xattr_test.go similarity index 81% rename from internal/restic/node_xattr_test.go rename to internal/fs/node_xattr_test.go index 5ce77bd28..3784dba45 100644 --- a/internal/restic/node_xattr_test.go +++ b/internal/fs/node_xattr_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "os" @@ -19,10 +19,10 @@ func TestIsListxattrPermissionError(t *testing.T) { } err := handleXattrErr(xerr) rtest.Assert(t, err != nil, "missing error") - rtest.Assert(t, IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err) + rtest.Assert(t, isListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err) xerr.Err = os.ErrNotExist err = handleXattrErr(xerr) rtest.Assert(t, err != nil, "missing error") - rtest.Assert(t, !IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err) + rtest.Assert(t, !isListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err) } diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 8dd8f71c1..731ad9a6a 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "io" "os" - "runtime" "sync" "github.com/restic/restic/internal/backend" @@ -186,14 +185,6 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *packe return errors.Wrap(err, "close tempfile") } - // on windows the tempfile is automatically deleted on close - if runtime.GOOS != "windows" { - err = fs.RemoveIfExists(p.tmpfile.Name()) - if err != nil { - return errors.WithStack(err) - } - } - // update blobs in the index debug.Log(" updating blobs %v to pack %v", p.Packer.Blobs(), id) r.idx.StorePack(id, p.Packer.Blobs()) diff --git a/internal/restic/node.go b/internal/restic/node.go index 6afdff64a..8bf97e59c 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -1,16 +1,13 @@ package restic import ( - "context" "encoding/json" "fmt" "os" - "os/user" "reflect" "strconv" "strings" "sync" - "syscall" "time" "unicode/utf8" @@ -19,7 +16,6 @@ import ( "bytes" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/fs" ) // ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. @@ -134,49 +130,6 @@ func (node Node) String() string { mode|node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name) } -// NodeFromFileInfo returns a new node from the given path and FileInfo. It -// returns the first error that is encountered, together with a node. -func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*Node, error) { - mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky - node := &Node{ - Path: path, - Name: fi.Name(), - Mode: fi.Mode() & mask, - ModTime: fi.ModTime(), - } - - node.Type = nodeTypeFromFileInfo(fi) - if node.Type == "file" { - node.Size = uint64(fi.Size()) - } - - err := node.fillExtra(path, fi, ignoreXattrListError) - return node, err -} - -func nodeTypeFromFileInfo(fi os.FileInfo) string { - switch fi.Mode() & os.ModeType { - case 0: - return "file" - case os.ModeDir: - return "dir" - case os.ModeSymlink: - return "symlink" - case os.ModeDevice | os.ModeCharDevice: - return "chardev" - case os.ModeDevice: - return "dev" - case os.ModeNamedPipe: - return "fifo" - case os.ModeSocket: - return "socket" - case os.ModeIrregular: - return "irregular" - } - - return "" -} - // GetExtendedAttribute gets the extended attribute. func (node Node) GetExtendedAttribute(a string) []byte { for _, attr := range node.ExtendedAttributes { @@ -187,186 +140,6 @@ func (node Node) GetExtendedAttribute(a string) []byte { return nil } -// CreateAt creates the node at the given path but does NOT restore node meta data. -func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error { - debug.Log("create node %v at %v", node.Name, path) - - switch node.Type { - case "dir": - if err := node.createDirAt(path); err != nil { - return err - } - case "file": - if err := node.createFileAt(ctx, path, repo); err != nil { - return err - } - case "symlink": - if err := node.createSymlinkAt(path); err != nil { - return err - } - case "dev": - if err := node.createDevAt(path); err != nil { - return err - } - case "chardev": - if err := node.createCharDevAt(path); err != nil { - return err - } - case "fifo": - if err := node.createFifoAt(path); err != nil { - return err - } - case "socket": - return nil - default: - return errors.Errorf("filetype %q not implemented", node.Type) - } - - return nil -} - -// RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string, warn func(msg string)) error { - err := node.restoreMetadata(path, warn) - if err != nil { - // It is common to have permission errors for folders like /home - // unless you're running as root, so ignore those. - if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { - debug.Log("not running as root, ignoring permission error for %v: %v", - path, err) - return nil - } - debug.Log("restoreMetadata(%s) error %v", path, err) - } - - return err -} - -func (node Node) restoreMetadata(path string, warn func(msg string)) error { - var firsterr error - - if err := lchown(path, int(node.UID), int(node.GID)); err != nil { - firsterr = errors.WithStack(err) - } - - if err := node.restoreExtendedAttributes(path); err != nil { - debug.Log("error restoring extended attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := node.restoreGenericAttributes(path, warn); err != nil { - debug.Log("error restoring generic attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := node.RestoreTimestamps(path); err != nil { - debug.Log("error restoring timestamps for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows - // calling Chmod below will no longer allow any modifications to be made on the file and the - // calls above would fail. - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr == nil { - firsterr = errors.WithStack(err) - } - } - } - - return firsterr -} - -func (node Node) RestoreTimestamps(path string) error { - var utimes = [...]syscall.Timespec{ - syscall.NsecToTimespec(node.AccessTime.UnixNano()), - syscall.NsecToTimespec(node.ModTime.UnixNano()), - } - - if node.Type == "symlink" { - return node.restoreSymlinkTimestamps(path, utimes) - } - - if err := syscall.UtimesNano(path, utimes[:]); err != nil { - return errors.Wrap(err, "UtimesNano") - } - - return nil -} - -func (node Node) createDirAt(path string) error { - err := fs.Mkdir(path, node.Mode) - if err != nil && !os.IsExist(err) { - return errors.WithStack(err) - } - - return nil -} - -func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error { - f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return errors.WithStack(err) - } - - err = node.writeNodeContent(ctx, repo, f) - closeErr := f.Close() - - if err != nil { - return err - } - - if closeErr != nil { - return errors.WithStack(closeErr) - } - - return nil -} - -func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error { - var buf []byte - for _, id := range node.Content { - buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) - if err != nil { - return err - } - - _, err = f.Write(buf) - if err != nil { - return errors.WithStack(err) - } - } - - return nil -} - -func (node Node) createSymlinkAt(path string) error { - if err := fs.Symlink(node.LinkTarget, path); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func (node *Node) createDevAt(path string) error { - return mknod(path, syscall.S_IFBLK|0600, node.Device) -} - -func (node *Node) createCharDevAt(path string) error { - return mknod(path, syscall.S_IFCHR|0600, node.Device) -} - -func (node *Node) createFifoAt(path string) error { - return mkfifo(path, 0600) -} - // FixTime returns a time.Time which can safely be used to marshal as JSON. If // the timestamp is earlier than year zero, the year is set to zero. In the same // way, if the year is larger than 9999, the year is set to 9999. Other than @@ -601,127 +374,6 @@ func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { return true } -func (node *Node) fillUser(stat *statT) { - uid, gid := stat.uid(), stat.gid() - node.UID, node.GID = uid, gid - node.User = lookupUsername(uid) - node.Group = lookupGroup(gid) -} - -var ( - uidLookupCache = make(map[uint32]string) - uidLookupCacheMutex = sync.RWMutex{} -) - -// Cached user name lookup by uid. Returns "" when no name can be found. -func lookupUsername(uid uint32) string { - uidLookupCacheMutex.RLock() - username, ok := uidLookupCache[uid] - uidLookupCacheMutex.RUnlock() - - if ok { - return username - } - - u, err := user.LookupId(strconv.Itoa(int(uid))) - if err == nil { - username = u.Username - } - - uidLookupCacheMutex.Lock() - uidLookupCache[uid] = username - uidLookupCacheMutex.Unlock() - - return username -} - -var ( - gidLookupCache = make(map[uint32]string) - gidLookupCacheMutex = sync.RWMutex{} -) - -// Cached group name lookup by gid. Returns "" when no name can be found. -func lookupGroup(gid uint32) string { - gidLookupCacheMutex.RLock() - group, ok := gidLookupCache[gid] - gidLookupCacheMutex.RUnlock() - - if ok { - return group - } - - g, err := user.LookupGroupId(strconv.Itoa(int(gid))) - if err == nil { - group = g.Name - } - - gidLookupCacheMutex.Lock() - gidLookupCache[gid] = group - gidLookupCacheMutex.Unlock() - - return group -} - -func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bool) error { - stat, ok := toStatT(fi.Sys()) - if !ok { - // fill minimal info with current values for uid, gid - node.UID = uint32(os.Getuid()) - node.GID = uint32(os.Getgid()) - node.ChangeTime = node.ModTime - return nil - } - - node.Inode = uint64(stat.ino()) - node.DeviceID = uint64(stat.dev()) - - node.fillTimes(stat) - - node.fillUser(stat) - - switch node.Type { - case "file": - node.Size = uint64(stat.size()) - node.Links = uint64(stat.nlink()) - case "dir": - case "symlink": - var err error - node.LinkTarget, err = fs.Readlink(path) - node.Links = uint64(stat.nlink()) - if err != nil { - return errors.WithStack(err) - } - case "dev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "chardev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "fifo": - case "socket": - default: - return errors.Errorf("unsupported file type %q", node.Type) - } - - allowExtended, err := node.fillGenericAttributes(path, fi, stat) - if allowExtended { - // Skip processing ExtendedAttributes if allowExtended is false. - err = errors.CombineErrors(err, node.fillExtendedAttributes(path, ignoreXattrListError)) - } - return err -} - -func mkfifo(path string, mode uint32) (err error) { - return mknod(path, mode|syscall.S_IFIFO, 0) -} - -func (node *Node) fillTimes(stat *statT) { - ctim := stat.ctim() - atim := stat.atim() - node.ChangeTime = time.Unix(ctim.Unix()) - node.AccessTime = time.Unix(atim.Unix()) -} - // HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { for _, unknownAttrib := range unknownAttribs { @@ -746,11 +398,11 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp } } -// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// HandleAllUnknownGenericAttributesFound performs validations for all generic attributes of a node. // This is not used on windows currently because windows has handling for generic attributes. // nolint:unused -func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { - for name := range node.GenericAttributes { +func HandleAllUnknownGenericAttributesFound(attributes map[GenericAttributeType]json.RawMessage, warn func(msg string)) error { + for name := range attributes { handleUnknownGenericAttributeFound(name, warn) } return nil @@ -770,9 +422,8 @@ func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool // The functions below are common helper functions which can be used for generic attributes support // across different OS. -// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection -// nolint:unused -func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { +// GenericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +func GenericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { attributeValue := *attributeValuePtr for key, rawMsg := range attrs { @@ -796,20 +447,17 @@ func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, } // getFQKey gets the fully qualified key for the field -// nolint:unused func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) } // getFQKeyByIndex gets the fully qualified key for the field index -// nolint:unused func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { return getFQKey(attributeType.Field(index), keyPrefix) } -// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection -// nolint:unused -func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { +// OSAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +func OSAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { attributeValue := *attributeValuePtr attrs = make(map[GenericAttributeType]json.RawMessage) diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go deleted file mode 100644 index 32f63af15..000000000 --- a/internal/restic/node_aix.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build aix -// +build aix - -package restic - -import ( - "os" - "syscall" -) - -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { - return nil -} - -// AIX has a funny timespec type in syscall, with 32-bit nanoseconds. -// golang.org/x/sys/unix handles this cleanly, but we're stuck with syscall -// because os.Stat returns a syscall type in its os.FileInfo.Sys(). -func toTimespec(t syscall.StTimespec_t) syscall.Timespec { - return syscall.Timespec{Sec: t.Sec, Nsec: int64(t.Nsec)} -} - -func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) } -func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } -func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } - -// restoreExtendedAttributes is a no-op on AIX. -func (node Node) restoreExtendedAttributes(_ string) error { - return nil -} - -// fillExtendedAttributes is a no-op on AIX. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { - return nil -} - -// IsListxattrPermissionError is a no-op on AIX. -func IsListxattrPermissionError(_ error) bool { - return false -} - -// restoreGenericAttributes is no-op on AIX. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) -} - -// fillGenericAttributes is a no-op on AIX. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { - return true, nil -} diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go deleted file mode 100644 index 0fe46a3f2..000000000 --- a/internal/restic/node_netbsd.go +++ /dev/null @@ -1,39 +0,0 @@ -package restic - -import ( - "os" - "syscall" -) - -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { - return nil -} - -func (s statT) atim() syscall.Timespec { return s.Atimespec } -func (s statT) mtim() syscall.Timespec { return s.Mtimespec } -func (s statT) ctim() syscall.Timespec { return s.Ctimespec } - -// restoreExtendedAttributes is a no-op on netbsd. -func (node Node) restoreExtendedAttributes(_ string) error { - return nil -} - -// fillExtendedAttributes is a no-op on netbsd. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { - return nil -} - -// IsListxattrPermissionError is a no-op on netbsd. -func IsListxattrPermissionError(_ error) bool { - return false -} - -// restoreGenericAttributes is no-op on netbsd. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) -} - -// fillGenericAttributes is a no-op on netbsd. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { - return true, nil -} diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go deleted file mode 100644 index 71841f59f..000000000 --- a/internal/restic/node_openbsd.go +++ /dev/null @@ -1,39 +0,0 @@ -package restic - -import ( - "os" - "syscall" -) - -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { - return nil -} - -func (s statT) atim() syscall.Timespec { return s.Atim } -func (s statT) mtim() syscall.Timespec { return s.Mtim } -func (s statT) ctim() syscall.Timespec { return s.Ctim } - -// restoreExtendedAttributes is a no-op on openbsd. -func (node Node) restoreExtendedAttributes(_ string) error { - return nil -} - -// fillExtendedAttributes is a no-op on openbsd. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { - return nil -} - -// IsListxattrPermissionError is a no-op on openbsd. -func IsListxattrPermissionError(_ error) bool { - return false -} - -// restoreGenericAttributes is no-op on openbsd. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) -} - -// fillGenericAttributes is a no-op on openbsd. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { - return true, nil -} diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index ab7f66e5b..38a17cb09 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -1,318 +1,14 @@ package restic import ( - "context" "encoding/json" "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" - rtest "github.com/restic/restic/internal/test" ) -func BenchmarkNodeFillUser(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - rtest.OK(t, err) - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func BenchmarkNodeFromFileInfo(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - if err != nil { - t.Fatal(err) - } - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func parseTime(s string) time.Time { - t, err := time.Parse("2006-01-02 15:04:05.999", s) - if err != nil { - panic(err) - } - - return t.Local() -} - -var nodeTests = []Node{ - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetuid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile2", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetgid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSticky", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSticky, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSymlink", - Type: "symlink", - LinkTarget: "invalid", - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0777 | os.ModeSymlink, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - - // include "testFile" and "testDir" again with slightly different - // metadata, so we can test if CreateAt works with pre-existing files. - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testXattrFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrFileMacOSResourceFork", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"com.apple.ResourceFork", []byte("bar")}, - }, - }, -} - -func TestNodeRestoreAt(t *testing.T) { - tempdir := t.TempDir() - - for _, test := range nodeTests { - t.Run("", func(t *testing.T) { - var nodePath string - if test.ExtendedAttributes != nil { - if runtime.GOOS == "windows" { - // In windows extended attributes are case insensitive and windows returns - // the extended attributes in UPPER case. - // Update the tests to use UPPER case xattr names for windows. - extAttrArr := test.ExtendedAttributes - // Iterate through the array using pointers - for i := 0; i < len(extAttrArr); i++ { - extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) - } - } - for _, attr := range test.ExtendedAttributes { - if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { - t.Skipf("attr %v only relevant on macOS", attr.Name) - } - } - - // tempdir might be backed by a filesystem that does not support - // extended attributes - nodePath = test.Name - defer func() { - _ = os.Remove(nodePath) - }() - } else { - nodePath = filepath.Join(tempdir, test.Name) - } - rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) - - fi, err := os.Lstat(nodePath) - rtest.OK(t, err) - - n2, err := NodeFromFileInfo(nodePath, fi, false) - rtest.OK(t, err) - n3, err := NodeFromFileInfo(nodePath, fi, true) - rtest.OK(t, err) - rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) - - rtest.Assert(t, test.Name == n2.Name, - "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) - rtest.Assert(t, test.Type == n2.Type, - "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) - rtest.Assert(t, test.Size == n2.Size, - "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - - if runtime.GOOS != "windows" { - rtest.Assert(t, test.UID == n2.UID, - "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) - rtest.Assert(t, test.GID == n2.GID, - "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != "symlink" { - // On OpenBSD only root can set sticky bit (see sticky(8)). - if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { - rtest.Assert(t, test.Mode == n2.Mode, - "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) - } - } - } - - AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) - AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) - if len(n2.ExtendedAttributes) == 0 { - n2.ExtendedAttributes = nil - } - rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), - "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) - }) - } -} - -func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { - var equal bool - - // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd - if nodeType == "symlink" { - switch runtime.GOOS { - case "darwin", "freebsd", "openbsd", "netbsd", "solaris": - return - } - } - - switch runtime.GOOS { - case "darwin": - // HFS+ timestamps don't support sub-second precision, - // see https://en.wikipedia.org/wiki/Comparison_of_file_systems - diff := int(t1.Sub(t2).Seconds()) - equal = diff == 0 - default: - equal = t1.Equal(t2) - } - - rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) -} - func parseTimeNano(t testing.TB, s string) time.Time { // 2006-01-02T15:04:05.999999999Z07:00 ts, err := time.Parse(time.RFC3339Nano, s) @@ -398,14 +94,3 @@ func TestSymlinkSerializationFormat(t *testing.T) { test.Assert(t, n2.LinkTargetRaw == nil, "quoted link target is just a helper field and must be unset after decoding") } } - -func TestNodeRestoreMetadataError(t *testing.T) { - tempdir := t.TempDir() - - node := nodeTests[0] - nodePath := filepath.Join(tempdir, node.Name) - - // This will fail because the target file does not exist - err := node.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) - test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") -} diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index 8e0b3587a..cdd6b3c18 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -86,7 +87,7 @@ func TestNodeComparison(t *testing.T) { fi, err := os.Lstat("tree_test.go") rtest.OK(t, err) - node, err := restic.NodeFromFileInfo("tree_test.go", fi, false) + node, err := fs.NodeFromFileInfo("tree_test.go", fi, false) rtest.OK(t, err) n2 := *node @@ -127,7 +128,7 @@ func TestTreeEqualSerialization(t *testing.T) { for _, fn := range files[:i] { fi, err := os.Lstat(fn) rtest.OK(t, err) - node, err := restic.NodeFromFileInfo(fn, fi, false) + node, err := fs.NodeFromFileInfo(fn, fi, false) rtest.OK(t, err) rtest.OK(t, tree.Insert(node)) diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 962f66619..d6f78f2d7 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -2,7 +2,6 @@ package restorer import ( "fmt" - stdfs "io/fs" "os" "sync" "syscall" @@ -82,7 +81,7 @@ func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete return nil, err } - var fi stdfs.FileInfo + var fi os.FileInfo if f != nil { // stat to check that we've opened a regular file fi, err = f.Stat() @@ -135,7 +134,7 @@ func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete return ensureSize(f, fi, createSize, sparse) } -func ensureSize(f *os.File, fi stdfs.FileInfo, createSize int64, sparse bool) (*os.File, error) { +func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os.File, error) { if sparse { err := truncateSparse(f, createSize) if err != nil { diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 0e30b82f8..26b6f3474 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -265,14 +265,14 @@ func (res *Restorer) traverseTreeInner(ctx context.Context, target, location str return filenames, hasRestored, nil } -func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string) error { +func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) error { if !res.opts.DryRun { debug.Log("restoreNode %v %v %v", node.Name, target, location) if err := fs.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { return errors.Wrap(err, "RemoveNode") } - err := node.CreateAt(ctx, target, res.repo) + err := fs.NodeCreateAt(node, target) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) return err @@ -288,7 +288,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return nil } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target, res.Warn) + err := fs.NodeRestoreMetadata(node, target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } @@ -435,7 +435,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) debug.Log("second pass, visitNode: restore node %q", location) if node.Type != "file" { _, err := res.withOverwriteCheck(ctx, node, target, location, false, nil, func(_ bool, _ *fileState) error { - return res.restoreNodeTo(ctx, node, target, location) + return res.restoreNodeTo(node, target, location) }) return err } diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 4764bed2d..9fcdfc48d 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -16,6 +16,7 @@ import ( "unsafe" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -263,7 +264,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer { //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY } - attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + attrs, err := fs.WindowsAttrsToGenericAttributes(fs.WindowsAttributes{FileAttributes: &fileattr}) test.OK(t, err) return attrs }