From 7080fed7ae0eb660e0ba9f352ccf3765ea2e9ed8 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Wed, 1 Jul 2020 14:35:49 +0200 Subject: [PATCH 1/3] Set O_NOATIME flag on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Citing Kerrisk, The Linux Programming Interface: The O_NOATIME flag is intended for use by indexing and backup programs. Its use can significantly reduce the amount of disk activity, because repeated disk seeks back and forth across the disk are not required to read the contents of a file and to update the last access time in the file’s i-node[.] restic used to do this, but the functionality was removed along with the fadvise call in #670. --- internal/fs/file.go | 14 ++++++- internal/fs/setflags_linux.go | 21 ++++++++++ internal/fs/setflags_linux_test.go | 67 ++++++++++++++++++++++++++++++ internal/fs/setflags_other.go | 12 ++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 internal/fs/setflags_linux.go create mode 100644 internal/fs/setflags_linux_test.go create mode 100644 internal/fs/setflags_other.go diff --git a/internal/fs/file.go b/internal/fs/file.go index e8e9080d7..a7c2b8886 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -85,7 +85,12 @@ func Create(name string) (*os.File, error) { // Open opens a file for reading. func Open(name string) (File, error) { - return os.Open(fixpath(name)) + f, err := os.Open(fixpath(name)) + if err != nil { + return nil, err + } + setFlags(f) + return f, err } // OpenFile is the generalized open call; most users will use Open @@ -94,7 +99,12 @@ func Open(name string) (File, error) { // methods on the returned File can be used for I/O. // If there is an error, it will be of type *PathError. func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { - return os.OpenFile(fixpath(name), flag, perm) + f, err := os.OpenFile(fixpath(name), flag, perm) + if err != nil { + return nil, err + } + setFlags(f) + return f, err } // Walk walks the file tree rooted at root, calling walkFn for each file or diff --git a/internal/fs/setflags_linux.go b/internal/fs/setflags_linux.go new file mode 100644 index 000000000..32e3d2683 --- /dev/null +++ b/internal/fs/setflags_linux.go @@ -0,0 +1,21 @@ +package fs + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// SetFlags tries to set the O_NOATIME flag on f, which prevents the kernel +// from updating the atime on a read call. +// +// The call fails when we're not the owner of the file or root. The caller +// should ignore the error, which is returned for testing only. +func setFlags(f *os.File) error { + fd := f.Fd() + flags, err := unix.FcntlInt(fd, unix.F_GETFL, 0) + if err == nil { + _, err = unix.FcntlInt(fd, unix.F_SETFL, flags|unix.O_NOATIME) + } + return err +} diff --git a/internal/fs/setflags_linux_test.go b/internal/fs/setflags_linux_test.go new file mode 100644 index 000000000..7818146ac --- /dev/null +++ b/internal/fs/setflags_linux_test.go @@ -0,0 +1,67 @@ +package fs + +import ( + "io" + "io/ioutil" + "os" + "testing" + "time" + + rtest "github.com/restic/restic/internal/test" + + "golang.org/x/sys/unix" +) + +func TestNoatime(t *testing.T) { + f, err := ioutil.TempFile("", "restic-test-noatime") + if err != nil { + t.Fatal(err) + } + + defer func() { + _ = f.Close() + err = Remove(f.Name()) + if err != nil { + t.Fatal(err) + } + }() + + // Only run this test on common filesystems that support O_NOATIME. + // On others, we may not get an error. + if !supportsNoatime(t, f) { + t.Skip("temp directory may not support O_NOATIME, skipping") + } + // From this point on, we own the file, so we should not get EPERM. + + _, err = io.WriteString(f, "Hello!") + rtest.OK(t, err) + _, err = f.Seek(0, io.SeekStart) + rtest.OK(t, err) + + getAtime := func() time.Time { + info, err := f.Stat() + rtest.OK(t, err) + return ExtendedStat(info).AccessTime + } + + atime := getAtime() + + err = setFlags(f) + rtest.OK(t, err) + + _, err = f.Read(make([]byte, 1)) + rtest.OK(t, err) + rtest.Equals(t, atime, getAtime()) +} + +func supportsNoatime(t *testing.T, f *os.File) bool { + var fsinfo unix.Statfs_t + err := unix.Fstatfs(int(f.Fd()), &fsinfo) + rtest.OK(t, err) + + return fsinfo.Type == unix.BTRFS_SUPER_MAGIC || + fsinfo.Type == unix.EXT2_SUPER_MAGIC || + fsinfo.Type == unix.EXT3_SUPER_MAGIC || + fsinfo.Type == unix.EXT4_SUPER_MAGIC || + fsinfo.Type == unix.TMPFS_MAGIC +} diff --git a/internal/fs/setflags_other.go b/internal/fs/setflags_other.go new file mode 100644 index 000000000..6485126e0 --- /dev/null +++ b/internal/fs/setflags_other.go @@ -0,0 +1,12 @@ +//go:build !linux +// +build !linux + +package fs + +import "os" + +// OS-specific replacements of setFlags can set file status flags +// that improve I/O performance. +func setFlags(*os.File) error { + return nil +} From 6b17a7110cfac2c67c6337f008f5adc661fa8fb8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 30 Dec 2021 15:32:42 +0100 Subject: [PATCH 2/3] backup: Set O_NOATIME in the right place The archiver uses FS.OpenFile, where FS is an instance of the FS interface. This is different from fs.OpenFile, which uses the OpenFile method provided by the fs package. --- internal/fs/file.go | 14 ++------------ internal/fs/fs_local.go | 2 ++ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/internal/fs/file.go b/internal/fs/file.go index a7c2b8886..e8e9080d7 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -85,12 +85,7 @@ func Create(name string) (*os.File, error) { // Open opens a file for reading. func Open(name string) (File, error) { - f, err := os.Open(fixpath(name)) - if err != nil { - return nil, err - } - setFlags(f) - return f, err + return os.Open(fixpath(name)) } // OpenFile is the generalized open call; most users will use Open @@ -99,12 +94,7 @@ func Open(name string) (File, error) { // methods on the returned File can be used for I/O. // If there is an error, it will be of type *PathError. func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { - f, err := os.OpenFile(fixpath(name), flag, perm) - if err != nil { - return nil, err - } - setFlags(f) - return f, err + return os.OpenFile(fixpath(name), flag, perm) } // Walk walks the file tree rooted at root, calling walkFn for each file or diff --git a/internal/fs/fs_local.go b/internal/fs/fs_local.go index dd1faafa0..48c40dc90 100644 --- a/internal/fs/fs_local.go +++ b/internal/fs/fs_local.go @@ -24,6 +24,7 @@ func (fs Local) Open(name string) (File, error) { if err != nil { return nil, err } + _ = setFlags(f) return f, nil } @@ -37,6 +38,7 @@ func (fs Local) OpenFile(name string, flag int, perm os.FileMode) (File, error) if err != nil { return nil, err } + _ = setFlags(f) return f, nil } From aaa7f941395de73d2c137aee9cdce36037574434 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 30 Dec 2021 15:41:04 +0100 Subject: [PATCH 3/3] Add changelog for O_NOATIME --- changelog/unreleased/pull-2816 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/pull-2816 diff --git a/changelog/unreleased/pull-2816 b/changelog/unreleased/pull-2816 new file mode 100644 index 000000000..4033a17ab --- /dev/null +++ b/changelog/unreleased/pull-2816 @@ -0,0 +1,10 @@ +Enhancement: backup no longer updates file access times on Linux + +When reading files during backup, restic caused the operating system to update +the file access times. Note that this does not apply to filesystems with +disabled file access times. + +Restic now instructs the operation system not to update the file access time, +if the user running restic is the file owner or has root permissions. + +https://github.com/restic/restic/pull/2816