Use fadvise() to not cache the content of files read

src/restic/archiver.go

@@ -13,6 +13,7 @@ import (
 
 	"restic/backend"
 	"restic/debug"
+	"restic/fs"
 	"restic/pack"
 	"restic/pipe"
 	"restic/repository"
@@ -126,7 +127,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) {
 	return arch.repo.SaveJSON(pack.Tree, item)
 }
 
-func (arch *Archiver) reloadFileIfChanged(node *Node, file *os.File) (*Node, error) {
+func (arch *Archiver) reloadFileIfChanged(node *Node, file fs.File) (*Node, error) {
 	fi, err := file.Stat()
 	if err != nil {
 		return nil, err
@@ -155,7 +156,7 @@ type saveResult struct {
 	bytes uint64
 }
 
-func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}, file *os.File, resultChannel chan<- saveResult) {
+func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}, file fs.File, resultChannel chan<- saveResult) {
 	defer freeBuf(chunk.Data)
 
 	id := backend.Hash(chunk.Data)
@@ -209,7 +210,7 @@ func updateNodeContent(node *Node, results []saveResult) error {
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
 func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
-	file, err := node.OpenForReading()
+	file, err := fs.Open(node.path)
 	defer file.Close()
 	if err != nil {
 		return err

src/restic/fs/doc.go

@@ -0,0 +1,3 @@
+// Package fs implements an OS independend abstraction of a file system
+// suitable for backup purposes.
+package fs

src/restic/fs/file.go

@@ -0,0 +1,18 @@
+package fs
+
+import (
+	"io"
+	"os"
+)
+
+// File is an open file on a file system.
+type File interface {
+	io.Reader
+	io.Writer
+	io.Closer
+
+	Stat() (os.FileInfo, error)
+
+	// ClearCache removes the file's content from the OS cache.
+	ClearCache() error
+}

src/restic/fs/file_all.go

@@ -0,0 +1,20 @@
+// +build !linux
+
+package fs
+
+import "os"
+
+// Open opens a file for reading.
+func Open(name string) (File, error) {
+	f, err := os.OpenFile(name, os.O_RDONLY, 0)
+	return osFile{File: f}, err
+}
+
+// osFile wraps an *os.File and adds a no-op ClearCache() method.
+type osFile struct {
+	*os.File
+}
+
+func (osFile) ClearCache() error {
+	return nil
+}

src/restic/fs/file_linux.go

@@ -0,0 +1,69 @@
+package fs
+
+import (
+	"os"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// Open opens a file for reading, without updating the atime and without caching data on read.
+func Open(name string) (File, error) {
+	file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0)
+	if os.IsPermission(err) {
+		file, err = os.OpenFile(name, os.O_RDONLY, 0)
+	}
+	return &nonCachingFile{File: file}, err
+}
+
+// osFile wraps an *os.File and adds a no-op ClearCache() method.
+type osFile struct {
+	*os.File
+}
+
+func (osFile) ClearCache() error {
+	return nil
+}
+
+// these constants should've been defined in x/sys/unix, but somehow aren't.
+const (
+	_POSIX_FADV_NORMAL = iota
+	_POSIX_FADV_RANDOM
+	_POSIX_FADV_SEQUENTIAL
+	_POSIX_FADV_WILLNEED
+	_POSIX_FADV_DONTNEED
+	_POSIX_FADV_NOREUSE
+)
+
+// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget
+// data that has been read or written.
+type nonCachingFile struct {
+	*os.File
+	readOffset int64
+}
+
+func (f *nonCachingFile) Read(p []byte) (int, error) {
+	n, err := f.File.Read(p)
+
+	if n > 0 {
+		ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), _POSIX_FADV_DONTNEED)
+
+		f.readOffset += int64(n)
+
+		if err == nil {
+			err = ferr
+		}
+	}
+
+	return n, err
+}
+
+func (f *nonCachingFile) ClearCache() error {
+	err := f.File.Sync()
+
+	if err != nil {
+		return err
+	}
+
+	return unix.Fadvise(int(f.File.Fd()), 0, 0, _POSIX_FADV_DONTNEED)
+}

src/restic/node_darwin.go

@@ -1,13 +1,6 @@
 package restic
 
-import (
+import "syscall"
-	"os"
-	"syscall"
-)
-
-func (node *Node) OpenForReading() (*os.File, error) {
-	return os.Open(node.path)
-}
 
 func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
 	return nil

src/restic/node_freebsd.go

@@ -1,13 +1,6 @@
 package restic
 
-import (
+import "syscall"
-	"os"
-	"syscall"
-)
-
-func (node *Node) OpenForReading() (*os.File, error) {
-	return os.OpenFile(node.path, os.O_RDONLY, 0)
-}
 
 func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
 	return nil

src/restic/node_linux.go

@@ -9,14 +9,6 @@ import (
 	"github.com/juju/errors"
 )
 
-func (node *Node) OpenForReading() (*os.File, error) {
-	file, err := os.OpenFile(node.path, os.O_RDONLY|syscall.O_NOATIME, 0)
-	if os.IsPermission(err) {
-		return os.OpenFile(node.path, os.O_RDONLY, 0)
-	}
-	return file, err
-}
-
 func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
 	dir, err := os.Open(filepath.Dir(path))
 	defer dir.Close()

src/restic/node_openbsd.go

@@ -1,17 +1,6 @@
 package restic
 
-import (
+import "syscall"
-	"os"
-	"syscall"
-)
-
-func (node *Node) OpenForReading() (*os.File, error) {
-	file, err := os.OpenFile(node.path, os.O_RDONLY, 0)
-	if os.IsPermission(err) {
-		return os.OpenFile(node.path, os.O_RDONLY, 0)
-	}
-	return file, err
-}
 
 func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
 	return nil

src/restic/node_windows.go

@@ -2,14 +2,9 @@ package restic
 
 import (
 	"errors"
-	"os"
 	"syscall"
 )
 
-func (node *Node) OpenForReading() (*os.File, error) {
-	return os.OpenFile(node.path, os.O_RDONLY, 0)
-}
-
 // mknod() creates a filesystem node (file, device
 // special file, or named pipe) named pathname, with attributes
 // specified by mode and dev.