restorer: pre-allocate files before loading chunks

changelog/unreleased/pull-2195

@@ -14,4 +14,10 @@ file can be written to the file before any of the preceeding file blobs.
 It is therefore possible to have gaps in the data written to the target
 files if restore fails or interrupted by the user.
 
+The implementation will try to preallocate space for the restored files
+on the filesystem to prevent file fragmentation. This ensures good read
+performance for large files, like for example VM images. If preallocating
+space is not supported by the filesystem, then this step is silently skipped.
+
 https://github.com/restic/restic/pull/2195
+https://github.com/restic/restic/pull/2893

internal/restorer/filerestorer.go

@@ -33,6 +33,7 @@ const (
 type fileInfo struct {
 	lock     sync.Mutex
 	flags    int
+	size     int64
 	location string      // file on local filesystem relative to restorer basedir
 	blobs    interface{} // blobs of the file
 }
@@ -74,8 +75,8 @@ func newFileRestorer(dst string,
 	}
 }
 
-func (r *fileRestorer) addFile(location string, content restic.IDs) {
-	r.files = append(r.files, &fileInfo{location: location, blobs: content})
+func (r *fileRestorer) addFile(location string, content restic.IDs, size int64) {
+	r.files = append(r.files, &fileInfo{location: location, blobs: content, size: size})
 }
 
 func (r *fileRestorer) targetPath(location string) string {
@@ -275,13 +276,15 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) {
 					// write other blobs after releasing the lock
 					file.lock.Lock()
 					create := file.flags&fileProgress == 0
+					createSize := int64(-1)
 					if create {
 						defer file.lock.Unlock()
 						file.flags |= fileProgress
+						createSize = file.size
 					} else {
 						file.lock.Unlock()
 					}
-					return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, create)
+					return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize)
 				}
 				err := writeToFile()
 				if err != nil {

internal/restorer/fileswriter.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/cespare/xxhash"
+	"github.com/restic/restic/internal/debug"
 )
 
 // writes blobs to target files.
@@ -33,7 +34,7 @@ func newFilesWriter(count int) *filesWriter {
 	}
 }
 
-func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bool) error {
+func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64) error {
 	bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
 
 	acquireWriter := func() (*os.File, error) {
@@ -46,7 +47,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 		}
 
 		var flags int
-		if create {
+		if createSize >= 0 {
 			flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY
 		} else {
 			flags = os.O_WRONLY
@@ -60,6 +61,18 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 		bucket.files[path] = wr
 		bucket.users[path] = 1
 
+		if createSize >= 0 {
+			err := preallocateFile(wr, createSize)
+			if err != nil {
+				// Just log the preallocate error but don't let it cause the restore process to fail.
+				// Preallocate might return an error if the filesystem (implementation) does not
+				// support preallocation or our parameters combination to the preallocate call
+				// This should yield a syscall.ENOTSUP error, but some other errors might also
+				// show up.
+				debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
+			}
+		}
+
 		return wr, nil
 	}
 

internal/restorer/fileswriter_test.go

@@ -16,19 +16,19 @@ func TestFilesWriterBasic(t *testing.T) {
 	f1 := dir + "/f1"
 	f2 := dir + "/f2"
 
-	rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, true))
+	rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, 2))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, true))
+	rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, 2))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, false))
+	rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, -1))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, false))
+	rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, -1))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 

internal/restorer/preallocate_darwin.go

@@ -0,0 +1,33 @@
+package restorer
+
+import (
+	"os"
+	"runtime"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+func preallocateFile(wr *os.File, size int64) error {
+	// try contiguous first
+	fst := unix.Fstore_t{
+		Flags:   unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,
+		Posmode: unix.F_PEOFPOSMODE,
+		Offset:  0,
+		Length:  size,
+	}
+	_, err := unix.FcntlInt(wr.Fd(), unix.F_PREALLOCATE, int(uintptr(unsafe.Pointer(&fst))))
+
+	if err == nil {
+		return nil
+	}
+
+	// just take preallocation in any form, but still ask for everything
+	fst.Flags = unix.F_ALLOCATEALL
+	_, err = unix.FcntlInt(wr.Fd(), unix.F_PREALLOCATE, int(uintptr(unsafe.Pointer(&fst))))
+
+	// Keep struct alive until fcntl has returned
+	runtime.KeepAlive(fst)
+
+	return err
+}

internal/restorer/preallocate_linux.go

@@ -0,0 +1,16 @@
+package restorer
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func preallocateFile(wr *os.File, size int64) error {
+	if size <= 0 {
+		return nil
+	}
+	// int fallocate(int fd, int mode, off_t offset, off_t len)
+	// use mode = 0 to also change the file size
+	return unix.Fallocate(int(wr.Fd()), 0, 0, size)
+}

internal/restorer/preallocate_other.go

@@ -0,0 +1,11 @@
+// +build !linux,!darwin
+
+package restorer
+
+import "os"
+
+func preallocateFile(wr *os.File, size int64) error {
+	// Maybe truncate can help?
+	// Windows: This calls SetEndOfFile which preallocates space on disk
+	return wr.Truncate(size)
+}

internal/restorer/restorer.go

@@ -238,7 +238,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
 				idx.Add(node.Inode, node.DeviceID, location)
 			}
 
-			filerestorer.addFile(location, node.Content)
+			filerestorer.addFile(location, node.Content, int64(node.Size))
 
 			return nil
 		},