Merge pull request #3261 from DRON-666/prealloc-packs

Prevent local backend file fragmentation by file preallocation.

changelog/unreleased/pull-3261

@@ -0,0 +1,8 @@
+Enhancement: Reduce file fragmentation for local backend
+
+Before this change, local backend files could become fragmented.
+Now restic will try to preallocate space for pack files to avoid
+their fragmentation.
+
+https://github.com/restic/restic/issues/2679
+https://github.com/restic/restic/pull/3261

internal/backend/local/local.go

@@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader)
 		}
 	}(f)
 
+	// preallocate disk space
+	if size := rd.Length(); size > 0 {
+		if err := fs.PreallocateFile(f, size); err != nil {
+			debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err)
+		}
+	}
+
 	// save data, then sync
 	wbytes, err := io.Copy(f, rd)
 	if err != nil {

internal/fs/preallocate_darwin.go

@@ -1,4 +1,4 @@
-package restorer
+package fs
 
 import (
 	"os"
@@ -6,7 +6,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func preallocateFile(wr *os.File, size int64) error {
+func PreallocateFile(wr *os.File, size int64) error {
 	// try contiguous first
 	fst := unix.Fstore_t{
 		Flags:   unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,

internal/fs/preallocate_linux.go

@@ -1,4 +1,4 @@
-package restorer
+package fs
 
 import (
 	"os"
@@ -6,7 +6,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func preallocateFile(wr *os.File, size int64) error {
+func PreallocateFile(wr *os.File, size int64) error {
 	if size <= 0 {
 		return nil
 	}

internal/fs/preallocate_other.go

@@ -1,11 +1,11 @@
 //go:build !linux && !darwin
 // +build !linux,!darwin
 
-package restorer
+package fs
 
 import "os"
 
-func preallocateFile(wr *os.File, size int64) error {
+func PreallocateFile(wr *os.File, size int64) error {
 	// Maybe truncate can help?
 	// Windows: This calls SetEndOfFile which preallocates space on disk
 	return wr.Truncate(size)

internal/fs/preallocate_test.go

@@ -1,4 +1,4 @@
-package restorer
+package fs
 
 import (
 	"os"
@@ -7,7 +7,6 @@ import (
 	"syscall"
 	"testing"
 
-	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/test"
 )
 
@@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) {
 				test.OK(t, wr.Close())
 			}()
 
-			err = preallocateFile(wr, i)
+			err = PreallocateFile(wr, i)
 			if err == syscall.ENOTSUP {
 				t.SkipNow()
 			}
@@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) {
 			fi, err := wr.Stat()
 			test.OK(t, err)
 
-			efi := fs.ExtendedStat(fi)
+			efi := ExtendedStat(fi)
 			test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks)
 		})
 	}

internal/restorer/fileswriter.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/fs"
 )
 
 // writes blobs to target files.
@@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 					return nil, err
 				}
 			} else {
-				err := preallocateFile(wr.File, createSize)
+				err := fs.PreallocateFile(wr.File, createSize)
 				if err != nil {
 					// Just log the preallocate error but don't let it cause the restore process to fail.
 					// Preallocate might return an error if the filesystem (implementation) does not