TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #2970 from labkode/fopenskip

Browse source 
Skip fopen for file change check to avoid network penalty
This commit is contained in:
MichaelEischer 2020-10-09 11:43:54 +02:00 committed by GitHub
commit 862ee4b2c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
changelog/unreleased/issue-2969 Normal file
View file

@ -0,0 +1,9 @@
Enhancement: Optimize check for unchanged files during backup
During a backup restic skips processing files which have not changed since the last backup run.
Previously this required opening each file once which can be slow on network filesystems. The
backup command now checks for file changes before opening a file. This considerably reduces
the time to create a backup on network filesystems.
https://github.com/restic/restic/issues/2969
https://github.com/restic/restic/pull/2970

48
internal/archiver/archiver.go
View file

@ -363,6 +363,30 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
debug.Log(" %v regular file", target)
start := time.Now()
// check if the file has not changed before performing a fopen operation (more expensive, specially
// in network filesystems)
if previous != nil && !fileChanged(fi, previous, arch.IgnoreInode) {
if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(snPath, previous.Size)
fn.node, err = arch.nodeFromFileInfo(target, fi)
if err != nil {
return FutureNode{}, false, err
}
// copy list of blobs
fn.node.Content = previous.Content
return fn, false, nil
}
debug.Log("%v hasn't changed, but contents are missing!", target)
// There are contents missing - inform user!
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
arch.error(abstarget, fi, err)
}
// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
@ -397,30 +421,6 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
return FutureNode{}, true, nil
}
// use previous list of blobs if the file hasn't changed
if previous != nil && !fileChanged(fi, previous, arch.IgnoreInode) {
if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(snPath, previous.Size)
fn.node, err = arch.nodeFromFileInfo(target, fi)
if err != nil {
return FutureNode{}, false, err
}
// copy list of blobs
fn.node.Content = previous.Content
_ = file.Close()
return fn, false, nil
}
debug.Log("%v hasn't changed, but contents are missing!", target)
// There are contents missing - inform user!
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
arch.error(abstarget, fi, err)
}
fn.isFile = true
// Save will close the file, we don't need to do that
fn.file = arch.fileSaver.Save(ctx, snPath, file, fi, func() {