Merge pull request #5145 from MichaelEischer/ignore-disappeared-files

backup: Ignore disappeared files

changelog/unreleased/issue-2165

@@ -0,0 +1,16 @@
+Bugfix: Ignore disappeared backup source files
+
+If during a backup files were removed between restic listing the directory
+content and backing up the file in question, the following error could occur:
+
+```
+error: lstat /some/file/name: no such file or directory
+```
+
+The backup command now ignores this particular error and silently skips the
+removed file.
+
+https://github.com/restic/restic/issues/2165
+https://github.com/restic/restic/issues/3098
+https://github.com/restic/restic/pull/5143
+https://github.com/restic/restic/pull/5145

internal/archiver/archiver.go

@@ -464,6 +464,12 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 		}
 		return futureNode{}, true, nil
 	}
+	filterNotExist := func(err error) error {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
 	// exclude files by path before running Lstat to reduce number of lstat calls
 	if !arch.SelectByName(abstarget) {
 		debug.Log("%v is excluded by path", target)
@@ -473,7 +479,8 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	meta, err := arch.FS.OpenFile(target, fs.O_NOFOLLOW, true)
 	if err != nil {
 		debug.Log("open metadata for %v returned error: %v", target, err)
-		return filterError(err)
+		// ignore if file disappeared since it was returned by readdir
+		return filterError(filterNotExist(err))
 	}
 	closeFile := true
 	defer func() {
@@ -489,7 +496,8 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	fi, err := meta.Stat()
 	if err != nil {
 		debug.Log("lstat() for %v returned error: %v", target, err)
-		return filterError(err)
+		// ignore if file disappeared since it was returned by readdir
+		return filterError(filterNotExist(err))
 	}
 	if !arch.Select(abstarget, fi, arch.FS) {
 		debug.Log("%v is excluded", target)

internal/archiver/archiver_test.go

@@ -2479,3 +2479,48 @@ func TestIrregularFile(t *testing.T) {
 		t.Errorf("Save() excluded the node, that's unexpected")
 	}
 }
+
+type missingFS struct {
+	fs.FS
+	errorOnOpen bool
+}
+
+func (fs *missingFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
+	if fs.errorOnOpen {
+		return nil, os.ErrNotExist
+	}
+
+	return &missingFile{}, nil
+}
+
+type missingFile struct {
+	fs.File
+}
+
+func (f *missingFile) Stat() (os.FileInfo, error) {
+	return nil, os.ErrNotExist
+}
+
+func (f *missingFile) Close() error {
+	// prevent segfault in test
+	return nil
+}
+
+func TestDisappearedFile(t *testing.T) {
+	tempdir, repo := prepareTempdirRepoSrc(t, TestDir{})
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// depending on the underlying FS implementation a missing file may be detected by OpenFile or
+	// the subsequent file.Stat() call. Thus test both cases.
+	for _, errorOnOpen := range []bool{false, true} {
+		arch := New(repo, fs.Track{FS: &missingFS{FS: &fs.Local{}, errorOnOpen: errorOnOpen}}, Options{})
+		_, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "testdir"), nil)
+		rtest.OK(t, err)
+		rtest.Assert(t, excluded, "testfile should have been excluded")
+	}
+}

internal/fs/vss_windows.go

@@ -171,6 +171,11 @@ func (h HRESULT) Str() string {
 	return "UNKNOWN"
 }
 
+// Error implements the error interface
+func (h HRESULT) Error() string {
+	return h.Str()
+}
+
 // VssError encapsulates errors returned from calling VSS api.
 type vssError struct {
 	text    string
@@ -195,6 +200,11 @@ func (e *vssError) Error() string {
 	return fmt.Sprintf("VSS error: %s: %s (%#x)", e.text, e.hresult.Str(), e.hresult)
 }
 
+// Unwrap returns the underlying HRESULT error
+func (e *vssError) Unwrap() error {
+	return e.hresult
+}
+
 // vssTextError encapsulates errors returned from calling VSS api.
 type vssTextError struct {
 	text string
@@ -943,10 +953,23 @@ func NewVssSnapshot(provider string,
 			"%s", volume))
 	}
 
-	snapshotSetID, err := iVssBackupComponents.StartSnapshotSet()
-	if err != nil {
-		iVssBackupComponents.Release()
-		return VssSnapshot{}, err
+	const retryStartSnapshotSetSleep = 5 * time.Second
+	var snapshotSetID ole.GUID
+	for {
+		var err error
+		snapshotSetID, err = iVssBackupComponents.StartSnapshotSet()
+		if errors.Is(err, VSS_E_SNAPSHOT_SET_IN_PROGRESS) && time.Now().Add(-retryStartSnapshotSetSleep).Before(deadline) {
+			// retry snapshot set creation while deadline is not reached
+			time.Sleep(retryStartSnapshotSetSleep)
+			continue
+		}
+
+		if err != nil {
+			iVssBackupComponents.Release()
+			return VssSnapshot{}, err
+		} else {
+			break
+		}
 	}
 
 	if err := iVssBackupComponents.AddToSnapshotSet(volume, providerID, &snapshotSetID); err != nil {