Merge pull request #4925 from MichaelEischer/fix-restore-truncation

restore: Fix truncation of uptodate files

internal/restorer/filerestorer.go

@@ -93,17 +93,20 @@ func (r *fileRestorer) targetPath(location string) string {
 	return filepath.Join(r.dst, location)
 }
 
-func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int)) error {
+func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error {
 	if len(blobIDs) == 0 {
 		return nil
 	}
 
+	fileOffset := int64(0)
 	for i, blobID := range blobIDs {
 		packs := r.idx(restic.DataBlob, blobID)
 		if len(packs) == 0 {
 			return errors.Errorf("Unknown blob %s", blobID.String())
 		}
-		fn(packs[0].PackID, packs[0].Blob, i)
+		pb := packs[0]
+		fn(pb.PackID, pb.Blob, i, fileOffset)
+		fileOffset += int64(pb.DataLength())
 	}
 
 	return nil
@@ -120,28 +123,24 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 	// create packInfo from fileInfo
 	for _, file := range r.files {
 		fileBlobs := file.blobs.(restic.IDs)
-		if len(fileBlobs) == 0 {
-			err := r.restoreEmptyFileAt(file.location)
-			if errFile := r.sanitizeError(file, err); errFile != nil {
-				return errFile
-			}
-		}
-
 		largeFile := len(fileBlobs) > largeFileBlobCount
 		var packsMap map[restic.ID][]fileBlobInfo
 		if largeFile {
 			packsMap = make(map[restic.ID][]fileBlobInfo)
+			file.blobs = packsMap
 		}
-		fileOffset := int64(0)
+		restoredBlobs := false
-		err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int) {
+		err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
-			if largeFile {
+			if !file.state.HasMatchingBlob(idx) {
-				if !file.state.HasMatchingBlob(idx) {
+				if largeFile {
 					packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset})
-				} else {
-					r.reportBlobProgress(file, uint64(blob.DataLength()))
 				}
+				restoredBlobs = true
+			} else {
+				r.reportBlobProgress(file, uint64(blob.DataLength()))
+				// completely ignore blob
+				return
 			}
-			fileOffset += int64(blob.DataLength())
 			pack, ok := packs[packID]
 			if !ok {
 				pack = &packInfo{
@@ -156,6 +155,11 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 				file.sparse = r.sparse
 			}
 		})
+		if err != nil {
+			// repository index is messed up, can't do anything
+			return err
+		}
+
 		if len(fileBlobs) == 1 {
 			// no need to preallocate files with a single block, thus we can always consider them to be sparse
 			// in addition, a short chunk will never match r.zeroChunk which would prevent sparseness for short files
@@ -168,12 +172,17 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 			file.sparse = false
 		}
 
-		if err != nil {
+		// empty file or one with already uptodate content. Make sure that the file size is correct
-			// repository index is messed up, can't do anything
+		if !restoredBlobs {
-			return err
+			err := r.truncateFileToSize(file.location, file.size)
-		}
+			if errFile := r.sanitizeError(file, err); errFile != nil {
-		if largeFile {
+				return errFile
-			file.blobs = packsMap
+			}
+
+			// the progress events were already sent for non-zero size files
+			if file.size == 0 {
+				r.reportBlobProgress(file, 0)
+			}
 		}
 	}
 	// drop no longer necessary file list
@@ -214,17 +223,12 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 	return wg.Wait()
 }
 
-func (r *fileRestorer) restoreEmptyFileAt(location string) error {
+func (r *fileRestorer) truncateFileToSize(location string, size int64) error {
-	f, err := createFile(r.targetPath(location), 0, false, r.allowRecursiveDelete)
+	f, err := createFile(r.targetPath(location), size, false, r.allowRecursiveDelete)
 	if err != nil {
 		return err
 	}
-	if err = f.Close(); err != nil {
+	return f.Close()
-		return err
-	}
-
-	r.progress.AddProgress(location, restore.ActionFileRestored, 0, 0)
-	return nil
 }
 
 type blobToFileOffsetsMapping map[restic.ID]struct {
@@ -246,16 +250,10 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 			blobInfo.files[file] = append(blobInfo.files[file], fileOffset)
 		}
 		if fileBlobs, ok := file.blobs.(restic.IDs); ok {
-			fileOffset := int64(0)
+			err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
-			err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int) {
+				if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) {
-				if packID.Equal(pack.id) {
+					addBlob(blob, fileOffset)
-					if !file.state.HasMatchingBlob(idx) {
-						addBlob(blob, fileOffset)
-					} else {
-						r.reportBlobProgress(file, uint64(blob.DataLength()))
-					}
 				}
-				fileOffset += int64(blob.DataLength())
 			})
 			if err != nil {
 				// restoreFiles should have caught this error before

internal/restorer/restorer_test.go

@@ -1003,6 +1003,7 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			"dirtest": Dir{
 				Nodes: map[string]Node{
 					"file": File{Data: "content: file\n", ModTime: baseTime},
+					"foo":  File{Data: "content: foobar", ModTime: baseTime},
 				},
 				ModTime: baseTime,
 			},
@@ -1014,6 +1015,7 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			"dirtest": Dir{
 				Nodes: map[string]Node{
 					"file": File{Data: "content: file2\n", ModTime: baseTime.Add(-time.Second)},
+					"foo":  File{Data: "content: foo", ModTime: baseTime},
 				},
 			},
 		},
@@ -1029,13 +1031,14 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			Files: map[string]string{
 				"foo":          "content: new\n",
 				"dirtest/file": "content: file2\n",
+				"dirtest/foo":  "content: foo",
 			},
 			Progress: restoreui.State{
-				FilesFinished:   3,
+				FilesFinished:   4,
-				FilesTotal:      3,
+				FilesTotal:      4,
 				FilesSkipped:    0,
-				AllBytesWritten: 28,
+				AllBytesWritten: 40,
-				AllBytesTotal:   28,
+				AllBytesTotal:   40,
 				AllBytesSkipped: 0,
 			},
 		},
@@ -1044,13 +1047,14 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			Files: map[string]string{
 				"foo":          "content: new\n",
 				"dirtest/file": "content: file2\n",
+				"dirtest/foo":  "content: foo",
 			},
 			Progress: restoreui.State{
-				FilesFinished:   3,
+				FilesFinished:   4,
-				FilesTotal:      3,
+				FilesTotal:      4,
 				FilesSkipped:    0,
-				AllBytesWritten: 28,
+				AllBytesWritten: 40,
-				AllBytesTotal:   28,
+				AllBytesTotal:   40,
 				AllBytesSkipped: 0,
 			},
 		},
@@ -1059,14 +1063,15 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			Files: map[string]string{
 				"foo":          "content: new\n",
 				"dirtest/file": "content: file\n",
+				"dirtest/foo":  "content: foobar",
 			},
 			Progress: restoreui.State{
 				FilesFinished:   2,
 				FilesTotal:      2,
-				FilesSkipped:    1,
+				FilesSkipped:    2,
 				AllBytesWritten: 13,
 				AllBytesTotal:   13,
-				AllBytesSkipped: 15,
+				AllBytesSkipped: 27,
 			},
 		},
 		{
@@ -1074,14 +1079,15 @@ func TestRestorerOverwriteBehavior(t *testing.T) {
 			Files: map[string]string{
 				"foo":          "content: foo\n",
 				"dirtest/file": "content: file\n",
+				"dirtest/foo":  "content: foobar",
 			},
 			Progress: restoreui.State{
 				FilesFinished:   1,
 				FilesTotal:      1,
-				FilesSkipped:    2,
+				FilesSkipped:    3,
 				AllBytesWritten: 0,
 				AllBytesTotal:   0,
-				AllBytesSkipped: 28,
+				AllBytesSkipped: 40,
 			},
 		},
 	}