Merge pull request #4530 from MichaelEischer/repair-packs

Add `repair packs` command to simplify recovery from damaged pack

cmd/restic/cmd_check.go

@@ -330,11 +330,28 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
+		var salvagePacks restic.IDs
+
 		for err := range errChan {
 			errorsFound = true
 			Warnf("%v\n", err)
+			if err, ok := err.(*checker.ErrPackData); ok {
+				if strings.Contains(err.Error(), "wrong data returned, hash is") {
+					salvagePacks = append(salvagePacks, err.PackID)
+				}
+			}
 		}
 		p.Done()
+
+		if len(salvagePacks) > 0 {
+			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n")
+			var strIds []string
+			for _, id := range salvagePacks {
+				strIds = append(strIds, id.String())
+			}
+			Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " "))
+			Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
+		}
 	}
 
 	switch {

cmd/restic/cmd_repair_packs.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+)
+
+var cmdRepairPacks = &cobra.Command{
+	Use:   "packs [packIDs...]",
+	Short: "Salvage damaged pack files",
+	Long: `
+WARNING: The CLI for this command is experimental and will likely change in the future!
+
+The "repair packs" command extracts intact blobs from the specified pack files, rebuilds
+the index to remove the damaged pack files and removes the pack files from the repository.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRepairPacks(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdRepair.AddCommand(cmdRepairPacks)
+}
+
+func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error {
+	// FIXME discuss and add proper feature flag mechanism
+	flag, _ := os.LookupEnv("RESTIC_FEATURES")
+	if flag != "repair-packs-v1" {
+		return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " +
+			"Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.")
+	}
+
+	ids := restic.NewIDSet()
+	for _, arg := range args {
+		id, err := restic.ParseID(arg)
+		if err != nil {
+			return err
+		}
+		ids.Insert(id)
+	}
+	if len(ids) == 0 {
+		return errors.Fatal("no ids specified")
+	}
+
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return repairPacks(ctx, gopts, repo, ids)
+}
+
+func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error {
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err := repo.LoadIndex(ctx, bar)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	Warnf("saving backup copies of pack files in current folder\n")
+	for id := range ids {
+		f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+
+		err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
+			_, err := f.Seek(0, 0)
+			if err != nil {
+				return err
+			}
+			_, err = io.Copy(f, rd)
+			return err
+		})
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	}
+
+	wg, wgCtx := errgroup.WithContext(ctx)
+	repo.StartPackUploader(wgCtx, wg)
+	repo.DisableAutoIndexUpdate()
+
+	Warnf("salvaging intact data from specified pack files\n")
+	bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files")
+	defer bar.Done()
+
+	wg.Go(func() error {
+		// examine all data the indexes have for the pack file
+		for b := range repo.Index().ListPacks(wgCtx, ids) {
+			blobs := b.Blobs
+			if len(blobs) == 0 {
+				Warnf("no blobs found for pack %v\n", b.PackID)
+				bar.Add(1)
+				continue
+			}
+
+			err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
+				if err != nil {
+					// Fallback path
+					buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil)
+					if err != nil {
+						Warnf("failed to load blob %v: %v\n", blob.ID, err)
+						return nil
+					}
+				}
+				id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true)
+				if !id.Equal(blob.ID) {
+					panic("pack id mismatch during upload")
+				}
+				return err
+			})
+			if err != nil {
+				return err
+			}
+			bar.Add(1)
+		}
+		return repo.Flush(wgCtx)
+	})
+
+	if err := wg.Wait(); err != nil {
+		return errors.Fatalf("%s", err)
+	}
+	bar.Done()
+
+	// remove salvaged packs from index
+	err = rebuildIndexFiles(ctx, gopts, repo, ids, nil)
+	if err != nil {
+		return errors.Fatalf("%s", err)
+	}
+
+	// cleanup
+	Warnf("removing salvaged pack files\n")
+	DeleteFiles(ctx, gopts, repo, ids, restic.PackFile)
+
+	Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n")
+	return nil
+}

internal/checker/checker.go

@@ -90,6 +90,16 @@ func (err *ErrOldIndexFormat) Error() string {
 	return fmt.Sprintf("index %v has old format", err.ID)
 }
 
+// ErrPackData is returned if errors are discovered while verifying a packfile
+type ErrPackData struct {
+	PackID restic.ID
+	errs   []error
+}
+
+func (e *ErrPackData) Error() string {
+	return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
+}
+
 func (c *Checker) LoadSnapshots(ctx context.Context) error {
 	var err error
 	c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile)
@@ -635,7 +645,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r
 	}
 
 	if len(errs) > 0 {
-		return errors.Errorf("pack %v contains %v errors: %v", id, len(errs), errs)
+		return &ErrPackData{PackID: id, errs: errs}
 	}
 
 	return nil