From b1250eead9bb6f69e618993509ef1964bd3e10ce Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 10:55:44 +0200 Subject: [PATCH 1/9] check: mark s3legacy layout and legacy indexes are error --- cmd/restic/cmd_check.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index cccb1986e..1a27ed6ea 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -264,7 +264,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n") } if suggestLegacyIndexRebuild { - printer.E("Found indexes using the legacy format, you must run `restic repair index' to correct this.\n") + printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n") } if mixedFound { term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") @@ -288,7 +288,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args orphanedPacks++ printer.P("%v\n", err) } else if err == checker.ErrLegacyLayout { - printer.P("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n") + errorsFound = true + printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n") } else { errorsFound = true printer.E("%v\n", err) From 1671a3fe2e6a454bab916182ea40d0e61aa5f72c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 10:57:12 +0200 Subject: [PATCH 2/9] check: hide message about additional files if error in repo The message says "[...] addition files were found [...]. This is non-critical [...]". Unless users are highly experienced with restic, it's hard to correctly interpret what "This" refers to. Thus, just hide the message if there is a real problem. --- cmd/restic/cmd_check.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 1a27ed6ea..2da877fad 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -296,7 +296,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } } - if orphanedPacks > 0 { + if orphanedPacks > 0 && !errorsFound { + // hide notice if repository is damaged printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) } if ctx.Err() != nil { From da338d5aa85113716423c0e62ff6896cb1c4c9e0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:02:36 +0200 Subject: [PATCH 3/9] check: tweak wording of repair packs message By now, the message is also shown for truncated or otherwise damaged pack files, not just those with corrupted blobs. --- cmd/restic/cmd_check.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 2da877fad..472cc742a 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -367,13 +367,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args p.Done() if len(salvagePacks) > 0 { - printer.E("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") + printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") var strIDs []string for _, id := range salvagePacks { strIDs = append(strIDs, id.String()) } printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) - printer.E("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") + printer.E("Damaged pack files can be caused by backend problem, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } } From c9a4a95848ec6d50dda861a2dad6c7329edc3686 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:06:49 +0200 Subject: [PATCH 4/9] check: suggest using `repair packs` to repair truncated pack files Previously, that help message was only shown for running `check --read-data`. --- cmd/restic/cmd_check.go | 40 ++++++++++++++++++++++--------------- internal/checker/checker.go | 16 +++++---------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 472cc742a..89bb30868 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -279,14 +279,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args orphanedPacks := 0 errChan := make(chan error) + salvagePacks := restic.NewIDSet() printer.P("check all packs\n") go chkr.Packs(ctx, errChan) for err := range errChan { - if checker.IsOrphanedPack(err) { - orphanedPacks++ - printer.P("%v\n", err) + var packErr *checker.PackError + if errors.As(err, &packErr) { + if packErr.Orphaned { + orphanedPacks++ + printer.P("%v\n", err) + } else { + if packErr.Truncated { + salvagePacks.Insert(packErr.ID) + } + errorsFound = true + printer.E("%v\n", err) + } } else if err == checker.ErrLegacyLayout { errorsFound = true printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n") @@ -355,26 +365,14 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args go chkr.ReadPacks(ctx, packs, p, errChan) - var salvagePacks restic.IDs - for err := range errChan { errorsFound = true printer.E("%v\n", err) if err, ok := err.(*repository.ErrPackData); ok { - salvagePacks = append(salvagePacks, err.PackID) + salvagePacks.Insert(err.PackID) } } p.Done() - - if len(salvagePacks) > 0 { - printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") - var strIDs []string - for _, id := range salvagePacks { - strIDs = append(strIDs, id.String()) - } - printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) - printer.E("Damaged pack files can be caused by backend problem, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") - } } switch { @@ -418,6 +416,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args doReadData(packs) } + if len(salvagePacks) > 0 { + printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") + var strIDs []string + for id := range salvagePacks { + strIDs = append(strIDs, id.String()) + } + printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) + printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") + } + if ctx.Err() != nil { return ctx.Err() } diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 82348c7ea..031e13807 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -183,22 +183,16 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e // PackError describes an error with a specific pack. type PackError struct { - ID restic.ID - Orphaned bool - Err error + ID restic.ID + Orphaned bool + Truncated bool + Err error } func (e *PackError) Error() string { return "pack " + e.ID.String() + ": " + e.Err.Error() } -// IsOrphanedPack returns true if the error describes a pack which is not -// contained in any index. -func IsOrphanedPack(err error) bool { - var e *PackError - return errors.As(err, &e) && e.Orphaned -} - func isS3Legacy(b backend.Backend) bool { be := backend.AsBackend[*s3.Backend](b) return be != nil && be.Layout.Name() == "s3legacy" @@ -250,7 +244,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { select { case <-ctx.Done(): return - case errChan <- &PackError{ID: id, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: + case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: } } } From 6d9dfff1cbe18846cb079f4c76b20910af907f73 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:08:52 +0200 Subject: [PATCH 5/9] check: point users towards the troubleshooting guide if repo has errors --- cmd/restic/cmd_check.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 89bb30868..ffb97ccdb 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -431,6 +431,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } if errorsFound { + if len(salvagePacks) == 0 { + printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n") + } return errors.Fatal("repository contains errors") } printer.P("no errors were found\n") From 879ba07a87d1819468e38f41d69dfa9544e30ffa Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:12:08 +0200 Subject: [PATCH 6/9] check: only show additional files if verbose output is enabled Additional files are nearly always caused by interrupted backup runs. This is unproblematic, thus don't pollute the check output with it. --- cmd/restic/cmd_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index ffb97ccdb..4cc9c666e 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -289,7 +289,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if errors.As(err, &packErr) { if packErr.Orphaned { orphanedPacks++ - printer.P("%v\n", err) + printer.V("%v\n", err) } else { if packErr.Truncated { salvagePacks.Insert(packErr.ID) From a03e00373ce9a701bf9a65c6a8a3053654ca8cf6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:18:53 +0200 Subject: [PATCH 7/9] update repair packs changelog --- changelog/unreleased/issue-828 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/changelog/unreleased/issue-828 b/changelog/unreleased/issue-828 index 6e1354225..2e8bcd0b0 100644 --- a/changelog/unreleased/issue-828 +++ b/changelog/unreleased/issue-828 @@ -1,11 +1,12 @@ Enhancement: Improve `repair packs` command The `repair packs` command has been improved to also be able to process -truncated pack files. The `check --read-data` command will provide instructions -on using the command if necessary to repair a repository. See the guide at -https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further -instructions. +truncated pack files. The `check` and `check --read-data` command will provide +instructions on using the command if necessary to repair a repository. See the +guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for +further instructions. https://github.com/restic/restic/issues/828 https://github.com/restic/restic/pull/4644 https://github.com/restic/restic/pull/4655 +https://github.com/restic/restic/pull/4882 From fdc7349aa4cad31587f84ea99dbe4000b88d13cb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:42:23 +0200 Subject: [PATCH 8/9] check: improve error on damaged index Always return the `repository contains errors` message if a repository is damaged and must be repaired. Also provide specific instructions how to repair the index. --- cmd/restic/cmd_check.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 4cc9c666e..f416c9269 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -274,7 +274,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args for _, err := range errs { printer.E("error: %v\n", err) } - return errors.Fatal("LoadIndex returned errors") + + printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n") + return errors.Fatal("repository contains errors") } orphanedPacks := 0 From 283038056e1c1769abbed77c1a48891a56538526 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Jun 2024 11:45:29 +0200 Subject: [PATCH 9/9] doc: suggest to follow troubleshooting steps if check reports error --- doc/045_working_with_repos.rst | 18 +++++++++++++++--- doc/077_troubleshooting.rst | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 6b9666693..8dba8439f 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -368,10 +368,22 @@ detect this and yield the same error as when you tried to restore: $ restic -r /srv/restic-repo check ... load indexes - error: error loading index de30f323: load : invalid data returned - Fatal: LoadIndex returned errors + error: error loading index de30f3231ca2e6a59af4aa84216dfe2ef7339c549dc11b09b84000997b139628: LoadRaw(): invalid data returned -If the repository structure is intact, restic will show that no errors were found: + The repository index is damaged and must be repaired. You must run `restic repair index' to correct this. + + Fatal: repository contains errors + +.. warning:: + + If ``check`` reports an error in the repository, then you must repair the repository. + As long as a repository is damaged, restoring some files or directories will fail. New + snapshots are not guaranteed to be restorable either. + + For instructions how to repair a damaged repository, see the :ref:`troubleshooting` + section or follow the instructions provided by the ``check`` command. + +If the repository structure is intact, restic will show that ``no errors were found``: .. code-block:: console diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 33302e9e0..36c9d63ec 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -10,6 +10,8 @@ ^ for subsubsections " for paragraphs +.. _troubleshooting: + ######################### Troubleshooting #########################