forked from TrueCloudLab/restic
Merge pull request #4882 from MichaelEischer/improve-check-output
check: improve output if repository is damaged
This commit is contained in:
commit
2a7d257036
5 changed files with 62 additions and 38 deletions
|
@ -1,11 +1,12 @@
|
||||||
Enhancement: Improve `repair packs` command
|
Enhancement: Improve `repair packs` command
|
||||||
|
|
||||||
The `repair packs` command has been improved to also be able to process
|
The `repair packs` command has been improved to also be able to process
|
||||||
truncated pack files. The `check --read-data` command will provide instructions
|
truncated pack files. The `check` and `check --read-data` command will provide
|
||||||
on using the command if necessary to repair a repository. See the guide at
|
instructions on using the command if necessary to repair a repository. See the
|
||||||
https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further
|
guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for
|
||||||
instructions.
|
further instructions.
|
||||||
|
|
||||||
https://github.com/restic/restic/issues/828
|
https://github.com/restic/restic/issues/828
|
||||||
https://github.com/restic/restic/pull/4644
|
https://github.com/restic/restic/pull/4644
|
||||||
https://github.com/restic/restic/pull/4655
|
https://github.com/restic/restic/pull/4655
|
||||||
|
https://github.com/restic/restic/pull/4882
|
||||||
|
|
|
@ -264,7 +264,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
|
term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
|
||||||
}
|
}
|
||||||
if suggestLegacyIndexRebuild {
|
if suggestLegacyIndexRebuild {
|
||||||
printer.E("Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
|
printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
|
||||||
}
|
}
|
||||||
if mixedFound {
|
if mixedFound {
|
||||||
term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
|
term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
|
||||||
|
@ -274,28 +274,42 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
for _, err := range errs {
|
for _, err := range errs {
|
||||||
printer.E("error: %v\n", err)
|
printer.E("error: %v\n", err)
|
||||||
}
|
}
|
||||||
return errors.Fatal("LoadIndex returned errors")
|
|
||||||
|
printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n")
|
||||||
|
return errors.Fatal("repository contains errors")
|
||||||
}
|
}
|
||||||
|
|
||||||
orphanedPacks := 0
|
orphanedPacks := 0
|
||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
|
salvagePacks := restic.NewIDSet()
|
||||||
|
|
||||||
printer.P("check all packs\n")
|
printer.P("check all packs\n")
|
||||||
go chkr.Packs(ctx, errChan)
|
go chkr.Packs(ctx, errChan)
|
||||||
|
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
if checker.IsOrphanedPack(err) {
|
var packErr *checker.PackError
|
||||||
|
if errors.As(err, &packErr) {
|
||||||
|
if packErr.Orphaned {
|
||||||
orphanedPacks++
|
orphanedPacks++
|
||||||
printer.P("%v\n", err)
|
printer.V("%v\n", err)
|
||||||
|
} else {
|
||||||
|
if packErr.Truncated {
|
||||||
|
salvagePacks.Insert(packErr.ID)
|
||||||
|
}
|
||||||
|
errorsFound = true
|
||||||
|
printer.E("%v\n", err)
|
||||||
|
}
|
||||||
} else if err == checker.ErrLegacyLayout {
|
} else if err == checker.ErrLegacyLayout {
|
||||||
printer.P("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
|
errorsFound = true
|
||||||
|
printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
|
||||||
} else {
|
} else {
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
printer.E("%v\n", err)
|
printer.E("%v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedPacks > 0 {
|
if orphanedPacks > 0 && !errorsFound {
|
||||||
|
// hide notice if repository is damaged
|
||||||
printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
|
printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
|
@ -353,26 +367,14 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
|
|
||||||
go chkr.ReadPacks(ctx, packs, p, errChan)
|
go chkr.ReadPacks(ctx, packs, p, errChan)
|
||||||
|
|
||||||
var salvagePacks restic.IDs
|
|
||||||
|
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
printer.E("%v\n", err)
|
printer.E("%v\n", err)
|
||||||
if err, ok := err.(*repository.ErrPackData); ok {
|
if err, ok := err.(*repository.ErrPackData); ok {
|
||||||
salvagePacks = append(salvagePacks, err.PackID)
|
salvagePacks.Insert(err.PackID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.Done()
|
p.Done()
|
||||||
|
|
||||||
if len(salvagePacks) > 0 {
|
|
||||||
printer.E("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
|
|
||||||
var strIDs []string
|
|
||||||
for _, id := range salvagePacks {
|
|
||||||
strIDs = append(strIDs, id.String())
|
|
||||||
}
|
|
||||||
printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
|
|
||||||
printer.E("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
|
@ -416,11 +418,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
doReadData(packs)
|
doReadData(packs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(salvagePacks) > 0 {
|
||||||
|
printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
|
||||||
|
var strIDs []string
|
||||||
|
for id := range salvagePacks {
|
||||||
|
strIDs = append(strIDs, id.String())
|
||||||
|
}
|
||||||
|
printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
|
||||||
|
printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
|
||||||
|
}
|
||||||
|
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if errorsFound {
|
if errorsFound {
|
||||||
|
if len(salvagePacks) == 0 {
|
||||||
|
printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
|
||||||
|
}
|
||||||
return errors.Fatal("repository contains errors")
|
return errors.Fatal("repository contains errors")
|
||||||
}
|
}
|
||||||
printer.P("no errors were found\n")
|
printer.P("no errors were found\n")
|
||||||
|
|
|
@ -368,10 +368,22 @@ detect this and yield the same error as when you tried to restore:
|
||||||
$ restic -r /srv/restic-repo check
|
$ restic -r /srv/restic-repo check
|
||||||
...
|
...
|
||||||
load indexes
|
load indexes
|
||||||
error: error loading index de30f323: load <index/de30f3231c>: invalid data returned
|
error: error loading index de30f3231ca2e6a59af4aa84216dfe2ef7339c549dc11b09b84000997b139628: LoadRaw(<index/de30f3231c>): invalid data returned
|
||||||
Fatal: LoadIndex returned errors
|
|
||||||
|
|
||||||
If the repository structure is intact, restic will show that no errors were found:
|
The repository index is damaged and must be repaired. You must run `restic repair index' to correct this.
|
||||||
|
|
||||||
|
Fatal: repository contains errors
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
If ``check`` reports an error in the repository, then you must repair the repository.
|
||||||
|
As long as a repository is damaged, restoring some files or directories will fail. New
|
||||||
|
snapshots are not guaranteed to be restorable either.
|
||||||
|
|
||||||
|
For instructions how to repair a damaged repository, see the :ref:`troubleshooting`
|
||||||
|
section or follow the instructions provided by the ``check`` command.
|
||||||
|
|
||||||
|
If the repository structure is intact, restic will show that ``no errors were found``:
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
|
|
|
@ -10,6 +10,8 @@
|
||||||
^ for subsubsections
|
^ for subsubsections
|
||||||
" for paragraphs
|
" for paragraphs
|
||||||
|
|
||||||
|
.. _troubleshooting:
|
||||||
|
|
||||||
#########################
|
#########################
|
||||||
Troubleshooting
|
Troubleshooting
|
||||||
#########################
|
#########################
|
||||||
|
|
|
@ -185,6 +185,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
|
||||||
type PackError struct {
|
type PackError struct {
|
||||||
ID restic.ID
|
ID restic.ID
|
||||||
Orphaned bool
|
Orphaned bool
|
||||||
|
Truncated bool
|
||||||
Err error
|
Err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -192,13 +193,6 @@ func (e *PackError) Error() string {
|
||||||
return "pack " + e.ID.String() + ": " + e.Err.Error()
|
return "pack " + e.ID.String() + ": " + e.Err.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsOrphanedPack returns true if the error describes a pack which is not
|
|
||||||
// contained in any index.
|
|
||||||
func IsOrphanedPack(err error) bool {
|
|
||||||
var e *PackError
|
|
||||||
return errors.As(err, &e) && e.Orphaned
|
|
||||||
}
|
|
||||||
|
|
||||||
func isS3Legacy(b backend.Backend) bool {
|
func isS3Legacy(b backend.Backend) bool {
|
||||||
be := backend.AsBackend[*s3.Backend](b)
|
be := backend.AsBackend[*s3.Backend](b)
|
||||||
return be != nil && be.Layout.Name() == "s3legacy"
|
return be != nil && be.Layout.Name() == "s3legacy"
|
||||||
|
@ -250,7 +244,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case errChan <- &PackError{ID: id, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
|
case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue