From 6c01078f3d24e99e1ced6ff420a36b2d9b26be88 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 May 2021 23:12:14 +0200 Subject: [PATCH 1/6] find: support resolving multiple pack ids to blobs Just passing the list of blobs to packsToBlobs would also work in most cases, however, it could cause unexpected results when multiple pack files have the same prefix. Forget found prefixes to prevent this. --- cmd/restic/cmd_find.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 45d42b45f..76ca0a090 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -411,7 +411,6 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { } allPacksFound := false - packsFound := 0 debug.Log("Looking for packs...") err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { @@ -424,6 +423,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { if _, ok := packIDs[id.Str()]; !ok { return nil } + delete(packIDs, id.Str()) + } else { + // forget found id + delete(packIDs, idStr) } debug.Log("Found pack %s", idStr) blobs, _, err := f.repo.ListPack(ctx, id, size) @@ -434,8 +437,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { f.blobIDs[b.ID.String()] = struct{}{} } // Stop searching when all packs have been found - packsFound++ - if packsFound >= len(packIDs) { + if len(packIDs) == 0 { allPacksFound = true } return nil @@ -561,7 +563,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { } if opts.PackID { - err := f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs + err := f.packsToBlobs(ctx, f.pat.pattern) if err != nil { return err } From 40745b4f8262f6d51de48d04dee146c581977279 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 May 2021 23:37:59 +0200 Subject: [PATCH 2/6] find: stop file listing after resolving all pack files --- cmd/restic/cmd_find.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 76ca0a090..b05f721a1 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "time" @@ -399,6 +400,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { }) } +var errorAllPacksFound = fmt.Errorf("all packs found") + // packsToBlobs converts the list of pack IDs to a list of blob IDs that // belong to those packs. func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { @@ -410,13 +413,8 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { f.blobIDs = make(map[string]struct{}) } - allPacksFound := false - debug.Log("Looking for packs...") err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { - if allPacksFound { - return nil - } idStr := id.String() if _, ok := packIDs[idStr]; !ok { // Look for short ID form @@ -438,16 +436,16 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { } // Stop searching when all packs have been found if len(packIDs) == 0 { - allPacksFound = true + return errorAllPacksFound } return nil }) - if err != nil { + if err != nil && err != errorAllPacksFound { return err } - if !allPacksFound { + if err != errorAllPacksFound { return errors.Fatal("unable to find all specified pack(s)") } From 3caab3c7ac07da6ff6620aaac0b4ae36962809a2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 May 2021 23:45:27 +0200 Subject: [PATCH 3/6] find: Print not found pack files --- cmd/restic/cmd_find.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index b05f721a1..7f03fc42b 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "strings" "time" @@ -446,7 +447,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { } if err != errorAllPacksFound { - return errors.Fatal("unable to find all specified pack(s)") + list := make([]string, 0, len(packIDs)) + for h := range packIDs { + list = append(list, h) + } + + sort.Strings(list) + return errors.Fatalf("unable to find pack(s): %v", list) } debug.Log("%d blobs found", len(f.blobIDs)) From 95b44490a04d7628e76f5b2930ce2aa3067e81da Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 16 May 2021 00:10:28 +0200 Subject: [PATCH 4/6] find: search blob ids for pack in index if pack is missing If a pack file is missing try to determine the contained pack ids based on the repository index. This helps with assessing the damage to a repository before running `rebuild-index`. --- cmd/restic/cmd_find.go | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 7f03fc42b..dba2d2cda 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -447,6 +447,11 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { } if err != errorAllPacksFound { + // try to resolve unknown pack ids from the index + packIDs = f.indexPacksToBlobs(ctx, packIDs) + } + + if len(packIDs) > 0 { list := make([]string, 0, len(packIDs)) for h := range packIDs { list = append(list, h) @@ -460,6 +465,46 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { return nil } +func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} { + wctx, cancel := context.WithCancel(ctx) + defer cancel() + + // remember which packs were found in the index + indexPackIDs := make(map[string]struct{}) + for pb := range f.repo.Index().Each(wctx) { + idStr := pb.PackID.String() + // keep entry in packIDs as Each() returns individual index entries + matchingID := false + if _, ok := packIDs[idStr]; ok { + matchingID = true + } else { + if _, ok := packIDs[pb.PackID.Str()]; ok { + // expand id + delete(packIDs, pb.PackID.Str()) + packIDs[idStr] = struct{}{} + matchingID = true + } + } + if matchingID { + f.blobIDs[pb.ID.String()] = struct{}{} + indexPackIDs[idStr] = struct{}{} + } + } + + for id := range indexPackIDs { + delete(packIDs, id) + } + + if len(indexPackIDs) > 0 { + list := make([]string, 0, len(indexPackIDs)) + for h := range indexPackIDs { + list = append(list, h) + } + Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list) + } + return packIDs +} + func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) { idx := f.repo.Index() From a81f34ae47e0191f7025d2bc54b4c4818a6337d6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 12 Jun 2021 23:02:26 +0200 Subject: [PATCH 5/6] Add changelog --- changelog/unreleased/pull-3427 | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog/unreleased/pull-3427 diff --git a/changelog/unreleased/pull-3427 b/changelog/unreleased/pull-3427 new file mode 100644 index 000000000..c875c8941 --- /dev/null +++ b/changelog/unreleased/pull-3427 @@ -0,0 +1,13 @@ +Enhancement: `find --pack` fallback to index if data file is missing + +When investigating a repository with missing data files, it might be useful +to determine affected snapshots before running `rebuild-index`. +`restic find --pack pack-id` previously returned no data as it required +accessing the data file. Now, if the necessary data is still available in the +repository index, it gets retrieved from there. + +The command now also supports looking up multiple pack files in a single `find` +run. + +https://github.com/restic/restic/pull/3427 +https://forum.restic.net/t/missing-packs-not-found/2600 From 3442dc87fb31ec4af276e12710ea0a7e49097415 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 6 Jul 2021 21:03:35 +0200 Subject: [PATCH 6/6] find: Address review comments --- cmd/restic/cmd_find.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index dba2d2cda..b46b8644b 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "fmt" "sort" "strings" "time" @@ -401,7 +400,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { }) } -var errorAllPacksFound = fmt.Errorf("all packs found") +var errAllPacksFound = errors.New("all packs found") // packsToBlobs converts the list of pack IDs to a list of blob IDs that // belong to those packs. @@ -437,16 +436,16 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { } // Stop searching when all packs have been found if len(packIDs) == 0 { - return errorAllPacksFound + return errAllPacksFound } return nil }) - if err != nil && err != errorAllPacksFound { + if err != nil && err != errAllPacksFound { return err } - if err != errorAllPacksFound { + if err != errAllPacksFound { // try to resolve unknown pack ids from the index packIDs = f.indexPacksToBlobs(ctx, packIDs) }