forked from TrueCloudLab/restic
Merge pull request #4419 from arjunajesh/issue#299
Progress Bar when loading index files
This commit is contained in:
commit
76ef94d15c
33 changed files with 158 additions and 60 deletions
6
changelog/unreleased/pull-299
Normal file
6
changelog/unreleased/pull-299
Normal file
|
@ -0,0 +1,6 @@
|
|||
Enhancement: Show progress bar while loading the index
|
||||
|
||||
Restic did not provide any feedback while loading index files. Now there is a progress bar for the index loading process.
|
||||
|
||||
https://github.com/restic/restic/issues/229
|
||||
https://github.com/restic/restic/pull/4419
|
|
@ -546,7 +546,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||
if !gopts.JSON {
|
||||
progressPrinter.V("load index files")
|
||||
}
|
||||
err = repo.LoadIndex(ctx)
|
||||
|
||||
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -252,7 +252,7 @@ func TestBackupTreeLoadError(t *testing.T) {
|
|||
|
||||
r, err := OpenRepository(context.TODO(), env.gopts)
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, r.LoadIndex(context.TODO()))
|
||||
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
|
||||
treePacks := restic.NewIDSet()
|
||||
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
if pb.Type == restic.TreeBlob {
|
||||
|
|
|
@ -169,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|||
return err
|
||||
|
||||
case "blob":
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -197,7 +198,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|||
return errors.Fatalf("could not find snapshot: %v\n", err)
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -226,7 +226,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
}
|
||||
|
||||
Verbosef("load indexes\n")
|
||||
hints, errs := chkr.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
hints, errs := chkr.LoadIndex(ctx, bar)
|
||||
|
||||
errorsFound := false
|
||||
suggestIndexRebuild := false
|
||||
|
|
|
@ -99,12 +99,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
|||
}
|
||||
|
||||
debug.Log("Loading source index")
|
||||
if err := srcRepo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err := srcRepo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bar = newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
debug.Log("Loading destination index")
|
||||
if err := dstRepo.LoadIndex(ctx); err != nil {
|
||||
if err := dstRepo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
|
|||
}
|
||||
|
||||
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
|
||||
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
Printf("index_id: %v\n", id)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -469,7 +469,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
|
|||
}
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -363,8 +363,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
|
|||
if !gopts.JSON {
|
||||
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
|
||||
}
|
||||
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -152,7 +152,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
|
|||
return errors.Fatalf("failed to find snapshot: %v", err)
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -589,8 +589,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
|
|||
case "locks":
|
||||
t = restic.LockFile
|
||||
case "blobs":
|
||||
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -173,7 +173,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
|
|||
return err
|
||||
}
|
||||
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -130,7 +130,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
|||
}
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -187,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
|
||||
Verbosef("loading indexes...\n")
|
||||
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
||||
err := repo.LoadIndex(ctx)
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
err := repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -58,7 +58,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
|
|||
}
|
||||
|
||||
Verbosef("load index files\n")
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -73,7 +74,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
|
|||
})
|
||||
|
||||
Verbosef("load %d trees\n", len(trees))
|
||||
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
|
||||
bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
|
||||
for id := range trees {
|
||||
tree, err := restic.LoadTree(ctx, repo, id)
|
||||
if err != nil {
|
||||
|
|
|
@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
|
|||
} else {
|
||||
Verbosef("loading indexes...\n")
|
||||
mi := index.NewMasterIndex()
|
||||
err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
if err != nil {
|
||||
Warnf("removing invalid index %v: %v\n", id, err)
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
|
|
|
@ -89,7 +89,8 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
|
|||
return err
|
||||
}
|
||||
|
||||
if err := repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err := repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -173,7 +173,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
|||
return errors.Fatalf("failed to find snapshot: %v", err)
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(ctx)
|
||||
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -212,7 +212,8 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
|
|||
return err
|
||||
}
|
||||
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -98,8 +98,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = repo.LoadIndex(ctx); err != nil {
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
@ -258,7 +258,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
|
|||
rtest.OK(t, err)
|
||||
|
||||
// Get all tree packs
|
||||
rtest.OK(t, r.LoadIndex(context.TODO()))
|
||||
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
|
||||
|
||||
treePacks := restic.NewIDSet()
|
||||
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
|
|
|
@ -29,13 +29,12 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
|
|||
return interval
|
||||
}
|
||||
|
||||
// newProgressMax returns a progress.Counter that prints to stdout.
|
||||
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
|
||||
// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided.
|
||||
func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter {
|
||||
if !show {
|
||||
return nil
|
||||
}
|
||||
interval := calculateProgressInterval(show, false)
|
||||
canUpdateStatus := stdoutCanUpdateStatus()
|
||||
|
||||
return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
|
||||
var status string
|
||||
|
@ -47,14 +46,28 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
|
|||
ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
|
||||
}
|
||||
|
||||
printProgress(status, canUpdateStatus)
|
||||
print(status)
|
||||
if final {
|
||||
fmt.Print("\n")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func printProgress(status string, canUpdateStatus bool) {
|
||||
func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter {
|
||||
return newGenericProgressMax(show, max, description, func(status string) {
|
||||
term.SetStatus([]string{status})
|
||||
})
|
||||
}
|
||||
|
||||
// newProgressMax calls newTerminalProgress without a terminal (print to stdout)
|
||||
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
|
||||
return newGenericProgressMax(show, max, description, printProgress)
|
||||
}
|
||||
|
||||
func printProgress(status string) {
|
||||
|
||||
canUpdateStatus := stdoutCanUpdateStatus()
|
||||
|
||||
w := stdoutTerminalWidth()
|
||||
if w > 0 {
|
||||
if w < 3 {
|
||||
|
@ -83,3 +96,11 @@ func printProgress(status string, canUpdateStatus bool) {
|
|||
|
||||
_, _ = os.Stdout.Write([]byte(clear + status + carriageControl))
|
||||
}
|
||||
|
||||
func newIndexProgress(quiet bool, json bool) *progress.Counter {
|
||||
return newProgressMax(!quiet && !json, 0, "index files loaded")
|
||||
}
|
||||
|
||||
func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter {
|
||||
return newTerminalProgressMax(!quiet && !json, 0, "index files loaded", term)
|
||||
}
|
||||
|
|
|
@ -113,13 +113,42 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
|
|||
}
|
||||
|
||||
// LoadIndex loads all index files.
|
||||
func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
|
||||
func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) {
|
||||
debug.Log("Start")
|
||||
|
||||
indexList, err := backend.MemorizeList(ctx, c.repo.Backend(), restic.IndexFile)
|
||||
if err != nil {
|
||||
// abort if an error occurs while listing the indexes
|
||||
return hints, append(errs, err)
|
||||
}
|
||||
|
||||
if p != nil {
|
||||
var numIndexFiles uint64
|
||||
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
|
||||
_, err := restic.ParseID(fi.Name)
|
||||
if err != nil {
|
||||
debug.Log("unable to parse %v as an ID", fi.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
numIndexFiles++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return hints, append(errs, err)
|
||||
}
|
||||
p.SetMax(numIndexFiles)
|
||||
defer p.Done()
|
||||
}
|
||||
|
||||
packToIndex := make(map[restic.ID]restic.IDSet)
|
||||
err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
err = index.ForAllIndexes(ctx, indexList, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
debug.Log("process index %v, err %v", id, err)
|
||||
|
||||
if p != nil {
|
||||
p.Add(1)
|
||||
}
|
||||
|
||||
if oldFormat {
|
||||
debug.Log("index %v has old format", id)
|
||||
hints = append(hints, &ErrOldIndexFormat{id})
|
||||
|
|
|
@ -77,7 +77,7 @@ func TestCheckRepo(t *testing.T) {
|
|||
repo := repository.TestOpenLocal(t, repodir)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -103,7 +103,7 @@ func TestMissingPack(t *testing.T) {
|
|||
test.OK(t, repo.Backend().Remove(context.TODO(), packHandle))
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -136,7 +136,7 @@ func TestUnreferencedPack(t *testing.T) {
|
|||
test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle))
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -178,7 +178,7 @@ func TestUnreferencedBlobs(t *testing.T) {
|
|||
sort.Sort(unusedBlobsBySnapshot)
|
||||
|
||||
chkr := checker.New(repo, true)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -258,7 +258,7 @@ func TestModifiedIndex(t *testing.T) {
|
|||
}
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) == 0 {
|
||||
t.Fatalf("expected errors not found")
|
||||
}
|
||||
|
@ -279,7 +279,7 @@ func TestDuplicatePacksInIndex(t *testing.T) {
|
|||
repo := repository.TestOpenLocal(t, repodir)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(hints) == 0 {
|
||||
t.Fatalf("did not get expected checker hints for duplicate packs in indexes")
|
||||
}
|
||||
|
@ -347,7 +347,7 @@ func TestCheckerModifiedData(t *testing.T) {
|
|||
|
||||
chkr := checker.New(checkRepo, false)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -408,7 +408,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
|
|||
}
|
||||
|
||||
chkr := checker.New(checkRepo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -524,7 +524,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
|
|||
delayRepo.Unblock()
|
||||
}()
|
||||
|
||||
hints, errs := chkr.LoadIndex(ctx)
|
||||
hints, errs := chkr.LoadIndex(ctx, nil)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
@ -553,7 +553,7 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
|
|||
repo := repository.TestOpenLocal(t, repodir)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
defer cleanup()
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
func TestCheckRepo(t testing.TB, repo restic.Repository) {
|
||||
chkr := New(repo, true)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO())
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("errors loading index: %v", errs)
|
||||
}
|
||||
|
|
|
@ -328,7 +328,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
err = d.root.repo.LoadIndex(ctx)
|
||||
err = d.root.repo.LoadIndex(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
// ForAllIndexes loads all index files in parallel and calls the given callback.
|
||||
// It is guaranteed that the function is not run concurrently. If the callback
|
||||
// returns an error, this function is cancelled and also returns that error.
|
||||
func ForAllIndexes(ctx context.Context, repo restic.Repository,
|
||||
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
|
||||
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
|
||||
|
||||
// decoding an index can take quite some time such that this can be both CPU- or IO-bound
|
||||
|
@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository,
|
|||
workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0))
|
||||
|
||||
var m sync.Mutex
|
||||
return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
|
||||
return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
|
||||
var err error
|
||||
var idx *Index
|
||||
oldFormat := false
|
||||
|
|
|
@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
|
|||
// check that all expected indexes are loaded without errors
|
||||
indexIDs := restic.NewIDSet()
|
||||
var indexErr error
|
||||
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
if err != nil {
|
||||
indexErr = err
|
||||
}
|
||||
|
@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
|
|||
// must failed with the returned error
|
||||
iterErr := errors.New("error to pass upwards")
|
||||
|
||||
err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||
return iterErr
|
||||
})
|
||||
|
||||
|
|
|
@ -357,7 +357,7 @@ func TestIndexSave(t *testing.T) {
|
|||
func testIndexSave(t *testing.T, version uint) {
|
||||
repo := createFilledRepo(t, 3, version)
|
||||
|
||||
err := repo.LoadIndex(context.TODO())
|
||||
err := repo.LoadIndex(context.TODO(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -382,7 +382,7 @@ func testIndexSave(t *testing.T, version uint) {
|
|||
t.Error(err)
|
||||
}
|
||||
|
||||
hints, errs := checker.LoadIndex(context.TODO())
|
||||
hints, errs := checker.LoadIndex(context.TODO(), nil)
|
||||
for _, h := range hints {
|
||||
t.Logf("hint: %v\n", h)
|
||||
}
|
||||
|
|
|
@ -213,7 +213,7 @@ func reloadIndex(t *testing.T, repo restic.Repository) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := repo.LoadIndex(context.TODO()); err != nil {
|
||||
if err := repo.LoadIndex(context.TODO(), nil); err != nil {
|
||||
t.Fatalf("error loading new index: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -581,15 +581,41 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
|||
}
|
||||
|
||||
// LoadIndex loads all index files from the backend in parallel and stores them
|
||||
// in the master index. The first error that occurred is returned.
|
||||
func (r *Repository) LoadIndex(ctx context.Context) error {
|
||||
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
||||
debug.Log("Loading index")
|
||||
|
||||
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if p != nil {
|
||||
var numIndexFiles uint64
|
||||
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
|
||||
_, err := restic.ParseID(fi.Name)
|
||||
if err != nil {
|
||||
debug.Log("unable to parse %v as an ID", fi.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
numIndexFiles++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.SetMax(numIndexFiles)
|
||||
defer p.Done()
|
||||
}
|
||||
|
||||
err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.idx.Insert(idx)
|
||||
if p != nil {
|
||||
p.Add(1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
|
|
|
@ -255,7 +255,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
|
|||
defer cleanup()
|
||||
|
||||
repo := repository.TestOpenLocal(t, repodir)
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO()))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
}
|
||||
|
||||
// loadIndex loads the index id from backend and returns it.
|
||||
|
@ -324,7 +324,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
|
|||
err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "")
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO()))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
}
|
||||
|
||||
func BenchmarkLoadIndex(b *testing.B) {
|
||||
|
|
|
@ -24,7 +24,7 @@ type Repository interface {
|
|||
Key() *crypto.Key
|
||||
|
||||
Index() MasterIndex
|
||||
LoadIndex(context.Context) error
|
||||
LoadIndex(context.Context, *progress.Counter) error
|
||||
SetIndex(MasterIndex) error
|
||||
LookupBlobSize(ID, BlobType) (uint, bool)
|
||||
|
||||
|
|
Loading…
Reference in a new issue