KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

prune/forget/repair index: convert output to use progress.Printer

Browse source
This commit is contained in:
Michael Eischer 2024-04-06 00:51:20 +02:00
parent 739d11c2eb
commit 32a234b67e
9 changed files with 176 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
cmd/restic/cmd_forget.go
View file

@ -8,6 +8,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
)
@ -33,7 +34,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args)
term, cancel := setupTermstatus()
defer cancel()
return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
},
}
@ -152,7 +155,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
return nil
}
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error {
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := verifyForgetOptions(&opts)
if err != nil {
return err
@ -173,6 +176,12 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
defer unlock()
verbosity := gopts.verbosity
if gopts.JSON {
verbosity = 0
}
printer := newTerminalProgressPrinter(verbosity, term)
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
@ -210,15 +219,11 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
if policy.Empty() && len(args) == 0 {
if !gopts.JSON {
Verbosef("no policy was specified, no snapshots will be removed\n")
}
printer.P("no policy was specified, no snapshots will be removed\n")
}
if !policy.Empty() {
if !gopts.JSON {
Verbosef("Applying Policy: %v\n", policy)
}
printer.P("Applying Policy: %v\n", policy)
for k, snapshotGroup := range snapshotGroups {
if gopts.Verbose >= 1 && !gopts.JSON {
@ -241,16 +246,16 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("keep %d snapshots:\n", len(keep))
printer.P("keep %d snapshots:\n", len(keep))
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
Printf("\n")
printer.P("\n")
}
fg.Keep = asJSONSnapshots(keep)
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("remove %d snapshots:\n", len(remove))
printer.P("remove %d snapshots:\n", len(remove))
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
Printf("\n")
printer.P("\n")
}
fg.Remove = asJSONSnapshots(remove)
@ -267,14 +272,12 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
if len(removeSnIDs) > 0 {
if !opts.DryRun {
bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted")
bar := printer.NewCounter("files deleted")
err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
if err != nil {
Warnf("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
}
if !gopts.JSON && gopts.verbosity > 2 {
Verbosef("removed %v/%v\n", restic.SnapshotFile, id)
printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
}
printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
return nil
}, bar)
bar.Done()
@ -282,9 +285,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
return err
}
} else {
if !gopts.JSON {
Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
}
printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
}
}
@ -296,15 +297,13 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
if len(removeSnIDs) > 0 && opts.Prune {
if !gopts.JSON {
if opts.DryRun {
Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
} else {
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
if opts.DryRun {
printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
} else {
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
}
return nil

5
cmd/restic/cmd_forget_integration_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
@ -12,5 +13,7 @@ func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
pruneOpts := PruneOptions{
MaxUnused: "5%",
}
rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args))
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
}))
}

162
cmd/restic/cmd_prune.go
View file

@ -16,6 +16,7 @@ import (
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
)
@ -38,7 +39,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return runPrune(cmd.Context(), pruneOptions, globalOptions)
term, cancel := setupTermstatus()
defer cancel()
return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
},
}
@ -138,7 +141,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
return nil
}
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
err := verifyPruneOptions(&opts)
if err != nil {
return err
@ -170,10 +173,10 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
opts.unsafeRecovery = true
}
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
}
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
// we do not need index updates while pruning!
repo.DisableAutoIndexUpdate()
@ -181,24 +184,26 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
Print("warning: running prune without a cache, this may be very slow!\n")
}
Verbosef("loading indexes...\n")
printer := newTerminalProgressPrinter(gopts.verbosity, term)
printer.P("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err := repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, printer)
if err != nil {
return err
}
if opts.DryRun {
Verbosef("\nWould have made the following changes:")
printer.P("\nWould have made the following changes:")
}
err = printPruneStats(stats)
err = printPruneStats(printer, stats)
if err != nil {
return err
}
@ -206,7 +211,7 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
// Trigger GC to reset garbage collection threshold
runtime.GC()
return doPrune(ctx, opts, gopts, repo, plan)
return doPrune(ctx, opts, repo, plan, printer)
}
type pruneStats struct {
@ -264,22 +269,22 @@ type packInfoWithID struct {
// planPrune selects which files to rewrite and which to delete and which blobs to keep.
// Also some summary statistics are returned.
func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) (prunePlan, pruneStats, error) {
var stats pruneStats
usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, printer)
if err != nil {
return prunePlan{}, stats, err
}
Verbosef("searching used packs...\n")
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
printer.P("searching used packs...\n")
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats, printer)
if err != nil {
return prunePlan{}, stats, err
}
Verbosef("collecting packs for deletion and repacking\n")
plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
printer.P("collecting packs for deletion and repacking\n")
plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, printer)
if err != nil {
return prunePlan{}, stats, err
}
@ -308,7 +313,7 @@ func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, i
return plan, stats, nil
}
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
// iterate over all blobs in index to find out which blobs are duplicates
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
@ -337,7 +342,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
}
if len(missingBlobs) != 0 {
Warnf("%v not found in the index\n\n"+
printer.E("%v not found in the index\n\n"+
"Integrity check failed: Data seems to be missing.\n"+
"Will not start prune to prevent (additional) data loss!\n"+
"Please report this error (along with the output of the 'prune' run) at\n"+
@ -458,7 +463,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
return usedBlobs, indexPack, nil
}
func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, printer progress.Printer) (prunePlan, error) {
removePacksFirst := restic.NewIDSet()
removePacks := restic.NewIDSet()
repackPacks := restic.NewIDSet()
@ -474,12 +479,13 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
}
// loop over all packs and decide what to do
bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
bar := printer.NewCounter("packs processed")
bar.SetMax(uint64(len(indexPack)))
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
p, ok := indexPack[id]
if !ok {
// Pack was not referenced in index and is not used => immediately remove!
Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
printer.V("will remove pack %v as it is unused and not indexed\n", id.Str())
removePacksFirst.Insert(id)
stats.size.unref += uint64(packSize)
return nil
@ -489,7 +495,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
// Pack size does not fit and pack is needed => error
// If the pack is not needed, this is no error, the pack can
// and will be simply removed, see below.
Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
printer.E("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
id.Str(), p.unusedSize+p.usedSize, packSize)
return errorSizeNotMatching
}
@ -562,16 +568,16 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
}
if len(indexPack) != 0 {
Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
printer.E("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
for id := range indexPack {
Warnf(" %v\n", id)
printer.E(" %v\n", id)
}
return prunePlan{}, errorPacksMissing
}
if len(ignorePacks) != 0 {
Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
printer.E("Missing but unneeded pack files are referenced in the index, will be repaired\n")
for id := range ignorePacks {
Warnf("will forget missing pack file %v\n", id)
printer.E("will forget missing pack file %v\n", id)
}
}
@ -657,43 +663,43 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
}
// printPruneStats prints out the statistics
func printPruneStats(stats pruneStats) error {
Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
func printPruneStats(printer progress.Printer, stats pruneStats) error {
printer.V("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
if stats.blobs.duplicate > 0 {
Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
printer.V("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
}
Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
printer.V("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
if stats.size.unref > 0 {
Verboseff("unreferenced: %s\n", ui.FormatBytes(stats.size.unref))
printer.V("unreferenced: %s\n", ui.FormatBytes(stats.size.unref))
}
totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
unusedSize := stats.size.duplicate + stats.size.unused
Verboseff("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
printer.V("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
printer.P("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
printer.P("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
printer.P("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
printer.P("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
if stats.size.uncompressed > 0 {
Verbosef("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed))
printer.P("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed))
}
Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
printer.P("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
Verbosef("unused size after prune: %s (%s of remaining size)\n",
printer.P("unused size after prune: %s (%s of remaining size)\n",
ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
Verbosef("\n")
Verboseff("totally used packs: %10d\n", stats.packs.used)
Verboseff("partly used packs: %10d\n", stats.packs.partlyUsed)
Verboseff("unused packs: %10d\n\n", stats.packs.unused)
printer.P("\n")
printer.V("totally used packs: %10d\n", stats.packs.used)
printer.V("partly used packs: %10d\n", stats.packs.partlyUsed)
printer.V("unused packs: %10d\n\n", stats.packs.unused)
Verboseff("to keep: %10d packs\n", stats.packs.keep)
Verboseff("to repack: %10d packs\n", stats.packs.repack)
Verboseff("to delete: %10d packs\n", stats.packs.remove)
printer.V("to keep: %10d packs\n", stats.packs.keep)
printer.V("to repack: %10d packs\n", stats.packs.repack)
printer.V("to delete: %10d packs\n", stats.packs.remove)
if stats.packs.unref > 0 {
Verboseff("to delete: %10d unreferenced packs\n\n", stats.packs.unref)
printer.V("to delete: %10d unreferenced packs\n\n", stats.packs.unref)
}
return nil
}
@ -704,29 +710,28 @@ func printPruneStats(stats pruneStats) error {
// - rebuild the index while ignoring all files that will be deleted
// - delete the files
// plan.removePacks and plan.ignorePacks are modified in this function.
func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
func doPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, plan prunePlan, printer progress.Printer) (err error) {
if opts.DryRun {
if !gopts.JSON && gopts.verbosity >= 2 {
Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
if len(plan.removePacksFirst) > 0 {
Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
}
Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
printer.V("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
if len(plan.removePacksFirst) > 0 {
printer.V("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
}
printer.V("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
printer.V("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
// Always quit here if DryRun was set!
return nil
}
// unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 {
Verbosef("deleting unreferenced packs\n")
DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
printer.P("deleting unreferenced packs\n")
DeleteFiles(ctx, repo, plan.removePacksFirst, restic.PackFile, printer)
}
if len(plan.repackPacks) != 0 {
Verbosef("repacking packs\n")
bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
printer.P("repacking packs\n")
bar := printer.NewCounter("packs repacked")
bar.SetMax(uint64(len(plan.repackPacks)))
_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
bar.Done()
if err != nil {
@ -737,7 +742,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
plan.removePacks.Merge(plan.repackPacks)
if len(plan.keepBlobs) != 0 {
Warnf("%v was not repacked\n\n"+
printer.E("%v was not repacked\n\n"+
"Integrity check failed.\n"+
"Please report this error (along with the output of the 'prune' run) at\n"+
"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
@ -755,56 +760,54 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
}
if opts.unsafeRecovery {
Verbosef("deleting index files\n")
printer.P("deleting index files\n")
indexFiles := repo.Index().(*index.MasterIndex).IDs()
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
err = DeleteFilesChecked(ctx, repo, indexFiles, restic.IndexFile, printer)
if err != nil {
return errors.Fatalf("%s", err)
}
} else if len(plan.ignorePacks) != 0 {
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false)
err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, false, printer)
if err != nil {
return errors.Fatalf("%s", err)
}
}
if len(plan.removePacks) != 0 {
Verbosef("removing %d old packs\n", len(plan.removePacks))
DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
printer.P("removing %d old packs\n", len(plan.removePacks))
DeleteFiles(ctx, repo, plan.removePacks, restic.PackFile, printer)
}
if opts.unsafeRecovery {
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true)
err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, true, printer)
if err != nil {
return errors.Fatalf("%s", err)
}
}
Verbosef("done\n")
printer.P("done\n")
return nil
}
func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error {
Verbosef("rebuilding index\n")
func rebuildIndexFiles(ctx context.Context, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool, printer progress.Printer) error {
printer.P("rebuilding index\n")
bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
bar := printer.NewCounter("packs processed")
return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
SaveProgress: bar,
DeleteProgress: func() *progress.Counter {
return newProgressMax(!gopts.Quiet, 0, "old indexes deleted")
return printer.NewCounter("old indexes deleted")
},
DeleteReport: func(id restic.ID, _ error) {
if gopts.verbosity > 2 {
Verbosef("removed index %v\n", id.String())
}
printer.VV("removed index %v\n", id.String())
},
SkipDeletion: skipDeletion,
})
}
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) (usedBlobs restic.CountedBlobSet, err error) {
var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n")
printer.P("loading all snapshots...\n")
err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
@ -819,11 +822,12 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r
return nil, errors.Fatalf("failed loading snapshot: %v", err)
}
Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
usedBlobs = restic.NewCountedBlobSet()
bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
bar := printer.NewCounter("snapshots")
bar.SetMax(uint64(len(snapshotTrees)))
defer bar.Done()
err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)

17
cmd/restic/cmd_prune_integration_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
@ -16,7 +17,9 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
defer func() {
gopts.backendTestHook = oldHook
}()
rtest.OK(t, runPrune(context.TODO(), opts, gopts))
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), opts, gopts, term)
}))
}
func TestPrune(t *testing.T) {
@ -84,7 +87,9 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
pruneOpts := PruneOptions{
MaxUnused: "5%",
}
return runForget(context.TODO(), opts, pruneOpts, gopts, args)
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
})
})
rtest.OK(t, err)
@ -138,7 +143,9 @@ func TestPruneWithDamagedRepository(t *testing.T) {
env.gopts.backendTestHook = oldHook
}()
// prune should fail
rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
}) == errorPacksMissing,
"prune should have reported index not complete error")
}
@ -218,7 +225,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
testRunPrune(t, env.gopts, optionsPrune)
testRunCheck(t, env.gopts)
} else {
rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), optionsPrune, env.gopts, term)
}) != nil,
"prune should have reported an error")
}
}

37
cmd/restic/cmd_repair_index.go
View file

@ -7,6 +7,8 @@ import (
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@ -25,7 +27,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
term, cancel := setupTermstatus()
defer cancel()
return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term)
},
}
@ -55,17 +59,19 @@ func init() {
}
}
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
return rebuildIndex(ctx, opts, gopts, repo)
printer := newTerminalProgressPrinter(gopts.verbosity, term)
return rebuildIndex(ctx, opts, repo, printer)
}
func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error {
func rebuildIndex(ctx context.Context, opts RepairIndexOptions, repo *repository.Repository, printer progress.Printer) error {
var obsoleteIndexes restic.IDs
packSizeFromList := make(map[restic.ID]int64)
packSizeFromIndex := make(map[restic.ID]int64)
@ -81,11 +87,11 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
return err
}
} else {
Verbosef("loading indexes...\n")
printer.P("loading indexes...\n")
mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
if err != nil {
Warnf("removing invalid index %v: %v\n", id, err)
printer.E("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id)
return nil
}
@ -109,7 +115,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
}
Verbosef("getting pack files to read...\n")
printer.P("getting pack files to read...\n")
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
size, ok := packSizeFromIndex[id]
if !ok || size != packSize {
@ -118,9 +124,9 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
removePacks.Insert(id)
}
if !ok {
Warnf("adding pack file to index %v\n", id)
printer.E("adding pack file to index %v\n", id)
} else if size != packSize {
Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
printer.E("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
}
delete(packSizeFromIndex, id)
return nil
@ -132,12 +138,13 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
// forget pack files that are referenced in the index but do not exist
// when rebuilding the index
removePacks.Insert(id)
Warnf("removing not found pack file %v\n", id)
printer.E("removing not found pack file %v\n", id)
}
if len(packSizeFromList) > 0 {
Verbosef("reading pack files\n")
bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs")
printer.P("reading pack files\n")
bar := printer.NewCounter("packs")
bar.SetMax(uint64(len(packSizeFromList)))
invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
bar.Done()
if err != nil {
@ -145,15 +152,15 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
}
for _, id := range invalidFiles {
Verboseff("skipped incomplete pack file: %v\n", id)
printer.V("skipped incomplete pack file: %v\n", id)
}
}
err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false)
err = rebuildIndexFiles(ctx, repo, removePacks, obsoleteIndexes, false, printer)
if err != nil {
return err
}
Verbosef("done\n")
printer.P("done\n")
return nil
}

14
cmd/restic/cmd_repair_index_integration_test.go
View file

@ -13,12 +13,15 @@ import (
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, withRestoreGlobalOptions(func() error {
globalOptions.stdout = io.Discard
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
globalOptions.stdout = io.Discard
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term)
})
}))
}
@ -126,12 +129,13 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
rtest.SetupTarTestFixture(t, env.base, datafile)
err := withRestoreGlobalOptions(func() error {
globalOptions.stdout = io.Discard
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil
}
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
globalOptions.stdout = io.Discard
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
})
})
if err == nil {

6
cmd/restic/cmd_repair_packs.go
View file

@ -58,14 +58,14 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.T
}
defer unlock()
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
printer := newTerminalProgressPrinter(gopts.verbosity, term)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return errors.Fatalf("%s", err)
}
printer := newTerminalProgressPrinter(gopts.verbosity, term)
printer.P("saving backup copies of pack files to current folder")
for id := range ids {
f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)

21
cmd/restic/delete.go
View file

@ -4,38 +4,35 @@ import (
"context"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
)
// DeleteFiles deletes the given fileList of fileType in parallel
// it will print a warning if there is an error, but continue deleting the remaining files
func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
_ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
func DeleteFiles(ctx context.Context, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) {
_ = deleteFiles(ctx, true, repo, fileList, fileType, printer)
}
// DeleteFilesChecked deletes the given fileList of fileType in parallel
// if an error occurs, it will cancel and return this error
func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
func DeleteFilesChecked(ctx context.Context, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) error {
return deleteFiles(ctx, false, repo, fileList, fileType, printer)
}
// deleteFiles deletes the given fileList of fileType in parallel
// if ignoreError=true, it will print a warning if there was an error, else it will abort.
func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted")
func deleteFiles(ctx context.Context, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) error {
bar := printer.NewCounter("files deleted")
defer bar.Done()
return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error {
if err != nil {
if !gopts.JSON {
Warnf("unable to remove %v/%v from the repository\n", fileType, id)
}
printer.E("unable to remove %v/%v from the repository\n", fileType, id)
if !ignoreError {
return err
}
}
if !gopts.JSON && gopts.verbosity > 2 {
Verbosef("removed %v/%v\n", fileType, id)
}
printer.VV("removed %v/%v\n", fileType, id)
return nil
}, bar)
}

9
cmd/restic/integration_test.go
View file

@ -12,6 +12,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func TestCheckRestoreNoLock(t *testing.T) {
@ -88,8 +89,12 @@ func TestListOnce(t *testing.T) {
testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts))
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
}))
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts, term)
}))
}
type writeToOnly struct {