KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #4314 from MichaelEischer/restore-json

Browse source 
Add JSON support for the restore progress bar
This commit is contained in:
Michael Eischer 2023-06-16 23:53:17 +02:00 committed by GitHub
commit 4df77e9f26
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 268 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
changelog/unreleased/issue-3627 → changelog/unreleased/issue-426
View file

@ -4,6 +4,11 @@ The `restore` command now shows a progress report while restoring files.
Example: [0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB
JSON output is now also supported.
https://github.com/restic/restic/issues/426
https://github.com/restic/restic/issues/3413
https://github.com/restic/restic/issues/3627
https://github.com/restic/restic/pull/3991
https://github.com/restic/restic/pull/4314
https://forum.restic.net/t/progress-bar-for-restore/5210

21
cmd/restic/cmd_backup_integration_test.go
View file

@ -13,17 +13,10 @@ import (
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
"golang.org/x/sync/errgroup"
)
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
var wg errgroup.Group
term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet)
wg.Go(func() error { term.Run(ctx); return nil })
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
t.Logf("backing up %v in %v", target, dir)
if dir != "" {
cleanup := rtest.Chdir(t, dir)
@ -31,16 +24,8 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
}
opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
backupErr := runBackup(ctx, opts, gopts, term, target)
cancel()
err := wg.Wait()
if err != nil {
t.Fatal(err)
}
return backupErr
return runBackup(ctx, opts, gopts, term, target)
})
}
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {

35
cmd/restic/cmd_restore.go
View file

@ -175,16 +175,20 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
return err
}
var progress *restoreui.Progress
if !gopts.Quiet && !gopts.JSON {
progress = restoreui.NewProgress(restoreui.NewProgressPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON))
msg := ui.NewMessage(term, gopts.verbosity)
var printer restoreui.ProgressPrinter
if gopts.JSON {
printer = restoreui.NewJSONProgress(term)
} else {
printer = restoreui.NewTextProgress(term)
}
progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON))
res := restorer.NewRestorer(repo, sn, opts.Sparse, progress)
totalErrors := 0
res.Error = func(location string, err error) error {
Warnf("ignoring error for %s: %s\n", location, err)
msg.E("ignoring error for %s: %s\n", location, err)
totalErrors++
return nil
}
@ -194,12 +198,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, err := filter.List(excludePatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
msg.E("error for exclude pattern: %v", err)
}
matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
msg.E("error for iexclude pattern: %v", err)
}
// An exclude filter is basically a 'wildcard but foo',
@ -217,12 +221,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
if err != nil {
Warnf("error for include pattern: %v", err)
msg.E("error for include pattern: %v", err)
}
matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
msg.E("error for iexclude pattern: %v", err)
}
selectedForRestore = matched || matchedInsensitive
@ -237,23 +241,25 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
res.SelectFilter = selectIncludeFilter
}
Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
if !gopts.JSON {
msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target)
}
err = res.RestoreTo(ctx, opts.Target)
if err != nil {
return err
}
if progress != nil {
progress.Finish()
}
if totalErrors > 0 {
return errors.Fatalf("There were %d errors\n", totalErrors)
}
if opts.Verify {
Verbosef("verifying files in %s\n", opts.Target)
if !gopts.JSON {
msg.P("verifying files in %s\n", opts.Target)
}
var count int
t0 := time.Now()
count, err = res.VerifyFiles(ctx, opts.Target)
@ -263,9 +269,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
if totalErrors > 0 {
return errors.Fatalf("There were %d errors\n", totalErrors)
}
Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target,
if !gopts.JSON {
msg.P("finished verifying %d files in %s (took %s)\n", count, opts.Target,
time.Since(t0).Round(time.Millisecond))
}
}
return nil
}

11
cmd/restic/cmd_restore_integration_test.go
View file

@ -14,6 +14,7 @@ import (
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) {
@ -26,11 +27,13 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Exclude: excludes,
}
rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()}))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func testRunRestoreAssumeFailure(snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
return runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID})
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runRestore(ctx, opts, gopts, term, []string{snapshotID})
})
}
func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
@ -42,7 +45,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [
},
}
rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{"latest"}))
rtest.OK(t, testRunRestoreAssumeFailure("latest", opts, gopts))
}
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
@ -51,7 +54,7 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Include: includes,
}
rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()}))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func TestRestoreFilter(t *testing.T) {

19
cmd/restic/integration_helpers_test.go
View file

@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"runtime"
"sync"
"testing"
"github.com/restic/restic/internal/backend/retry"
@ -17,6 +18,7 @@ import (
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
type dirEntry struct {
@ -356,3 +358,20 @@ func withCaptureStdout(inner func() error) (*bytes.Buffer, error) {
return buf, err
}
func withTermStatus(gopts GlobalOptions, callback func(ctx context.Context, term *termstatus.Terminal) error) error {
ctx, cancel := context.WithCancel(context.TODO())
var wg sync.WaitGroup
term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet)
wg.Add(1)
go func() {
defer wg.Done()
term.Run(ctx)
}()
defer wg.Wait()
defer cancel()
return callback(ctx, term)
}

15
internal/ui/backup/json.go
View file

@ -1,8 +1,6 @@
package backup
import (
"bytes"
"encoding/json"
"sort"
"time"
@ -32,21 +30,12 @@ func NewJSONProgress(term *termstatus.Terminal, verbosity uint) *JSONProgress {
}
}
func toJSONString(status interface{}) string {
buf := new(bytes.Buffer)
err := json.NewEncoder(buf).Encode(status)
if err != nil {
panic(err)
}
return buf.String()
}
func (b *JSONProgress) print(status interface{}) {
b.term.Print(toJSONString(status))
b.term.Print(ui.ToJSONString(status))
}
func (b *JSONProgress) error(status interface{}) {
b.term.Error(toJSONString(status))
b.term.Error(ui.ToJSONString(status))
}
// Update updates the status lines.

11
internal/ui/format.go
View file

@ -1,6 +1,8 @@
package ui
import (
"bytes"
"encoding/json"
"fmt"
"time"
)
@ -53,3 +55,12 @@ func FormatSeconds(sec uint64) string {
}
return fmt.Sprintf("%d:%02d", min, sec)
}
func ToJSONString(status interface{}) string {
buf := new(bytes.Buffer)
err := json.NewEncoder(buf).Encode(status)
if err != nil {
panic(err)
}
return buf.String()
}

69
internal/ui/restore/json.go Normal file
View file

@ -0,0 +1,69 @@
package restore
import (
"time"
"github.com/restic/restic/internal/ui"
)
type jsonPrinter struct {
terminal term
}
func NewJSONProgress(terminal term) ProgressPrinter {
return &jsonPrinter{
terminal: terminal,
}
}
func (t *jsonPrinter) print(status interface{}) {
t.terminal.Print(ui.ToJSONString(status))
}
func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
status := statusUpdate{
MessageType: "status",
SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal,
FilesRestored: filesFinished,
TotalBytes: allBytesTotal,
BytesRestored: allBytesWritten,
}
if allBytesTotal > 0 {
status.PercentDone = float64(allBytesWritten) / float64(allBytesTotal)
}
t.print(status)
}
func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
status := summaryOutput{
MessageType: "summary",
SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal,
FilesRestored: filesFinished,
TotalBytes: allBytesTotal,
BytesRestored: allBytesWritten,
}
t.print(status)
}
type statusUpdate struct {
MessageType string `json:"message_type"` // "status"
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
PercentDone float64 `json:"percent_done"`
TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
}
type summaryOutput struct {
MessageType string `json:"message_type"` // "summary"
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
}

29
internal/ui/restore/json_test.go Normal file
View file

@ -0,0 +1,29 @@
package restore
import (
"testing"
"time"
"github.com/restic/restic/internal/test"
)
func TestJSONPrintUpdate(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
}
func TestJSONPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output)
}
func TestJSONPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
}

51
internal/ui/restore/progressformatter.go → internal/ui/restore/progress.go
View file

@ -1,11 +1,9 @@
package restore
import (
"fmt"
"sync"
"time"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
@ -28,6 +26,11 @@ type progressInfoEntry struct {
bytesTotal uint64
}
type term interface {
Print(line string)
SetStatus(lines []string)
}
type ProgressPrinter interface {
Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration)
Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration)
@ -85,47 +88,3 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot
func (p *Progress) Finish() {
p.updater.Done()
}
type term interface {
Print(line string)
SetStatus(lines []string)
}
type textPrinter struct {
terminal term
}
func NewProgressPrinter(terminal term) ProgressPrinter {
return &textPrinter{
terminal: terminal,
}
}
func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
timeLeft := ui.FormatDuration(duration)
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal)
progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v",
timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal)
t.terminal.SetStatus([]string{progress})
}
func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
t.terminal.SetStatus([]string{})
timeLeft := ui.FormatDuration(duration)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
var summary string
if filesFinished == filesTotal && allBytesWritten == allBytesTotal {
summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft)
} else {
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s",
filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft)
}
t.terminal.Print(summary)
}

33
internal/ui/restore/progressformatter_test.go → internal/ui/restore/progress_test.go
View file

@ -135,36 +135,3 @@ func TestSummaryOnErrors(t *testing.T) {
printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true},
}, result)
}
type mockTerm struct {
output []string
}
func (m *mockTerm) Print(line string) {
m.output = append(m.output, line)
}
func (m *mockTerm) SetStatus(lines []string) {
m.output = append([]string{}, lines...)
}
func TestPrintUpdate(t *testing.T) {
term := &mockTerm{}
printer := NewProgressPrinter(term)
printer.Update(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output)
}
func TestPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{}
printer := NewProgressPrinter(term)
printer.Finish(11, 11, 47, 47, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output)
}
func TestPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{}
printer := NewProgressPrinter(term)
printer.Finish(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output)
}

47
internal/ui/restore/text.go Normal file
View file

@ -0,0 +1,47 @@
package restore
import (
"fmt"
"time"
"github.com/restic/restic/internal/ui"
)
type textPrinter struct {
terminal term
}
func NewTextProgress(terminal term) ProgressPrinter {
return &textPrinter{
terminal: terminal,
}
}
func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
timeLeft := ui.FormatDuration(duration)
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal)
progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v",
timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal)
t.terminal.SetStatus([]string{progress})
}
func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
t.terminal.SetStatus([]string{})
timeLeft := ui.FormatDuration(duration)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
var summary string
if filesFinished == filesTotal && allBytesWritten == allBytesTotal {
summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft)
} else {
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s",
filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft)
}
t.terminal.Print(summary)
}

41
internal/ui/restore/text_test.go Normal file
View file

@ -0,0 +1,41 @@
package restore
import (
"testing"
"time"
"github.com/restic/restic/internal/test"
)
type mockTerm struct {
output []string
}
func (m *mockTerm) Print(line string) {
m.output = append(m.output, line)
}
func (m *mockTerm) SetStatus(lines []string) {
m.output = append([]string{}, lines...)
}
func TestPrintUpdate(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output)
}
func TestPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output)
}
func TestPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output)
}