Merge pull request #4837 from MichaelEischer/restore-options

Make restore overwrite behavior configurable
This commit is contained in:
Michael Eischer 2024-06-12 22:52:55 +02:00 committed by GitHub
commit 663151db57
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 486 additions and 228 deletions

View file

@ -0,0 +1,11 @@
Enhancement: Make overwrite behavior of `restore` customizable
The `restore` command now supports an `--overwrite` option to configure whether
already existing files are overwritten. The default is `--overwrite always`,
which overwrites existing files. `--overwrite if-newer` only restores files
from the snapshot that are newer than the local state. And `--overwrite never`
does not modify existing files.
https://github.com/restic/restic/issues/4817
https://github.com/restic/restic/issues/200
https://github.com/restic/restic/pull/4837

View file

@ -47,8 +47,9 @@ type RestoreOptions struct {
includePatternOptions includePatternOptions
Target string Target string
restic.SnapshotFilter restic.SnapshotFilter
Sparse bool Sparse bool
Verify bool Verify bool
Overwrite restorer.OverwriteBehavior
} }
var restoreOptions RestoreOptions var restoreOptions RestoreOptions
@ -65,6 +66,7 @@ func init() {
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
flags.Var(&restoreOptions.Overwrite, "overwrite", "overwrite behavior, one of (always|if-newer|never) (default: always)")
} }
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
@ -137,7 +139,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
} }
progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON)) progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON))
res := restorer.NewRestorer(repo, sn, opts.Sparse, progress) res := restorer.NewRestorer(repo, sn, restorer.Options{
Sparse: opts.Sparse,
Progress: progress,
Overwrite: opts.Overwrite,
})
totalErrors := 0 totalErrors := 0
res.Error = func(location string, err error) error { res.Error = func(location string, err error) error {

View file

@ -88,6 +88,15 @@ disk space. Note that the exact location of the holes can differ from those in
the original file, as their location is determined while restoring and is not the original file, as their location is determined while restoring and is not
stored explicitly. stored explicitly.
Restoring in-place
------------------
By default, the ``restore`` command overwrites already existing files in the target
directory. This behavior can be configured via the ``--overwrite`` option. The
default is ``--overwrite always``. To only overwrite existing files if the file in
the snapshot is newer, use ``--overwrite if-newer``. To never overwrite existing files,
use ``--overwrite never``.
Restore using mount Restore using mount
=================== ===================

View file

@ -502,11 +502,14 @@ Status
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``files_restored`` | Files restored | |``files_restored`` | Files restored |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set | |``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored | |``bytes_restored`` | Number of bytes restored |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``bytes_skipped`` | Total size of skipped files |
+----------------------+------------------------------------------------------------+
Summary Summary
^^^^^^^ ^^^^^^^
@ -520,10 +523,14 @@ Summary
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``files_restored`` | Files restored | |``files_restored`` | Files restored |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set | |``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored | |``bytes_restored`` | Number of bytes restored |
+----------------------+------------------------------------------------------------+ +----------------------+------------------------------------------------------------+
|``bytes_skipped`` | Total size of skipped files |
+----------------------+------------------------------------------------------------+
snapshots snapshots

View file

@ -14,11 +14,6 @@ import (
"github.com/restic/restic/internal/ui/restore" "github.com/restic/restic/internal/ui/restore"
) )
// TODO if a blob is corrupt, there may be good blob copies in other packs
// TODO evaluate if it makes sense to split download and processing workers
// pro: can (slowly) read network and decrypt/write files concurrently
// con: each worker needs to keep one pack in memory
const ( const (
largeFileBlobCount = 25 largeFileBlobCount = 25
) )
@ -120,6 +115,13 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
// create packInfo from fileInfo // create packInfo from fileInfo
for _, file := range r.files { for _, file := range r.files {
fileBlobs := file.blobs.(restic.IDs) fileBlobs := file.blobs.(restic.IDs)
if len(fileBlobs) == 0 {
err := r.restoreEmptyFileAt(file.location)
if errFile := r.sanitizeError(file, err); errFile != nil {
return errFile
}
}
largeFile := len(fileBlobs) > largeFileBlobCount largeFile := len(fileBlobs) > largeFileBlobCount
var packsMap map[restic.ID][]fileBlobInfo var packsMap map[restic.ID][]fileBlobInfo
if largeFile { if largeFile {
@ -159,6 +161,8 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
file.blobs = packsMap file.blobs = packsMap
} }
} }
// drop no longer necessary file list
r.files = nil
wg, ctx := errgroup.WithContext(ctx) wg, ctx := errgroup.WithContext(ctx)
downloadCh := make(chan *packInfo) downloadCh := make(chan *packInfo)
@ -195,6 +199,19 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
return wg.Wait() return wg.Wait()
} }
func (r *fileRestorer) restoreEmptyFileAt(location string) error {
f, err := createFile(r.targetPath(location), 0, false)
if err != nil {
return err
}
if err = f.Close(); err != nil {
return err
}
r.progress.AddProgress(location, 0, 0)
return nil
}
type blobToFileOffsetsMapping map[restic.ID]struct { type blobToFileOffsetsMapping map[restic.ID]struct {
files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
blob restic.Blob blob restic.Blob
@ -240,32 +257,6 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
// track already processed blobs for precise error reporting // track already processed blobs for precise error reporting
processedBlobs := restic.NewBlobSet() processedBlobs := restic.NewBlobSet()
for _, entry := range blobs {
occurrences := 0
for _, offsets := range entry.files {
occurrences += len(offsets)
}
// With a maximum blob size of 8MB, the normal blob streaming has to write
// at most 800MB for a single blob. This should be short enough to avoid
// network connection timeouts. Based on a quick test, a limit of 100 only
// selects a very small number of blobs (the number of references per blob
// - aka. `count` - seem to follow a expontential distribution)
if occurrences > 100 {
// process frequently referenced blobs first as these can take a long time to write
// which can cause backend connections to time out
delete(blobs, entry.blob.ID)
partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry}
err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs)
if err := r.reportError(blobs, processedBlobs, err); err != nil {
return err
}
}
}
if len(blobs) == 0 {
return nil
}
err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs)
return r.reportError(blobs, processedBlobs, err) return r.reportError(blobs, processedBlobs, err)
} }
@ -339,11 +330,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID,
createSize = file.size createSize = file.size
} }
writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
if r.progress != nil {
r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
}
return writeErr return writeErr
} }
err := r.sanitizeError(file, writeToFile()) err := r.sanitizeError(file, writeToFile())

View file

@ -206,6 +206,10 @@ func TestFileRestorerBasic(t *testing.T) {
{"data3-1", "pack3-1"}, {"data3-1", "pack3-1"},
}, },
}, },
{
name: "empty",
blobs: []TestBlob{},
},
}, nil, sparse) }, nil, sparse)
} }
} }

View file

@ -39,6 +39,48 @@ func newFilesWriter(count int) *filesWriter {
} }
} }
func createFile(path string, createSize int64, sparse bool) (*os.File, error) {
var f *os.File
var err error
if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
if !fs.IsAccessDenied(err) {
return nil, err
}
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return nil, err
}
}
if createSize > 0 {
if sparse {
err = truncateSparse(f, createSize)
if err != nil {
_ = f.Close()
return nil, err
}
} else {
err := fs.PreallocateFile(f, createSize)
if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not
// support preallocation or our parameters combination to the preallocate call
// This should yield a syscall.ENOTSUP error, but some other errors might also
// show up.
debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
}
}
}
return f, err
}
func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error { func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error {
bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))] bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
@ -53,21 +95,9 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
var f *os.File var f *os.File
var err error var err error
if createSize >= 0 { if createSize >= 0 {
if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil { f, err = createFile(path, createSize, sparse)
if fs.IsAccessDenied(err) { if err != nil {
// If file is readonly, clear the readonly flag by resetting the return nil, err
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return nil, err
}
} else {
return nil, err
}
} }
} else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil { } else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil {
return nil, err return nil, err
@ -76,25 +106,6 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
wr := &partialFile{File: f, users: 1, sparse: sparse} wr := &partialFile{File: f, users: 1, sparse: sparse}
bucket.files[path] = wr bucket.files[path] = wr
if createSize >= 0 {
if sparse {
err = truncateSparse(f, createSize)
if err != nil {
return nil, err
}
} else {
err := fs.PreallocateFile(wr.File, createSize)
if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not
// support preallocation or our parameters combination to the preallocate call
// This should yield a syscall.ENOTSUP error, but some other errors might also
// show up.
debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
}
}
}
return wr, nil return wr, nil
} }

View file

@ -2,6 +2,7 @@ package restorer
import ( import (
"context" "context"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"sync/atomic" "sync/atomic"
@ -17,11 +18,13 @@ import (
// Restorer is used to restore a snapshot to a directory. // Restorer is used to restore a snapshot to a directory.
type Restorer struct { type Restorer struct {
repo restic.Repository repo restic.Repository
sn *restic.Snapshot sn *restic.Snapshot
sparse bool sparse bool
progress *restoreui.Progress
overwrite OverwriteBehavior
progress *restoreui.Progress fileList map[string]struct{}
Error func(location string, err error) error Error func(location string, err error) error
Warn func(message string) Warn func(message string)
@ -30,15 +33,66 @@ type Restorer struct {
var restorerAbortOnAllErrors = func(_ string, err error) error { return err } var restorerAbortOnAllErrors = func(_ string, err error) error { return err }
type Options struct {
Sparse bool
Progress *restoreui.Progress
Overwrite OverwriteBehavior
}
type OverwriteBehavior int
// Constants for different overwrite behavior
const (
OverwriteAlways OverwriteBehavior = 0
OverwriteIfNewer OverwriteBehavior = 1
OverwriteNever OverwriteBehavior = 2
OverwriteInvalid OverwriteBehavior = 3
)
// Set implements the method needed for pflag command flag parsing.
func (c *OverwriteBehavior) Set(s string) error {
switch s {
case "always":
*c = OverwriteAlways
case "if-newer":
*c = OverwriteIfNewer
case "never":
*c = OverwriteNever
default:
*c = OverwriteInvalid
return fmt.Errorf("invalid overwrite behavior %q, must be one of (always|if-newer|never)", s)
}
return nil
}
func (c *OverwriteBehavior) String() string {
switch *c {
case OverwriteAlways:
return "always"
case OverwriteIfNewer:
return "if-newer"
case OverwriteNever:
return "never"
default:
return "invalid"
}
}
func (c *OverwriteBehavior) Type() string {
return "behavior"
}
// NewRestorer creates a restorer preloaded with the content from the snapshot id. // NewRestorer creates a restorer preloaded with the content from the snapshot id.
func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool, func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer {
progress *restoreui.Progress) *Restorer {
r := &Restorer{ r := &Restorer{
repo: repo, repo: repo,
sparse: sparse, sparse: opts.Sparse,
progress: opts.Progress,
overwrite: opts.Overwrite,
fileList: make(map[string]struct{}),
Error: restorerAbortOnAllErrors, Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true }, SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true },
progress: progress,
sn: sn, sn: sn,
} }
@ -170,10 +224,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe
return err return err
} }
if res.progress != nil { res.progress.AddProgress(location, 0, 0)
res.progress.AddProgress(location, 0, 0)
}
return res.restoreNodeMetadataTo(node, target, location) return res.restoreNodeMetadataTo(node, target, location)
} }
@ -195,39 +246,12 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location
return errors.WithStack(err) return errors.WithStack(err)
} }
if res.progress != nil { res.progress.AddProgress(location, 0, 0)
res.progress.AddProgress(location, 0, 0)
}
// TODO investigate if hardlinks have separate metadata on any supported system // TODO investigate if hardlinks have separate metadata on any supported system
return res.restoreNodeMetadataTo(node, path, location) return res.restoreNodeMetadataTo(node, path, location)
} }
func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error {
wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(target); err != nil {
return err
}
if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return err
}
}
if err = wr.Close(); err != nil {
return err
}
if res.progress != nil {
res.progress.AddProgress(location, 0, 0)
}
return res.restoreNodeMetadataTo(node, target, location)
}
// RestoreTo creates the directories and files in the snapshot below dst. // RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called. // Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
@ -250,9 +274,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
_, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
enterDir: func(_ *restic.Node, target, location string) error { enterDir: func(_ *restic.Node, target, location string) error {
debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location)
if res.progress != nil { res.progress.AddFile(0)
res.progress.AddFile(0)
}
// create dir with default permissions // create dir with default permissions
// #leaveDir restores dir metadata after visiting all children // #leaveDir restores dir metadata after visiting all children
return fs.MkdirAll(target, 0700) return fs.MkdirAll(target, 0700)
@ -268,37 +290,25 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
} }
if node.Type != "file" { if node.Type != "file" {
if res.progress != nil { res.progress.AddFile(0)
res.progress.AddFile(0)
}
return nil return nil
} }
if node.Size == 0 {
if res.progress != nil {
res.progress.AddFile(node.Size)
}
return nil // deal with empty files later
}
if node.Links > 1 { if node.Links > 1 {
if idx.Has(node.Inode, node.DeviceID) { if idx.Has(node.Inode, node.DeviceID) {
if res.progress != nil { // a hardlinked file does not increase the restore size
// a hardlinked file does not increase the restore size res.progress.AddFile(0)
res.progress.AddFile(0)
}
return nil return nil
} }
idx.Add(node.Inode, node.DeviceID, location) idx.Add(node.Inode, node.DeviceID, location)
} }
if res.progress != nil { return res.withOverwriteCheck(node, target, false, func() error {
res.progress.AddFile(node.Size) res.progress.AddFile(node.Size)
} filerestorer.addFile(location, node.Content, int64(node.Size))
res.trackFile(location)
filerestorer.addFile(location, node.Content, int64(node.Size)) return nil
})
return nil
}, },
}) })
if err != nil { if err != nil {
@ -317,26 +327,26 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
visitNode: func(node *restic.Node, target, location string) error { visitNode: func(node *restic.Node, target, location string) error {
debug.Log("second pass, visitNode: restore node %q", location) debug.Log("second pass, visitNode: restore node %q", location)
if node.Type != "file" { if node.Type != "file" {
return res.restoreNodeTo(ctx, node, target, location) return res.withOverwriteCheck(node, target, false, func() error {
} return res.restoreNodeTo(ctx, node, target, location)
})
// create empty files, but not hardlinks to empty files
if node.Size == 0 && (node.Links < 2 || !idx.Has(node.Inode, node.DeviceID)) {
if node.Links > 1 {
idx.Add(node.Inode, node.DeviceID, location)
}
return res.restoreEmptyFileAt(node, target, location)
} }
if idx.Has(node.Inode, node.DeviceID) && idx.Value(node.Inode, node.DeviceID) != location { if idx.Has(node.Inode, node.DeviceID) && idx.Value(node.Inode, node.DeviceID) != location {
return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location) return res.withOverwriteCheck(node, target, true, func() error {
return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location)
})
} }
return res.restoreNodeMetadataTo(node, target, location) if res.hasRestoredFile(location) {
return res.restoreNodeMetadataTo(node, target, location)
}
// don't touch skipped files
return nil
}, },
leaveDir: func(node *restic.Node, target, location string) error { leaveDir: func(node *restic.Node, target, location string) error {
err := res.restoreNodeMetadataTo(node, target, location) err := res.restoreNodeMetadataTo(node, target, location)
if err == nil && res.progress != nil { if err == nil {
res.progress.AddProgress(location, 0, 0) res.progress.AddProgress(location, 0, 0)
} }
return err return err
@ -345,6 +355,53 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
return err return err
} }
func (res *Restorer) trackFile(location string) {
res.fileList[location] = struct{}{}
}
func (res *Restorer) hasRestoredFile(location string) bool {
_, ok := res.fileList[location]
return ok
}
func (res *Restorer) withOverwriteCheck(node *restic.Node, target string, isHardlink bool, cb func() error) error {
overwrite, err := shouldOverwrite(res.overwrite, node, target)
if err != nil {
return err
} else if !overwrite {
size := node.Size
if isHardlink {
size = 0
}
res.progress.AddSkippedFile(size)
return nil
}
return cb()
}
func shouldOverwrite(overwrite OverwriteBehavior, node *restic.Node, destination string) (bool, error) {
if overwrite == OverwriteAlways {
return true, nil
}
fi, err := fs.Lstat(destination)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return true, nil
}
return false, err
}
if overwrite == OverwriteIfNewer {
// return if node is newer
return node.ModTime.After(fi.ModTime()), nil
} else if overwrite == OverwriteNever {
// file exists
return false, nil
}
panic("unknown overwrite behavior")
}
// Snapshot returns the snapshot this restorer is configured to use. // Snapshot returns the snapshot this restorer is configured to use.
func (res *Restorer) Snapshot() *restic.Snapshot { func (res *Restorer) Snapshot() *restic.Snapshot {
return res.sn return res.sn
@ -375,8 +432,8 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) {
defer close(work) defer close(work)
_, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ _, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
visitNode: func(node *restic.Node, target, _ string) error { visitNode: func(node *restic.Node, target, location string) error {
if node.Type != "file" { if node.Type != "file" || !res.hasRestoredFile(location) {
return nil return nil
} }
select { select {

View file

@ -343,7 +343,7 @@ func TestRestorer(t *testing.T) {
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str()) t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
// make sure we're creating a new subdir of the tempdir // make sure we're creating a new subdir of the tempdir
@ -460,7 +460,7 @@ func TestRestorerRelative(t *testing.T) {
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str()) t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
cleanup := rtest.Chdir(t, tempdir) cleanup := rtest.Chdir(t, tempdir)
@ -689,7 +689,7 @@ func TestRestorerTraverseTree(t *testing.T) {
repo := repository.TestRepository(t) repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
res.SelectFilter = test.Select res.SelectFilter = test.Select
@ -765,7 +765,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
}, },
}, noopGetGenericAttributes) }, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
switch filepath.ToSlash(item) { switch filepath.ToSlash(item) {
@ -820,7 +820,7 @@ func TestVerifyCancel(t *testing.T) {
repo := repository.TestRepository(t) repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -862,7 +862,7 @@ func TestRestorerSparseFiles(t *testing.T) {
archiver.SnapshotOptions{}) archiver.SnapshotOptions{})
rtest.OK(t, err) rtest.OK(t, err)
res := NewRestorer(repo, sn, true, nil) res := NewRestorer(repo, sn, Options{Sparse: true})
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -893,3 +893,92 @@ func TestRestorerSparseFiles(t *testing.T) {
t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse", t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse",
len(zeros), blocks, 100*sparsity) len(zeros), blocks, 100*sparsity)
} }
func TestRestorerOverwriteBehavior(t *testing.T) {
baseTime := time.Now()
baseSnapshot := Snapshot{
Nodes: map[string]Node{
"foo": File{Data: "content: foo\n", ModTime: baseTime},
"dirtest": Dir{
Nodes: map[string]Node{
"file": File{Data: "content: file\n", ModTime: baseTime},
},
ModTime: baseTime,
},
},
}
overwriteSnapshot := Snapshot{
Nodes: map[string]Node{
"foo": File{Data: "content: new\n", ModTime: baseTime.Add(time.Second)},
"dirtest": Dir{
Nodes: map[string]Node{
"file": File{Data: "content: file2\n", ModTime: baseTime.Add(-time.Second)},
},
},
},
}
var tests = []struct {
Overwrite OverwriteBehavior
Files map[string]string
}{
{
Overwrite: OverwriteAlways,
Files: map[string]string{
"foo": "content: new\n",
"dirtest/file": "content: file2\n",
},
},
{
Overwrite: OverwriteIfNewer,
Files: map[string]string{
"foo": "content: new\n",
"dirtest/file": "content: file\n",
},
},
{
Overwrite: OverwriteNever,
Files: map[string]string{
"foo": "content: foo\n",
"dirtest/file": "content: file\n",
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
repo := repository.TestRepository(t)
tempdir := filepath.Join(rtest.TempDir(t), "target")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// base snapshot
sn, id := saveSnapshot(t, repo, baseSnapshot, noopGetGenericAttributes)
t.Logf("base snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, Options{})
rtest.OK(t, res.RestoreTo(ctx, tempdir))
// overwrite snapshot
sn, id = saveSnapshot(t, repo, overwriteSnapshot, noopGetGenericAttributes)
t.Logf("overwrite snapshot saved as %v", id.Str())
res = NewRestorer(repo, sn, Options{Overwrite: test.Overwrite})
rtest.OK(t, res.RestoreTo(ctx, tempdir))
_, err := res.VerifyFiles(ctx, tempdir)
rtest.OK(t, err)
for filename, content := range test.Files {
data, err := os.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename)))
if err != nil {
t.Errorf("unable to read file %v: %v", filename, err)
continue
}
if !bytes.Equal(data, []byte(content)) {
t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data)
}
}
})
}
}

View file

@ -31,7 +31,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
}, },
}, noopGetGenericAttributes) }, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true return true, true
@ -70,16 +70,13 @@ func getBlockCount(t *testing.T, filename string) int64 {
} }
type printerMock struct { type printerMock struct {
filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 s restoreui.State
} }
func (p *printerMock) Update(_, _, _, _ uint64, _ time.Duration) { func (p *printerMock) Update(_ restoreui.State, _ time.Duration) {
} }
func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) { func (p *printerMock) Finish(s restoreui.State, _ time.Duration) {
p.filesFinished = filesFinished p.s = s
p.filesTotal = filesTotal
p.allBytesWritten = allBytesWritten
p.allBytesTotal = allBytesTotal
} }
func TestRestorerProgressBar(t *testing.T) { func TestRestorerProgressBar(t *testing.T) {
@ -99,7 +96,7 @@ func TestRestorerProgressBar(t *testing.T) {
mock := &printerMock{} mock := &printerMock{}
progress := restoreui.NewProgress(mock, 0) progress := restoreui.NewProgress(mock, 0)
res := NewRestorer(repo, sn, false, progress) res := NewRestorer(repo, sn, Options{Progress: progress})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true return true, true
} }
@ -112,12 +109,12 @@ func TestRestorerProgressBar(t *testing.T) {
rtest.OK(t, err) rtest.OK(t, err)
progress.Finish() progress.Finish()
const filesFinished = 4 rtest.Equals(t, restoreui.State{
const filesTotal = filesFinished FilesFinished: 4,
const allBytesWritten = 10 FilesTotal: 4,
const allBytesTotal = allBytesWritten FilesSkipped: 0,
rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished) AllBytesWritten: 10,
rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal) AllBytesTotal: 10,
rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten) AllBytesSkipped: 0,
rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal) }, mock.s)
} }

View file

@ -269,7 +269,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer {
sn, _ := saveSnapshot(t, repo, Snapshot{ sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: nodesMap, Nodes: nodesMap,
}, getFileAttributes) }, getFileAttributes)
res := NewRestorer(repo, sn, false, nil) res := NewRestorer(repo, sn, Options{})
return res return res
} }

View file

@ -20,31 +20,35 @@ func (t *jsonPrinter) print(status interface{}) {
t.terminal.Print(ui.ToJSONString(status)) t.terminal.Print(ui.ToJSONString(status))
} }
func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { func (t *jsonPrinter) Update(p State, duration time.Duration) {
status := statusUpdate{ status := statusUpdate{
MessageType: "status", MessageType: "status",
SecondsElapsed: uint64(duration / time.Second), SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal, TotalFiles: p.FilesTotal,
FilesRestored: filesFinished, FilesRestored: p.FilesFinished,
TotalBytes: allBytesTotal, FilesSkipped: p.FilesSkipped,
BytesRestored: allBytesWritten, TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
} }
if allBytesTotal > 0 { if p.AllBytesTotal > 0 {
status.PercentDone = float64(allBytesWritten) / float64(allBytesTotal) status.PercentDone = float64(p.AllBytesWritten) / float64(p.AllBytesTotal)
} }
t.print(status) t.print(status)
} }
func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { func (t *jsonPrinter) Finish(p State, duration time.Duration) {
status := summaryOutput{ status := summaryOutput{
MessageType: "summary", MessageType: "summary",
SecondsElapsed: uint64(duration / time.Second), SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal, TotalFiles: p.FilesTotal,
FilesRestored: filesFinished, FilesRestored: p.FilesFinished,
TotalBytes: allBytesTotal, FilesSkipped: p.FilesSkipped,
BytesRestored: allBytesWritten, TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
} }
t.print(status) t.print(status)
} }
@ -55,8 +59,10 @@ type statusUpdate struct {
PercentDone float64 `json:"percent_done"` PercentDone float64 `json:"percent_done"`
TotalFiles uint64 `json:"total_files,omitempty"` TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
} }
type summaryOutput struct { type summaryOutput struct {
@ -64,6 +70,8 @@ type summaryOutput struct {
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
TotalFiles uint64 `json:"total_files,omitempty"` TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
} }

View file

@ -10,20 +10,34 @@ import (
func TestJSONPrintUpdate(t *testing.T) { func TestJSONPrintUpdate(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewJSONProgress(term) printer := NewJSONProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second) printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
} }
func TestJSONPrintUpdateWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.output)
}
func TestJSONPrintSummaryOnSuccess(t *testing.T) { func TestJSONPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewJSONProgress(term) printer := NewJSONProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second) printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output)
} }
func TestJSONPrintSummaryOnErrors(t *testing.T) { func TestJSONPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewJSONProgress(term) printer := NewJSONProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second) printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
} }
func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.output)
}

View file

@ -7,15 +7,21 @@ import (
"github.com/restic/restic/internal/ui/progress" "github.com/restic/restic/internal/ui/progress"
) )
type State struct {
FilesFinished uint64
FilesTotal uint64
FilesSkipped uint64
AllBytesWritten uint64
AllBytesTotal uint64
AllBytesSkipped uint64
}
type Progress struct { type Progress struct {
updater progress.Updater updater progress.Updater
m sync.Mutex m sync.Mutex
progressInfoMap map[string]progressInfoEntry progressInfoMap map[string]progressInfoEntry
filesFinished uint64 s State
filesTotal uint64
allBytesWritten uint64
allBytesTotal uint64
started time.Time started time.Time
printer ProgressPrinter printer ProgressPrinter
@ -32,8 +38,8 @@ type term interface {
} }
type ProgressPrinter interface { type ProgressPrinter interface {
Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) Update(progress State, duration time.Duration)
Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) Finish(progress State, duration time.Duration)
} }
func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
@ -51,23 +57,31 @@ func (p *Progress) update(runtime time.Duration, final bool) {
defer p.m.Unlock() defer p.m.Unlock()
if !final { if !final {
p.printer.Update(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) p.printer.Update(p.s, runtime)
} else { } else {
p.printer.Finish(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) p.printer.Finish(p.s, runtime)
} }
} }
// AddFile starts tracking a new file with the given size // AddFile starts tracking a new file with the given size
func (p *Progress) AddFile(size uint64) { func (p *Progress) AddFile(size uint64) {
if p == nil {
return
}
p.m.Lock() p.m.Lock()
defer p.m.Unlock() defer p.m.Unlock()
p.filesTotal++ p.s.FilesTotal++
p.allBytesTotal += size p.s.AllBytesTotal += size
} }
// AddProgress accumulates the number of bytes written for a file // AddProgress accumulates the number of bytes written for a file
func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) { func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) {
if p == nil {
return
}
p.m.Lock() p.m.Lock()
defer p.m.Unlock() defer p.m.Unlock()
@ -78,13 +92,25 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot
entry.bytesWritten += bytesWrittenPortion entry.bytesWritten += bytesWrittenPortion
p.progressInfoMap[name] = entry p.progressInfoMap[name] = entry
p.allBytesWritten += bytesWrittenPortion p.s.AllBytesWritten += bytesWrittenPortion
if entry.bytesWritten == entry.bytesTotal { if entry.bytesWritten == entry.bytesTotal {
delete(p.progressInfoMap, name) delete(p.progressInfoMap, name)
p.filesFinished++ p.s.FilesFinished++
} }
} }
func (p *Progress) AddSkippedFile(size uint64) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
p.s.FilesSkipped++
p.s.AllBytesSkipped += size
}
func (p *Progress) Finish() { func (p *Progress) Finish() {
p.updater.Done() p.updater.Done()
} }

View file

@ -8,7 +8,7 @@ import (
) )
type printerTraceEntry struct { type printerTraceEntry struct {
filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 progress State
duration time.Duration duration time.Duration
isFinished bool isFinished bool
@ -22,11 +22,11 @@ type mockPrinter struct {
const mockFinishDuration = 42 * time.Second const mockFinishDuration = 42 * time.Second
func (p *mockPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { func (p *mockPrinter) Update(progress State, duration time.Duration) {
p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, duration, false}) p.trace = append(p.trace, printerTraceEntry{progress, duration, false})
} }
func (p *mockPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) { func (p *mockPrinter) Finish(progress State, _ time.Duration) {
p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, mockFinishDuration, true}) p.trace = append(p.trace, printerTraceEntry{progress, mockFinishDuration, true})
} }
func testProgress(fn func(progress *Progress) bool) printerTrace { func testProgress(fn func(progress *Progress) bool) printerTrace {
@ -45,7 +45,7 @@ func TestNew(t *testing.T) {
return false return false
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{0, 0, 0, 0, 0, false}, printerTraceEntry{State{0, 0, 0, 0, 0, 0}, 0, false},
}, result) }, result)
} }
@ -57,7 +57,7 @@ func TestAddFile(t *testing.T) {
return false return false
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{0, 1, 0, fileSize, 0, false}, printerTraceEntry{State{0, 1, 0, 0, fileSize, 0}, 0, false},
}, result) }, result)
} }
@ -71,7 +71,7 @@ func TestFirstProgressOnAFile(t *testing.T) {
return false return false
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{0, 1, expectedBytesWritten, expectedBytesTotal, 0, false}, printerTraceEntry{State{0, 1, 0, expectedBytesWritten, expectedBytesTotal, 0}, 0, false},
}, result) }, result)
} }
@ -86,7 +86,7 @@ func TestLastProgressOnAFile(t *testing.T) {
return false return false
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{1, 1, fileSize, fileSize, 0, false}, printerTraceEntry{State{1, 1, 0, fileSize, fileSize, 0}, 0, false},
}, result) }, result)
} }
@ -102,7 +102,7 @@ func TestLastProgressOnLastFile(t *testing.T) {
return false return false
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, 0, false}, printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, 0, false},
}, result) }, result)
} }
@ -117,7 +117,7 @@ func TestSummaryOnSuccess(t *testing.T) {
return true return true
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, mockFinishDuration, true}, printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, mockFinishDuration, true},
}, result) }, result)
} }
@ -132,6 +132,18 @@ func TestSummaryOnErrors(t *testing.T) {
return true return true
}) })
test.Equals(t, printerTrace{ test.Equals(t, printerTrace{
printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true}, printerTraceEntry{State{1, 2, 0, 50 + fileSize/2, 50 + fileSize, 0}, mockFinishDuration, true},
}, result)
}
func TestSkipFile(t *testing.T) {
fileSize := uint64(100)
result := testProgress(func(progress *Progress) bool {
progress.AddSkippedFile(fileSize)
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 1, 0, 0, fileSize}, mockFinishDuration, true},
}, result) }, result)
} }

View file

@ -17,30 +17,36 @@ func NewTextProgress(terminal term) ProgressPrinter {
} }
} }
func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { func (t *textPrinter) Update(p State, duration time.Duration) {
timeLeft := ui.FormatDuration(duration) timeLeft := ui.FormatDuration(duration)
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal)
allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) allPercent := ui.FormatPercent(p.AllBytesWritten, p.AllBytesTotal)
progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v", progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v",
timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) timeLeft, allPercent, p.FilesFinished, formattedAllBytesWritten, p.FilesTotal, formattedAllBytesTotal)
if p.FilesSkipped > 0 {
progress += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped))
}
t.terminal.SetStatus([]string{progress}) t.terminal.SetStatus([]string{progress})
} }
func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { func (t *textPrinter) Finish(p State, duration time.Duration) {
t.terminal.SetStatus([]string{}) t.terminal.SetStatus([]string{})
timeLeft := ui.FormatDuration(duration) timeLeft := ui.FormatDuration(duration)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal)
var summary string var summary string
if filesFinished == filesTotal && allBytesWritten == allBytesTotal { if p.FilesFinished == p.FilesTotal && p.AllBytesWritten == p.AllBytesTotal {
summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", p.FilesTotal, formattedAllBytesTotal, timeLeft)
} else { } else {
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten)
summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s", summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s",
filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) p.FilesFinished, p.FilesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft)
}
if p.FilesSkipped > 0 {
summary += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped))
} }
t.terminal.Print(summary) t.terminal.Print(summary)

View file

@ -22,20 +22,34 @@ func (m *mockTerm) SetStatus(lines []string) {
func TestPrintUpdate(t *testing.T) { func TestPrintUpdate(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewTextProgress(term) printer := NewTextProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second) printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output) test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output)
} }
func TestPrintUpdateWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.output)
}
func TestPrintSummaryOnSuccess(t *testing.T) { func TestPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewTextProgress(term) printer := NewTextProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second) printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output) test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output)
} }
func TestPrintSummaryOnErrors(t *testing.T) { func TestPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{} term := &mockTerm{}
printer := NewTextProgress(term) printer := NewTextProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second) printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output) test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output)
} }
func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.output)
}