Merge pull request #4705 from MichaelEischer/snapshot-statistics
Store snapshot statistics & print snapshot size
This commit is contained in:
Michael Eischer
GitHub
commit
7f9ad1c3db
16 changed files with 369 additions and 160 deletions
12
changelog/unreleased/issue-693
Normal file
12
changelog/unreleased/issue-693
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
Enhancement: Support printing snapshot size in `snapshots` command
|
||||
|
||||
The `snapshots` command now supports printing the snapshot size for snapshots
|
||||
created using this or a future restic version. For this, the `backup` command
|
||||
now stores the backup summary statistics in the snapshot.
|
||||
|
||||
The text output of the `snapshots` command only shows the snapshot size. The
|
||||
other statistics are only included in the JSON output. To inspect these
|
||||
statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
|
||||
|
||||
https://github.com/restic/restic/issues/693
|
||||
https://github.com/restic/restic/pull/4705
|
||||
|
|
@ -451,6 +451,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||
}
|
||||
|
||||
timeStamp := time.Now()
|
||||
backupStart := timeStamp
|
||||
if opts.TimeStamp != "" {
|
||||
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
|
||||
if err != nil {
|
||||
|
|
@ -640,6 +641,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||
snapshotOpts := archiver.SnapshotOptions{
|
||||
Excludes: opts.Excludes,
|
||||
Tags: opts.Tags.Flatten(),
|
||||
BackupStart: backupStart,
|
||||
Time: timeStamp,
|
||||
Hostname: opts.Host,
|
||||
ParentSnapshot: parentSnapshot,
|
||||
|
|
@ -649,7 +651,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||
if !gopts.JSON {
|
||||
progressPrinter.V("start backup on %v", targets)
|
||||
}
|
||||
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
|
||||
_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
|
||||
|
||||
// cleanly shutdown all running goroutines
|
||||
cancel()
|
||||
|
|
@ -663,7 +665,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||
}
|
||||
|
||||
// Report finished execution
|
||||
progressReporter.Finish(id, opts.DryRun)
|
||||
progressReporter.Finish(id, summary, opts.DryRun)
|
||||
if !gopts.JSON && !opts.DryRun {
|
||||
progressPrinter.P("snapshot %s saved\n", id.Str())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
"github.com/restic/restic/internal/ui/table"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -163,6 +164,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
|
|||
keepReasons[*id] = reasons[i]
|
||||
}
|
||||
}
|
||||
// check if any snapshot contains a summary
|
||||
hasSize := false
|
||||
for _, sn := range list {
|
||||
hasSize = hasSize || (sn.Summary != nil)
|
||||
}
|
||||
|
||||
// always sort the snapshots so that the newer ones are listed last
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
|
|
@ -198,6 +204,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
|
|||
tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
|
||||
}
|
||||
tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
|
||||
if hasSize {
|
||||
tab.AddColumn("Size", `{{ .Size }}`)
|
||||
}
|
||||
}
|
||||
|
||||
type snapshot struct {
|
||||
|
|
@ -207,6 +216,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
|
|||
Tags []string
|
||||
Reasons []string
|
||||
Paths []string
|
||||
Size string
|
||||
}
|
||||
|
||||
var multiline bool
|
||||
|
|
@ -228,6 +238,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
|
|||
multiline = true
|
||||
}
|
||||
|
||||
if sn.Summary != nil {
|
||||
data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed)
|
||||
}
|
||||
|
||||
tab.AddRow(data)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,19 +18,21 @@ Working with repositories
|
|||
Listing all snapshots
|
||||
=====================
|
||||
|
||||
Now, you can list all the snapshots stored in the repository:
|
||||
Now, you can list all the snapshots stored in the repository. The size column
|
||||
only exists for snapshots created using restic 0.17.0 or later. It reflects the
|
||||
size of the contained files at the time when the snapshot was created.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo snapshots
|
||||
enter password for repository:
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work
|
||||
79766175 2015-05-08 21:40:19 kasimir /home/user/work
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv
|
||||
ID Date Host Tags Directory Size
|
||||
-------------------------------------------------------------------------
|
||||
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
|
||||
79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
|
||||
|
||||
You can filter the listing by directory path:
|
||||
|
||||
|
|
@ -38,10 +40,10 @@ You can filter the listing by directory path:
|
|||
|
||||
$ restic -r /srv/restic-repo snapshots --path="/srv"
|
||||
enter password for repository:
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv
|
||||
ID Date Host Tags Directory Size
|
||||
-------------------------------------------------------------------
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
|
||||
|
||||
Or filter by host:
|
||||
|
||||
|
|
@ -49,10 +51,10 @@ Or filter by host:
|
|||
|
||||
$ restic -r /srv/restic-repo snapshots --host luigi
|
||||
enter password for repository:
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv
|
||||
ID Date Host Tags Directory Size
|
||||
-------------------------------------------------------------------
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
|
||||
|
||||
Combining filters is also possible.
|
||||
|
||||
|
|
@ -64,21 +66,21 @@ Furthermore you can group the output by the same filters (host, paths, tags):
|
|||
|
||||
enter password for repository:
|
||||
snapshots for (host [kasimir])
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work
|
||||
79766175 2015-05-08 21:40:19 kasimir /home/user/work
|
||||
ID Date Host Tags Directory Size
|
||||
------------------------------------------------------------------------
|
||||
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
|
||||
79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
|
||||
2 snapshots
|
||||
snapshots for (host [luigi])
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv
|
||||
ID Date Host Tags Directory Size
|
||||
-------------------------------------------------------------------
|
||||
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
|
||||
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
|
||||
2 snapshots
|
||||
snapshots for (host [kazik])
|
||||
ID Date Host Tags Directory
|
||||
----------------------------------------------------------------------
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv
|
||||
ID Date Host Tags Directory Size
|
||||
-------------------------------------------------------------------
|
||||
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
|
||||
1 snapshots
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -163,7 +163,9 @@ Summary is the last output line in a successful backup.
|
|||
+---------------------------+---------------------------------------------------------+
|
||||
| ``tree_blobs`` | Number of tree blobs |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``data_added`` | Amount of data added, in bytes |
|
||||
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``total_files_processed`` | Total number of files processed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
|
|
@ -551,11 +553,48 @@ The snapshots command returns a single JSON object, an array with objects of the
|
|||
+---------------------+--------------------------------------------------+
|
||||
| ``program_version`` | restic version used to create snapshot |
|
||||
+---------------------+--------------------------------------------------+
|
||||
| ``summary`` | Snapshot statistics, see "Summary object" |
|
||||
+---------------------+--------------------------------------------------+
|
||||
| ``id`` | Snapshot ID |
|
||||
+---------------------+--------------------------------------------------+
|
||||
| ``short_id`` | Snapshot ID, short form |
|
||||
+---------------------+--------------------------------------------------+
|
||||
|
||||
Summary object
|
||||
|
||||
The contained statistics reflect the information at the point in time when the snapshot
|
||||
was created.
|
||||
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``backup_start`` | Time at which the backup was started |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``backup_end`` | Time at which the backup was completed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``files_new`` | Number of new files |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``files_changed`` | Number of files that changed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``files_unmodified`` | Number of files that did not change |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``dirs_new`` | Number of new directories |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``dirs_changed`` | Number of directories that changed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``dirs_unmodified`` | Number of directories that did not change |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``data_blobs`` | Number of data blobs |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``tree_blobs`` | Number of tree blobs |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``total_files_processed`` | Total number of files processed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
| ``total_bytes_processed`` | Total number of bytes processed |
|
||||
+---------------------------+---------------------------------------------------------+
|
||||
|
||||
|
||||
stats
|
||||
-----
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
|
@ -41,6 +42,18 @@ type ItemStats struct {
|
|||
TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
|
||||
}
|
||||
|
||||
type ChangeStats struct {
|
||||
New uint
|
||||
Changed uint
|
||||
Unchanged uint
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
Files, Dirs ChangeStats
|
||||
ProcessedBytes uint64
|
||||
ItemStats
|
||||
}
|
||||
|
||||
// Add adds other to the current ItemStats.
|
||||
func (s *ItemStats) Add(other ItemStats) {
|
||||
s.DataBlobs += other.DataBlobs
|
||||
|
|
@ -62,6 +75,8 @@ type Archiver struct {
|
|||
blobSaver *BlobSaver
|
||||
fileSaver *FileSaver
|
||||
treeSaver *TreeSaver
|
||||
mu sync.Mutex
|
||||
summary *Summary
|
||||
|
||||
// Error is called for all errors that occur during backup.
|
||||
Error ErrorFunc
|
||||
|
|
@ -183,6 +198,44 @@ func (arch *Archiver) error(item string, err error) error {
|
|||
return errf
|
||||
}
|
||||
|
||||
func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
|
||||
arch.CompleteItem(item, previous, current, s, d)
|
||||
|
||||
arch.mu.Lock()
|
||||
defer arch.mu.Unlock()
|
||||
|
||||
arch.summary.ItemStats.Add(s)
|
||||
|
||||
if current != nil {
|
||||
arch.summary.ProcessedBytes += current.Size
|
||||
} else {
|
||||
// last item or an error occurred
|
||||
return
|
||||
}
|
||||
|
||||
switch current.Type {
|
||||
case "dir":
|
||||
switch {
|
||||
case previous == nil:
|
||||
arch.summary.Dirs.New++
|
||||
case previous.Equals(*current):
|
||||
arch.summary.Dirs.Unchanged++
|
||||
default:
|
||||
arch.summary.Dirs.Changed++
|
||||
}
|
||||
|
||||
case "file":
|
||||
switch {
|
||||
case previous == nil:
|
||||
arch.summary.Files.New++
|
||||
case previous.Equals(*current):
|
||||
arch.summary.Files.Unchanged++
|
||||
default:
|
||||
arch.summary.Files.Changed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nodeFromFileInfo returns the restic node from an os.FileInfo.
|
||||
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
|
||||
node, err := restic.NodeFromFileInfo(filename, fi)
|
||||
|
|
@ -231,9 +284,9 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// SaveDir stores a directory in the repo and returns the node. snPath is the
|
||||
// saveDir stores a directory in the repo and returns the node. snPath is the
|
||||
// path within the current snapshot.
|
||||
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
|
||||
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
|
||||
debug.Log("%v %v", snPath, dir)
|
||||
|
||||
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
|
||||
|
|
@ -259,7 +312,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi
|
|||
pathname := arch.FS.Join(dir, name)
|
||||
oldNode := previous.Find(name)
|
||||
snItem := join(snPath, name)
|
||||
fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
|
||||
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
|
||||
|
||||
// return error early if possible
|
||||
if err != nil {
|
||||
|
|
@ -343,14 +396,14 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// Save saves a target (file or directory) to the repo. If the item is
|
||||
// save saves a target (file or directory) to the repo. If the item is
|
||||
// excluded, this function returns a nil node and error, with excluded set to
|
||||
// true.
|
||||
//
|
||||
// Errors and completion needs to be handled by the caller.
|
||||
//
|
||||
// snPath is the path within the current snapshot.
|
||||
func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
|
||||
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
|
||||
start := time.Now()
|
||||
|
||||
debug.Log("%v target %q, previous %v", snPath, target, previous)
|
||||
|
|
@ -389,7 +442,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
|
|||
if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
|
||||
if arch.allBlobsPresent(previous) {
|
||||
debug.Log("%v hasn't changed, using old list of blobs", target)
|
||||
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
||||
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
||||
arch.CompleteBlob(previous.Size)
|
||||
node, err := arch.nodeFromFileInfo(snPath, target, fi)
|
||||
if err != nil {
|
||||
|
|
@ -454,9 +507,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
|
|||
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
|
||||
arch.StartFile(snPath)
|
||||
}, func() {
|
||||
arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0)
|
||||
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
|
||||
}, func(node *restic.Node, stats ItemStats) {
|
||||
arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
|
||||
arch.trackItem(snPath, previous, node, stats, time.Since(start))
|
||||
})
|
||||
|
||||
case fi.IsDir():
|
||||
|
|
@ -471,9 +524,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
|
|||
return FutureNode{}, false, err
|
||||
}
|
||||
|
||||
fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
|
||||
fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
|
||||
func(node *restic.Node, stats ItemStats) {
|
||||
arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
|
||||
arch.trackItem(snItem, previous, node, stats, time.Since(start))
|
||||
})
|
||||
if err != nil {
|
||||
debug.Log("SaveDir for %v returned error: %v", snPath, err)
|
||||
|
|
@ -554,9 +607,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
|
|||
return fi, nil
|
||||
}
|
||||
|
||||
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
|
||||
// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
|
||||
// within the current snapshot.
|
||||
func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
|
||||
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
|
||||
|
||||
var node *restic.Node
|
||||
if snPath != "/" {
|
||||
|
|
@ -594,7 +647,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
|
|||
|
||||
// this is a leaf node
|
||||
if subatree.Leaf() {
|
||||
fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
|
||||
fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
|
||||
|
||||
if err != nil {
|
||||
err = arch.error(subatree.Path, err)
|
||||
|
|
@ -628,8 +681,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
|
|||
}
|
||||
|
||||
// not a leaf node, archive subtree
|
||||
fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
|
||||
arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
|
||||
fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
|
||||
arch.trackItem(snItem, oldNode, n, is, time.Since(start))
|
||||
})
|
||||
if err != nil {
|
||||
return FutureNode{}, 0, err
|
||||
|
|
@ -697,6 +750,7 @@ type SnapshotOptions struct {
|
|||
Tags restic.TagList
|
||||
Hostname string
|
||||
Excludes []string
|
||||
BackupStart time.Time
|
||||
Time time.Time
|
||||
ParentSnapshot *restic.Snapshot
|
||||
ProgramVersion string
|
||||
|
|
@ -747,15 +801,17 @@ func (arch *Archiver) stopWorkers() {
|
|||
}
|
||||
|
||||
// Snapshot saves several targets and returns a snapshot.
|
||||
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
|
||||
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
|
||||
arch.summary = &Summary{}
|
||||
|
||||
cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
return nil, restic.ID{}, nil, err
|
||||
}
|
||||
|
||||
atree, err := NewTree(arch.FS, cleanTargets)
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
return nil, restic.ID{}, nil, err
|
||||
}
|
||||
|
||||
var rootTreeID restic.ID
|
||||
|
|
@ -771,8 +827,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
|
|||
arch.runWorkers(wgCtx, wg)
|
||||
|
||||
debug.Log("starting snapshot")
|
||||
fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
|
||||
arch.CompleteItem("/", nil, nil, is, time.Since(start))
|
||||
fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
|
||||
arch.trackItem("/", nil, nil, is, time.Since(start))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -808,12 +864,12 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
|
|||
})
|
||||
err = wgUp.Wait()
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
return nil, restic.ID{}, nil, err
|
||||
}
|
||||
|
||||
sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
return nil, restic.ID{}, nil, err
|
||||
}
|
||||
|
||||
sn.ProgramVersion = opts.ProgramVersion
|
||||
|
|
@ -822,11 +878,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
|
|||
sn.Parent = opts.ParentSnapshot.ID()
|
||||
}
|
||||
sn.Tree = &rootTreeID
|
||||
sn.Summary = &restic.SnapshotSummary{
|
||||
BackupStart: opts.BackupStart,
|
||||
BackupEnd: time.Now(),
|
||||
|
||||
FilesNew: arch.summary.Files.New,
|
||||
FilesChanged: arch.summary.Files.Changed,
|
||||
FilesUnmodified: arch.summary.Files.Unchanged,
|
||||
DirsNew: arch.summary.Dirs.New,
|
||||
DirsChanged: arch.summary.Dirs.Changed,
|
||||
DirsUnmodified: arch.summary.Dirs.Unchanged,
|
||||
DataBlobs: arch.summary.ItemStats.DataBlobs,
|
||||
TreeBlobs: arch.summary.ItemStats.TreeBlobs,
|
||||
DataAdded: arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize,
|
||||
DataAddedPacked: arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo,
|
||||
TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged,
|
||||
TotalBytesProcessed: arch.summary.ProcessedBytes,
|
||||
}
|
||||
|
||||
id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
return nil, restic.ID{}, nil, err
|
||||
}
|
||||
|
||||
return sn, id, nil
|
||||
return sn, id, arch.summary, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -227,8 +227,9 @@ func TestArchiverSave(t *testing.T) {
|
|||
return err
|
||||
}
|
||||
arch.runWorkers(ctx, wg)
|
||||
arch.summary = &Summary{}
|
||||
|
||||
node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil)
|
||||
node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -304,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) {
|
|||
return err
|
||||
}
|
||||
arch.runWorkers(ctx, wg)
|
||||
arch.summary = &Summary{}
|
||||
|
||||
node, excluded, err := arch.Save(ctx, "/", filename, nil)
|
||||
node, excluded, err := arch.save(ctx, "/", filename, nil)
|
||||
t.Logf("Save returned %v %v", node, err)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -832,6 +834,7 @@ func TestArchiverSaveDir(t *testing.T) {
|
|||
|
||||
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
||||
arch.runWorkers(ctx, wg)
|
||||
arch.summary = &Summary{}
|
||||
|
||||
chdir := tempdir
|
||||
if test.chdir != "" {
|
||||
|
|
@ -846,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil)
|
||||
ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -913,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
|
|||
|
||||
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
||||
arch.runWorkers(ctx, wg)
|
||||
arch.summary = &Summary{}
|
||||
|
||||
fi, err := fs.Lstat(tempdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil)
|
||||
ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -983,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
|
|||
|
||||
// bothZeroOrNeither fails the test if only one of exp, act is zero.
|
||||
func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
|
||||
tb.Helper()
|
||||
if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
|
||||
restictest.Equals(tb, exp, act)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1005,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
prepare func(t testing.TB)
|
||||
targets []string
|
||||
want TestDir
|
||||
stat ItemStats
|
||||
stat Summary
|
||||
}{
|
||||
{
|
||||
src: TestDir{
|
||||
|
|
@ -1015,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
want: TestDir{
|
||||
"targetfile": TestFile{Content: string("foobar")},
|
||||
},
|
||||
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
||||
stat: Summary{
|
||||
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
||||
ProcessedBytes: 6,
|
||||
Files: ChangeStats{1, 0, 0},
|
||||
Dirs: ChangeStats{0, 0, 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
src: TestDir{
|
||||
|
|
@ -1027,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
"targetfile": TestFile{Content: string("foobar")},
|
||||
"filesymlink": TestSymlink{Target: "targetfile"},
|
||||
},
|
||||
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
||||
stat: Summary{
|
||||
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
||||
ProcessedBytes: 6,
|
||||
Files: ChangeStats{1, 0, 0},
|
||||
Dirs: ChangeStats{0, 0, 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
src: TestDir{
|
||||
|
|
@ -1047,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
"symlink": TestSymlink{Target: "subdir"},
|
||||
},
|
||||
},
|
||||
stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
|
||||
stat: Summary{
|
||||
ItemStats: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
|
||||
ProcessedBytes: 0,
|
||||
Files: ChangeStats{0, 0, 0},
|
||||
Dirs: ChangeStats{1, 0, 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
src: TestDir{
|
||||
|
|
@ -1071,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
|
||||
stat: Summary{
|
||||
ItemStats: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
|
||||
ProcessedBytes: 6,
|
||||
Files: ChangeStats{1, 0, 0},
|
||||
Dirs: ChangeStats{3, 0, 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -1083,18 +1107,11 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
|
||||
arch := New(repo, testFS, Options{})
|
||||
|
||||
var stat ItemStats
|
||||
lock := &sync.Mutex{}
|
||||
arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
stat.Add(s)
|
||||
}
|
||||
|
||||
wg, ctx := errgroup.WithContext(context.TODO())
|
||||
repo.StartPackUploader(ctx, wg)
|
||||
|
||||
arch.runWorkers(ctx, wg)
|
||||
arch.summary = &Summary{}
|
||||
|
||||
back := restictest.Chdir(t, tempdir)
|
||||
defer back()
|
||||
|
|
@ -1108,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil)
|
||||
fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1135,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) {
|
|||
want = test.src
|
||||
}
|
||||
TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
|
||||
stat := arch.summary
|
||||
bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
|
||||
bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
|
||||
bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
|
||||
bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
|
||||
bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
|
||||
restictest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
|
||||
restictest.Equals(t, test.stat.Files, stat.Files)
|
||||
restictest.Equals(t, test.stat.Dirs, stat.Dirs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1396,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) {
|
|||
}
|
||||
|
||||
t.Logf("targets: %v", targets)
|
||||
sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
||||
sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1544,7 +1565,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
|
|||
defer back()
|
||||
|
||||
targets := []string{"."}
|
||||
_, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
||||
_, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
||||
if test.err != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
|
||||
|
|
@ -1617,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) {
|
|||
return n, err
|
||||
}
|
||||
|
||||
func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
|
||||
restictest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
|
||||
restictest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
|
||||
restictest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
|
||||
restictest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
|
||||
restictest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
|
||||
restictest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
|
||||
restictest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
|
||||
restictest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
|
||||
bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
|
||||
bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
|
||||
bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
|
||||
bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
|
||||
}
|
||||
|
||||
func TestArchiverParent(t *testing.T) {
|
||||
var tests = []struct {
|
||||
src TestDir
|
||||
read map[string]int // tracks number of times a file must have been read
|
||||
src TestDir
|
||||
modify func(path string)
|
||||
statInitial Summary
|
||||
statSecond Summary
|
||||
}{
|
||||
{
|
||||
src: TestDir{
|
||||
"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
|
||||
},
|
||||
read: map[string]int{
|
||||
"targetfile": 1,
|
||||
statInitial: Summary{
|
||||
Files: ChangeStats{1, 0, 0},
|
||||
Dirs: ChangeStats{0, 0, 0},
|
||||
ProcessedBytes: 2102152,
|
||||
ItemStats: ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
|
||||
},
|
||||
statSecond: Summary{
|
||||
Files: ChangeStats{0, 0, 1},
|
||||
Dirs: ChangeStats{0, 0, 0},
|
||||
ProcessedBytes: 2102152,
|
||||
},
|
||||
},
|
||||
{
|
||||
src: TestDir{
|
||||
"targetDir": TestDir{
|
||||
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
|
||||
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
|
||||
},
|
||||
},
|
||||
statInitial: Summary{
|
||||
Files: ChangeStats{2, 0, 0},
|
||||
Dirs: ChangeStats{1, 0, 0},
|
||||
ProcessedBytes: 2469,
|
||||
ItemStats: ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
|
||||
},
|
||||
statSecond: Summary{
|
||||
Files: ChangeStats{0, 0, 2},
|
||||
Dirs: ChangeStats{0, 0, 1},
|
||||
ProcessedBytes: 2469,
|
||||
},
|
||||
},
|
||||
{
|
||||
src: TestDir{
|
||||
"targetDir": TestDir{
|
||||
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
|
||||
},
|
||||
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
|
||||
},
|
||||
modify: func(path string) {
|
||||
remove(t, filepath.Join(path, "targetDir", "targetfile"))
|
||||
save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
|
||||
},
|
||||
statInitial: Summary{
|
||||
Files: ChangeStats{2, 0, 0},
|
||||
Dirs: ChangeStats{1, 0, 0},
|
||||
ProcessedBytes: 2469,
|
||||
ItemStats: ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
|
||||
},
|
||||
statSecond: Summary{
|
||||
Files: ChangeStats{0, 1, 0},
|
||||
Dirs: ChangeStats{0, 1, 0},
|
||||
ProcessedBytes: 6,
|
||||
ItemStats: ItemStats{1, 0x305, 0x233, 2, 0, 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -1649,7 +1738,7 @@ func TestArchiverParent(t *testing.T) {
|
|||
back := restictest.Chdir(t, tempdir)
|
||||
defer back()
|
||||
|
||||
firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1674,33 +1763,33 @@ func TestArchiverParent(t *testing.T) {
|
|||
}
|
||||
return nil
|
||||
})
|
||||
restictest.Equals(t, test.statInitial.Files, summary.Files)
|
||||
restictest.Equals(t, test.statInitial.Dirs, summary.Dirs)
|
||||
restictest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
|
||||
checkSnapshotStats(t, firstSnapshot, test.statInitial)
|
||||
|
||||
if test.modify != nil {
|
||||
test.modify(tempdir)
|
||||
}
|
||||
|
||||
opts := SnapshotOptions{
|
||||
Time: time.Now(),
|
||||
ParentSnapshot: firstSnapshot,
|
||||
}
|
||||
_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts)
|
||||
testFS.bytesRead = map[string]int{}
|
||||
secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// check that all files still been read exactly once
|
||||
TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error {
|
||||
file, ok := item.(TestFile)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
n, ok := testFS.bytesRead[filename]
|
||||
if !ok {
|
||||
t.Fatalf("file %v was not read at all", filename)
|
||||
}
|
||||
|
||||
if n != len(file.Content) {
|
||||
t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if test.modify == nil {
|
||||
// check that no files were read this time
|
||||
restictest.Equals(t, map[string]int{}, testFS.bytesRead)
|
||||
}
|
||||
restictest.Equals(t, test.statSecond.Files, summary.Files)
|
||||
restictest.Equals(t, test.statSecond.Dirs, summary.Dirs)
|
||||
restictest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
|
||||
checkSnapshotStats(t, secondSnapshot, test.statSecond)
|
||||
|
||||
t.Logf("second backup saved as %v", secondSnapshotID.Str())
|
||||
t.Logf("testfs: %v", testFS)
|
||||
|
|
@ -1815,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) {
|
|||
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
||||
arch.Error = test.errFn
|
||||
|
||||
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
if test.mustError {
|
||||
if err != nil {
|
||||
t.Logf("found expected error (%v), skipping further checks", err)
|
||||
|
|
@ -1888,7 +1977,7 @@ func TestArchiverContextCanceled(t *testing.T) {
|
|||
|
||||
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
||||
|
||||
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
|
||||
if err != nil {
|
||||
t.Logf("found expected error (%v)", err)
|
||||
|
|
@ -2027,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
|
|||
SaveBlobConcurrency: 1,
|
||||
})
|
||||
|
||||
_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Errorf("expected error (%v) not found, got %v", test.err, err)
|
||||
}
|
||||
|
|
@ -2055,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna
|
|||
Time: time.Now(),
|
||||
ParentSnapshot: parent,
|
||||
}
|
||||
snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
|
||||
snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -2240,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) {
|
|||
arch.runWorkers(ctx, wg)
|
||||
|
||||
// fs.Track will panic if the file was not closed
|
||||
_, excluded, err := arch.Save(ctx, "/", tempfile, nil)
|
||||
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
|
||||
if err == nil {
|
||||
t.Errorf("Save() should have failed")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
|
|||
}
|
||||
opts.ParentSnapshot = sn
|
||||
}
|
||||
sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
|
||||
sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
|
|||
Hostname: "localhost",
|
||||
Tags: []string{"test"},
|
||||
}
|
||||
_, id, err := arch.Snapshot(ctx, []string{"."}, opts)
|
||||
_, id, _, err := arch.Snapshot(ctx, []string{"."}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
|
|||
back := rtest.Chdir(t, tmpdir)
|
||||
defer back()
|
||||
|
||||
sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
|
||||
sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
|
||||
rtest.OK(t, err)
|
||||
|
||||
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
|
||||
|
|
|
|||
|
|
@ -25,11 +25,31 @@ type Snapshot struct {
|
|||
Tags []string `json:"tags,omitempty"`
|
||||
Original *ID `json:"original,omitempty"`
|
||||
|
||||
ProgramVersion string `json:"program_version,omitempty"`
|
||||
ProgramVersion string `json:"program_version,omitempty"`
|
||||
Summary *SnapshotSummary `json:"summary,omitempty"`
|
||||
|
||||
id *ID // plaintext ID, used during restore
|
||||
}
|
||||
|
||||
type SnapshotSummary struct {
|
||||
BackupStart time.Time `json:"backup_start"`
|
||||
BackupEnd time.Time `json:"backup_end"`
|
||||
|
||||
// statistics from the backup json output
|
||||
FilesNew uint `json:"files_new"`
|
||||
FilesChanged uint `json:"files_changed"`
|
||||
FilesUnmodified uint `json:"files_unmodified"`
|
||||
DirsNew uint `json:"dirs_new"`
|
||||
DirsChanged uint `json:"dirs_changed"`
|
||||
DirsUnmodified uint `json:"dirs_unmodified"`
|
||||
DataBlobs int `json:"data_blobs"`
|
||||
TreeBlobs int `json:"tree_blobs"`
|
||||
DataAdded uint64 `json:"data_added"`
|
||||
DataAddedPacked uint64 `json:"data_added_packed"`
|
||||
TotalFilesProcessed uint `json:"total_files_processed"`
|
||||
TotalBytesProcessed uint64 `json:"total_bytes_processed"`
|
||||
}
|
||||
|
||||
// NewSnapshot returns an initialized snapshot struct for the current user and
|
||||
// time.
|
||||
func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {
|
||||
|
|
|
|||
|
|
@ -858,7 +858,7 @@ func TestRestorerSparseFiles(t *testing.T) {
|
|||
rtest.OK(t, err)
|
||||
|
||||
arch := archiver.New(repo, target, archiver.Options{})
|
||||
sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
|
||||
sn, _, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
|
||||
archiver.SnapshotOptions{})
|
||||
rtest.OK(t, err)
|
||||
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) {
|
|||
}
|
||||
|
||||
// Finish prints the finishing messages.
|
||||
func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) {
|
||||
func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
|
||||
b.print(summaryOutput{
|
||||
MessageType: "summary",
|
||||
FilesNew: summary.Files.New,
|
||||
|
|
@ -175,6 +175,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
|
|||
DataBlobs: summary.ItemStats.DataBlobs,
|
||||
TreeBlobs: summary.ItemStats.TreeBlobs,
|
||||
DataAdded: summary.ItemStats.DataSize + summary.ItemStats.TreeSize,
|
||||
DataAddedPacked: summary.ItemStats.DataSizeInRepo + summary.ItemStats.TreeSizeInRepo,
|
||||
TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged,
|
||||
TotalBytesProcessed: summary.ProcessedBytes,
|
||||
TotalDuration: time.Since(start).Seconds(),
|
||||
|
|
@ -230,6 +231,7 @@ type summaryOutput struct {
|
|||
DataBlobs int `json:"data_blobs"`
|
||||
TreeBlobs int `json:"tree_blobs"`
|
||||
DataAdded uint64 `json:"data_added"`
|
||||
DataAddedPacked uint64 `json:"data_added_packed"`
|
||||
TotalFilesProcessed uint `json:"total_files_processed"`
|
||||
TotalBytesProcessed uint64 `json:"total_bytes_processed"`
|
||||
TotalDuration float64 `json:"total_duration"` // in seconds
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ type ProgressPrinter interface {
|
|||
ScannerError(item string, err error) error
|
||||
CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration)
|
||||
ReportTotal(start time.Time, s archiver.ScanStats)
|
||||
Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool)
|
||||
Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool)
|
||||
Reset()
|
||||
|
||||
P(msg string, args ...interface{})
|
||||
|
|
@ -28,16 +28,6 @@ type Counter struct {
|
|||
Files, Dirs, Bytes uint64
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
Files, Dirs struct {
|
||||
New uint
|
||||
Changed uint
|
||||
Unchanged uint
|
||||
}
|
||||
ProcessedBytes uint64
|
||||
archiver.ItemStats
|
||||
}
|
||||
|
||||
// Progress reports progress for the `backup` command.
|
||||
type Progress struct {
|
||||
progress.Updater
|
||||
|
|
@ -52,7 +42,6 @@ type Progress struct {
|
|||
processed, total Counter
|
||||
errors uint
|
||||
|
||||
summary Summary
|
||||
printer ProgressPrinter
|
||||
}
|
||||
|
||||
|
|
@ -126,16 +115,6 @@ func (p *Progress) CompleteBlob(bytes uint64) {
|
|||
// CompleteItem is the status callback function for the archiver when a
|
||||
// file/dir has been saved successfully.
|
||||
func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
|
||||
p.mu.Lock()
|
||||
p.summary.ItemStats.Add(s)
|
||||
|
||||
// for the last item "/", current is nil
|
||||
if current != nil {
|
||||
p.summary.ProcessedBytes += current.Size
|
||||
}
|
||||
|
||||
p.mu.Unlock()
|
||||
|
||||
if current == nil {
|
||||
// error occurred, tell the status display to remove the line
|
||||
p.mu.Lock()
|
||||
|
|
@ -153,21 +132,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
|
|||
switch {
|
||||
case previous == nil:
|
||||
p.printer.CompleteItem("dir new", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Dirs.New++
|
||||
p.mu.Unlock()
|
||||
|
||||
case previous.Equals(*current):
|
||||
p.printer.CompleteItem("dir unchanged", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Dirs.Unchanged++
|
||||
p.mu.Unlock()
|
||||
|
||||
default:
|
||||
p.printer.CompleteItem("dir modified", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Dirs.Changed++
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
case "file":
|
||||
|
|
@ -179,21 +147,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
|
|||
switch {
|
||||
case previous == nil:
|
||||
p.printer.CompleteItem("file new", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Files.New++
|
||||
p.mu.Unlock()
|
||||
|
||||
case previous.Equals(*current):
|
||||
p.printer.CompleteItem("file unchanged", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Files.Unchanged++
|
||||
p.mu.Unlock()
|
||||
|
||||
default:
|
||||
p.printer.CompleteItem("file modified", item, s, d)
|
||||
p.mu.Lock()
|
||||
p.summary.Files.Changed++
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -213,8 +170,8 @@ func (p *Progress) ReportTotal(item string, s archiver.ScanStats) {
|
|||
}
|
||||
|
||||
// Finish prints the finishing messages.
|
||||
func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) {
|
||||
func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryrun bool) {
|
||||
// wait for the status update goroutine to shut down
|
||||
p.Updater.Done()
|
||||
p.printer.Finish(snapshotID, p.start, &p.summary, dryrun)
|
||||
p.printer.Finish(snapshotID, p.start, summary, dryrun)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,11 +33,10 @@ func (p *mockPrinter) CompleteItem(messageType string, _ string, _ archiver.Item
|
|||
}
|
||||
|
||||
func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {}
|
||||
func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, _ bool) {
|
||||
func (p *mockPrinter) Finish(id restic.ID, _ time.Time, _ *archiver.Summary, _ bool) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
|
||||
_ = *summary // Should not be nil.
|
||||
p.id = id
|
||||
}
|
||||
|
||||
|
|
@ -64,7 +63,7 @@ func TestProgress(t *testing.T) {
|
|||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
id := restic.NewRandomID()
|
||||
prog.Finish(id, false)
|
||||
prog.Finish(id, nil, false)
|
||||
|
||||
if !prnt.dirUnchanged {
|
||||
t.Error(`"dir unchanged" event not seen`)
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ func (b *TextProgress) Reset() {
|
|||
}
|
||||
|
||||
// Finish prints the finishing messages.
|
||||
func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dryRun bool) {
|
||||
func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
|
||||
b.P("\n")
|
||||
b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged)
|
||||
b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged)
|
||||
|
|
|
|||
Loading…
Reference in a new issue