KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #4705 from MichaelEischer/snapshot-statistics

Browse source 
Store snapshot statistics & print snapshot size
This commit is contained in:
Michael Eischer 2024-03-28 22:41:45 +01:00 committed by GitHub
commit 7f9ad1c3db
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 369 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
changelog/unreleased/issue-693 Normal file
View file

@ -0,0 +1,12 @@
Enhancement: Support printing snapshot size in `snapshots` command
The `snapshots` command now supports printing the snapshot size for snapshots
created using this or a future restic version. For this, the `backup` command
now stores the backup summary statistics in the snapshot.
The text output of the `snapshots` command only shows the snapshot size. The
other statistics are only included in the JSON output. To inspect these
statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
https://github.com/restic/restic/issues/693
https://github.com/restic/restic/pull/4705

6
cmd/restic/cmd_backup.go
View file

@ -451,6 +451,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
} }
timeStamp := time.Now() timeStamp := time.Now()
backupStart := timeStamp
if opts.TimeStamp != "" { if opts.TimeStamp != "" {
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local) timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
if err != nil { if err != nil {
@ -640,6 +641,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
snapshotOpts := archiver.SnapshotOptions{ snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes, Excludes: opts.Excludes,
Tags: opts.Tags.Flatten(), Tags: opts.Tags.Flatten(),
BackupStart: backupStart,
Time: timeStamp, Time: timeStamp,
Hostname: opts.Host, Hostname: opts.Host,
ParentSnapshot: parentSnapshot, ParentSnapshot: parentSnapshot,
@ -649,7 +651,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
if !gopts.JSON { if !gopts.JSON {
progressPrinter.V("start backup on %v", targets) progressPrinter.V("start backup on %v", targets)
} }
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts) _, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines // cleanly shutdown all running goroutines
cancel() cancel()
@ -663,7 +665,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
} }
// Report finished execution // Report finished execution
progressReporter.Finish(id, opts.DryRun) progressReporter.Finish(id, summary, opts.DryRun)
if !gopts.JSON && !opts.DryRun { if !gopts.JSON && !opts.DryRun {
progressPrinter.P("snapshot %s saved\n", id.Str()) progressPrinter.P("snapshot %s saved\n", id.Str())
} }

14
cmd/restic/cmd_snapshots.go
View file

@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -163,6 +164,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
keepReasons[*id] = reasons[i] keepReasons[*id] = reasons[i]
} }
} }
// check if any snapshot contains a summary
hasSize := false
for _, sn := range list {
hasSize = hasSize || (sn.Summary != nil)
}
// always sort the snapshots so that the newer ones are listed last // always sort the snapshots so that the newer ones are listed last
sort.SliceStable(list, func(i, j int) bool { sort.SliceStable(list, func(i, j int) bool {
@ -198,6 +204,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`) tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
} }
tab.AddColumn("Paths", `{{ join .Paths "\n" }}`) tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
if hasSize {
tab.AddColumn("Size", `{{ .Size }}`)
}
} }
type snapshot struct { type snapshot struct {
@ -207,6 +216,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
Tags []string Tags []string
Reasons []string Reasons []string
Paths []string Paths []string
Size string
} }
var multiline bool var multiline bool
@ -228,6 +238,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
multiline = true multiline = true
} }
if sn.Summary != nil {
data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed)
}
tab.AddRow(data) tab.AddRow(data)
} }

56
doc/045_working_with_repos.rst
View file

@ -18,19 +18,21 @@ Working with repositories
Listing all snapshots Listing all snapshots
===================== =====================
Now, you can list all the snapshots stored in the repository: Now, you can list all the snapshots stored in the repository. The size column
only exists for snapshots created using restic 0.17.0 or later. It reflects the
size of the contained files at the time when the snapshot was created.
.. code-block:: console .. code-block:: console
$ restic -r /srv/restic-repo snapshots $ restic -r /srv/restic-repo snapshots
enter password for repository: enter password for repository:
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- -------------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
79766175 2015-05-08 21:40:19 kasimir /home/user/work 79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
bdbd3439 2015-05-08 21:45:17 luigi /home/art bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
590c8fc8 2015-05-08 21:47:38 kazik /srv 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
You can filter the listing by directory path: You can filter the listing by directory path:
@ -38,10 +40,10 @@ You can filter the listing by directory path:
$ restic -r /srv/restic-repo snapshots --path="/srv" $ restic -r /srv/restic-repo snapshots --path="/srv"
enter password for repository: enter password for repository:
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- -------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
Or filter by host: Or filter by host:
@ -49,10 +51,10 @@ Or filter by host:
$ restic -r /srv/restic-repo snapshots --host luigi $ restic -r /srv/restic-repo snapshots --host luigi
enter password for repository: enter password for repository:
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- -------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
Combining filters is also possible. Combining filters is also possible.
@ -64,21 +66,21 @@ Furthermore you can group the output by the same filters (host, paths, tags):
enter password for repository: enter password for repository:
snapshots for (host [kasimir]) snapshots for (host [kasimir])
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- ------------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
79766175 2015-05-08 21:40:19 kasimir /home/user/work 79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
2 snapshots 2 snapshots
snapshots for (host [luigi]) snapshots for (host [luigi])
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- -------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
2 snapshots 2 snapshots
snapshots for (host [kazik]) snapshots for (host [kazik])
ID Date Host Tags Directory ID Date Host Tags Directory Size
---------------------------------------------------------------------- -------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
1 snapshots 1 snapshots

41
doc/075_scripting.rst
View file

@ -163,7 +163,9 @@ Summary is the last output line in a successful backup.
+---------------------------+---------------------------------------------------------+ +---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs | | ``tree_blobs`` | Number of tree blobs |
+---------------------------+---------------------------------------------------------+ +---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of data added, in bytes | | ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
+---------------------------+---------------------------------------------------------+ +---------------------------+---------------------------------------------------------+
| ``total_files_processed`` | Total number of files processed | | ``total_files_processed`` | Total number of files processed |
+---------------------------+---------------------------------------------------------+ +---------------------------+---------------------------------------------------------+
@ -551,11 +553,48 @@ The snapshots command returns a single JSON object, an array with objects of the
+---------------------+--------------------------------------------------+ +---------------------+--------------------------------------------------+
| ``program_version`` | restic version used to create snapshot | | ``program_version`` | restic version used to create snapshot |
+---------------------+--------------------------------------------------+ +---------------------+--------------------------------------------------+
| ``summary`` | Snapshot statistics, see "Summary object" |
+---------------------+--------------------------------------------------+
| ``id`` | Snapshot ID | | ``id`` | Snapshot ID |
+---------------------+--------------------------------------------------+ +---------------------+--------------------------------------------------+
| ``short_id`` | Snapshot ID, short form | | ``short_id`` | Snapshot ID, short form |
+---------------------+--------------------------------------------------+ +---------------------+--------------------------------------------------+
Summary object
The contained statistics reflect the information at the point in time when the snapshot
was created.
+---------------------------+---------------------------------------------------------+
| ``backup_start`` | Time at which the backup was started |
+---------------------------+---------------------------------------------------------+
| ``backup_end`` | Time at which the backup was completed |
+---------------------------+---------------------------------------------------------+
| ``files_new`` | Number of new files |
+---------------------------+---------------------------------------------------------+
| ``files_changed`` | Number of files that changed |
+---------------------------+---------------------------------------------------------+
| ``files_unmodified`` | Number of files that did not change |
+---------------------------+---------------------------------------------------------+
| ``dirs_new`` | Number of new directories |
+---------------------------+---------------------------------------------------------+
| ``dirs_changed`` | Number of directories that changed |
+---------------------------+---------------------------------------------------------+
| ``dirs_unmodified`` | Number of directories that did not change |
+---------------------------+---------------------------------------------------------+
| ``data_blobs`` | Number of data blobs |
+---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs |
+---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
+---------------------------+---------------------------------------------------------+
| ``total_files_processed`` | Total number of files processed |
+---------------------------+---------------------------------------------------------+
| ``total_bytes_processed`` | Total number of bytes processed |
+---------------------------+---------------------------------------------------------+
stats stats
----- -----

121
internal/archiver/archiver.go
View file

@ -8,6 +8,7 @@ import (
"runtime" "runtime"
"sort" "sort"
"strings" "strings"
"sync"
"time" "time"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -41,6 +42,18 @@ type ItemStats struct {
TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
} }
type ChangeStats struct {
New uint
Changed uint
Unchanged uint
}
type Summary struct {
Files, Dirs ChangeStats
ProcessedBytes uint64
ItemStats
}
// Add adds other to the current ItemStats. // Add adds other to the current ItemStats.
func (s *ItemStats) Add(other ItemStats) { func (s *ItemStats) Add(other ItemStats) {
s.DataBlobs += other.DataBlobs s.DataBlobs += other.DataBlobs
@ -62,6 +75,8 @@ type Archiver struct {
blobSaver *BlobSaver blobSaver *BlobSaver
fileSaver *FileSaver fileSaver *FileSaver
treeSaver *TreeSaver treeSaver *TreeSaver
mu sync.Mutex
summary *Summary
// Error is called for all errors that occur during backup. // Error is called for all errors that occur during backup.
Error ErrorFunc Error ErrorFunc
@ -183,6 +198,44 @@ func (arch *Archiver) error(item string, err error) error {
return errf return errf
} }
func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
arch.CompleteItem(item, previous, current, s, d)
arch.mu.Lock()
defer arch.mu.Unlock()
arch.summary.ItemStats.Add(s)
if current != nil {
arch.summary.ProcessedBytes += current.Size
} else {
// last item or an error occurred
return
}
switch current.Type {
case "dir":
switch {
case previous == nil:
arch.summary.Dirs.New++
case previous.Equals(*current):
arch.summary.Dirs.Unchanged++
default:
arch.summary.Dirs.Changed++
}
case "file":
switch {
case previous == nil:
arch.summary.Files.New++
case previous.Equals(*current):
arch.summary.Files.Unchanged++
default:
arch.summary.Files.Changed++
}
}
}
// nodeFromFileInfo returns the restic node from an os.FileInfo. // nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) { func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
node, err := restic.NodeFromFileInfo(filename, fi) node, err := restic.NodeFromFileInfo(filename, fi)
@ -231,9 +284,9 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
return err return err
} }
// SaveDir stores a directory in the repo and returns the node. snPath is the // saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot. // path within the current snapshot.
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
debug.Log("%v %v", snPath, dir) debug.Log("%v %v", snPath, dir)
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi) treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
@ -259,7 +312,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi
pathname := arch.FS.Join(dir, name) pathname := arch.FS.Join(dir, name)
oldNode := previous.Find(name) oldNode := previous.Find(name)
snItem := join(snPath, name) snItem := join(snPath, name)
fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode) fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
// return error early if possible // return error early if possible
if err != nil { if err != nil {
@ -343,14 +396,14 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
return true return true
} }
// Save saves a target (file or directory) to the repo. If the item is // save saves a target (file or directory) to the repo. If the item is
// excluded, this function returns a nil node and error, with excluded set to // excluded, this function returns a nil node and error, with excluded set to
// true. // true.
// //
// Errors and completion needs to be handled by the caller. // Errors and completion needs to be handled by the caller.
// //
// snPath is the path within the current snapshot. // snPath is the path within the current snapshot.
func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
start := time.Now() start := time.Now()
debug.Log("%v target %q, previous %v", snPath, target, previous) debug.Log("%v target %q, previous %v", snPath, target, previous)
@ -389,7 +442,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) { if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
if arch.allBlobsPresent(previous) { if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target) debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size) arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, fi) node, err := arch.nodeFromFileInfo(snPath, target, fi)
if err != nil { if err != nil {
@ -454,9 +507,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() { fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
arch.StartFile(snPath) arch.StartFile(snPath)
}, func() { }, func() {
arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0) arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
}, func(node *restic.Node, stats ItemStats) { }, func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snPath, previous, node, stats, time.Since(start)) arch.trackItem(snPath, previous, node, stats, time.Since(start))
}) })
case fi.IsDir(): case fi.IsDir():
@ -471,9 +524,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
return FutureNode{}, false, err return FutureNode{}, false, err
} }
fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree, fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
func(node *restic.Node, stats ItemStats) { func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snItem, previous, node, stats, time.Since(start)) arch.trackItem(snItem, previous, node, stats, time.Since(start))
}) })
if err != nil { if err != nil {
debug.Log("SaveDir for %v returned error: %v", snPath, err) debug.Log("SaveDir for %v returned error: %v", snPath, err)
@ -554,9 +607,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
return fi, nil return fi, nil
} }
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path // saveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot. // within the current snapshot.
func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) { func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
var node *restic.Node var node *restic.Node
if snPath != "/" { if snPath != "/" {
@ -594,7 +647,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
// this is a leaf node // this is a leaf node
if subatree.Leaf() { if subatree.Leaf() {
fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
if err != nil { if err != nil {
err = arch.error(subatree.Path, err) err = arch.error(subatree.Path, err)
@ -628,8 +681,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
} }
// not a leaf node, archive subtree // not a leaf node, archive subtree
fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
arch.CompleteItem(snItem, oldNode, n, is, time.Since(start)) arch.trackItem(snItem, oldNode, n, is, time.Since(start))
}) })
if err != nil { if err != nil {
return FutureNode{}, 0, err return FutureNode{}, 0, err
@ -697,6 +750,7 @@ type SnapshotOptions struct {
Tags restic.TagList Tags restic.TagList
Hostname string Hostname string
Excludes []string Excludes []string
BackupStart time.Time
Time time.Time Time time.Time
ParentSnapshot *restic.Snapshot ParentSnapshot *restic.Snapshot
ProgramVersion string ProgramVersion string
@ -747,15 +801,17 @@ func (arch *Archiver) stopWorkers() {
} }
// Snapshot saves several targets and returns a snapshot. // Snapshot saves several targets and returns a snapshot.
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) { func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
arch.summary = &Summary{}
cleanTargets, err := resolveRelativeTargets(arch.FS, targets) cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
if err != nil { if err != nil {
return nil, restic.ID{}, err return nil, restic.ID{}, nil, err
} }
atree, err := NewTree(arch.FS, cleanTargets) atree, err := NewTree(arch.FS, cleanTargets)
if err != nil { if err != nil {
return nil, restic.ID{}, err return nil, restic.ID{}, nil, err
} }
var rootTreeID restic.ID var rootTreeID restic.ID
@ -771,8 +827,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
arch.runWorkers(wgCtx, wg) arch.runWorkers(wgCtx, wg)
debug.Log("starting snapshot") debug.Log("starting snapshot")
fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) { fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
arch.CompleteItem("/", nil, nil, is, time.Since(start)) arch.trackItem("/", nil, nil, is, time.Since(start))
}) })
if err != nil { if err != nil {
return err return err
@ -808,12 +864,12 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
}) })
err = wgUp.Wait() err = wgUp.Wait()
if err != nil { if err != nil {
return nil, restic.ID{}, err return nil, restic.ID{}, nil, err
} }
sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
if err != nil { if err != nil {
return nil, restic.ID{}, err return nil, restic.ID{}, nil, err
} }
sn.ProgramVersion = opts.ProgramVersion sn.ProgramVersion = opts.ProgramVersion
@ -822,11 +878,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
sn.Parent = opts.ParentSnapshot.ID() sn.Parent = opts.ParentSnapshot.ID()
} }
sn.Tree = &rootTreeID sn.Tree = &rootTreeID
sn.Summary = &restic.SnapshotSummary{
BackupStart: opts.BackupStart,
BackupEnd: time.Now(),
FilesNew: arch.summary.Files.New,
FilesChanged: arch.summary.Files.Changed,
FilesUnmodified: arch.summary.Files.Unchanged,
DirsNew: arch.summary.Dirs.New,
DirsChanged: arch.summary.Dirs.Changed,
DirsUnmodified: arch.summary.Dirs.Unchanged,
DataBlobs: arch.summary.ItemStats.DataBlobs,
TreeBlobs: arch.summary.ItemStats.TreeBlobs,
DataAdded: arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize,
DataAddedPacked: arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo,
TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged,
TotalBytesProcessed: arch.summary.ProcessedBytes,
}
id, err := restic.SaveSnapshot(ctx, arch.Repo, sn) id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
if err != nil { if err != nil {
return nil, restic.ID{}, err return nil, restic.ID{}, nil, err
} }
return sn, id, nil return sn, id, arch.summary, nil
} }

185
internal/archiver/archiver_test.go
View file

@ -227,8 +227,9 @@ func TestArchiverSave(t *testing.T) {
return err return err
} }
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil) node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -304,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) {
return err return err
} }
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
node, excluded, err := arch.Save(ctx, "/", filename, nil) node, excluded, err := arch.save(ctx, "/", filename, nil)
t.Logf("Save returned %v %v", node, err) t.Logf("Save returned %v %v", node, err)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -832,6 +834,7 @@ func TestArchiverSaveDir(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
chdir := tempdir chdir := tempdir
if test.chdir != "" { if test.chdir != "" {
@ -846,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil) ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -913,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
fi, err := fs.Lstat(tempdir) fi, err := fs.Lstat(tempdir)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil) ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -983,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
// bothZeroOrNeither fails the test if only one of exp, act is zero. // bothZeroOrNeither fails the test if only one of exp, act is zero.
func bothZeroOrNeither(tb testing.TB, exp, act uint64) { func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
tb.Helper()
if (exp == 0 && act != 0) || (exp != 0 && act == 0) { if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
_, file, line, _ := runtime.Caller(1) restictest.Equals(tb, exp, act)
tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
} }
} }
@ -1005,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) {
prepare func(t testing.TB) prepare func(t testing.TB)
targets []string targets []string
want TestDir want TestDir
stat ItemStats stat Summary
}{ }{
{ {
src: TestDir{ src: TestDir{
@ -1015,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) {
want: TestDir{ want: TestDir{
"targetfile": TestFile{Content: string("foobar")}, "targetfile": TestFile{Content: string("foobar")},
}, },
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
},
}, },
{ {
src: TestDir{ src: TestDir{
@ -1027,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) {
"targetfile": TestFile{Content: string("foobar")}, "targetfile": TestFile{Content: string("foobar")},
"filesymlink": TestSymlink{Target: "targetfile"}, "filesymlink": TestSymlink{Target: "targetfile"},
}, },
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
},
}, },
{ {
src: TestDir{ src: TestDir{
@ -1047,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) {
"symlink": TestSymlink{Target: "subdir"}, "symlink": TestSymlink{Target: "subdir"},
}, },
}, },
stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, stat: Summary{
ItemStats: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
ProcessedBytes: 0,
Files: ChangeStats{0, 0, 0},
Dirs: ChangeStats{1, 0, 0},
},
}, },
{ {
src: TestDir{ src: TestDir{
@ -1071,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) {
}, },
}, },
}, },
stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{3, 0, 0},
},
}, },
} }
@ -1083,18 +1107,11 @@ func TestArchiverSaveTree(t *testing.T) {
arch := New(repo, testFS, Options{}) arch := New(repo, testFS, Options{})
var stat ItemStats
lock := &sync.Mutex{}
arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
lock.Lock()
defer lock.Unlock()
stat.Add(s)
}
wg, ctx := errgroup.WithContext(context.TODO()) wg, ctx := errgroup.WithContext(context.TODO())
repo.StartPackUploader(ctx, wg) repo.StartPackUploader(ctx, wg)
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
back := restictest.Chdir(t, tempdir) back := restictest.Chdir(t, tempdir)
defer back() defer back()
@ -1108,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil) fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1135,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) {
want = test.src want = test.src
} }
TestEnsureTree(context.TODO(), t, "/", repo, treeID, want) TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
stat := arch.summary
bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs)) bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs)) bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize) bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo) bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo) bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
restictest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
restictest.Equals(t, test.stat.Files, stat.Files)
restictest.Equals(t, test.stat.Dirs, stat.Dirs)
}) })
} }
} }
@ -1396,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) {
} }
t.Logf("targets: %v", targets) t.Logf("targets: %v", targets)
sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1544,7 +1565,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
defer back() defer back()
targets := []string{"."} targets := []string{"."}
_, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) _, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
if test.err != "" { if test.err != "" {
if err == nil { if err == nil {
t.Fatalf("expected error not found, got %v, wanted %q", err, test.err) t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
@ -1617,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) {
return n, err return n, err
} }
func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
restictest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
restictest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
restictest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
restictest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
restictest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
restictest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
restictest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
restictest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
}
func TestArchiverParent(t *testing.T) { func TestArchiverParent(t *testing.T) {
var tests = []struct { var tests = []struct {
src TestDir src TestDir
read map[string]int // tracks number of times a file must have been read modify func(path string)
statInitial Summary
statSecond Summary
}{ }{
{ {
src: TestDir{ src: TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))}, "targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
}, },
read: map[string]int{ statInitial: Summary{
"targetfile": 1, Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
ProcessedBytes: 2102152,
ItemStats: ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 0, 1},
Dirs: ChangeStats{0, 0, 0},
ProcessedBytes: 2102152,
},
},
{
src: TestDir{
"targetDir": TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
},
},
statInitial: Summary{
Files: ChangeStats{2, 0, 0},
Dirs: ChangeStats{1, 0, 0},
ProcessedBytes: 2469,
ItemStats: ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 0, 2},
Dirs: ChangeStats{0, 0, 1},
ProcessedBytes: 2469,
},
},
{
src: TestDir{
"targetDir": TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
},
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
},
modify: func(path string) {
remove(t, filepath.Join(path, "targetDir", "targetfile"))
save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
},
statInitial: Summary{
Files: ChangeStats{2, 0, 0},
Dirs: ChangeStats{1, 0, 0},
ProcessedBytes: 2469,
ItemStats: ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 1, 0},
Dirs: ChangeStats{0, 1, 0},
ProcessedBytes: 6,
ItemStats: ItemStats{1, 0x305, 0x233, 2, 0, 0},
}, },
}, },
} }
@ -1649,7 +1738,7 @@ func TestArchiverParent(t *testing.T) {
back := restictest.Chdir(t, tempdir) back := restictest.Chdir(t, tempdir)
defer back() defer back()
firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1674,33 +1763,33 @@ func TestArchiverParent(t *testing.T) {
} }
return nil return nil
}) })
restictest.Equals(t, test.statInitial.Files, summary.Files)
restictest.Equals(t, test.statInitial.Dirs, summary.Dirs)
restictest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
checkSnapshotStats(t, firstSnapshot, test.statInitial)
if test.modify != nil {
test.modify(tempdir)
}
opts := SnapshotOptions{ opts := SnapshotOptions{
Time: time.Now(), Time: time.Now(),
ParentSnapshot: firstSnapshot, ParentSnapshot: firstSnapshot,
} }
_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts) testFS.bytesRead = map[string]int{}
secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// check that all files still been read exactly once if test.modify == nil {
TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error { // check that no files were read this time
file, ok := item.(TestFile) restictest.Equals(t, map[string]int{}, testFS.bytesRead)
if !ok {
return nil
} }
restictest.Equals(t, test.statSecond.Files, summary.Files)
n, ok := testFS.bytesRead[filename] restictest.Equals(t, test.statSecond.Dirs, summary.Dirs)
if !ok { restictest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
t.Fatalf("file %v was not read at all", filename) checkSnapshotStats(t, secondSnapshot, test.statSecond)
}
if n != len(file.Content) {
t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
}
return nil
})
t.Logf("second backup saved as %v", secondSnapshotID.Str()) t.Logf("second backup saved as %v", secondSnapshotID.Str())
t.Logf("testfs: %v", testFS) t.Logf("testfs: %v", testFS)
@ -1815,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.Error = test.errFn arch.Error = test.errFn
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) _, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if test.mustError { if test.mustError {
if err != nil { if err != nil {
t.Logf("found expected error (%v), skipping further checks", err) t.Logf("found expected error (%v), skipping further checks", err)
@ -1888,7 +1977,7 @@ func TestArchiverContextCanceled(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) _, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if err != nil { if err != nil {
t.Logf("found expected error (%v)", err) t.Logf("found expected error (%v)", err)
@ -2027,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
SaveBlobConcurrency: 1, SaveBlobConcurrency: 1,
}) })
_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) _, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if !errors.Is(err, test.err) { if !errors.Is(err, test.err) {
t.Errorf("expected error (%v) not found, got %v", test.err, err) t.Errorf("expected error (%v) not found, got %v", test.err, err)
} }
@ -2055,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna
Time: time.Now(), Time: time.Now(),
ParentSnapshot: parent, ParentSnapshot: parent,
} }
snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts) snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -2240,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) {
arch.runWorkers(ctx, wg) arch.runWorkers(ctx, wg)
// fs.Track will panic if the file was not closed // fs.Track will panic if the file was not closed
_, excluded, err := arch.Save(ctx, "/", tempfile, nil) _, excluded, err := arch.save(ctx, "/", tempfile, nil)
if err == nil { if err == nil {
t.Errorf("Save() should have failed") t.Errorf("Save() should have failed")
} }

2
internal/archiver/testing.go
View file

@ -32,7 +32,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
} }
opts.ParentSnapshot = sn opts.ParentSnapshot = sn
} }
sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts) sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

2
internal/archiver/testing_test.go
View file

@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
Hostname: "localhost", Hostname: "localhost",
Tags: []string{"test"}, Tags: []string{"test"},
} }
_, id, err := arch.Snapshot(ctx, []string{"."}, opts) _, id, _, err := arch.Snapshot(ctx, []string{"."}, opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

2
internal/dump/common_test.go
View file

@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
back := rtest.Chdir(t, tmpdir) back := rtest.Chdir(t, tmpdir)
defer back() defer back()
sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
rtest.OK(t, err) rtest.OK(t, err)
tree, err := restic.LoadTree(ctx, repo, *sn.Tree) tree, err := restic.LoadTree(ctx, repo, *sn.Tree)

20
internal/restic/snapshot.go
View file

@ -26,10 +26,30 @@ type Snapshot struct {
Original *ID `json:"original,omitempty"` Original *ID `json:"original,omitempty"`
ProgramVersion string `json:"program_version,omitempty"` ProgramVersion string `json:"program_version,omitempty"`
Summary *SnapshotSummary `json:"summary,omitempty"`
id *ID // plaintext ID, used during restore id *ID // plaintext ID, used during restore
} }
type SnapshotSummary struct {
BackupStart time.Time `json:"backup_start"`
BackupEnd time.Time `json:"backup_end"`
// statistics from the backup json output
FilesNew uint `json:"files_new"`
FilesChanged uint `json:"files_changed"`
FilesUnmodified uint `json:"files_unmodified"`
DirsNew uint `json:"dirs_new"`
DirsChanged uint `json:"dirs_changed"`
DirsUnmodified uint `json:"dirs_unmodified"`
DataBlobs int `json:"data_blobs"`
TreeBlobs int `json:"tree_blobs"`
DataAdded uint64 `json:"data_added"`
DataAddedPacked uint64 `json:"data_added_packed"`
TotalFilesProcessed uint `json:"total_files_processed"`
TotalBytesProcessed uint64 `json:"total_bytes_processed"`
}
// NewSnapshot returns an initialized snapshot struct for the current user and // NewSnapshot returns an initialized snapshot struct for the current user and
// time. // time.
func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) { func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {

2
internal/restorer/restorer_test.go
View file

@ -858,7 +858,7 @@ func TestRestorerSparseFiles(t *testing.T) {
rtest.OK(t, err) rtest.OK(t, err)
arch := archiver.New(repo, target, archiver.Options{}) arch := archiver.New(repo, target, archiver.Options{})
sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"}, sn, _, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
archiver.SnapshotOptions{}) archiver.SnapshotOptions{})
rtest.OK(t, err) rtest.OK(t, err)

4
internal/ui/backup/json.go
View file

@ -163,7 +163,7 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) {
} }
// Finish prints the finishing messages. // Finish prints the finishing messages.
func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) { func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
b.print(summaryOutput{ b.print(summaryOutput{
MessageType: "summary", MessageType: "summary",
FilesNew: summary.Files.New, FilesNew: summary.Files.New,
@ -175,6 +175,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
DataBlobs: summary.ItemStats.DataBlobs, DataBlobs: summary.ItemStats.DataBlobs,
TreeBlobs: summary.ItemStats.TreeBlobs, TreeBlobs: summary.ItemStats.TreeBlobs,
DataAdded: summary.ItemStats.DataSize + summary.ItemStats.TreeSize, DataAdded: summary.ItemStats.DataSize + summary.ItemStats.TreeSize,
DataAddedPacked: summary.ItemStats.DataSizeInRepo + summary.ItemStats.TreeSizeInRepo,
TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged, TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged,
TotalBytesProcessed: summary.ProcessedBytes, TotalBytesProcessed: summary.ProcessedBytes,
TotalDuration: time.Since(start).Seconds(), TotalDuration: time.Since(start).Seconds(),
@ -230,6 +231,7 @@ type summaryOutput struct {
DataBlobs int `json:"data_blobs"` DataBlobs int `json:"data_blobs"`
TreeBlobs int `json:"tree_blobs"` TreeBlobs int `json:"tree_blobs"`
DataAdded uint64 `json:"data_added"` DataAdded uint64 `json:"data_added"`
DataAddedPacked uint64 `json:"data_added_packed"`
TotalFilesProcessed uint `json:"total_files_processed"` TotalFilesProcessed uint `json:"total_files_processed"`
TotalBytesProcessed uint64 `json:"total_bytes_processed"` TotalBytesProcessed uint64 `json:"total_bytes_processed"`
TotalDuration float64 `json:"total_duration"` // in seconds TotalDuration float64 `json:"total_duration"` // in seconds

49
internal/ui/backup/progress.go
View file

@ -17,7 +17,7 @@ type ProgressPrinter interface {
ScannerError(item string, err error) error ScannerError(item string, err error) error
CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration) CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration)
ReportTotal(start time.Time, s archiver.ScanStats) ReportTotal(start time.Time, s archiver.ScanStats)
Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool)
Reset() Reset()
P(msg string, args ...interface{}) P(msg string, args ...interface{})
@ -28,16 +28,6 @@ type Counter struct {
Files, Dirs, Bytes uint64 Files, Dirs, Bytes uint64
} }
type Summary struct {
Files, Dirs struct {
New uint
Changed uint
Unchanged uint
}
ProcessedBytes uint64
archiver.ItemStats
}
// Progress reports progress for the `backup` command. // Progress reports progress for the `backup` command.
type Progress struct { type Progress struct {
progress.Updater progress.Updater
@ -52,7 +42,6 @@ type Progress struct {
processed, total Counter processed, total Counter
errors uint errors uint
summary Summary
printer ProgressPrinter printer ProgressPrinter
} }
@ -126,16 +115,6 @@ func (p *Progress) CompleteBlob(bytes uint64) {
// CompleteItem is the status callback function for the archiver when a // CompleteItem is the status callback function for the archiver when a
// file/dir has been saved successfully. // file/dir has been saved successfully.
func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
p.mu.Lock()
p.summary.ItemStats.Add(s)
// for the last item "/", current is nil
if current != nil {
p.summary.ProcessedBytes += current.Size
}
p.mu.Unlock()
if current == nil { if current == nil {
// error occurred, tell the status display to remove the line // error occurred, tell the status display to remove the line
p.mu.Lock() p.mu.Lock()
@ -153,21 +132,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
switch { switch {
case previous == nil: case previous == nil:
p.printer.CompleteItem("dir new", item, s, d) p.printer.CompleteItem("dir new", item, s, d)
p.mu.Lock()
p.summary.Dirs.New++
p.mu.Unlock()
case previous.Equals(*current): case previous.Equals(*current):
p.printer.CompleteItem("dir unchanged", item, s, d) p.printer.CompleteItem("dir unchanged", item, s, d)
p.mu.Lock()
p.summary.Dirs.Unchanged++
p.mu.Unlock()
default: default:
p.printer.CompleteItem("dir modified", item, s, d) p.printer.CompleteItem("dir modified", item, s, d)
p.mu.Lock()
p.summary.Dirs.Changed++
p.mu.Unlock()
} }
case "file": case "file":
@ -179,21 +147,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
switch { switch {
case previous == nil: case previous == nil:
p.printer.CompleteItem("file new", item, s, d) p.printer.CompleteItem("file new", item, s, d)
p.mu.Lock()
p.summary.Files.New++
p.mu.Unlock()
case previous.Equals(*current): case previous.Equals(*current):
p.printer.CompleteItem("file unchanged", item, s, d) p.printer.CompleteItem("file unchanged", item, s, d)
p.mu.Lock()
p.summary.Files.Unchanged++
p.mu.Unlock()
default: default:
p.printer.CompleteItem("file modified", item, s, d) p.printer.CompleteItem("file modified", item, s, d)
p.mu.Lock()
p.summary.Files.Changed++
p.mu.Unlock()
} }
} }
} }
@ -213,8 +170,8 @@ func (p *Progress) ReportTotal(item string, s archiver.ScanStats) {
} }
// Finish prints the finishing messages. // Finish prints the finishing messages.
func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) { func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryrun bool) {
// wait for the status update goroutine to shut down // wait for the status update goroutine to shut down
p.Updater.Done() p.Updater.Done()
p.printer.Finish(snapshotID, p.start, &p.summary, dryrun) p.printer.Finish(snapshotID, p.start, summary, dryrun)
} }

5
internal/ui/backup/progress_test.go
View file

@ -33,11 +33,10 @@ func (p *mockPrinter) CompleteItem(messageType string, _ string, _ archiver.Item
} }
func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {} func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {}
func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, _ bool) { func (p *mockPrinter) Finish(id restic.ID, _ time.Time, _ *archiver.Summary, _ bool) {
p.Lock() p.Lock()
defer p.Unlock() defer p.Unlock()
_ = *summary // Should not be nil.
p.id = id p.id = id
} }
@ -64,7 +63,7 @@ func TestProgress(t *testing.T) {
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
id := restic.NewRandomID() id := restic.NewRandomID()
prog.Finish(id, false) prog.Finish(id, nil, false)
if !prnt.dirUnchanged { if !prnt.dirUnchanged {
t.Error(`"dir unchanged" event not seen`) t.Error(`"dir unchanged" event not seen`)

2
internal/ui/backup/text.go
View file

@ -126,7 +126,7 @@ func (b *TextProgress) Reset() {
} }
// Finish prints the finishing messages. // Finish prints the finishing messages.
func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dryRun bool) { func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
b.P("\n") b.P("\n")
b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged) b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged)
b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged) b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged)