From 6869bdaaa8b86ae99afd7ce2dc3d467c00e186fa Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 22 May 2024 16:38:00 +0200 Subject: [PATCH 1/4] backup: implement --skip-if-unchanged --- cmd/restic/cmd_backup.go | 20 ++++++++++---------- cmd/restic/cmd_backup_integration_test.go | 15 +++++++++++++++ doc/075_scripting.rst | 3 ++- internal/archiver/archiver.go | 9 +++++++++ internal/ui/backup/json.go | 9 +++++++-- internal/ui/backup/text.go | 10 +++++++++- 6 files changed, 52 insertions(+), 14 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index e5369f7b9..434469683 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -87,6 +87,7 @@ type BackupOptions struct { DryRun bool ReadConcurrency uint NoScan bool + SkipIfUnchanged bool } var backupOptions BackupOptions @@ -133,6 +134,7 @@ func init() { if runtime.GOOS == "windows" { f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)") } + f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") // parse read concurrency from env, on error the default value will be used readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32) @@ -638,13 +640,14 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } snapshotOpts := archiver.SnapshotOptions{ - Excludes: opts.Excludes, - Tags: opts.Tags.Flatten(), - BackupStart: backupStart, - Time: timeStamp, - Hostname: opts.Host, - ParentSnapshot: parentSnapshot, - ProgramVersion: "restic " + version, + Excludes: opts.Excludes, + Tags: opts.Tags.Flatten(), + BackupStart: backupStart, + Time: timeStamp, + Hostname: opts.Host, + ParentSnapshot: parentSnapshot, + ProgramVersion: "restic " + version, + SkipIfUnchanged: opts.SkipIfUnchanged, } if !gopts.JSON { @@ -665,9 +668,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter // Report finished execution progressReporter.Finish(id, summary, opts.DryRun) - if !gopts.JSON && !opts.DryRun { - progressPrinter.P("snapshot %s saved\n", id.Str()) - } if !success { return ErrInvalidSourceData } diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index f7372851f..5e00b84b0 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -641,3 +641,18 @@ func TestBackupEmptyPassword(t *testing.T) { testListSnapshots(t, env.gopts, 1) testRunCheck(t, env.gopts) } + +func TestBackupSkipIfUnchanged(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{SkipIfUnchanged: true} + + for i := 0; i < 3; i++ { + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + } + + testRunCheck(t, env.gopts) +} diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 28419c292..e413e349f 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -173,7 +173,8 @@ Summary is the last output line in a successful backup. +---------------------------+---------------------------------------------------------+ | ``total_duration`` | Total time it took for the operation to complete | +---------------------------+---------------------------------------------------------+ -| ``snapshot_id`` | ID of the new snapshot | +| ``snapshot_id`` | ID of the new snapshot. Field is omitted if snapshot | +| | creation was skipped | +---------------------------+---------------------------------------------------------+ diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 86b329a9a..9a31911b9 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -767,6 +767,8 @@ type SnapshotOptions struct { Time time.Time ParentSnapshot *restic.Snapshot ProgramVersion string + // SkipIfUnchanged omits the snapshot creation if it is identical to the parent snapshot. + SkipIfUnchanged bool } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. @@ -880,6 +882,13 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, nil, err } + if opts.ParentSnapshot != nil && opts.SkipIfUnchanged { + ps := opts.ParentSnapshot + if ps.Tree != nil && rootTreeID.Equal(*ps.Tree) { + return nil, restic.ID{}, arch.summary, nil + } + } + sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { return nil, restic.ID{}, nil, err diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index a14c7ccec..64b5de13b 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -164,6 +164,11 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) { // Finish prints the finishing messages. func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) { + id := "" + // empty if snapshot creation was skipped + if !snapshotID.IsNull() { + id = snapshotID.String() + } b.print(summaryOutput{ MessageType: "summary", FilesNew: summary.Files.New, @@ -179,7 +184,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *ar TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged, TotalBytesProcessed: summary.ProcessedBytes, TotalDuration: time.Since(start).Seconds(), - SnapshotID: snapshotID.String(), + SnapshotID: id, DryRun: dryRun, }) } @@ -235,6 +240,6 @@ type summaryOutput struct { TotalFilesProcessed uint `json:"total_files_processed"` TotalBytesProcessed uint64 `json:"total_bytes_processed"` TotalDuration float64 `json:"total_duration"` // in seconds - SnapshotID string `json:"snapshot_id"` + SnapshotID string `json:"snapshot_id,omitempty"` DryRun bool `json:"dry_run,omitempty"` } diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 00d025e51..43e963b82 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -126,7 +126,7 @@ func (b *TextProgress) Reset() { } // Finish prints the finishing messages. -func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) { +func (b *TextProgress) Finish(id restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) { b.P("\n") b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged) b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged) @@ -145,4 +145,12 @@ func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Su ui.FormatBytes(summary.ProcessedBytes), ui.FormatDuration(time.Since(start)), ) + + if !dryRun { + if id.IsNull() { + b.P("skipped creating snapshot\n") + } else { + b.P("snapshot %s saved\n", id.Str()) + } + } } From 5649334099ec4b9113182085f0eb70ac82706e05 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 22 May 2024 16:38:39 +0200 Subject: [PATCH 2/4] backup: document --skip-if-unchanged --- doc/040_backup.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index efaa73255..88b896280 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -231,6 +231,40 @@ On **Windows**, a file is considered unchanged when its path, size and modification time match, and only ``--force`` has any effect. The other options are recognized but ignored. +Skip creating snapshots if unchanged +************************************ + +By default, restic always creates a new snapshot even if nothing has changed +compared to the parent snapshot. To omit the creation of a new snapshot in this +case, specify the ``--skip-if-unchanged`` option. + +Note that when using absolute paths to specify the backup target, then also +changes to the parent folders result in a changed snapshot. For example, a backup +of ``/home/user/work`` will create a new snapshot if the metadata of either + ``/``, ``/home`` or ``/home/user`` change. To avoid this problem run restic from +the corresponding folder and use relative paths. + +.. code-block:: console + + $ cd /home/user/work && restic -r /srv/restic-repo backup . --skip-if-unchanged + + open repository + enter password for repository: + repository a14e5863 opened (version 2, compression level auto) + load index files + using parent snapshot 40dc1520 + start scan on [.] + start backup on [.] + scan finished in 1.814s: 5307 files, 1.200GiB + + Files: 0 new, 0 changed, 5307 unmodified + Dirs: 0 new, 0 changed, 1867 unmodified + Added to the repository: 0 B (0 B stored) + + processed 5307 files, 1.720 GiB in 0:03 + skipped creating snapshot + + Dry Runs ******** From 513135b2389309429dedbb1a03bb7823eb2e2555 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 22 May 2024 16:39:10 +0200 Subject: [PATCH 3/4] doc: slightly modernize backup examples --- doc/040_backup.rst | 38 +++++++++++++++++----------------- doc/045_working_with_repos.rst | 8 +++---- doc/060_forget.rst | 6 +++--- doc/manual_rest.rst | 3 --- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 88b896280..a07413ad9 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -24,16 +24,17 @@ again: $ restic -r /srv/restic-repo --verbose backup ~/work open repository enter password for repository: - password is correct - lock repository + repository a14e5863 opened (version 2, compression level auto) load index files - start scan - start backup - scan finished in 1.837s - processed 1.720 GiB in 0:12 + start scan on [/home/user/work] + start backup on [/home/user/work] + scan finished in 1.837s: 5307 files, 1.720 GiB + Files: 5307 new, 0 changed, 0 unmodified Dirs: 1867 new, 0 changed, 0 unmodified - Added: 1.200 GiB + Added to the repository: 1.200 GiB (1.103 GiB stored) + + processed 5307 files, 1.720 GiB in 0:12 snapshot 40dc1520 saved As you can see, restic created a backup of the directory and was pretty @@ -44,6 +45,7 @@ You can see that restic tells us it processed 1.720 GiB of data, this is the size of the files and directories in ``~/work`` on the local file system. It also tells us that only 1.200 GiB was added to the repository. This means that some of the data was duplicate and restic was able to efficiently reduce it. +The data compression also managed to compress the data down to 1.103 GiB. If you don't pass the ``--verbose`` option, restic will print less data. You'll still get a nice live status display. Be aware that the live status shows the @@ -109,17 +111,18 @@ repository (since all data is already there). This is de-duplication at work! $ restic -r /srv/restic-repo --verbose backup ~/work open repository enter password for repository: - password is correct - lock repository + repository a14e5863 opened (version 2, compression level auto) load index files - using parent snapshot d875ae93 - start scan - start backup - scan finished in 1.881s - processed 1.720 GiB in 0:03 + using parent snapshot 40dc1520 + start scan on [/home/user/work] + start backup on [/home/user/work] + scan finished in 1.881s: 5307 files, 1.720 GiB + Files: 0 new, 0 changed, 5307 unmodified Dirs: 0 new, 0 changed, 1867 unmodified - Added: 0 B + Added to the repository: 0 B (0 B stored) + + processed 5307 files, 1.720 GiB in 0:03 snapshot 79766175 saved You can even backup individual files in the same repository (not passing @@ -129,7 +132,6 @@ You can even backup individual files in the same repository (not passing $ restic -r /srv/restic-repo backup ~/work.txt enter password for repository: - password is correct snapshot 249d0210 saved If you're interested in what restic does, pass ``--verbose`` twice (or @@ -143,7 +145,6 @@ restic encounters: $ restic -r /srv/restic-repo --verbose --verbose backup ~/work.txt open repository enter password for repository: - password is correct lock repository load index files using parent snapshot f3f8d56b @@ -255,7 +256,7 @@ the corresponding folder and use relative paths. using parent snapshot 40dc1520 start scan on [.] start backup on [.] - scan finished in 1.814s: 5307 files, 1.200GiB + scan finished in 1.814s: 5307 files, 1.720 GiB Files: 0 new, 0 changed, 5307 unmodified Dirs: 0 new, 0 changed, 1867 unmodified @@ -503,7 +504,6 @@ and displays a small statistic, just pass the command two snapshot IDs: .. code-block:: console $ restic -r /srv/restic-repo diff 5845b002 2ab627a6 - password is correct comparing snapshot ea657ce5 to 2ab627a6: C /restic/cmd_diff.go diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 85c022580..9d6167895 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -163,8 +163,8 @@ example from a local to a remote repository, you can use the ``copy`` command: .. code-block:: console $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo - repository d6504c63 opened successfully, password is correct - repository 3dd0878c opened successfully, password is correct + repository d6504c63 opened successfully + repository 3dd0878c opened successfully snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir copy started, this may take a while... @@ -263,7 +263,7 @@ the unwanted files from affected snapshots by rewriting them using the .. code-block:: console $ restic -r /srv/restic-repo rewrite --exclude secret-file - repository c881945a opened (repository version 2) successfully, password is correct + repository c881945a opened (repository version 2) successfully snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file @@ -274,7 +274,7 @@ the unwanted files from affected snapshots by rewriting them using the modified 1 snapshots $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2 - repository c881945a opened (repository version 2) successfully, password is correct + repository c881945a opened (repository version 2) successfully snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file diff --git a/doc/060_forget.rst b/doc/060_forget.rst index b8d206c97..fe0236f12 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -80,7 +80,7 @@ command must be run: $ restic -r /srv/restic-repo prune enter password for repository: - repository 33002c5e opened successfully, password is correct + repository 33002c5e opened successfully loading all snapshots... loading indexes... finding data that is still in use for 4 snapshots @@ -265,7 +265,7 @@ Sunday for 12 weeks: .. code-block:: console $ restic snapshots - repository f00c6e2a opened successfully, password is correct + repository f00c6e2a opened successfully ID Time Host Tags Paths --------------------------------------------------------------- 0a1f9759 2019-09-01 11:00:00 mopped /home/user/work @@ -289,7 +289,7 @@ four Sundays, and remove the other snapshots: .. code-block:: console $ restic forget --keep-daily 4 --dry-run - repository f00c6e2a opened successfully, password is correct + repository f00c6e2a opened successfully Applying Policy: keep the last 4 daily snapshots keep 4 snapshots: ID Time Host Tags Reasons Paths diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 188ebcb70..3f8b3a2c7 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -329,7 +329,6 @@ required to restore the latest snapshot (from any host that made it): .. code-block:: console $ restic stats latest - password is correct Total File Count: 10538 Total Size: 37.824 GiB @@ -340,7 +339,6 @@ host by using the ``--host`` flag: .. code-block:: console $ restic stats --host myserver latest - password is correct Total File Count: 21766 Total Size: 481.783 GiB @@ -357,7 +355,6 @@ has restic's deduplication helped? We can check: .. code-block:: console $ restic stats --host myserver --mode raw-data latest - password is correct Total Blob Count: 340847 Total Size: 458.663 GiB From c103c0830e41a7fe9aba599094c87b33f27be517 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 22 May 2024 16:45:09 +0200 Subject: [PATCH 4/4] add changelog for skip-if-unchanged --- changelog/unreleased/issue-662 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-662 diff --git a/changelog/unreleased/issue-662 b/changelog/unreleased/issue-662 new file mode 100644 index 000000000..e5a7c6fe8 --- /dev/null +++ b/changelog/unreleased/issue-662 @@ -0,0 +1,11 @@ +Enhancement: `backup` can omit snapshot creation if there was no change + +The `backup` command always created a snapshot even if nothing changed +compared to the parent snapshot. + +Restic now supports the `--skip-if-unchanged` option for the `backup` +command to omit creating a snapshot if the new snapshot's content would +be identical to that of the parent snapshot. + +https://github.com/restic/restic/issues/662 +https://github.com/restic/restic/pull/4816