Merge pull request #4573 from gab50000/rewrite_time

Rewrite metadata

changelog/unreleased/pull-4573

@@ -0,0 +1,5 @@
+Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
+
+`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
+
+https://github.com/restic/restic/pull/4573

cmd/restic/cmd_repair_snapshots.go

@@ -148,7 +148,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
 			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
 				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-			}, opts.DryRun, opts.Forget, "repaired")
+			}, opts.DryRun, opts.Forget, nil, "repaired")
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
 		}

cmd/restic/cmd_rewrite.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
@@ -46,11 +47,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+type snapshotMetadata struct {
+	Hostname string
+	Time     *time.Time
+}
+
+type snapshotMetadataArgs struct {
+	Hostname string
+	Time     string
+}
+
+func (sma snapshotMetadataArgs) empty() bool {
+	return sma.Hostname == "" && sma.Time == ""
+}
+
+func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
+	if sma.empty() {
+		return nil, nil
+	}
+
+	var timeStamp *time.Time
+	if sma.Time != "" {
+		t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local)
+		if err != nil {
+			return nil, errors.Fatalf("error in time option: %v\n", err)
+		}
+		timeStamp = &t
+	}
+	return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil
+}
+
 // RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
 	Forget bool
 	DryRun bool
 
+	Metadata snapshotMetadataArgs
 	restic.SnapshotFilter
 	excludePatternOptions
 }
@@ -63,11 +95,15 @@ func init() {
 	f := cmdRewrite.Flags()
 	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+	f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
+	f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
 
 	initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
 	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 
+type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
+
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
 	if sn.Tree == nil {
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
@@ -78,33 +114,50 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, err
 	}
 
-	selectByName := func(nodepath string) bool {
+	metadata, err := opts.Metadata.convert()
-		for _, reject := range rejectByNameFuncs {
+
-			if reject(nodepath) {
+	if err != nil {
-				return false
+		return false, err
-			}
-		}
-		return true
 	}
 
-	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
+	var filter rewriteFilterFunc
-		RewriteNode: func(node *restic.Node, path string) *restic.Node {
+
-			if selectByName(path) {
+	if len(rejectByNameFuncs) > 0 {
-				return node
+		selectByName := func(nodepath string) bool {
+			for _, reject := range rejectByNameFuncs {
+				if reject(nodepath) {
+					return false
+				}
 			}
-			Verbosef(fmt.Sprintf("excluding %s\n", path))
+			return true
-			return nil
+		}
-		},
+
-		DisableNodeCache: true,
+		rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
-	})
+			RewriteNode: func(node *restic.Node, path string) *restic.Node {
+				if selectByName(path) {
+					return node
+				}
+				Verbosef(fmt.Sprintf("excluding %s\n", path))
+				return nil
+			},
+			DisableNodeCache: true,
+		})
+
+		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+		}
+	} else {
+		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return *sn.Tree, nil
+		}
+	}
 
 	return filterAndReplaceSnapshot(ctx, repo, sn,
-		func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+		filter, opts.DryRun, opts.Forget, metadata, "rewrite")
-			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-		}, opts.DryRun, opts.Forget, "rewrite")
 }
 
-func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) {
+func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot,
+	filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) {
 
 	wg, wgCtx := errgroup.WithContext(ctx)
 	repo.StartPackUploader(wgCtx, wg)
@@ -138,7 +191,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		return true, nil
 	}
 
-	if filteredTree == *sn.Tree {
+	if filteredTree == *sn.Tree && newMetadata == nil {
 		debug.Log("Snapshot %v not modified", sn)
 		return false, nil
 	}
@@ -151,6 +204,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 			Verbosef("would remove old snapshot\n")
 		}
 
+		if newMetadata != nil && newMetadata.Time != nil {
+			Verbosef("would set time to %s\n", newMetadata.Time)
+		}
+
+		if newMetadata != nil && newMetadata.Hostname != "" {
+			Verbosef("would set time to %s\n", newMetadata.Hostname)
+		}
+
 		return true, nil
 	}
 
@@ -162,6 +223,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		sn.AddTags([]string{addTag})
 	}
 
+	if newMetadata != nil && newMetadata.Time != nil {
+		Verbosef("setting time to %s\n", *newMetadata.Time)
+		sn.Time = *newMetadata.Time
+	}
+
+	if newMetadata != nil && newMetadata.Hostname != "" {
+		Verbosef("setting host to %s\n", newMetadata.Hostname)
+		sn.Hostname = newMetadata.Hostname
+	}
+
 	// Save the new snapshot.
 	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
@@ -181,8 +252,8 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if opts.excludePatternOptions.Empty() {
+	if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
-		return errors.Fatal("Nothing to do: no excludes provided")
+		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 
 	repo, err := OpenRepository(ctx, gopts)

cmd/restic/cmd_rewrite_integration_test.go

@@ -9,12 +9,13 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
 	opts := RewriteOptions{
 		excludePatternOptions: excludePatternOptions{
 			Excludes: excludes,
 		},
-		Forget: forget,
+		Forget:   forget,
+		Metadata: metadata,
 	}
 
 	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
@@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) {
 	createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	snapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
 	testRunCheck(t, env.gopts)
@@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// use an exclude that will not exclude anything
-	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
 	rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
@@ -63,11 +64,44 @@ func TestRewriteReplace(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""})
-	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+	newSnapshotIDs := testListSnapshots(t, env.gopts, 1)
-	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
 	rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
 	// check forbids unused blobs, thus remove them first
 	testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
 	testRunCheck(t, env.gopts)
 }
+
+func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+	testRunRewriteExclude(t, env.gopts, []string{}, true, metadata)
+
+	repo, _ := OpenRepository(context.TODO(), env.gopts)
+	snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil)
+	rtest.OK(t, err)
+	rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots))
+	newSnapshot := snapshots[0]
+
+	if metadata.Time != "" {
+		rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time)
+	}
+
+	if metadata.Hostname != "" {
+		rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname)
+	}
+}
+
+func TestRewriteMetadata(t *testing.T) {
+	newHost := "new host"
+	newTime := "1999-01-01 11:11:11"
+
+	for _, metadata := range []snapshotMetadataArgs{
+		{Hostname: "", Time: newTime},
+		{Hostname: newHost, Time: ""},
+		{Hostname: newHost, Time: newTime},
+	} {
+		testRewriteMetadata(t, metadata)
+	}
+}

doc/045_working_with_repos.rst

@@ -234,6 +234,27 @@ modifying the repository. Instead restic will only print the actions it would
 perform.
 
 
+Modifying metadata of snapshots
+===============================
+
+Sometimes it may be desirable to change the metadata of an existing snapshot.
+Currently, rewriting the hostname and the time of the backup is supported. 
+This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp.
+
+.. code-block:: console
+    $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11"
+
+    repository b7dbade3 opened (version 2, compression level auto)
+    [0:00] 100.00%  1 / 1 index files loaded
+
+    snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET)
+    setting time to 1999-01-01 11:11:11 +0100 CET
+    setting host to newhost
+    saved new snapshot c05da643
+
+    modified 1 snapshots
+
+
 .. _checking-integrity:
 
 Checking integrity and consistency

internal/restic/testing.go

@@ -187,3 +187,22 @@ func ParseDurationOrPanic(s string) Duration {
 
 	return d
 }
+
+// TestLoadAllSnapshots returns a list of all snapshots in the repo.
+// If a snapshot ID is in excludeIDs, it will not be included in the result.
+func TestLoadAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet) (snapshots Snapshots, err error) {
+	err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error {
+		if err != nil {
+			return err
+		}
+
+		snapshots = append(snapshots, sn)
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return snapshots, nil
+}

internal/restic/testing_test.go

@@ -17,32 +17,13 @@ const (
 	testDepth           = 2
 )
 
-// LoadAllSnapshots returns a list of all snapshots in the repo.
-// If a snapshot ID is in excludeIDs, it will not be included in the result.
-func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) {
-	err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
-		if err != nil {
-			return err
-		}
-
-		snapshots = append(snapshots, sn)
-		return nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return snapshots, nil
-}
-
 func TestCreateSnapshot(t *testing.T) {
 	repo := repository.TestRepository(t)
 	for i := 0; i < testCreateSnapshots; i++ {
 		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
 	}
 
-	snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
+	snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
 	if err != nil {
 		t.Fatal(err)
 	}