Merge pull request #3310 from MichaelEischer/copy-unstable-tree

`copy` raw tree blobs

changelog/unreleased/issue-3267

@@ -0,0 +1,11 @@
+Bugfix: `copy` failed to copy snapshots in rare cases
+
+The `copy` command could in rare cases fail with the error message `SaveTree(...)
+returned unexpected id ...`.  This has been fixed.
+
+On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
+target paths. Note that, due to limitations in the repository format, these are
+not stored properly and should be avoided if possible.
+
+https://github.com/restic/restic/issues/3267
+https://github.com/restic/restic/pull/3310

cmd/restic/cmd_copy.go

@@ -196,13 +196,16 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 
 			// Do we already have this tree blob?
 			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
-				newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree)
+				// copy raw tree bytes to avoid problems if the serialization changes
+				var err error
+				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
 				if err != nil {
-					return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err)
+					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
 				}
-				// Assurance only.
+
-				if newTreeID != tree.ID {
+				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
-					return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str())
+				if err != nil {
+					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
 				}
 			}
 

cmd/restic/integration_test.go

@@ -799,6 +799,25 @@ func TestCopyIncremental(t *testing.T) {
 		len(copiedSnapshotIDs), len(snapshotIDs))
 }
 
+func TestCopyUnstableJSON(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	// contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink`
+	datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz")
+	rtest.SetupTarTestFixture(t, env.base, datafile)
+
+	testRunInit(t, env2.gopts)
+	testRunCopy(t, env.gopts, env2.gopts)
+	testRunCheck(t, env2.gopts)
+
+	copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts)
+	rtest.Assert(t, 1 == len(copiedSnapshotIDs), "still expected %v snapshot, found %v",
+		1, len(copiedSnapshotIDs))
+}
+
 func TestInitCopyChunkerParams(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()