Merge pull request #1772 from ifedorenko/restore-verify

Restore verify

changelog/unreleased/pull-1772

@@ -0,0 +1,6 @@
+Enhancement: Add restore --verify to verify restored file content
+
+Restore will print error message if restored file content does not match
+expected SHA256 checksum
+
+https://github.com/restic/restic/pull/1772

cmd/restic/cmd_restore.go

@@ -5,6 +5,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/restorer"
 
 	"github.com/spf13/cobra"
 )
@@ -33,6 +34,7 @@ type RestoreOptions struct {
 	Host    string
 	Paths   []string
 	Tags    restic.TagLists
+	Verify  bool
 }
 
 var restoreOptions RestoreOptions
@@ -48,6 +50,7 @@ func init() {
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
 	flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
@@ -104,7 +107,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	res, err := restic.NewRestorer(repo, id)
+	res, err := restorer.NewRestorer(repo, id)
 	if err != nil {
 		Exitf(2, "creating restorer failed: %v\n", err)
 	}
@@ -153,6 +156,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
 
 	err = res.RestoreTo(ctx, opts.Target)
+	if err == nil && opts.Verify {
+		Verbosef("verifying files in %s\n", opts.Target)
+		var count int
+		count, err = res.VerifyFiles(ctx, opts.Target)
+		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
+	}
 	if totalErrors > 0 {
 		Printf("There were %d errors\n", totalErrors)
 	}

cmd/restic/integration_test.go

@@ -875,9 +875,6 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
 	fi, err := os.Stat(f1)
 	rtest.OK(t, err)
 
-	rtest.Assert(t, fi.ModTime() != time.Unix(0, 0),
-		"meta data of intermediate directory has been restore although it was ignored")
-
 	// restore with filter "*", this should restore meta data on everything.
 	testRunRestoreIncludes(t, env.gopts, filepath.Join(env.base, "restore1"), snapshotID, []string{"*"})
 

internal/restic/node.go

@@ -134,7 +134,7 @@ func (node Node) GetExtendedAttribute(a string) []byte {
 	return nil
 }
 
-// CreateAt creates the node at the given path and restores all the meta data.
+// CreateAt creates the node at the given path but does NOT restore node meta data.
 func (node *Node) CreateAt(ctx context.Context, path string, repo Repository, idx *HardlinkIndex) error {
 	debug.Log("create node %v at %v", node.Name, path)
 
@@ -169,6 +169,11 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo Repository, id
 		return errors.Errorf("filetype %q not implemented!\n", node.Type)
 	}
 
+	return nil
+}
+
+// RestoreMetadata restores node metadata
+func (node Node) RestoreMetadata(path string) error {
 	err := node.restoreMetadata(path)
 	if err != nil {
 		debug.Log("restoreMetadata(%s) error %v", path, err)
@@ -192,14 +197,12 @@ func (node Node) restoreMetadata(path string) error {
 		}
 	}
 
-	if node.Type != "dir" {
 	if err := node.RestoreTimestamps(path); err != nil {
 		debug.Log("error restoring timestamps for dir %v: %v", path, err)
 		if firsterr != nil {
 			firsterr = err
 		}
 	}
-	}
 
 	if err := node.restoreExtendedAttributes(path); err != nil {
 		debug.Log("error restoring extended attributes for %v: %v", path, err)

internal/restic/node_test.go

@@ -182,6 +182,7 @@ func TestNodeRestoreAt(t *testing.T) {
 	for _, test := range nodeTests {
 		nodePath := filepath.Join(tempdir, test.Name)
 		rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil, idx))
+		rtest.OK(t, test.RestoreMetadata(nodePath))
 
 		if test.Type == "symlink" && runtime.GOOS == "windows" {
 			continue

internal/restic/restorer.go

@@ -1,173 +0,0 @@
-package restic
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-
-	"github.com/restic/restic/internal/errors"
-
-	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/fs"
-)
-
-// Restorer is used to restore a snapshot to a directory.
-type Restorer struct {
-	repo Repository
-	sn   *Snapshot
-
-	Error        func(dir string, node *Node, err error) error
-	SelectFilter func(item string, dstpath string, node *Node) (selectedForRestore bool, childMayBeSelected bool)
-}
-
-var restorerAbortOnAllErrors = func(str string, node *Node, err error) error { return err }
-
-// NewRestorer creates a restorer preloaded with the content from the snapshot id.
-func NewRestorer(repo Repository, id ID) (*Restorer, error) {
-	r := &Restorer{
-		repo: repo, Error: restorerAbortOnAllErrors,
-		SelectFilter: func(string, string, *Node) (bool, bool) { return true, true },
-	}
-
-	var err error
-
-	r.sn, err = LoadSnapshot(context.TODO(), repo, id)
-	if err != nil {
-		return nil, err
-	}
-
-	return r, nil
-}
-
-// restoreTo restores a tree from the repo to a destination. target is the path in
-// the file system, location within the snapshot.
-func (res *Restorer) restoreTo(ctx context.Context, target, location string, treeID ID, idx *HardlinkIndex) error {
-	debug.Log("%v %v %v", target, location, treeID)
-	tree, err := res.repo.LoadTree(ctx, treeID)
-	if err != nil {
-		debug.Log("error loading tree %v: %v", treeID, err)
-		return res.Error(location, nil, err)
-	}
-
-	for _, node := range tree.Nodes {
-
-		// ensure that the node name does not contain anything that refers to a
-		// top-level directory.
-		nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name))
-		if nodeName != node.Name {
-			debug.Log("node %q has invalid name %q", node.Name, nodeName)
-			err := res.Error(location, node, errors.New("node has invalid name"))
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		nodeTarget := filepath.Join(target, nodeName)
-		nodeLocation := filepath.Join(location, nodeName)
-
-		if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
-			debug.Log("target: %v %v", target, nodeTarget)
-			debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
-			err := res.Error(nodeLocation, node, errors.New("node has invalid path"))
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		// sockets cannot be restored
-		if node.Type == "socket" {
-			continue
-		}
-
-		selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node)
-		debug.Log("SelectFilter returned %v %v", selectedForRestore, childMayBeSelected)
-
-		if node.Type == "dir" && childMayBeSelected {
-			if node.Subtree == nil {
-				return errors.Errorf("Dir without subtree in tree %v", treeID.Str())
-			}
-
-			err = res.restoreTo(ctx, nodeTarget, nodeLocation, *node.Subtree, idx)
-			if err != nil {
-				err = res.Error(nodeLocation, node, err)
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		if selectedForRestore {
-			err = res.restoreNodeTo(ctx, node, nodeTarget, nodeLocation, idx)
-			if err != nil {
-				err = res.Error(nodeLocation, node, errors.Wrap(err, "restoreNodeTo"))
-				if err != nil {
-					return err
-				}
-			}
-
-			// Restore directory timestamp at the end. If we would do it earlier, restoring files within
-			// the directory would overwrite the timestamp of the directory they are in.
-			err = node.RestoreTimestamps(nodeTarget)
-			if err != nil {
-				err = res.Error(nodeLocation, node, errors.Wrap(err, "RestoreTimestamps"))
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (res *Restorer) restoreNodeTo(ctx context.Context, node *Node, target, location string, idx *HardlinkIndex) error {
-	debug.Log("%v %v %v", node.Name, target, location)
-
-	err := node.CreateAt(ctx, target, res.repo, idx)
-	if err != nil {
-		debug.Log("node.CreateAt(%s) error %v", target, err)
-	}
-
-	// Did it fail because of ENOENT?
-	if err != nil && os.IsNotExist(errors.Cause(err)) {
-		debug.Log("create intermediate paths")
-
-		// Create parent directories and retry
-		err = fs.MkdirAll(filepath.Dir(target), 0700)
-		if err == nil || os.IsExist(errors.Cause(err)) {
-			err = node.CreateAt(ctx, target, res.repo, idx)
-		}
-	}
-
-	if err != nil {
-		debug.Log("error %v", err)
-		err = res.Error(location, node, err)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// RestoreTo creates the directories and files in the snapshot below dst.
-// Before an item is created, res.Filter is called.
-func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
-	var err error
-	if !filepath.IsAbs(dst) {
-		dst, err = filepath.Abs(dst)
-		if err != nil {
-			return errors.Wrap(err, "Abs")
-		}
-	}
-
-	idx := NewHardlinkIndex()
-	return res.restoreTo(ctx, dst, string(filepath.Separator), *res.sn.Tree, idx)
-}
-
-// Snapshot returns the snapshot this restorer is configured to use.
-func (res *Restorer) Snapshot() *Snapshot {
-	return res.sn
-}

internal/restorer/restorer.go

@@ -0,0 +1,270 @@
+package restorer
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/restic/restic/internal/crypto"
+	"github.com/restic/restic/internal/errors"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+)
+
+// Restorer is used to restore a snapshot to a directory.
+type Restorer struct {
+	repo restic.Repository
+	sn   *restic.Snapshot
+
+	Error        func(dir string, node *restic.Node, err error) error
+	SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool)
+}
+
+var restorerAbortOnAllErrors = func(str string, node *restic.Node, err error) error { return err }
+
+// NewRestorer creates a restorer preloaded with the content from the snapshot id.
+func NewRestorer(repo restic.Repository, id restic.ID) (*Restorer, error) {
+	r := &Restorer{
+		repo: repo, Error: restorerAbortOnAllErrors,
+		SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true },
+	}
+
+	var err error
+
+	r.sn, err = restic.LoadSnapshot(context.TODO(), repo, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+type treeVisitor struct {
+	enterDir  func(node *restic.Node, target, location string) error
+	visitNode func(node *restic.Node, target, location string) error
+	leaveDir  func(node *restic.Node, target, location string) error
+}
+
+// traverseTree traverses a tree from the repo and calls treeVisitor.
+// target is the path in the file system, location within the snapshot.
+func (res *Restorer) traverseTree(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) error {
+	debug.Log("%v %v %v", target, location, treeID)
+	tree, err := res.repo.LoadTree(ctx, treeID)
+	if err != nil {
+		debug.Log("error loading tree %v: %v", treeID, err)
+		return res.Error(location, nil, err)
+	}
+
+	for _, node := range tree.Nodes {
+
+		// ensure that the node name does not contain anything that refers to a
+		// top-level directory.
+		nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name))
+		if nodeName != node.Name {
+			debug.Log("node %q has invalid name %q", node.Name, nodeName)
+			err := res.Error(location, node, errors.New("node has invalid name"))
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		nodeTarget := filepath.Join(target, nodeName)
+		nodeLocation := filepath.Join(location, nodeName)
+
+		if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
+			debug.Log("target: %v %v", target, nodeTarget)
+			debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
+			err := res.Error(nodeLocation, node, errors.New("node has invalid path"))
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// sockets cannot be restored
+		if node.Type == "socket" {
+			continue
+		}
+
+		selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node)
+		debug.Log("SelectFilter returned %v %v", selectedForRestore, childMayBeSelected)
+
+		sanitizeError := func(err error) error {
+			if err != nil {
+				err = res.Error(nodeTarget, node, err)
+			}
+			return err
+		}
+
+		enteredDir := false
+		if node.Type == "dir" {
+			if node.Subtree == nil {
+				return errors.Errorf("Dir without subtree in tree %v", treeID.Str())
+			}
+
+			// ifedorenko: apparently a dir can be selected explicitly or implicitly when a child is selected
+			// to support implicit selection, visit the directory from within visitor#visitNode
+			if selectedForRestore {
+				enteredDir = true
+				err = sanitizeError(visitor.enterDir(node, nodeTarget, nodeLocation))
+				if err != nil {
+					return err
+				}
+			} else {
+				_visitor := visitor
+				visitor = treeVisitor{
+					enterDir: _visitor.enterDir,
+					visitNode: func(node *restic.Node, nodeTarget, nodeLocation string) error {
+						if !enteredDir {
+							enteredDir = true
+							derr := sanitizeError(_visitor.enterDir(node, nodeTarget, nodeLocation))
+							if derr != nil {
+								return derr
+							}
+						}
+						return _visitor.visitNode(node, nodeTarget, nodeLocation)
+					},
+					leaveDir: _visitor.leaveDir,
+				}
+			}
+
+			if childMayBeSelected {
+				err = sanitizeError(res.traverseTree(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor))
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		if selectedForRestore && node.Type != "dir" {
+			err = sanitizeError(visitor.visitNode(node, nodeTarget, nodeLocation))
+			if err != nil {
+				err = res.Error(nodeLocation, node, errors.Wrap(err, "restoreNodeTo"))
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		if enteredDir {
+			err = sanitizeError(visitor.leaveDir(node, nodeTarget, nodeLocation))
+			if err != nil {
+				err = res.Error(nodeLocation, node, errors.Wrap(err, "RestoreTimestamps"))
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string, idx *restic.HardlinkIndex) error {
+	debug.Log("restoreNode %v %v %v", node.Name, target, location)
+
+	err := node.CreateAt(ctx, target, res.repo, idx)
+	if err != nil {
+		debug.Log("node.CreateAt(%s) error %v", target, err)
+	}
+	if err == nil {
+		err = res.restoreNodeMetadataTo(node, target, location)
+	}
+
+	return err
+}
+
+func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error {
+	debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location)
+	err := node.RestoreMetadata(target)
+	if err != nil {
+		debug.Log("node.RestoreMetadata(%s) error %v", target, err)
+	}
+	return err
+}
+
+// RestoreTo creates the directories and files in the snapshot below dst.
+// Before an item is created, res.Filter is called.
+func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
+	var err error
+	if !filepath.IsAbs(dst) {
+		dst, err = filepath.Abs(dst)
+		if err != nil {
+			return errors.Wrap(err, "Abs")
+		}
+	}
+
+	idx := restic.NewHardlinkIndex()
+	return res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
+		enterDir: func(node *restic.Node, target, location string) error {
+			// create dir with default permissions
+			// #leaveDir restores dir metadata after visiting all children
+			return fs.MkdirAll(target, 0700)
+		},
+		visitNode: func(node *restic.Node, target, location string) error {
+			return res.restoreNodeTo(ctx, node, target, location, idx)
+		},
+		leaveDir: func(node *restic.Node, target, location string) error {
+			// Restore directory permissions and timestamp at the end. If we did it earlier
+			// - children restore could fail because of restictive directory permission
+			// - children restore could overwrite the timestamp of the directory they are in
+			return res.restoreNodeMetadataTo(node, target, location)
+		},
+	})
+}
+
+// Snapshot returns the snapshot this restorer is configured to use.
+func (res *Restorer) Snapshot() *restic.Snapshot {
+	return res.sn
+}
+
+// VerifyFiles reads all snapshot files and verifies their contents
+func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) {
+	// TODO multithreaded?
+
+	count := 0
+	err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
+		enterDir: func(node *restic.Node, target, location string) error { return nil },
+		visitNode: func(node *restic.Node, target, location string) error {
+			if node.Type != "file" {
+				return nil
+			}
+
+			count++
+			stat, err := os.Stat(target)
+			if err != nil {
+				return err
+			}
+			if int64(node.Size) != stat.Size() {
+				return errors.Errorf("Invalid file size: expected %d got %d", node.Size, stat.Size())
+			}
+
+			offset := int64(0)
+			for _, blobID := range node.Content {
+				rd, err := os.Open(target)
+				if err != nil {
+					return err
+				}
+				blobs, _ := res.repo.Index().Lookup(blobID, restic.DataBlob)
+				length := blobs[0].Length - uint(crypto.Extension)
+				buf := make([]byte, length) // TODO do I want to reuse the buffer somehow?
+				_, err = rd.ReadAt(buf, offset)
+				if err != nil {
+					return err
+				}
+				if !blobID.Equal(restic.Hash(buf)) {
+					return errors.Errorf("Unexpected contents starting at offset %d", offset)
+				}
+				offset += int64(length)
+			}
+
+			return nil
+		},
+		leaveDir: func(node *restic.Node, target, location string) error { return nil },
+	})
+
+	return count, err
+}

internal/restorer/restorer_test.go

@@ -1,4 +1,4 @@
-package restic_test
+package restorer_test
 
 import (
 	"bytes"
@@ -13,6 +13,7 @@ import (
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/restorer"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -264,7 +265,7 @@ func TestRestorer(t *testing.T) {
 			_, id := saveSnapshot(t, repo, test.Snapshot)
 			t.Logf("snapshot saved as %v", id.Str())
 
-			res, err := restic.NewRestorer(repo, id)
+			res, err := restorer.NewRestorer(repo, id)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -377,7 +378,7 @@ func TestRestorerRelative(t *testing.T) {
 			_, id := saveSnapshot(t, repo, test.Snapshot)
 			t.Logf("snapshot saved as %v", id.Str())
 
-			res, err := restic.NewRestorer(repo, id)
+			res, err := restorer.NewRestorer(repo, id)
 			if err != nil {
 				t.Fatal(err)
 			}