Merge pull request #4881 from MichaelEischer/restore-delete-actual

restore: add `--delete` option
This commit is contained in:
Michael Eischer 2024-07-05 22:52:54 +02:00 committed by GitHub
commit 8e27a934de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 567 additions and 130 deletions

View file

@ -0,0 +1,10 @@
Enhancement: Add `--delete` option to `restore` command
The `restore` command now supports a `--delete` option that allows removing files and directories
from the target directory that do not exist in the snapshot. This option also allows files in the
snapshot to replace non-empty directories.
To check that only the expected files are deleted add the `--dry-run --verbose=2` options.
https://github.com/restic/restic/issues/2348
https://github.com/restic/restic/pull/4881

View file

@ -51,6 +51,7 @@ type RestoreOptions struct {
Sparse bool Sparse bool
Verify bool Verify bool
Overwrite restorer.OverwriteBehavior Overwrite restorer.OverwriteBehavior
Delete bool
} }
var restoreOptions RestoreOptions var restoreOptions RestoreOptions
@ -69,6 +70,7 @@ func init() {
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
flags.Var(&restoreOptions.Overwrite, "overwrite", "overwrite behavior, one of (always|if-changed|if-newer|never) (default: always)") flags.Var(&restoreOptions.Overwrite, "overwrite", "overwrite behavior, one of (always|if-changed|if-newer|never) (default: always)")
flags.BoolVar(&restoreOptions.Delete, "delete", false, "delete files from target directory if they do not exist in snapshot. Use '--dry-run -vv' to check what would be deleted")
} }
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
@ -149,6 +151,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
Sparse: opts.Sparse, Sparse: opts.Sparse,
Progress: progress, Progress: progress,
Overwrite: opts.Overwrite, Overwrite: opts.Overwrite,
Delete: opts.Delete,
}) })
totalErrors := 0 totalErrors := 0
@ -161,7 +164,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
msg.E("Warning: %s\n", message) msg.E("Warning: %s\n", message)
} }
selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { selectExcludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
matched := false matched := false
for _, rejectFn := range excludePatternFns { for _, rejectFn := range excludePatternFns {
matched = matched || rejectFn(item) matched = matched || rejectFn(item)
@ -178,12 +181,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
// therefore childMayMatch does not matter, but we should not go down // therefore childMayMatch does not matter, but we should not go down
// unless the dir is selected for restore // unless the dir is selected for restore
selectedForRestore = !matched selectedForRestore = !matched
childMayBeSelected = selectedForRestore && node.Type == "dir" childMayBeSelected = selectedForRestore && isDir
return selectedForRestore, childMayBeSelected return selectedForRestore, childMayBeSelected
} }
selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { selectIncludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
selectedForRestore = false selectedForRestore = false
childMayBeSelected = false childMayBeSelected = false
for _, includeFn := range includePatternFns { for _, includeFn := range includePatternFns {
@ -195,7 +198,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
break break
} }
} }
childMayBeSelected = childMayBeSelected && node.Type == "dir" childMayBeSelected = childMayBeSelected && isDir
return selectedForRestore, childMayBeSelected return selectedForRestore, childMayBeSelected
} }

View file

@ -111,6 +111,27 @@ values are supported:
newer modification time (mtime). newer modification time (mtime).
* ``--overwrite never``: never overwrite existing files. * ``--overwrite never``: never overwrite existing files.
Delete files not in snapshot
----------------------------
When restoring into a directory that already contains files, it can be useful to remove all
files that do not exist in the snapshot. For this, pass the ``--delete`` option to the ``restore``
command. The command will then **delete all files** from the target directory that do not
exist in the snapshot.
The ``--delete`` option also allows overwriting a non-empty directory if the snapshot contains a
file with the same name.
.. warning::
Always use the ``--dry-run -vv`` option to verify what would be deleted before running the actual
command.
When specifying ``--include`` or ``--exclude`` options, only files or directories matched by those
options will be deleted. For example, the command
``restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo --delete``
would only delete files within ``/tmp/restore-work/foo``.
Dry run Dry run
------- -------

View file

@ -520,7 +520,7 @@ Only printed if `--verbose=2` is specified.
+----------------------+-----------------------------------------------------------+ +----------------------+-----------------------------------------------------------+
| ``message_type`` | Always "verbose_status" | | ``message_type`` | Always "verbose_status" |
+----------------------+-----------------------------------------------------------+ +----------------------+-----------------------------------------------------------+
| ``action`` | Either "restored", "updated" or "unchanged" | | ``action`` | Either "restored", "updated", "unchanged" or "deleted" |
+----------------------+-----------------------------------------------------------+ +----------------------+-----------------------------------------------------------+
| ``item`` | The item in question | | ``item`` | The item in question |
+----------------------+-----------------------------------------------------------+ +----------------------+-----------------------------------------------------------+

View file

@ -304,7 +304,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi
return FutureNode{}, err return FutureNode{}, err
} }
names, err := readdirnames(arch.FS, dir, fs.O_NOFOLLOW) names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
if err != nil { if err != nil {
return FutureNode{}, err return FutureNode{}, err
} }
@ -707,27 +707,6 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree,
return fn, len(nodes), nil return fn, len(nodes), nil
} }
// flags are passed to fs.OpenFile. O_RDONLY is implied.
func readdirnames(filesystem fs.FS, dir string, flags int) ([]string, error) {
f, err := filesystem.OpenFile(dir, fs.O_RDONLY|flags, 0)
if err != nil {
return nil, errors.WithStack(err)
}
entries, err := f.Readdirnames(-1)
if err != nil {
_ = f.Close()
return nil, errors.Wrapf(err, "Readdirnames %v failed", dir)
}
err = f.Close()
if err != nil {
return nil, err
}
return entries, nil
}
// resolveRelativeTargets replaces targets that only contain relative // resolveRelativeTargets replaces targets that only contain relative
// directories ("." or "../../") with the contents of the directory. Each // directories ("." or "../../") with the contents of the directory. Each
// element of target is processed with fs.Clean(). // element of target is processed with fs.Clean().
@ -743,7 +722,7 @@ func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) {
} }
debug.Log("replacing %q with readdir(%q)", target, target) debug.Log("replacing %q with readdir(%q)", target, target)
entries, err := readdirnames(filesys, target, fs.O_NOFOLLOW) entries, err := fs.Readdirnames(filesys, target, fs.O_NOFOLLOW)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -124,7 +124,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca
stats.Files++ stats.Files++
stats.Bytes += uint64(fi.Size()) stats.Bytes += uint64(fi.Size())
case fi.Mode().IsDir(): case fi.Mode().IsDir():
names, err := readdirnames(s.FS, target, fs.O_NOFOLLOW) names, err := fs.Readdirnames(s.FS, target, fs.O_NOFOLLOW)
if err != nil { if err != nil {
return stats, s.Error(target, err) return stats, s.Error(target, err)
} }

View file

@ -233,7 +233,7 @@ func unrollTree(f fs.FS, t *Tree) error {
// nodes, add the contents of Path to the nodes. // nodes, add the contents of Path to the nodes.
if t.Path != "" && len(t.Nodes) > 0 { if t.Path != "" && len(t.Nodes) > 0 {
debug.Log("resolve path %v", t.Path) debug.Log("resolve path %v", t.Path)
entries, err := readdirnames(f, t.Path, 0) entries, err := fs.Readdirnames(f, t.Path, 0)
if err != nil { if err != nil {
return err return err
} }

View file

@ -1,6 +1,7 @@
package fs package fs
import ( import (
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
@ -138,3 +139,24 @@ func ResetPermissions(path string) error {
} }
return nil return nil
} }
// Readdirnames returns a list of file in a directory. Flags are passed to fs.OpenFile. O_RDONLY is implied.
func Readdirnames(filesystem FS, dir string, flags int) ([]string, error) {
f, err := filesystem.OpenFile(dir, O_RDONLY|flags, 0)
if err != nil {
return nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}
entries, err := f.Readdirnames(-1)
if err != nil {
_ = f.Close()
return nil, fmt.Errorf("readdirnames %v failed: %w", dir, err)
}
err = f.Close()
if err != nil {
return nil, err
}
return entries, nil
}

View file

@ -53,6 +53,8 @@ type fileRestorer struct {
sparse bool sparse bool
progress *restore.Progress progress *restore.Progress
allowRecursiveDelete bool
dst string dst string
files []*fileInfo files []*fileInfo
Error func(string, error) error Error func(string, error) error
@ -63,21 +65,23 @@ func newFileRestorer(dst string,
idx func(restic.BlobType, restic.ID) []restic.PackedBlob, idx func(restic.BlobType, restic.ID) []restic.PackedBlob,
connections uint, connections uint,
sparse bool, sparse bool,
allowRecursiveDelete bool,
progress *restore.Progress) *fileRestorer { progress *restore.Progress) *fileRestorer {
// as packs are streamed the concurrency is limited by IO // as packs are streamed the concurrency is limited by IO
workerCount := int(connections) workerCount := int(connections)
return &fileRestorer{ return &fileRestorer{
idx: idx, idx: idx,
blobsLoader: blobsLoader, blobsLoader: blobsLoader,
filesWriter: newFilesWriter(workerCount), filesWriter: newFilesWriter(workerCount, allowRecursiveDelete),
zeroChunk: repository.ZeroChunk(), zeroChunk: repository.ZeroChunk(),
sparse: sparse, sparse: sparse,
progress: progress, progress: progress,
workerCount: workerCount, allowRecursiveDelete: allowRecursiveDelete,
dst: dst, workerCount: workerCount,
Error: restorerAbortOnAllErrors, dst: dst,
Error: restorerAbortOnAllErrors,
} }
} }
@ -207,7 +211,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
} }
func (r *fileRestorer) restoreEmptyFileAt(location string) error { func (r *fileRestorer) restoreEmptyFileAt(location string) error {
f, err := createFile(r.targetPath(location), 0, false) f, err := createFile(r.targetPath(location), 0, false, r.allowRecursiveDelete)
if err != nil { if err != nil {
return err return err
} }

View file

@ -144,7 +144,7 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma
t.Helper() t.Helper()
repo := newTestRepo(content) repo := newTestRepo(content)
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, nil) r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, nil)
if files == nil { if files == nil {
r.files = repo.files r.files = repo.files
@ -285,7 +285,7 @@ func TestErrorRestoreFiles(t *testing.T) {
return loadError return loadError
} }
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil)
r.files = repo.files r.files = repo.files
err := r.restoreFiles(context.TODO()) err := r.restoreFiles(context.TODO())
@ -326,7 +326,7 @@ func TestFatalDownloadError(t *testing.T) {
}) })
} }
r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil)
r.files = repo.files r.files = repo.files
var errors []string var errors []string

View file

@ -19,7 +19,8 @@ import (
// TODO I am not 100% convinced this is necessary, i.e. it may be okay // TODO I am not 100% convinced this is necessary, i.e. it may be okay
// to use multiple os.File to write to the same target file // to use multiple os.File to write to the same target file
type filesWriter struct { type filesWriter struct {
buckets []filesWriterBucket buckets []filesWriterBucket
allowRecursiveDelete bool
} }
type filesWriterBucket struct { type filesWriterBucket struct {
@ -33,13 +34,14 @@ type partialFile struct {
sparse bool sparse bool
} }
func newFilesWriter(count int) *filesWriter { func newFilesWriter(count int, allowRecursiveDelete bool) *filesWriter {
buckets := make([]filesWriterBucket, count) buckets := make([]filesWriterBucket, count)
for b := 0; b < count; b++ { for b := 0; b < count; b++ {
buckets[b].files = make(map[string]*partialFile) buckets[b].files = make(map[string]*partialFile)
} }
return &filesWriter{ return &filesWriter{
buckets: buckets, buckets: buckets,
allowRecursiveDelete: allowRecursiveDelete,
} }
} }
@ -60,7 +62,7 @@ func openFile(path string) (*os.File, error) {
return f, nil return f, nil
} }
func createFile(path string, createSize int64, sparse bool) (*os.File, error) { func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete bool) (*os.File, error) {
f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
if err != nil && fs.IsAccessDenied(err) { if err != nil && fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the // If file is readonly, clear the readonly flag by resetting the
@ -109,8 +111,14 @@ func createFile(path string, createSize int64, sparse bool) (*os.File, error) {
} }
// not what we expected, try to get rid of it // not what we expected, try to get rid of it
if err := fs.Remove(path); err != nil { if allowRecursiveDelete {
return nil, err if err := fs.RemoveAll(path); err != nil {
return nil, err
}
} else {
if err := fs.Remove(path); err != nil {
return nil, err
}
} }
// create a new file, pass O_EXCL to make sure there are no surprises // create a new file, pass O_EXCL to make sure there are no surprises
f, err = fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_EXCL|fs.O_NOFOLLOW, 0600) f, err = fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_EXCL|fs.O_NOFOLLOW, 0600)
@ -169,7 +177,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
var f *os.File var f *os.File
var err error var err error
if createSize >= 0 { if createSize >= 0 {
f, err = createFile(path, createSize, sparse) f, err = createFile(path, createSize, sparse, w.allowRecursiveDelete)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -13,7 +13,7 @@ import (
func TestFilesWriterBasic(t *testing.T) { func TestFilesWriterBasic(t *testing.T) {
dir := rtest.TempDir(t) dir := rtest.TempDir(t)
w := newFilesWriter(1) w := newFilesWriter(1, false)
f1 := dir + "/f1" f1 := dir + "/f1"
f2 := dir + "/f2" f2 := dir + "/f2"
@ -39,6 +39,29 @@ func TestFilesWriterBasic(t *testing.T) {
rtest.Equals(t, []byte{2, 2}, buf) rtest.Equals(t, []byte{2, 2}, buf)
} }
func TestFilesWriterRecursiveOverwrite(t *testing.T) {
path := filepath.Join(t.TempDir(), "test")
// create filled directory
rtest.OK(t, os.Mkdir(path, 0o700))
rtest.OK(t, os.WriteFile(filepath.Join(path, "file"), []byte("data"), 0o400))
// must error if recursive delete is not allowed
w := newFilesWriter(1, false)
err := w.writeToFile(path, []byte{1}, 0, 2, false)
rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexepected error got %v", err)
rtest.Equals(t, 0, len(w.buckets[0].files))
// must replace directory
w = newFilesWriter(1, true)
rtest.OK(t, w.writeToFile(path, []byte{1, 1}, 0, 2, false))
rtest.Equals(t, 0, len(w.buckets[0].files))
buf, err := os.ReadFile(path)
rtest.OK(t, err)
rtest.Equals(t, []byte{1, 1}, buf)
}
func TestCreateFile(t *testing.T) { func TestCreateFile(t *testing.T) {
basepath := filepath.Join(t.TempDir(), "test") basepath := filepath.Join(t.TempDir(), "test")
@ -110,7 +133,7 @@ func TestCreateFile(t *testing.T) {
for j, test := range tests { for j, test := range tests {
path := basepath + fmt.Sprintf("%v%v", i, j) path := basepath + fmt.Sprintf("%v%v", i, j)
sc.create(t, path) sc.create(t, path)
f, err := createFile(path, test.size, test.isSparse) f, err := createFile(path, test.size, test.isSparse, false)
if sc.err == nil { if sc.err == nil {
rtest.OK(t, err) rtest.OK(t, err)
fi, err := f.Stat() fi, err := f.Stat()
@ -129,3 +152,19 @@ func TestCreateFile(t *testing.T) {
}) })
} }
} }
func TestCreateFileRecursiveDelete(t *testing.T) {
path := filepath.Join(t.TempDir(), "test")
// create filled directory
rtest.OK(t, os.Mkdir(path, 0o700))
rtest.OK(t, os.WriteFile(filepath.Join(path, "file"), []byte("data"), 0o400))
// replace it
f, err := createFile(path, 42, false, true)
rtest.OK(t, err)
fi, err := f.Stat()
rtest.OK(t, err)
rtest.Assert(t, fi.Mode().IsRegular(), "wrong filetype %v", fi.Mode())
rtest.OK(t, f.Close())
}

View file

@ -25,9 +25,11 @@ type Restorer struct {
fileList map[string]bool fileList map[string]bool
Error func(location string, err error) error Error func(location string, err error) error
Warn func(message string) Warn func(message string)
SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) // SelectFilter determines whether the item is selectedForRestore or whether a childMayBeSelected.
// selectedForRestore must not depend on isDir as `removeUnexpectedFiles` always passes false to isDir.
SelectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool)
} }
var restorerAbortOnAllErrors = func(_ string, err error) error { return err } var restorerAbortOnAllErrors = func(_ string, err error) error { return err }
@ -37,6 +39,7 @@ type Options struct {
Sparse bool Sparse bool
Progress *restoreui.Progress Progress *restoreui.Progress
Overwrite OverwriteBehavior Overwrite OverwriteBehavior
Delete bool
} }
type OverwriteBehavior int type OverwriteBehavior int
@ -97,7 +100,7 @@ func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Res
opts: opts, opts: opts,
fileList: make(map[string]bool), fileList: make(map[string]bool),
Error: restorerAbortOnAllErrors, Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true }, SelectFilter: func(string, bool) (bool, bool) { return true, true },
sn: sn, sn: sn,
} }
@ -107,20 +110,61 @@ func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Res
type treeVisitor struct { type treeVisitor struct {
enterDir func(node *restic.Node, target, location string) error enterDir func(node *restic.Node, target, location string) error
visitNode func(node *restic.Node, target, location string) error visitNode func(node *restic.Node, target, location string) error
leaveDir func(node *restic.Node, target, location string) error // 'entries' contains all files the snapshot contains for this node. This also includes files
// ignored by the SelectFilter.
leaveDir func(node *restic.Node, target, location string, entries []string) error
} }
// traverseTree traverses a tree from the repo and calls treeVisitor. // traverseTree traverses a tree from the repo and calls treeVisitor.
// target is the path in the file system, location within the snapshot. // target is the path in the file system, location within the snapshot.
func (res *Restorer) traverseTree(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (hasRestored bool, err error) { func (res *Restorer) traverseTree(ctx context.Context, target string, treeID restic.ID, visitor treeVisitor) error {
location := string(filepath.Separator)
sanitizeError := func(err error) error {
switch err {
case nil, context.Canceled, context.DeadlineExceeded:
// Context errors are permanent.
return err
default:
return res.Error(location, err)
}
}
if visitor.enterDir != nil {
err := sanitizeError(visitor.enterDir(nil, target, location))
if err != nil {
return err
}
}
childFilenames, hasRestored, err := res.traverseTreeInner(ctx, target, location, treeID, visitor)
if err != nil {
return err
}
if hasRestored && visitor.leaveDir != nil {
err = sanitizeError(visitor.leaveDir(nil, target, location, childFilenames))
}
return err
}
func (res *Restorer) traverseTreeInner(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (filenames []string, hasRestored bool, err error) {
debug.Log("%v %v %v", target, location, treeID) debug.Log("%v %v %v", target, location, treeID)
tree, err := restic.LoadTree(ctx, res.repo, treeID) tree, err := restic.LoadTree(ctx, res.repo, treeID)
if err != nil { if err != nil {
debug.Log("error loading tree %v: %v", treeID, err) debug.Log("error loading tree %v: %v", treeID, err)
return hasRestored, res.Error(location, err) return nil, hasRestored, res.Error(location, err)
} }
for _, node := range tree.Nodes { if res.opts.Delete {
filenames = make([]string, 0, len(tree.Nodes))
}
for i, node := range tree.Nodes {
// allow GC of tree node
tree.Nodes[i] = nil
if res.opts.Delete {
// just track all files included in the tree node to simplify the control flow.
// tracking too many files does not matter except for a slightly elevated memory usage
filenames = append(filenames, node.Name)
}
// ensure that the node name does not contain anything that refers to a // ensure that the node name does not contain anything that refers to a
// top-level directory. // top-level directory.
@ -129,8 +173,10 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
debug.Log("node %q has invalid name %q", node.Name, nodeName) debug.Log("node %q has invalid name %q", node.Name, nodeName)
err := res.Error(location, errors.Errorf("invalid child node name %s", node.Name)) err := res.Error(location, errors.Errorf("invalid child node name %s", node.Name))
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
// force disable deletion to prevent unexpected behavior
res.opts.Delete = false
continue continue
} }
@ -142,8 +188,10 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
debug.Log("node %q has invalid target path %q", node.Name, nodeTarget) debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
err := res.Error(nodeLocation, errors.New("node has invalid path")) err := res.Error(nodeLocation, errors.New("node has invalid path"))
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
// force disable deletion to prevent unexpected behavior
res.opts.Delete = false
continue continue
} }
@ -152,7 +200,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
continue continue
} }
selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node) selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, node.Type == "dir")
debug.Log("SelectFilter returned %v %v for %q", selectedForRestore, childMayBeSelected, nodeLocation) debug.Log("SelectFilter returned %v %v for %q", selectedForRestore, childMayBeSelected, nodeLocation)
if selectedForRestore { if selectedForRestore {
@ -171,25 +219,26 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
if node.Type == "dir" { if node.Type == "dir" {
if node.Subtree == nil { if node.Subtree == nil {
return hasRestored, errors.Errorf("Dir without subtree in tree %v", treeID.Str()) return nil, hasRestored, errors.Errorf("Dir without subtree in tree %v", treeID.Str())
} }
if selectedForRestore && visitor.enterDir != nil { if selectedForRestore && visitor.enterDir != nil {
err = sanitizeError(visitor.enterDir(node, nodeTarget, nodeLocation)) err = sanitizeError(visitor.enterDir(node, nodeTarget, nodeLocation))
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
} }
// keep track of restored child status // keep track of restored child status
// so metadata of the current directory are restored on leaveDir // so metadata of the current directory are restored on leaveDir
childHasRestored := false childHasRestored := false
var childFilenames []string
if childMayBeSelected { if childMayBeSelected {
childHasRestored, err = res.traverseTree(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor) childFilenames, childHasRestored, err = res.traverseTreeInner(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor)
err = sanitizeError(err) err = sanitizeError(err)
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
// inform the parent directory to restore parent metadata on leaveDir if needed // inform the parent directory to restore parent metadata on leaveDir if needed
if childHasRestored { if childHasRestored {
@ -200,9 +249,9 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
// metadata need to be restore when leaving the directory in both cases // metadata need to be restore when leaving the directory in both cases
// selected for restore or any child of any subtree have been restored // selected for restore or any child of any subtree have been restored
if (selectedForRestore || childHasRestored) && visitor.leaveDir != nil { if (selectedForRestore || childHasRestored) && visitor.leaveDir != nil {
err = sanitizeError(visitor.leaveDir(node, nodeTarget, nodeLocation)) err = sanitizeError(visitor.leaveDir(node, nodeTarget, nodeLocation, childFilenames))
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
} }
@ -212,12 +261,12 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
if selectedForRestore { if selectedForRestore {
err = sanitizeError(visitor.visitNode(node, nodeTarget, nodeLocation)) err = sanitizeError(visitor.visitNode(node, nodeTarget, nodeLocation))
if err != nil { if err != nil {
return hasRestored, err return nil, hasRestored, err
} }
} }
} }
return hasRestored, nil return filenames, hasRestored, nil
} }
func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string) error { func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string) error {
@ -300,7 +349,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
idx := NewHardlinkIndex[string]() idx := NewHardlinkIndex[string]()
filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob, filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob,
res.repo.Connections(), res.opts.Sparse, res.opts.Progress) res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.opts.Progress)
filerestorer.Error = res.Error filerestorer.Error = res.Error
debug.Log("first pass for %q", dst) debug.Log("first pass for %q", dst)
@ -308,10 +357,12 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
var buf []byte var buf []byte
// first tree pass: create directories and collect all files to restore // first tree pass: create directories and collect all files to restore
_, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{
enterDir: func(_ *restic.Node, target, location string) error { enterDir: func(_ *restic.Node, target, location string) error {
debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location)
res.opts.Progress.AddFile(0) if location != "/" {
res.opts.Progress.AddFile(0)
}
return res.ensureDir(target) return res.ensureDir(target)
}, },
@ -371,7 +422,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
debug.Log("second pass for %q", dst) debug.Log("second pass for %q", dst)
// second tree pass: restore special files and filesystem metadata // second tree pass: restore special files and filesystem metadata
_, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{
visitNode: func(node *restic.Node, target, location string) error { visitNode: func(node *restic.Node, target, location string) error {
debug.Log("second pass, visitNode: restore node %q", location) debug.Log("second pass, visitNode: restore node %q", location)
if node.Type != "file" { if node.Type != "file" {
@ -394,7 +445,17 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
// don't touch skipped files // don't touch skipped files
return nil return nil
}, },
leaveDir: func(node *restic.Node, target, location string) error { leaveDir: func(node *restic.Node, target, location string, expectedFilenames []string) error {
if res.opts.Delete {
if err := res.removeUnexpectedFiles(target, location, expectedFilenames); err != nil {
return err
}
}
if node == nil {
return nil
}
err := res.restoreNodeMetadataTo(node, target, location) err := res.restoreNodeMetadataTo(node, target, location)
if err == nil { if err == nil {
res.opts.Progress.AddProgress(location, restoreui.ActionDirRestored, 0, 0) res.opts.Progress.AddProgress(location, restoreui.ActionDirRestored, 0, 0)
@ -405,6 +466,51 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
return err return err
} }
func (res *Restorer) removeUnexpectedFiles(target, location string, expectedFilenames []string) error {
if !res.opts.Delete {
panic("internal error")
}
entries, err := fs.Readdirnames(fs.Local{}, target, fs.O_NOFOLLOW)
if errors.Is(err, os.ErrNotExist) {
return nil
} else if err != nil {
return err
}
keep := map[string]struct{}{}
for _, name := range expectedFilenames {
keep[toComparableFilename(name)] = struct{}{}
}
for _, entry := range entries {
if _, ok := keep[toComparableFilename(entry)]; ok {
continue
}
nodeTarget := filepath.Join(target, entry)
nodeLocation := filepath.Join(location, entry)
if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
return fmt.Errorf("skipping deletion due to invalid filename: %v", entry)
}
// TODO pass a proper value to the isDir parameter once this becomes relevant for the filters
selectedForRestore, _ := res.SelectFilter(nodeLocation, false)
// only delete files that were selected for restore
if selectedForRestore {
res.opts.Progress.ReportDeletedFile(nodeLocation)
if !res.opts.DryRun {
if err := fs.RemoveAll(nodeTarget); err != nil {
return err
}
}
}
}
return nil
}
func (res *Restorer) trackFile(location string, metadataOnly bool) { func (res *Restorer) trackFile(location string, metadataOnly bool) {
res.fileList[location] = metadataOnly res.fileList[location] = metadataOnly
} }
@ -491,7 +597,7 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) {
g.Go(func() error { g.Go(func() error {
defer close(work) defer close(work)
_, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ err := res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{
visitNode: func(node *restic.Node, target, location string) error { visitNode: func(node *restic.Node, target, location string) error {
if node.Type != "file" { if node.Type != "file" {
return nil return nil

View file

@ -8,6 +8,7 @@ import (
"math" "math"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"runtime" "runtime"
"strings" "strings"
"syscall" "syscall"
@ -192,7 +193,7 @@ func TestRestorer(t *testing.T) {
Files map[string]string Files map[string]string
ErrorsMust map[string]map[string]struct{} ErrorsMust map[string]map[string]struct{}
ErrorsMay map[string]map[string]struct{} ErrorsMay map[string]map[string]struct{}
Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) Select func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool)
}{ }{
// valid test cases // valid test cases
{ {
@ -284,7 +285,7 @@ func TestRestorer(t *testing.T) {
Files: map[string]string{ Files: map[string]string{
"dir/file": "content: file\n", "dir/file": "content: file\n",
}, },
Select: func(item, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
switch item { switch item {
case filepath.FromSlash("/dir"): case filepath.FromSlash("/dir"):
childMayBeSelected = true childMayBeSelected = true
@ -370,16 +371,10 @@ func TestRestorer(t *testing.T) {
// make sure we're creating a new subdir of the tempdir // make sure we're creating a new subdir of the tempdir
tempdir = filepath.Join(tempdir, "target") tempdir = filepath.Join(tempdir, "target")
res.SelectFilter = func(item, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { res.SelectFilter = func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
t.Logf("restore %v to %v", item, dstpath) t.Logf("restore %v", item)
if !fs.HasPathPrefix(tempdir, dstpath) {
t.Errorf("would restore %v to %v, which is not within the target dir %v",
item, dstpath, tempdir)
return false, false
}
if test.Select != nil { if test.Select != nil {
return test.Select(item, dstpath, node) return test.Select(item, isDir)
} }
return true, true return true, true
@ -527,16 +522,17 @@ func TestRestorerRelative(t *testing.T) {
type TraverseTreeCheck func(testing.TB) treeVisitor type TraverseTreeCheck func(testing.TB) treeVisitor
type TreeVisit struct { type TreeVisit struct {
funcName string // name of the function funcName string // name of the function
location string // location passed to the function location string // location passed to the function
files []string // file list passed to the function
} }
func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { func checkVisitOrder(list []TreeVisit) TraverseTreeCheck {
var pos int var pos int
return func(t testing.TB) treeVisitor { return func(t testing.TB) treeVisitor {
check := func(funcName string) func(*restic.Node, string, string) error { check := func(funcName string) func(*restic.Node, string, string, []string) error {
return func(node *restic.Node, target, location string) error { return func(node *restic.Node, target, location string, expectedFilenames []string) error {
if pos >= len(list) { if pos >= len(list) {
t.Errorf("step %v, %v(%v): expected no more than %d function calls", pos, funcName, location, len(list)) t.Errorf("step %v, %v(%v): expected no more than %d function calls", pos, funcName, location, len(list))
pos++ pos++
@ -554,14 +550,24 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck {
t.Errorf("step %v: want location %v, got %v", pos, list[pos].location, location) t.Errorf("step %v: want location %v, got %v", pos, list[pos].location, location)
} }
if !reflect.DeepEqual(expectedFilenames, v.files) {
t.Errorf("step %v: want files %v, got %v", pos, list[pos].files, expectedFilenames)
}
pos++ pos++
return nil return nil
} }
} }
checkNoFilename := func(funcName string) func(*restic.Node, string, string) error {
f := check(funcName)
return func(node *restic.Node, target, location string) error {
return f(node, target, location, nil)
}
}
return treeVisitor{ return treeVisitor{
enterDir: check("enterDir"), enterDir: checkNoFilename("enterDir"),
visitNode: check("visitNode"), visitNode: checkNoFilename("visitNode"),
leaveDir: check("leaveDir"), leaveDir: check("leaveDir"),
} }
} }
@ -570,7 +576,7 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck {
func TestRestorerTraverseTree(t *testing.T) { func TestRestorerTraverseTree(t *testing.T) {
var tests = []struct { var tests = []struct {
Snapshot Snapshot
Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) Select func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool)
Visitor TraverseTreeCheck Visitor TraverseTreeCheck
}{ }{
{ {
@ -586,17 +592,19 @@ func TestRestorerTraverseTree(t *testing.T) {
"foo": File{Data: "content: foo\n"}, "foo": File{Data: "content: foo\n"},
}, },
}, },
Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) {
return true, true return true, true
}, },
Visitor: checkVisitOrder([]TreeVisit{ Visitor: checkVisitOrder([]TreeVisit{
{"enterDir", "/dir"}, {"enterDir", "/", nil},
{"visitNode", "/dir/otherfile"}, {"enterDir", "/dir", nil},
{"enterDir", "/dir/subdir"}, {"visitNode", "/dir/otherfile", nil},
{"visitNode", "/dir/subdir/file"}, {"enterDir", "/dir/subdir", nil},
{"leaveDir", "/dir/subdir"}, {"visitNode", "/dir/subdir/file", nil},
{"leaveDir", "/dir"}, {"leaveDir", "/dir/subdir", []string{"file"}},
{"visitNode", "/foo"}, {"leaveDir", "/dir", []string{"otherfile", "subdir"}},
{"visitNode", "/foo", nil},
{"leaveDir", "/", []string{"dir", "foo"}},
}), }),
}, },
@ -613,14 +621,16 @@ func TestRestorerTraverseTree(t *testing.T) {
"foo": File{Data: "content: foo\n"}, "foo": File{Data: "content: foo\n"},
}, },
}, },
Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) {
if item == "/foo" { if item == "/foo" {
return true, false return true, false
} }
return false, false return false, false
}, },
Visitor: checkVisitOrder([]TreeVisit{ Visitor: checkVisitOrder([]TreeVisit{
{"visitNode", "/foo"}, {"enterDir", "/", nil},
{"visitNode", "/foo", nil},
{"leaveDir", "/", []string{"dir", "foo"}},
}), }),
}, },
{ {
@ -635,14 +645,16 @@ func TestRestorerTraverseTree(t *testing.T) {
}}, }},
}, },
}, },
Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) {
if item == "/aaa" { if item == "/aaa" {
return true, false return true, false
} }
return false, false return false, false
}, },
Visitor: checkVisitOrder([]TreeVisit{ Visitor: checkVisitOrder([]TreeVisit{
{"visitNode", "/aaa"}, {"enterDir", "/", nil},
{"visitNode", "/aaa", nil},
{"leaveDir", "/", []string{"aaa", "dir"}},
}), }),
}, },
@ -659,19 +671,21 @@ func TestRestorerTraverseTree(t *testing.T) {
"foo": File{Data: "content: foo\n"}, "foo": File{Data: "content: foo\n"},
}, },
}, },
Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) {
if strings.HasPrefix(item, "/dir") { if strings.HasPrefix(item, "/dir") {
return true, true return true, true
} }
return false, false return false, false
}, },
Visitor: checkVisitOrder([]TreeVisit{ Visitor: checkVisitOrder([]TreeVisit{
{"enterDir", "/dir"}, {"enterDir", "/", nil},
{"visitNode", "/dir/otherfile"}, {"enterDir", "/dir", nil},
{"enterDir", "/dir/subdir"}, {"visitNode", "/dir/otherfile", nil},
{"visitNode", "/dir/subdir/file"}, {"enterDir", "/dir/subdir", nil},
{"leaveDir", "/dir/subdir"}, {"visitNode", "/dir/subdir/file", nil},
{"leaveDir", "/dir"}, {"leaveDir", "/dir/subdir", []string{"file"}},
{"leaveDir", "/dir", []string{"otherfile", "subdir"}},
{"leaveDir", "/", []string{"dir", "foo"}},
}), }),
}, },
@ -688,7 +702,7 @@ func TestRestorerTraverseTree(t *testing.T) {
"foo": File{Data: "content: foo\n"}, "foo": File{Data: "content: foo\n"},
}, },
}, },
Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) {
switch item { switch item {
case "/dir": case "/dir":
return false, true return false, true
@ -699,8 +713,10 @@ func TestRestorerTraverseTree(t *testing.T) {
} }
}, },
Visitor: checkVisitOrder([]TreeVisit{ Visitor: checkVisitOrder([]TreeVisit{
{"visitNode", "/dir/otherfile"}, {"enterDir", "/", nil},
{"leaveDir", "/dir"}, {"visitNode", "/dir/otherfile", nil},
{"leaveDir", "/dir", []string{"otherfile", "subdir"}},
{"leaveDir", "/", []string{"dir", "foo"}},
}), }),
}, },
} }
@ -710,7 +726,8 @@ func TestRestorerTraverseTree(t *testing.T) {
repo := repository.TestRepository(t) repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, Options{}) // set Delete option to enable tracking filenames in a directory
res := NewRestorer(repo, sn, Options{Delete: true})
res.SelectFilter = test.Select res.SelectFilter = test.Select
@ -721,7 +738,7 @@ func TestRestorerTraverseTree(t *testing.T) {
// make sure we're creating a new subdir of the tempdir // make sure we're creating a new subdir of the tempdir
target := filepath.Join(tempdir, "target") target := filepath.Join(tempdir, "target")
_, err := res.traverseTree(ctx, target, string(filepath.Separator), *sn.Tree, test.Visitor(t)) err := res.traverseTree(ctx, target, *sn.Tree, test.Visitor(t))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -788,7 +805,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
res := NewRestorer(repo, sn, Options{}) res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { res.SelectFilter = func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
switch filepath.ToSlash(item) { switch filepath.ToSlash(item) {
case "/dir": case "/dir":
childMayBeSelected = true childMayBeSelected = true
@ -1196,3 +1213,162 @@ func TestRestoreDryRun(t *testing.T) {
_, err := os.Stat(tempdir) _, err := os.Stat(tempdir)
rtest.Assert(t, errors.Is(err, os.ErrNotExist), "expected no file to be created, got %v", err) rtest.Assert(t, errors.Is(err, os.ErrNotExist), "expected no file to be created, got %v", err)
} }
func TestRestoreDryRunDelete(t *testing.T) {
snapshot := Snapshot{
Nodes: map[string]Node{
"foo": File{Data: "content: foo\n"},
},
}
repo := repository.TestRepository(t)
tempdir := filepath.Join(rtest.TempDir(t), "target")
tempfile := filepath.Join(tempdir, "existing")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rtest.OK(t, os.Mkdir(tempdir, 0o755))
f, err := os.Create(tempfile)
rtest.OK(t, err)
rtest.OK(t, f.Close())
sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, Options{DryRun: true, Delete: true})
rtest.OK(t, res.RestoreTo(ctx, tempdir))
_, err = os.Stat(tempfile)
rtest.Assert(t, err == nil, "expected file to still exist, got error %v", err)
}
func TestRestoreOverwriteDirectory(t *testing.T) {
saveSnapshotsAndOverwrite(t,
Snapshot{
Nodes: map[string]Node{
"dir": Dir{
Mode: normalizeFileMode(0755 | os.ModeDir),
Nodes: map[string]Node{
"anotherfile": File{Data: "content: file\n"},
},
},
},
},
Snapshot{
Nodes: map[string]Node{
"dir": File{Data: "content: file\n"},
},
},
Options{Delete: true},
)
}
func TestRestoreDelete(t *testing.T) {
repo := repository.TestRepository(t)
tempdir := rtest.TempDir(t)
sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"dir": Dir{
Mode: normalizeFileMode(0755 | os.ModeDir),
Nodes: map[string]Node{
"file1": File{Data: "content: file\n"},
"anotherfile": File{Data: "content: file\n"},
},
},
"dir2": Dir{
Mode: normalizeFileMode(0755 | os.ModeDir),
Nodes: map[string]Node{
"anotherfile": File{Data: "content: file\n"},
},
},
"anotherfile": File{Data: "content: file\n"},
},
}, noopGetGenericAttributes)
// should delete files that no longer exist in the snapshot
deleteSn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"dir": Dir{
Mode: normalizeFileMode(0755 | os.ModeDir),
Nodes: map[string]Node{
"file1": File{Data: "content: file\n"},
},
},
},
}, noopGetGenericAttributes)
tests := []struct {
selectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool)
fileState map[string]bool
}{
{
selectFilter: nil,
fileState: map[string]bool{
"dir": true,
filepath.Join("dir", "anotherfile"): false,
filepath.Join("dir", "file1"): true,
"dir2": false,
filepath.Join("dir2", "anotherfile"): false,
"anotherfile": false,
},
},
{
selectFilter: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
return false, false
},
fileState: map[string]bool{
"dir": true,
filepath.Join("dir", "anotherfile"): true,
filepath.Join("dir", "file1"): true,
"dir2": true,
filepath.Join("dir2", "anotherfile"): true,
"anotherfile": true,
},
},
{
selectFilter: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
switch item {
case filepath.FromSlash("/dir"):
selectedForRestore = true
case filepath.FromSlash("/dir2"):
selectedForRestore = true
}
return
},
fileState: map[string]bool{
"dir": true,
filepath.Join("dir", "anotherfile"): true,
filepath.Join("dir", "file1"): true,
"dir2": false,
filepath.Join("dir2", "anotherfile"): false,
"anotherfile": true,
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
res := NewRestorer(repo, sn, Options{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
res = NewRestorer(repo, deleteSn, Options{Delete: true})
if test.selectFilter != nil {
res.SelectFilter = test.selectFilter
}
err = res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
for fn, shouldExist := range test.fileState {
_, err := os.Stat(filepath.Join(tempdir, fn))
if shouldExist {
rtest.OK(t, err)
} else {
rtest.Assert(t, errors.Is(err, os.ErrNotExist), "file %v: unexpected error got %v, expected ErrNotExist", fn, err)
}
}
})
}
}

View file

@ -0,0 +1,10 @@
//go:build !windows
// +build !windows
package restorer
// toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the
// uppercase version of the string. On all other systems, it returns the unmodified filename.
func toComparableFilename(path string) string {
return path
}

View file

@ -13,7 +13,6 @@ import (
"time" "time"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
restoreui "github.com/restic/restic/internal/ui/restore" restoreui "github.com/restic/restic/internal/ui/restore"
) )
@ -34,10 +33,6 @@ func TestRestorerRestoreEmptyHardlinkedFields(t *testing.T) {
res := NewRestorer(repo, sn, Options{}) res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true
}
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -108,9 +103,6 @@ func testRestorerProgressBar(t *testing.T, dryRun bool) {
mock := &printerMock{} mock := &printerMock{}
progress := restoreui.NewProgress(mock, 0) progress := restoreui.NewProgress(mock, 0)
res := NewRestorer(repo, sn, Options{Progress: progress, DryRun: dryRun}) res := NewRestorer(repo, sn, Options{Progress: progress, DryRun: dryRun})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true
}
tempdir := rtest.TempDir(t) tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())

View file

@ -0,0 +1,13 @@
//go:build windows
// +build windows
package restorer
import "strings"
// toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the
// uppercase version of the string. On all other systems, it returns the unmodified filename.
func toComparableFilename(path string) string {
// apparently NTFS internally uppercases filenames for comparision
return strings.ToUpper(path)
}

View file

@ -9,6 +9,7 @@ import (
"math" "math"
"os" "os"
"path" "path"
"path/filepath"
"syscall" "syscall"
"testing" "testing"
"time" "time"
@ -539,3 +540,36 @@ func TestDirAttributeCombinationsOverwrite(t *testing.T) {
} }
} }
} }
func TestRestoreDeleteCaseInsensitive(t *testing.T) {
repo := repository.TestRepository(t)
tempdir := rtest.TempDir(t)
sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"anotherfile": File{Data: "content: file\n"},
},
}, noopGetGenericAttributes)
// should delete files that no longer exist in the snapshot
deleteSn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"AnotherfilE": File{Data: "content: file\n"},
},
}, noopGetGenericAttributes)
res := NewRestorer(repo, sn, Options{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
res = NewRestorer(repo, deleteSn, Options{Delete: true})
err = res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
// anotherfile must still exist
_, err = os.Stat(filepath.Join(tempdir, "anotherfile"))
rtest.OK(t, err)
}

View file

@ -56,6 +56,8 @@ func (t *jsonPrinter) CompleteItem(messageType ItemAction, item string, size uin
action = "updated" action = "updated"
case ActionFileUnchanged: case ActionFileUnchanged:
action = "unchanged" action = "unchanged"
case ActionDeleted:
action = "deleted"
default: default:
panic("unknown message type") panic("unknown message type")
} }

View file

@ -53,6 +53,7 @@ func TestJSONPrintCompleteItem(t *testing.T) {
{ActionFileRestored, 123, "{\"message_type\":\"verbose_status\",\"action\":\"restored\",\"item\":\"test\",\"size\":123}\n"}, {ActionFileRestored, 123, "{\"message_type\":\"verbose_status\",\"action\":\"restored\",\"item\":\"test\",\"size\":123}\n"},
{ActionFileUpdated, 123, "{\"message_type\":\"verbose_status\",\"action\":\"updated\",\"item\":\"test\",\"size\":123}\n"}, {ActionFileUpdated, 123, "{\"message_type\":\"verbose_status\",\"action\":\"updated\",\"item\":\"test\",\"size\":123}\n"},
{ActionFileUnchanged, 123, "{\"message_type\":\"verbose_status\",\"action\":\"unchanged\",\"item\":\"test\",\"size\":123}\n"}, {ActionFileUnchanged, 123, "{\"message_type\":\"verbose_status\",\"action\":\"unchanged\",\"item\":\"test\",\"size\":123}\n"},
{ActionDeleted, 0, "{\"message_type\":\"verbose_status\",\"action\":\"deleted\",\"item\":\"test\",\"size\":0}\n"},
} { } {
term, printer := createJSONProgress() term, printer := createJSONProgress()
printer.CompleteItem(data.action, "test", data.size) printer.CompleteItem(data.action, "test", data.size)

View file

@ -51,6 +51,7 @@ const (
ActionFileRestored ItemAction = "file restored" ActionFileRestored ItemAction = "file restored"
ActionFileUpdated ItemAction = "file updated" ActionFileUpdated ItemAction = "file updated"
ActionFileUnchanged ItemAction = "file unchanged" ActionFileUnchanged ItemAction = "file unchanged"
ActionDeleted ItemAction = "deleted"
) )
func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
@ -126,6 +127,17 @@ func (p *Progress) AddSkippedFile(name string, size uint64) {
p.printer.CompleteItem(ActionFileUnchanged, name, size) p.printer.CompleteItem(ActionFileUnchanged, name, size)
} }
func (p *Progress) ReportDeletedFile(name string) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
p.printer.CompleteItem(ActionDeleted, name, 0)
}
func (p *Progress) Finish() { func (p *Progress) Finish() {
p.updater.Done() p.updater.Done()
} }

View file

@ -181,10 +181,12 @@ func TestProgressTypes(t *testing.T) {
progress.AddFile(0) progress.AddFile(0)
progress.AddProgress("dir", ActionDirRestored, fileSize, fileSize) progress.AddProgress("dir", ActionDirRestored, fileSize, fileSize)
progress.AddProgress("new", ActionFileRestored, 0, 0) progress.AddProgress("new", ActionFileRestored, 0, 0)
progress.ReportDeletedFile("del")
return true return true
}) })
test.Equals(t, itemTrace{ test.Equals(t, itemTrace{
itemTraceEntry{ActionDirRestored, "dir", fileSize}, itemTraceEntry{ActionDirRestored, "dir", fileSize},
itemTraceEntry{ActionFileRestored, "new", 0}, itemTraceEntry{ActionFileRestored, "new", 0},
itemTraceEntry{ActionDeleted, "del", 0},
}, items) }, items)
} }

View file

@ -48,12 +48,14 @@ func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uin
action = "updated" action = "updated"
case ActionFileUnchanged: case ActionFileUnchanged:
action = "unchanged" action = "unchanged"
case ActionDeleted:
action = "deleted"
default: default:
panic("unknown message type") panic("unknown message type")
} }
if messageType == ActionDirRestored { if messageType == ActionDirRestored || messageType == ActionDeleted {
t.terminal.Print(fmt.Sprintf("restored %v", item)) t.terminal.Print(fmt.Sprintf("%-9v %v", action, item))
} else { } else {
t.terminal.Print(fmt.Sprintf("%-9v %v with size %v", action, item, ui.FormatBytes(size))) t.terminal.Print(fmt.Sprintf("%-9v %v with size %v", action, item, ui.FormatBytes(size)))
} }

View file

@ -65,6 +65,7 @@ func TestPrintCompleteItem(t *testing.T) {
{ActionFileRestored, 123, "restored test with size 123 B"}, {ActionFileRestored, 123, "restored test with size 123 B"},
{ActionFileUpdated, 123, "updated test with size 123 B"}, {ActionFileUpdated, 123, "updated test with size 123 B"},
{ActionFileUnchanged, 123, "unchanged test with size 123 B"}, {ActionFileUnchanged, 123, "unchanged test with size 123 B"},
{ActionDeleted, 0, "deleted test"},
} { } {
term, printer := createTextProgress() term, printer := createTextProgress()
printer.CompleteItem(data.action, "test", data.size) printer.CompleteItem(data.action, "test", data.size)