diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 1a342800a..03a990cc6 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -70,9 +70,6 @@ func init() { func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - hasExcludes := len(opts.Excludes) > 0 || len(opts.InsensitiveExcludes) > 0 - hasIncludes := len(opts.Includes) > 0 || len(opts.InsensitiveIncludes) > 0 - excludePatternFns, err := opts.excludePatternOptions.CollectPatterns() if err != nil { return err @@ -83,6 +80,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return err } + hasExcludes := len(excludePatternFns) > 0 + hasIncludes := len(includePatternFns) > 0 + switch { case len(args) == 0: return errors.Fatal("no snapshot ID specified") @@ -153,6 +153,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, matched := false for _, rejectFn := range excludePatternFns { matched = matched || rejectFn(item) + + // implementing a short-circuit here to improve the performance + // to prevent additional pattern matching once the first pattern + // matches. + if matched { + break + } } // An exclude filter is basically a 'wildcard but foo', // so even if a childMayMatch, other children of a dir may not, @@ -171,9 +178,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, matched, childMayMatch := includeFn(item) selectedForRestore = selectedForRestore || matched childMayBeSelected = childMayBeSelected || childMayMatch - } + childMayBeSelected = childMayBeSelected && node.Type == "dir" - childMayBeSelected = childMayBeSelected && node.Type == "dir" + if selectedForRestore || childMayBeSelected { + break + } + } return selectedForRestore, childMayBeSelected } diff --git a/cmd/restic/cmd_restore_integration_test.go b/cmd/restic/cmd_restore_integration_test.go index 27272fe40..0e1620f7e 100644 --- a/cmd/restic/cmd_restore_integration_test.go +++ b/cmd/restic/cmd_restore_integration_test.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "path/filepath" + "strings" "syscall" "testing" "time" @@ -58,6 +59,67 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) } +func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) { + opts := RestoreOptions{ + Target: dir, + } + opts.IncludeFiles = []string{includesFile} + + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) +} + +func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) { + opts := RestoreOptions{ + Target: dir, + } + opts.ExcludeFiles = []string{excludesFile} + + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) +} + +func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) { + testfiles := []struct { + path string + size uint + }{ + {"dir1/include_me.txt", 100}, + } + + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + // Create test files and directories + for _, testFile := range testfiles { + fullPath := filepath.Join(env.testdata, testFile.path) + rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) + rtest.OK(t, appendRandomData(fullPath, testFile.size)) + } + + opts := BackupOptions{} + // Perform backup + testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) + testRunCheck(t, env.gopts) + + snapshotID := testListSnapshots(t, env.gopts, 1)[0] + + // Add both include and exclude patterns + includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"} + excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"} + + restoredir := filepath.Join(env.base, "restore") + + restoreOpts := RestoreOptions{ + Target: restoredir, + } + restoreOpts.Includes = includePatterns + restoreOpts.Excludes = excludePatterns + + err := testRunRestoreAssumeFailure(snapshotID.String(), restoreOpts, env.gopts) + rtest.Assert(t, err != nil, "restore must fail if include and exclude patterns are provided") +} + func TestRestoreIncludes(t *testing.T) { testfiles := []struct { path string @@ -97,16 +159,33 @@ func TestRestoreIncludes(t *testing.T) { restoredir := filepath.Join(env.base, "restore") testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns) - // Check that only the included files are restored - for _, testFile := range testfiles { - restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path) - _, err := os.Stat(restoredFilePath) - if testFile.include { - rtest.OK(t, err) - } else { - rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path) + testRestoreFileInclusions := func(t *testing.T, env *testEnvironment, includePatterns []string) { + // Check that only the included files are restored + for _, testFile := range testfiles { + restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path) + _, err := os.Stat(restoredFilePath) + if testFile.include { + rtest.OK(t, err) + } else { + rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path) + } } } + + testRestoreFileInclusions(t, env, includePatterns) + + // Create an include file with some patterns + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644) + if fileErr != nil { + t.Fatalf("Could not write include file: %v", fileErr) + } + + restoredir = filepath.Join(env.base, "restore-include-from-file") + + testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile) + + testRestoreFileInclusions(t, env, includePatterns) } func TestRestoreFilter(t *testing.T) { @@ -144,19 +223,38 @@ func TestRestoreFilter(t *testing.T) { rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size))) } - for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} { - base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1)) - testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat}) - for _, testFile := range testfiles { - err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size)) - if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok { - rtest.OK(t, err) - } else { - rtest.Assert(t, os.IsNotExist(err), - "expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err) + excludePatterns := []string{"*.c", "*.exe", "*", "*file3*"} + + testRestoreFileExclusions := func(t *testing.T, env *testEnvironment, excludePatterns []string) { + for i, pat := range excludePatterns { + base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1)) + testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat}) + for _, testFile := range testfiles { + err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size)) + if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok { + rtest.OK(t, err) + } else { + rtest.Assert(t, os.IsNotExist(err), + "expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err) + } } } } + + testRestoreFileExclusions(t, env, excludePatterns) + + // Create an include file with some patterns + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644) + if fileErr != nil { + t.Fatalf("Could not write include file: %v", fileErr) + } + + restoredir := filepath.Join(env.base, "restore-exclude-from-file") + + testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile) + + testRestoreFileExclusions(t, env, excludePatterns) } func TestRestore(t *testing.T) { diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 4657e4915..9f5f40511 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -443,7 +443,7 @@ func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) { f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") + f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of filenames in patterns") } func (opts *excludePatternOptions) Empty() bool { diff --git a/cmd/restic/include.go b/cmd/restic/include.go index dcc4c7f37..64659d98f 100644 --- a/cmd/restic/include.go +++ b/cmd/restic/include.go @@ -23,7 +23,7 @@ func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) { f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames") f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns") + f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of filenames in patterns") } func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) { diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go index 0f547bdc2..dccbcc0a0 100644 --- a/cmd/restic/integration_filter_pattern_test.go +++ b/cmd/restic/integration_filter_pattern_test.go @@ -105,49 +105,28 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { testRunInit(t, env.gopts) // Create an include file with some invalid patterns - includeFile := env.base + "/includefile" - fileErr := os.WriteFile(includeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) if fileErr != nil { t.Fatalf("Could not write include file: %v", fileErr) } - err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{includeFile}}}, env.gopts) + err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - // Create an exclude file with some invalid patterns - excludeFile := env.base + "/excludefile" - fileErr = os.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) - if fileErr != nil { - t.Fatalf("Could not write exclude file: %v", fileErr) - } - - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - // Create an insentive include file with some invalid patterns - insensitiveIncludeFile := env.base + "/insensitiveincludefile" - fileErr = os.WriteFile(insensitiveIncludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) - if fileErr != nil { - t.Fatalf("Could not write insensitive include file: %v", fileErr) - } - - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{insensitiveIncludeFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - // Create an insensitive exclude file with some invalid patterns - insensitiveExcludeFile := env.base + "/insensitiveexcludefile" - fileErr = os.WriteFile(insensitiveExcludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) - if fileErr != nil { - t.Fatalf("Could not write insensitive exclude file: %v", fileErr) - } - - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{insensitiveExcludeFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error())