KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

fix: include and exclude logic, add tests for include file and exclude file

Browse source
This commit is contained in:
Srigovind Nayak 2024-06-09 19:07:23 +05:30
parent 24a247a0dc
commit 188684ee9e
No known key found for this signature in database
GPG key ID: 3C4A72A34ABD4C43
5 changed files with 139 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cmd/restic/cmd_restore.go
View file

@ -70,9 +70,6 @@ func init() {
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
term *termstatus.Terminal, args []string) error {
hasExcludes := len(opts.Excludes) > 0 || len(opts.InsensitiveExcludes) > 0
hasIncludes := len(opts.Includes) > 0 || len(opts.InsensitiveIncludes) > 0
excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
if err != nil {
return err
@ -83,6 +80,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
return err
}
hasExcludes := len(excludePatternFns) > 0
hasIncludes := len(includePatternFns) > 0
switch {
case len(args) == 0:
return errors.Fatal("no snapshot ID specified")
@ -153,6 +153,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
matched := false
for _, rejectFn := range excludePatternFns {
matched = matched || rejectFn(item)
// implementing a short-circuit here to improve the performance
// to prevent additional pattern matching once the first pattern
// matches.
if matched {
break
}
}
// An exclude filter is basically a 'wildcard but foo',
// so even if a childMayMatch, other children of a dir may not,
@ -171,9 +178,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
matched, childMayMatch := includeFn(item)
selectedForRestore = selectedForRestore || matched
childMayBeSelected = childMayBeSelected || childMayMatch
}
childMayBeSelected = childMayBeSelected && node.Type == "dir"
childMayBeSelected = childMayBeSelected && node.Type == "dir"
if selectedForRestore || childMayBeSelected {
break
}
}
return selectedForRestore, childMayBeSelected
}

134
cmd/restic/cmd_restore_integration_test.go
View file

@ -7,6 +7,7 @@ import (
"math/rand"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
@ -58,6 +59,67 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) {
opts := RestoreOptions{
Target: dir,
}
opts.IncludeFiles = []string{includesFile}
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) {
opts := RestoreOptions{
Target: dir,
}
opts.ExcludeFiles = []string{excludesFile}
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
testfiles := []struct {
path string
size uint
}{
{"dir1/include_me.txt", 100},
}
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
// Create test files and directories
for _, testFile := range testfiles {
fullPath := filepath.Join(env.testdata, testFile.path)
rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
rtest.OK(t, appendRandomData(fullPath, testFile.size))
}
opts := BackupOptions{}
// Perform backup
testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
testRunCheck(t, env.gopts)
snapshotID := testListSnapshots(t, env.gopts, 1)[0]
// Add both include and exclude patterns
includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
restoredir := filepath.Join(env.base, "restore")
restoreOpts := RestoreOptions{
Target: restoredir,
}
restoreOpts.Includes = includePatterns
restoreOpts.Excludes = excludePatterns
err := testRunRestoreAssumeFailure(snapshotID.String(), restoreOpts, env.gopts)
rtest.Assert(t, err != nil, "restore must fail if include and exclude patterns are provided")
}
func TestRestoreIncludes(t *testing.T) {
testfiles := []struct {
path string
@ -97,16 +159,33 @@ func TestRestoreIncludes(t *testing.T) {
restoredir := filepath.Join(env.base, "restore")
testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns)
// Check that only the included files are restored
for _, testFile := range testfiles {
restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
_, err := os.Stat(restoredFilePath)
if testFile.include {
rtest.OK(t, err)
} else {
rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
testRestoreFileInclusions := func(t *testing.T, env *testEnvironment, includePatterns []string) {
// Check that only the included files are restored
for _, testFile := range testfiles {
restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
_, err := os.Stat(restoredFilePath)
if testFile.include {
rtest.OK(t, err)
} else {
rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
}
}
}
testRestoreFileInclusions(t, env, includePatterns)
// Create an include file with some patterns
patternsFile := env.base + "/patternsFile"
fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644)
if fileErr != nil {
t.Fatalf("Could not write include file: %v", fileErr)
}
restoredir = filepath.Join(env.base, "restore-include-from-file")
testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
testRestoreFileInclusions(t, env, includePatterns)
}
func TestRestoreFilter(t *testing.T) {
@ -144,19 +223,38 @@ func TestRestoreFilter(t *testing.T) {
rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
}
for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} {
base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat})
for _, testFile := range testfiles {
err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size))
if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
rtest.OK(t, err)
} else {
rtest.Assert(t, os.IsNotExist(err),
"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
excludePatterns := []string{"*.c", "*.exe", "*", "*file3*"}
testRestoreFileExclusions := func(t *testing.T, env *testEnvironment, excludePatterns []string) {
for i, pat := range excludePatterns {
base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat})
for _, testFile := range testfiles {
err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size))
if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
rtest.OK(t, err)
} else {
rtest.Assert(t, os.IsNotExist(err),
"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
}
}
}
}
testRestoreFileExclusions(t, env, excludePatterns)
// Create an include file with some patterns
patternsFile := env.base + "/patternsFile"
fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644)
if fileErr != nil {
t.Fatalf("Could not write include file: %v", fileErr)
}
restoredir := filepath.Join(env.base, "restore-exclude-from-file")
testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
testRestoreFileExclusions(t, env, excludePatterns)
}
func TestRestore(t *testing.T) {

2
cmd/restic/exclude.go
View file

@ -443,7 +443,7 @@ func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of filenames in patterns")
}
func (opts *excludePatternOptions) Empty() bool {

2
cmd/restic/include.go
View file

@ -23,7 +23,7 @@ func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) {
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of filenames in patterns")
}
func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {

33
cmd/restic/integration_filter_pattern_test.go
View file

@ -105,49 +105,28 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
testRunInit(t, env.gopts)
// Create an include file with some invalid patterns
includeFile := env.base + "/includefile"
fileErr := os.WriteFile(includeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
patternsFile := env.base + "/patternsFile"
fileErr := os.WriteFile(patternsFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
if fileErr != nil {
t.Fatalf("Could not write include file: %v", fileErr)
}
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{includeFile}}}, env.gopts)
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Create an exclude file with some invalid patterns
excludeFile := env.base + "/excludefile"
fileErr = os.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
if fileErr != nil {
t.Fatalf("Could not write exclude file: %v", fileErr)
}
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Create an insentive include file with some invalid patterns
insensitiveIncludeFile := env.base + "/insensitiveincludefile"
fileErr = os.WriteFile(insensitiveIncludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
if fileErr != nil {
t.Fatalf("Could not write insensitive include file: %v", fileErr)
}
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{insensitiveIncludeFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Create an insensitive exclude file with some invalid patterns
insensitiveExcludeFile := env.base + "/insensitiveexcludefile"
fileErr = os.WriteFile(insensitiveExcludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
if fileErr != nil {
t.Fatalf("Could not write insensitive exclude file: %v", fileErr)
}
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{insensitiveExcludeFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())