fix: include and exclude logic, add tests for include file and exclude file

cmd/restic/cmd_restore.go

@@ -70,9 +70,6 @@ func init() {
 func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 	term *termstatus.Terminal, args []string) error {
 
-	hasExcludes := len(opts.Excludes) > 0 || len(opts.InsensitiveExcludes) > 0
-	hasIncludes := len(opts.Includes) > 0 || len(opts.InsensitiveIncludes) > 0
-
 	excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
 	if err != nil {
 		return err
@@ -83,6 +80,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		return err
 	}
 
+	hasExcludes := len(excludePatternFns) > 0
+	hasIncludes := len(includePatternFns) > 0
+
 	switch {
 	case len(args) == 0:
 		return errors.Fatal("no snapshot ID specified")
@@ -153,6 +153,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		matched := false
 		for _, rejectFn := range excludePatternFns {
 			matched = matched || rejectFn(item)
+
+			// implementing a short-circuit here to improve the performance
+			// to prevent additional pattern matching once the first pattern
+			// matches.
+			if matched {
+				break
+			}
 		}
 		// An exclude filter is basically a 'wildcard but foo',
 		// so even if a childMayMatch, other children of a dir may not,
@@ -171,9 +178,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 			matched, childMayMatch := includeFn(item)
 			selectedForRestore = selectedForRestore || matched
 			childMayBeSelected = childMayBeSelected || childMayMatch
-		}
+			childMayBeSelected = childMayBeSelected && node.Type == "dir"
 
-		childMayBeSelected = childMayBeSelected && node.Type == "dir"
+			if selectedForRestore || childMayBeSelected {
+				break
+			}
+		}
 
 		return selectedForRestore, childMayBeSelected
 	}

cmd/restic/cmd_restore_integration_test.go

@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -58,6 +59,67 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
 	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
 }
 
+func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) {
+	opts := RestoreOptions{
+		Target: dir,
+	}
+	opts.IncludeFiles = []string{includesFile}
+
+	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
+}
+
+func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) {
+	opts := RestoreOptions{
+		Target: dir,
+	}
+	opts.ExcludeFiles = []string{excludesFile}
+
+	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
+}
+
+func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
+	testfiles := []struct {
+		path string
+		size uint
+	}{
+		{"dir1/include_me.txt", 100},
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	// Create test files and directories
+	for _, testFile := range testfiles {
+		fullPath := filepath.Join(env.testdata, testFile.path)
+		rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
+		rtest.OK(t, appendRandomData(fullPath, testFile.size))
+	}
+
+	opts := BackupOptions{}
+	// Perform backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	testRunCheck(t, env.gopts)
+
+	snapshotID := testListSnapshots(t, env.gopts, 1)[0]
+
+	// Add both include and exclude patterns
+	includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
+	excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
+
+	restoredir := filepath.Join(env.base, "restore")
+
+	restoreOpts := RestoreOptions{
+		Target: restoredir,
+	}
+	restoreOpts.Includes = includePatterns
+	restoreOpts.Excludes = excludePatterns
+
+	err := testRunRestoreAssumeFailure(snapshotID.String(), restoreOpts, env.gopts)
+	rtest.Assert(t, err != nil, "restore must fail if include and exclude patterns are provided")
+}
+
 func TestRestoreIncludes(t *testing.T) {
 	testfiles := []struct {
 		path    string
@@ -97,16 +159,33 @@ func TestRestoreIncludes(t *testing.T) {
 	restoredir := filepath.Join(env.base, "restore")
 	testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns)
 
-	// Check that only the included files are restored
+	testRestoreFileInclusions := func(t *testing.T, env *testEnvironment, includePatterns []string) {
-	for _, testFile := range testfiles {
+		// Check that only the included files are restored
-		restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
+		for _, testFile := range testfiles {
-		_, err := os.Stat(restoredFilePath)
+			restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
-		if testFile.include {
+			_, err := os.Stat(restoredFilePath)
-			rtest.OK(t, err)
+			if testFile.include {
-		} else {
+				rtest.OK(t, err)
-			rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
+			} else {
+				rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
+			}
 		}
 	}
+
+	testRestoreFileInclusions(t, env, includePatterns)
+
+	// Create an include file with some patterns
+	patternsFile := env.base + "/patternsFile"
+	fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644)
+	if fileErr != nil {
+		t.Fatalf("Could not write include file: %v", fileErr)
+	}
+
+	restoredir = filepath.Join(env.base, "restore-include-from-file")
+
+	testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
+
+	testRestoreFileInclusions(t, env, includePatterns)
 }
 
 func TestRestoreFilter(t *testing.T) {
@@ -144,19 +223,38 @@ func TestRestoreFilter(t *testing.T) {
 		rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
 	}
 
-	for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} {
+	excludePatterns := []string{"*.c", "*.exe", "*", "*file3*"}
-		base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
+
-		testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat})
+	testRestoreFileExclusions := func(t *testing.T, env *testEnvironment, excludePatterns []string) {
-		for _, testFile := range testfiles {
+		for i, pat := range excludePatterns {
-			err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size))
+			base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
-			if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
+			testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat})
-				rtest.OK(t, err)
+			for _, testFile := range testfiles {
-			} else {
+				err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size))
-				rtest.Assert(t, os.IsNotExist(err),
+				if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
-					"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
+					rtest.OK(t, err)
+				} else {
+					rtest.Assert(t, os.IsNotExist(err),
+						"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
+				}
 			}
 		}
 	}
+
+	testRestoreFileExclusions(t, env, excludePatterns)
+
+	// Create an include file with some patterns
+	patternsFile := env.base + "/patternsFile"
+	fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644)
+	if fileErr != nil {
+		t.Fatalf("Could not write include file: %v", fileErr)
+	}
+
+	restoredir := filepath.Join(env.base, "restore-exclude-from-file")
+
+	testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
+
+	testRestoreFileExclusions(t, env, excludePatterns)
 }
 
 func TestRestore(t *testing.T) {

cmd/restic/exclude.go

@@ -443,7 +443,7 @@ func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
 	f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of filenames in patterns")
 }
 
 func (opts *excludePatternOptions) Empty() bool {

cmd/restic/include.go

@@ -23,7 +23,7 @@ func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) {
 	f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
+	f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of filenames in patterns")
 }
 
 func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {

cmd/restic/integration_filter_pattern_test.go

@@ -105,49 +105,28 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
 	testRunInit(t, env.gopts)
 
 	// Create an include file with some invalid patterns
-	includeFile := env.base + "/includefile"
+	patternsFile := env.base + "/patternsFile"
-	fileErr := os.WriteFile(includeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
+	fileErr := os.WriteFile(patternsFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
 	if fileErr != nil {
 		t.Fatalf("Could not write include file: %v", fileErr)
 	}
 
-	err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{includeFile}}}, env.gopts)
+	err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	// Create an exclude file with some invalid patterns
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
-	excludeFile := env.base + "/excludefile"
-	fileErr = os.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
-	if fileErr != nil {
-		t.Fatalf("Could not write exclude file: %v", fileErr)
-	}
-
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	// Create an insentive include file with some invalid patterns
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
-	insensitiveIncludeFile := env.base + "/insensitiveincludefile"
-	fileErr = os.WriteFile(insensitiveIncludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
-	if fileErr != nil {
-		t.Fatalf("Could not write insensitive include file: %v", fileErr)
-	}
-
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{insensitiveIncludeFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	// Create an insensitive exclude file with some invalid patterns
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
-	insensitiveExcludeFile := env.base + "/insensitiveexcludefile"
-	fileErr = os.WriteFile(insensitiveExcludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
-	if fileErr != nil {
-		t.Fatalf("Could not write insensitive exclude file: %v", fileErr)
-	}
-
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{insensitiveExcludeFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())