Merge pull request 2032 from j6s/feature.case-insensitive-exclude

Add options for case insensitive includes & excludes
This commit is contained in:
Alexander Neumann 2019-02-10 12:40:33 +01:00
commit 56e5467096
7 changed files with 109 additions and 27 deletions

View file

@ -0,0 +1,7 @@
Enhancement: Add case insensitive include & exclude options
The backup and restore commands now have --iexclude and --iinclude flags
as case insensitive variants of --exclude and --include.
https://github.com/restic/restic/issues/1895
https://github.com/restic/restic/pull/2032

View file

@ -73,6 +73,7 @@ type BackupOptions struct {
Parent string Parent string
Force bool Force bool
Excludes []string Excludes []string
InsensitiveExcludes []string
ExcludeFiles []string ExcludeFiles []string
ExcludeOtherFS bool ExcludeOtherFS bool
ExcludeIfPresent []string ExcludeIfPresent []string
@ -95,6 +96,7 @@ func init() {
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)") f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames")
f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
@ -224,6 +226,10 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
opts.Excludes = append(opts.Excludes, excludes...) opts.Excludes = append(opts.Excludes, excludes...)
} }
if len(opts.InsensitiveExcludes) > 0 {
fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
}
if len(opts.Excludes) > 0 { if len(opts.Excludes) > 0 {
fs = append(fs, rejectByPattern(opts.Excludes)) fs = append(fs, rejectByPattern(opts.Excludes))
} }

View file

@ -6,6 +6,7 @@ import (
"github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/restorer"
"strings"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -29,7 +30,9 @@ repository.
// RestoreOptions collects all options for the restore command. // RestoreOptions collects all options for the restore command.
type RestoreOptions struct { type RestoreOptions struct {
Exclude []string Exclude []string
InsensitiveExclude []string
Include []string Include []string
InsensitiveInclude []string
Target string Target string
Host string Host string
Paths []string Paths []string
@ -44,7 +47,9 @@ func init() {
flags := cmdRestore.Flags() flags := cmdRestore.Flags()
flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames")
flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)") flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
@ -55,6 +60,16 @@ func init() {
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx ctx := gopts.ctx
hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
for i, str := range opts.InsensitiveExclude {
opts.InsensitiveExclude[i] = strings.ToLower(str)
}
for i, str := range opts.InsensitiveInclude {
opts.InsensitiveInclude[i] = strings.ToLower(str)
}
switch { switch {
case len(args) == 0: case len(args) == 0:
@ -67,7 +82,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("please specify a directory to restore to (--target)") return errors.Fatal("please specify a directory to restore to (--target)")
} }
if len(opts.Exclude) > 0 && len(opts.Include) > 0 { if hasExcludes && hasIncludes {
return errors.Fatal("exclude and include patterns are mutually exclusive") return errors.Fatal("exclude and include patterns are mutually exclusive")
} }
@ -125,11 +140,16 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
Warnf("error for exclude pattern: %v", err) Warnf("error for exclude pattern: %v", err)
} }
matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
}
// An exclude filter is basically a 'wildcard but foo', // An exclude filter is basically a 'wildcard but foo',
// so even if a childMayMatch, other children of a dir may not, // so even if a childMayMatch, other children of a dir may not,
// therefore childMayMatch does not matter, but we should not go down // therefore childMayMatch does not matter, but we should not go down
// unless the dir is selected for restore // unless the dir is selected for restore
selectedForRestore = !matched selectedForRestore = !matched && !matchedInsensitive
childMayBeSelected = selectedForRestore && node.Type == "dir" childMayBeSelected = selectedForRestore && node.Type == "dir"
return selectedForRestore, childMayBeSelected return selectedForRestore, childMayBeSelected
@ -141,15 +161,20 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
Warnf("error for include pattern: %v", err) Warnf("error for include pattern: %v", err)
} }
selectedForRestore = matched matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item))
childMayBeSelected = childMayMatch && node.Type == "dir" if err != nil {
Warnf("error for iexclude pattern: %v", err)
}
selectedForRestore = matched || matchedInsensitive
childMayBeSelected = (childMayMatch || childMayMatchInsensitive) && node.Type == "dir"
return selectedForRestore, childMayBeSelected return selectedForRestore, childMayBeSelected
} }
if len(opts.Exclude) > 0 { if hasExcludes {
res.SelectFilter = selectExcludeFilter res.SelectFilter = selectExcludeFilter
} else if len(opts.Include) > 0 { } else if hasIncludes {
res.SelectFilter = selectIncludeFilter res.SelectFilter = selectIncludeFilter
} }

View file

@ -88,6 +88,18 @@ func rejectByPattern(patterns []string) RejectByNameFunc {
} }
} }
// Same as `rejectByPattern` but case insensitive.
func rejectByInsensitivePattern(patterns []string) RejectByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
rejFunc := rejectByPattern(patterns)
return func(item string) bool {
return rejFunc(strings.ToLower(item))
}
}
// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path // rejectIfPresent returns a RejectByNameFunc which itself returns whether a path
// should be excluded. The RejectByNameFunc considers a file to be excluded when // should be excluded. The RejectByNameFunc considers a file to be excluded when
// it resides in a directory with an exclusion file, that is specified by // it resides in a directory with an exclusion file, that is specified by

View file

@ -36,6 +36,33 @@ func TestRejectByPattern(t *testing.T) {
} }
} }
func TestRejectByInsensitivePattern(t *testing.T) {
var tests = []struct {
filename string
reject bool
}{
{filename: "/home/user/foo.GO", reject: true},
{filename: "/home/user/foo.c", reject: false},
{filename: "/home/user/foobar", reject: false},
{filename: "/home/user/FOObar/x", reject: true},
{filename: "/home/user/README", reject: false},
{filename: "/home/user/readme.md", reject: true},
}
patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
for _, tc := range tests {
t.Run("", func(t *testing.T) {
reject := rejectByInsensitivePattern(patterns)
res := reject(tc.filename)
if res != tc.reject {
t.Fatalf("wrong result for filename %v: want %v, got %v",
tc.filename, tc.reject, res)
}
})
}
}
func TestIsExcludedByFile(t *testing.T) { func TestIsExcludedByFile(t *testing.T) {
const ( const (
tagFilename = "CACHEDIR.TAG" tagFilename = "CACHEDIR.TAG"

View file

@ -139,6 +139,7 @@ You can exclude folders and files by specifying exclude patterns, currently
the exclude options are: the exclude options are:
- ``--exclude`` Specified one or more times to exclude one or more items - ``--exclude`` Specified one or more times to exclude one or more items
- ``--iexclude`` Same as ``--exclude`` but ignores the case of paths
- ``--exclude-caches`` Specified once to exclude folders containing a special file - ``--exclude-caches`` Specified once to exclude folders containing a special file
- ``--exclude-file`` Specified one or more times to exclude items listed in a given file - ``--exclude-file`` Specified one or more times to exclude items listed in a given file
- ``--exclude-if-present`` Specified one or more times to exclude a folders content - ``--exclude-if-present`` Specified one or more times to exclude a folders content

View file

@ -52,6 +52,10 @@ You can use the command ``restic ls latest`` or ``restic find foo`` to find the
path to the file within the snapshot. This path you can then pass to path to the file within the snapshot. This path you can then pass to
`--include` in verbatim to only restore the single file or directory. `--include` in verbatim to only restore the single file or directory.
There are case insensitive variants of of ``--exclude`` and ``--include`` called
``--iexclude`` and ``--iinclude``. These options will behave the same way but
ignore the casing of paths.
Restore using mount Restore using mount
=================== ===================