forked from TrueCloudLab/restic
filter: Parse filter patterns only once
name old time/op new time/op delta FilterPatterns/Relative-4 30.3ms ±10% 23.6ms ±20% -22.12% (p=0.000 n=10+10) FilterPatterns/Absolute-4 49.0ms ± 3% 32.3ms ± 8% -33.94% (p=0.000 n=8+10) FilterPatterns/Wildcard-4 345ms ± 9% 334ms ±17% ~ (p=0.315 n=10+10) FilterPatterns/ManyNoMatch-4 3.93s ± 2% 0.71s ± 7% -81.98% (p=0.000 n=9+10) name old alloc/op new alloc/op delta FilterPatterns/Relative-4 4.63MB ± 0% 3.57MB ± 0% -22.98% (p=0.000 n=9+9) FilterPatterns/Absolute-4 8.54MB ± 0% 3.57MB ± 0% -58.20% (p=0.000 n=10+10) FilterPatterns/Wildcard-4 146MB ± 0% 141MB ± 0% -2.93% (p=0.000 n=9+9) FilterPatterns/ManyNoMatch-4 907MB ± 0% 4MB ± 0% -99.61% (p=0.000 n=9+9) name old allocs/op new allocs/op delta FilterPatterns/Relative-4 66.6k ± 0% 22.2k ± 0% -66.67% (p=0.000 n=10+10) FilterPatterns/Absolute-4 88.7k ± 0% 22.2k ± 0% -75.00% (p=0.000 n=10+10) FilterPatterns/Wildcard-4 1.70M ± 0% 1.63M ± 0% -3.92% (p=0.000 n=10+10) FilterPatterns/ManyNoMatch-4 4.46M ± 0% 0.02M ± 0% -99.50% (p=0.000 n=10+10)
This commit is contained in:
Michael Eischer
parent
b8eacd1364
commit
375c2a56de
4 changed files with 50 additions and 34 deletions
|
|
@ -140,13 +140,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
excludePatterns := filter.ParsePatterns(opts.Exclude)
|
||||
insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
|
||||
selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
||||
matched, _, err := filter.List(opts.Exclude, item)
|
||||
matched, _, err := filter.List(excludePatterns, item)
|
||||
if err != nil {
|
||||
Warnf("error for exclude pattern: %v", err)
|
||||
}
|
||||
|
||||
matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item))
|
||||
matchedInsensitive, _, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
|
||||
if err != nil {
|
||||
Warnf("error for iexclude pattern: %v", err)
|
||||
}
|
||||
|
|
@ -161,13 +163,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
|
|||
return selectedForRestore, childMayBeSelected
|
||||
}
|
||||
|
||||
includePatterns := filter.ParsePatterns(opts.Include)
|
||||
insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
|
||||
selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
||||
matched, childMayMatch, err := filter.List(opts.Include, item)
|
||||
matched, childMayMatch, err := filter.List(includePatterns, item)
|
||||
if err != nil {
|
||||
Warnf("error for include pattern: %v", err)
|
||||
}
|
||||
|
||||
matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item))
|
||||
matchedInsensitive, childMayMatchInsensitive, err := filter.List(insensitiveIncludePatterns, strings.ToLower(item))
|
||||
if err != nil {
|
||||
Warnf("error for iexclude pattern: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,8 +74,9 @@ type RejectFunc func(path string, fi os.FileInfo) bool
|
|||
// rejectByPattern returns a RejectByNameFunc which rejects files that match
|
||||
// one of the patterns.
|
||||
func rejectByPattern(patterns []string) RejectByNameFunc {
|
||||
parsedPatterns := filter.ParsePatterns(patterns)
|
||||
return func(item string) bool {
|
||||
matched, _, err := filter.List(patterns, item)
|
||||
matched, _, err := filter.List(parsedPatterns, item)
|
||||
if err != nil {
|
||||
Warnf("error for exclude pattern: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ import (
|
|||
// second argument.
|
||||
var ErrBadString = errors.New("filter.Match: string is empty")
|
||||
|
||||
type filterPattern []string
|
||||
// Pattern represents a preparsed filter pattern
|
||||
type Pattern []string
|
||||
|
||||
func prepareStr(str string) ([]string, error) {
|
||||
if str == "" {
|
||||
|
|
@ -26,7 +27,7 @@ func prepareStr(str string) ([]string, error) {
|
|||
return strings.Split(str, "/"), nil
|
||||
}
|
||||
|
||||
func preparePattern(pattern string) filterPattern {
|
||||
func preparePattern(pattern string) Pattern {
|
||||
pattern = filepath.Clean(pattern)
|
||||
|
||||
// convert file path separator to '/'
|
||||
|
|
@ -87,7 +88,7 @@ func ChildMatch(pattern, str string) (matched bool, err error) {
|
|||
return childMatch(patterns, strs)
|
||||
}
|
||||
|
||||
func childMatch(patterns, strs []string) (matched bool, err error) {
|
||||
func childMatch(patterns Pattern, strs []string) (matched bool, err error) {
|
||||
if patterns[0] != "" {
|
||||
// relative pattern can always be nested down
|
||||
return true, nil
|
||||
|
|
@ -109,7 +110,7 @@ func childMatch(patterns, strs []string) (matched bool, err error) {
|
|||
return match(patterns[0:l], strs)
|
||||
}
|
||||
|
||||
func hasDoubleWildcard(list []string) (ok bool, pos int) {
|
||||
func hasDoubleWildcard(list Pattern) (ok bool, pos int) {
|
||||
for i, item := range list {
|
||||
if item == "**" {
|
||||
return true, i
|
||||
|
|
@ -119,11 +120,11 @@ func hasDoubleWildcard(list []string) (ok bool, pos int) {
|
|||
return false, 0
|
||||
}
|
||||
|
||||
func match(patterns, strs []string) (matched bool, err error) {
|
||||
func match(patterns Pattern, strs []string) (matched bool, err error) {
|
||||
if ok, pos := hasDoubleWildcard(patterns); ok {
|
||||
// gradually expand '**' into separate wildcards
|
||||
for i := 0; i <= len(strs)-len(patterns)+1; i++ {
|
||||
newPat := make([]string, pos)
|
||||
newPat := make(Pattern, pos)
|
||||
copy(newPat, patterns[:pos])
|
||||
for k := 0; k < i; k++ {
|
||||
newPat = append(newPat, "*")
|
||||
|
|
@ -169,9 +170,22 @@ func match(patterns, strs []string) (matched bool, err error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// List returns true if str matches one of the patterns. Empty patterns are
|
||||
// ignored.
|
||||
func List(patterns []string, str string) (matched bool, childMayMatch bool, err error) {
|
||||
// ParsePatterns prepares a list of patterns for use with List.
|
||||
func ParsePatterns(patterns []string) []Pattern {
|
||||
patpat := make([]Pattern, 0)
|
||||
for _, pat := range patterns {
|
||||
if pat == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
pats := preparePattern(pat)
|
||||
patpat = append(patpat, pats)
|
||||
}
|
||||
return patpat
|
||||
}
|
||||
|
||||
// List returns true if str matches one of the patterns. Empty patterns are ignored.
|
||||
func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) {
|
||||
if len(patterns) == 0 {
|
||||
return false, false, nil
|
||||
}
|
||||
|
|
@ -181,17 +195,12 @@ func List(patterns []string, str string) (matched bool, childMayMatch bool, err
|
|||
return false, false, err
|
||||
}
|
||||
for _, pat := range patterns {
|
||||
if pat == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
pats := preparePattern(pat)
|
||||
m, err := match(pats, strs)
|
||||
m, err := match(pat, strs)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
c, err := childMatch(pats, strs)
|
||||
c, err := childMatch(pat, strs)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -259,7 +259,8 @@ var filterListTests = []struct {
|
|||
|
||||
func TestList(t *testing.T) {
|
||||
for i, test := range filterListTests {
|
||||
match, _, err := filter.List(test.patterns, test.path)
|
||||
patterns := filter.ParsePatterns(test.patterns)
|
||||
match, _, err := filter.List(patterns, test.path)
|
||||
if err != nil {
|
||||
t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v",
|
||||
i, test.patterns, err)
|
||||
|
|
@ -274,7 +275,8 @@ func TestList(t *testing.T) {
|
|||
}
|
||||
|
||||
func ExampleList() {
|
||||
match, _, _ := filter.List([]string{"*.c", "*.go"}, "/home/user/file.go")
|
||||
patterns := filter.ParsePatterns([]string{"*.c", "*.go"})
|
||||
match, _, _ := filter.List(patterns, "/home/user/file.go")
|
||||
fmt.Printf("match: %v\n", match)
|
||||
// Output:
|
||||
// match: true
|
||||
|
|
@ -292,7 +294,7 @@ func TestInvalidStrs(t *testing.T) {
|
|||
}
|
||||
|
||||
patterns := []string{"test"}
|
||||
_, _, err = filter.List(patterns, "")
|
||||
_, _, err = filter.List(filter.ParsePatterns(patterns), "")
|
||||
if err == nil {
|
||||
t.Error("List accepted invalid path")
|
||||
}
|
||||
|
|
@ -300,13 +302,13 @@ func TestInvalidStrs(t *testing.T) {
|
|||
|
||||
func TestInvalidPattern(t *testing.T) {
|
||||
patterns := []string{"test/["}
|
||||
_, _, err := filter.List(patterns, "test/example")
|
||||
_, _, err := filter.List(filter.ParsePatterns(patterns), "test/example")
|
||||
if err == nil {
|
||||
t.Error("List accepted invalid pattern")
|
||||
}
|
||||
|
||||
patterns = []string{"test/**/["}
|
||||
_, _, err = filter.List(patterns, "test/example")
|
||||
_, _, err = filter.List(filter.ParsePatterns(patterns), "test/example")
|
||||
if err == nil {
|
||||
t.Error("List accepted invalid pattern")
|
||||
}
|
||||
|
|
@ -403,25 +405,25 @@ func BenchmarkFilterPatterns(b *testing.B) {
|
|||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns []string
|
||||
patterns []filter.Pattern
|
||||
matches uint
|
||||
}{
|
||||
{"Relative", []string{
|
||||
{"Relative", filter.ParsePatterns([]string{
|
||||
"does-not-match",
|
||||
"sdk/*",
|
||||
"*.html",
|
||||
}, 22185},
|
||||
{"Absolute", []string{
|
||||
}), 22185},
|
||||
{"Absolute", filter.ParsePatterns([]string{
|
||||
"/etc",
|
||||
"/home/*/test",
|
||||
"/usr/share/doc/libreoffice/sdk/docs/java",
|
||||
}, 150},
|
||||
{"Wildcard", []string{
|
||||
}), 150},
|
||||
{"Wildcard", filter.ParsePatterns([]string{
|
||||
"/etc/**/example",
|
||||
"/home/**/test",
|
||||
"/usr/**/java",
|
||||
}, 150},
|
||||
{"ManyNoMatch", modlines, 0},
|
||||
}), 150},
|
||||
{"ManyNoMatch", filter.ParsePatterns(modlines), 0},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
|
|
|||
Loading…
Reference in a new issue