TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

filter: only check whether a child path could match when necessary

Browse source 
When checking excludes there is no need to test whether a child path
could also match the pattern, as it is by definition excluded.
Previously childMayMatch was calculated but then discarded. For simple
absolute paths this can account for half the time spent for checking
pattern matches.

name                          old time/op    new time/op    delta
FilterPatterns/Relative-4       23.3ms ± 9%    21.7ms ± 6%   -6.68%  (p=0.004 n=10+10)
FilterPatterns/Absolute-4       13.9ms ± 7%    10.0ms ± 5%  -27.61%  (p=0.000 n=10+10)
FilterPatterns/Wildcard-4       51.4ms ± 7%    47.0ms ± 7%   -8.51%  (p=0.001 n=9+9)
FilterPatterns/ManyNoMatch-4     551ms ± 9%     190ms ± 1%  -65.41%  (p=0.000 n=10+8)

name                          old alloc/op   new alloc/op   delta
FilterPatterns/Relative-4       3.57MB ± 0%    3.57MB ± 0%     ~     (p=0.665 n=10+9)
FilterPatterns/Absolute-4       3.57MB ± 0%    3.57MB ± 0%     ~     (p=0.480 n=9+10)
FilterPatterns/Wildcard-4       14.3MB ± 0%    14.3MB ± 0%     ~     (p=0.431 n=9+10)
FilterPatterns/ManyNoMatch-4    3.57MB ± 0%    3.57MB ± 0%     ~     (all equal)

name                          old allocs/op  new allocs/op  delta
FilterPatterns/Relative-4        22.2k ± 0%     22.2k ± 0%     ~     (all equal)
FilterPatterns/Absolute-4        22.2k ± 0%     22.2k ± 0%     ~     (all equal)
FilterPatterns/Wildcard-4        88.7k ± 0%     88.7k ± 0%     ~     (all equal)
FilterPatterns/ManyNoMatch-4     22.2k ± 0%     22.2k ± 0%     ~     (all equal)
This commit is contained in:
Michael Eischer 2020-10-07 19:46:41 +02:00
parent 17c53efb0d
commit bcc3bddcf4
4 changed files with 31 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/restic/cmd_restore.go
View file

@ -143,12 +143,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
excludePatterns := filter.ParsePatterns(opts.Exclude)
insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, _, err := filter.List(excludePatterns, item)
matched, err := filter.List(excludePatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}
matchedInsensitive, _, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
}
@ -166,12 +166,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
includePatterns := filter.ParsePatterns(opts.Include)
insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, childMayMatch, err := filter.List(includePatterns, item)
matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
if err != nil {
Warnf("error for include pattern: %v", err)
}
matchedInsensitive, childMayMatchInsensitive, err := filter.List(insensitiveIncludePatterns, strings.ToLower(item))
matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
}

2
cmd/restic/exclude.go
View file

@ -76,7 +76,7 @@ type RejectFunc func(path string, fi os.FileInfo) bool
func rejectByPattern(patterns []string) RejectByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
return func(item string) bool {
matched, _, err := filter.List(parsedPatterns, item)
matched, err := filter.List(parsedPatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}

24
internal/filter/filter.go
View file

@ -194,7 +194,18 @@ func ParsePatterns(patterns []string) []Pattern {
}
// List returns true if str matches one of the patterns. Empty patterns are ignored.
func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) {
func List(patterns []Pattern, str string) (matched bool, err error) {
matched, _, err = list(patterns, false, str)
return matched, err
}
// ListWithChild returns true if str matches one of the patterns. Empty patterns are ignored.
func ListWithChild(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) {
return list(patterns, true, str)
}
// List returns true if str matches one of the patterns. Empty patterns are ignored.
func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool, childMayMatch bool, err error) {
if len(patterns) == 0 {
return false, false, nil
}
@ -209,9 +220,14 @@ func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err
return false, false, err
}
c, err := childMatch(pat, strs)
if err != nil {
return false, false, err
var c bool
if checkChildMatches {
c, err = childMatch(pat, strs)
if err != nil {
return false, false, err
}
} else {
c = true
}
matched = matched || m

12
internal/filter/filter_test.go
View file

@ -260,7 +260,7 @@ var filterListTests = []struct {
func TestList(t *testing.T) {
for i, test := range filterListTests {
patterns := filter.ParsePatterns(test.patterns)
match, _, err := filter.List(patterns, test.path)
match, err := filter.List(patterns, test.path)
if err != nil {
t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v",
i, test.patterns, err)
@ -276,7 +276,7 @@ func TestList(t *testing.T) {
func ExampleList() {
patterns := filter.ParsePatterns([]string{"*.c", "*.go"})
match, _, _ := filter.List(patterns, "/home/user/file.go")
match, _ := filter.List(patterns, "/home/user/file.go")
fmt.Printf("match: %v\n", match)
// Output:
// match: true
@ -294,7 +294,7 @@ func TestInvalidStrs(t *testing.T) {
}
patterns := []string{"test"}
_, _, err = filter.List(filter.ParsePatterns(patterns), "")
_, err = filter.List(filter.ParsePatterns(patterns), "")
if err == nil {
t.Error("List accepted invalid path")
}
@ -302,13 +302,13 @@ func TestInvalidStrs(t *testing.T) {
func TestInvalidPattern(t *testing.T) {
patterns := []string{"test/["}
_, _, err := filter.List(filter.ParsePatterns(patterns), "test/example")
_, err := filter.List(filter.ParsePatterns(patterns), "test/example")
if err == nil {
t.Error("List accepted invalid pattern")
}
patterns = []string{"test/**/["}
_, _, err = filter.List(filter.ParsePatterns(patterns), "test/example")
_, err = filter.List(filter.ParsePatterns(patterns), "test/example")
if err == nil {
t.Error("List accepted invalid pattern")
}
@ -435,7 +435,7 @@ func BenchmarkFilterPatterns(b *testing.B) {
for i := 0; i < b.N; i++ {
c = 0
for _, line := range lines {
match, _, err := filter.List(test.patterns, line)
match, err := filter.List(test.patterns, line)
if err != nil {
b.Fatal(err)
}