TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Allows multiple --include/--exclude/--filter options - fixes #875

Browse source
This commit is contained in:
Nick Craig-Wood 2016-12-07 13:37:40 +00:00
parent 3bdfa284a9
commit dcf53a1d12
3 changed files with 127 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
docs/content/filtering.md
View file

@ -19,10 +19,6 @@ and exclude rules like `--include`, `--exclude`, `--include-from`,
try them out is using the `ls` command, or `--dry-run` together with
`-v`.
**Important** Due to limitations of the command line parser you can
only use any of these options once - if you duplicate them then rclone
will use the last one only.
## Patterns ##
The patterns used to match files for inclusion or exclusion are based
@ -159,16 +155,44 @@ based remotes (eg s3, swift, google compute storage, b2).
Filtering rules are added with the following command line flags.
### Repeating options ##
You can repeat the following options to add more than one rule of that
type.
* `--include`
* `--include-from`
* `--exclude`
* `--exclude-from`
* `--filter`
* `--filter-from`
Note that all the options of the same type are processed together in
the order above, regardless of what order they were placed on the
command line.
So all `--include` options are processed first in the order they
appeared on the command line, then all `--include-from` options etc.
To mix up the order includes and excludes, the `--filter` flag can be
used.
### `--exclude` - Exclude files matching pattern ###
Add a single exclude rule with `--exclude`.
This flag can be repeated. See above for the order the flags are
processed in.
Eg `--exclude *.bak` to exclude all bak files from the sync.
### `--exclude-from` - Read exclude patterns from file ###
Add exclude rules from a file.
This flag can be repeated. See above for the order the flags are
processed in.
Prepare a file like this `exclude-file.txt`
# a sample exclude rule file
@ -184,6 +208,9 @@ This is useful if you have a lot of rules.
Add a single include rule with `--include`.
This flag can be repeated. See above for the order the flags are
processed in.
Eg `--include *.{png,jpg}` to include all `png` and `jpg` files in the
backup and no others.
@ -197,6 +224,9 @@ flexibility then you must use `--filter-from`.
Add include rules from a file.
This flag can be repeated. See above for the order the flags are
processed in.
Prepare a file like this `include-file.txt`
# a sample include rule file
@ -221,12 +251,18 @@ This can be used to add a single include or exclude rule. Include
rules start with `+ ` and exclude rules start with `- `. A special
rule called `!` can be used to clear the existing rules.
This flag can be repeated. See above for the order the flags are
processed in.
Eg `--filter "- *.bak"` to exclude all bak files from the sync.
### `--filter-from` - Read filtering patterns from a file ###
Add include/exclude rules from a file.
This flag can be repeated. See above for the order the flags are
processed in.
Prepare a file like this `filter-file.txt`
# a sample exclude rule file
@ -250,6 +286,9 @@ This reads a list of file names from the file passed in and **only**
these files are transferred. The filtering rules are ignored
completely if you use this option.
This option can be repeated to read from more than one file. These
are read in the order that they are placed on the command line.
Prepare a file like this `files-from.txt`
# comment

108
fs/filter.go
View file

@ -20,13 +20,13 @@ import (
var (
// Flags
deleteExcluded = pflag.BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync")
filterRule = pflag.StringP("filter", "f", "", "Add a file-filtering rule")
filterFrom = pflag.StringP("filter-from", "", "", "Read filtering patterns from a file")
excludeRule = pflag.StringP("exclude", "", "", "Exclude files matching pattern")
excludeFrom = pflag.StringP("exclude-from", "", "", "Read exclude patterns from file")
includeRule = pflag.StringP("include", "", "", "Include files matching pattern")
includeFrom = pflag.StringP("include-from", "", "", "Read include patterns from file")
filesFrom = pflag.StringP("files-from", "", "", "Read list of source-file names from file")
filterRule = pflag.StringArrayP("filter", "f", nil, "Add a file-filtering rule")
filterFrom = pflag.StringArrayP("filter-from", "", nil, "Read filtering patterns from a file")
excludeRule = pflag.StringArrayP("exclude", "", nil, "Exclude files matching pattern")
excludeFrom = pflag.StringArrayP("exclude-from", "", nil, "Read exclude patterns from file")
includeRule = pflag.StringArrayP("include", "", nil, "Include files matching pattern")
includeFrom = pflag.StringArrayP("include-from", "", nil, "Read include patterns from file")
filesFrom = pflag.StringArrayP("files-from", "", nil, "Read list of source-file names from file")
minAge = pflag.StringP("min-age", "", "", "Don't transfer any file younger than this in s or suffix ms|s|m|h|d|w|M|y")
maxAge = pflag.StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
minSize = SizeSuffix(-1)
@ -157,54 +157,68 @@ func NewFilter() (f *Filter, err error) {
}
addImplicitExclude := false
if *includeRule != "" {
err = f.Add(true, *includeRule)
if err != nil {
return nil, err
}
addImplicitExclude = true
}
if *includeFrom != "" {
err := forEachLine(*includeFrom, func(line string) error {
return f.Add(true, line)
})
if err != nil {
return nil, err
}
addImplicitExclude = true
}
if *excludeRule != "" {
err = f.Add(false, *excludeRule)
if err != nil {
return nil, err
if includeRule != nil {
for _, rule := range *includeRule {
err = f.Add(true, rule)
if err != nil {
return nil, err
}
addImplicitExclude = true
}
}
if *excludeFrom != "" {
err := forEachLine(*excludeFrom, func(line string) error {
return f.Add(false, line)
})
if err != nil {
return nil, err
if includeFrom != nil {
for _, rule := range *includeFrom {
err := forEachLine(rule, func(line string) error {
return f.Add(true, line)
})
if err != nil {
return nil, err
}
addImplicitExclude = true
}
}
if *filterRule != "" {
err = f.AddRule(*filterRule)
if err != nil {
return nil, err
if excludeRule != nil {
for _, rule := range *excludeRule {
err = f.Add(false, rule)
if err != nil {
return nil, err
}
}
}
if *filterFrom != "" {
err := forEachLine(*filterFrom, f.AddRule)
if err != nil {
return nil, err
if excludeFrom != nil {
for _, rule := range *excludeFrom {
err := forEachLine(rule, func(line string) error {
return f.Add(false, line)
})
if err != nil {
return nil, err
}
}
}
if *filesFrom != "" {
err := forEachLine(*filesFrom, func(line string) error {
return f.AddFile(line)
})
if err != nil {
return nil, err
if filterRule != nil {
for _, rule := range *filterRule {
err = f.AddRule(rule)
if err != nil {
return nil, err
}
}
}
if filterFrom != nil {
for _, rule := range *filterFrom {
err := forEachLine(rule, f.AddRule)
if err != nil {
return nil, err
}
}
}
if filesFrom != nil {
for _, rule := range *filesFrom {
err := forEachLine(rule, func(line string) error {
return f.AddFile(line)
})
if err != nil {
return nil, err
}
}
}
if addImplicitExclude {

51
fs/filter_test.go
View file

@ -54,13 +54,8 @@ func TestNewFilterDefault(t *testing.T) {
assert.True(t, f.InActive())
}
// return a pointer to the string
func stringP(s string) *string {
return &s
}
// testFile creates a temp file with the contents
func testFile(t *testing.T, contents string) *string {
func testFile(t *testing.T, contents string) string {
out, err := ioutil.TempFile("", "filter_test")
require.NoError(t, err)
defer func() {
@ -70,25 +65,24 @@ func testFile(t *testing.T, contents string) *string {
_, err = out.Write([]byte(contents))
require.NoError(t, err)
s := out.Name()
return &s
return s
}
func TestNewFilterFull(t *testing.T) {
mins := int64(100 * 1024)
maxs := int64(1000 * 1024)
emptyString := ""
isFalse := false
isTrue := true
// Set up the input
deleteExcluded = &isTrue
filterRule = stringP("- filter1")
filterFrom = testFile(t, "#comment\n+ filter2\n- filter3\n")
excludeRule = stringP("exclude1")
excludeFrom = testFile(t, "#comment\nexclude2\nexclude3\n")
includeRule = stringP("include1")
includeFrom = testFile(t, "#comment\ninclude2\ninclude3\n")
filesFrom = testFile(t, "#comment\nfiles1\nfiles2\n")
filterRule = &[]string{"- filter1", "- filter1b"}
filterFrom = &[]string{testFile(t, "#comment\n+ filter2\n- filter3\n")}
excludeRule = &[]string{"exclude1"}
excludeFrom = &[]string{testFile(t, "#comment\nexclude2\nexclude3\n")}
includeRule = &[]string{"include1"}
includeFrom = &[]string{testFile(t, "#comment\ninclude2\ninclude3\n")}
filesFrom = &[]string{testFile(t, "#comment\nfiles1\nfiles2\n")}
minSize = SizeSuffix(mins)
maxSize = SizeSuffix(maxs)
@ -100,20 +94,20 @@ func TestNewFilterFull(t *testing.T) {
}
// Reset the input
defer func() {
rm(*filterFrom)
rm(*excludeFrom)
rm(*includeFrom)
rm(*filesFrom)
rm((*filterFrom)[0])
rm((*excludeFrom)[0])
rm((*includeFrom)[0])
rm((*filesFrom)[0])
minSize = -1
maxSize = -1
deleteExcluded = &isFalse
filterRule = &emptyString
filterFrom = &emptyString
excludeRule = &emptyString
excludeFrom = &emptyString
includeRule = &emptyString
includeFrom = &emptyString
filesFrom = &emptyString
filterRule = nil
filterFrom = nil
excludeRule = nil
excludeFrom = nil
includeRule = nil
includeFrom = nil
filesFrom = nil
}()
f, err := NewFilter()
@ -130,6 +124,7 @@ func TestNewFilterFull(t *testing.T) {
- (^|/)exclude2$
- (^|/)exclude3$
- (^|/)filter1$
- (^|/)filter1b$
+ (^|/)filter2$
- (^|/)filter3$
- ^.*$
@ -356,11 +351,11 @@ four
five
six `)
defer func() {
err := os.Remove(*file)
err := os.Remove(file)
require.NoError(t, err)
}()
lines := []string{}
err := forEachLine(*file, func(s string) error {
err := forEachLine(file, func(s string) error {
lines = append(lines, s)
return nil
})