Allows multiple --include/--exclude/--filter options - fixes #875

docs/content/filtering.md

@@ -19,10 +19,6 @@ and exclude rules like `--include`, `--exclude`, `--include-from`,
 try them out is using the `ls` command, or `--dry-run` together with
 `-v`.
 
-**Important** Due to limitations of the command line parser you can
-only use any of these options once - if you duplicate them then rclone
-will use the last one only.
-
 ## Patterns ##
 
 The patterns used to match files for inclusion or exclusion are based
@@ -159,16 +155,44 @@ based remotes (eg s3, swift, google compute storage, b2).
 
 Filtering rules are added with the following command line flags.
 
+### Repeating options ##
+
+You can repeat the following options to add more than one rule of that
+type.
+
+  * `--include`
+  * `--include-from`
+  * `--exclude`
+  * `--exclude-from`
+  * `--filter`
+  * `--filter-from`
+
+Note that all the options of the same type are processed together in
+the order above, regardless of what order they were placed on the
+command line.
+
+So all `--include` options are processed first in the order they
+appeared on the command line, then all `--include-from` options etc.
+
+To mix up the order includes and excludes, the `--filter` flag can be
+used.
+
 ### `--exclude` - Exclude files matching pattern ###
 
 Add a single exclude rule with `--exclude`.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Eg `--exclude *.bak` to exclude all bak files from the sync.
 
 ### `--exclude-from` - Read exclude patterns from file ###
 
 Add exclude rules from a file.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Prepare a file like this `exclude-file.txt`
 
     # a sample exclude rule file
@@ -184,6 +208,9 @@ This is useful if you have a lot of rules.
 
 Add a single include rule with `--include`.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Eg `--include *.{png,jpg}` to include all `png` and `jpg` files in the
 backup and no others.
 
@@ -197,6 +224,9 @@ flexibility then you must use `--filter-from`.
 
 Add include rules from a file.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Prepare a file like this `include-file.txt`
 
     # a sample include rule file
@@ -221,12 +251,18 @@ This can be used to add a single include or exclude rule.  Include
 rules start with `+ ` and exclude rules start with `- `.  A special
 rule called `!` can be used to clear the existing rules.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Eg `--filter "- *.bak"` to exclude all bak files from the sync.
 
 ### `--filter-from` - Read filtering patterns from a file ###
 
 Add include/exclude rules from a file.
 
+This flag can be repeated.  See above for the order the flags are
+processed in.
+
 Prepare a file like this `filter-file.txt`
 
     # a sample exclude rule file
@@ -250,6 +286,9 @@ This reads a list of file names from the file passed in and **only**
 these files are transferred.  The filtering rules are ignored
 completely if you use this option.
 
+This option can be repeated to read from more than one file.  These
+are read in the order that they are placed on the command line.
+
 Prepare a file like this `files-from.txt`
 
     # comment

fs/filter.go

@@ -20,13 +20,13 @@ import (
 var (
 	// Flags
 	deleteExcluded = pflag.BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync")
-	filterRule     = pflag.StringP("filter", "f", "", "Add a file-filtering rule")
+	filterRule     = pflag.StringArrayP("filter", "f", nil, "Add a file-filtering rule")
-	filterFrom     = pflag.StringP("filter-from", "", "", "Read filtering patterns from a file")
+	filterFrom     = pflag.StringArrayP("filter-from", "", nil, "Read filtering patterns from a file")
-	excludeRule    = pflag.StringP("exclude", "", "", "Exclude files matching pattern")
+	excludeRule    = pflag.StringArrayP("exclude", "", nil, "Exclude files matching pattern")
-	excludeFrom    = pflag.StringP("exclude-from", "", "", "Read exclude patterns from file")
+	excludeFrom    = pflag.StringArrayP("exclude-from", "", nil, "Read exclude patterns from file")
-	includeRule    = pflag.StringP("include", "", "", "Include files matching pattern")
+	includeRule    = pflag.StringArrayP("include", "", nil, "Include files matching pattern")
-	includeFrom    = pflag.StringP("include-from", "", "", "Read include patterns from file")
+	includeFrom    = pflag.StringArrayP("include-from", "", nil, "Read include patterns from file")
-	filesFrom      = pflag.StringP("files-from", "", "", "Read list of source-file names from file")
+	filesFrom      = pflag.StringArrayP("files-from", "", nil, "Read list of source-file names from file")
 	minAge         = pflag.StringP("min-age", "", "", "Don't transfer any file younger than this in s or suffix ms|s|m|h|d|w|M|y")
 	maxAge         = pflag.StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
 	minSize        = SizeSuffix(-1)
@@ -157,15 +157,18 @@ func NewFilter() (f *Filter, err error) {
 	}
 	addImplicitExclude := false
 
-	if *includeRule != "" {
+	if includeRule != nil {
-		err = f.Add(true, *includeRule)
+		for _, rule := range *includeRule {
+			err = f.Add(true, rule)
 			if err != nil {
 				return nil, err
 			}
 			addImplicitExclude = true
 		}
-	if *includeFrom != "" {
+	}
-		err := forEachLine(*includeFrom, func(line string) error {
+	if includeFrom != nil {
+		for _, rule := range *includeFrom {
+			err := forEachLine(rule, func(line string) error {
 				return f.Add(true, line)
 			})
 			if err != nil {
@@ -173,40 +176,51 @@ func NewFilter() (f *Filter, err error) {
 			}
 			addImplicitExclude = true
 		}
-	if *excludeRule != "" {
+	}
-		err = f.Add(false, *excludeRule)
+	if excludeRule != nil {
+		for _, rule := range *excludeRule {
+			err = f.Add(false, rule)
 			if err != nil {
 				return nil, err
 			}
 		}
-	if *excludeFrom != "" {
+	}
-		err := forEachLine(*excludeFrom, func(line string) error {
+	if excludeFrom != nil {
+		for _, rule := range *excludeFrom {
+			err := forEachLine(rule, func(line string) error {
 				return f.Add(false, line)
 			})
 			if err != nil {
 				return nil, err
 			}
 		}
-	if *filterRule != "" {
+	}
-		err = f.AddRule(*filterRule)
+	if filterRule != nil {
+		for _, rule := range *filterRule {
+			err = f.AddRule(rule)
 			if err != nil {
 				return nil, err
 			}
 		}
-	if *filterFrom != "" {
+	}
-		err := forEachLine(*filterFrom, f.AddRule)
+	if filterFrom != nil {
+		for _, rule := range *filterFrom {
+			err := forEachLine(rule, f.AddRule)
 			if err != nil {
 				return nil, err
 			}
 		}
-	if *filesFrom != "" {
+	}
-		err := forEachLine(*filesFrom, func(line string) error {
+	if filesFrom != nil {
+		for _, rule := range *filesFrom {
+			err := forEachLine(rule, func(line string) error {
 				return f.AddFile(line)
 			})
 			if err != nil {
 				return nil, err
 			}
 		}
+	}
 	if addImplicitExclude {
 		err = f.Add(false, "/**")
 		if err != nil {

fs/filter_test.go

@@ -54,13 +54,8 @@ func TestNewFilterDefault(t *testing.T) {
 	assert.True(t, f.InActive())
 }
 
-// return a pointer to the string
-func stringP(s string) *string {
-	return &s
-}
-
 // testFile creates a temp file with the contents
-func testFile(t *testing.T, contents string) *string {
+func testFile(t *testing.T, contents string) string {
 	out, err := ioutil.TempFile("", "filter_test")
 	require.NoError(t, err)
 	defer func() {
@@ -70,25 +65,24 @@ func testFile(t *testing.T, contents string) *string {
 	_, err = out.Write([]byte(contents))
 	require.NoError(t, err)
 	s := out.Name()
-	return &s
+	return s
 }
 
 func TestNewFilterFull(t *testing.T) {
 	mins := int64(100 * 1024)
 	maxs := int64(1000 * 1024)
-	emptyString := ""
 	isFalse := false
 	isTrue := true
 
 	// Set up the input
 	deleteExcluded = &isTrue
-	filterRule = stringP("- filter1")
+	filterRule = &[]string{"- filter1", "- filter1b"}
-	filterFrom = testFile(t, "#comment\n+ filter2\n- filter3\n")
+	filterFrom = &[]string{testFile(t, "#comment\n+ filter2\n- filter3\n")}
-	excludeRule = stringP("exclude1")
+	excludeRule = &[]string{"exclude1"}
-	excludeFrom = testFile(t, "#comment\nexclude2\nexclude3\n")
+	excludeFrom = &[]string{testFile(t, "#comment\nexclude2\nexclude3\n")}
-	includeRule = stringP("include1")
+	includeRule = &[]string{"include1"}
-	includeFrom = testFile(t, "#comment\ninclude2\ninclude3\n")
+	includeFrom = &[]string{testFile(t, "#comment\ninclude2\ninclude3\n")}
-	filesFrom = testFile(t, "#comment\nfiles1\nfiles2\n")
+	filesFrom = &[]string{testFile(t, "#comment\nfiles1\nfiles2\n")}
 	minSize = SizeSuffix(mins)
 	maxSize = SizeSuffix(maxs)
 
@@ -100,20 +94,20 @@ func TestNewFilterFull(t *testing.T) {
 	}
 	// Reset the input
 	defer func() {
-		rm(*filterFrom)
+		rm((*filterFrom)[0])
-		rm(*excludeFrom)
+		rm((*excludeFrom)[0])
-		rm(*includeFrom)
+		rm((*includeFrom)[0])
-		rm(*filesFrom)
+		rm((*filesFrom)[0])
 		minSize = -1
 		maxSize = -1
 		deleteExcluded = &isFalse
-		filterRule = &emptyString
+		filterRule = nil
-		filterFrom = &emptyString
+		filterFrom = nil
-		excludeRule = &emptyString
+		excludeRule = nil
-		excludeFrom = &emptyString
+		excludeFrom = nil
-		includeRule = &emptyString
+		includeRule = nil
-		includeFrom = &emptyString
+		includeFrom = nil
-		filesFrom = &emptyString
+		filesFrom = nil
 	}()
 
 	f, err := NewFilter()
@@ -130,6 +124,7 @@ func TestNewFilterFull(t *testing.T) {
 - (^|/)exclude2$
 - (^|/)exclude3$
 - (^|/)filter1$
+- (^|/)filter1b$
 + (^|/)filter2$
 - (^|/)filter3$
 - ^.*$
@@ -356,11 +351,11 @@ four
 five
   six  `)
 	defer func() {
-		err := os.Remove(*file)
+		err := os.Remove(file)
 		require.NoError(t, err)
 	}()
 	lines := []string{}
-	err := forEachLine(*file, func(s string) error {
+	err := forEachLine(file, func(s string) error {
 		lines = append(lines, s)
 		return nil
 	})