filter: convert options to new style

fs/filter/filter.go

@@ -19,24 +19,137 @@ import (
 // This is accessed through GetConfig and AddConfig
 var globalConfig = mustNewFilter(nil)
 
+// OptionsInfo describes the Options in use
+var OptionsInfo = fs.Options{{
+	Name:    "delete_excluded",
+	Default: false,
+	Help:    "Delete files on dest excluded from sync",
+	Groups:  "Filter",
+}, {
+	Name:    "exclude_if_present",
+	Default: []string{},
+	Help:    "Exclude directories if filename is present",
+	Groups:  "Filter",
+}, {
+	Name:    "files_from",
+	Default: []string{},
+	Help:    "Read list of source-file names from file (use - to read from stdin)",
+	Groups:  "Filter",
+}, {
+	Name:    "files_from_raw",
+	Default: []string{},
+	Help:    "Read list of source-file names from file without any processing of lines (use - to read from stdin)",
+	Groups:  "Filter",
+}, {
+	Name:    "min_age",
+	Default: fs.DurationOff,
+	Help:    "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y",
+	Groups:  "Filter",
+}, {
+	Name:    "max_age",
+	Default: fs.DurationOff,
+	Help:    "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y",
+	Groups:  "Filter",
+}, {
+	Name:    "min_size",
+	Default: fs.SizeSuffix(-1),
+	Help:    "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P",
+	Groups:  "Filter",
+}, {
+	Name:    "max_size",
+	Default: fs.SizeSuffix(-1),
+	Help:    "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P",
+	Groups:  "Filter",
+}, {
+	Name:    "ignore_case",
+	Default: false,
+	Help:    "Ignore case in filters (case insensitive)",
+	Groups:  "Filter",
+}, {
+	Name:     "filter",
+	Default:  []string{},
+	ShortOpt: "f",
+	Help:     "Add a file filtering rule",
+	Groups:   "Filter",
+}, {
+	Name:    "filter_from",
+	Default: []string{},
+	Help:    "Read file filtering patterns from a file (use - to read from stdin)",
+	Groups:  "Filter",
+}, {
+	Name:    "exclude",
+	Default: []string{},
+	Help:    "Exclude files matching pattern",
+	Groups:  "Filter",
+}, {
+	Name:    "exclude_from",
+	Default: []string{},
+	Help:    "Read file exclude patterns from file (use - to read from stdin)",
+	Groups:  "Filter",
+}, {
+	Name:    "include",
+	Default: []string{},
+	Help:    "Include files matching pattern",
+	Groups:  "Filter",
+}, {
+	Name:    "include_from",
+	Default: []string{},
+	Help:    "Read file include patterns from file (use - to read from stdin)",
+	Groups:  "Filter",
+}, {
+	Name:    "metadata_filter",
+	Default: []string{},
+	Help:    "Add a metadata filtering rule",
+	Groups:  "Filter,Metadata",
+}, {
+	Name:    "metadata_filter_from",
+	Default: []string{},
+	Help:    "Read metadata filtering patterns from a file (use - to read from stdin)",
+	Groups:  "Filter,Metadata",
+}, {
+	Name:    "metadata_exclude",
+	Default: []string{},
+	Help:    "Exclude metadatas matching pattern",
+	Groups:  "Filter,Metadata",
+}, {
+	Name:    "metadata_exclude_from",
+	Default: []string{},
+	Help:    "Read metadata exclude patterns from file (use - to read from stdin)",
+	Groups:  "Filter,Metadata",
+}, {
+	Name:    "metadata_include",
+	Default: []string{},
+	Help:    "Include metadatas matching pattern",
+	Groups:  "Filter,Metadata",
+}, {
+	Name:    "metadata_include_from",
+	Default: []string{},
+	Help:    "Read metadata include patterns from file (use - to read from stdin)",
+	Groups:  "Filter,Metadata",
+}}
+
 // Options configures the filter
 type Options struct {
-	DeleteExcluded bool
+	DeleteExcluded bool          `config:"delete_excluded"`
 	RulesOpt                     // embedded so we don't change the JSON API
-	ExcludeFile    []string
+	ExcludeFile    []string      `config:"exclude_if_present"`
-	FilesFrom      []string
+	FilesFrom      []string      `config:"files_from"`
-	FilesFromRaw   []string
+	FilesFromRaw   []string      `config:"files_from_raw"`
-	MetaRules      RulesOpt
+	MetaRules      RulesOpt      `config:"metadata"`
-	MinAge         fs.Duration
+	MinAge         fs.Duration   `config:"min_age"`
-	MaxAge         fs.Duration
+	MaxAge         fs.Duration   `config:"max_age"`
-	MinSize        fs.SizeSuffix
+	MinSize        fs.SizeSuffix `config:"min_size"`
-	MaxSize        fs.SizeSuffix
+	MaxSize        fs.SizeSuffix `config:"max_size"`
-	IgnoreCase     bool
+	IgnoreCase     bool          `config:"ignore_case"`
 }
 
-// DefaultOpt is the default config for the filter
+func init() {
-var DefaultOpt = Options{
+	fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "filter", Opt: &Opt, Options: OptionsInfo, Reload: Reload})
-	MinAge:  fs.DurationOff,
+}
+
+// Opt is the default config for the filter
+var Opt = Options{
+	MinAge:  fs.DurationOff, // These have to be set here as the options are parsed once before the defaults are set
 	MaxAge:  fs.DurationOff,
 	MinSize: fs.SizeSuffix(-1),
 	MaxSize: fs.SizeSuffix(-1),
@@ -66,7 +179,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
 	if opt != nil {
 		f.Opt = *opt
 	} else {
-		f.Opt = DefaultOpt
+		f.Opt = Opt
 	}
 
 	// Filter flags
@@ -77,7 +190,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
 	if f.Opt.MaxAge.IsSet() {
 		f.ModTimeFrom = time.Now().Add(-time.Duration(f.Opt.MaxAge))
 		if !f.ModTimeTo.IsZero() && f.ModTimeTo.Before(f.ModTimeFrom) {
-			log.Fatal("filter: --min-age can't be larger than --max-age")
+			log.Fatalf("filter: --min-age %q can't be larger than --max-age %q", opt.MinAge, opt.MaxAge)
 		}
 		fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom)
 	}
@@ -568,3 +681,14 @@ func SetUseFilter(ctx context.Context, useFilter bool) context.Context {
 	*pVal = useFilter
 	return context.WithValue(ctx, useFlagContextKey, pVal)
 }
+
+// Reload the filters from the flags
+func Reload(ctx context.Context) (err error) {
+	fi := GetConfig(ctx)
+	newFilter, err := NewFilter(&Opt)
+	if err != nil {
+		return err
+	}
+	*fi = *newFilter
+	return nil
+}

fs/filter/filter_test.go

@@ -43,7 +43,7 @@ func testFile(t *testing.T, contents string) string {
 }
 
 func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {
-	Opt := DefaultOpt
+	Opt := Opt
 
 	// Set up the input
 	Opt.FilterRule = []string{"- filter1", "- filter1b"}
@@ -66,7 +66,7 @@ func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {
 }
 
 func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {
-	Opt := DefaultOpt
+	Opt := Opt
 
 	// Set up the input
 	Opt.FilterRule = []string{"- filter1", "- filter1b"}
@@ -89,7 +89,7 @@ func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {
 }
 
 func TestNewFilterWithFilesFromAlone(t *testing.T) {
-	Opt := DefaultOpt
+	Opt := Opt
 
 	// Set up the input
 	Opt.FilesFrom = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}
@@ -117,7 +117,7 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) {
 }
 
 func TestNewFilterWithFilesFromRaw(t *testing.T) {
-	Opt := DefaultOpt
+	Opt := Opt
 
 	// Set up the input
 	Opt.FilesFromRaw = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}
@@ -145,7 +145,7 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {
 }
 
 func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {
-	Opt := DefaultOpt
+	Opt := Opt
 
 	mins := fs.SizeSuffix(100 * 1024)
 	maxs := fs.SizeSuffix(1000 * 1024)

fs/filter/filterflags/filterflags.go

@@ -2,62 +2,12 @@
 package filterflags
 
 import (
-	"context"
-	"fmt"
-
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/filter"
-	"github.com/rclone/rclone/fs/rc"
 	"github.com/spf13/pflag"
 )
 
-// Options set by command line flags
-var (
-	Opt = filter.DefaultOpt
-)
-
-// Reload the filters from the flags
-func Reload(ctx context.Context) (err error) {
-	fi := filter.GetConfig(ctx)
-	newFilter, err := filter.NewFilter(&Opt)
-	if err != nil {
-		return err
-	}
-	*fi = *newFilter
-	return nil
-}
-
-// AddRuleFlags add a set of rules flags with prefix
-func AddRuleFlags(flagSet *pflag.FlagSet, Opt *filter.RulesOpt, what, prefix string) {
-	shortFilter := ""
-	if prefix == "" {
-		shortFilter = "f"
-	}
-	group := "Filter"
-	if prefix == "metadata-" {
-		group += ",Metadata"
-	}
-	flags.StringArrayVarP(flagSet, &Opt.FilterRule, prefix+"filter", shortFilter, nil, fmt.Sprintf("Add a %s filtering rule", what), group)
-	flags.StringArrayVarP(flagSet, &Opt.FilterFrom, prefix+"filter-from", "", nil, fmt.Sprintf("Read %s filtering patterns from a file (use - to read from stdin)", what), group)
-	flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, prefix+"exclude", "", nil, fmt.Sprintf("Exclude %ss matching pattern", what), group)
-	flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, prefix+"exclude-from", "", nil, fmt.Sprintf("Read %s exclude patterns from file (use - to read from stdin)", what), group)
-	flags.StringArrayVarP(flagSet, &Opt.IncludeRule, prefix+"include", "", nil, fmt.Sprintf("Include %ss matching pattern", what), group)
-	flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, prefix+"include-from", "", nil, fmt.Sprintf("Read %s include patterns from file (use - to read from stdin)", what), group)
-}
-
 // AddFlags adds the non filing system specific flags to the command
 func AddFlags(flagSet *pflag.FlagSet) {
-	rc.AddOptionReload("filter", &Opt, Reload)
+	flags.AddFlagsFromOptions(flagSet, "", filter.OptionsInfo)
-	flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync", "Filter")
-	AddRuleFlags(flagSet, &Opt.RulesOpt, "file", "")
-	AddRuleFlags(flagSet, &Opt.MetaRules, "metadata", "metadata-")
-	flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present", "Filter")
-	flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)", "Filter")
-	flags.StringArrayVarP(flagSet, &Opt.FilesFromRaw, "files-from-raw", "", nil, "Read list of source-file names from file without any processing of lines (use - to read from stdin)", "Filter")
-	flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")
-	flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")
-	flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P", "Filter")
-	flags.FVarP(flagSet, &Opt.MaxSize, "max-size", "", "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P", "Filter")
-	flags.BoolVarP(flagSet, &Opt.IgnoreCase, "ignore-case", "", false, "Ignore case in filters (case insensitive)", "Filter")
-	//cvsExclude     = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
 }

fs/filter/rules.go

@@ -12,12 +12,12 @@ import (
 
 // RulesOpt is configuration for a rule set
 type RulesOpt struct {
-	FilterRule  []string
+	FilterRule  []string `config:"filter"`
-	FilterFrom  []string
+	FilterFrom  []string `config:"filter_from"`
-	ExcludeRule []string
+	ExcludeRule []string `config:"exclude"`
-	ExcludeFrom []string
+	ExcludeFrom []string `config:"exclude_from"`
-	IncludeRule []string
+	IncludeRule []string `config:"include"`
-	IncludeFrom []string
+	IncludeFrom []string `config:"include_from"`
 }
 
 // rule is one filter rule