fs: add Options registry and rework rc to use it

cmd/cmd.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configflags"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/filter"
-	"github.com/rclone/rclone/fs/filter/filterflags"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fspath"
 	fslog "github.com/rclone/rclone/fs/log"
@@ -383,6 +382,12 @@ func StartStats() func() {
 
 // initConfig is run by cobra after initialising the flags
 func initConfig() {
+	// Set the global options from the flags
+	err := fs.GlobalOptionsInit()
+	if err != nil {
+		log.Fatalf("Failed to initialise global options: %v", err)
+	}
+
 	ctx := context.Background()
 	ci := fs.GetConfig(ctx)
 
@@ -409,12 +414,6 @@ func initConfig() {
 		terminal.EnableColorsStdout()
 	}
 
-	// Load filters
-	err := filterflags.Reload(ctx)
-	if err != nil {
-		log.Fatalf("Failed to load filters: %v", err)
-	}
-
 	// Write the args for debug purposes
 	fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
 

fs/configmap.go

@@ -33,10 +33,15 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {
 	if opt == nil {
 		return "", false
 	}
-	envKey := OptionToEnv(oev.prefix + "-" + key)
+	var envKey string
+	if oev.prefix == "" {
+		envKey = OptionToEnv(key)
+	} else {
+		envKey = OptionToEnv(oev.prefix + "-" + key)
+	}
 	value, ok = os.LookupEnv(envKey)
 	if ok {
-		Debugf(nil, "Setting %s_%s=%q from environment variable %s", oev.prefix, key, value, envKey)
+		Debugf(nil, "Setting %s %s=%q from environment variable %s", oev.prefix, key, value, envKey)
 	} else if opt.NoPrefix {
 		// For options with NoPrefix set, check without prefix too
 		envKey := OptionToEnv(key)
@@ -115,10 +120,12 @@ func ConfigMap(prefix string, options Options, configName string, connectionStri
 	}
 
 	// remote specific environment vars
-	config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
+	if configName != "" {
+		config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
+	}
 
 	// backend specific environment vars
-	if options != nil && prefix != "" {
+	if options != nil {
 		config.AddGetter(optionEnvVars{prefix: prefix, options: options}, configmap.PriorityNormal)
 	}
 

fs/rc/config.go

@@ -12,21 +12,17 @@ import (
 	"github.com/rclone/rclone/fs/filter"
 )
 
-var (
-	optionBlock  = map[string]interface{}{}
-	optionReload = map[string]func(context.Context) error{}
-)
-
 // AddOption adds an option set
 func AddOption(name string, option interface{}) {
-	optionBlock[name] = option
+	// FIXME remove this function when conversion to options is complete
+	fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option})
 }
 
 // AddOptionReload adds an option set with a reload function to be
 // called when options are changed
 func AddOptionReload(name string, option interface{}, reload func(context.Context) error) {
-	optionBlock[name] = option
-	optionReload[name] = reload
+	// FIXME remove this function when conversion to options is complete
+	fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option, Reload: reload})
 }
 
 func init() {
@@ -42,8 +38,8 @@ func init() {
 // Show the list of all the option blocks
 func rcOptionsBlocks(ctx context.Context, in Params) (out Params, err error) {
 	options := []string{}
-	for name := range optionBlock {
-		options = append(options, name)
+	for _, opt := range fs.OptionsRegistry {
+		options = append(options, opt.Name)
 	}
 	out = make(Params)
 	out["options"] = options
@@ -71,8 +67,8 @@ map to the external options very easily with a few exceptions.
 // Show the list of all the option blocks
 func rcOptionsGet(ctx context.Context, in Params) (out Params, err error) {
 	out = make(Params)
-	for name, options := range optionBlock {
-		out[name] = options
+	for _, opt := range fs.OptionsRegistry {
+		out[opt.Name] = opt.Opt
 	}
 	return out, nil
 }
@@ -144,16 +140,16 @@ And this sets NOTICE level logs (normal without -v)
 // Set an option in an option block
 func rcOptionsSet(ctx context.Context, in Params) (out Params, err error) {
 	for name, options := range in {
-		current := optionBlock[name]
-		if current == nil {
+		opt, ok := fs.OptionsRegistry[name]
+		if !ok {
 			return nil, fmt.Errorf("unknown option block %q", name)
 		}
-		err := Reshape(current, options)
+		err := Reshape(opt.Opt, options)
 		if err != nil {
 			return nil, fmt.Errorf("failed to write options from block %q: %w", name, err)
 		}
-		if reload := optionReload[name]; reload != nil {
-			err = reload(ctx)
+		if opt.Reload != nil {
+			err = opt.Reload(ctx)
 			if err != nil {
 				return nil, fmt.Errorf("failed to reload options from block %q: %w", name, err)
 			}

fs/rc/config_test.go

@@ -13,10 +13,10 @@ import (
 )
 
 func clearOptionBlock() func() {
-	oldOptionBlock := optionBlock
-	optionBlock = map[string]interface{}{}
+	oldOptionBlock := fs.OptionsRegistry
+	fs.OptionsRegistry = map[string]fs.OptionsInfo{}
 	return func() {
-		optionBlock = oldOptionBlock
+		fs.OptionsRegistry = oldOptionBlock
 	}
 }
 
@@ -30,22 +30,20 @@ var testOptions = struct {
 
 func TestAddOption(t *testing.T) {
 	defer clearOptionBlock()()
-	assert.Equal(t, len(optionBlock), 0)
+	assert.Equal(t, len(fs.OptionsRegistry), 0)
 	AddOption("potato", &testOptions)
-	assert.Equal(t, len(optionBlock), 1)
-	assert.Equal(t, len(optionReload), 0)
-	assert.Equal(t, &testOptions, optionBlock["potato"])
+	assert.Equal(t, len(fs.OptionsRegistry), 1)
+	assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)
 }
 
 func TestAddOptionReload(t *testing.T) {
 	defer clearOptionBlock()()
-	assert.Equal(t, len(optionBlock), 0)
+	assert.Equal(t, len(fs.OptionsRegistry), 0)
 	reload := func(ctx context.Context) error { return nil }
 	AddOptionReload("potato", &testOptions, reload)
-	assert.Equal(t, len(optionBlock), 1)
-	assert.Equal(t, len(optionReload), 1)
-	assert.Equal(t, &testOptions, optionBlock["potato"])
-	assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", optionReload["potato"]))
+	assert.Equal(t, len(fs.OptionsRegistry), 1)
+	assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)
+	assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", fs.OptionsRegistry["potato"].Reload))
 }
 
 func TestOptionsBlocks(t *testing.T) {

fs/registry.go

@@ -61,6 +61,22 @@ func (ri *RegInfo) FileName() string {
 // Options is a slice of configuration Option for a backend
 type Options []Option
 
+// Add more options returning a new options slice
+func (os Options) Add(newOptions Options) Options {
+	return append(os, newOptions...)
+}
+
+// AddPrefix adds more options with a prefix returning a new options slice
+func (os Options) AddPrefix(newOptions Options, prefix string, groups string) Options {
+	for _, opt := range newOptions {
+		// opt is a copy so can modify
+		opt.Name = prefix + "_" + opt.Name
+		opt.Groups = groups
+		os = append(os, opt)
+	}
+	return os
+}
+
 // Set the default values for the options
 func (os Options) setValues() {
 	for i := range os {
@@ -91,6 +107,19 @@ func (os Options) Get(name string) *Option {
 	return nil
 }
 
+// SetDefault sets the default for the Option corresponding to name
+//
+// Writes an ERROR level log if the option is not found
+func (os Options) SetDefault(name string, def any) Options {
+	opt := os.Get(name)
+	if opt == nil {
+		Errorf(nil, "Couldn't find option %q to SetDefault on", name)
+	} else {
+		opt.Default = def
+	}
+	return os
+}
+
 // Overridden discovers which config items have been overridden in the
 // configmap passed in, either by the config string, command line
 // flags or environment variables
@@ -370,6 +399,62 @@ func MustFind(name string) *RegInfo {
 	return fs
 }
 
+// OptionsInfo holds info about an block of options
+type OptionsInfo struct {
+	Name    string                      // name of this options block for the rc
+	Opt     interface{}                 // pointer to a struct to set the options in
+	Options Options                     // description of the options
+	Reload  func(context.Context) error // if not nil, call when options changed and on init
+}
+
+// OptionsRegistry is a registry of global options
+var OptionsRegistry = map[string]OptionsInfo{}
+
+// RegisterGlobalOptions registers global options to be made into
+// command line options, rc options and environment variable reading.
+//
+// Packages which need global options should use this in an init() function
+func RegisterGlobalOptions(oi OptionsInfo) {
+	OptionsRegistry[oi.Name] = oi
+}
+
+// load the defaults from the options
+//
+// Reload the options if required
+func (oi *OptionsInfo) load() error {
+	if oi.Options == nil {
+		Errorf(nil, "No options defined for config block %q", oi.Name)
+		return nil
+	}
+
+	m := ConfigMap("", oi.Options, "", nil)
+	err := configstruct.Set(m, oi.Opt)
+	if err != nil {
+		return fmt.Errorf("failed to initialise %q options: %w", oi.Name, err)
+	}
+
+	if oi.Reload != nil {
+		err = oi.Reload(context.Background())
+		if err != nil {
+			return fmt.Errorf("failed to reload %q options: %w", oi.Name, err)
+		}
+	}
+	return nil
+}
+
+// GlobalOptionsInit initialises the defaults of global options to
+// their values read from the options, environment variables and
+// command line parameters.
+func GlobalOptionsInit() error {
+	for _, opt := range OptionsRegistry {
+		err := opt.load()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Type returns a textual string to identify the type of the remote
 func Type(f Fs) string {
 	typeName := fmt.Sprintf("%T", f)