diff --git a/cmd/cmd.go b/cmd/cmd.go index 203253601..835033ef9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -29,7 +29,6 @@ import ( "github.com/rclone/rclone/fs/config/configflags" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/filter" - "github.com/rclone/rclone/fs/filter/filterflags" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fspath" fslog "github.com/rclone/rclone/fs/log" @@ -383,6 +382,12 @@ func StartStats() func() { // initConfig is run by cobra after initialising the flags func initConfig() { + // Set the global options from the flags + err := fs.GlobalOptionsInit() + if err != nil { + log.Fatalf("Failed to initialise global options: %v", err) + } + ctx := context.Background() ci := fs.GetConfig(ctx) @@ -409,12 +414,6 @@ func initConfig() { terminal.EnableColorsStdout() } - // Load filters - err := filterflags.Reload(ctx) - if err != nil { - log.Fatalf("Failed to load filters: %v", err) - } - // Write the args for debug purposes fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) diff --git a/fs/configmap.go b/fs/configmap.go index 9a0610c8e..4155f830d 100644 --- a/fs/configmap.go +++ b/fs/configmap.go @@ -33,10 +33,15 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) { if opt == nil { return "", false } - envKey := OptionToEnv(oev.prefix + "-" + key) + var envKey string + if oev.prefix == "" { + envKey = OptionToEnv(key) + } else { + envKey = OptionToEnv(oev.prefix + "-" + key) + } value, ok = os.LookupEnv(envKey) if ok { - Debugf(nil, "Setting %s_%s=%q from environment variable %s", oev.prefix, key, value, envKey) + Debugf(nil, "Setting %s %s=%q from environment variable %s", oev.prefix, key, value, envKey) } else if opt.NoPrefix { // For options with NoPrefix set, check without prefix too envKey := OptionToEnv(key) @@ -115,10 +120,12 @@ func ConfigMap(prefix string, options Options, configName string, connectionStri } // remote specific environment vars - config.AddGetter(configEnvVars(configName), configmap.PriorityNormal) + if configName != "" { + config.AddGetter(configEnvVars(configName), configmap.PriorityNormal) + } // backend specific environment vars - if options != nil && prefix != "" { + if options != nil { config.AddGetter(optionEnvVars{prefix: prefix, options: options}, configmap.PriorityNormal) } diff --git a/fs/rc/config.go b/fs/rc/config.go index f0cc4503a..1f538472d 100644 --- a/fs/rc/config.go +++ b/fs/rc/config.go @@ -12,21 +12,17 @@ import ( "github.com/rclone/rclone/fs/filter" ) -var ( - optionBlock = map[string]interface{}{} - optionReload = map[string]func(context.Context) error{} -) - // AddOption adds an option set func AddOption(name string, option interface{}) { - optionBlock[name] = option + // FIXME remove this function when conversion to options is complete + fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option}) } // AddOptionReload adds an option set with a reload function to be // called when options are changed func AddOptionReload(name string, option interface{}, reload func(context.Context) error) { - optionBlock[name] = option - optionReload[name] = reload + // FIXME remove this function when conversion to options is complete + fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option, Reload: reload}) } func init() { @@ -42,8 +38,8 @@ func init() { // Show the list of all the option blocks func rcOptionsBlocks(ctx context.Context, in Params) (out Params, err error) { options := []string{} - for name := range optionBlock { - options = append(options, name) + for _, opt := range fs.OptionsRegistry { + options = append(options, opt.Name) } out = make(Params) out["options"] = options @@ -71,8 +67,8 @@ map to the external options very easily with a few exceptions. // Show the list of all the option blocks func rcOptionsGet(ctx context.Context, in Params) (out Params, err error) { out = make(Params) - for name, options := range optionBlock { - out[name] = options + for _, opt := range fs.OptionsRegistry { + out[opt.Name] = opt.Opt } return out, nil } @@ -144,16 +140,16 @@ And this sets NOTICE level logs (normal without -v) // Set an option in an option block func rcOptionsSet(ctx context.Context, in Params) (out Params, err error) { for name, options := range in { - current := optionBlock[name] - if current == nil { + opt, ok := fs.OptionsRegistry[name] + if !ok { return nil, fmt.Errorf("unknown option block %q", name) } - err := Reshape(current, options) + err := Reshape(opt.Opt, options) if err != nil { return nil, fmt.Errorf("failed to write options from block %q: %w", name, err) } - if reload := optionReload[name]; reload != nil { - err = reload(ctx) + if opt.Reload != nil { + err = opt.Reload(ctx) if err != nil { return nil, fmt.Errorf("failed to reload options from block %q: %w", name, err) } diff --git a/fs/rc/config_test.go b/fs/rc/config_test.go index f3c75d164..b046f24bd 100644 --- a/fs/rc/config_test.go +++ b/fs/rc/config_test.go @@ -13,10 +13,10 @@ import ( ) func clearOptionBlock() func() { - oldOptionBlock := optionBlock - optionBlock = map[string]interface{}{} + oldOptionBlock := fs.OptionsRegistry + fs.OptionsRegistry = map[string]fs.OptionsInfo{} return func() { - optionBlock = oldOptionBlock + fs.OptionsRegistry = oldOptionBlock } } @@ -30,22 +30,20 @@ var testOptions = struct { func TestAddOption(t *testing.T) { defer clearOptionBlock()() - assert.Equal(t, len(optionBlock), 0) + assert.Equal(t, len(fs.OptionsRegistry), 0) AddOption("potato", &testOptions) - assert.Equal(t, len(optionBlock), 1) - assert.Equal(t, len(optionReload), 0) - assert.Equal(t, &testOptions, optionBlock["potato"]) + assert.Equal(t, len(fs.OptionsRegistry), 1) + assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt) } func TestAddOptionReload(t *testing.T) { defer clearOptionBlock()() - assert.Equal(t, len(optionBlock), 0) + assert.Equal(t, len(fs.OptionsRegistry), 0) reload := func(ctx context.Context) error { return nil } AddOptionReload("potato", &testOptions, reload) - assert.Equal(t, len(optionBlock), 1) - assert.Equal(t, len(optionReload), 1) - assert.Equal(t, &testOptions, optionBlock["potato"]) - assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", optionReload["potato"])) + assert.Equal(t, len(fs.OptionsRegistry), 1) + assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt) + assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", fs.OptionsRegistry["potato"].Reload)) } func TestOptionsBlocks(t *testing.T) { diff --git a/fs/registry.go b/fs/registry.go index 2257623df..44f1606c1 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -61,6 +61,22 @@ func (ri *RegInfo) FileName() string { // Options is a slice of configuration Option for a backend type Options []Option +// Add more options returning a new options slice +func (os Options) Add(newOptions Options) Options { + return append(os, newOptions...) +} + +// AddPrefix adds more options with a prefix returning a new options slice +func (os Options) AddPrefix(newOptions Options, prefix string, groups string) Options { + for _, opt := range newOptions { + // opt is a copy so can modify + opt.Name = prefix + "_" + opt.Name + opt.Groups = groups + os = append(os, opt) + } + return os +} + // Set the default values for the options func (os Options) setValues() { for i := range os { @@ -91,6 +107,19 @@ func (os Options) Get(name string) *Option { return nil } +// SetDefault sets the default for the Option corresponding to name +// +// Writes an ERROR level log if the option is not found +func (os Options) SetDefault(name string, def any) Options { + opt := os.Get(name) + if opt == nil { + Errorf(nil, "Couldn't find option %q to SetDefault on", name) + } else { + opt.Default = def + } + return os +} + // Overridden discovers which config items have been overridden in the // configmap passed in, either by the config string, command line // flags or environment variables @@ -370,6 +399,62 @@ func MustFind(name string) *RegInfo { return fs } +// OptionsInfo holds info about an block of options +type OptionsInfo struct { + Name string // name of this options block for the rc + Opt interface{} // pointer to a struct to set the options in + Options Options // description of the options + Reload func(context.Context) error // if not nil, call when options changed and on init +} + +// OptionsRegistry is a registry of global options +var OptionsRegistry = map[string]OptionsInfo{} + +// RegisterGlobalOptions registers global options to be made into +// command line options, rc options and environment variable reading. +// +// Packages which need global options should use this in an init() function +func RegisterGlobalOptions(oi OptionsInfo) { + OptionsRegistry[oi.Name] = oi +} + +// load the defaults from the options +// +// Reload the options if required +func (oi *OptionsInfo) load() error { + if oi.Options == nil { + Errorf(nil, "No options defined for config block %q", oi.Name) + return nil + } + + m := ConfigMap("", oi.Options, "", nil) + err := configstruct.Set(m, oi.Opt) + if err != nil { + return fmt.Errorf("failed to initialise %q options: %w", oi.Name, err) + } + + if oi.Reload != nil { + err = oi.Reload(context.Background()) + if err != nil { + return fmt.Errorf("failed to reload %q options: %w", oi.Name, err) + } + } + return nil +} + +// GlobalOptionsInit initialises the defaults of global options to +// their values read from the options, environment variables and +// command line parameters. +func GlobalOptionsInit() error { + for _, opt := range OptionsRegistry { + err := opt.load() + if err != nil { + return err + } + } + return nil +} + // Type returns a textual string to identify the type of the remote func Type(f Fs) string { typeName := fmt.Sprintf("%T", f)