config create: add --non-interactive and --continue parameters #3455

This adds a mechanism to add external interfaces to rclone using the
state based configuration.

cmd/config/config.go

@@ -105,12 +105,9 @@ var configProvidersCommand = &cobra.Command{
 	},
 }
 
-var (
+var updateRemoteOpt config.UpdateRemoteOpt
-	configObscure   bool
-	configNoObscure bool
-)
 
-const configPasswordHelp = `
+var configPasswordHelp = strings.ReplaceAll(`
 If any of the parameters passed is a password field, then rclone will
 automatically obscure them if they aren't already obscured before
 putting them in the config file.
@@ -119,12 +116,57 @@ putting them in the config file.
 consists only of base64 characters then rclone can get confused about
 whether the password is already obscured or not and put unobscured
 passwords into the config file. If you want to be 100% certain that
-the passwords get obscured then use the "--obscure" flag, or if you
+the passwords get obscured then use the |--obscure| flag, or if you
 are 100% certain you are already passing obscured passwords then use
-"--no-obscure".  You can also set obscured passwords using the
+|--no-obscure|.  You can also set obscured passwords using the
-"rclone config password" command.
+|rclone config password| command.
-`
 
+The flag |--non-interactive| is for use by applications that wish to
+configure rclone themeselves, rather than using rclone's text based
+configuration questions. If this flag is set, and rclone needs to ask
+the user a question, a JSON blob will be returned with the question in
+it.
+
+This will look something like (some irrelevant detail removed):
+
+{
+    "State": "*oauth-islocal,teamdrive,,",
+    "Option": {
+        "Name": "config_is_local",
+        "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n",
+        "Default": true,
+        "Examples": [
+            {
+                "Value": "true",
+                "Help": "Yes"
+            },
+            {
+                "Value": "false",
+                "Help": "No"
+            }
+        ],
+        "Required": false,
+        "IsPassword": false,
+        "Type": "bool"
+    },
+    "Error": "",
+}
+
+The format of |Option| is the same as returned by |rclone config
+providers|. The question should be asked to the user and returned to
+rclone as a string parameter along with the state parameter.
+
+If |Error| is set then it should be shown to the user at the same
+time as the question.
+
+    rclone config update name --continue state "*oauth-islocal,teamdrive,," result "true"
+
+Note that when using |--continue| all passwords should be passed in
+the clear (not obscured).
+
+At the end of the non interactive process, rclone will return a result
+with |State| as empty string.
+`, "|", "`")
 var configCreateCommand = &cobra.Command{
 	Use:   "create `name` `type` [`key` `value`]*",
 	Short: `Create a new remote with name, type and options.`,
@@ -152,19 +194,39 @@ using remote authorization you would do this:
 		if err != nil {
 			return err
 		}
-		err = config.CreateRemote(context.Background(), args[0], args[1], in, configObscure, configNoObscure)
+		return doConfig(args[0], in, func(opts config.UpdateRemoteOpt) (*fs.ConfigOut, error) {
+			return config.CreateRemote(context.Background(), args[0], args[1], in, opts)
+		})
+	},
+}
+
+func doConfig(name string, in rc.Params, do func(config.UpdateRemoteOpt) (*fs.ConfigOut, error)) error {
+	out, err := do(updateRemoteOpt)
 	if err != nil {
 		return err
 	}
-		config.ShowRemote(args[0])
+	if !(updateRemoteOpt.NonInteractive || updateRemoteOpt.Continue) {
+		config.ShowRemote(name)
+	} else {
+		if out == nil {
+			out = &fs.ConfigOut{}
+		}
+		outBytes, err := json.MarshalIndent(out, "", "\t")
+		if err != nil {
+			return err
+		}
+		_, _ = os.Stdout.Write(outBytes)
+		_, _ = os.Stdout.WriteString("\n")
+	}
 	return nil
-	},
 }
 
 func init() {
 	for _, cmdFlags := range []*pflag.FlagSet{configCreateCommand.Flags(), configUpdateCommand.Flags()} {
-		flags.BoolVarP(cmdFlags, &configObscure, "obscure", "", false, "Force any passwords to be obscured.")
+		flags.BoolVarP(cmdFlags, &updateRemoteOpt.Obscure, "obscure", "", false, "Force any passwords to be obscured.")
-		flags.BoolVarP(cmdFlags, &configNoObscure, "no-obscure", "", false, "Force any passwords not to be obscured.")
+		flags.BoolVarP(cmdFlags, &updateRemoteOpt.NoObscure, "no-obscure", "", false, "Force any passwords not to be obscured.")
+		flags.BoolVarP(cmdFlags, &updateRemoteOpt.NonInteractive, "non-interactive", "", false, "Don't interact with user and return questions.")
+		flags.BoolVarP(cmdFlags, &updateRemoteOpt.Continue, "continue", "", false, "Continue the configuration process with an answer.")
 	}
 }
 
@@ -191,12 +253,9 @@ require this add an extra parameter thus:
 		if err != nil {
 			return err
 		}
-		err = config.UpdateRemote(context.Background(), args[0], in, configObscure, configNoObscure)
+		return doConfig(args[0], in, func(opts config.UpdateRemoteOpt) (*fs.ConfigOut, error) {
-		if err != nil {
+			return config.UpdateRemote(context.Background(), args[0], in, opts)
-			return err
+		})
-		}
-		config.ShowRemote(args[0])
-		return nil
 	},
 }
 

fs/backend_config.go

@@ -258,10 +258,42 @@ func StatePop(state string) (newState string, value string) {
 //
 // It wraps any OAuth transactions as necessary so only straight forward config questions are emitted
 func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
+	for {
+		out, err = backendConfigStep(ctx, name, m, ri, in)
+		if err != nil {
+			break
+		}
+		if out == nil || out.State == "" {
+			// finished
+			break
+		}
+		if out.Option != nil {
+			// question to ask user
+			break
+		}
+		if out.Error != "" {
+			// error to show user
+			break
+		}
+		// non terminal state, but no question to ask or error to show - loop here
+		in = ConfigIn{
+			State:  out.State,
+			Result: out.Result,
+		}
+	}
+	return out, err
+}
+
+func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
 	ci := GetConfig(ctx)
 	if ri.Config == nil {
 		return nil, nil
 	}
+	Debugf(name, "config in: state=%q, result=%q", in.State, in.Result)
+	defer func() {
+		Debugf(name, "config out: out=%+v, err=%v", out, err)
+	}()
+
 	switch {
 	case strings.HasPrefix(in.State, "*oauth"):
 		// Do internal oauth states
@@ -299,7 +331,7 @@ func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *Reg
 		// If AutoConfirm is set, choose the default value
 		if ci.AutoConfirm {
 			result := fmt.Sprint(out.Option.Default)
-			Debugf(nil, "Auto confirm is set, choosing default %q for state %q", result, out.State)
+			Debugf(nil, "Auto confirm is set, choosing default %q for state %q, override by setting config parameter %q", result, out.State, out.Option.Name)
 			return ConfigResult(out.State, result)
 		}
 	}

fs/config/config.go

@@ -408,35 +408,53 @@ func getWithDefault(name, key, defaultValue string) string {
 	return value
 }
 
+// UpdateRemoteOpt configures the remote update
+type UpdateRemoteOpt struct {
+	// Treat all passwords as plain that need obscuring
+	Obscure bool `json:"obscure"`
+	// Treat all passwords as obscured
+	NoObscure bool `json:"noObscure"`
+	// Don't interact with the user - return questions
+	NonInteractive bool `json:"nonInteractive"`
+	// If set then supply state and result parameters to continue the process
+	Continue bool `json:"continue"`
+}
+
 // UpdateRemote adds the keyValues passed in to the remote of name.
 // keyValues should be key, value pairs.
-func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscure, noObscure bool) error {
+func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, opt UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
-	if doObscure && noObscure {
+	if opt.Obscure && opt.NoObscure {
-		return errors.New("can't use --obscure and --no-obscure together")
+		return nil, errors.New("can't use --obscure and --no-obscure together")
 	}
-	err := fspath.CheckConfigName(name)
+	err = fspath.CheckConfigName(name)
 	if err != nil {
-		return err
+		return nil, err
 	}
+	interactive := !(opt.NonInteractive || opt.Continue)
+	if interactive {
 		ctx = suppressConfirm(ctx)
+	}
 
+	fsType := FileGet(name, "type")
+	if fsType == "" {
+		return nil, errors.New("couldn't find type field in config")
+	}
+
+	ri, err := fs.Find(fsType)
+	if err != nil {
+		return nil, errors.Errorf("couldn't find backend for type %q", fsType)
+	}
+
+	if !opt.Continue {
 		// Work out which options need to be obscured
 		needsObscure := map[string]struct{}{}
-	if !noObscure {
+		if !opt.NoObscure {
-		if fsType := FileGet(name, "type"); fsType != "" {
+			for _, option := range ri.Options {
-			if ri, err := fs.Find(fsType); err != nil {
+				if option.IsPassword {
-				fs.Debugf(nil, "Couldn't find fs for type %q", fsType)
+					needsObscure[option.Name] = struct{}{}
-			} else {
-				for _, opt := range ri.Options {
-					if opt.IsPassword {
-						needsObscure[opt.Name] = struct{}{}
 				}
 			}
 		}
-		} else {
-			fs.Debugf(nil, "UpdateRemote: Couldn't find fs type")
-		}
-	}
 
 		// Set the config
 		for k, v := range keyValues {
@@ -444,40 +462,63 @@ func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscu
 			// Obscure parameter if necessary
 			if _, ok := needsObscure[k]; ok {
 				_, err := obscure.Reveal(vStr)
-			if err != nil || doObscure {
+				if err != nil || opt.Obscure {
 					// If error => not already obscured, so obscure it
 					// or we are forced to obscure
 					vStr, err = obscure.Obscure(vStr)
 					if err != nil {
-					return errors.Wrap(err, "UpdateRemote: obscure failed")
+						return nil, errors.Wrap(err, "UpdateRemote: obscure failed")
 					}
 				}
 			}
 			LoadedData().SetValue(name, k, vStr)
 		}
+	}
+
+	if interactive {
 		err = RemoteConfig(ctx, name)
+	} else {
+		// Start the config state machine
+		m := fs.ConfigMap(ri, name, nil)
+		in := fs.ConfigIn{}
+		if opt.Continue {
+			if state, ok := keyValues["state"]; ok {
+				in.State = fmt.Sprint(state)
+			} else {
+				return nil, errors.New("UpdateRemote: need state parameter with --continue")
+			}
+			if result, ok := keyValues["result"]; ok {
+				in.Result = fmt.Sprint(result)
+			} else {
+				return nil, errors.New("UpdateRemote: need result parameter with --continue")
+			}
+		}
+		out, err = fs.BackendConfig(ctx, name, m, ri, in)
+	}
 	if err != nil {
-		return err
+		return nil, err
 	}
 	SaveConfig()
 	cache.ClearConfig(name) // remove any remotes based on this config from the cache
-	return nil
+	return out, nil
 }
 
 // CreateRemote creates a new remote with name, provider and a list of
 // parameters which are key, value pairs.  If update is set then it
 // adds the new keys rather than replacing all of them.
-func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, doObscure, noObscure bool) error {
+func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, opts UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
-	err := fspath.CheckConfigName(name)
+	err = fspath.CheckConfigName(name)
 	if err != nil {
-		return err
+		return nil, err
 	}
+	if !opts.Continue {
 		// Delete the old config if it exists
 		LoadedData().DeleteSection(name)
 		// Set the type
 		LoadedData().SetValue(name, "type", provider)
+	}
 	// Set the remaining values
-	return UpdateRemote(ctx, name, keyValues, doObscure, noObscure)
+	return UpdateRemote(ctx, name, keyValues, opts)
 }
 
 // PasswordRemote adds the keyValues passed in to the remote of name.
@@ -491,7 +532,10 @@ func PasswordRemote(ctx context.Context, name string, keyValues rc.Params) error
 	for k, v := range keyValues {
 		keyValues[k] = obscure.MustObscure(fmt.Sprint(v))
 	}
-	return UpdateRemote(ctx, name, keyValues, false, true)
+	_, err = UpdateRemote(ctx, name, keyValues, UpdateRemoteOpt{
+		NoObscure: true,
+	})
+	return err
 }
 
 // JSONListProviders prints all the providers and options in JSON format

fs/config/rc.go

@@ -3,6 +3,7 @@ package config
 import (
 	"context"
 
+	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/rc"
 )
@@ -112,8 +113,12 @@ func init() {
 			extraHelp = "- type - type of the new remote\n"
 		}
 		if name == "create" || name == "update" {
-			extraHelp += "- obscure - optional bool - forces obscuring of passwords\n"
+			extraHelp += `- opt - a dictionary of options to control the configuration
-			extraHelp += "- noObscure - optional bool - forces passwords not to be obscured\n"
+    - obscure - declare passwords are plain and need obscuring
+    - noObscure - declare passwords are already obscured and don't need obscuring
+    - nonInteractive - don't interact with a user, return questions
+    - continue - continue the config process with an answer
+`
 		}
 		rc.Add(rc.Call{
 			Path:         "config/" + name,
@@ -144,21 +149,47 @@ func rcConfig(ctx context.Context, in rc.Params, what string) (out rc.Params, er
 	if err != nil {
 		return nil, err
 	}
-	doObscure, _ := in.GetBool("obscure")
+	var opt UpdateRemoteOpt
-	noObscure, _ := in.GetBool("noObscure")
+	err = in.GetStruct("opt", &opt)
+	if err != nil && !rc.IsErrParamNotFound(err) {
+		return nil, err
+	}
+	// Backwards compatibility
+	if value, err := in.GetBool("obscure"); err == nil {
+		opt.Obscure = value
+	}
+	if value, err := in.GetBool("noObscure"); err == nil {
+		opt.NoObscure = value
+	}
+	var configOut *fs.ConfigOut
 	switch what {
 	case "create":
-		remoteType, err := in.GetString("type")
+		remoteType, typeErr := in.GetString("type")
+		if typeErr != nil {
+			return nil, typeErr
+		}
+		configOut, err = CreateRemote(ctx, name, remoteType, parameters, opt)
+	case "update":
+		configOut, err = UpdateRemote(ctx, name, parameters, opt)
+	case "password":
+		err = PasswordRemote(ctx, name, parameters)
+	default:
+		err = errors.New("unknown rcConfig type")
+	}
 	if err != nil {
 		return nil, err
 	}
-		return nil, CreateRemote(ctx, name, remoteType, parameters, doObscure, noObscure)
+	if !opt.NonInteractive {
-	case "update":
+		return nil, nil
-		return nil, UpdateRemote(ctx, name, parameters, doObscure, noObscure)
-	case "password":
-		return nil, PasswordRemote(ctx, name, parameters)
 	}
-	panic("unknown rcConfig type")
+	if configOut == nil {
+		configOut = &fs.ConfigOut{}
+	}
+	err = rc.Reshape(&out, configOut)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
 }
 
 func init() {

fs/config/ui.go

@@ -252,7 +252,6 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
 		State: "",
 	}
 	for {
-		fs.Debugf(name, "config: state=%q, result=%q", in.State, in.Result)
 		out, err := fs.BackendConfig(ctx, name, m, ri, in)
 		if err != nil {
 			return err
@@ -266,7 +265,7 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
 		in.State = out.State
 		in.Result = out.Result
 		if out.Option != nil {
-			fs.Debugf(name, "config: reading config item named %q", out.Option.Name)
+			fs.Debugf(name, "config: reading config parameter %q", out.Option.Name)
 			if out.Option.Default == nil {
 				out.Option.Default = ""
 			}

fs/config/ui_test.go

@@ -197,10 +197,15 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
 				break
 			}
 			t.Run(fmt.Sprintf("doObscure=%v,noObscure=%v", doObscure, noObscure), func(t *testing.T) {
-				require.NoError(t, config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
+				opt := config.UpdateRemoteOpt{
+					Obscure:   doObscure,
+					NoObscure: noObscure,
+				}
+				_, err := config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
 					"bool": true,
 					"pass": "potato",
-				}, doObscure, noObscure))
+				}, opt)
+				require.NoError(t, err)
 
 				assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
 				assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
@@ -212,11 +217,12 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
 				assert.Equal(t, "potato", gotPw)
 
 				wantPw := obscure.MustObscure("potato2")
-				require.NoError(t, config.UpdateRemote(ctx, "test2", rc.Params{
+				_, err = config.UpdateRemote(ctx, "test2", rc.Params{
 					"bool":  false,
 					"pass":  wantPw,
 					"spare": "spare",
-				}, doObscure, noObscure))
+				}, opt)
+				require.NoError(t, err)
 
 				assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
 				assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))