diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index c5a7278cb..5a52ebb07 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -66,7 +66,7 @@ func init() {
 			IsPassword: true,
 		}, {
 			Name: "key_file",
-			Help: "Path to PEM-encoded private key file, leave blank to use ssh-agent.",
+			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
 		}, {
 			Name: "key_file_pass",
 			Help: `The passphrase to decrypt the PEM-encoded private key file.
@@ -74,6 +74,14 @@ func init() {
 Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
+		}, {
+			Name: "key_use_agent",
+			Help: `When set forces the usage of the ssh-agent.
+
+When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
+requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
+when the ssh-agent contains many keys.`,
+			Default: false,
 		}, {
 			Name:    "use_insecure_cipher",
 			Help:    "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
@@ -130,6 +138,7 @@ type Options struct {
 	Pass              string `config:"pass"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
+	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
 	AskPassword       bool   `config:"ask_password"`
@@ -334,7 +343,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Add ssh agent-auth if no password or file specified
-	if opt.Pass == "" && opt.KeyFile == "" {
+	if (opt.Pass == "" && opt.KeyFile == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -343,7 +352,30 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't read ssh agent signers")
 		}
-		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
+		if opt.KeyFile != "" {
+			pubBytes, err := ioutil.ReadFile(opt.KeyFile + ".pub")
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read public key file")
+			}
+			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse public key file")
+			}
+			pubM := pub.Marshal()
+			found := false
+			for _, s := range signers {
+				if bytes.Equal(pubM, s.PublicKey().Marshal()) {
+					sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
+					found = true
+					break
+				}
+			}
+			if !found {
+				return nil, errors.New("private key not found in the ssh-agent")
+			}
+		} else {
+			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
+		}
 	}
 
 	// Load key file if specified
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index 6554735be..feac88d6d 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -124,11 +124,15 @@ The SFTP remote supports three authentication methods:
   * Key file
   * ssh-agent
 
-Key files should be unencrypted PEM-encoded private key files.  For
-instance `/home/$USER/.ssh/id_rsa`.
+Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`.
+Only unencrypted OpenSSH or PEM encrypted files are supported.
 
-If you don't specify `pass` or `key_file` then rclone will attempt to
-contact an ssh-agent.
+If you don't specify `pass` or `key_file` then rclone will attempt to contact an ssh-agent.
+
+You can also specify `key_use_agent` to force the usage of an ssh-agent. In this case
+`key_file` can also be specified to force the usage of a specific key in the ssh-agent.
+
+Using an ssh-agent is the only way to load encrypted OpenSSH keys at the moment.
 
 If you set the `--sftp-ask-password` option, rclone will prompt for a
 password when needed and no password has been configured.
@@ -204,7 +208,7 @@ SSH password, leave blank to use ssh-agent.
 
 #### --sftp-key-file
 
-Path to PEM-encoded private key file, leave blank to use ssh-agent.
+Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
 
 - Config:      key_file
 - Env Var:     RCLONE_SFTP_KEY_FILE
@@ -223,6 +227,19 @@ in the new OpenSSH format can't be used.
 - Type:        string
 - Default:     ""
 
+#### --sftp-key-use-agent
+
+When set forces the usage of the ssh-agent.
+
+When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
+requested from the ssh-agent. This allows to avoid `Too many authentication failures for *username*` errors
+when the ssh-agent contains many keys.
+
+- Config:      key_use_agent
+- Env Var:     RCLONE_SFTP_KEY_USE_AGENT
+- Type:        bool
+- Default:     false
+
 #### --sftp-use-insecure-cipher
 
 Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.