sftp: add symlink support - fixes #5011

Add new flag (--sftp-links) for backend sftp to recreate symlink from source to
destination.

backend/sftp/sftp.go

@@ -178,13 +178,13 @@ E.g. if shared folders can be found in directories representing volumes:
 E.g. if home directory can be found in a shared folder called "home":
 
     rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory
-	
+
 To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path.
 
 E.g. the first example above could be rewritten as:
 
 	rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2
-	
+
 Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home".
 
 E.g. the second example above should be rewritten as:
@@ -232,6 +232,19 @@ E.g. the second example above should be rewritten as:
 			Help:     "Set to skip any symlinks and any other non regular files.",
 			Advanced: true,
 		}, {
+			Name:    "links",
+			Default: false,
+			Help: `Copy symlinks instead of following them.
+
+This permit to recreate the same symlink structure on the destination.
+
+Only works between two remotes with symlink support. [Currently only supported between two SFTP remotes].
+
+Symlink is validate if target file on destination and on source have same size and same hash. [Except if target is a directory, size and hash are not checked].
+`,
+			Advanced: true,
+		}, {
+
 			Name:     "subsystem",
 			Default:  "sftp",
 			Help:     "Specifies the SSH2 subsystem on the remote host.",
@@ -243,7 +256,7 @@ E.g. the second example above should be rewritten as:
 
 The subsystem option is ignored when server_command is defined.
 
-If adding server_command to the configuration file please note that 
+If adding server_command to the configuration file please note that
 it should not be enclosed in quotes, since that will make rclone fail.
 
 A working example is:
@@ -469,7 +482,7 @@ connection for every hash it calculates.
 			Name:    "socks_proxy",
 			Default: "",
 			Help: `Socks 5 proxy host.
-	
+
 Supports the format user:pass@host:port, user@host:port, host:port.
 
 Example:
@@ -523,6 +536,7 @@ type Options struct {
 	Md5sumCommand           string          `config:"md5sum_command"`
 	Sha1sumCommand          string          `config:"sha1sum_command"`
 	SkipLinks               bool            `config:"skip_links"`
+	TranslateSymlinks       bool            `config:"links"`
 	Subsystem               string          `config:"subsystem"`
 	ServerCommand           string          `config:"server_command"`
 	UseFstat                bool            `config:"use_fstat"`
@@ -568,13 +582,15 @@ type Fs struct {
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
 type Object struct {
-	fs      *Fs
+	fs              *Fs
-	remote  string
+	remote          string
-	size    int64       // size of the object
+	size            int64       // size of the object
-	modTime uint32      // modification time of the object as unix time
+	modTime         uint32      // modification time of the object as unix time
-	mode    os.FileMode // mode bits from the file
+	mode            os.FileMode // mode bits from the file
-	md5sum  *string     // Cached MD5 checksum
+	md5sum          *string     // Cached MD5 checksum
-	sha1sum *string     // Cached SHA1 checksum
+	sha1sum         *string     // Cached SHA1 checksum
+	linkTarget      string      // If object isSymlink, this is the target
+	linkTargetIsDir bool        // If object isSymlink, this is true if the target is a directory
 }
 
 // conn encapsulates an ssh client and corresponding sftp client
@@ -852,6 +868,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if len(opt.SSH) != 0 && ((opt.User != currentUser && opt.User != "") || opt.Host != "" || (opt.Port != "22" && opt.Port != "")) {
 		fs.Logf(name, "--sftp-ssh is in use - ignoring user/host/port from config - set in the parameters to --sftp-ssh (remove them from the config to silence this warning)")
 	}
+	if opt.TranslateSymlinks && opt.SkipLinks {
+		return nil, errors.New("can't use --sftp-links and --sftp-skip-links together")
+	}
 	f.tokens = pacer.NewTokenDispenser(opt.Connections)
 
 	if opt.User == "" {
@@ -1115,6 +1134,12 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 		// Disable server side copy unless --sftp-copy-is-hardlink is set
 		f.features.Copy = nil
 	}
+	if opt.TranslateSymlinks {
+		// Enable symlink translation Feature when --sftp-links is set
+		// Not used yet but may be used in the future on shared backend operations.
+		// Maybe to check if src and dst backend support this feature before proceeding with the operation.
+		f.features.TranslateSymlink = true
+	}
 	// Make a connection and pool it to return errors early
 	c, err := f.getSftpConnection(ctx)
 	if err != nil {
@@ -1331,6 +1356,18 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				remote: remote,
 			}
 			o.setMetadata(info)
+			if o.IsSymlink() {
+				// Read the link target if it is a symlink to get path to real object and its size
+				linkTarget, linkTargetIsDir, sizeTarget, err := f.Readlink(ctx, o.path())
+				if err != nil {
+					// If we can't read the link target, log the error and continue
+					fs.Errorf(remote, "Readlink failed: %v", err)
+					continue
+				}
+				o.size = sizeTarget
+				o.linkTarget = linkTarget
+				o.linkTargetIsDir = linkTargetIsDir
+			}
 			entries = append(entries, o)
 		}
 	}
@@ -1340,14 +1377,36 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime(ctx)>
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	err := f.mkParentDir(ctx, src.Remote())
+
+	// Init two variables to store symlink object and check if source object is a symlink
+	// If TranslateSymlinks is set
+	var srcSymlinkObject fs.Object
+	isSymlink := false
+
 	if err != nil {
 		return nil, fmt.Errorf("Put mkParentDir failed: %w", err)
 	}
+
+	if f.opt.TranslateSymlinks {
+		// If TranslateSymlinks is set, we need to check if source object is a symlink
+		if or, ok := src.(*fs.OverrideRemote); ok {
+			srcSymlinkObject = or.UnWrap()
+			isSymlink = srcSymlinkObject.(*Object).IsSymlink()
+		}
+	}
 	// Temporary object under construction
 	o := &Object{
 		fs:     f,
 		remote: src.Remote(),
 	}
+	// if source file is a symlink, we need to specify target path to temporary object
+	if isSymlink {
+		o.linkTarget = srcSymlinkObject.(*Object).linkTarget
+		o.size = srcSymlinkObject.(*Object).size
+		o.mode = srcSymlinkObject.(*Object).mode
+		o.linkTargetIsDir = srcSymlinkObject.(*Object).linkTargetIsDir
+	}
+
 	err = o.Update(ctx, in, src, options...)
 	if err != nil {
 		return nil, err
@@ -1822,7 +1881,9 @@ func (o *Object) Remote() string {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
-	if o.fs.opt.DisableHashCheck {
+	// Check if HashCheck is disabled or
+	// if TranslateSymlinks is enabled and the target is a directory
+	if o.fs.opt.DisableHashCheck || (o.fs.opt.TranslateSymlinks && o.linkTargetIsDir) {
 		return "", nil
 	}
 	_ = o.fs.Hashes()
@@ -1987,6 +2048,11 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	o.mode = info.Mode()
 }
 
+// IsSymlink returns true if the remote sftp file is a symlink
+func (o *Object) IsSymlink() bool {
+	return o.mode&os.ModeSymlink != 0
+}
+
 // statRemote stats the file or directory at the remote given
 func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
 	absPath := remote
@@ -1997,7 +2063,13 @@ func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err err
 	if err != nil {
 		return nil, fmt.Errorf("stat: %w", err)
 	}
-	info, err = c.sftpClient.Stat(absPath)
+	if f.opt.TranslateSymlinks {
+		// Lstat is used to get the info of the symlink itself instead of the target
+		// We use Lstat only if the user has requested --sftp-links flag
+		info, err = c.sftpClient.Lstat(absPath)
+	} else {
+		info, err = c.sftpClient.Stat(absPath)
+	}
 	f.putSftpConnection(&c, err)
 	return info, err
 }
@@ -2041,8 +2113,13 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	return nil
 }
 
-// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.)
+// Storable returns whether the remote sftp file is a regular file (not a directory, block device, character device, named pipe, etc.)
+// if TranslateSymlinks is set, symlinks are also allowed
 func (o *Object) Storable() bool {
+	if o.fs.opt.TranslateSymlinks {
+		// If TranslateSymlinks is set, we also allow symlinks
+		return o.mode.IsRegular() || o.IsSymlink()
+	}
 	return o.mode.IsRegular()
 }
 
@@ -2156,12 +2233,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return fmt.Errorf("Update: %w", err)
 	}
-	// Hang on to the connection for the whole upload so it doesn't get reused while we are uploading
-	file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
-	if err != nil {
-		o.fs.putSftpConnection(&c, err)
-		return fmt.Errorf("Update Create failed: %w", err)
-	}
 	// remove the file if upload failed
 	remove := func() {
 		c, removeErr := o.fs.getSftpConnection(ctx)
@@ -2177,6 +2248,44 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			fs.Debugf(src, "Removed after failed upload: %v", err)
 		}
 	}
+	// Check if TranslateSymlinks flag is set and if temporary object given by Put() is Symlink
+	if o.fs.opt.TranslateSymlinks && o.IsSymlink() {
+		// Create or update symlink
+		err = o.fs.Symlink(ctx, o.linkTarget, o.path())
+		if err != nil {
+			o.fs.putSftpConnection(&c, err)
+			return fmt.Errorf("Update Symlink failed: %w", err)
+		}
+		_, linkTargetIsDir, sizeTarget, err := o.fs.Readlink(ctx, o.path())
+		if o.linkTargetIsDir == linkTargetIsDir {
+			// if symlink target is a directory, in will be closed with ErrorReadIsDirectory
+			// FIXME : Needed to bypass closeErr := inAcc.Close() in updateOrPut function in copy.go <== Need Help here if bad practice
+			err = in.(*accounting.Account).Close()
+			if err == fs.ErrorReadIsDirectory {
+				fs.Debugf(o, "Readlink returned directory, Continue normally")
+				err = nil
+			}
+		}
+		if err != nil {
+			o.fs.putSftpConnection(&c, err)
+			remove()
+			return fmt.Errorf("Update Readlink failed: %w", err)
+		}
+		if sizeTarget != src.Size() {
+			o.fs.putSftpConnection(&c, err)
+			remove()
+			return fmt.Errorf("Update Readlink target's size mismatch: %d != %d", sizeTarget, src.Size())
+		}
+		o.fs.putSftpConnection(&c, err)
+		o.size = sizeTarget
+		return nil
+	}
+	// Hang on to the connection for the whole upload so it doesn't get reused while we are uploading
+	file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		o.fs.putSftpConnection(&c, err)
+		return fmt.Errorf("Update Create failed: %w", err)
+	}
 	_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
 	if err != nil {
 		o.fs.putSftpConnection(&c, err)
@@ -2227,6 +2336,33 @@ func (o *Object) Remove(ctx context.Context) error {
 	return err
 }
 
+// Symlink creates a symbolic link on remote.
+func (f *Fs) Symlink(ctx context.Context, targetPath, linkName string) error {
+	c, err := f.getSftpConnection(ctx)
+	if err != nil {
+		return fmt.Errorf("Symlink: %w", err)
+	}
+	err = c.sftpClient.Symlink(targetPath, linkName)
+	f.putSftpConnection(&c, err)
+	return err
+}
+
+// Readlink reads the target of a symbolic link.
+func (f *Fs) Readlink(ctx context.Context, path string) (string, bool, int64, error) {
+	c, err := f.getSftpConnection(ctx)
+	if err != nil {
+		return "", false, 0, err
+	}
+	target, err := c.sftpClient.Stat(path)
+	if err != nil {
+		f.putSftpConnection(&c, err)
+		return "", false, 0, err
+	}
+	link, err := c.sftpClient.ReadLink(path)
+	f.putSftpConnection(&c, err)
+	return link, target.IsDir(), target.Size(), err
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs             = &Fs{}

docs/content/sftp.md

@@ -21,9 +21,9 @@ SSH installations.
 
 Paths are specified as `remote:path`. If the path does not begin with
 a `/` it is relative to the home directory of the user.  An empty path
-`remote:` refers to the user's home directory. For example, `rclone lsd remote:` 
+`remote:` refers to the user's home directory. For example, `rclone lsd remote:`
-would list the home directory of the user configured in the rclone remote config 
+would list the home directory of the user configured in the rclone remote config
-(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root 
+(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root
 directory for remote machine (i.e. `/`)
 
 Note that some SFTP servers will need the leading / - Synology is a
@@ -128,7 +128,7 @@ The SFTP remote supports three authentication methods:
 Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`.
 Only unencrypted OpenSSH or PEM encrypted files are supported.
 
-The key file can be specified in either an external file (key_file) or contained within the 
+The key file can be specified in either an external file (key_file) or contained within the
 rclone config file (key_pem).  If using key_pem in the config file, the entry should be on a
 single line with new line ('\n' or '\r\n') separating lines.  i.e.
 
@@ -199,7 +199,7 @@ e.g. using the OpenSSH `known_hosts` file:
 type = sftp
 host = example.com
 user = sftpuser
-pass = 
+pass =
 known_hosts_file = ~/.ssh/known_hosts
 ````
 
@@ -593,7 +593,7 @@ Properties:
 - Config:      ssh
 - Env Var:     RCLONE_SFTP_SSH
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 ### Advanced options
 
@@ -647,13 +647,13 @@ E.g. if shared folders can be found in directories representing volumes:
 E.g. if home directory can be found in a shared folder called "home":
 
     rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory
-	
+
 To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path.
 
 E.g. the first example above could be rewritten as:
 
 	rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2
-	
+
 Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home".
 
 E.g. the second example above should be rewritten as:
@@ -730,6 +730,8 @@ Properties:
 
 Set to skip any symlinks and any other non regular files.
 
+Mutually exclusive with `--sftp-links`.
+
 Properties:
 
 - Config:      skip_links
@@ -737,6 +739,19 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --sftp-links
+
+Set to follow symlinks. Recreate or update symlinks as symlinks and not copy the underlying file/directory.
+
+Mutually exclusive with `--sftp-skip-links`.
+
+Properties:
+
+- Config:      links
+- Env Var:     RCLONE_SFTP_LINKS
+- Type:        bool
+- Default:     false
+
 #### --sftp-subsystem
 
 Specifies the SSH2 subsystem on the remote host.
@@ -754,7 +769,7 @@ Specifies the path or command to run a sftp server on the remote host.
 
 The subsystem option is ignored when server_command is defined.
 
-If adding server_command to the configuration file please note that 
+If adding server_command to the configuration file please note that
 it should not be enclosed in quotes, since that will make rclone fail.
 
 A working example is:
@@ -946,7 +961,7 @@ Properties:
 - Config:      set_env
 - Env Var:     RCLONE_SFTP_SET_ENV
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 #### --sftp-ciphers
 
@@ -966,7 +981,7 @@ Properties:
 - Config:      ciphers
 - Env Var:     RCLONE_SFTP_CIPHERS
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 #### --sftp-key-exchange
 
@@ -986,7 +1001,7 @@ Properties:
 - Config:      key_exchange
 - Env Var:     RCLONE_SFTP_KEY_EXCHANGE
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 #### --sftp-macs
 
@@ -1004,7 +1019,7 @@ Properties:
 - Config:      macs
 - Env Var:     RCLONE_SFTP_MACS
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 #### --sftp-host-key-algorithms
 
@@ -1024,18 +1039,18 @@ Properties:
 - Config:      host_key_algorithms
 - Env Var:     RCLONE_SFTP_HOST_KEY_ALGORITHMS
 - Type:        SpaceSepList
-- Default:     
+- Default:
 
 #### --sftp-socks-proxy
 
 Socks 5 proxy host.
-	
+
 Supports the format user:pass@host:port, user@host:port, host:port.
 
 Example:
 
 	myUser:myPass@localhost:9005
-	
+
 
 Properties:
 

fs/features.go

@@ -39,6 +39,7 @@ type Features struct {
 	NoMultiThreading         bool // set if can't have multiplethreads on one download open
 	Overlay                  bool // this wraps one or more backends to add functionality
 	ChunkWriterDoesntSeek    bool // set if the chunk writer doesn't need to read the data more than once
+	TranslateSymlink         bool // set if backend can Read and Write symlinks
 
 	// Purge all files in the directory specified
 	//

fs/fs.go

@@ -48,6 +48,7 @@ var (
 	ErrorNotImplemented              = errors.New("optional feature not implemented")
 	ErrorCommandNotFound             = errors.New("command not found")
 	ErrorFileNameTooLong             = errors.New("file name too long")
+	ErrorReadIsDirectory             = errors.New("read is a directory")
 )
 
 // CheckClose is a utility function used to check the return from