diff --git a/cmd/mountlib/daemon.go b/cmd/mountlib/daemon.go index 88e0ae3bb..8d27e402d 100644 --- a/cmd/mountlib/daemon.go +++ b/cmd/mountlib/daemon.go @@ -1,6 +1,6 @@ // Daemonization interface for non-Unix variants only -// +build windows +// +build windows plan9 js package mountlib diff --git a/cmd/mountlib/daemon_unix.go b/cmd/mountlib/daemon_unix.go index e1988cb32..5842b9393 100644 --- a/cmd/mountlib/daemon_unix.go +++ b/cmd/mountlib/daemon_unix.go @@ -1,6 +1,6 @@ // Daemonization interface for Unix variants only -// +build !windows +// +build !windows,!plan9,!js package mountlib diff --git a/cmd/mountlib/help.go b/cmd/mountlib/help.go new file mode 100644 index 000000000..973840863 --- /dev/null +++ b/cmd/mountlib/help.go @@ -0,0 +1,302 @@ +package mountlib + +// "@" will be replaced by the command name, "|" will be replaced by backticks +var mountHelp = ` +rclone @ allows Linux, FreeBSD, macOS and Windows to +mount any of Rclone's cloud storage systems as a file system with +FUSE. + +First set up your remote using |rclone config|. Check it works with |rclone ls| etc. + +On Linux and OSX, you can either run mount in foreground mode or background (daemon) mode. +Mount runs in foreground mode by default, use the |--daemon| flag to specify background mode. +You can only run mount in foreground mode on Windows. + +On Linux/macOS/FreeBSD start the mount like this, where |/path/to/local/mount| +is an **empty** **existing** directory: + + rclone @ remote:path/to/files /path/to/local/mount + +On Windows you can start a mount in different ways. See [below](#mounting-modes-on-windows) +for details. The following examples will mount to an automatically assigned drive, +to specific drive letter |X:|, to path |C:\path\parent\mount| +(where parent directory or drive must exist, and mount must **not** exist, +and is not supported when [mounting as a network drive](#mounting-modes-on-windows)), and +the last example will mount as network share |\\cloud\remote| and map it to an +automatically assigned drive: + + rclone @ remote:path/to/files * + rclone @ remote:path/to/files X: + rclone @ remote:path/to/files C:\path\parent\mount + rclone @ remote:path/to/files \\cloud\remote + +When the program ends while in foreground mode, either via Ctrl+C or receiving +a SIGINT or SIGTERM signal, the mount should be automatically stopped. + +When running in background mode the user will have to stop the mount manually: + + # Linux + fusermount -u /path/to/local/mount + # OS X + umount /path/to/local/mount + +The umount operation can fail, for example when the mountpoint is busy. +When that happens, it is the user's responsibility to stop the mount manually. + +The size of the mounted file system will be set according to information retrieved +from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/) +command. Remotes with unlimited storage may report the used size only, +then an additional 1 PiB of free space is assumed. If the remote does not +[support](https://rclone.org/overview/#optional-features) the about feature +at all, then 1 PiB is set as both the total and the free size. + +**Note**: As of |rclone| 1.52.2, |rclone mount| now requires Go version 1.13 +or newer on some platforms depending on the underlying FUSE library in use. + +### Installing on Windows + +To run rclone @ on Windows, you will need to +download and install [WinFsp](http://www.secfs.net/winfsp/). + +[WinFsp](https://github.com/billziss-gh/winfsp) is an open source +Windows File System Proxy which makes it easy to write user space file +systems for Windows. It provides a FUSE emulation layer which rclone +uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse). +Both of these packages are by Bill Zissimopoulos who was very helpful +during the implementation of rclone @ for Windows. + +#### Mounting modes on windows + +Unlike other operating systems, Microsoft Windows provides a different filesystem +type for network and fixed drives. It optimises access on the assumption fixed +disk drives are fast and reliable, while network drives have relatively high latency +and less reliability. Some settings can also be differentiated between the two types, +for example that Windows Explorer should just display icons and not create preview +thumbnails for image and video files on network drives. + +In most cases, rclone will mount the remote as a normal, fixed disk drive by default. +However, you can also choose to mount it as a remote network drive, often described +as a network share. If you mount an rclone remote using the default, fixed drive mode +and experience unexpected program errors, freezes or other issues, consider mounting +as a network drive instead. + +When mounting as a fixed disk drive you can either mount to an unused drive letter, +or to a path representing a **non-existent** subdirectory of an **existing** parent +directory or drive. Using the special value |*| will tell rclone to +automatically assign the next available drive letter, starting with Z: and moving backward. +Examples: + + rclone @ remote:path/to/files * + rclone @ remote:path/to/files X: + rclone @ remote:path/to/files C:\path\parent\mount + rclone @ remote:path/to/files X: + +Option |--volname| can be used to set a custom volume name for the mounted +file system. The default is to use the remote name and path. + +To mount as network drive, you can add option |--network-mode| +to your @ command. Mounting to a directory path is not supported in +this mode, it is a limitation Windows imposes on junctions, so the remote must always +be mounted to a drive letter. + + rclone @ remote:path/to/files X: --network-mode + +A volume name specified with |--volname| will be used to create the network share path. +A complete UNC path, such as |\\cloud\remote|, optionally with path +|\\cloud\remote\madeup\path|, will be used as is. Any other +string will be used as the share part, after a default prefix |\\server\|. +If no volume name is specified then |\\server\share| will be used. +You must make sure the volume name is unique when you are mounting more than one drive, +or else the mount command will fail. The share name will treated as the volume label for +the mapped drive, shown in Windows Explorer etc, while the complete +|\\server\share| will be reported as the remote UNC path by +|net use| etc, just like a normal network drive mapping. + +If you specify a full network share UNC path with |--volname|, this will implicitely +set the |--network-mode| option, so the following two examples have same result: + + rclone @ remote:path/to/files X: --network-mode + rclone @ remote:path/to/files X: --volname \\server\share + +You may also specify the network share UNC path as the mountpoint itself. Then rclone +will automatically assign a drive letter, same as with |*| and use that as +mountpoint, and instead use the UNC path specified as the volume name, as if it were +specified with the |--volname| option. This will also implicitely set +the |--network-mode| option. This means the following two examples have same result: + + rclone @ remote:path/to/files \\cloud\remote + rclone @ remote:path/to/files * --volname \\cloud\remote + +There is yet another way to enable network mode, and to set the share path, +and that is to pass the "native" libfuse/WinFsp option directly: +|--fuse-flag --VolumePrefix=\server\share|. Note that the path +must be with just a single backslash prefix in this case. + + +*Note:* In previous versions of rclone this was the only supported method. + +[Read more about drive mapping](https://en.wikipedia.org/wiki/Drive_mapping) + +See also [Limitations](#limitations) section below. + +#### Windows filesystem permissions + +The FUSE emulation layer on Windows must convert between the POSIX-based +permission model used in FUSE, and the permission model used in Windows, +based on access-control lists (ACL). + +The mounted filesystem will normally get three entries in its access-control list (ACL), +representing permissions for the POSIX permission scopes: Owner, group and others. +By default, the owner and group will be taken from the current user, and the built-in +group "Everyone" will be used to represent others. The user/group can be customized +with FUSE options "UserName" and "GroupName", +e.g. |-o UserName=user123 -o GroupName="Authenticated Users"|. + +The permissions on each entry will be set according to +[options](#options) |--dir-perms| and |--file-perms|, +which takes a value in traditional [numeric notation](https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation), +where the default corresponds to |--file-perms 0666 --dir-perms 0777|. + +Note that the mapping of permissions is not always trivial, and the result +you see in Windows Explorer may not be exactly like you expected. +For example, when setting a value that includes write access, this will be +mapped to individual permissions "write attributes", "write data" and "append data", +but not "write extended attributes". Windows will then show this as basic +permission "Special" instead of "Write", because "Write" includes the +"write extended attributes" permission. + +If you set POSIX permissions for only allowing access to the owner, using +|--file-perms 0600 --dir-perms 0700|, the user group and the built-in "Everyone" +group will still be given some special permissions, such as "read attributes" +and "read permissions", in Windows. This is done for compatibility reasons, +e.g. to allow users without additional permissions to be able to read basic +metadata about files like in UNIX. One case that may arise is that other programs +(incorrectly) interprets this as the file being accessible by everyone. For example +an SSH client may warn about "unprotected private key file". + +WinFsp 2021 (version 1.9) introduces a new FUSE option "FileSecurity", +that allows the complete specification of file security descriptors using +[SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format). +With this you can work around issues such as the mentioned "unprotected private key file" +by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to the owner (OW). + +#### Windows caveats + +Drives created as Administrator are not visible to other accounts, +not even an account that was elevated to Administrator with the +User Account Control (UAC) feature. A result of this is that if you mount +to a drive letter from a Command Prompt run as Administrator, and then try +to access the same drive from Windows Explorer (which does not run as +Administrator), you will not be able to see the mounted drive. + +If you don't need to access the drive from applications running with +administrative privileges, the easiest way around this is to always +create the mount from a non-elevated command prompt. + +To make mapped drives available to the user account that created them +regardless if elevated or not, there is a special Windows setting called +[linked connections](https://docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry) +that can be enabled. + +It is also possible to make a drive mount available to everyone on the system, +by running the process creating it as the built-in SYSTEM account. +There are several ways to do this: One is to use the command-line +utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec), +from Microsoft's Sysinternals suite, which has option |-s| to start +processes as the SYSTEM account. Another alternative is to run the mount +command from a Windows Scheduled Task, or a Windows Service, configured +to run as the SYSTEM account. A third alternative is to use the +[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)). +Note that when running rclone as another user, it will not use +the configuration file from your profile unless you tell it to +with the [|--config|](https://rclone.org/docs/#config-config-file) option. +Read more in the [install documentation](https://rclone.org/install/). + +Note that mapping to a directory path, instead of a drive letter, +does not suffer from the same limitations. + +### Limitations + +Without the use of |--vfs-cache-mode| this can only write files +sequentially, it can only seek when reading. This means that many +applications won't work with their files on an rclone mount without +|--vfs-cache-mode writes| or |--vfs-cache-mode full|. +See the [VFS File Caching](#vfs-file-caching) section for more info. + +The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2, +Hubic) do not support the concept of empty directories, so empty +directories will have a tendency to disappear once they fall out of +the directory cache. + +Only supported on Linux, FreeBSD, OS X and Windows at the moment. + +### rclone @ vs rclone sync/copy + +File systems expect things to be 100% reliable, whereas cloud storage +systems are a long way from 100% reliable. The rclone sync/copy +commands cope with this with lots of retries. However rclone @ +can't use retries in the same way without making local copies of the +uploads. Look at the [VFS File Caching](#vfs-file-caching) +for solutions to make @ more reliable. + +### Attribute caching + +You can use the flag |--attr-timeout| to set the time the kernel caches +the attributes (size, modification time, etc.) for directory entries. + +The default is |1s| which caches files just long enough to avoid +too many callbacks to rclone from the kernel. + +In theory 0s should be the correct value for filesystems which can +change outside the control of the kernel. However this causes quite a +few problems such as +[rclone using too much memory](https://github.com/rclone/rclone/issues/2157), +[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112) +and [excessive time listing directories](https://github.com/rclone/rclone/issues/2095#issuecomment-371141147). + +The kernel can cache the info about a file for the time given by +|--attr-timeout|. You may see corruption if the remote file changes +length during this window. It will show up as either a truncated file +or a file with garbage on the end. With |--attr-timeout 1s| this is +very unlikely but not impossible. The higher you set |--attr-timeout| +the more likely it is. The default setting of "1s" is the lowest +setting which mitigates the problems above. + +If you set it higher (|10s| or |1m| say) then the kernel will call +back to rclone less often making it more efficient, however there is +more chance of the corruption issue above. + +If files don't change on the remote outside of the control of rclone +then there is no chance of corruption. + +This is the same as setting the attr_timeout option in mount.fuse. + +### Filters + +Note that all the rclone filters can be used to select a subset of the +files to be visible in the mount. + +### systemd + +When running rclone @ as a systemd service, it is possible +to use Type=notify. In this case the service will enter the started state +after the mountpoint has been successfully set up. +Units having the rclone @ service specified as a requirement +will see all files and folders immediately in this mode. + +### chunked reading + +|--vfs-read-chunk-size| will enable reading the source objects in parts. +This can reduce the used download quota for some remotes by requesting only chunks +from the remote that are actually read at the cost of an increased number of requests. + +When |--vfs-read-chunk-size-limit| is also specified and greater than +|--vfs-read-chunk-size|, the chunk size for each open file will get doubled +for each chunk read, until the specified value is reached. A value of |-1| will disable +the limit and the chunk size will grow indefinitely. + +With |--vfs-read-chunk-size 100M| and |--vfs-read-chunk-size-limit 0| +the following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. +When |--vfs-read-chunk-size-limit 500M| is specified, the result would be +0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on. +` diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go index b417223e6..1c99d44ac 100644 --- a/cmd/mountlib/mount.go +++ b/cmd/mountlib/mount.go @@ -1,19 +1,14 @@ package mountlib import ( - "io" "log" "os" - "os/signal" "path/filepath" "runtime" "strings" "sync" - "syscall" "time" - sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify" - "github.com/pkg/errors" "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" @@ -21,7 +16,11 @@ import ( "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/rclone/rclone/vfs/vfsflags" + + sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -63,6 +62,19 @@ type ( MountFn func(VFS *vfs.VFS, mountpoint string, opt *Options) (<-chan error, func() error, error) ) +// MountPoint represents a mount with options and runtime state +type MountPoint struct { + MountPoint string + MountedOn time.Time + MountOpt Options + VFSOpt vfscommon.Options + Fs fs.Fs + VFS *vfs.VFS + MountFn MountFn + UnmountFn UnmountFn + ErrChan <-chan error +} + // Global constants const ( MaxLeafSize = 1024 // don't pass file names longer than this @@ -106,424 +118,37 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &Opt.NetworkMode, "network-mode", "", Opt.NetworkMode, "Mount as remote network drive, instead of fixed disk drive. Supported on Windows only") } -// Check if folder is empty -func checkMountEmpty(mountpoint string) error { - fp, fpErr := os.Open(mountpoint) - - if fpErr != nil { - return errors.Wrap(fpErr, "Can not open: "+mountpoint) - } - defer fs.CheckClose(fp, &fpErr) - - _, fpErr = fp.Readdirnames(1) - - // directory is not empty - if fpErr != io.EOF { - var e error - var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option" - if fpErr == nil { - e = errors.New(errorMsg) - } else { - e = errors.Wrap(fpErr, errorMsg) - } - return e - } - return nil -} - -// Check the root doesn't overlap the mountpoint -func checkMountpointOverlap(root, mountpoint string) error { - abs := func(x string) string { - if absX, err := filepath.EvalSymlinks(x); err == nil { - x = absX - } - if absX, err := filepath.Abs(x); err == nil { - x = absX - } - x = filepath.ToSlash(x) - if !strings.HasSuffix(x, "/") { - x += "/" - } - return x - } - rootAbs, mountpointAbs := abs(root), abs(mountpoint) - if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) { - return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root) - } - return nil -} - // NewMountCommand makes a mount command with the given name and Mount function func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Command { var commandDefinition = &cobra.Command{ Use: commandName + " remote:path /path/to/mountpoint", Hidden: hidden, Short: `Mount the remote as file system on a mountpoint.`, - // Warning! "|" will be replaced by backticks below - // "@" will be replaced by the command name - Long: strings.ReplaceAll(strings.ReplaceAll(` -rclone @ allows Linux, FreeBSD, macOS and Windows to -mount any of Rclone's cloud storage systems as a file system with -FUSE. - -First set up your remote using |rclone config|. Check it works with |rclone ls| etc. - -On Linux and OSX, you can either run mount in foreground mode or background (daemon) mode. -Mount runs in foreground mode by default, use the |--daemon| flag to specify background mode. -You can only run mount in foreground mode on Windows. - -On Linux/macOS/FreeBSD start the mount like this, where |/path/to/local/mount| -is an **empty** **existing** directory: - - rclone @ remote:path/to/files /path/to/local/mount - -On Windows you can start a mount in different ways. See [below](#mounting-modes-on-windows) -for details. The following examples will mount to an automatically assigned drive, -to specific drive letter |X:|, to path |C:\path\parent\mount| -(where parent directory or drive must exist, and mount must **not** exist, -and is not supported when [mounting as a network drive](#mounting-modes-on-windows)), and -the last example will mount as network share |\\cloud\remote| and map it to an -automatically assigned drive: - - rclone @ remote:path/to/files * - rclone @ remote:path/to/files X: - rclone @ remote:path/to/files C:\path\parent\mount - rclone @ remote:path/to/files \\cloud\remote - -When the program ends while in foreground mode, either via Ctrl+C or receiving -a SIGINT or SIGTERM signal, the mount should be automatically stopped. - -When running in background mode the user will have to stop the mount manually: - - # Linux - fusermount -u /path/to/local/mount - # OS X - umount /path/to/local/mount - -The umount operation can fail, for example when the mountpoint is busy. -When that happens, it is the user's responsibility to stop the mount manually. - -The size of the mounted file system will be set according to information retrieved -from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/) -command. Remotes with unlimited storage may report the used size only, -then an additional 1 PiB of free space is assumed. If the remote does not -[support](https://rclone.org/overview/#optional-features) the about feature -at all, then 1 PiB is set as both the total and the free size. - -**Note**: As of |rclone| 1.52.2, |rclone mount| now requires Go version 1.13 -or newer on some platforms depending on the underlying FUSE library in use. - -### Installing on Windows - -To run rclone @ on Windows, you will need to -download and install [WinFsp](http://www.secfs.net/winfsp/). - -[WinFsp](https://github.com/billziss-gh/winfsp) is an open source -Windows File System Proxy which makes it easy to write user space file -systems for Windows. It provides a FUSE emulation layer which rclone -uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse). -Both of these packages are by Bill Zissimopoulos who was very helpful -during the implementation of rclone @ for Windows. - -#### Mounting modes on windows - -Unlike other operating systems, Microsoft Windows provides a different filesystem -type for network and fixed drives. It optimises access on the assumption fixed -disk drives are fast and reliable, while network drives have relatively high latency -and less reliability. Some settings can also be differentiated between the two types, -for example that Windows Explorer should just display icons and not create preview -thumbnails for image and video files on network drives. - -In most cases, rclone will mount the remote as a normal, fixed disk drive by default. -However, you can also choose to mount it as a remote network drive, often described -as a network share. If you mount an rclone remote using the default, fixed drive mode -and experience unexpected program errors, freezes or other issues, consider mounting -as a network drive instead. - -When mounting as a fixed disk drive you can either mount to an unused drive letter, -or to a path representing a **non-existent** subdirectory of an **existing** parent -directory or drive. Using the special value |*| will tell rclone to -automatically assign the next available drive letter, starting with Z: and moving backward. -Examples: - - rclone @ remote:path/to/files * - rclone @ remote:path/to/files X: - rclone @ remote:path/to/files C:\path\parent\mount - rclone @ remote:path/to/files X: - -Option |--volname| can be used to set a custom volume name for the mounted -file system. The default is to use the remote name and path. - -To mount as network drive, you can add option |--network-mode| -to your @ command. Mounting to a directory path is not supported in -this mode, it is a limitation Windows imposes on junctions, so the remote must always -be mounted to a drive letter. - - rclone @ remote:path/to/files X: --network-mode - -A volume name specified with |--volname| will be used to create the network share path. -A complete UNC path, such as |\\cloud\remote|, optionally with path -|\\cloud\remote\madeup\path|, will be used as is. Any other -string will be used as the share part, after a default prefix |\\server\|. -If no volume name is specified then |\\server\share| will be used. -You must make sure the volume name is unique when you are mounting more than one drive, -or else the mount command will fail. The share name will treated as the volume label for -the mapped drive, shown in Windows Explorer etc, while the complete -|\\server\share| will be reported as the remote UNC path by -|net use| etc, just like a normal network drive mapping. - -If you specify a full network share UNC path with |--volname|, this will implicitely -set the |--network-mode| option, so the following two examples have same result: - - rclone @ remote:path/to/files X: --network-mode - rclone @ remote:path/to/files X: --volname \\server\share - -You may also specify the network share UNC path as the mountpoint itself. Then rclone -will automatically assign a drive letter, same as with |*| and use that as -mountpoint, and instead use the UNC path specified as the volume name, as if it were -specified with the |--volname| option. This will also implicitely set -the |--network-mode| option. This means the following two examples have same result: - - rclone @ remote:path/to/files \\cloud\remote - rclone @ remote:path/to/files * --volname \\cloud\remote - -There is yet another way to enable network mode, and to set the share path, -and that is to pass the "native" libfuse/WinFsp option directly: -|--fuse-flag --VolumePrefix=\server\share|. Note that the path -must be with just a single backslash prefix in this case. - - -*Note:* In previous versions of rclone this was the only supported method. - -[Read more about drive mapping](https://en.wikipedia.org/wiki/Drive_mapping) - -See also [Limitations](#limitations) section below. - -#### Windows filesystem permissions - -The FUSE emulation layer on Windows must convert between the POSIX-based -permission model used in FUSE, and the permission model used in Windows, -based on access-control lists (ACL). - -The mounted filesystem will normally get three entries in its access-control list (ACL), -representing permissions for the POSIX permission scopes: Owner, group and others. -By default, the owner and group will be taken from the current user, and the built-in -group "Everyone" will be used to represent others. The user/group can be customized -with FUSE options "UserName" and "GroupName", -e.g. |-o UserName=user123 -o GroupName="Authenticated Users"|. - -The permissions on each entry will be set according to -[options](#options) |--dir-perms| and |--file-perms|, -which takes a value in traditional [numeric notation](https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation), -where the default corresponds to |--file-perms 0666 --dir-perms 0777|. - -Note that the mapping of permissions is not always trivial, and the result -you see in Windows Explorer may not be exactly like you expected. -For example, when setting a value that includes write access, this will be -mapped to individual permissions "write attributes", "write data" and "append data", -but not "write extended attributes". Windows will then show this as basic -permission "Special" instead of "Write", because "Write" includes the -"write extended attributes" permission. - -If you set POSIX permissions for only allowing access to the owner, using -|--file-perms 0600 --dir-perms 0700|, the user group and the built-in "Everyone" -group will still be given some special permissions, such as "read attributes" -and "read permissions", in Windows. This is done for compatibility reasons, -e.g. to allow users without additional permissions to be able to read basic -metadata about files like in UNIX. One case that may arise is that other programs -(incorrectly) interprets this as the file being accessible by everyone. For example -an SSH client may warn about "unprotected private key file". - -WinFsp 2021 (version 1.9) introduces a new FUSE option "FileSecurity", -that allows the complete specification of file security descriptors using -[SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format). -With this you can work around issues such as the mentioned "unprotected private key file" -by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to the owner (OW). - -#### Windows caveats - -Drives created as Administrator are not visible to other accounts, -not even an account that was elevated to Administrator with the -User Account Control (UAC) feature. A result of this is that if you mount -to a drive letter from a Command Prompt run as Administrator, and then try -to access the same drive from Windows Explorer (which does not run as -Administrator), you will not be able to see the mounted drive. - -If you don't need to access the drive from applications running with -administrative privileges, the easiest way around this is to always -create the mount from a non-elevated command prompt. - -To make mapped drives available to the user account that created them -regardless if elevated or not, there is a special Windows setting called -[linked connections](https://docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry) -that can be enabled. - -It is also possible to make a drive mount available to everyone on the system, -by running the process creating it as the built-in SYSTEM account. -There are several ways to do this: One is to use the command-line -utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec), -from Microsoft's Sysinternals suite, which has option |-s| to start -processes as the SYSTEM account. Another alternative is to run the mount -command from a Windows Scheduled Task, or a Windows Service, configured -to run as the SYSTEM account. A third alternative is to use the -[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)). -Note that when running rclone as another user, it will not use -the configuration file from your profile unless you tell it to -with the [|--config|](https://rclone.org/docs/#config-config-file) option. -Read more in the [install documentation](https://rclone.org/install/). - -Note that mapping to a directory path, instead of a drive letter, -does not suffer from the same limitations. - -### Limitations - -Without the use of |--vfs-cache-mode| this can only write files -sequentially, it can only seek when reading. This means that many -applications won't work with their files on an rclone mount without -|--vfs-cache-mode writes| or |--vfs-cache-mode full|. -See the [VFS File Caching](#vfs-file-caching) section for more info. - -The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2, -Hubic) do not support the concept of empty directories, so empty -directories will have a tendency to disappear once they fall out of -the directory cache. - -Only supported on Linux, FreeBSD, OS X and Windows at the moment. - -### rclone @ vs rclone sync/copy - -File systems expect things to be 100% reliable, whereas cloud storage -systems are a long way from 100% reliable. The rclone sync/copy -commands cope with this with lots of retries. However rclone @ -can't use retries in the same way without making local copies of the -uploads. Look at the [VFS File Caching](#vfs-file-caching) -for solutions to make @ more reliable. - -### Attribute caching - -You can use the flag |--attr-timeout| to set the time the kernel caches -the attributes (size, modification time, etc.) for directory entries. - -The default is |1s| which caches files just long enough to avoid -too many callbacks to rclone from the kernel. - -In theory 0s should be the correct value for filesystems which can -change outside the control of the kernel. However this causes quite a -few problems such as -[rclone using too much memory](https://github.com/rclone/rclone/issues/2157), -[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112) -and [excessive time listing directories](https://github.com/rclone/rclone/issues/2095#issuecomment-371141147). - -The kernel can cache the info about a file for the time given by -|--attr-timeout|. You may see corruption if the remote file changes -length during this window. It will show up as either a truncated file -or a file with garbage on the end. With |--attr-timeout 1s| this is -very unlikely but not impossible. The higher you set |--attr-timeout| -the more likely it is. The default setting of "1s" is the lowest -setting which mitigates the problems above. - -If you set it higher (|10s| or |1m| say) then the kernel will call -back to rclone less often making it more efficient, however there is -more chance of the corruption issue above. - -If files don't change on the remote outside of the control of rclone -then there is no chance of corruption. - -This is the same as setting the attr_timeout option in mount.fuse. - -### Filters - -Note that all the rclone filters can be used to select a subset of the -files to be visible in the mount. - -### systemd - -When running rclone @ as a systemd service, it is possible -to use Type=notify. In this case the service will enter the started state -after the mountpoint has been successfully set up. -Units having the rclone @ service specified as a requirement -will see all files and folders immediately in this mode. - -### chunked reading - -|--vfs-read-chunk-size| will enable reading the source objects in parts. -This can reduce the used download quota for some remotes by requesting only chunks -from the remote that are actually read at the cost of an increased number of requests. - -When |--vfs-read-chunk-size-limit| is also specified and greater than -|--vfs-read-chunk-size|, the chunk size for each open file will get doubled -for each chunk read, until the specified value is reached. A value of |-1| will disable -the limit and the chunk size will grow indefinitely. - -With |--vfs-read-chunk-size 100M| and |--vfs-read-chunk-size-limit 0| -the following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. -When |--vfs-read-chunk-size-limit 500M| is specified, the result would be -0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on. -`, "|", "`"), "@", commandName) + vfs.Help, + Long: strings.ReplaceAll(strings.ReplaceAll(mountHelp, "|", "`"), "@", commandName) + vfs.Help, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(2, 2, command, args) - opt := Opt // make a copy of the options - if opt.Daemon { + if Opt.Daemon { config.PassConfigKeyForDaemonization = true } - mountpoint := args[1] - fdst := cmd.NewFsDir(args) - if fdst.Name() == "" || fdst.Name() == "local" { - err := checkMountpointOverlap(fdst.Root(), mountpoint) - if err != nil { - log.Fatalf("Fatal error: %v", err) - } - } - // Show stats if the user has specifically requested them if cmd.ShowStats() { defer cmd.StartStats()() } - // Inform about ignored flags on Windows, - // and if not on Windows and not --allow-non-empty flag is used - // verify that mountpoint is empty. - if runtime.GOOS == "windows" { - if opt.AllowNonEmpty { - fs.Logf(nil, "--allow-non-empty flag does nothing on Windows") - } - if opt.AllowRoot { - fs.Logf(nil, "--allow-root flag does nothing on Windows") - } - if opt.AllowOther { - fs.Logf(nil, "--allow-other flag does nothing on Windows") - } - } else if !opt.AllowNonEmpty { - err := checkMountEmpty(mountpoint) - if err != nil { - log.Fatalf("Fatal error: %v", err) - } + mnt := &MountPoint{ + MountFn: mount, + MountPoint: args[1], + Fs: cmd.NewFsDir(args), + MountOpt: Opt, + VFSOpt: vfsflags.Opt, } - // Work out the volume name, removing special - // characters from it if necessary - if opt.VolumeName == "" { - opt.VolumeName = fdst.Name() + ":" + fdst.Root() + err := mnt.Mount() + if err == nil { + err = mnt.Wait() } - opt.VolumeName = strings.Replace(opt.VolumeName, ":", " ", -1) - opt.VolumeName = strings.Replace(opt.VolumeName, "/", " ", -1) - opt.VolumeName = strings.TrimSpace(opt.VolumeName) - if runtime.GOOS == "windows" && len(opt.VolumeName) > 32 { - opt.VolumeName = opt.VolumeName[:32] - } - - // Start background task if --background is specified - if opt.Daemon { - daemonized := startBackgroundMode() - if daemonized { - return - } - } - - VFS := vfs.New(fdst, &vfsflags.Opt) - err := Mount(VFS, mountpoint, mount, &opt) if err != nil { log.Fatalf("Fatal error: %v", err) } @@ -541,49 +166,94 @@ When |--vfs-read-chunk-size-limit 500M| is specified, the result would be return commandDefinition } -// ClipBlocks clips the blocks pointed to the OS max -func ClipBlocks(b *uint64) { - var max uint64 - switch runtime.GOOS { - case "windows": - if runtime.GOARCH == "386" { - max = (1 << 32) - 1 - } else { - max = (1 << 43) - 1 +// Mount the remote at mountpoint +func (m *MountPoint) Mount() (err error) { + if err = m.CheckOverlap(); err != nil { + return err + } + + if err = m.CheckAllowings(); err != nil { + return err + } + m.SetVolumeName(m.MountOpt.VolumeName) + + // Start background task if --background is specified + if m.MountOpt.Daemon { + daemonized := startBackgroundMode() + if daemonized { + return nil } - case "darwin": - // OSX FUSE only supports 32 bit number of blocks - // https://github.com/osxfuse/osxfuse/issues/396 - max = (1 << 32) - 1 - default: - // no clipping - return - } - if *b > max { - *b = max - } -} - -// Mount mounts the remote at mountpoint. -// -// If noModTime is set then it -func Mount(VFS *vfs.VFS, mountpoint string, mount MountFn, opt *Options) error { - if opt == nil { - opt = &DefaultOpt } - // Mount it - errChan, unmount, err := mount(VFS, mountpoint, opt) + m.VFS = vfs.New(m.Fs, &m.VFSOpt) + + m.ErrChan, m.UnmountFn, err = m.MountFn(m.VFS, m.MountPoint, &m.MountOpt) if err != nil { return errors.Wrap(err, "failed to mount FUSE fs") } + return nil +} +// CheckOverlap checks that root doesn't overlap with mountpoint +func (m *MountPoint) CheckOverlap() error { + name := m.Fs.Name() + if name != "" && name != "local" { + return nil + } + rootAbs := absPath(m.Fs.Root()) + mountpointAbs := absPath(m.MountPoint) + if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) { + const msg = "mount point %q and directory to be mounted %q mustn't overlap" + return errors.Errorf(msg, m.MountPoint, m.Fs.Root()) + } + return nil +} + +// absPath is a helper function for MountPoint.CheckOverlap +func absPath(path string) string { + if abs, err := filepath.EvalSymlinks(path); err == nil { + path = abs + } + if abs, err := filepath.Abs(path); err == nil { + path = abs + } + path = filepath.ToSlash(path) + if !strings.HasSuffix(path, "/") { + path += "/" + } + return path +} + +// CheckAllowings informs about ignored flags on Windows. If not on Windows +// and not --allow-non-empty flag is used, verify that mountpoint is empty. +func (m *MountPoint) CheckAllowings() error { + opt := &m.MountOpt + if runtime.GOOS == "windows" { + if opt.AllowNonEmpty { + fs.Logf(nil, "--allow-non-empty flag does nothing on Windows") + } + if opt.AllowRoot { + fs.Logf(nil, "--allow-root flag does nothing on Windows") + } + if opt.AllowOther { + fs.Logf(nil, "--allow-other flag does nothing on Windows") + } + return nil + } + if !opt.AllowNonEmpty { + return CheckMountEmpty(m.MountPoint) + } + return nil +} + +// Wait for mount end +func (m *MountPoint) Wait() error { // Unmount on exit var finaliseOnce sync.Once finalise := func() { finaliseOnce.Do(func() { _ = sysdnotify.Stopping() - _ = unmount() + _ = m.UnmountFn() }) } fnHandle := atexit.Register(finalise) @@ -596,19 +266,20 @@ func Mount(VFS *vfs.VFS, mountpoint string, mount MountFn, opt *Options) error { // Reload VFS cache on SIGHUP sigHup := make(chan os.Signal, 1) - signal.Notify(sigHup, syscall.SIGHUP) + NotifyOnSigHup(sigHup) + var err error -waitloop: - for { + waiting := true + for waiting { select { // umount triggered outside the app - case err = <-errChan: - break waitloop + case err = <-m.ErrChan: + waiting = false // user sent SIGHUP to clear the cache case <-sigHup: - root, err := VFS.Root() + root, err := m.VFS.Root() if err != nil { - fs.Errorf(VFS.Fs(), "Error reading root: %v", err) + fs.Errorf(m.VFS.Fs(), "Error reading root: %v", err) } else { root.ForgetAll() } @@ -620,6 +291,29 @@ waitloop: if err != nil { return errors.Wrap(err, "failed to umount FUSE fs") } - return nil } + +// Unmount the specified mountpoint +func (m *MountPoint) Unmount() (err error) { + return m.UnmountFn() +} + +// SetVolumeName with sensible default +func (m *MountPoint) SetVolumeName(vol string) { + if vol == "" { + vol = m.Fs.Name() + ":" + m.Fs.Root() + } + m.MountOpt.SetVolumeName(vol) +} + +// SetVolumeName removes special characters from volume name if necessary +func (opt *Options) SetVolumeName(vol string) { + vol = strings.ReplaceAll(vol, ":", " ") + vol = strings.ReplaceAll(vol, "/", " ") + vol = strings.TrimSpace(vol) + if runtime.GOOS == "windows" && len(vol) > 32 { + vol = vol[:32] + } + opt.VolumeName = vol +} diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index 728c94b73..3042c48fa 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -11,29 +11,33 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/vfs" - "github.com/rclone/rclone/vfs/vfscommon" "github.com/rclone/rclone/vfs/vfsflags" ) -// MountInfo defines the configuration for a mount -type MountInfo struct { - unmountFn UnmountFn - MountPoint string `json:"MountPoint"` - MountedOn time.Time `json:"MountedOn"` - Fs string `json:"Fs"` - MountOpt *Options - VFSOpt *vfscommon.Options -} - var ( // mutex to protect all the variables in this block mountMu sync.Mutex // Mount functions available mountFns = map[string]MountFn{} // Map of mounted path => MountInfo - liveMounts = map[string]MountInfo{} + liveMounts = map[string]*MountPoint{} + // Supported mount types + supportedMountTypes = []string{"mount", "cmount", "mount2"} ) +// ResolveMountMethod returns mount function by name +func ResolveMountMethod(mountType string) (string, MountFn) { + if mountType != "" { + return mountType, mountFns[mountType] + } + for _, mountType := range supportedMountTypes { + if mountFns[mountType] != nil { + return mountType, mountFns[mountType] + } + } + return "", nil +} + // AddRc adds mount and unmount functionality to rc func AddRc(mountUtilName string, mountFunction MountFn) { mountMu.Lock() @@ -99,14 +103,12 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) { mountMu.Lock() defer mountMu.Unlock() - if err != nil || mountType == "" { - if mountFns["mount"] != nil { - mountType = "mount" - } else if mountFns["cmount"] != nil { - mountType = "cmount" - } else if mountFns["mount2"] != nil { - mountType = "mount2" - } + if err != nil { + mountType = "" + } + mountType, mountFn := ResolveMountMethod(mountType) + if mountFn == nil { + return nil, errors.New("Mount Option specified is not registered, or is invalid") } // Get Fs.fs to be mounted from fs parameter in the params @@ -115,28 +117,26 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) { return nil, err } - if mountFns[mountType] != nil { - VFS := vfs.New(fdst, &vfsOpt) - _, unmountFn, err := mountFns[mountType](VFS, mountPoint, &mountOpt) - - if err != nil { - log.Printf("mount FAILED: %v", err) - return nil, err - } - // Add mount to list if mount point was successfully created - liveMounts[mountPoint] = MountInfo{ - unmountFn: unmountFn, - MountedOn: time.Now(), - Fs: fdst.Name(), - MountPoint: mountPoint, - VFSOpt: &vfsOpt, - MountOpt: &mountOpt, - } - - fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType) - return nil, nil + VFS := vfs.New(fdst, &vfsOpt) + _, unmountFn, err := mountFn(VFS, mountPoint, &mountOpt) + if err != nil { + log.Printf("mount FAILED: %v", err) + return nil, err } - return nil, errors.New("Mount Option specified is not registered, or is invalid") + + // Add mount to list if mount point was successfully created + liveMounts[mountPoint] = &MountPoint{ + MountPoint: mountPoint, + MountedOn: time.Now(), + MountFn: mountFn, + UnmountFn: unmountFn, + MountOpt: mountOpt, + VFSOpt: vfsOpt, + Fs: fdst, + } + + fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType) + return nil, nil } func init() { @@ -169,10 +169,14 @@ func unMountRc(_ context.Context, in rc.Params) (out rc.Params, err error) { } mountMu.Lock() defer mountMu.Unlock() - err = performUnMount(mountPoint) - if err != nil { + mountInfo, found := liveMounts[mountPoint] + if !found { + return nil, errors.New("mount not found") + } + if err = mountInfo.Unmount(); err != nil { return nil, err } + delete(liveMounts, mountPoint) return nil, nil } @@ -231,16 +235,34 @@ Eg }) } -// listMountsRc returns a list of current mounts +// MountInfo is a transitional structure for json marshaling +type MountInfo struct { + Fs string `json:"Fs"` + MountPoint string `json:"MountPoint"` + MountedOn time.Time `json:"MountedOn"` +} + +// listMountsRc returns a list of current mounts sorted by mount path func listMountsRc(_ context.Context, in rc.Params) (out rc.Params, err error) { - var mountTypes = []MountInfo{} mountMu.Lock() defer mountMu.Unlock() - for _, a := range liveMounts { - mountTypes = append(mountTypes, a) + var keys []string + for key := range liveMounts { + keys = append(keys, key) + } + sort.Strings(keys) + mountPoints := []MountInfo{} + for _, k := range keys { + m := liveMounts[k] + info := MountInfo{ + Fs: m.Fs.Name(), + MountPoint: m.MountPoint, + MountedOn: m.MountedOn, + } + mountPoints = append(mountPoints, info) } return rc.Params{ - "mountPoints": mountTypes, + "mountPoints": mountPoints, }, nil } @@ -265,27 +287,12 @@ Eg func unmountAll(_ context.Context, in rc.Params) (out rc.Params, err error) { mountMu.Lock() defer mountMu.Unlock() - for key, mountInfo := range liveMounts { - err = performUnMount(mountInfo.MountPoint) - if err != nil { - fs.Debugf(nil, "Couldn't unmount : %s", key) + for mountPoint, mountInfo := range liveMounts { + if err = mountInfo.Unmount(); err != nil { + fs.Debugf(nil, "Couldn't unmount : %s", mountPoint) return nil, err } + delete(liveMounts, mountPoint) } return nil, nil } - -// performUnMount unmounts the specified mountPoint -func performUnMount(mountPoint string) (err error) { - mountInfo, ok := liveMounts[mountPoint] - if ok { - err := mountInfo.unmountFn() - if err != nil { - return err - } - delete(liveMounts, mountPoint) - } else { - return errors.New("mount not found") - } - return nil -} diff --git a/cmd/mountlib/rc_test.go b/cmd/mountlib/rc_test.go index 2a079032b..203269901 100644 --- a/cmd/mountlib/rc_test.go +++ b/cmd/mountlib/rc_test.go @@ -13,6 +13,7 @@ import ( _ "github.com/rclone/rclone/cmd/cmount" _ "github.com/rclone/rclone/cmd/mount" _ "github.com/rclone/rclone/cmd/mount2" + "github.com/rclone/rclone/cmd/mountlib" "github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/rc" "github.com/stretchr/testify/assert" @@ -95,6 +96,22 @@ func TestRc(t *testing.T) { assert.Equal(t, os.FileMode(0400), fi.Mode()) } + // check mount point list + checkMountList := func() []mountlib.MountInfo { + listCall := rc.Calls.Get("mount/listmounts") + require.NotNil(t, listCall) + listReply, err := listCall.Fn(ctx, rc.Params{}) + require.NoError(t, err) + mountPointsReply, err := listReply.Get("mountPoints") + require.NoError(t, err) + mountPoints, ok := mountPointsReply.([]mountlib.MountInfo) + require.True(t, ok) + return mountPoints + } + mountPoints := checkMountList() + require.Equal(t, 1, len(mountPoints)) + require.Equal(t, mountPoint, mountPoints[0].MountPoint) + // FIXME the OS sometimes appears to be using the mount // immediately after it appears so wait a moment time.Sleep(100 * time.Millisecond) @@ -102,6 +119,7 @@ func TestRc(t *testing.T) { t.Run("Unmount", func(t *testing.T) { _, err := unmount.Fn(ctx, in) require.NoError(t, err) + assert.Equal(t, 0, len(checkMountList())) }) }) } diff --git a/cmd/mountlib/sighup.go b/cmd/mountlib/sighup.go new file mode 100644 index 000000000..a912b8d47 --- /dev/null +++ b/cmd/mountlib/sighup.go @@ -0,0 +1,14 @@ +// +build !plan9,!js + +package mountlib + +import ( + "os" + "os/signal" + "syscall" +) + +// NotifyOnSigHup makes SIGHUP notify given channel on supported systems +func NotifyOnSigHup(sighupChan chan os.Signal) { + signal.Notify(sighupChan, syscall.SIGHUP) +} diff --git a/cmd/mountlib/sighup_unsupported.go b/cmd/mountlib/sighup_unsupported.go new file mode 100644 index 000000000..fb8c550aa --- /dev/null +++ b/cmd/mountlib/sighup_unsupported.go @@ -0,0 +1,10 @@ +// +build plan9 js + +package mountlib + +import ( + "os" +) + +// NotifyOnSigHup makes SIGHUP notify given channel on supported systems +func NotifyOnSigHup(sighupChan chan os.Signal) {} diff --git a/cmd/mountlib/utils.go b/cmd/mountlib/utils.go new file mode 100644 index 000000000..5c106ad25 --- /dev/null +++ b/cmd/mountlib/utils.go @@ -0,0 +1,55 @@ +package mountlib + +import ( + "io" + "os" + "runtime" + + "github.com/pkg/errors" + "github.com/rclone/rclone/fs" +) + +// CheckMountEmpty checks if folder is empty +func CheckMountEmpty(mountpoint string) error { + fp, fpErr := os.Open(mountpoint) + + if fpErr != nil { + return errors.Wrap(fpErr, "Can not open: "+mountpoint) + } + defer fs.CheckClose(fp, &fpErr) + + _, fpErr = fp.Readdirnames(1) + + if fpErr == io.EOF { + return nil + } + + msg := "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option" + if fpErr == nil { + return errors.New(msg) + } + return errors.Wrap(fpErr, msg) +} + +// ClipBlocks clips the blocks pointed to the OS max +func ClipBlocks(b *uint64) { + var max uint64 + switch runtime.GOOS { + case "windows": + if runtime.GOARCH == "386" { + max = (1 << 32) - 1 + } else { + max = (1 << 43) - 1 + } + case "darwin": + // OSX FUSE only supports 32 bit number of blocks + // https://github.com/osxfuse/osxfuse/issues/396 + max = (1 << 32) - 1 + default: + // no clipping + return + } + if *b > max { + *b = max + } +}