diff --git a/cmd/all/all.go b/cmd/all/all.go index 39cddd363..92d9ac1cc 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -19,6 +19,7 @@ import ( _ "github.com/ncw/rclone/cmd/md5sum" _ "github.com/ncw/rclone/cmd/memtest" _ "github.com/ncw/rclone/cmd/mkdir" + _ "github.com/ncw/rclone/cmd/mount" _ "github.com/ncw/rclone/cmd/move" _ "github.com/ncw/rclone/cmd/purge" _ "github.com/ncw/rclone/cmd/rmdir" diff --git a/cmd/mount/createinfo.go b/cmd/mount/createinfo.go new file mode 100644 index 000000000..babc0c259 --- /dev/null +++ b/cmd/mount/createinfo.go @@ -0,0 +1,57 @@ +// +build linux darwin freebsd + +package mount + +import ( + "time" + + "github.com/ncw/rclone/fs" +) + +// info to create a new object +type createInfo struct { + f fs.Fs + remote string +} + +func newCreateInfo(f fs.Fs, remote string) *createInfo { + return &createInfo{ + f: f, + remote: remote, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (ci *createInfo) Fs() fs.Info { + return ci.f +} + +// Remote returns the remote path +func (ci *createInfo) Remote() string { + return ci.remote +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (ci *createInfo) Hash(fs.HashType) (string, error) { + return "", fs.ErrHashUnsupported +} + +// ModTime returns the modification date of the file +// It should return a best guess if one isn't available +func (ci *createInfo) ModTime() time.Time { + return time.Now() +} + +// Size returns the size of the file +func (ci *createInfo) Size() int64 { + // FIXME this means this won't work with all remotes... + return 0 +} + +// Storable says whether this object can be stored +func (ci *createInfo) Storable() bool { + return true +} + +var _ fs.ObjectInfo = (*createInfo)(nil) diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go new file mode 100644 index 000000000..e99c71ca2 --- /dev/null +++ b/cmd/mount/dir.go @@ -0,0 +1,377 @@ +// +build linux darwin freebsd + +package mount + +import ( + "os" + "path" + "sync" + "time" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// DirEntry describes the contents of a directory entry +// +// It can be a file or a directory +// +// node may be nil, but o may not +type DirEntry struct { + o fs.BasicInfo + node fusefs.Node +} + +// Dir represents a directory entry +type Dir struct { + f fs.Fs + path string + mu sync.RWMutex // protects the following + read bool + items map[string]*DirEntry +} + +func newDir(f fs.Fs, path string) *Dir { + return &Dir{ + f: f, + path: path, + } +} + +// addObject adds a new object or directory to the directory +// +// note that we add new objects rather than updating old ones +func (d *Dir) addObject(o fs.BasicInfo, node fusefs.Node) *DirEntry { + item := &DirEntry{ + o: o, + node: node, + } + d.mu.Lock() + d.items[path.Base(o.Remote())] = item + d.mu.Unlock() + return item +} + +// delObject removes an object from the directory +func (d *Dir) delObject(leaf string) { + d.mu.Lock() + delete(d.items, leaf) + d.mu.Unlock() +} + +// read the directory +func (d *Dir) readDir() error { + d.mu.Lock() + defer d.mu.Unlock() + if d.read { + return nil + } + objs, dirs, err := fs.NewLister().SetLevel(1).Start(d.f, d.path).GetAll() + if err == fs.ErrorDirNotFound { + // We treat directory not found as empty because we + // create directories on the fly + } else if err != nil { + return err + } + // Cache the items by name + d.items = make(map[string]*DirEntry, len(objs)+len(dirs)) + for _, obj := range objs { + name := path.Base(obj.Remote()) + d.items[name] = &DirEntry{ + o: obj, + node: nil, + } + } + for _, dir := range dirs { + name := path.Base(dir.Remote()) + d.items[name] = &DirEntry{ + o: dir, + node: nil, + } + } + d.read = true + return nil +} + +// lookup a single item in the directory +// +// returns fuse.ENOENT if not found. +func (d *Dir) lookup(leaf string) (*DirEntry, error) { + err := d.readDir() + if err != nil { + return nil, err + } + d.mu.RLock() + item, ok := d.items[leaf] + d.mu.RUnlock() + if !ok { + return nil, fuse.ENOENT + } + return item, nil +} + +// Check to see if a directory is empty +func (d *Dir) isEmpty() (bool, error) { + err := d.readDir() + if err != nil { + return false, err + } + d.mu.RLock() + defer d.mu.RUnlock() + return len(d.items) == 0, nil +} + +// Check interface satsified +var _ fusefs.Node = (*Dir)(nil) + +// Attr updates the attribes of a directory +func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error { + fs.Debug(d.path, "Dir.Attr") + a.Mode = os.ModeDir | dirPerms + // FIXME include Valid so get some caching? Also mtime + return nil +} + +// lookupNode calls lookup then makes sure the node is not nil in the DirEntry +func (d *Dir) lookupNode(leaf string) (item *DirEntry, err error) { + item, err = d.lookup(leaf) + if err != nil { + return nil, err + } + if item.node != nil { + return item, nil + } + var node fusefs.Node + switch x := item.o.(type) { + case fs.Object: + node, err = newFile(d, x), nil + case *fs.Dir: + node, err = newDir(d.f, x.Remote()), nil + default: + err = errors.Errorf("unknown type %T", item) + } + if err != nil { + return nil, err + } + item = d.addObject(item.o, node) + return item, err +} + +// Check interface satisfied +var _ fusefs.NodeRequestLookuper = (*Dir)(nil) + +// Lookup looks up a specific entry in the receiver. +// +// Lookup should return a Node corresponding to the entry. If the +// name does not exist in the directory, Lookup should return ENOENT. +// +// Lookup need not to handle the names "." and "..". +func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fusefs.Node, err error) { + path := path.Join(d.path, req.Name) + fs.Debug(path, "Dir.Lookup") + item, err := d.lookupNode(req.Name) + if err != nil { + if err != fuse.ENOENT { + fs.ErrorLog(path, "Dir.Lookup error: %v", err) + } + return nil, err + } + fs.Debug(path, "Dir.Lookup OK") + return item.node, nil +} + +// Check interface satisfied +var _ fusefs.HandleReadDirAller = (*Dir)(nil) + +// ReadDirAll reads the contents of the directory +func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error) { + fs.Debug(d.path, "Dir.ReadDirAll") + err = d.readDir() + if err != nil { + fs.Debug(d.path, "Dir.ReadDirAll error: %v", err) + return nil, err + } + d.mu.RLock() + defer d.mu.RUnlock() + for _, item := range d.items { + var dirent fuse.Dirent + switch x := item.o.(type) { + case fs.Object: + dirent = fuse.Dirent{ + // Inode FIXME ??? + Type: fuse.DT_File, + Name: path.Base(x.Remote()), + } + case *fs.Dir: + dirent = fuse.Dirent{ + // Inode FIXME ??? + Type: fuse.DT_Dir, + Name: path.Base(x.Remote()), + } + default: + err = errors.Errorf("unknown type %T", item) + fs.ErrorLog(d.path, "Dir.ReadDirAll error: %v", err) + return nil, err + } + dirents = append(dirents, dirent) + } + fs.Debug(d.path, "Dir.ReadDirAll OK with %d entries", len(dirents)) + return dirents, nil +} + +var _ fusefs.NodeCreater = (*Dir)(nil) + +// Create makes a new file +func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fusefs.Node, fusefs.Handle, error) { + path := path.Join(d.path, req.Name) + fs.Debug(path, "Dir.Create") + src := newCreateInfo(d.f, path) + // This gets added to the directory when the file is written + file := newFile(d, nil) + fh, err := newWriteFileHandle(d, file, src) + if err != nil { + fs.ErrorLog(path, "Dir.Create error: %v", err) + return nil, nil, err + } + fs.Debug(path, "Dir.Create OK") + return file, fh, nil +} + +var _ fusefs.NodeMkdirer = (*Dir)(nil) + +// Mkdir creates a new directory +func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fusefs.Node, error) { + // We just pretend to have created the directory - rclone will + // actually create the directory if we write files into it + path := path.Join(d.path, req.Name) + fs.Debug(path, "Dir.Mkdir") + fsDir := &fs.Dir{ + Name: path, + When: time.Now(), + } + dir := newDir(d.f, path) + d.addObject(fsDir, dir) + fs.Debug(path, "Dir.Mkdir OK") + return dir, nil +} + +var _ fusefs.NodeRemover = (*Dir)(nil) + +// Remove removes the entry with the given name from +// the receiver, which must be a directory. The entry to be removed +// may correspond to a file (unlink) or to a directory (rmdir). +func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + path := path.Join(d.path, req.Name) + fs.Debug(path, "Dir.Remove") + item, err := d.lookupNode(req.Name) + if err != nil { + fs.ErrorLog(path, "Dir.Remove error: %v", err) + return err + } + switch x := item.o.(type) { + case fs.Object: + err = x.Remove() + if err != nil { + fs.ErrorLog(path, "Dir.Remove file error: %v", err) + return err + } + case *fs.Dir: + // Do nothing for deleting directory - rclone can't + // currently remote a random directory + // + // Check directory is empty first though + dir := item.node.(*Dir) + empty, err := dir.isEmpty() + if err != nil { + fs.ErrorLog(path, "Dir.Remove dir error: %v", err) + return err + } + if !empty { + // return fuse.ENOTEMPTY - doesn't exist though so use EEXIST + fs.ErrorLog(path, "Dir.Remove not empty") + return fuse.EEXIST + } + default: + fs.ErrorLog(path, "Dir.Remove unknown type %T", item) + return errors.Errorf("unknown type %T", item) + } + // Remove the item from the directory listing + d.delObject(req.Name) + fs.Debug(path, "Dir.Remove OK") + return nil +} + +// Check interface satisfied +var _ fusefs.NodeRenamer = (*Dir)(nil) + +// Rename the file +func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs.Node) error { + oldPath := path.Join(d.path, req.OldName) + destDir, ok := newDir.(*Dir) + if !ok { + err := errors.Errorf("Unknown Dir type %T", newDir) + fs.ErrorLog(oldPath, "Dir.Rename error: %v", err) + return err + } + newPath := path.Join(destDir.path, req.NewName) + fs.Debug(oldPath, "Dir.Rename to %q", newPath) + oldItem, err := d.lookupNode(req.OldName) + if err != nil { + fs.ErrorLog(oldPath, "Dir.Rename error: %v", err) + return err + } + var newObj fs.BasicInfo + switch x := oldItem.o.(type) { + case fs.Object: + oldObject := x + do, ok := d.f.(fs.Mover) + if !ok { + err := errors.Errorf("Fs %q can't Move files", d.f) + fs.ErrorLog(oldPath, "Dir.Rename error: %v", err) + return err + } + newObject, err := do.Move(oldObject, newPath) + if err != nil { + fs.ErrorLog(oldPath, "Dir.Rename error: %v", err) + return err + } + newObj = newObject + case *fs.Dir: + oldDir := oldItem.node.(*Dir) + empty, err := oldDir.isEmpty() + if err != nil { + fs.ErrorLog(oldPath, "Dir.Rename dir error: %v", err) + return err + } + if !empty { + // return fuse.ENOTEMPTY - doesn't exist though so use EEXIST + fs.ErrorLog(oldPath, "Dir.Rename can't rename non empty directory") + return fuse.EEXIST + } + newObj = &fs.Dir{ + Name: newPath, + When: time.Now(), + } + default: + err = errors.Errorf("unknown type %T", oldItem) + fs.ErrorLog(d.path, "Dir.ReadDirAll error: %v", err) + return err + } + + // Show moved - delete from old dir and add to new + d.delObject(req.OldName) + destDir.addObject(newObj, nil) + + // FIXME need to flush the dir also + + // FIXME use DirMover to move a directory? + // or maybe use MoveDir which can move anything + // fallback to Copy/Delete if no Move? + // if dir is empty then can move it + + fs.ErrorLog(newPath, "Dir.Rename renamed from %q", oldPath) + return nil +} diff --git a/cmd/mount/dir_test.go b/cmd/mount/dir_test.go new file mode 100644 index 000000000..7b4f71438 --- /dev/null +++ b/cmd/mount/dir_test.go @@ -0,0 +1,121 @@ +// +build linux darwin freebsd + +package mount + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDirLs(t *testing.T) { + run.checkDir(t, "") + + run.mkdir(t, "a directory") + run.createFile(t, "a file", "hello") + + run.checkDir(t, "a directory/|a file 5") + + run.rmdir(t, "a directory") + run.rm(t, "a file") + + run.checkDir(t, "") +} + +func TestDirCreateAndRemoveDir(t *testing.T) { + run.mkdir(t, "dir") + run.mkdir(t, "dir/subdir") + run.checkDir(t, "dir/|dir/subdir/") + + // Check we can't delete a directory with stuff in + err := os.Remove(run.path("dir")) + assert.Error(t, err, "file exists") + + // Now delete subdir then dir - should produce no errors + run.rmdir(t, "dir/subdir") + run.checkDir(t, "dir/") + run.rmdir(t, "dir") + run.checkDir(t, "") +} + +func TestDirCreateAndRemoveFile(t *testing.T) { + run.mkdir(t, "dir") + run.createFile(t, "dir/file", "potato") + run.checkDir(t, "dir/|dir/file 6") + + // Check we can't delete a directory with stuff in + err := os.Remove(run.path("dir")) + assert.Error(t, err, "file exists") + + // Now delete file + run.rm(t, "dir/file") + + run.checkDir(t, "dir/") + run.rmdir(t, "dir") + run.checkDir(t, "") +} + +func TestDirRenameFile(t *testing.T) { + run.mkdir(t, "dir") + run.createFile(t, "file", "potato") + run.checkDir(t, "dir/|file 6") + + err := os.Rename(run.path("file"), run.path("dir/file2")) + require.NoError(t, err) + run.checkDir(t, "dir/|dir/file2 6") + + err = os.Rename(run.path("dir/file2"), run.path("dir/file3")) + require.NoError(t, err) + run.checkDir(t, "dir/|dir/file3 6") + + run.rm(t, "dir/file3") + run.rmdir(t, "dir") + run.checkDir(t, "") +} + +func TestDirRenameEmptyDir(t *testing.T) { + run.mkdir(t, "dir") + run.mkdir(t, "dir1") + run.checkDir(t, "dir/|dir1/") + + err := os.Rename(run.path("dir1"), run.path("dir/dir2")) + require.NoError(t, err) + run.checkDir(t, "dir/|dir/dir2/") + + err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3")) + require.NoError(t, err) + run.checkDir(t, "dir/|dir/dir3/") + + run.rmdir(t, "dir/dir3") + run.rmdir(t, "dir") + run.checkDir(t, "") +} + +func TestDirRenameFullDir(t *testing.T) { + run.mkdir(t, "dir") + run.mkdir(t, "dir1") + run.createFile(t, "dir1/potato.txt", "maris piper") + run.checkDir(t, "dir/|dir1/|dir1/potato.txt 11") + + err := os.Rename(run.path("dir1"), run.path("dir/dir2")) + require.Error(t, err, "file exists") + // Can't currently rename directories with stuff in + /* + require.NoError(t, err) + run.checkDir(t, "dir/|dir/dir2/|dir/dir2/potato.txt 11") + + err = os.Rename(run.path("dir/dir2"), run.path("dir/dir3")) + require.NoError(t, err) + run.checkDir(t, "dir/|dir/dir3/|dir/dir3/potato.txt 11") + + run.rm(t, "dir/dir3/potato.txt") + run.rmdir(t, "dir/dir3") + */ + + run.rm(t, "dir1/potato.txt") + run.rmdir(t, "dir1") + run.rmdir(t, "dir") + run.checkDir(t, "") +} diff --git a/cmd/mount/file.go b/cmd/mount/file.go new file mode 100644 index 000000000..c287be687 --- /dev/null +++ b/cmd/mount/file.go @@ -0,0 +1,142 @@ +// +build linux darwin freebsd + +package mount + +import ( + "sync" + "sync/atomic" + "time" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// File represents a file +type File struct { + d *Dir // parent directory - read only + size int64 // size of file - read and written with atomic + mu sync.RWMutex // protects the following + o fs.Object // NB o may be nil if file is being written + writers int // number of writers for this file +} + +// newFile creates a new File +func newFile(d *Dir, o fs.Object) *File { + return &File{ + d: d, + o: o, + } +} + +// addWriters increments or decrements the writers +func (f *File) addWriters(n int) { + f.mu.Lock() + f.writers += n + f.mu.Unlock() +} + +// Check interface satisfied +var _ fusefs.Node = (*File)(nil) + +// Attr fills out the attributes for the file +func (f *File) Attr(ctx context.Context, a *fuse.Attr) error { + f.mu.Lock() + defer f.mu.Unlock() + fs.Debug(f.o, "File.Attr") + a.Mode = filePerms + // if o is nil it isn't valid yet, so return the size so far + if f.o == nil { + a.Size = uint64(atomic.LoadInt64(&f.size)) + } else { + a.Size = uint64(f.o.Size()) + if !noModTime { + modTime := f.o.ModTime() + a.Atime = modTime + a.Mtime = modTime + a.Ctime = modTime + a.Crtime = modTime + } + } + return nil +} + +// Update the size while writing +func (f *File) written(n int64) { + atomic.AddInt64(&f.size, n) +} + +// Update the object when written +func (f *File) setObject(o fs.Object) { + f.mu.Lock() + defer f.mu.Unlock() + f.o = o + f.d.addObject(o, f) +} + +// Wait for f.o to become non nil for a short time returning it or an +// error +// +// Call without the mutex held +func (f *File) waitForValidObject() (o fs.Object, err error) { + for i := 0; i < 50; i++ { + f.mu.Lock() + o = f.o + writers := f.writers + f.mu.Unlock() + if o != nil { + return o, nil + } + if writers == 0 { + return nil, errors.New("can't open file - writer failed") + } + time.Sleep(100 * time.Millisecond) + } + return nil, fuse.ENOENT +} + +// Check interface satisfied +var _ fusefs.NodeOpener = (*File)(nil) + +// Open the file for read or write +func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fusefs.Handle, error) { + // if o is nil it isn't valid yet + o, err := f.waitForValidObject() + if err != nil { + return nil, err + } + + fs.Debug(o, "File.Open") + + // Files aren't seekable + resp.Flags |= fuse.OpenNonSeekable + + switch { + case req.Flags.IsReadOnly(): + return newReadFileHandle(o) + case req.Flags.IsWriteOnly(): + src := newCreateInfo(f.d.f, o.Remote()) + fh, err := newWriteFileHandle(f.d, f, src) + if err != nil { + return nil, err + } + return fh, nil + case req.Flags.IsReadWrite(): + return nil, errors.New("can't open read and write") + } + + /* + // File was opened in append-only mode, all writes will go to end + // of file. OS X does not provide this information. + OpenAppend OpenFlags = syscall.O_APPEND + OpenCreate OpenFlags = syscall.O_CREAT + OpenDirectory OpenFlags = syscall.O_DIRECTORY + OpenExclusive OpenFlags = syscall.O_EXCL + OpenNonblock OpenFlags = syscall.O_NONBLOCK + OpenSync OpenFlags = syscall.O_SYNC + OpenTruncate OpenFlags = syscall.O_TRUNC + */ + return nil, errors.New("can't figure out how to open") +} diff --git a/cmd/mount/fs.go b/cmd/mount/fs.go new file mode 100644 index 000000000..f3b1fb493 --- /dev/null +++ b/cmd/mount/fs.go @@ -0,0 +1,67 @@ +// FUSE main Fs + +// +build linux darwin freebsd + +package mount + +import ( + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/fs" +) + +// Default permissions +const ( + dirPerms = 0755 + filePerms = 0644 +) + +// FS represents the top level filing system +type FS struct { + f fs.Fs +} + +// Check interface satistfied +var _ fusefs.FS = (*FS)(nil) + +// Root returns the root node +func (f *FS) Root() (fusefs.Node, error) { + fs.Debug(f.f, "Root()") + return newDir(f.f, ""), nil +} + +// mount the file system +// +// The mount point will be ready when this returns. +// +// returns an error, and an error channel for the serve process to +// report an error when fusermount is called. +func mount(f fs.Fs, mountpoint string) (<-chan error, error) { + c, err := fuse.Mount(mountpoint) + if err != nil { + return nil, err + } + + filesys := &FS{ + f: f, + } + + // Serve the mount point in the background returning error to errChan + errChan := make(chan error, 1) + go func() { + err := fusefs.Serve(c, filesys) + closeErr := c.Close() + if err == nil { + err = closeErr + } + errChan <- err + }() + + // check if the mount process has an error to report + <-c.Ready + if err := c.MountError; err != nil { + return nil, err + } + + return errChan, nil +} diff --git a/cmd/mount/fs_test.go b/cmd/mount/fs_test.go new file mode 100644 index 000000000..2c589800e --- /dev/null +++ b/cmd/mount/fs_test.go @@ -0,0 +1,248 @@ +// +build linux darwin freebsd + +// Test suite for rclonefs + +package mount + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "strings" + "testing" + + "github.com/ncw/rclone/fs" + _ "github.com/ncw/rclone/fs/all" + "github.com/ncw/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Globals +var ( + RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem") + SubDir = flag.Bool("subdir", false, "Set to test with a sub directory") + Verbose = flag.Bool("verbose", false, "Set to enable logging") + DumpHeaders = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)") + DumpBodies = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)") + Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower") + LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries") +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + flag.Parse() + run = newRun() + rc := m.Run() + run.Finalise() + os.Exit(rc) +} + +// Run holds the remotes for a test run +type Run struct { + mountPath string + fremote fs.Fs + fremoteName string + cleanRemote func() + umountResult <-chan error +} + +// run holds the master Run data +var run *Run + +// newRun initialise the remote mount for testing and returns a run +// object. +// +// r.fremote is an empty remote Fs +// +// Finalise() will tidy them away when done. +func newRun() *Run { + r := &Run{ + umountResult: make(chan error, 1), + } + + // Never ask for passwords, fail instead. + // If your local config is encrypted set environment variable + // "RCLONE_CONFIG_PASS=hunter2" (or your password) + *fs.AskPassword = false + fs.LoadConfig() + fs.Config.Verbose = *Verbose + fs.Config.Quiet = !*Verbose + fs.Config.DumpHeaders = *DumpHeaders + fs.Config.DumpBodies = *DumpBodies + fs.Config.LowLevelRetries = *LowLevelRetries + var err error + r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir) + if err != nil { + log.Fatalf("Failed to open remote %q: %v", *RemoteName, err) + } + + r.mountPath, err = ioutil.TempDir("", "rclonefs-mount") + if err != nil { + log.Fatalf("Failed to create mount dir: %v", err) + } + + // Mount it up + r.mount() + + return r +} + +func (r *Run) mount() { + log.Printf("mount %q %q", r.fremote, r.mountPath) + var err error + r.umountResult, err = mount(r.fremote, r.mountPath) + if err != nil { + log.Fatalf("umount failed: %v", err) + } + log.Printf("mount OK") +} + +func (r *Run) umount() { + log.Printf("Calling fusermount -u %q", r.mountPath) + err := exec.Command("fusermount", "-u", r.mountPath).Run() + if err != nil { + log.Printf("fusermount failed: %v", err) + } + log.Printf("Waiting for umount") + err = <-r.umountResult + if err != nil { + log.Fatalf("umount failed: %v", err) + } +} + +// Finalise cleans the remote and unmounts +func (r *Run) Finalise() { + r.umount() + r.cleanRemote() + err := os.RemoveAll(r.mountPath) + if err != nil { + log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err) + } +} + +func (r *Run) path(filepath string) string { + return path.Join(run.mountPath, filepath) +} + +type dirMap map[string]struct{} + +// Create a dirMap from a string +func newDirMap(dirString string) (dm dirMap) { + dm = make(dirMap) + for _, entry := range strings.Split(dirString, "|") { + if entry != "" { + dm[entry] = struct{}{} + } + } + return dm +} + +// Returns a dirmap with only the files in +func (dm dirMap) filesOnly() dirMap { + newDm := make(dirMap) + for name := range dm { + if !strings.HasSuffix(name, "/") { + newDm[name] = struct{}{} + } + } + return newDm +} + +// reads the local tree into dir +func (r *Run) readLocal(t *testing.T, dir dirMap, filepath string) { + realPath := r.path(filepath) + files, err := ioutil.ReadDir(realPath) + require.NoError(t, err) + for _, fi := range files { + name := path.Join(filepath, fi.Name()) + if fi.IsDir() { + dir[name+"/"] = struct{}{} + r.readLocal(t, dir, name) + assert.Equal(t, fi.Mode().Perm(), os.FileMode(dirPerms)) + } else { + dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{} + assert.Equal(t, fi.Mode().Perm(), os.FileMode(filePerms)) + } + } +} + +// reads the remote tree into dir +func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) { + objs, dirs, err := fs.NewLister().SetLevel(1).Start(r.fremote, filepath).GetAll() + if err == fs.ErrorDirNotFound { + return + } + require.NoError(t, err) + for _, obj := range objs { + dir[fmt.Sprintf("%s %d", obj.Remote(), obj.Size())] = struct{}{} + } + for _, d := range dirs { + name := d.Remote() + dir[name+"/"] = struct{}{} + r.readRemote(t, dir, name) + } +} + +// checkDir checks the local and remote against the string passed in +func (r *Run) checkDir(t *testing.T, dirString string) { + dm := newDirMap(dirString) + localDm := make(dirMap) + r.readLocal(t, localDm, "") + remoteDm := make(dirMap) + r.readRemote(t, remoteDm, "") + // Ignore directories for remote compare + assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote") + assert.Equal(t, dm, localDm, "expected vs fuse mount") +} + +func (r *Run) createFile(t *testing.T, filepath string, contents string) { + filepath = r.path(filepath) + err := ioutil.WriteFile(filepath, []byte(contents), 0600) + require.NoError(t, err) +} + +func (r *Run) readFile(t *testing.T, filepath string) string { + filepath = r.path(filepath) + result, err := ioutil.ReadFile(filepath) + require.NoError(t, err) + return string(result) +} + +func (r *Run) mkdir(t *testing.T, filepath string) { + filepath = r.path(filepath) + err := os.Mkdir(filepath, 0700) + require.NoError(t, err) +} + +func (r *Run) rm(t *testing.T, filepath string) { + filepath = r.path(filepath) + err := os.Remove(filepath) + require.NoError(t, err) +} + +func (r *Run) rmdir(t *testing.T, filepath string) { + filepath = r.path(filepath) + err := os.Remove(filepath) + require.NoError(t, err) +} + +// Check that the Fs is mounted by seeing if the mountpoint is +// in the mount output +func TestMount(t *testing.T) { + out, err := exec.Command("mount").Output() + require.NoError(t, err) + assert.Contains(t, string(out), run.mountPath) +} + +// Check root directory is present and correct +func TestRoot(t *testing.T) { + fi, err := os.Lstat(run.mountPath) + require.NoError(t, err) + assert.True(t, fi.IsDir()) + assert.Equal(t, fi.Mode().Perm(), os.FileMode(dirPerms)) +} diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go new file mode 100644 index 000000000..4d38b153f --- /dev/null +++ b/cmd/mount/mount.go @@ -0,0 +1,117 @@ +// Package mount implents a FUSE mounting system for rclone remotes. + +// +build linux darwin freebsd + +package mount + +import ( + "bazil.org/fuse" + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// Globals +var ( + noModTime = false + debugFUSE = false +) + +func init() { + cmd.Root.AddCommand(mountCmd) + mountCmd.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).") + mountCmd.Flags().BoolVarP(&debugFUSE, "debug-fuse", "", false, "Debug the FUSE internals - needs -v.") +} + +var mountCmd = &cobra.Command{ + Use: "mount remote:path /path/to/mountpoint", + Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`, + Long: ` +rclone mount allows Linux and macOS to mount any of Rclone's cloud storage +systems as a file system with FUSE. + +This is **EXPERIMENTAL** - use with care. + +First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc. + +Start the mount like this + + rclone mount remote:path/to/files /path/to/local/mount & + +Stop the mount with + + fusermount -u /path/to/local/mount + +Or with OS X + + umount -u /path/to/local/mount + +### Limitations ### + +This can only read files seqentially, or write files sequentially. It +can't read and write or seek in files. + +rclonefs inherits rclone's directory handling. In rclone's world +directories don't really exist. This means that empty directories +will have a tendency to disappear once they fall out of the directory +cache. + +The bucket based FSes (eg swift, s3, google compute storage, b2) won't +work from the root - you will need to specify a bucket, or a path +within the bucket. So ` + "`swift:`" + ` won't work whereas ` + "`swift:bucket`" + ` will +as will ` + "`swift:bucket/path`" + `. + +### rclone mount vs rclone sync/copy ## + +File systems expect things to be 100% reliable, whereas cloud storage +systems are a long way from 100% reliable. The rclone sync/copy +commands cope with this with lots of retries. However rclone mount +can't use retries in the same way without making local copies of the +uploads. This might happen in the future, but for the moment rclone +mount won't do that, so will be less reliable than the rclone command. + +### Bugs ### + + * All the remotes should work for read, but some may not for write + * those which need to know the size in advance won't - eg B2 + * maybe should pass in size as -1 to mean work it out + +### TODO ### + + * Tests + * Check hashes on upload/download + * Preserve timestamps + * Move directories +`, + RunE: func(command *cobra.Command, args []string) error { + cmd.CheckArgs(2, 2, command, args) + fdst := cmd.NewFsDst(args) + return Mount(fdst, args[1]) + }, +} + +// Mount mounts the remote at mountpoint. +// +// If noModTime is set then it +func Mount(f fs.Fs, mountpoint string) error { + if debugFUSE { + fuse.Debug = func(msg interface{}) { + fs.Debug("fuse", "%v", msg) + } + } + + // Mount it + errChan, err := mount(f, mountpoint) + if err != nil { + return errors.Wrap(err, "failed to mount FUSE fs") + } + + // Wait for umount + err = <-errChan + if err != nil { + return errors.Wrap(err, "failed to umount FUSE fs") + } + + return nil +} diff --git a/cmd/mount/mount_unsupported.go b/cmd/mount/mount_unsupported.go new file mode 100644 index 000000000..5c7bf60db --- /dev/null +++ b/cmd/mount/mount_unsupported.go @@ -0,0 +1,6 @@ +// Build for mount for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build !linux,!darwin,!freebsd + +package mount diff --git a/cmd/mount/read.go b/cmd/mount/read.go new file mode 100644 index 000000000..97f7aaa7c --- /dev/null +++ b/cmd/mount/read.go @@ -0,0 +1,130 @@ +// +build linux darwin freebsd + +package mount + +import ( + "io" + "sync" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/fs" + "golang.org/x/net/context" +) + +// ReadFileHandle is an open for read file handle on a File +type ReadFileHandle struct { + mu sync.Mutex + closed bool // set if handle has been closed + r io.ReadCloser + o fs.Object + readCalled bool // set if read has been called +} + +func newReadFileHandle(o fs.Object) (*ReadFileHandle, error) { + r, err := o.Open() + if err != nil { + return nil, err + } + return &ReadFileHandle{ + r: r, + o: o, + }, nil +} + +// Check interface satisfied +var _ fusefs.Handle = (*ReadFileHandle)(nil) + +// Check interface satisfied +var _ fusefs.HandleReader = (*ReadFileHandle)(nil) + +// Read from the file handle +func (fh *ReadFileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + fs.Debug(fh.o, "ReadFileHandle.Open") + if fh.closed { + fs.ErrorLog(fh.o, "ReadFileHandle.Read error: %v", errClosedFileHandle) + return errClosedFileHandle + } + fh.readCalled = true + // We don't actually enforce Offset to match where previous read + // ended. Maybe we should, but that would mean'd we need to track + // it. The kernel *should* do it for us, based on the + // fuse.OpenNonSeekable flag. + // + // One exception to the above is if we fail to fully populate a + // page cache page; a read into page cache is always page aligned. + // Make sure we never serve a partial read, to avoid that. + buf := make([]byte, req.Size) + n, err := io.ReadFull(fh.r, buf) + if err == io.ErrUnexpectedEOF || err == io.EOF { + err = nil + } + resp.Data = buf[:n] + if err != nil { + fs.ErrorLog(fh.o, "ReadFileHandle.Open error: %v", err) + } else { + fs.Debug(fh.o, "ReadFileHandle.Open OK") + } + return err +} + +// close the file handle returning errClosedFileHandle if it has been +// closed already. +// +// Must be called with fh.mu held +func (fh *ReadFileHandle) close() error { + if fh.closed { + return errClosedFileHandle + } + fh.closed = true + return fh.r.Close() +} + +// Check interface satisfied +var _ fusefs.HandleFlusher = (*ReadFileHandle)(nil) + +// Flush is called each time the file or directory is closed. +// Because there can be multiple file descriptors referring to a +// single opened file, Flush can be called multiple times. +func (fh *ReadFileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + fs.Debug(fh.o, "ReadFileHandle.Flush") + // If Read hasn't been called then ignore the Flush - Release + // will pick it up + if !fh.readCalled { + fs.Debug(fh.o, "ReadFileHandle.Flush ignoring flush on unread handle") + return nil + + } + err := fh.close() + if err != nil { + fs.ErrorLog(fh.o, "ReadFileHandle.Flush error: %v", err) + return err + } + fs.Debug(fh.o, "ReadFileHandle.Flush OK") + return nil +} + +var _ fusefs.HandleReleaser = (*ReadFileHandle)(nil) + +// Release is called when we are finished with the file handle +// +// It isn't called directly from userspace so the error is ignored by +// the kernel +func (fh *ReadFileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + if fh.closed { + fs.Debug(fh.o, "ReadFileHandle.Release nothing to do") + return nil + } + fs.Debug(fh.o, "ReadFileHandle.Release closing") + err := fh.close() + if err != nil { + fs.ErrorLog(fh.o, "ReadFileHandle.Release error: %v", err) + } else { + fs.Debug(fh.o, "ReadFileHandle.Release OK") + } + return err +} diff --git a/cmd/mount/read_test.go b/cmd/mount/read_test.go new file mode 100644 index 000000000..7afb79965 --- /dev/null +++ b/cmd/mount/read_test.go @@ -0,0 +1,75 @@ +// +build linux darwin freebsd + +package mount + +import ( + "io" + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Read by byte including don't read any bytes +func TestReadByByte(t *testing.T) { + var data = []byte("hellohello") + run.createFile(t, "testfile", string(data)) + run.checkDir(t, "testfile 10") + + for i := 0; i < len(data); i++ { + fd, err := os.Open(run.path("testfile")) + assert.NoError(t, err) + for j := 0; j < i; j++ { + buf := make([]byte, 1) + n, err := io.ReadFull(fd, buf) + assert.NoError(t, err) + assert.Equal(t, 1, n) + assert.Equal(t, buf[0], data[j]) + } + err = fd.Close() + assert.NoError(t, err) + } + + run.rm(t, "testfile") +} + +// Test double close +func TestReadFileDoubleClose(t *testing.T) { + run.createFile(t, "testdoubleclose", "hello") + + in, err := os.Open(run.path("testdoubleclose")) + assert.NoError(t, err) + fd := in.Fd() + + fd1, err := syscall.Dup(int(fd)) + assert.NoError(t, err) + + fd2, err := syscall.Dup(int(fd)) + assert.NoError(t, err) + + // close one of the dups - should produce no error + err = syscall.Close(fd1) + assert.NoError(t, err) + + // read from the file + buf := make([]byte, 1) + _, err = in.Read(buf) + assert.NoError(t, err) + + // close it + err = in.Close() + assert.NoError(t, err) + + // read from the other dup - should produce no error as this + // file is now buffered + n, err := syscall.Read(fd2, buf) + assert.NoError(t, err) + assert.Equal(t, 1, n) + + // close the dup - should produce an error + err = syscall.Close(fd2) + assert.Error(t, err, "input/output error") + + run.rm(t, "testdoubleclose") +} diff --git a/cmd/mount/write.go b/cmd/mount/write.go new file mode 100644 index 000000000..a58c17a56 --- /dev/null +++ b/cmd/mount/write.go @@ -0,0 +1,157 @@ +// +build linux darwin freebsd + +package mount + +import ( + "errors" + "io" + "sync" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/fs" + "golang.org/x/net/context" +) + +var errClosedFileHandle = errors.New("Attempt to use closed file handle") + +// WriteFileHandle is an open for write handle on a File +type WriteFileHandle struct { + mu sync.Mutex + closed bool // set if handle has been closed + remote string + pipeReader *io.PipeReader + pipeWriter *io.PipeWriter + o fs.Object + result chan error + file *File + writeCalled bool // set the first time Write() is called +} + +// Check interface satisfied +var _ fusefs.Handle = (*WriteFileHandle)(nil) + +func newWriteFileHandle(d *Dir, f *File, src fs.ObjectInfo) (*WriteFileHandle, error) { + fh := &WriteFileHandle{ + remote: src.Remote(), + result: make(chan error, 1), + file: f, + } + fh.pipeReader, fh.pipeWriter = io.Pipe() + go func() { + o, err := d.f.Put(fh.pipeReader, src) + fh.o = o + fh.result <- err + }() + fh.file.addWriters(1) + return fh, nil +} + +// Check interface satisfied +var _ fusefs.HandleWriter = (*WriteFileHandle)(nil) + +// Write data to the file handle +func (fh *WriteFileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + fs.Debug(fh.remote, "WriteFileHandle.Write len=%d", len(req.Data)) + fh.mu.Lock() + defer fh.mu.Unlock() + if fh.closed { + fs.ErrorLog(fh.remote, "WriteFileHandle.Write error: %v", errClosedFileHandle) + return errClosedFileHandle + } + fh.writeCalled = true + // FIXME should probably check the file isn't being seeked? + n, err := fh.pipeWriter.Write(req.Data) + resp.Size = n + fh.file.written(int64(n)) + if err != nil { + fs.ErrorLog(fh.remote, "WriteFileHandle.Write error: %v", err) + return err + } + fs.Debug(fh.remote, "WriteFileHandle.Write OK (%d bytes written)", n) + return nil +} + +// close the file handle returning errClosedFileHandle if it has been +// closed already. +// +// Must be called with fh.mu held +func (fh *WriteFileHandle) close() error { + if fh.closed { + return errClosedFileHandle + } + fh.closed = true + fh.file.addWriters(-1) + writeCloseErr := fh.pipeWriter.Close() + err := <-fh.result + readCloseErr := fh.pipeReader.Close() + if err == nil { + fh.file.setObject(fh.o) + err = writeCloseErr + } + if err == nil { + err = readCloseErr + } + return err +} + +// Check interface satisfied +var _ fusefs.HandleFlusher = (*WriteFileHandle)(nil) + +// Flush is called on each close() of a file descriptor. So if a +// filesystem wants to return write errors in close() and the file has +// cached dirty data, this is a good place to write back data and +// return any errors. Since many applications ignore close() errors +// this is not always useful. +// +// NOTE: The flush() method may be called more than once for each +// open(). This happens if more than one file descriptor refers to an +// opened file due to dup(), dup2() or fork() calls. It is not +// possible to determine if a flush is final, so each flush should be +// treated equally. Multiple write-flush sequences are relatively +// rare, so this shouldn't be a problem. +// +// Filesystems shouldn't assume that flush will always be called after +// some writes, or that if will be called at all. +func (fh *WriteFileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + fs.Debug(fh.remote, "WriteFileHandle.Flush") + // If Write hasn't been called then ignore the Flush - Release + // will pick it up + if !fh.writeCalled { + fs.Debug(fh.remote, "WriteFileHandle.Flush ignoring flush on unwritten handle") + return nil + + } + err := fh.close() + if err != nil { + fs.ErrorLog(fh.remote, "WriteFileHandle.Flush error: %v", err) + } else { + fs.Debug(fh.remote, "WriteFileHandle.Flush OK") + } + return err +} + +var _ fusefs.HandleReleaser = (*WriteFileHandle)(nil) + +// Release is called when we are finished with the file handle +// +// It isn't called directly from userspace so the error is ignored by +// the kernel +func (fh *WriteFileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + fh.mu.Lock() + defer fh.mu.Unlock() + if fh.closed { + fs.Debug(fh.remote, "WriteFileHandle.Release nothing to do") + return nil + } + fs.Debug(fh.remote, "WriteFileHandle.Release closing") + err := fh.close() + if err != nil { + fs.ErrorLog(fh.remote, "WriteFileHandle.Release error: %v", err) + } else { + fs.Debug(fh.remote, "WriteFileHandle.Release OK") + } + return err +} diff --git a/cmd/mount/write_test.go b/cmd/mount/write_test.go new file mode 100644 index 000000000..c94bb7f96 --- /dev/null +++ b/cmd/mount/write_test.go @@ -0,0 +1,93 @@ +// +build linux darwin freebsd + +package mount + +import ( + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test writing a file with no write()'s to it +func TestWriteFileNoWrite(t *testing.T) { + fd, err := os.Create(run.path("testnowrite")) + assert.NoError(t, err) + + err = fd.Close() + assert.NoError(t, err) + + run.checkDir(t, "testnowrite 0") + + run.rm(t, "testnowrite") +} + +// Test open file in directory listing +func FIXMETestWriteOpenFileInDirListing(t *testing.T) { + fd, err := os.Create(run.path("testnowrite")) + assert.NoError(t, err) + + run.checkDir(t, "testnowrite 0") + + err = fd.Close() + assert.NoError(t, err) + + run.rm(t, "testnowrite") +} + +// Test writing a file and reading it back +func TestWriteFileWrite(t *testing.T) { + run.createFile(t, "testwrite", "data") + run.checkDir(t, "testwrite 4") + contents := run.readFile(t, "testwrite") + assert.Equal(t, "data", contents) + run.rm(t, "testwrite") +} + +// Test overwriting a file +func TestWriteFileOverwrite(t *testing.T) { + run.createFile(t, "testwrite", "data") + run.checkDir(t, "testwrite 4") + run.createFile(t, "testwrite", "potato") + contents := run.readFile(t, "testwrite") + assert.Equal(t, "potato", contents) + run.rm(t, "testwrite") +} + +// Test double close +func TestWriteFileDoubleClose(t *testing.T) { + out, err := os.Create(run.path("testdoubleclose")) + assert.NoError(t, err) + fd := out.Fd() + + fd1, err := syscall.Dup(int(fd)) + assert.NoError(t, err) + + fd2, err := syscall.Dup(int(fd)) + assert.NoError(t, err) + + // close one of the dups - should produce no error + err = syscall.Close(fd1) + assert.NoError(t, err) + + // write to the file + buf := []byte("hello") + n, err := out.Write(buf) + assert.NoError(t, err) + assert.Equal(t, 5, n) + + // close it + err = out.Close() + assert.NoError(t, err) + + // write to the other dup - should produce an error + n, err = syscall.Write(fd2, buf) + assert.Error(t, err, "input/output error") + + // close the dup - should produce an error + err = syscall.Close(fd2) + assert.Error(t, err, "input/output error") + + run.rm(t, "testdoubleclose") +}