diff --git a/fs/all/all.go b/fs/all/all.go index c144c75b4..7155d22ba 100644 --- a/fs/all/all.go +++ b/fs/all/all.go @@ -7,6 +7,7 @@ import ( _ "github.com/ncw/rclone/crypt" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" + _ "github.com/ncw/rclone/ftp" _ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/hubic" _ "github.com/ncw/rclone/local" @@ -15,5 +16,4 @@ import ( _ "github.com/ncw/rclone/sftp" _ "github.com/ncw/rclone/swift" _ "github.com/ncw/rclone/yandex" - _ "github.com/ncw/rclone/ftp" ) diff --git a/ftp/ftp.go b/ftp/ftp.go index e7e900f58..f73dd6ac1 100644 --- a/ftp/ftp.go +++ b/ftp/ftp.go @@ -3,23 +3,24 @@ package ftp import ( "fmt" - "github.com/jlaffaye/ftp" - "github.com/ncw/rclone/fs" "io" "path/filepath" "regexp" "strconv" - "time" - "sync" "strings" + "sync" + "time" + + "github.com/jlaffaye/ftp" + "github.com/ncw/rclone/fs" ) // Register with Fs func init() { fs.Register(&fs.RegInfo{ - Name: "Ftp", + Name: "Ftp", Description: "FTP interface", - NewFs: NewFs, + NewFs: NewFs, Options: []fs.Option{ { Name: "username", @@ -37,23 +38,24 @@ func init() { type Url struct { Scheme string - Host string - Port int - Path string + Host string + Port int + Path string } type Fs struct { - name string // name of this remote - c *ftp.ServerConn // the connection to the FTP server - root string // the path we are working on if any - url Url - mu sync.Mutex + name string // name of this remote + root string // the path we are working on if any + features *fs.Features // optional features + c *ftp.ServerConn // the connection to the FTP server + url Url + mu sync.Mutex } type Object struct { - fs *Fs - remote string - info *FileInfo + fs *Fs + remote string + info *FileInfo } type FileInfo struct { @@ -63,13 +65,11 @@ type FileInfo struct { IsDir bool } - - // Implements ReadCloser for FTP objects. type FtpReadCloser struct { - remote string - c *ftp.ServerConn - fd io.ReadCloser + remote string + c *ftp.ServerConn + fd io.ReadCloser } ///////////////// @@ -87,10 +87,12 @@ func parseUrl(url string) Url { // This is *similar* to the RFC 3986 regexp but it matches the // port independently from the host r, _ := regexp.Compile("^(([^:/?#]+):)?(//([^/?#:]*))?(:([0-9]+))?([^?#]*)(\\?([^#]*))?(#(.*))?") - + data := r.FindAllStringSubmatch(url, -1) - if data[0][5] == "" { data[0][5] = "21" } + if data[0][5] == "" { + data[0][5] = "21" + } port, _ := strconv.Atoi(data[0][5]) return Url{data[0][2], data[0][4], port, data[0][7]} } @@ -100,9 +102,9 @@ func parseUrl(url string) Url { //////////////// func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { - fs.Debug(f, "Trying to put file %s", src.Remote()) + fs.Debugf(f, "Trying to put file %s", src.Remote()) o := &Object{ - fs: f, + fs: f, remote: src.Remote(), } err := o.Update(in, src) @@ -114,11 +116,11 @@ func (f *Fs) Rmdir(dir string) error { f.mu.Lock() files, _ := f.c.List(filepath.Join(f.root, dir)) f.mu.Unlock() - for i:= range files { + for i := range files { if files[i].Type == ftp.EntryTypeFolder { f.Rmdir(filepath.Join(dir, files[i].Name)) } - } + } f.mu.Lock() err := f.c.RemoveDir(filepath.Join(f.root, dir)) f.mu.Unlock() @@ -126,7 +128,7 @@ func (f *Fs) Rmdir(dir string) error { } func (f *Fs) Name() string { - return f.name + return f.name } // Root of the remote (as passed into NewFs) @@ -138,6 +140,11 @@ func (f *Fs) String() string { return fmt.Sprintf("FTP Connection to %s", f.url.String()) } +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + // Hash are not supported func (f *Fs) Hashes() fs.HashSet { return 0 @@ -151,7 +158,7 @@ func (f *Fs) Precision() time.Duration { func (f *Fs) mkdir(abspath string) error { _, err := f.GetInfo(abspath) if err != nil { - fs.Debug(f, "Trying to create directory %s", abspath) + fs.Debugf(f, "Trying to create directory %s", abspath) f.mu.Lock() err := f.c.MakeDir(abspath) f.mu.Unlock() @@ -159,17 +166,17 @@ func (f *Fs) mkdir(abspath string) error { return err } } - return err + return err } func (f *Fs) Mkdir(dir string) error { // This actually works as mkdir -p - fs.Debug(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir) - defer fs.Debug(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir) + fs.Debugf(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir) + defer fs.Debugf(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir) abspath := filepath.Join(f.root, dir) tokens := strings.Split(abspath, "/") curdir := "" - for i:= range tokens { + for i := range tokens { curdir += "/" + tokens[i] f.mkdir(curdir) } @@ -177,21 +184,21 @@ func (f *Fs) Mkdir(dir string) error { } func (f *Fs) GetInfo(remote string) (*FileInfo, error) { - fs.Debug(f, "ENTER function 'GetInfo' on file %s", remote) - defer fs.Debug(f, "EXIT function 'GetInfo'") + fs.Debugf(f, "ENTER function 'GetInfo' on file %s", remote) + defer fs.Debugf(f, "EXIT function 'GetInfo'") dir := filepath.Dir(remote) base := filepath.Base(remote) f.mu.Lock() files, _ := f.c.List(dir) f.mu.Unlock() - for i:= range files { + for i := range files { if files[i].Name == base { info := &FileInfo{ - Name: remote, - Size: files[i].Size, + Name: remote, + Size: files[i].Size, ModTime: files[i].Time, - IsDir: files[i].Type == ftp.EntryTypeFolder, + IsDir: files[i].Type == ftp.EntryTypeFolder, } return info, nil } @@ -200,53 +207,53 @@ func (f *Fs) GetInfo(remote string) (*FileInfo, error) { } func (f *Fs) NewObject(remote string) (fs.Object, error) { - fs.Debug(f, "ENTER function 'NewObject' called with remote %s", remote) - defer fs.Debug(f, "EXIT function 'NewObject'") + fs.Debugf(f, "ENTER function 'NewObject' called with remote %s", remote) + defer fs.Debugf(f, "EXIT function 'NewObject'") dir := filepath.Dir(remote) base := filepath.Base(remote) f.mu.Lock() files, _ := f.c.List(dir) f.mu.Unlock() - for i:= range files { + for i := range files { if files[i].Name == base { o := &Object{ fs: f, remote: remote, } info := &FileInfo{ - Name: remote, - Size: files[i].Size, + Name: remote, + Size: files[i].Size, ModTime: files[i].Time, } o.info = info - + return o, nil } } return nil, fs.ErrorObjectNotFound } -func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { - fs.Debug(f, "ENTER function 'list'") - defer fs.Debug(f, "EXIT function 'list'") +func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { + fs.Debugf(f, "ENTER function 'list'") + defer fs.Debugf(f, "EXIT function 'list'") f.mu.Lock() files, _ := f.c.List(filepath.Join(f.root, dir)) f.mu.Unlock() - for i:= range files { + for i := range files { object := files[i] newremote := filepath.Join(dir, object.Name) switch object.Type { case ftp.EntryTypeFolder: - if out.IncludeDirectory(newremote){ + if out.IncludeDirectory(newremote) { d := &fs.Dir{ Name: newremote, When: object.Time, Bytes: 0, Count: -1, } - if curlevel < out.Level(){ - f.list(out, filepath.Join(dir, object.Name), curlevel +1 ) + if curlevel < out.Level() { + f.list(out, filepath.Join(dir, object.Name), curlevel+1) } if out.AddDir(d) { return @@ -258,8 +265,8 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { remote: newremote, } info := &FileInfo{ - Name: newremote, - Size: object.Size, + Name: newremote, + Size: object.Size, ModTime: object.Time, } o.info = info @@ -271,8 +278,8 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { } func (f *Fs) List(out fs.ListOpts, dir string) { - fs.Debug(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir) - defer fs.Debug(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir) + fs.Debugf(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir) + defer fs.Debugf(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir) f.list(out, dir, 1) out.Finished() } @@ -287,8 +294,8 @@ func (o *Object) Hash(t fs.HashType) (string, error) { func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { path := filepath.Join(o.fs.root, o.remote) - fs.Debug(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root) - defer fs.Debug(o.fs, "EXIT function 'Open' %s", path) + fs.Debugf(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root) + defer fs.Debugf(o.fs, "EXIT function 'Open' %s", path) c, _, err := ftpConnection(o.fs.name, o.fs.root) if err != nil { return nil, err @@ -306,8 +313,8 @@ func (o *Object) Remote() string { func (o *Object) Remove() error { path := filepath.Join(o.fs.root, o.remote) - fs.Debug(o, "ENTER function 'Remove' for obejct at %s", path) - defer fs.Debug(o, "EXIT function 'Remove' for obejct at %s", path) + fs.Debugf(o, "ENTER function 'Remove' for obejct at %s", path) + defer fs.Debugf(o, "EXIT function 'Remove' for obejct at %s", path) // Check if it's a directory or a file info, _ := o.fs.GetInfo(path) var err error @@ -348,8 +355,8 @@ func (o *Object) String() string { func (o *Object) MakeAllDir() { tokens := strings.Split(filepath.Dir(o.remote), "/") dir := "" - for i:= range tokens { - dir += tokens[i]+"/" + for i := range tokens { + dir += tokens[i] + "/" o.fs.Mkdir(dir) } } @@ -392,42 +399,43 @@ func ftpConnection(name, root string) (*ftp.ServerConn, Url, error) { pass := fs.ConfigFileGet(name, "password") u := parseUrl(url) u.Path = filepath.Join(u.Path, root) - fs.Debug(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path) + fs.Debugf(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path) globalMux.Lock() defer globalMux.Unlock() c, err := ftp.DialTimeout(u.ToDial(), 30*time.Second) if err != nil { - fs.ErrorLog(nil, "Error while Dialing %s: %s", u.ToDial(), err) + fs.Errorf(nil, "Error while Dialing %s: %s", u.ToDial(), err) return nil, u, err } err = c.Login(user, pass) if err != nil { - fs.ErrorLog(nil, "Error while Logging in into %s: %s", u.ToDial(), err) + fs.Errorf(nil, "Error while Logging in into %s: %s", u.ToDial(), err) return nil, u, err } return c, u, nil } - - // Register the FS func NewFs(name, root string) (fs.Fs, error) { - fs.Debug(nil, "ENTER function 'NewFs' with name %s and root %s", name, root) - defer fs.Debug(nil, "EXIT function 'NewFs'") + fs.Debugf(nil, "ENTER function 'NewFs' with name %s and root %s", name, root) + defer fs.Debugf(nil, "EXIT function 'NewFs'") c, u, err := ftpConnection(name, root) if err != nil { return nil, err } - fs := &Fs{ + f := &Fs{ name: name, root: u.Path, - c: c, - url: u, - mu: sync.Mutex{}, + c: c, + url: u, + mu: sync.Mutex{}, } - return fs, err + f.features = (&fs.Features{}).Fill(f) + return f, err } +// Check the interfaces are satisfied var ( - _ fs.Fs = &Fs{} + _ fs.Fs = &Fs{} + _ fs.Object = &Object{} ) diff --git a/ftp/ftp_test.go b/ftp/ftp_test.go index 7b223fdcd..63a792240 100644 --- a/ftp/ftp_test.go +++ b/ftp/ftp_test.go @@ -38,6 +38,7 @@ func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }