TrueCloudLab/rclone
16
0
Fork 4
Code Issues3 Pull requests2 Projects Releases3 Packages Wiki Activity Actions

ftp: fix bitrot

Browse source
This commit is contained in:
Nick Craig-Wood 2017-05-03 13:18:17 +01:00
parent c72fca2711
commit e0ba1a2cd2
3 changed files with 86 additions and 77 deletions

2
fs/all/all.go
View file

@ -7,6 +7,7 @@ import (
_ "github.com/ncw/rclone/crypt" _ "github.com/ncw/rclone/crypt"
_ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/ftp"
_ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/hubic" _ "github.com/ncw/rclone/hubic"
_ "github.com/ncw/rclone/local" _ "github.com/ncw/rclone/local"
@ -15,5 +16,4 @@ import (
_ "github.com/ncw/rclone/sftp" _ "github.com/ncw/rclone/sftp"
_ "github.com/ncw/rclone/swift" _ "github.com/ncw/rclone/swift"
_ "github.com/ncw/rclone/yandex" _ "github.com/ncw/rclone/yandex"
_ "github.com/ncw/rclone/ftp"
) )

160
ftp/ftp.go
View file

@ -3,23 +3,24 @@ package ftp
import ( import (
"fmt" "fmt"
"github.com/jlaffaye/ftp"
"github.com/ncw/rclone/fs"
"io" "io"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strconv" "strconv"
"time"
"sync"
"strings" "strings"
"sync"
"time"
"github.com/jlaffaye/ftp"
"github.com/ncw/rclone/fs"
) )
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.RegInfo{ fs.Register(&fs.RegInfo{
Name: "Ftp", Name: "Ftp",
Description: "FTP interface", Description: "FTP interface",
NewFs: NewFs, NewFs: NewFs,
Options: []fs.Option{ Options: []fs.Option{
{ {
Name: "username", Name: "username",
@ -37,23 +38,24 @@ func init() {
type Url struct { type Url struct {
Scheme string Scheme string
Host string Host string
Port int Port int
Path string Path string
} }
type Fs struct { type Fs struct {
name string // name of this remote name string // name of this remote
c *ftp.ServerConn // the connection to the FTP server root string // the path we are working on if any
root string // the path we are working on if any features *fs.Features // optional features
url Url c *ftp.ServerConn // the connection to the FTP server
mu sync.Mutex url Url
mu sync.Mutex
} }
type Object struct { type Object struct {
fs *Fs fs *Fs
remote string remote string
info *FileInfo info *FileInfo
} }
type FileInfo struct { type FileInfo struct {
@ -63,13 +65,11 @@ type FileInfo struct {
IsDir bool IsDir bool
} }
// Implements ReadCloser for FTP objects. // Implements ReadCloser for FTP objects.
type FtpReadCloser struct { type FtpReadCloser struct {
remote string remote string
c *ftp.ServerConn c *ftp.ServerConn
fd io.ReadCloser fd io.ReadCloser
} }
///////////////// /////////////////
@ -87,10 +87,12 @@ func parseUrl(url string) Url {
// This is *similar* to the RFC 3986 regexp but it matches the // This is *similar* to the RFC 3986 regexp but it matches the
// port independently from the host // port independently from the host
r, _ := regexp.Compile("^(([^:/?#]+):)?(//([^/?#:]*))?(:([0-9]+))?([^?#]*)(\\?([^#]*))?(#(.*))?") r, _ := regexp.Compile("^(([^:/?#]+):)?(//([^/?#:]*))?(:([0-9]+))?([^?#]*)(\\?([^#]*))?(#(.*))?")
data := r.FindAllStringSubmatch(url, -1) data := r.FindAllStringSubmatch(url, -1)
if data[0][5] == "" { data[0][5] = "21" } if data[0][5] == "" {
data[0][5] = "21"
}
port, _ := strconv.Atoi(data[0][5]) port, _ := strconv.Atoi(data[0][5])
return Url{data[0][2], data[0][4], port, data[0][7]} return Url{data[0][2], data[0][4], port, data[0][7]}
} }
@ -100,9 +102,9 @@ func parseUrl(url string) Url {
//////////////// ////////////////
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
fs.Debug(f, "Trying to put file %s", src.Remote()) fs.Debugf(f, "Trying to put file %s", src.Remote())
o := &Object{ o := &Object{
fs: f, fs: f,
remote: src.Remote(), remote: src.Remote(),
} }
err := o.Update(in, src) err := o.Update(in, src)
@ -114,11 +116,11 @@ func (f *Fs) Rmdir(dir string) error {
f.mu.Lock() f.mu.Lock()
files, _ := f.c.List(filepath.Join(f.root, dir)) files, _ := f.c.List(filepath.Join(f.root, dir))
f.mu.Unlock() f.mu.Unlock()
for i:= range files { for i := range files {
if files[i].Type == ftp.EntryTypeFolder { if files[i].Type == ftp.EntryTypeFolder {
f.Rmdir(filepath.Join(dir, files[i].Name)) f.Rmdir(filepath.Join(dir, files[i].Name))
} }
} }
f.mu.Lock() f.mu.Lock()
err := f.c.RemoveDir(filepath.Join(f.root, dir)) err := f.c.RemoveDir(filepath.Join(f.root, dir))
f.mu.Unlock() f.mu.Unlock()
@ -126,7 +128,7 @@ func (f *Fs) Rmdir(dir string) error {
} }
func (f *Fs) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
@ -138,6 +140,11 @@ func (f *Fs) String() string {
return fmt.Sprintf("FTP Connection to %s", f.url.String()) return fmt.Sprintf("FTP Connection to %s", f.url.String())
} }
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// Hash are not supported // Hash are not supported
func (f *Fs) Hashes() fs.HashSet { func (f *Fs) Hashes() fs.HashSet {
return 0 return 0
@ -151,7 +158,7 @@ func (f *Fs) Precision() time.Duration {
func (f *Fs) mkdir(abspath string) error { func (f *Fs) mkdir(abspath string) error {
_, err := f.GetInfo(abspath) _, err := f.GetInfo(abspath)
if err != nil { if err != nil {
fs.Debug(f, "Trying to create directory %s", abspath) fs.Debugf(f, "Trying to create directory %s", abspath)
f.mu.Lock() f.mu.Lock()
err := f.c.MakeDir(abspath) err := f.c.MakeDir(abspath)
f.mu.Unlock() f.mu.Unlock()
@ -159,17 +166,17 @@ func (f *Fs) mkdir(abspath string) error {
return err return err
} }
} }
return err return err
} }
func (f *Fs) Mkdir(dir string) error { func (f *Fs) Mkdir(dir string) error {
// This actually works as mkdir -p // This actually works as mkdir -p
fs.Debug(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir) fs.Debugf(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir)
defer fs.Debug(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir) defer fs.Debugf(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir)
abspath := filepath.Join(f.root, dir) abspath := filepath.Join(f.root, dir)
tokens := strings.Split(abspath, "/") tokens := strings.Split(abspath, "/")
curdir := "" curdir := ""
for i:= range tokens { for i := range tokens {
curdir += "/" + tokens[i] curdir += "/" + tokens[i]
f.mkdir(curdir) f.mkdir(curdir)
} }
@ -177,21 +184,21 @@ func (f *Fs) Mkdir(dir string) error {
} }
func (f *Fs) GetInfo(remote string) (*FileInfo, error) { func (f *Fs) GetInfo(remote string) (*FileInfo, error) {
fs.Debug(f, "ENTER function 'GetInfo' on file %s", remote) fs.Debugf(f, "ENTER function 'GetInfo' on file %s", remote)
defer fs.Debug(f, "EXIT function 'GetInfo'") defer fs.Debugf(f, "EXIT function 'GetInfo'")
dir := filepath.Dir(remote) dir := filepath.Dir(remote)
base := filepath.Base(remote) base := filepath.Base(remote)
f.mu.Lock() f.mu.Lock()
files, _ := f.c.List(dir) files, _ := f.c.List(dir)
f.mu.Unlock() f.mu.Unlock()
for i:= range files { for i := range files {
if files[i].Name == base { if files[i].Name == base {
info := &FileInfo{ info := &FileInfo{
Name: remote, Name: remote,
Size: files[i].Size, Size: files[i].Size,
ModTime: files[i].Time, ModTime: files[i].Time,
IsDir: files[i].Type == ftp.EntryTypeFolder, IsDir: files[i].Type == ftp.EntryTypeFolder,
} }
return info, nil return info, nil
} }
@ -200,53 +207,53 @@ func (f *Fs) GetInfo(remote string) (*FileInfo, error) {
} }
func (f *Fs) NewObject(remote string) (fs.Object, error) { func (f *Fs) NewObject(remote string) (fs.Object, error) {
fs.Debug(f, "ENTER function 'NewObject' called with remote %s", remote) fs.Debugf(f, "ENTER function 'NewObject' called with remote %s", remote)
defer fs.Debug(f, "EXIT function 'NewObject'") defer fs.Debugf(f, "EXIT function 'NewObject'")
dir := filepath.Dir(remote) dir := filepath.Dir(remote)
base := filepath.Base(remote) base := filepath.Base(remote)
f.mu.Lock() f.mu.Lock()
files, _ := f.c.List(dir) files, _ := f.c.List(dir)
f.mu.Unlock() f.mu.Unlock()
for i:= range files { for i := range files {
if files[i].Name == base { if files[i].Name == base {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
} }
info := &FileInfo{ info := &FileInfo{
Name: remote, Name: remote,
Size: files[i].Size, Size: files[i].Size,
ModTime: files[i].Time, ModTime: files[i].Time,
} }
o.info = info o.info = info
return o, nil return o, nil
} }
} }
return nil, fs.ErrorObjectNotFound return nil, fs.ErrorObjectNotFound
} }
func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
fs.Debug(f, "ENTER function 'list'") fs.Debugf(f, "ENTER function 'list'")
defer fs.Debug(f, "EXIT function 'list'") defer fs.Debugf(f, "EXIT function 'list'")
f.mu.Lock() f.mu.Lock()
files, _ := f.c.List(filepath.Join(f.root, dir)) files, _ := f.c.List(filepath.Join(f.root, dir))
f.mu.Unlock() f.mu.Unlock()
for i:= range files { for i := range files {
object := files[i] object := files[i]
newremote := filepath.Join(dir, object.Name) newremote := filepath.Join(dir, object.Name)
switch object.Type { switch object.Type {
case ftp.EntryTypeFolder: case ftp.EntryTypeFolder:
if out.IncludeDirectory(newremote){ if out.IncludeDirectory(newremote) {
d := &fs.Dir{ d := &fs.Dir{
Name: newremote, Name: newremote,
When: object.Time, When: object.Time,
Bytes: 0, Bytes: 0,
Count: -1, Count: -1,
} }
if curlevel < out.Level(){ if curlevel < out.Level() {
f.list(out, filepath.Join(dir, object.Name), curlevel +1 ) f.list(out, filepath.Join(dir, object.Name), curlevel+1)
} }
if out.AddDir(d) { if out.AddDir(d) {
return return
@ -258,8 +265,8 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
remote: newremote, remote: newremote,
} }
info := &FileInfo{ info := &FileInfo{
Name: newremote, Name: newremote,
Size: object.Size, Size: object.Size,
ModTime: object.Time, ModTime: object.Time,
} }
o.info = info o.info = info
@ -271,8 +278,8 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
} }
func (f *Fs) List(out fs.ListOpts, dir string) { func (f *Fs) List(out fs.ListOpts, dir string) {
fs.Debug(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir) fs.Debugf(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir)
defer fs.Debug(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir) defer fs.Debugf(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir)
f.list(out, dir, 1) f.list(out, dir, 1)
out.Finished() out.Finished()
} }
@ -287,8 +294,8 @@ func (o *Object) Hash(t fs.HashType) (string, error) {
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
path := filepath.Join(o.fs.root, o.remote) path := filepath.Join(o.fs.root, o.remote)
fs.Debug(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root) fs.Debugf(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root)
defer fs.Debug(o.fs, "EXIT function 'Open' %s", path) defer fs.Debugf(o.fs, "EXIT function 'Open' %s", path)
c, _, err := ftpConnection(o.fs.name, o.fs.root) c, _, err := ftpConnection(o.fs.name, o.fs.root)
if err != nil { if err != nil {
return nil, err return nil, err
@ -306,8 +313,8 @@ func (o *Object) Remote() string {
func (o *Object) Remove() error { func (o *Object) Remove() error {
path := filepath.Join(o.fs.root, o.remote) path := filepath.Join(o.fs.root, o.remote)
fs.Debug(o, "ENTER function 'Remove' for obejct at %s", path) fs.Debugf(o, "ENTER function 'Remove' for obejct at %s", path)
defer fs.Debug(o, "EXIT function 'Remove' for obejct at %s", path) defer fs.Debugf(o, "EXIT function 'Remove' for obejct at %s", path)
// Check if it's a directory or a file // Check if it's a directory or a file
info, _ := o.fs.GetInfo(path) info, _ := o.fs.GetInfo(path)
var err error var err error
@ -348,8 +355,8 @@ func (o *Object) String() string {
func (o *Object) MakeAllDir() { func (o *Object) MakeAllDir() {
tokens := strings.Split(filepath.Dir(o.remote), "/") tokens := strings.Split(filepath.Dir(o.remote), "/")
dir := "" dir := ""
for i:= range tokens { for i := range tokens {
dir += tokens[i]+"/" dir += tokens[i] + "/"
o.fs.Mkdir(dir) o.fs.Mkdir(dir)
} }
} }
@ -392,42 +399,43 @@ func ftpConnection(name, root string) (*ftp.ServerConn, Url, error) {
pass := fs.ConfigFileGet(name, "password") pass := fs.ConfigFileGet(name, "password")
u := parseUrl(url) u := parseUrl(url)
u.Path = filepath.Join(u.Path, root) u.Path = filepath.Join(u.Path, root)
fs.Debug(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path) fs.Debugf(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path)
globalMux.Lock() globalMux.Lock()
defer globalMux.Unlock() defer globalMux.Unlock()
c, err := ftp.DialTimeout(u.ToDial(), 30*time.Second) c, err := ftp.DialTimeout(u.ToDial(), 30*time.Second)
if err != nil { if err != nil {
fs.ErrorLog(nil, "Error while Dialing %s: %s", u.ToDial(), err) fs.Errorf(nil, "Error while Dialing %s: %s", u.ToDial(), err)
return nil, u, err return nil, u, err
} }
err = c.Login(user, pass) err = c.Login(user, pass)
if err != nil { if err != nil {
fs.ErrorLog(nil, "Error while Logging in into %s: %s", u.ToDial(), err) fs.Errorf(nil, "Error while Logging in into %s: %s", u.ToDial(), err)
return nil, u, err return nil, u, err
} }
return c, u, nil return c, u, nil
} }
// Register the FS // Register the FS
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
fs.Debug(nil, "ENTER function 'NewFs' with name %s and root %s", name, root) fs.Debugf(nil, "ENTER function 'NewFs' with name %s and root %s", name, root)
defer fs.Debug(nil, "EXIT function 'NewFs'") defer fs.Debugf(nil, "EXIT function 'NewFs'")
c, u, err := ftpConnection(name, root) c, u, err := ftpConnection(name, root)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fs := &Fs{ f := &Fs{
name: name, name: name,
root: u.Path, root: u.Path,
c: c, c: c,
url: u, url: u,
mu: sync.Mutex{}, mu: sync.Mutex{},
} }
return fs, err f.features = (&fs.Features{}).Fill(f)
return f, err
} }
// Check the interfaces are satisfied
var ( var (
_ fs.Fs = &Fs{} _ fs.Fs = &Fs{}
_ fs.Object = &Object{}
) )

1
ftp/ftp_test.go
View file

@ -38,6 +38,7 @@ func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }