ftp: rework mkdir to be more efficient

This commit is contained in:
Nick Craig-Wood 2017-05-18 17:23:02 +01:00
parent 9627a6142d
commit 0ca6408580

View file

@ -336,11 +336,15 @@ func (f *Fs) Precision() time.Duration {
// nil and the error // nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// fs.Debugf(f, "Trying to put file %s", src.Remote()) // fs.Debugf(f, "Trying to put file %s", src.Remote())
err := f.mkParentDir(src.Remote())
if err != nil {
return nil, errors.Wrap(err, "Put mkParentDir failed")
}
o := &Object{ o := &Object{
fs: f, fs: f,
remote: src.Remote(), remote: src.Remote(),
} }
err := o.Update(in, src) err = o.Update(in, src)
return o, err return o, err
} }
@ -374,39 +378,46 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
return nil, fs.ErrorObjectNotFound return nil, fs.ErrorObjectNotFound
} }
// mkdir makes the directory and parents using unrooted paths
func (f *Fs) mkdir(abspath string) error { func (f *Fs) mkdir(abspath string) error {
_, err := f.getInfo(abspath) if abspath == "." || abspath == "/" {
if err == fs.ErrorObjectNotFound { return nil
// fs.Debugf(f, "Trying to create directory %s", abspath)
c, connErr := f.getFtpConnection()
if connErr != nil {
return errors.Wrap(connErr, "mkdir")
}
err = c.MakeDir(abspath)
f.putFtpConnection(&c)
} }
fi, err := f.getInfo(abspath)
if err == nil {
if fi.IsDir {
return nil
}
return fs.ErrorIsFile
} else if err != fs.ErrorObjectNotFound {
return errors.Wrapf(err, "mkdir %q failed", abspath)
}
parent := path.Dir(abspath)
err = f.mkdir(parent)
if err != nil {
return err
}
c, connErr := f.getFtpConnection()
if connErr != nil {
return errors.Wrap(connErr, "mkdir")
}
err = c.MakeDir(abspath)
f.putFtpConnection(&c)
return err return err
} }
// mkParentDir makes the parent of remote if necessary and any
// directories above that
func (f *Fs) mkParentDir(remote string) error {
parent := path.Dir(remote)
return f.mkdir(path.Join(f.root, parent))
}
// Mkdir creates the directory if it doesn't exist // Mkdir creates the directory if it doesn't exist
func (f *Fs) Mkdir(dir string) (err error) { func (f *Fs) Mkdir(dir string) (err error) {
// defer fs.Trace(dir, "")("err=%v", &err) // defer fs.Trace(dir, "")("err=%v", &err)
// This actually works as mkdir -p root := path.Join(f.root, dir)
abspath := path.Join(f.root, dir) return f.mkdir(root)
tokens := strings.Split(abspath, "/")
curdir := ""
for i := range tokens {
curdir += tokens[i]
if curdir == "" {
continue
}
err := f.mkdir(curdir)
if err != nil {
return err
}
curdir += "/"
}
return nil
} }
// Rmdir removes the directory (container, bucket) if empty // Rmdir removes the directory (container, bucket) if empty
@ -509,21 +520,6 @@ func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
return rc, nil return rc, nil
} }
// makeAllDir creates the parent directories for the object
func (o *Object) makeAllDir() error {
dir, _ := path.Split(o.remote)
tokens := strings.Split(dir, "/")
dir = ""
for i := range tokens {
dir += tokens[i] + "/"
err := o.fs.Mkdir(dir)
if err != nil {
return err
}
}
return nil
}
// Update the already existing object // Update the already existing object
// //
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
@ -531,11 +527,6 @@ func (o *Object) makeAllDir() error {
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
// defer fs.Trace(o, "src=%v", src)("err=%v", &err) // defer fs.Trace(o, "src=%v", src)("err=%v", &err)
// Create all upper directory first...
err = o.makeAllDir()
if err != nil {
return errors.Wrap(err, "update mkdir")
}
path := path.Join(o.fs.root, o.remote) path := path.Join(o.fs.root, o.remote)
// remove the file if upload failed // remove the file if upload failed
remove := func() { remove := func() {