From 1628ca0d46b9ec313c425735ab85d11ec2695317 Mon Sep 17 00:00:00 2001
From: Anthony Pessy <anthony@cogniteev.com>
Date: Mon, 5 Dec 2022 17:19:04 +0100
Subject: [PATCH] ftp: Improve performance to speed up --files-from and
NewObject
This commit uses the MLST command (where available) to get the status
for single files rather than listing the parent directory and looking
for the file. This makes actions such as using `--files-from` much quicker.
* use getEntry to lookup remote files when supported
* findItem now expects the full path directly
It makes the expected argument similar to the getInfo method, the
difference now is that one is returning a FileInfo whereas
the other is returning an ftp Entry.
Fixes #6225
Co-authored-by: Nick Craig-Wood <nick@craig-wood.com>
---
backend/ftp/ftp.go | 64 ++++++++++++++++++++++++----------------------
1 file changed, 34 insertions(+), 30 deletions(-)
diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index c68e7dc4a..01d686c8e 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -657,8 +657,7 @@ func (f *Fs) dirFromStandardPath(dir string) string {
// findItem finds a directory entry for the name in its parent directory
func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
- fullPath := path.Join(f.root, remote)
- if fullPath == "" || fullPath == "." || fullPath == "/" {
+ if remote == "" || remote == "." || remote == "/" {
// if root, assume exists and synthesize an entry
return &ftp.Entry{
Name: "",
@@ -666,13 +665,32 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
Time: time.Now(),
}, nil
}
- dir := path.Dir(fullPath)
- base := path.Base(fullPath)
c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, fmt.Errorf("findItem: %w", err)
}
+
+ // returns TRUE if MLST is supported which is required to call GetEntry
+ if c.IsTimePreciseInList() {
+ entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
+ f.putFtpConnection(&c, err)
+ if err != nil {
+ err = translateErrorFile(err)
+ if err == fs.ErrorObjectNotFound {
+ return nil, nil
+ }
+ return nil, err
+ }
+ if entry != nil {
+ f.entryToStandard(entry)
+ }
+ return entry, nil
+ }
+
+ dir := path.Dir(remote)
+ base := path.Base(remote)
+
files, err := c.List(f.dirFromStandardPath(dir))
f.putFtpConnection(&c, err)
if err != nil {
@@ -691,7 +709,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
- entry, err := f.findItem(ctx, remote)
+ entry, err := f.findItem(ctx, path.Join(f.root, remote))
if err != nil {
return nil, err
}
@@ -713,7 +731,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
// dirExists checks the directory pointed to by remote exists or not
func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
- entry, err := f.findItem(ctx, remote)
+ entry, err := f.findItem(ctx, path.Join(f.root, remote))
if err != nil {
return false, fmt.Errorf("dirExists: %w", err)
}
@@ -857,32 +875,18 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// getInfo reads the FileInfo for a path
func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
- dir := path.Dir(remote)
- base := path.Base(remote)
-
- c, err := f.getFtpConnection(ctx)
+ file, err := f.findItem(ctx, remote)
if err != nil {
- return nil, fmt.Errorf("getInfo: %w", err)
- }
- files, err := c.List(f.dirFromStandardPath(dir))
- f.putFtpConnection(&c, err)
- if err != nil {
- return nil, translateErrorFile(err)
- }
-
- for i := range files {
- file := files[i]
- f.entryToStandard(file)
- if file.Name == base {
- info := &FileInfo{
- Name: remote,
- Size: file.Size,
- ModTime: file.Time,
- precise: f.fLstTime,
- IsDir: file.Type == ftp.EntryTypeFolder,
- }
- return info, nil
+ return nil, err
+ } else if file != nil {
+ info := &FileInfo{
+ Name: remote,
+ Size: file.Size,
+ ModTime: file.Time,
+ precise: f.fLstTime,
+ IsDir: file.Type == ftp.EntryTypeFolder,
}
+ return info, nil
}
return nil, fs.ErrorObjectNotFound
}