2017-05-03 12:51:03 +00:00
|
|
|
// Package ftp interfaces with FTP servers
|
2016-12-27 20:52:30 +00:00
|
|
|
package ftp
|
|
|
|
|
|
|
|
import (
|
2019-06-17 08:34:30 +00:00
|
|
|
"context"
|
2019-04-30 16:16:08 +00:00
|
|
|
"crypto/tls"
|
2021-11-04 10:12:57 +00:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
2016-12-27 20:52:30 +00:00
|
|
|
"io"
|
2021-01-29 03:41:50 +00:00
|
|
|
"net"
|
2017-05-03 20:45:01 +00:00
|
|
|
"net/textproto"
|
2017-05-03 15:42:04 +00:00
|
|
|
"path"
|
2020-07-18 08:52:22 +00:00
|
|
|
"runtime"
|
|
|
|
"strings"
|
2017-05-03 12:18:17 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2022-06-08 10:28:33 +00:00
|
|
|
"github.com/jlaffaye/ftp"
|
2019-07-28 17:47:38 +00:00
|
|
|
"github.com/rclone/rclone/fs"
|
2021-01-06 12:19:23 +00:00
|
|
|
"github.com/rclone/rclone/fs/accounting"
|
2020-01-14 17:33:35 +00:00
|
|
|
"github.com/rclone/rclone/fs/config"
|
2019-07-28 17:47:38 +00:00
|
|
|
"github.com/rclone/rclone/fs/config/configmap"
|
|
|
|
"github.com/rclone/rclone/fs/config/configstruct"
|
|
|
|
"github.com/rclone/rclone/fs/config/obscure"
|
2020-03-03 11:07:03 +00:00
|
|
|
"github.com/rclone/rclone/fs/fserrors"
|
2021-01-29 03:41:50 +00:00
|
|
|
"github.com/rclone/rclone/fs/fshttp"
|
2019-07-28 17:47:38 +00:00
|
|
|
"github.com/rclone/rclone/fs/hash"
|
2020-01-14 17:33:35 +00:00
|
|
|
"github.com/rclone/rclone/lib/encoder"
|
2020-09-27 10:40:58 +00:00
|
|
|
"github.com/rclone/rclone/lib/env"
|
2019-07-28 17:47:38 +00:00
|
|
|
"github.com/rclone/rclone/lib/pacer"
|
|
|
|
"github.com/rclone/rclone/lib/readers"
|
2016-12-27 20:52:30 +00:00
|
|
|
)
|
|
|
|
|
2020-09-27 10:40:58 +00:00
|
|
|
var (
|
|
|
|
currentUser = env.CurrentUser()
|
|
|
|
)
|
|
|
|
|
2020-03-03 11:07:03 +00:00
|
|
|
const (
|
|
|
|
minSleep = 10 * time.Millisecond
|
|
|
|
maxSleep = 2 * time.Second
|
|
|
|
decayConstant = 2 // bigger for slower decay, exponential
|
|
|
|
)
|
|
|
|
|
2016-12-27 20:52:30 +00:00
|
|
|
// Register with Fs
|
|
|
|
func init() {
|
|
|
|
fs.Register(&fs.RegInfo{
|
2017-05-03 15:07:40 +00:00
|
|
|
Name: "ftp",
|
2022-06-19 17:29:02 +00:00
|
|
|
Description: "FTP",
|
2017-05-03 12:18:17 +00:00
|
|
|
NewFs: NewFs,
|
2020-01-14 17:33:35 +00:00
|
|
|
Options: []fs.Option{{
|
|
|
|
Name: "host",
|
2021-08-22 13:11:41 +00:00
|
|
|
Help: "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
|
2020-01-14 17:33:35 +00:00
|
|
|
Required: true,
|
|
|
|
}, {
|
2021-09-11 16:29:57 +00:00
|
|
|
Name: "user",
|
|
|
|
Help: "FTP username.",
|
|
|
|
Default: currentUser,
|
2020-01-14 17:33:35 +00:00
|
|
|
}, {
|
2021-09-11 16:29:57 +00:00
|
|
|
Name: "port",
|
|
|
|
Help: "FTP port number.",
|
|
|
|
Default: 21,
|
2020-01-14 17:33:35 +00:00
|
|
|
}, {
|
|
|
|
Name: "pass",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: "FTP password.",
|
2020-01-14 17:33:35 +00:00
|
|
|
IsPassword: true,
|
|
|
|
}, {
|
2020-05-30 14:02:35 +00:00
|
|
|
Name: "tls",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: `Use Implicit FTPS (FTP over TLS).
|
|
|
|
|
2020-10-28 15:45:52 +00:00
|
|
|
When using implicit FTP over TLS the client connects using TLS
|
|
|
|
right from the start which breaks compatibility with
|
2020-05-30 14:02:35 +00:00
|
|
|
non-TLS-aware servers. This is usually served over port 990 rather
|
|
|
|
than port 21. Cannot be used in combination with explicit FTP.`,
|
|
|
|
Default: false,
|
|
|
|
}, {
|
|
|
|
Name: "explicit_tls",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: `Use Explicit FTPS (FTP over TLS).
|
|
|
|
|
2020-10-28 15:45:52 +00:00
|
|
|
When using explicit FTP over TLS the client explicitly requests
|
2020-05-30 14:02:35 +00:00
|
|
|
security from the server in order to upgrade a plain text connection
|
|
|
|
to an encrypted one. Cannot be used in combination with implicit FTP.`,
|
2020-01-14 17:33:35 +00:00
|
|
|
Default: false,
|
|
|
|
}, {
|
|
|
|
Name: "concurrency",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: "Maximum number of FTP simultaneous connections, 0 for unlimited.",
|
2020-01-14 17:33:35 +00:00
|
|
|
Default: 0,
|
|
|
|
Advanced: true,
|
|
|
|
}, {
|
|
|
|
Name: "no_check_certificate",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: "Do not verify the TLS certificate of the server.",
|
2020-01-14 17:33:35 +00:00
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
|
|
|
}, {
|
|
|
|
Name: "disable_epsv",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: "Disable using EPSV even if server advertises support.",
|
2020-01-14 17:33:35 +00:00
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2020-09-04 17:07:08 +00:00
|
|
|
}, {
|
|
|
|
Name: "disable_mlsd",
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: "Disable using MLSD even if server advertises support.",
|
2020-09-04 17:07:08 +00:00
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2022-06-01 17:15:09 +00:00
|
|
|
}, {
|
|
|
|
Name: "disable_utf8",
|
|
|
|
Help: "Disable using UTF-8 even if server advertises support.",
|
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2021-07-22 23:04:08 +00:00
|
|
|
}, {
|
|
|
|
Name: "writing_mdtm",
|
|
|
|
Help: "Use MDTM to set modification time (VsFtpd quirk)",
|
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2021-02-16 11:47:41 +00:00
|
|
|
}, {
|
|
|
|
Name: "idle_timeout",
|
|
|
|
Default: fs.Duration(60 * time.Second),
|
2021-08-16 09:30:01 +00:00
|
|
|
Help: `Max time before closing idle connections.
|
2021-02-16 11:47:41 +00:00
|
|
|
|
|
|
|
If no connections have been returned to the connection pool in the time
|
|
|
|
given, rclone will empty the connection pool.
|
|
|
|
|
|
|
|
Set to 0 to keep connections indefinitely.
|
|
|
|
`,
|
|
|
|
Advanced: true,
|
2020-02-25 16:51:18 +00:00
|
|
|
}, {
|
|
|
|
Name: "close_timeout",
|
|
|
|
Help: "Maximum time to wait for a response to close.",
|
|
|
|
Default: fs.Duration(60 * time.Second),
|
|
|
|
Advanced: true,
|
2021-06-05 02:34:57 +00:00
|
|
|
}, {
|
|
|
|
Name: "tls_cache_size",
|
|
|
|
Help: `Size of TLS session cache for all control and data connections.
|
2021-08-16 12:50:03 +00:00
|
|
|
|
2021-06-05 02:34:57 +00:00
|
|
|
TLS cache allows to resume TLS sessions and reuse PSK between connections.
|
|
|
|
Increase if default size is not enough resulting in TLS resumption errors.
|
|
|
|
Enabled by default. Use 0 to disable.`,
|
|
|
|
Default: 32,
|
|
|
|
Advanced: true,
|
2021-07-26 11:32:29 +00:00
|
|
|
}, {
|
|
|
|
Name: "disable_tls13",
|
|
|
|
Help: "Disable TLS 1.3 (workaround for FTP servers with buggy TLS)",
|
|
|
|
Default: false,
|
|
|
|
Advanced: true,
|
2021-08-24 14:37:04 +00:00
|
|
|
}, {
|
|
|
|
Name: "shut_timeout",
|
|
|
|
Help: "Maximum time to wait for data connection closing status.",
|
|
|
|
Default: fs.Duration(60 * time.Second),
|
|
|
|
Advanced: true,
|
2021-03-26 21:13:03 +00:00
|
|
|
}, {
|
|
|
|
Name: "ask_password",
|
|
|
|
Default: false,
|
|
|
|
Help: `Allow asking for FTP password when needed.
|
|
|
|
|
|
|
|
If this is set and no password is supplied then rclone will ask for a password
|
|
|
|
`,
|
|
|
|
Advanced: true,
|
2020-01-14 17:33:35 +00:00
|
|
|
}, {
|
|
|
|
Name: config.ConfigEncoding,
|
|
|
|
Help: config.ConfigEncodingHelp,
|
|
|
|
Advanced: true,
|
2021-07-22 23:01:17 +00:00
|
|
|
// The FTP protocol can't handle trailing spaces
|
|
|
|
// (for instance, pureftpd turns them into '_')
|
2020-01-14 21:51:49 +00:00
|
|
|
Default: (encoder.Display |
|
|
|
|
encoder.EncodeRightSpace),
|
2021-07-22 23:01:17 +00:00
|
|
|
Examples: []fs.OptionExample{{
|
|
|
|
Value: "Asterisk,Ctl,Dot,Slash",
|
|
|
|
Help: "ProFTPd can't handle '*' in file names",
|
|
|
|
}, {
|
|
|
|
Value: "BackSlash,Ctl,Del,Dot,RightSpace,Slash,SquareBracket",
|
|
|
|
Help: "PureFTPd can't handle '[]' or '*' in file names",
|
|
|
|
}, {
|
|
|
|
Value: "Ctl,LeftPeriod,Slash",
|
|
|
|
Help: "VsFTPd can't handle file names starting with dot",
|
|
|
|
}},
|
2020-01-14 17:33:35 +00:00
|
|
|
}},
|
2016-12-27 20:52:30 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
// Options defines the configuration for this backend
|
|
|
|
type Options struct {
|
2020-01-14 17:33:35 +00:00
|
|
|
Host string `config:"host"`
|
|
|
|
User string `config:"user"`
|
|
|
|
Pass string `config:"pass"`
|
|
|
|
Port string `config:"port"`
|
|
|
|
TLS bool `config:"tls"`
|
2020-05-30 14:02:35 +00:00
|
|
|
ExplicitTLS bool `config:"explicit_tls"`
|
2021-06-05 02:34:57 +00:00
|
|
|
TLSCacheSize int `config:"tls_cache_size"`
|
2021-07-26 11:32:29 +00:00
|
|
|
DisableTLS13 bool `config:"disable_tls13"`
|
2020-01-14 17:33:35 +00:00
|
|
|
Concurrency int `config:"concurrency"`
|
|
|
|
SkipVerifyTLSCert bool `config:"no_check_certificate"`
|
|
|
|
DisableEPSV bool `config:"disable_epsv"`
|
2020-09-04 17:07:08 +00:00
|
|
|
DisableMLSD bool `config:"disable_mlsd"`
|
2022-06-01 17:15:09 +00:00
|
|
|
DisableUTF8 bool `config:"disable_utf8"`
|
2021-07-22 23:04:08 +00:00
|
|
|
WritingMDTM bool `config:"writing_mdtm"`
|
2021-02-16 11:47:41 +00:00
|
|
|
IdleTimeout fs.Duration `config:"idle_timeout"`
|
2020-02-25 16:51:18 +00:00
|
|
|
CloseTimeout fs.Duration `config:"close_timeout"`
|
2021-08-24 14:37:04 +00:00
|
|
|
ShutTimeout fs.Duration `config:"shut_timeout"`
|
2021-03-26 21:13:03 +00:00
|
|
|
AskPassword bool `config:"ask_password"`
|
2020-01-14 17:33:35 +00:00
|
|
|
Enc encoder.MultiEncoder `config:"encoding"`
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Fs represents a remote FTP server
|
2016-12-27 20:52:30 +00:00
|
|
|
type Fs struct {
|
2020-11-05 11:33:32 +00:00
|
|
|
name string // name of this remote
|
|
|
|
root string // the path we are working on if any
|
|
|
|
opt Options // parsed options
|
|
|
|
ci *fs.ConfigInfo // global config
|
|
|
|
features *fs.Features // optional features
|
2017-05-25 09:16:51 +00:00
|
|
|
url string
|
2017-05-03 20:45:01 +00:00
|
|
|
user string
|
|
|
|
pass string
|
|
|
|
dialAddr string
|
2017-05-18 14:51:35 +00:00
|
|
|
poolMu sync.Mutex
|
|
|
|
pool []*ftp.ServerConn
|
2021-02-16 11:47:41 +00:00
|
|
|
drain *time.Timer // used to drain the pool when we stop using the connections
|
2019-02-13 21:53:43 +00:00
|
|
|
tokens *pacer.TokenDispenser
|
2021-01-29 03:41:50 +00:00
|
|
|
tlsConf *tls.Config
|
2020-03-03 11:07:03 +00:00
|
|
|
pacer *fs.Pacer // pacer for FTP connections
|
2021-07-22 23:04:08 +00:00
|
|
|
fGetTime bool // true if the ftp library accepts GetTime
|
|
|
|
fSetTime bool // true if the ftp library accepts SetTime
|
|
|
|
fLstTime bool // true if the List call returns precise time
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Object describes an FTP file
|
2016-12-27 20:52:30 +00:00
|
|
|
type Object struct {
|
2017-05-03 12:18:17 +00:00
|
|
|
fs *Fs
|
|
|
|
remote string
|
|
|
|
info *FileInfo
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// FileInfo is the metadata known about an FTP file
|
2016-12-27 20:52:30 +00:00
|
|
|
type FileInfo struct {
|
|
|
|
Name string
|
|
|
|
Size uint64
|
|
|
|
ModTime time.Time
|
2021-07-22 23:04:08 +00:00
|
|
|
precise bool // true if the time is precise
|
2016-12-27 20:52:30 +00:00
|
|
|
IsDir bool
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// ------------------------------------------------------------
|
2016-12-27 20:52:30 +00:00
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Name of this fs
|
2016-12-27 20:52:30 +00:00
|
|
|
func (f *Fs) Name() string {
|
2017-05-03 12:18:17 +00:00
|
|
|
return f.name
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
|
|
func (f *Fs) Root() string {
|
|
|
|
return f.root
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// String returns a description of the FS
|
2016-12-27 20:52:30 +00:00
|
|
|
func (f *Fs) String() string {
|
2017-05-25 09:16:51 +00:00
|
|
|
return f.url
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:18:17 +00:00
|
|
|
// Features returns the optional features of this Fs
|
|
|
|
func (f *Fs) Features() *fs.Features {
|
|
|
|
return f.features
|
|
|
|
}
|
|
|
|
|
2020-07-18 08:52:22 +00:00
|
|
|
// Enable debugging output
|
|
|
|
type debugLog struct {
|
|
|
|
mu sync.Mutex
|
|
|
|
auth bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write writes len(p) bytes from p to the underlying data stream. It returns
|
|
|
|
// the number of bytes written from p (0 <= n <= len(p)) and any error
|
|
|
|
// encountered that caused the write to stop early. Write must return a non-nil
|
|
|
|
// error if it returns n < len(p). Write must not modify the slice data, even
|
|
|
|
// temporarily.
|
|
|
|
//
|
|
|
|
// Implementations must not retain p.
|
|
|
|
//
|
|
|
|
// This writes debug info to the log
|
|
|
|
func (dl *debugLog) Write(p []byte) (n int, err error) {
|
|
|
|
dl.mu.Lock()
|
|
|
|
defer dl.mu.Unlock()
|
|
|
|
_, file, _, ok := runtime.Caller(1)
|
|
|
|
direction := "FTP Rx"
|
|
|
|
if ok && strings.Contains(file, "multi") {
|
|
|
|
direction = "FTP Tx"
|
|
|
|
}
|
|
|
|
lines := strings.Split(string(p), "\r\n")
|
|
|
|
if lines[len(lines)-1] == "" {
|
|
|
|
lines = lines[:len(lines)-1]
|
|
|
|
}
|
|
|
|
for _, line := range lines {
|
|
|
|
if !dl.auth && strings.HasPrefix(line, "PASS") {
|
|
|
|
fs.Debugf(direction, "PASS *****")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
fs.Debugf(direction, "%q", line)
|
|
|
|
}
|
|
|
|
return len(p), nil
|
|
|
|
}
|
|
|
|
|
2020-03-03 11:07:03 +00:00
|
|
|
// shouldRetry returns a boolean as to whether this err deserve to be
|
|
|
|
// retried. It returns the err as a convenience
|
|
|
|
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
|
|
|
if fserrors.ContextError(ctx, &err) {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
|
|
|
case ftp.StatusNotAvailable:
|
|
|
|
return true, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return fserrors.ShouldRetry(err), err
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Open a new connection to the FTP server.
|
2020-03-03 11:07:03 +00:00
|
|
|
func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
fs.Debugf(f, "Connecting to FTP server")
|
2021-04-08 22:43:50 +00:00
|
|
|
|
|
|
|
// Make ftp library dial with fshttp dialer optionally using TLS
|
|
|
|
dial := func(network, address string) (conn net.Conn, err error) {
|
|
|
|
conn, err = fshttp.NewDialer(ctx).Dial(network, address)
|
|
|
|
if f.tlsConf != nil && err == nil {
|
|
|
|
conn = tls.Client(conn, f.tlsConf)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
|
|
|
|
|
|
|
|
if f.opt.TLS {
|
|
|
|
// Our dialer takes care of TLS but ftp library also needs tlsConf
|
|
|
|
// as a trigger for sending PSBZ and PROT options to server.
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
|
|
|
|
} else if f.opt.ExplicitTLS {
|
2021-01-29 03:41:50 +00:00
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
|
|
|
|
// Initial connection needs to be cleartext for explicit TLS
|
|
|
|
conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2019-04-30 16:16:08 +00:00
|
|
|
}
|
2021-01-29 03:41:50 +00:00
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
|
2019-04-30 16:16:08 +00:00
|
|
|
}
|
2019-10-10 17:07:48 +00:00
|
|
|
if f.opt.DisableEPSV {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
|
|
|
|
}
|
2020-09-04 17:07:08 +00:00
|
|
|
if f.opt.DisableMLSD {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
|
|
|
|
}
|
2022-06-01 17:15:09 +00:00
|
|
|
if f.opt.DisableUTF8 {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
|
|
|
|
}
|
2021-08-24 14:37:04 +00:00
|
|
|
if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
|
|
|
|
}
|
2021-07-22 23:04:08 +00:00
|
|
|
if f.opt.WritingMDTM {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
|
|
|
|
ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
|
2020-07-18 08:52:22 +00:00
|
|
|
}
|
2020-03-03 11:07:03 +00:00
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
|
|
c, err = ftp.Dial(f.dialAddr, ftpConfig...)
|
|
|
|
if err != nil {
|
|
|
|
return shouldRetry(ctx, err)
|
|
|
|
}
|
|
|
|
err = c.Login(f.user, f.pass)
|
|
|
|
if err != nil {
|
|
|
|
_ = c.Quit()
|
|
|
|
return shouldRetry(ctx, err)
|
|
|
|
}
|
|
|
|
return false, nil
|
|
|
|
})
|
2017-05-03 20:45:01 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
err = fmt.Errorf("failed to make FTP connection to %q: %w", f.dialAddr, err)
|
2017-05-03 20:45:01 +00:00
|
|
|
}
|
2020-03-03 11:07:03 +00:00
|
|
|
return c, err
|
2017-05-03 20:45:01 +00:00
|
|
|
}
|
|
|
|
|
2017-05-18 14:51:35 +00:00
|
|
|
// Get an FTP connection from the pool, or open a new one
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
2019-02-13 21:53:43 +00:00
|
|
|
if f.opt.Concurrency > 0 {
|
|
|
|
f.tokens.Get()
|
|
|
|
}
|
2021-01-06 12:19:23 +00:00
|
|
|
accounting.LimitTPS(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
f.poolMu.Lock()
|
|
|
|
if len(f.pool) > 0 {
|
|
|
|
c = f.pool[0]
|
|
|
|
f.pool = f.pool[1:]
|
|
|
|
}
|
|
|
|
f.poolMu.Unlock()
|
|
|
|
if c != nil {
|
|
|
|
return c, nil
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err = f.ftpConnection(ctx)
|
2020-02-22 10:37:01 +00:00
|
|
|
if err != nil && f.opt.Concurrency > 0 {
|
|
|
|
f.tokens.Put()
|
|
|
|
}
|
|
|
|
return c, err
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return an FTP connection to the pool
|
|
|
|
//
|
|
|
|
// It nils the pointed to connection out so it can't be reused
|
2017-05-24 13:47:13 +00:00
|
|
|
//
|
|
|
|
// if err is not nil then it checks the connection is alive using a
|
|
|
|
// NOOP request
|
|
|
|
func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
|
2019-02-13 21:53:43 +00:00
|
|
|
if f.opt.Concurrency > 0 {
|
|
|
|
defer f.tokens.Put()
|
|
|
|
}
|
2020-02-25 11:38:13 +00:00
|
|
|
if pc == nil {
|
|
|
|
return
|
|
|
|
}
|
2017-05-24 13:47:13 +00:00
|
|
|
c := *pc
|
2020-02-25 11:38:13 +00:00
|
|
|
if c == nil {
|
|
|
|
return
|
|
|
|
}
|
2017-05-24 13:47:13 +00:00
|
|
|
*pc = nil
|
|
|
|
if err != nil {
|
|
|
|
// If not a regular FTP error code then check the connection
|
2021-11-04 10:12:57 +00:00
|
|
|
var tpErr *textproto.Error
|
|
|
|
if !errors.As(err, &tpErr) {
|
2017-05-24 13:47:13 +00:00
|
|
|
nopErr := c.NoOp()
|
|
|
|
if nopErr != nil {
|
|
|
|
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
|
|
|
|
_ = c.Quit()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-05-18 14:51:35 +00:00
|
|
|
f.poolMu.Lock()
|
2017-05-24 13:47:13 +00:00
|
|
|
f.pool = append(f.pool, c)
|
2021-02-16 11:47:41 +00:00
|
|
|
if f.opt.IdleTimeout > 0 {
|
|
|
|
f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
|
|
|
|
}
|
2017-05-18 14:51:35 +00:00
|
|
|
f.poolMu.Unlock()
|
|
|
|
}
|
|
|
|
|
2021-02-16 11:36:32 +00:00
|
|
|
// Drain the pool of any connections
|
|
|
|
func (f *Fs) drainPool(ctx context.Context) (err error) {
|
|
|
|
f.poolMu.Lock()
|
|
|
|
defer f.poolMu.Unlock()
|
2021-02-16 11:47:41 +00:00
|
|
|
if f.opt.IdleTimeout > 0 {
|
|
|
|
f.drain.Stop()
|
|
|
|
}
|
|
|
|
if len(f.pool) != 0 {
|
|
|
|
fs.Debugf(f, "closing %d unused connections", len(f.pool))
|
|
|
|
}
|
2021-02-16 11:36:32 +00:00
|
|
|
for i, c := range f.pool {
|
|
|
|
if cErr := c.Quit(); cErr != nil {
|
|
|
|
err = cErr
|
|
|
|
}
|
|
|
|
f.pool[i] = nil
|
|
|
|
}
|
|
|
|
f.pool = nil
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-02-07 17:41:17 +00:00
|
|
|
// NewFs constructs an Fs from the path, container:path
|
2020-11-05 15:18:51 +00:00
|
|
|
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
|
2018-05-14 17:06:57 +00:00
|
|
|
// Parse config into Options struct
|
|
|
|
opt := new(Options)
|
|
|
|
err = configstruct.Set(m, opt)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-05-25 09:16:51 +00:00
|
|
|
}
|
2021-03-26 21:13:03 +00:00
|
|
|
pass := ""
|
|
|
|
if opt.AskPassword && opt.Pass == "" {
|
|
|
|
pass = config.GetPassword("FTP server password")
|
|
|
|
} else {
|
|
|
|
pass, err = obscure.Reveal(opt.Pass)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("NewFS decrypt password: %w", err)
|
|
|
|
}
|
2017-05-03 15:07:40 +00:00
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
user := opt.User
|
2017-05-24 22:24:46 +00:00
|
|
|
if user == "" {
|
2020-09-27 10:40:58 +00:00
|
|
|
user = currentUser
|
2017-05-03 15:07:40 +00:00
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
port := opt.Port
|
2017-05-24 22:24:46 +00:00
|
|
|
if port == "" {
|
|
|
|
port = "21"
|
2017-05-03 20:45:01 +00:00
|
|
|
}
|
2017-05-24 22:24:46 +00:00
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
dialAddr := opt.Host + ":" + port
|
2019-04-30 16:16:08 +00:00
|
|
|
protocol := "ftp://"
|
|
|
|
if opt.TLS {
|
|
|
|
protocol = "ftps://"
|
|
|
|
}
|
2021-01-29 03:41:50 +00:00
|
|
|
if opt.TLS && opt.ExplicitTLS {
|
2022-06-08 20:54:39 +00:00
|
|
|
return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config")
|
2021-01-29 03:41:50 +00:00
|
|
|
}
|
|
|
|
var tlsConfig *tls.Config
|
|
|
|
if opt.TLS || opt.ExplicitTLS {
|
|
|
|
tlsConfig = &tls.Config{
|
|
|
|
ServerName: opt.Host,
|
|
|
|
InsecureSkipVerify: opt.SkipVerifyTLSCert,
|
|
|
|
}
|
2021-06-05 02:34:57 +00:00
|
|
|
if opt.TLSCacheSize > 0 {
|
|
|
|
tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(opt.TLSCacheSize)
|
|
|
|
}
|
2021-07-26 11:32:29 +00:00
|
|
|
if opt.DisableTLS13 {
|
|
|
|
tlsConfig.MaxVersion = tls.VersionTLS12
|
|
|
|
}
|
2021-01-29 03:41:50 +00:00
|
|
|
}
|
2019-04-30 16:16:08 +00:00
|
|
|
u := protocol + path.Join(dialAddr+"/", root)
|
2020-11-05 11:33:32 +00:00
|
|
|
ci := fs.GetConfig(ctx)
|
2017-05-03 20:45:01 +00:00
|
|
|
f := &Fs{
|
|
|
|
name: name,
|
|
|
|
root: root,
|
2018-05-14 17:06:57 +00:00
|
|
|
opt: *opt,
|
2020-11-05 11:33:32 +00:00
|
|
|
ci: ci,
|
2017-05-03 20:45:01 +00:00
|
|
|
url: u,
|
|
|
|
user: user,
|
|
|
|
pass: pass,
|
|
|
|
dialAddr: dialAddr,
|
2019-02-13 21:53:43 +00:00
|
|
|
tokens: pacer.NewTokenDispenser(opt.Concurrency),
|
2021-01-29 03:41:50 +00:00
|
|
|
tlsConf: tlsConfig,
|
2020-03-03 11:07:03 +00:00
|
|
|
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2017-08-09 14:27:43 +00:00
|
|
|
f.features = (&fs.Features{
|
|
|
|
CanHaveEmptyDirectories: true,
|
2020-11-05 16:00:40 +00:00
|
|
|
}).Fill(ctx, f)
|
2021-02-16 11:47:41 +00:00
|
|
|
// set the pool drainer timer going
|
|
|
|
if f.opt.IdleTimeout > 0 {
|
|
|
|
f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
|
|
|
}
|
2017-05-18 14:51:35 +00:00
|
|
|
// Make a connection and pool it to return errors early
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-03 12:51:03 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("NewFs: %w", err)
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
2021-07-22 23:04:08 +00:00
|
|
|
f.fGetTime = c.IsGetTimeSupported()
|
|
|
|
f.fSetTime = c.IsSetTimeSupported()
|
|
|
|
f.fLstTime = c.IsTimePreciseInList()
|
|
|
|
if !f.fLstTime && f.fGetTime {
|
|
|
|
f.features.SlowModTime = true
|
|
|
|
}
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, nil)
|
2017-05-03 20:45:01 +00:00
|
|
|
if root != "" {
|
|
|
|
// Check to see if the root actually an existing file
|
|
|
|
remote := path.Base(root)
|
|
|
|
f.root = path.Dir(root)
|
|
|
|
if f.root == "." {
|
|
|
|
f.root = ""
|
|
|
|
}
|
2019-06-17 08:34:30 +00:00
|
|
|
_, err := f.NewObject(ctx, remote)
|
2017-05-03 20:45:01 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// File doesn't exist so return old f
|
|
|
|
f.root = root
|
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// return an error with an fs which points to the parent
|
|
|
|
return f, fs.ErrorIsFile
|
|
|
|
}
|
|
|
|
return f, err
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2021-02-16 11:36:32 +00:00
|
|
|
// Shutdown the backend, closing any background tasks and any
|
|
|
|
// cached connections.
|
|
|
|
func (f *Fs) Shutdown(ctx context.Context) error {
|
|
|
|
return f.drainPool(ctx)
|
|
|
|
}
|
|
|
|
|
2017-05-03 20:45:01 +00:00
|
|
|
// translateErrorFile turns FTP errors into rclone errors if possible for a file
|
|
|
|
func translateErrorFile(err error) error {
|
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
2018-04-08 17:09:49 +00:00
|
|
|
case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
|
2017-05-03 20:45:01 +00:00
|
|
|
err = fs.ErrorObjectNotFound
|
|
|
|
}
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2017-05-03 20:45:01 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// translateErrorDir turns FTP errors into rclone errors if possible for a directory
|
|
|
|
func translateErrorDir(err error) error {
|
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
2018-04-26 13:53:03 +00:00
|
|
|
case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
|
2017-05-03 20:45:01 +00:00
|
|
|
err = fs.ErrorDirNotFound
|
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
2017-05-03 20:45:01 +00:00
|
|
|
return err
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2019-07-27 15:58:33 +00:00
|
|
|
// entryToStandard converts an incoming ftp.Entry to Standard encoding
|
2020-01-14 17:33:35 +00:00
|
|
|
func (f *Fs) entryToStandard(entry *ftp.Entry) {
|
2019-07-27 15:58:33 +00:00
|
|
|
// Skip . and .. as we don't want these encoded
|
|
|
|
if entry.Name == "." || entry.Name == ".." {
|
|
|
|
return
|
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
entry.Name = f.opt.Enc.ToStandardName(entry.Name)
|
|
|
|
entry.Target = f.opt.Enc.ToStandardPath(entry.Target)
|
2019-07-27 15:58:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// dirFromStandardPath returns dir in encoded form.
|
2020-01-14 17:33:35 +00:00
|
|
|
func (f *Fs) dirFromStandardPath(dir string) string {
|
2019-07-27 15:58:33 +00:00
|
|
|
// Skip . and .. as we don't want these encoded
|
|
|
|
if dir == "." || dir == ".." {
|
|
|
|
return dir
|
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
return f.opt.Enc.FromStandardPath(dir)
|
2019-07-27 15:58:33 +00:00
|
|
|
}
|
|
|
|
|
2018-04-20 22:08:08 +00:00
|
|
|
// findItem finds a directory entry for the name in its parent directory
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
|
|
|
|
fullPath := path.Join(f.root, remote)
|
2019-09-28 10:39:24 +00:00
|
|
|
if fullPath == "" || fullPath == "." || fullPath == "/" {
|
|
|
|
// if root, assume exists and synthesize an entry
|
|
|
|
return &ftp.Entry{
|
|
|
|
Name: "",
|
|
|
|
Type: ftp.EntryTypeFolder,
|
|
|
|
Time: time.Now(),
|
|
|
|
}, nil
|
|
|
|
}
|
2017-05-03 20:45:01 +00:00
|
|
|
dir := path.Dir(fullPath)
|
|
|
|
base := path.Base(fullPath)
|
2016-12-27 20:52:30 +00:00
|
|
|
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("findItem: %w", err)
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
files, err := c.List(f.dirFromStandardPath(dir))
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2017-05-03 15:42:04 +00:00
|
|
|
if err != nil {
|
2017-05-03 20:45:01 +00:00
|
|
|
return nil, translateErrorFile(err)
|
2017-05-03 15:42:04 +00:00
|
|
|
}
|
2018-04-20 22:08:08 +00:00
|
|
|
for _, file := range files {
|
2020-01-14 17:33:35 +00:00
|
|
|
f.entryToStandard(file)
|
2018-04-20 22:08:08 +00:00
|
|
|
if file.Name == base {
|
|
|
|
return file, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, nil
|
|
|
|
}
|
2017-05-03 12:18:17 +00:00
|
|
|
|
2018-04-20 22:08:08 +00:00
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
|
|
// it returns the error fs.ErrorObjectNotFound.
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
2018-04-20 22:08:08 +00:00
|
|
|
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
|
2020-11-05 11:33:32 +00:00
|
|
|
entry, err := f.findItem(ctx, remote)
|
2018-04-20 22:08:08 +00:00
|
|
|
if err != nil {
|
2018-04-26 13:53:03 +00:00
|
|
|
return nil, err
|
2018-04-20 22:08:08 +00:00
|
|
|
}
|
|
|
|
if entry != nil && entry.Type != ftp.EntryTypeFolder {
|
|
|
|
o := &Object{
|
|
|
|
fs: f,
|
|
|
|
remote: remote,
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2021-07-22 23:04:08 +00:00
|
|
|
o.info = &FileInfo{
|
2018-04-20 22:08:08 +00:00
|
|
|
Name: remote,
|
|
|
|
Size: entry.Size,
|
|
|
|
ModTime: entry.Time,
|
2021-07-22 23:04:08 +00:00
|
|
|
precise: f.fLstTime,
|
2018-04-20 22:08:08 +00:00
|
|
|
}
|
|
|
|
return o, nil
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
return nil, fs.ErrorObjectNotFound
|
|
|
|
}
|
|
|
|
|
2018-04-20 22:08:08 +00:00
|
|
|
// dirExists checks the directory pointed to by remote exists or not
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
|
|
|
|
entry, err := f.findItem(ctx, remote)
|
2018-04-20 22:08:08 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return false, fmt.Errorf("dirExists: %w", err)
|
2018-04-20 22:08:08 +00:00
|
|
|
}
|
|
|
|
if entry != nil && entry.Type == ftp.EntryTypeFolder {
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
2017-06-11 21:43:31 +00:00
|
|
|
// List the objects and directories in dir into entries. The
|
|
|
|
// entries can be returned in any order but should be for a
|
|
|
|
// complete directory.
|
|
|
|
//
|
|
|
|
// dir should be "" to list the root, and should not have
|
|
|
|
// trailing slashes.
|
|
|
|
//
|
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
2019-09-28 10:39:24 +00:00
|
|
|
// defer log.Trace(dir, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("list: %w", err)
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2019-04-03 18:30:22 +00:00
|
|
|
|
|
|
|
var listErr error
|
|
|
|
var files []*ftp.Entry
|
|
|
|
|
|
|
|
resultchan := make(chan []*ftp.Entry, 1)
|
|
|
|
errchan := make(chan error, 1)
|
|
|
|
go func() {
|
2020-01-14 17:33:35 +00:00
|
|
|
result, err := c.List(f.dirFromStandardPath(path.Join(f.root, dir)))
|
2019-04-03 18:30:22 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
|
|
|
if err != nil {
|
|
|
|
errchan <- err
|
|
|
|
return
|
|
|
|
}
|
|
|
|
resultchan <- result
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Wait for List for up to Timeout seconds
|
2021-03-01 12:05:36 +00:00
|
|
|
timer := time.NewTimer(f.ci.TimeoutOrInfinite())
|
2019-04-03 18:30:22 +00:00
|
|
|
select {
|
|
|
|
case listErr = <-errchan:
|
|
|
|
timer.Stop()
|
|
|
|
return nil, translateErrorDir(listErr)
|
|
|
|
case files = <-resultchan:
|
|
|
|
timer.Stop()
|
|
|
|
case <-timer.C:
|
|
|
|
// if timer fired assume no error but connection dead
|
|
|
|
fs.Errorf(f, "Timeout when waiting for List")
|
2022-06-08 20:54:39 +00:00
|
|
|
return nil, errors.New("timeout when waiting for List")
|
2017-05-03 15:42:04 +00:00
|
|
|
}
|
2019-04-03 18:30:22 +00:00
|
|
|
|
2018-04-20 22:08:08 +00:00
|
|
|
// Annoyingly FTP returns success for a directory which
|
|
|
|
// doesn't exist, so check it really doesn't exist if no
|
|
|
|
// entries found.
|
|
|
|
if len(files) == 0 {
|
2020-11-05 11:33:32 +00:00
|
|
|
exists, err := f.dirExists(ctx, dir)
|
2018-04-20 22:08:08 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("list: %w", err)
|
2018-04-20 22:08:08 +00:00
|
|
|
}
|
|
|
|
if !exists {
|
|
|
|
return nil, fs.ErrorDirNotFound
|
|
|
|
}
|
|
|
|
}
|
2017-05-03 12:18:17 +00:00
|
|
|
for i := range files {
|
2016-12-27 20:52:30 +00:00
|
|
|
object := files[i]
|
2020-01-14 17:33:35 +00:00
|
|
|
f.entryToStandard(object)
|
2017-05-03 15:42:04 +00:00
|
|
|
newremote := path.Join(dir, object.Name)
|
2016-12-27 20:52:30 +00:00
|
|
|
switch object.Type {
|
|
|
|
case ftp.EntryTypeFolder:
|
2017-05-03 20:45:01 +00:00
|
|
|
if object.Name == "." || object.Name == ".." {
|
|
|
|
continue
|
|
|
|
}
|
2017-06-30 12:37:29 +00:00
|
|
|
d := fs.NewDir(newremote, object.Time)
|
2017-06-11 21:43:31 +00:00
|
|
|
entries = append(entries, d)
|
2016-12-27 20:52:30 +00:00
|
|
|
default:
|
|
|
|
o := &Object{
|
|
|
|
fs: f,
|
|
|
|
remote: newremote,
|
|
|
|
}
|
|
|
|
info := &FileInfo{
|
2017-05-03 12:18:17 +00:00
|
|
|
Name: newremote,
|
|
|
|
Size: object.Size,
|
2016-12-27 20:52:30 +00:00
|
|
|
ModTime: object.Time,
|
2021-07-22 23:04:08 +00:00
|
|
|
precise: f.fLstTime,
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
o.info = info
|
2017-06-11 21:43:31 +00:00
|
|
|
entries = append(entries, o)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
}
|
2017-06-11 21:43:31 +00:00
|
|
|
return entries, nil
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Hashes are not supported
|
2018-01-12 16:30:54 +00:00
|
|
|
func (f *Fs) Hashes() hash.Set {
|
2017-05-03 12:51:03 +00:00
|
|
|
return 0
|
|
|
|
}
|
2016-12-27 20:52:30 +00:00
|
|
|
|
2021-07-22 23:04:08 +00:00
|
|
|
// Precision shows whether modified time is supported or not depending on the
|
|
|
|
// FTP server capabilities, namely whether FTP server:
|
|
|
|
// - accepts the MDTM command to get file time (fGetTime)
|
|
|
|
// or supports MLSD returning precise file time in the list (fLstTime)
|
|
|
|
// - accepts the MFMT command to set file time (fSetTime)
|
|
|
|
// or non-standard form of the MDTM command (fSetTime, too)
|
|
|
|
// used by VsFtpd for the same purpose (WritingMDTM)
|
|
|
|
// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
|
2017-05-03 12:51:03 +00:00
|
|
|
func (f *Fs) Precision() time.Duration {
|
2021-07-22 23:04:08 +00:00
|
|
|
if (f.fGetTime || f.fLstTime) && f.fSetTime {
|
|
|
|
return time.Second
|
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
return fs.ModTimeNotSupported
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// fs.Debugf(f, "Trying to put file %s", src.Remote())
|
2020-11-05 11:33:32 +00:00
|
|
|
err := f.mkParentDir(ctx, src.Remote())
|
2017-05-18 16:23:02 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("Put mkParentDir failed: %w", err)
|
2017-05-18 16:23:02 +00:00
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
o := &Object{
|
|
|
|
fs: f,
|
|
|
|
remote: src.Remote(),
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2019-06-17 08:34:30 +00:00
|
|
|
err = o.Update(ctx, in, src, options...)
|
2017-05-03 12:51:03 +00:00
|
|
|
return o, err
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
|
|
return f.Put(ctx, in, src, options...)
|
2017-08-03 19:42:35 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// getInfo reads the FileInfo for a path
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
|
2017-05-03 15:42:04 +00:00
|
|
|
dir := path.Dir(remote)
|
|
|
|
base := path.Base(remote)
|
2017-05-03 12:51:03 +00:00
|
|
|
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("getInfo: %w", err)
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
files, err := c.List(f.dirFromStandardPath(dir))
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2017-05-03 15:42:04 +00:00
|
|
|
if err != nil {
|
2017-05-03 20:45:01 +00:00
|
|
|
return nil, translateErrorFile(err)
|
2017-05-03 15:42:04 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
for i := range files {
|
2019-07-27 15:58:33 +00:00
|
|
|
file := files[i]
|
2020-01-14 17:33:35 +00:00
|
|
|
f.entryToStandard(file)
|
2019-07-27 15:58:33 +00:00
|
|
|
if file.Name == base {
|
2017-05-03 12:51:03 +00:00
|
|
|
info := &FileInfo{
|
|
|
|
Name: remote,
|
2019-07-27 15:58:33 +00:00
|
|
|
Size: file.Size,
|
|
|
|
ModTime: file.Time,
|
2021-07-22 23:04:08 +00:00
|
|
|
precise: f.fLstTime,
|
2019-07-27 15:58:33 +00:00
|
|
|
IsDir: file.Type == ftp.EntryTypeFolder,
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
return info, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, fs.ErrorObjectNotFound
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-18 16:23:02 +00:00
|
|
|
// mkdir makes the directory and parents using unrooted paths
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) mkdir(ctx context.Context, abspath string) error {
|
2019-07-27 15:58:33 +00:00
|
|
|
abspath = path.Clean(abspath)
|
2017-05-18 16:23:02 +00:00
|
|
|
if abspath == "." || abspath == "/" {
|
|
|
|
return nil
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
fi, err := f.getInfo(ctx, abspath)
|
2017-05-18 16:23:02 +00:00
|
|
|
if err == nil {
|
|
|
|
if fi.IsDir {
|
|
|
|
return nil
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2017-05-18 16:23:02 +00:00
|
|
|
return fs.ErrorIsFile
|
|
|
|
} else if err != fs.ErrorObjectNotFound {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("mkdir %q failed: %w", abspath, err)
|
2017-05-18 16:23:02 +00:00
|
|
|
}
|
|
|
|
parent := path.Dir(abspath)
|
2020-11-05 11:33:32 +00:00
|
|
|
err = f.mkdir(ctx, parent)
|
2017-05-18 16:23:02 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
c, connErr := f.getFtpConnection(ctx)
|
2017-05-18 16:23:02 +00:00
|
|
|
if connErr != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("mkdir: %w", connErr)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
err = c.MakeDir(f.dirFromStandardPath(abspath))
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2018-04-26 13:53:03 +00:00
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
|
|
|
case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
|
|
|
|
err = nil
|
2018-06-28 08:01:17 +00:00
|
|
|
case 521: // dir already exists: error number according to RFC 959: issue #2363
|
|
|
|
err = nil
|
2018-04-26 13:53:03 +00:00
|
|
|
}
|
|
|
|
}
|
2016-12-27 20:52:30 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-05-18 16:23:02 +00:00
|
|
|
// mkParentDir makes the parent of remote if necessary and any
|
|
|
|
// directories above that
|
2020-11-05 11:33:32 +00:00
|
|
|
func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
|
2017-05-18 16:23:02 +00:00
|
|
|
parent := path.Dir(remote)
|
2020-11-05 11:33:32 +00:00
|
|
|
return f.mkdir(ctx, path.Join(f.root, parent))
|
2017-05-18 16:23:02 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 15:42:04 +00:00
|
|
|
// Mkdir creates the directory if it doesn't exist
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(dir, "")("err=%v", &err)
|
2017-05-18 16:23:02 +00:00
|
|
|
root := path.Join(f.root, dir)
|
2020-11-05 11:33:32 +00:00
|
|
|
return f.mkdir(ctx, root)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Rmdir removes the directory (container, bucket) if empty
|
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist or isn't empty
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("Rmdir: %w", translateErrorFile(err))
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
err = c.RemoveDir(f.dirFromStandardPath(path.Join(f.root, dir)))
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2017-05-18 14:51:35 +00:00
|
|
|
return translateErrorDir(err)
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
|
2017-05-18 16:32:45 +00:00
|
|
|
// Move renames a remote file object
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
2017-05-18 16:32:45 +00:00
|
|
|
srcObj, ok := src.(*Object)
|
|
|
|
if !ok {
|
|
|
|
fs.Debugf(src, "Can't move - not same remote type")
|
|
|
|
return nil, fs.ErrorCantMove
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
err := f.mkParentDir(ctx, remote)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("Move mkParentDir failed: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("Move: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
err = c.Rename(
|
2020-01-14 17:33:35 +00:00
|
|
|
f.opt.Enc.FromStandardPath(path.Join(srcObj.fs.root, srcObj.remote)),
|
|
|
|
f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
2017-05-18 16:32:45 +00:00
|
|
|
)
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("Move Rename failed: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
2019-06-17 08:34:30 +00:00
|
|
|
dstObj, err := f.NewObject(ctx, remote)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("Move NewObject failed: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
return dstObj, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
2020-10-13 21:43:40 +00:00
|
|
|
// using server-side move operations.
|
2017-05-18 16:32:45 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
2019-06-17 08:34:30 +00:00
|
|
|
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
2017-05-18 16:32:45 +00:00
|
|
|
srcFs, ok := src.(*Fs)
|
|
|
|
if !ok {
|
|
|
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
|
|
|
return fs.ErrorCantDirMove
|
|
|
|
}
|
|
|
|
srcPath := path.Join(srcFs.root, srcRemote)
|
|
|
|
dstPath := path.Join(f.root, dstRemote)
|
|
|
|
|
|
|
|
// Check if destination exists
|
2020-11-05 11:33:32 +00:00
|
|
|
fi, err := f.getInfo(ctx, dstPath)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err == nil {
|
|
|
|
if fi.IsDir {
|
|
|
|
return fs.ErrorDirExists
|
|
|
|
}
|
|
|
|
return fs.ErrorIsFile
|
|
|
|
} else if err != fs.ErrorObjectNotFound {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("DirMove getInfo failed: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure the parent directory exists
|
2020-11-05 11:33:32 +00:00
|
|
|
err = f.mkdir(ctx, path.Dir(dstPath))
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("DirMove mkParentDir dst failed: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Do the move
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := f.getFtpConnection(ctx)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("DirMove: %w", err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
err = c.Rename(
|
2020-01-14 17:33:35 +00:00
|
|
|
f.dirFromStandardPath(srcPath),
|
|
|
|
f.dirFromStandardPath(dstPath),
|
2017-05-18 16:32:45 +00:00
|
|
|
)
|
2017-05-24 13:47:13 +00:00
|
|
|
f.putFtpConnection(&c, err)
|
2017-05-18 16:32:45 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("DirMove Rename(%q,%q) failed: %w", srcPath, dstPath, err)
|
2017-05-18 16:32:45 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// ------------------------------------------------------------
|
|
|
|
|
|
|
|
// Fs returns the parent Fs
|
2016-12-27 20:52:30 +00:00
|
|
|
func (o *Object) Fs() fs.Info {
|
|
|
|
return o.fs
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// String version of o
|
|
|
|
func (o *Object) String() string {
|
|
|
|
if o == nil {
|
|
|
|
return "<nil>"
|
|
|
|
}
|
|
|
|
return o.remote
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Remote returns the remote path
|
|
|
|
func (o *Object) Remote() string {
|
|
|
|
return o.remote
|
|
|
|
}
|
|
|
|
|
|
|
|
// Hash returns the hash of an object returning a lowercase hex string
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
2018-01-18 20:27:52 +00:00
|
|
|
return "", hash.ErrUnsupported
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Size returns the size of an object in bytes
|
2016-12-27 20:52:30 +00:00
|
|
|
func (o *Object) Size() int64 {
|
|
|
|
return int64(o.info.Size)
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// ModTime returns the modification time of the object
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) ModTime(ctx context.Context) time.Time {
|
2021-07-22 23:04:08 +00:00
|
|
|
if !o.info.precise && o.fs.fGetTime {
|
|
|
|
c, err := o.fs.getFtpConnection(ctx)
|
|
|
|
if err == nil {
|
|
|
|
path := path.Join(o.fs.root, o.remote)
|
|
|
|
path = o.fs.opt.Enc.FromStandardPath(path)
|
|
|
|
modTime, err := c.GetTime(path)
|
|
|
|
if err == nil && o.info != nil {
|
|
|
|
o.info.ModTime = modTime
|
|
|
|
o.info.precise = true
|
|
|
|
}
|
|
|
|
o.fs.putFtpConnection(&c, err)
|
|
|
|
}
|
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
return o.info.ModTime
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetModTime sets the modification time of the object
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
2021-07-22 23:04:08 +00:00
|
|
|
if !o.fs.fSetTime {
|
|
|
|
fs.Errorf(o.fs, "SetModTime is not supported")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
c, err := o.fs.getFtpConnection(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
path := path.Join(o.fs.root, o.remote)
|
|
|
|
path = o.fs.opt.Enc.FromStandardPath(path)
|
|
|
|
err = c.SetTime(path, modTime.In(time.UTC))
|
|
|
|
if err == nil && o.info != nil {
|
|
|
|
o.info.ModTime = modTime
|
|
|
|
o.info.precise = true
|
|
|
|
}
|
|
|
|
o.fs.putFtpConnection(&c, err)
|
|
|
|
return err
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Storable returns a boolean as to whether this object is storable
|
2016-12-27 20:52:30 +00:00
|
|
|
func (o *Object) Storable() bool {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// ftpReadCloser implements io.ReadCloser for FTP objects.
|
|
|
|
type ftpReadCloser struct {
|
2017-05-19 16:45:22 +00:00
|
|
|
rc io.ReadCloser
|
|
|
|
c *ftp.ServerConn
|
|
|
|
f *Fs
|
|
|
|
err error // errors found during read
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read bytes into p
|
|
|
|
func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
|
|
|
|
n, err = f.rc.Read(p)
|
|
|
|
if err != nil && err != io.EOF {
|
|
|
|
f.err = err // store any errors for Close to examine
|
|
|
|
}
|
|
|
|
return
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
|
2017-05-18 14:51:35 +00:00
|
|
|
// Close the FTP reader and return the connection to the pool
|
2017-05-03 12:51:03 +00:00
|
|
|
func (f *ftpReadCloser) Close() error {
|
2019-01-15 17:32:14 +00:00
|
|
|
var err error
|
|
|
|
errchan := make(chan error, 1)
|
|
|
|
go func() {
|
|
|
|
errchan <- f.rc.Close()
|
|
|
|
}()
|
2020-02-25 16:51:18 +00:00
|
|
|
// Wait for Close for up to 60 seconds by default
|
2021-08-24 14:37:04 +00:00
|
|
|
closeTimeout := f.f.opt.CloseTimeout
|
|
|
|
if closeTimeout == 0 {
|
|
|
|
closeTimeout = fs.DurationOff
|
|
|
|
}
|
|
|
|
timer := time.NewTimer(time.Duration(closeTimeout))
|
2019-01-15 17:32:14 +00:00
|
|
|
select {
|
|
|
|
case err = <-errchan:
|
|
|
|
timer.Stop()
|
|
|
|
case <-timer.C:
|
|
|
|
// if timer fired assume no error but connection dead
|
|
|
|
fs.Errorf(f.f, "Timeout when waiting for connection Close")
|
2020-02-25 11:44:29 +00:00
|
|
|
f.f.putFtpConnection(nil, nil)
|
2019-01-15 17:32:14 +00:00
|
|
|
return nil
|
|
|
|
}
|
2017-05-19 16:45:22 +00:00
|
|
|
// if errors while reading or closing, dump the connection
|
|
|
|
if err != nil || f.err != nil {
|
|
|
|
_ = f.c.Quit()
|
2020-02-25 11:44:29 +00:00
|
|
|
f.f.putFtpConnection(nil, nil)
|
2017-05-19 16:45:22 +00:00
|
|
|
} else {
|
2017-05-24 13:47:13 +00:00
|
|
|
f.f.putFtpConnection(&f.c, nil)
|
2017-05-19 16:45:22 +00:00
|
|
|
}
|
|
|
|
// mask the error if it was caused by a premature close
|
2020-02-28 11:40:24 +00:00
|
|
|
// NB StatusAboutToSend is to work around a bug in pureftpd
|
|
|
|
// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
|
2017-05-19 11:21:24 +00:00
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
2020-02-28 11:40:24 +00:00
|
|
|
case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
|
2017-05-19 11:21:24 +00:00
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
return err
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Open an object for read
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(o, "")("rc=%v, err=%v", &rc, &err)
|
2017-05-03 15:42:04 +00:00
|
|
|
path := path.Join(o.fs.root, o.remote)
|
2018-01-27 10:07:17 +00:00
|
|
|
var offset, limit int64 = 0, -1
|
2017-05-03 20:45:01 +00:00
|
|
|
for _, option := range options {
|
|
|
|
switch x := option.(type) {
|
|
|
|
case *fs.SeekOption:
|
2018-01-27 10:07:17 +00:00
|
|
|
offset = x.Offset
|
2018-01-23 20:09:11 +00:00
|
|
|
case *fs.RangeOption:
|
|
|
|
offset, limit = x.Decode(o.Size())
|
2017-05-03 20:45:01 +00:00
|
|
|
default:
|
|
|
|
if option.Mandatory() {
|
|
|
|
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := o.fs.getFtpConnection(ctx)
|
2017-05-03 12:51:03 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("open: %w", err)
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
|
2017-05-03 12:51:03 +00:00
|
|
|
if err != nil {
|
2017-05-24 13:47:13 +00:00
|
|
|
o.fs.putFtpConnection(&c, err)
|
2021-11-04 10:12:57 +00:00
|
|
|
return nil, fmt.Errorf("open: %w", err)
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
2018-01-23 20:09:11 +00:00
|
|
|
rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
|
2017-05-03 20:45:01 +00:00
|
|
|
return rc, nil
|
2017-05-03 12:51:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Update the already existing object
|
|
|
|
//
|
|
|
|
// Copy the reader into the object updating modTime and size
|
|
|
|
//
|
|
|
|
// The new object may have been created if an error is returned
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
2017-05-18 14:51:35 +00:00
|
|
|
// defer fs.Trace(o, "src=%v", src)("err=%v", &err)
|
2017-05-03 15:42:04 +00:00
|
|
|
path := path.Join(o.fs.root, o.remote)
|
2017-05-03 20:45:01 +00:00
|
|
|
// remove the file if upload failed
|
|
|
|
remove := func() {
|
2018-09-17 07:46:06 +00:00
|
|
|
// Give the FTP server a chance to get its internal state in order after the error.
|
|
|
|
// The error may have been local in which case we closed the connection. The server
|
|
|
|
// may still be dealing with it for a moment. A sleep isn't ideal but I haven't been
|
|
|
|
// able to think of a better method to find out if the server has finished - ncw
|
|
|
|
time.Sleep(1 * time.Second)
|
2019-06-17 08:34:30 +00:00
|
|
|
removeErr := o.Remove(ctx)
|
2017-05-03 20:45:01 +00:00
|
|
|
if removeErr != nil {
|
|
|
|
fs.Debugf(o, "Failed to remove: %v", removeErr)
|
|
|
|
} else {
|
|
|
|
fs.Debugf(o, "Removed after failed upload: %v", err)
|
|
|
|
}
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := o.fs.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("Update: %w", err)
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
|
2021-06-16 13:34:53 +00:00
|
|
|
// Ignore error 250 here - send by some servers
|
|
|
|
if err != nil {
|
|
|
|
switch errX := err.(type) {
|
|
|
|
case *textproto.Error:
|
|
|
|
switch errX.Code {
|
|
|
|
case ftp.StatusRequestedFileActionOK:
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-12-27 20:52:30 +00:00
|
|
|
if err != nil {
|
2018-09-17 07:46:06 +00:00
|
|
|
_ = c.Quit() // toss this connection to avoid sync errors
|
2021-08-19 15:58:44 +00:00
|
|
|
// recycle connection in advance to let remove() find free token
|
2020-02-25 11:38:13 +00:00
|
|
|
o.fs.putFtpConnection(nil, err)
|
2021-08-19 15:58:44 +00:00
|
|
|
remove()
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("update stor: %w", err)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2017-05-24 13:47:13 +00:00
|
|
|
o.fs.putFtpConnection(&c, nil)
|
2021-07-22 23:04:08 +00:00
|
|
|
if err = o.SetModTime(ctx, src.ModTime(ctx)); err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("SetModTime: %w", err)
|
2021-07-22 23:04:08 +00:00
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
o.info, err = o.fs.getInfo(ctx, path)
|
2016-12-27 20:52:30 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("update getinfo: %w", err)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
return nil
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:51:03 +00:00
|
|
|
// Remove an object
|
2019-06-17 08:34:30 +00:00
|
|
|
func (o *Object) Remove(ctx context.Context) (err error) {
|
2017-05-03 20:45:01 +00:00
|
|
|
// defer fs.Trace(o, "")("err=%v", &err)
|
2017-05-03 15:42:04 +00:00
|
|
|
path := path.Join(o.fs.root, o.remote)
|
2017-05-03 12:51:03 +00:00
|
|
|
// Check if it's a directory or a file
|
2020-11-05 11:33:32 +00:00
|
|
|
info, err := o.fs.getInfo(ctx, path)
|
2017-05-03 15:42:04 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
if info.IsDir {
|
2019-06-17 08:34:30 +00:00
|
|
|
err = o.fs.Rmdir(ctx, o.remote)
|
2017-05-03 12:51:03 +00:00
|
|
|
} else {
|
2020-11-05 11:33:32 +00:00
|
|
|
c, err := o.fs.getFtpConnection(ctx)
|
2017-05-18 14:51:35 +00:00
|
|
|
if err != nil {
|
2021-11-04 10:12:57 +00:00
|
|
|
return fmt.Errorf("Remove: %w", err)
|
2017-05-18 14:51:35 +00:00
|
|
|
}
|
2020-01-14 17:33:35 +00:00
|
|
|
err = c.Delete(o.fs.opt.Enc.FromStandardPath(path))
|
2017-05-24 13:47:13 +00:00
|
|
|
o.fs.putFtpConnection(&c, err)
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
2017-05-03 12:51:03 +00:00
|
|
|
return err
|
2016-12-27 20:52:30 +00:00
|
|
|
}
|
|
|
|
|
2017-05-03 12:18:17 +00:00
|
|
|
// Check the interfaces are satisfied
|
2016-12-27 20:52:30 +00:00
|
|
|
var (
|
2017-08-03 19:42:35 +00:00
|
|
|
_ fs.Fs = &Fs{}
|
|
|
|
_ fs.Mover = &Fs{}
|
|
|
|
_ fs.DirMover = &Fs{}
|
|
|
|
_ fs.PutStreamer = &Fs{}
|
2021-02-16 11:36:32 +00:00
|
|
|
_ fs.Shutdowner = &Fs{}
|
2017-08-03 19:42:35 +00:00
|
|
|
_ fs.Object = &Object{}
|
2016-12-27 20:52:30 +00:00
|
|
|
)
|