potyarkin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

fs: deglobalise the config #4685

Browse source 
This is done by making fs.Config private and attaching it to the
context instead.

The Config should be obtained with fs.GetConfig and fs.AddConfig
should be used to get a new mutable config that can be changed.
This commit is contained in:
Nick Craig-Wood 2020-11-05 11:33:32 +00:00
parent 506342317b
commit 2e21c58e6a
93 changed files with 1128 additions and 847 deletions
Download patch file Download diff file Expand all files Collapse all files

96
backend/sftp/sftp.go
View file

@ -226,6 +226,7 @@ type Fs struct {
root string
absRoot string
opt Options // parsed options
ci *fs.ConfigInfo // global config
m configmap.Mapper // config
features *fs.Features // optional features
config *ssh.ClientConfig
@ -252,8 +253,8 @@ type Object struct {
// dial starts a client connection to the given SSH server. It is a
// convenience function that connects to the given network address,
// initiates the SSH handshake, and then sets up a Client.
func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
dialer := fshttp.NewDialer(fs.Config)
func (f *Fs) dial(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
dialer := fshttp.NewDialer(fs.GetConfig(ctx))
conn, err := dialer.Dial(network, addr)
if err != nil {
return nil, err
@ -299,12 +300,12 @@ func (c *conn) closed() error {
}
// Open a new connection to the SFTP server.
func (f *Fs) sftpConnection() (c *conn, err error) {
func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
// Rate limit rate of new connections
c = &conn{
err: make(chan error, 1),
}
c.sshClient, err = f.dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
c.sshClient, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port, f.config)
if err != nil {
return nil, errors.Wrap(err, "couldn't connect SSH")
}
@ -347,7 +348,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
}
// Get an SFTP connection from the pool, or open a new one
func (f *Fs) getSftpConnection() (c *conn, err error) {
func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) {
f.poolMu.Lock()
for len(f.pool) > 0 {
c = f.pool[0]
@ -364,7 +365,7 @@ func (f *Fs) getSftpConnection() (c *conn, err error) {
return c, nil
}
err = f.pacer.Call(func() (bool, error) {
c, err = f.sftpConnection()
c, err = f.sftpConnection(ctx)
if err != nil {
return true, err
}
@ -417,7 +418,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// This will hold the Fs object. We need to create it here
// so we can refer to it in the SSH callback, but it's populated
// in NewFsWithConnection
f := &Fs{}
f := &Fs{
ci: fs.GetConfig(ctx),
}
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@ -435,8 +438,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
User: opt.User,
Auth: []ssh.AuthMethod{},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: fs.Config.ConnectTimeout,
ClientVersion: "SSH-2.0-" + fs.Config.UserAgent,
Timeout: f.ci.ConnectTimeout,
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
}
if opt.KnownHostsFile != "" {
@ -603,7 +606,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
f.config = sshConfig
f.url = "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root
f.mkdirLock = newStringLock()
f.pacer = fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.savedpswd = ""
f.features = (&fs.Features{
@ -611,7 +614,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
SlowHash: true,
}).Fill(ctx, f)
// Make a connection and pool it to return errors early
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "NewFs")
}
@ -679,7 +682,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
fs: f,
remote: remote,
}
err := o.stat()
err := o.stat(ctx)
if err != nil {
return nil, err
}
@ -688,11 +691,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
// dirExists returns true,nil if the directory exists, false, nil if
// it doesn't or false, err
func (f *Fs) dirExists(dir string) (bool, error) {
func (f *Fs) dirExists(ctx context.Context, dir string) (bool, error) {
if dir == "" {
dir = "."
}
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return false, errors.Wrap(err, "dirExists")
}
@ -721,7 +724,7 @@ func (f *Fs) dirExists(dir string) (bool, error) {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
root := path.Join(f.absRoot, dir)
ok, err := f.dirExists(root)
ok, err := f.dirExists(ctx, root)
if err != nil {
return nil, errors.Wrap(err, "List failed")
}
@ -732,7 +735,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if sftpDir == "" {
sftpDir = "."
}
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "List")
}
@ -751,7 +754,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
continue
}
oldInfo := info
info, err = f.stat(remote)
info, err = f.stat(ctx, remote)
if err != nil {
if !os.IsNotExist(err) {
fs.Errorf(remote, "stat of non-regular file failed: %v", err)
@ -776,7 +779,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime(ctx)>
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
err := f.mkParentDir(src.Remote())
err := f.mkParentDir(ctx, src.Remote())
if err != nil {
return nil, errors.Wrap(err, "Put mkParentDir failed")
}
@ -799,19 +802,19 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// mkParentDir makes the parent of remote if necessary and any
// directories above that
func (f *Fs) mkParentDir(remote string) error {
func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
parent := path.Dir(remote)
return f.mkdir(path.Join(f.absRoot, parent))
return f.mkdir(ctx, path.Join(f.absRoot, parent))
}
// mkdir makes the directory and parents using native paths
func (f *Fs) mkdir(dirPath string) error {
func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
f.mkdirLock.Lock(dirPath)
defer f.mkdirLock.Unlock(dirPath)
if dirPath == "." || dirPath == "/" {
return nil
}
ok, err := f.dirExists(dirPath)
ok, err := f.dirExists(ctx, dirPath)
if err != nil {
return errors.Wrap(err, "mkdir dirExists failed")
}
@ -819,11 +822,11 @@ func (f *Fs) mkdir(dirPath string) error {
return nil
}
parent := path.Dir(dirPath)
err = f.mkdir(parent)
err = f.mkdir(ctx, parent)
if err != nil {
return err
}
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "mkdir")
}
@ -838,7 +841,7 @@ func (f *Fs) mkdir(dirPath string) error {
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
root := path.Join(f.absRoot, dir)
return f.mkdir(root)
return f.mkdir(ctx, root)
}
// Rmdir removes the root directory of the Fs object
@ -854,7 +857,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
}
// Remove the directory
root := path.Join(f.absRoot, dir)
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Rmdir")
}
@ -870,11 +873,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
err := f.mkParentDir(remote)
err := f.mkParentDir(ctx, remote)
if err != nil {
return nil, errors.Wrap(err, "Move mkParentDir failed")
}
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "Move")
}
@ -911,7 +914,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := path.Join(f.absRoot, dstRemote)
// Check if destination exists
ok, err := f.dirExists(dstPath)
ok, err := f.dirExists(ctx, dstPath)
if err != nil {
return errors.Wrap(err, "DirMove dirExists dst failed")
}
@ -920,13 +923,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// Make sure the parent directory exists
err = f.mkdir(path.Dir(dstPath))
err = f.mkdir(ctx, path.Dir(dstPath))
if err != nil {
return errors.Wrap(err, "DirMove mkParentDir dst failed")
}
// Do the move
c, err := f.getSftpConnection()
c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "DirMove")
}
@ -942,8 +945,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// run runds cmd on the remote end returning standard output
func (f *Fs) run(cmd string) ([]byte, error) {
c, err := f.getSftpConnection()
func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "run: get SFTP connection")
}
@ -971,6 +974,7 @@ func (f *Fs) run(cmd string) ([]byte, error) {
// Hashes returns the supported hash types of the filesystem
func (f *Fs) Hashes() hash.Set {
ctx := context.TODO()
if f.opt.DisableHashCheck {
return hash.Set(hash.None)
}
@ -989,7 +993,7 @@ func (f *Fs) Hashes() hash.Set {
}
*changed = true
for _, command := range commands {
output, err := f.run(command)
output, err := f.run(ctx, command)
if err != nil {
continue
}
@ -1034,7 +1038,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
if len(escapedPath) == 0 {
escapedPath = "/"
}
stdout, err := f.run("df -k " + escapedPath)
stdout, err := f.run(ctx, "df -k "+escapedPath)
if err != nil {
return nil, errors.Wrap(err, "your remote may not support About")
}
@ -1097,7 +1101,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
c, err := o.fs.getSftpConnection()
c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return "", errors.Wrap(err, "Hash get SFTP connection")
}
@ -1205,8 +1209,8 @@ func (o *Object) setMetadata(info os.FileInfo) {
}
// statRemote stats the file or directory at the remote given
func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
c, err := f.getSftpConnection()
func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "stat")
}
@ -1217,8 +1221,8 @@ func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
}
// stat updates the info in the Object
func (o *Object) stat() error {
info, err := o.fs.stat(o.remote)
func (o *Object) stat(ctx context.Context) error {
info, err := o.fs.stat(ctx, o.remote)
if err != nil {
if os.IsNotExist(err) {
return fs.ErrorObjectNotFound
@ -1237,7 +1241,7 @@ func (o *Object) stat() error {
// it also updates the info field
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
if o.fs.opt.SetModTime {
c, err := o.fs.getSftpConnection()
c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "SetModTime")
}
@ -1247,7 +1251,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
return errors.Wrap(err, "SetModTime failed")
}
}
err := o.stat()
err := o.stat(ctx)
if err != nil {
return errors.Wrap(err, "SetModTime stat failed")
}
@ -1320,7 +1324,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
}
}
}
c, err := o.fs.getSftpConnection()
c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "Open")
}
@ -1344,7 +1348,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Clear the hash cache since we are about to update the object
o.md5sum = nil
o.sha1sum = nil
c, err := o.fs.getSftpConnection()
c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Update")
}
@ -1355,7 +1359,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
// remove the file if upload failed
remove := func() {
c, removeErr := o.fs.getSftpConnection()
c, removeErr := o.fs.getSftpConnection(ctx)
if removeErr != nil {
fs.Debugf(src, "Failed to open new SSH connection for delete: %v", removeErr)
return
@ -1387,7 +1391,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove a remote sftp file object
func (o *Object) Remove(ctx context.Context) error {
c, err := o.fs.getSftpConnection()
c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Remove")
}