diff --git a/README.md b/README.md index 9880d4f9e..d46293f9a 100644 --- a/README.md +++ b/README.md @@ -133,14 +133,16 @@ This help. General options: ``` - --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix K|M|G + --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G --checkers=8: Number of checkers to run in parallel. --config="~/.rclone.conf": Config file. + --contimeout=1m0s: Connect timeout -n, --dry-run=false: Do a trial run with no permanent changes --log-file="": Log everything to this file --modify-window=1ns: Max time diff to be considered the same -q, --quiet=false: Print as little stuff as possible --stats=1m0s: Interval to print stats (0 to disable) + --timeout=5m0s: IO idle timeout --transfers=4: Number of file transfers to run in parallel. -v, --verbose=false: Print lots more stuff -V, --version=false: Print the version number diff --git a/docs/content/docs.md b/docs/content/docs.md index 95da136bd..054d7ac4a 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -114,14 +114,16 @@ Enter an interactive configuration session. This help. ``` - --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix K|M|G + --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G --checkers=8: Number of checkers to run in parallel. --config="~/.rclone.conf": Config file. + --contimeout=1m0s: Connect timeout -n, --dry-run=false: Do a trial run with no permanent changes --log-file="": Log everything to this file --modify-window=1ns: Max time diff to be considered the same -q, --quiet=false: Print as little stuff as possible --stats=1m0s: Interval to print stats (0 to disable) + --timeout=5m0s: IO idle timeout --transfers=4: Number of file transfers to run in parallel. -v, --verbose=false: Print lots more stuff -V, --version=false: Print the version number diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 0d1fee204..5c5a46261 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -17,6 +17,20 @@ This is a JSON decode error - from Update / UploadByChunk - Caused by 500 error from dropbox - See https://github.com/stacktic/dropbox/issues/1 - Possibly confusing dropbox with excess concurrency? + +FIXME implement timeouts - need to get "github.com/stacktic/dropbox" +and hence "golang.org/x/oauth2" which uses DefaultTransport unless it +is set in the context passed into .Client() + +func (db *Dropbox) client() *http.Client { + return db.config.Client(oauth2.NoContext, db.token) +} + +// HTTPClient is the context key to use with golang.org/x/net/context's +// WithValue function to associate an *http.Client value with a context. +var HTTPClient ContextKey + +So pass in a context with HTTPClient set... */ import ( diff --git a/fs/config.go b/fs/config.go index 1b5987fd3..38a4cb6ad 100644 --- a/fs/config.go +++ b/fs/config.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "math" + "net/http" "os" "os/user" "path" @@ -16,6 +17,7 @@ import ( "time" "github.com/Unknwon/goconfig" + "github.com/mreiferson/go-httpclient" "github.com/ogier/pflag" ) @@ -36,14 +38,16 @@ var ( // Global config Config = &ConfigInfo{} // Flags - verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff") - quiet = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible") - modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same") - checkers = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.") - transfers = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.") - configFile = pflag.StringP("config", "", ConfigPath, "Config file.") - dryRun = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes") - bwLimit SizeSuffix + verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff") + quiet = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible") + modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same") + checkers = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.") + transfers = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.") + configFile = pflag.StringP("config", "", ConfigPath, "Config file.") + dryRun = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes") + connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout") + timeout = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout") + bwLimit SizeSuffix ) func init() { @@ -112,12 +116,48 @@ var _ pflag.Value = (*SizeSuffix)(nil) // Filesystem config options type ConfigInfo struct { - Verbose bool - Quiet bool - DryRun bool - ModifyWindow time.Duration - Checkers int - Transfers int + Verbose bool + Quiet bool + DryRun bool + ModifyWindow time.Duration + Checkers int + Transfers int + ConnectTimeout time.Duration // Connect timeout + Timeout time.Duration // Data channel timeout +} + +// Transport returns an http.RoundTripper with the correct timeouts +func (ci *ConfigInfo) Transport() http.RoundTripper { + return &httpclient.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: ci.Checkers + ci.Transfers + 1, + + // ConnectTimeout, if non-zero, is the maximum amount of time a dial will wait for + // a connect to complete. + ConnectTimeout: ci.ConnectTimeout, + + // ResponseHeaderTimeout, if non-zero, specifies the amount of + // time to wait for a server's response headers after fully + // writing the request (including its body, if any). This + // time does not include the time to read the response body. + ResponseHeaderTimeout: ci.Timeout, + + // RequestTimeout, if non-zero, specifies the amount of time for the entire + // request to complete (including all of the above timeouts + entire response body). + // This should never be less than the sum total of the above two timeouts. + //RequestTimeout: NOT SET, + + // ReadWriteTimeout, if non-zero, will set a deadline for every Read and + // Write operation on the request connection. + ReadWriteTimeout: ci.Timeout, + } +} + +// Transport returns an http.Client with the correct timeouts +func (ci *ConfigInfo) Client() *http.Client { + return &http.Client{ + Transport: ci.Transport(), + } } // Find the config directory @@ -152,6 +192,8 @@ func LoadConfig() { Config.Checkers = *checkers Config.Transfers = *transfers Config.DryRun = *dryRun + Config.Timeout = *timeout + Config.ConnectTimeout = *connectTimeout ConfigPath = *configFile diff --git a/googleauth/googleauth.go b/googleauth/googleauth.go index 113d53b1d..59a6853f2 100644 --- a/googleauth/googleauth.go +++ b/googleauth/googleauth.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "net/http" "code.google.com/p/goauth2/oauth" "github.com/ncw/rclone/fs" @@ -82,7 +81,7 @@ func (auth *Auth) newTransport(name string) (*oauth.Transport, error) { t := &oauth.Transport{ Config: config, - Transport: http.DefaultTransport, + Transport: fs.Config.Transport(), } return t, nil diff --git a/s3/s3.go b/s3/s3.go index 0ad57cd7b..d5fb1cc08 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -184,6 +184,7 @@ func s3Connection(name string) (*s3.S3, error) { } c := s3.New(auth, region) + c.Client = fs.Config.Client() return c, nil } diff --git a/swift/swift.go b/swift/swift.go index 4dfd95ce1..d299bcb23 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -113,12 +113,15 @@ func swiftConnection(name string) (*swift.Connection, error) { return nil, errors.New("auth not found") } c := &swift.Connection{ - UserName: userName, - ApiKey: apiKey, - AuthUrl: authUrl, - UserAgent: fs.UserAgent, - Tenant: fs.ConfigFile.MustValue(name, "tenant"), - Region: fs.ConfigFile.MustValue(name, "region"), + UserName: userName, + ApiKey: apiKey, + AuthUrl: authUrl, + UserAgent: fs.UserAgent, + Tenant: fs.ConfigFile.MustValue(name, "tenant"), + Region: fs.ConfigFile.MustValue(name, "region"), + ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport + Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport + Transport: fs.Config.Transport(), } err := c.Authenticate() if err != nil {