From f471a7e3f5c490ae589040faabdbc960b4de2eb5 Mon Sep 17 00:00:00 2001 From: qip Date: Sat, 12 Jan 2019 01:35:29 +0800 Subject: [PATCH] fshttp: Add cookie support with cmdline switch --use-cookies Cookies are handled by cookiejar in memory with fshttp module through the entire session. One useful scenario is, with HTTP storage system where index server adds authentication cookie while redirecting to CDN for actual files. Also, it can be helpful to reuse fshttp in other storage systems requiring cookie. --- fs/config.go | 1 + fs/config/configflags/configflags.go | 1 + fs/fshttp/http.go | 15 +++++++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/fs/config.go b/fs/config.go index aaa392ae4..3499a9fec 100644 --- a/fs/config.go +++ b/fs/config.go @@ -85,6 +85,7 @@ type ConfigInfo struct { MaxBacklog int StatsOneLine bool Progress bool + Cookie bool } // NewConfig creates a new config with everything set to the default diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 9e9ca06a3..f9201bcda 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -87,6 +87,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.") flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.") flags.BoolVarP(flagSet, &fs.Config.Progress, "progress", "P", fs.Config.Progress, "Show progress during transfer.") + flags.BoolVarP(flagSet, &fs.Config.Cookie, "use-cookies", "", fs.Config.Cookie, "Enable session cookiejar.") } // SetFlags converts any flags into config which weren't straight foward diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index e8240c9a5..a11dcc0fc 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -7,12 +7,14 @@ import ( "crypto/tls" "net" "net/http" + "net/http/cookiejar" "net/http/httputil" "reflect" "sync" "time" "github.com/ncw/rclone/fs" + "golang.org/x/net/publicsuffix" "golang.org/x/time/rate" ) @@ -22,9 +24,10 @@ const ( ) var ( - transport http.RoundTripper - noTransport sync.Once - tpsBucket *rate.Limiter // for limiting number of http transactions per second + transport http.RoundTripper + noTransport sync.Once + tpsBucket *rate.Limiter // for limiting number of http transactions per second + cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) ) // StartHTTPTokenBucket starts the token bucket if necessary @@ -142,9 +145,13 @@ func NewTransport(ci *fs.ConfigInfo) http.RoundTripper { // NewClient returns an http.Client with the correct timeouts func NewClient(ci *fs.ConfigInfo) *http.Client { - return &http.Client{ + transport := &http.Client{ Transport: NewTransport(ci), } + if ci.Cookie { + transport.Jar = cookieJar + } + return transport } // Transport is a our http Transport which wraps an http.Transport