From 318e335137eb198a04e4981c1f610e7f8c3fbe4b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 2 Nov 2016 15:53:43 +0000 Subject: [PATCH] Remove `Authorization:` headers from `--dump-headers` output Add in `--dump-auth` flag to put it back. --- docs/content/docs.md | 14 ++++++++++++-- fs/config.go | 3 +++ fs/http.go | 46 ++++++++++++++++++++++++++++++++++++++++---- fs/http_test.go | 24 +++++++++++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 2a6112b29..d9d29905b 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -543,6 +543,13 @@ here which are used for testing. These start with remote name eg Write CPU profile to file. This can be analysed with `go tool pprof`. +### --dump-auth ### + +Dump HTTP headers - will contain sensitive info such as +`Authorization:` headers - use `--dump-headers` to dump without +`Authorization:` headers. Can be very verbose. Useful for debugging +only. + ### --dump-bodies ### Dump HTTP headers and bodies - may contain sensitive info. Can be @@ -555,8 +562,11 @@ and exclude options are filtering on. ### --dump-headers ### -Dump HTTP headers - may contain sensitive info. Can be very verbose. -Useful for debugging only. +Dump HTTP headers with `Authorization:` lines removed. May still +contain sensitive info. Can be very verbose. Useful for debugging +only. + +Use `--dump-auth` if you do want the `Authorization:` headers. ### --memprofile=FILE ### diff --git a/fs/config.go b/fs/config.go index 5d9610813..89556f23e 100644 --- a/fs/config.go +++ b/fs/config.go @@ -76,6 +76,7 @@ var ( timeout = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout") dumpHeaders = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info") dumpBodies = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info") + dumpAuth = pflag.BoolP("dump-auth", "", false, "Dump HTTP headers with auth info") skipVerify = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.") AskPassword = pflag.BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.") deleteBefore = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering") @@ -287,6 +288,7 @@ type ConfigInfo struct { Timeout time.Duration // Data channel timeout DumpHeaders bool DumpBodies bool + DumpAuth bool Filter *Filter InsecureSkipVerify bool // Skip server certificate verification DeleteBefore bool // Delete before checking @@ -341,6 +343,7 @@ func LoadConfig() { Config.IgnoreExisting = *ignoreExisting Config.DumpHeaders = *dumpHeaders Config.DumpBodies = *dumpBodies + Config.DumpAuth = *dumpAuth Config.InsecureSkipVerify = *skipVerify Config.LowLevelRetries = *lowLevelRetries Config.UpdateOlder = *updateOlder diff --git a/fs/http.go b/fs/http.go index accd9f41f..c6734c9f2 100644 --- a/fs/http.go +++ b/fs/http.go @@ -3,6 +3,7 @@ package fs import ( + "bytes" "crypto/tls" "net" "net/http" @@ -112,7 +113,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper { // t.ExpectContinueTimeout ci.initTransport(t) // Wrap that http.Transport in our own transport - transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies) + transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies, ci.DumpAuth) }) return transport } @@ -131,15 +132,17 @@ type Transport struct { *http.Transport logHeader bool logBody bool + logAuth bool } // NewTransport wraps the http.Transport passed in and logs all // roundtrips including the body if logBody is set. -func NewTransport(transport *http.Transport, logHeader, logBody bool) *Transport { +func NewTransport(transport *http.Transport, logHeader, logBody, logAuth bool) *Transport { return &Transport{ Transport: transport, logHeader: logHeader, logBody: logBody, + logAuth: logAuth, } } @@ -180,13 +183,48 @@ func checkServerTime(req *http.Request, resp *http.Response) { checkedHostMu.Unlock() } +var authBuf = []byte("Authorization: ") + +// cleanAuth gets rid of one Authorization: header within the first 4k +func cleanAuth(buf []byte) []byte { + // Find how much buffer to check + n := 4096 + if len(buf) < n { + n = len(buf) + } + // See if there is an Authorization: header + i := bytes.Index(buf[:n], authBuf) + if i < 0 { + return buf + } + i += len(authBuf) + // Overwrite the next 4 chars with 'X' + for j := 0; i < len(buf) && j < 4; j++ { + if buf[i] == '\n' { + break + } + buf[i] = 'X' + i++ + } + // Snip out to the next '\n' + j := bytes.IndexByte(buf[i:], '\n') + if j < 0 { + return buf[:i] + } + n = copy(buf[i:], buf[i+j:]) + return buf[:i+n] +} + // RoundTrip implements the RoundTripper interface. func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { // Force user agent req.Header.Set("User-Agent", UserAgent) // Log request - if t.logHeader || t.logBody { + if t.logHeader || t.logBody || t.logAuth { buf, _ := httputil.DumpRequestOut(req, t.logBody) + if !t.logAuth { + buf = cleanAuth(buf) + } Debug(nil, "%s", separatorReq) Debug(nil, "%s (req %p)", "HTTP REQUEST", req) Debug(nil, "%s", string(buf)) @@ -195,7 +233,7 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error // Do round trip resp, err = t.Transport.RoundTrip(req) // Log response - if t.logHeader || t.logBody { + if t.logHeader || t.logBody || t.logAuth { Debug(nil, "%s", separatorResp) Debug(nil, "%s (req %p)", "HTTP RESPONSE", req) if err != nil { diff --git a/fs/http_test.go b/fs/http_test.go index b5b8e7837..a1e2c55f3 100644 --- a/fs/http_test.go +++ b/fs/http_test.go @@ -38,3 +38,27 @@ func TestSetDefaults(t *testing.T) { assert.Equal(t, old.TLSNextProto, new.TLSNextProto, "when checking .TLSNextProto") assert.Equal(t, old.MaxResponseHeaderBytes, new.MaxResponseHeaderBytes, "when checking .MaxResponseHeaderBytes") } + +func TestCleanAuth(t *testing.T) { + for _, test := range []struct { + in string + want string + }{ + {"", ""}, + {"floo", "floo"}, + {"Authorization: ", "Authorization: "}, + {"Authorization: \n", "Authorization: \n"}, + {"Authorization: A", "Authorization: X"}, + {"Authorization: A\n", "Authorization: X\n"}, + {"Authorization: AAAA", "Authorization: XXXX"}, + {"Authorization: AAAA\n", "Authorization: XXXX\n"}, + {"Authorization: AAAAA", "Authorization: XXXX"}, + {"Authorization: AAAAA\n", "Authorization: XXXX\n"}, + {"Authorization: AAAA\n", "Authorization: XXXX\n"}, + {"Authorization: AAAAAAAAA\nPotato: Help\n", "Authorization: XXXX\nPotato: Help\n"}, + {"Sausage: 1\nAuthorization: AAAAAAAAA\nPotato: Help\n", "Sausage: 1\nAuthorization: XXXX\nPotato: Help\n"}, + } { + got := string(cleanAuth([]byte(test.in))) + assert.Equal(t, test.want, got, test.in) + } +}