diff --git a/docs/content/docs.md b/docs/content/docs.md index 328ebd503..6e30fba7f 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -888,14 +888,20 @@ here which are used for testing. These start with remote name eg Write CPU profile to file. This can be analysed with `go tool pprof`. -### --dump-auth ### +#### --dump flag,flag,flag #### -Dump HTTP headers - will contain sensitive info such as -`Authorization:` headers - use `--dump-headers` to dump without -`Authorization:` headers. Can be very verbose. Useful for debugging +The `--dump` flag takes a comma separated list of flags to dump info +about. These are: + +#### --dump headers #### + +Dump HTTP headers with `Authorization:` lines removed. May still +contain sensitive info. Can be very verbose. Useful for debugging only. -### --dump-bodies ### +Use `--dump auth` if you do want the `Authorization:` headers. + +#### --dump bodies #### Dump HTTP headers and bodies - may contain sensitive info. Can be very verbose. Useful for debugging only. @@ -903,19 +909,28 @@ very verbose. Useful for debugging only. Note that the bodies are buffered in memory so don't use this for enormous files. -### --dump-filters ### +#### --dump requests #### + +Like `--dump bodies` but dumps the request bodies and the response +headers. Useful for debugging download problems. + +#### --dump responses #### + +Like `--dump bodies` but dumps the response bodies and the request +headers. Useful for debugging upload problems. + +#### --dump auth #### + +Dump HTTP headers - will contain sensitive info such as +`Authorization:` headers - use `--dump headers` to dump without +`Authorization:` headers. Can be very verbose. Useful for debugging +only. + +#### --dump filters #### Dump the filters to the output. Useful to see exactly what include and exclude options are filtering on. -### --dump-headers ### - -Dump HTTP headers with `Authorization:` lines removed. May still -contain sensitive info. Can be very verbose. Useful for debugging -only. - -Use `--dump-auth` if you do want the `Authorization:` headers. - ### --memprofile=FILE ### Write memory profile to file. This can be analysed with `go tool pprof`. @@ -969,7 +984,7 @@ For the filtering options * `--max-size` * `--min-age` * `--max-age` - * `--dump-filters` + * `--dump filters` See the [filtering section](/filtering/). diff --git a/docs/content/filtering.md b/docs/content/filtering.md index 19c2c23f0..5898134a2 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -400,7 +400,7 @@ these are now excluded from the sync. Always test first with `--dry-run` and `-v` before using this flag. -### `--dump-filters` - dump the filters to the output ### +### `--dump filters` - dump the filters to the output ### This dumps the defined filters to the output as regular expressions. diff --git a/fs/config.go b/fs/config.go index d5fbf62ac..93f110be9 100644 --- a/fs/config.go +++ b/fs/config.go @@ -90,7 +90,6 @@ var ( timeout = DurationP("timeout", "", 5*60*time.Second, "IO idle timeout") dumpHeaders = BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info") dumpBodies = BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info") - dumpAuth = BoolP("dump-auth", "", false, "Dump HTTP headers with auth info") skipVerify = BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.") AskPassword = BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.") deleteBefore = BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering") @@ -116,6 +115,7 @@ var ( immutable = BoolP("immutable", "", false, "Do not modify files. Fail if existing files have been modified.") autoConfirm = BoolP("auto-confirm", "", false, "If enabled, do not request console confirmation.") streamingUploadCutoff = SizeSuffix(100 * 1024) + dump DumpFlags logLevel = LogLevelNotice statsLogLevel = LogLevelInfo bwLimit BwTimetable @@ -132,6 +132,7 @@ func init() { VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.") VarP(&bufferSize, "buffer-size", "", "Buffer size when copying files.") VarP(&streamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.") + VarP(&dump, "dump", "", "List of items to dump from: "+dumpFlagsList) } // crypt internals @@ -229,9 +230,7 @@ type ConfigInfo struct { Transfers int ConnectTimeout time.Duration // Connect timeout Timeout time.Duration // Data channel timeout - DumpHeaders bool - DumpBodies bool - DumpAuth bool + Dump DumpFlags Filter *Filter InsecureSkipVerify bool // Skip server certificate verification DeleteMode DeleteMode @@ -377,9 +376,6 @@ func LoadConfig() { Config.SizeOnly = *sizeOnly Config.IgnoreTimes = *ignoreTimes Config.IgnoreExisting = *ignoreExisting - Config.DumpHeaders = *dumpHeaders - Config.DumpBodies = *dumpBodies - Config.DumpAuth = *dumpAuth Config.InsecureSkipVerify = *skipVerify Config.LowLevelRetries = *lowLevelRetries Config.UpdateOlder = *updateOlder @@ -398,6 +394,15 @@ func LoadConfig() { Config.AutoConfirm = *autoConfirm Config.BufferSize = bufferSize Config.StreamingUploadCutoff = streamingUploadCutoff + Config.Dump = dump + if *dumpHeaders { + Config.Dump |= DumpHeaders + Infof(nil, "--dump-headers is obsolete - please use --dump headers instead") + } + if *dumpBodies { + Config.Dump |= DumpBodies + Infof(nil, "--dump-bodies is obsolete - please use --dump bodies instead") + } Config.TrackRenames = *trackRenames @@ -1460,3 +1465,87 @@ func makeCacheDir() (dir string) { } return filepath.Join(dir, "rclone") } + +// DumpFlags describes the Dump options in force +type DumpFlags int + +// DumpFlags definitions +const ( + DumpHeaders DumpFlags = 1 << iota + DumpBodies + DumpRequests + DumpResponses + DumpAuth + DumpFilters +) + +var dumpFlags = []struct { + flag DumpFlags + name string +}{ + {DumpHeaders, "headers"}, + {DumpBodies, "bodies"}, + {DumpRequests, "requests"}, + {DumpResponses, "responses"}, + {DumpAuth, "auth"}, + {DumpFilters, "filters"}, +} + +// list of dump flags used in the help +var dumpFlagsList string + +func init() { + // calculate the dump flags list + var out []string + for _, info := range dumpFlags { + out = append(out, info.name) + } + dumpFlagsList = strings.Join(out, ",") +} + +// String turns a DumpFlags into a string +func (f DumpFlags) String() string { + var out []string + for _, info := range dumpFlags { + if f&info.flag != 0 { + out = append(out, info.name) + f &^= info.flag + } + } + if f != 0 { + out = append(out, fmt.Sprintf("Unknown-0x%X", int(f))) + } + return strings.Join(out, ",") +} + +// Set a DumpFlags as a comma separated list of flags +func (f *DumpFlags) Set(s string) error { + var flags DumpFlags + parts := strings.Split(s, ",") + for _, part := range parts { + found := false + part = strings.ToLower(strings.TrimSpace(part)) + if part == "" { + continue + } + for _, info := range dumpFlags { + if part == info.name { + found = true + flags |= info.flag + } + } + if !found { + return errors.Errorf("Unknown dump flag %q", part) + } + } + *f = flags + return nil +} + +// Type of the value +func (f *DumpFlags) Type() string { + return "string" +} + +// Check it satisfies the interface +var _ pflag.Value = (*DumpFlags)(nil) diff --git a/fs/config_test.go b/fs/config_test.go index 44a20f548..544d51aee 100644 --- a/fs/config_test.go +++ b/fs/config_test.go @@ -226,3 +226,50 @@ func hashedKeyCompare(t *testing.T, a, b string, shouldMatch bool) { assert.NotEqual(t, k1, k2) } } + +func TestDumpFlagsString(t *testing.T) { + assert.Equal(t, "", DumpFlags(0).String()) + assert.Equal(t, "headers", (DumpHeaders).String()) + assert.Equal(t, "headers,bodies", (DumpHeaders | DumpBodies).String()) + assert.Equal(t, "headers,bodies,requests,responses,auth,filters", (DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters).String()) + assert.Equal(t, "headers,Unknown-0x8000", (DumpHeaders | DumpFlags(0x8000)).String()) +} + +func TestDumpFlagsSet(t *testing.T) { + for _, test := range []struct { + in string + want DumpFlags + wantErr string + }{ + {"", DumpFlags(0), ""}, + {"bodies", DumpBodies, ""}, + {"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""}, + {"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""}, + {"headers,bodies,requests,responses,auth,filters", DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters, ""}, + {"headers,bodies,unknown,auth", 0, "Unknown dump flag \"unknown\""}, + } { + f := DumpFlags(-1) + initial := f + err := f.Set(test.in) + if err != nil { + if test.wantErr == "" { + t.Errorf("Got an error when not expecting one on %q: %v", test.in, err) + } else { + assert.Contains(t, err.Error(), test.wantErr) + } + assert.Equal(t, initial, f, test.want) + } else { + if test.wantErr != "" { + t.Errorf("Got no error when expecting one on %q", test.in) + } else { + assert.Equal(t, test.want, f) + } + } + + } +} + +func TestDumpFlagsType(t *testing.T) { + f := DumpFlags(0) + assert.Equal(t, "string", f.Type()) +} diff --git a/fs/filter.go b/fs/filter.go index 4e74adedf..966d014bf 100644 --- a/fs/filter.go +++ b/fs/filter.go @@ -31,7 +31,6 @@ var ( maxAge = StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y") minSize = SizeSuffix(-1) maxSize = SizeSuffix(-1) - dumpFilters = BoolP("dump-filters", "", false, "Dump the filters to the output") //cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does") ) @@ -249,7 +248,7 @@ func NewFilter() (f *Filter, err error) { } Debugf(nil, "--max-age %v to %v", duration, f.ModTimeFrom) } - if *dumpFilters { + if Config.Dump&DumpFilters != 0 { fmt.Println("--- start filters ---") fmt.Println(f.DumpFilters()) fmt.Println("--- end filters ---") diff --git a/fs/http.go b/fs/http.go index 3b76a2f5d..e5de19244 100644 --- a/fs/http.go +++ b/fs/http.go @@ -129,7 +129,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper { // t.ExpectContinueTimeout ci.initTransport(t) // Wrap that http.Transport in our own transport - transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies, ci.DumpAuth) + transport = NewTransport(t, ci.Dump) }) return transport } @@ -146,19 +146,15 @@ func (ci *ConfigInfo) Client() *http.Client { // * Does logging type Transport struct { *http.Transport - logHeader bool - logBody bool - logAuth bool + dump DumpFlags } // NewTransport wraps the http.Transport passed in and logs all // roundtrips including the body if logBody is set. -func NewTransport(transport *http.Transport, logHeader, logBody, logAuth bool) *Transport { +func NewTransport(transport *http.Transport, dump DumpFlags) *Transport { return &Transport{ Transport: transport, - logHeader: logHeader, - logBody: logBody, - logAuth: logAuth, + dump: dump, } } @@ -243,9 +239,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error // Force user agent req.Header.Set("User-Agent", *userAgent) // Logf request - if t.logHeader || t.logBody || t.logAuth { - buf, _ := httputil.DumpRequestOut(req, t.logBody) - if !t.logAuth { + if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 { + buf, _ := httputil.DumpRequestOut(req, t.dump&(DumpBodies|DumpRequests) != 0) + if t.dump&DumpAuth == 0 { buf = cleanAuth(buf) } Debugf(nil, "%s", separatorReq) @@ -256,13 +252,13 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error // Do round trip resp, err = t.Transport.RoundTrip(req) // Logf response - if t.logHeader || t.logBody || t.logAuth { + if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 { Debugf(nil, "%s", separatorResp) Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req) if err != nil { Debugf(nil, "Error: %v", err) } else { - buf, _ := httputil.DumpResponse(resp, t.logBody) + buf, _ := httputil.DumpResponse(resp, t.dump&(DumpBodies|DumpResponses) != 0) Debugf(nil, "%s", string(buf)) } Debugf(nil, "%s", separatorResp) diff --git a/fstest/fstest.go b/fstest/fstest.go index de39a3227..99fda2117 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -59,8 +59,12 @@ func Initialise() { if *Verbose { fs.Config.LogLevel = fs.LogLevelDebug } - fs.Config.DumpHeaders = *DumpHeaders - fs.Config.DumpBodies = *DumpBodies + if *DumpHeaders { + fs.Config.Dump |= fs.DumpHeaders + } + if *DumpBodies { + fs.Config.Dump |= fs.DumpBodies + } fs.Config.LowLevelRetries = *LowLevelRetries fs.Config.UseListR = *UseListR }