fs: Add --dump flag, introduce --dump requests, responses and remove --dump-auth, --dump-filters

Now --dump-flag is written as --dump flag. This is a comma separated list which can contain

  * headers - HTTP headers as before
  * bodies  - HTTP bodies as before
  * requests - HTTP request bodies
  * responses - HTTP response bodies
  * auth - HTTP auth
  * filters - Filter rexeps

Leave --dump-headers and --dump-bodies for the time being but remove
the other --dump-* flags as they aren't used very often.
This commit is contained in:
Nick Craig-Wood 2017-11-20 20:21:44 +00:00
parent bcf1ece43b
commit 1a65a4e769
7 changed files with 190 additions and 40 deletions

View file

@ -888,14 +888,20 @@ here which are used for testing. These start with remote name eg
Write CPU profile to file. This can be analysed with `go tool pprof`.
### --dump-auth ###
#### --dump flag,flag,flag ####
Dump HTTP headers - will contain sensitive info such as
`Authorization:` headers - use `--dump-headers` to dump without
`Authorization:` headers. Can be very verbose. Useful for debugging
The `--dump` flag takes a comma separated list of flags to dump info
about. These are:
#### --dump headers ####
Dump HTTP headers with `Authorization:` lines removed. May still
contain sensitive info. Can be very verbose. Useful for debugging
only.
### --dump-bodies ###
Use `--dump auth` if you do want the `Authorization:` headers.
#### --dump bodies ####
Dump HTTP headers and bodies - may contain sensitive info. Can be
very verbose. Useful for debugging only.
@ -903,19 +909,28 @@ very verbose. Useful for debugging only.
Note that the bodies are buffered in memory so don't use this for
enormous files.
### --dump-filters ###
#### --dump requests ####
Like `--dump bodies` but dumps the request bodies and the response
headers. Useful for debugging download problems.
#### --dump responses ####
Like `--dump bodies` but dumps the response bodies and the request
headers. Useful for debugging upload problems.
#### --dump auth ####
Dump HTTP headers - will contain sensitive info such as
`Authorization:` headers - use `--dump headers` to dump without
`Authorization:` headers. Can be very verbose. Useful for debugging
only.
#### --dump filters ####
Dump the filters to the output. Useful to see exactly what include
and exclude options are filtering on.
### --dump-headers ###
Dump HTTP headers with `Authorization:` lines removed. May still
contain sensitive info. Can be very verbose. Useful for debugging
only.
Use `--dump-auth` if you do want the `Authorization:` headers.
### --memprofile=FILE ###
Write memory profile to file. This can be analysed with `go tool pprof`.
@ -969,7 +984,7 @@ For the filtering options
* `--max-size`
* `--min-age`
* `--max-age`
* `--dump-filters`
* `--dump filters`
See the [filtering section](/filtering/).

View file

@ -400,7 +400,7 @@ these are now excluded from the sync.
Always test first with `--dry-run` and `-v` before using this flag.
### `--dump-filters` - dump the filters to the output ###
### `--dump filters` - dump the filters to the output ###
This dumps the defined filters to the output as regular expressions.

View file

@ -90,7 +90,6 @@ var (
timeout = DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
dumpHeaders = BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
dumpAuth = BoolP("dump-auth", "", false, "Dump HTTP headers with auth info")
skipVerify = BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
AskPassword = BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
deleteBefore = BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
@ -116,6 +115,7 @@ var (
immutable = BoolP("immutable", "", false, "Do not modify files. Fail if existing files have been modified.")
autoConfirm = BoolP("auto-confirm", "", false, "If enabled, do not request console confirmation.")
streamingUploadCutoff = SizeSuffix(100 * 1024)
dump DumpFlags
logLevel = LogLevelNotice
statsLogLevel = LogLevelInfo
bwLimit BwTimetable
@ -132,6 +132,7 @@ func init() {
VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
VarP(&bufferSize, "buffer-size", "", "Buffer size when copying files.")
VarP(&streamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.")
VarP(&dump, "dump", "", "List of items to dump from: "+dumpFlagsList)
}
// crypt internals
@ -229,9 +230,7 @@ type ConfigInfo struct {
Transfers int
ConnectTimeout time.Duration // Connect timeout
Timeout time.Duration // Data channel timeout
DumpHeaders bool
DumpBodies bool
DumpAuth bool
Dump DumpFlags
Filter *Filter
InsecureSkipVerify bool // Skip server certificate verification
DeleteMode DeleteMode
@ -377,9 +376,6 @@ func LoadConfig() {
Config.SizeOnly = *sizeOnly
Config.IgnoreTimes = *ignoreTimes
Config.IgnoreExisting = *ignoreExisting
Config.DumpHeaders = *dumpHeaders
Config.DumpBodies = *dumpBodies
Config.DumpAuth = *dumpAuth
Config.InsecureSkipVerify = *skipVerify
Config.LowLevelRetries = *lowLevelRetries
Config.UpdateOlder = *updateOlder
@ -398,6 +394,15 @@ func LoadConfig() {
Config.AutoConfirm = *autoConfirm
Config.BufferSize = bufferSize
Config.StreamingUploadCutoff = streamingUploadCutoff
Config.Dump = dump
if *dumpHeaders {
Config.Dump |= DumpHeaders
Infof(nil, "--dump-headers is obsolete - please use --dump headers instead")
}
if *dumpBodies {
Config.Dump |= DumpBodies
Infof(nil, "--dump-bodies is obsolete - please use --dump bodies instead")
}
Config.TrackRenames = *trackRenames
@ -1460,3 +1465,87 @@ func makeCacheDir() (dir string) {
}
return filepath.Join(dir, "rclone")
}
// DumpFlags describes the Dump options in force
type DumpFlags int
// DumpFlags definitions
const (
DumpHeaders DumpFlags = 1 << iota
DumpBodies
DumpRequests
DumpResponses
DumpAuth
DumpFilters
)
var dumpFlags = []struct {
flag DumpFlags
name string
}{
{DumpHeaders, "headers"},
{DumpBodies, "bodies"},
{DumpRequests, "requests"},
{DumpResponses, "responses"},
{DumpAuth, "auth"},
{DumpFilters, "filters"},
}
// list of dump flags used in the help
var dumpFlagsList string
func init() {
// calculate the dump flags list
var out []string
for _, info := range dumpFlags {
out = append(out, info.name)
}
dumpFlagsList = strings.Join(out, ",")
}
// String turns a DumpFlags into a string
func (f DumpFlags) String() string {
var out []string
for _, info := range dumpFlags {
if f&info.flag != 0 {
out = append(out, info.name)
f &^= info.flag
}
}
if f != 0 {
out = append(out, fmt.Sprintf("Unknown-0x%X", int(f)))
}
return strings.Join(out, ",")
}
// Set a DumpFlags as a comma separated list of flags
func (f *DumpFlags) Set(s string) error {
var flags DumpFlags
parts := strings.Split(s, ",")
for _, part := range parts {
found := false
part = strings.ToLower(strings.TrimSpace(part))
if part == "" {
continue
}
for _, info := range dumpFlags {
if part == info.name {
found = true
flags |= info.flag
}
}
if !found {
return errors.Errorf("Unknown dump flag %q", part)
}
}
*f = flags
return nil
}
// Type of the value
func (f *DumpFlags) Type() string {
return "string"
}
// Check it satisfies the interface
var _ pflag.Value = (*DumpFlags)(nil)

View file

@ -226,3 +226,50 @@ func hashedKeyCompare(t *testing.T, a, b string, shouldMatch bool) {
assert.NotEqual(t, k1, k2)
}
}
func TestDumpFlagsString(t *testing.T) {
assert.Equal(t, "", DumpFlags(0).String())
assert.Equal(t, "headers", (DumpHeaders).String())
assert.Equal(t, "headers,bodies", (DumpHeaders | DumpBodies).String())
assert.Equal(t, "headers,bodies,requests,responses,auth,filters", (DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters).String())
assert.Equal(t, "headers,Unknown-0x8000", (DumpHeaders | DumpFlags(0x8000)).String())
}
func TestDumpFlagsSet(t *testing.T) {
for _, test := range []struct {
in string
want DumpFlags
wantErr string
}{
{"", DumpFlags(0), ""},
{"bodies", DumpBodies, ""},
{"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""},
{"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""},
{"headers,bodies,requests,responses,auth,filters", DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters, ""},
{"headers,bodies,unknown,auth", 0, "Unknown dump flag \"unknown\""},
} {
f := DumpFlags(-1)
initial := f
err := f.Set(test.in)
if err != nil {
if test.wantErr == "" {
t.Errorf("Got an error when not expecting one on %q: %v", test.in, err)
} else {
assert.Contains(t, err.Error(), test.wantErr)
}
assert.Equal(t, initial, f, test.want)
} else {
if test.wantErr != "" {
t.Errorf("Got no error when expecting one on %q", test.in)
} else {
assert.Equal(t, test.want, f)
}
}
}
}
func TestDumpFlagsType(t *testing.T) {
f := DumpFlags(0)
assert.Equal(t, "string", f.Type())
}

View file

@ -31,7 +31,6 @@ var (
maxAge = StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
minSize = SizeSuffix(-1)
maxSize = SizeSuffix(-1)
dumpFilters = BoolP("dump-filters", "", false, "Dump the filters to the output")
//cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
)
@ -249,7 +248,7 @@ func NewFilter() (f *Filter, err error) {
}
Debugf(nil, "--max-age %v to %v", duration, f.ModTimeFrom)
}
if *dumpFilters {
if Config.Dump&DumpFilters != 0 {
fmt.Println("--- start filters ---")
fmt.Println(f.DumpFilters())
fmt.Println("--- end filters ---")

View file

@ -129,7 +129,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper {
// t.ExpectContinueTimeout
ci.initTransport(t)
// Wrap that http.Transport in our own transport
transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies, ci.DumpAuth)
transport = NewTransport(t, ci.Dump)
})
return transport
}
@ -146,19 +146,15 @@ func (ci *ConfigInfo) Client() *http.Client {
// * Does logging
type Transport struct {
*http.Transport
logHeader bool
logBody bool
logAuth bool
dump DumpFlags
}
// NewTransport wraps the http.Transport passed in and logs all
// roundtrips including the body if logBody is set.
func NewTransport(transport *http.Transport, logHeader, logBody, logAuth bool) *Transport {
func NewTransport(transport *http.Transport, dump DumpFlags) *Transport {
return &Transport{
Transport: transport,
logHeader: logHeader,
logBody: logBody,
logAuth: logAuth,
dump: dump,
}
}
@ -243,9 +239,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
// Force user agent
req.Header.Set("User-Agent", *userAgent)
// Logf request
if t.logHeader || t.logBody || t.logAuth {
buf, _ := httputil.DumpRequestOut(req, t.logBody)
if !t.logAuth {
if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 {
buf, _ := httputil.DumpRequestOut(req, t.dump&(DumpBodies|DumpRequests) != 0)
if t.dump&DumpAuth == 0 {
buf = cleanAuth(buf)
}
Debugf(nil, "%s", separatorReq)
@ -256,13 +252,13 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
// Do round trip
resp, err = t.Transport.RoundTrip(req)
// Logf response
if t.logHeader || t.logBody || t.logAuth {
if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 {
Debugf(nil, "%s", separatorResp)
Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
if err != nil {
Debugf(nil, "Error: %v", err)
} else {
buf, _ := httputil.DumpResponse(resp, t.logBody)
buf, _ := httputil.DumpResponse(resp, t.dump&(DumpBodies|DumpResponses) != 0)
Debugf(nil, "%s", string(buf))
}
Debugf(nil, "%s", separatorResp)

View file

@ -59,8 +59,12 @@ func Initialise() {
if *Verbose {
fs.Config.LogLevel = fs.LogLevelDebug
}
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
if *DumpHeaders {
fs.Config.Dump |= fs.DumpHeaders
}
if *DumpBodies {
fs.Config.Dump |= fs.DumpBodies
}
fs.Config.LowLevelRetries = *LowLevelRetries
fs.Config.UseListR = *UseListR
}