Add custom HTTP transport

src/restic/backend/http_transport.go

@@ -0,0 +1,28 @@
+package backend
+
+import (
+	"net"
+	"net/http"
+	"restic/debug"
+	"time"
+)
+
+// Transport returns a new http.RoundTripper with default settings applied.
+func Transport() http.RoundTripper {
+	// copied from net/http
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	// wrap in the debug round tripper
+	return debug.RoundTripper(tr)
+}

src/restic/backend/rest/rest.go

@@ -35,8 +35,8 @@ func Open(cfg Config) (restic.Backend, error) {
 	for i := 0; i < connLimit; i++ {
 		connChan <- struct{}{}
 	}
-	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+
-	client := http.Client{Transport: tr}
+	client := http.Client{Transport: backend.Transport()}
 
 	// use url without trailing slash for layout
 	url := cfg.URL.String()

src/restic/backend/s3/s3.go

@@ -3,7 +3,6 @@ package s3
 import (
 	"bytes"
 	"io"
-	"net/http"
 	"path"
 	"restic"
 	"strings"
@@ -48,8 +47,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		Layout:       &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
 	}
 
-	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+	client.SetCustomTransport(backend.Transport())
-	client.SetCustomTransport(tr)
 
 	be.createConnections()
 

src/restic/debug/round_tripper_debug.go

@@ -3,10 +3,59 @@
 package debug
 
 import (
+	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
+	"os"
+	"restic/errors"
 )
 
+type eofDetectRoundTripper struct {
+	http.RoundTripper
+}
+
+type eofDetectReader struct {
+	eofSeen bool
+	rd      io.ReadCloser
+}
+
+func (rd *eofDetectReader) Read(p []byte) (n int, err error) {
+	n, err = rd.rd.Read(p)
+	if err == io.EOF {
+		rd.eofSeen = true
+	}
+	return n, err
+}
+
+func (rd *eofDetectReader) Close() error {
+	if !rd.eofSeen {
+		buf, err := ioutil.ReadAll(rd)
+		msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf))
+		if err != nil {
+			msg += fmt.Sprintf(", error: %v", err)
+		}
+
+		if len(buf) > 0 {
+			if len(buf) > 20 {
+				buf = append(buf[:20], []byte("...")...)
+			}
+			msg += fmt.Sprintf(", body: %q", buf)
+		}
+
+		fmt.Fprintln(os.Stderr, msg)
+		Log("%s: %+v", msg, errors.New("Close()"))
+	}
+	return rd.rd.Close()
+}
+
+func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
+	res, err = tr.RoundTripper.RoundTrip(req)
+	res.Body = &eofDetectReader{rd: res.Body}
+	return res, err
+}
+
 type loggingRoundTripper struct {
 	http.RoundTripper
 }
@@ -14,7 +63,7 @@ type loggingRoundTripper struct {
 // RoundTripper returns a new http.RoundTripper which logs all requests (if
 // debug is enabled). When debug is not enabled, upstream is returned.
 func RoundTripper(upstream http.RoundTripper) http.RoundTripper {
-	return loggingRoundTripper{upstream}
+	return loggingRoundTripper{eofDetectRoundTripper{upstream}}
 }
 
 func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {