From 72721f4c8d11481618ef9d97aa9085b7ccd4064e Mon Sep 17 00:00:00 2001
From: Stefan Breunig <stefan-github@yrden.de>
Date: Fri, 8 Mar 2019 21:33:22 +0100
Subject: [PATCH] copyurl: honor --no-check-certificate
---
fs/fshttp/http.go | 10 ++++++++--
fs/operations/operations.go | 6 ++++--
fs/operations/operations_test.go | 25 ++++++++++++++++++++++---
3 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index fd44648fe..eb7dd5679 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -28,7 +28,7 @@ const (
var (
transport http.RoundTripper
- noTransport sync.Once
+ noTransport = new(sync.Once)
tpsBucket *rate.Limiter // for limiting number of http transactions per second
cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
)
@@ -121,9 +121,15 @@ func dialContextTimeout(ctx context.Context, network, address string, ci *fs.Con
return newTimeoutConn(c, ci.Timeout)
}
+// ResetTransport resets the existing transport, allowing it to take new settings.
+// Should only be used for testing.
+func ResetTransport() {
+ noTransport = new(sync.Once)
+}
+
// NewTransport returns an http.RoundTripper with the correct timeouts
func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
- noTransport.Do(func() {
+ (*noTransport).Do(func() {
// Start with a sensible set of defaults then override.
// This also means we get new stuff when it gets added to go
t := new(http.Transport)
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 6eba7f401..a8e544078 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"io/ioutil"
- "net/http"
"path"
"path/filepath"
"sort"
@@ -21,6 +20,7 @@ import (
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/fs/fserrors"
+ "github.com/ncw/rclone/fs/fshttp"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fs/march"
"github.com/ncw/rclone/fs/object"
@@ -1411,7 +1411,9 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
// CopyURL copies the data from the url to (fdst, dstFileName)
func CopyURL(fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
- resp, err := http.Get(url)
+ client := fshttp.NewClient(fs.Config)
+ resp, err := client.Get(url)
+
if err != nil {
return nil, err
}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 01cc9f2d4..42cbd8945 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -38,6 +38,7 @@ import (
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/fs/filter"
+ "github.com/ncw/rclone/fs/fshttp"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fstest"
@@ -673,15 +674,18 @@ func TestCopyURL(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
- contents := "file1 contents\n"
+ contents := "file contents\n"
file1 := r.WriteFile("file1", contents, t1)
+ file2 := r.WriteFile("file2", contents, t1)
r.Mkdir(r.Fremote)
fstest.CheckItems(t, r.Fremote)
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // check when reading from regular HTTP server
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(contents))
assert.NoError(t, err)
- }))
+ })
+ ts := httptest.NewServer(handler)
defer ts.Close()
o, err := operations.CopyURL(r.Fremote, "file1", ts.URL)
@@ -689,6 +693,21 @@ func TestCopyURL(t *testing.T) {
assert.Equal(t, int64(len(contents)), o.Size())
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+
+ // check when reading from unverified HTTPS server
+ fs.Config.InsecureSkipVerify = true
+ fshttp.ResetTransport()
+ defer func() {
+ fs.Config.InsecureSkipVerify = false
+ fshttp.ResetTransport()
+ }()
+ tss := httptest.NewTLSServer(handler)
+ defer tss.Close()
+
+ o, err = operations.CopyURL(r.Fremote, "file2", tss.URL)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
}
func TestMoveFile(t *testing.T) {