Merge pull request #1594 from restic/gcs-use-http-transport

GS: Use generic http transport
This commit is contained in:
Alexander Neumann 2018-01-27 22:16:49 +01:00
commit b1b1f6e04f
5 changed files with 39 additions and 14 deletions

2
Gopkg.lock generated
View file

@ -231,6 +231,6 @@
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a" inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

View file

@ -0,0 +1,7 @@
Bugfix: Google Cloud Storage: Use generic HTTP transport
It was discovered that the Google Cloud Storage backend did not use the generic
HTTP transport, so things such as bandwidth limiting with `--limit-upload` did
not work. This is resolved now.
https://github.com/restic/restic/pull/1594

View file

@ -567,7 +567,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
case "s3": case "s3":
be, err = s3.Open(cfg.(s3.Config), rt) be, err = s3.Open(cfg.(s3.Config), rt)
case "gs": case "gs":
be, err = gs.Open(cfg.(gs.Config)) be, err = gs.Open(cfg.(gs.Config), rt)
case "azure": case "azure":
be, err = azure.Open(cfg.(azure.Config), rt) be, err = azure.Open(cfg.(azure.Config), rt)
case "swift": case "swift":
@ -628,7 +628,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
case "s3": case "s3":
return s3.Create(cfg.(s3.Config), rt) return s3.Create(cfg.(s3.Config), rt)
case "gs": case "gs":
return gs.Create(cfg.(gs.Config)) return gs.Create(cfg.(gs.Config), rt)
case "azure": case "azure":
return azure.Create(cfg.(azure.Config), rt) return azure.Create(cfg.(azure.Config), rt)
case "swift": case "swift":

View file

@ -5,6 +5,7 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"net/http"
"os" "os"
"path" "path"
"strings" "strings"
@ -16,6 +17,7 @@ import (
"io/ioutil" "io/ioutil"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google" "golang.org/x/oauth2/google"
"google.golang.org/api/googleapi" "google.golang.org/api/googleapi"
storage "google.golang.org/api/storage/v1" storage "google.golang.org/api/storage/v1"
@ -41,7 +43,7 @@ type Backend struct {
// Ensure that *Backend implements restic.Backend. // Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{} var _ restic.Backend = &Backend{}
func getStorageService(jsonKeyPath string) (*storage.Service, error) { func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) {
raw, err := ioutil.ReadFile(jsonKeyPath) raw, err := ioutil.ReadFile(jsonKeyPath)
if err != nil { if err != nil {
@ -53,8 +55,18 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
return nil, err return nil, err
} }
client := conf.Client(context.TODO()) // create a new HTTP client
httpClient := &http.Client{
Transport: rt,
}
// create a now context with the HTTP client stored at the oauth2.HTTPClient key
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient)
// then pass this context to Client(), which returns a new HTTP client
client := conf.Client(ctx)
// that we can then pass to New()
service, err := storage.New(client) service, err := storage.New(client)
if err != nil { if err != nil {
return nil, err return nil, err
@ -65,10 +77,10 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
const defaultListMaxItems = 1000 const defaultListMaxItems = 1000
func open(cfg Config) (*Backend, error) { func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg) debug.Log("open, config %#v", cfg)
service, err := getStorageService(cfg.JSONKeyPath) service, err := getStorageService(cfg.JSONKeyPath, rt)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "getStorageService") return nil, errors.Wrap(err, "getStorageService")
} }
@ -95,8 +107,8 @@ func open(cfg Config) (*Backend, error) {
} }
// Open opens the gs backend at the specified bucket. // Open opens the gs backend at the specified bucket.
func Open(cfg Config) (restic.Backend, error) { func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg) return open(cfg, rt)
} }
// Create opens the gs backend at the specified bucket and attempts to creates // Create opens the gs backend at the specified bucket and attempts to creates
@ -104,8 +116,8 @@ func Open(cfg Config) (restic.Backend, error) {
// //
// The service account must have the "storage.buckets.create" permission to // The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist. // create a bucket the does not yet exist.
func Create(cfg Config) (restic.Backend, error) { func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := open(cfg) be, err := open(cfg, rt)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "open") return nil, errors.Wrap(err, "open")
} }

View file

@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -15,6 +16,11 @@ import (
) )
func newGSTestSuite(t testing.TB) *test.Suite { func newGSTestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{ return &test.Suite{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
@ -37,7 +43,7 @@ func newGSTestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) { Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config) cfg := config.(gs.Config)
be, err := gs.Create(cfg) be, err := gs.Create(cfg, tr)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -57,14 +63,14 @@ func newGSTestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config) cfg := config.(gs.Config)
return gs.Open(cfg) return gs.Open(cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(config interface{}) error {
cfg := config.(gs.Config) cfg := config.(gs.Config)
be, err := gs.Open(cfg) be, err := gs.Open(cfg, tr)
if err != nil { if err != nil {
return err return err
} }