TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #1594 from restic/gcs-use-http-transport

Browse source 
GS: Use generic http transport
This commit is contained in:
Alexander Neumann 2018-01-27 22:16:49 +01:00
commit b1b1f6e04f
5 changed files with 39 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Gopkg.lock generated
View file

@ -231,6 +231,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
solver-name = "gps-cdcl"
solver-version = 1

7
changelog/0.8.2/pull-1594 Normal file
View file

@ -0,0 +1,7 @@
Bugfix: Google Cloud Storage: Use generic HTTP transport
It was discovered that the Google Cloud Storage backend did not use the generic
HTTP transport, so things such as bandwidth limiting with `--limit-upload` did
not work. This is resolved now.
https://github.com/restic/restic/pull/1594

4
cmd/restic/global.go
View file

@ -567,7 +567,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
case "s3":
be, err = s3.Open(cfg.(s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config))
be, err = gs.Open(cfg.(gs.Config), rt)
case "azure":
be, err = azure.Open(cfg.(azure.Config), rt)
case "swift":
@ -628,7 +628,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
case "s3":
return s3.Create(cfg.(s3.Config), rt)
case "gs":
return gs.Create(cfg.(gs.Config))
return gs.Create(cfg.(gs.Config), rt)
case "azure":
return azure.Create(cfg.(azure.Config), rt)
case "swift":

28
internal/backend/gs/gs.go
View file

@ -5,6 +5,7 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
@ -16,6 +17,7 @@ import (
"io/ioutil"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
storage "google.golang.org/api/storage/v1"
@ -41,7 +43,7 @@ type Backend struct {
// Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
func getStorageService(jsonKeyPath string) (*storage.Service, error) {
func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) {
raw, err := ioutil.ReadFile(jsonKeyPath)
if err != nil {
@ -53,8 +55,18 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
return nil, err
}
client := conf.Client(context.TODO())
// create a new HTTP client
httpClient := &http.Client{
Transport: rt,
}
// create a now context with the HTTP client stored at the oauth2.HTTPClient key
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient)
// then pass this context to Client(), which returns a new HTTP client
client := conf.Client(ctx)
// that we can then pass to New()
service, err := storage.New(client)
if err != nil {
return nil, err
@ -65,10 +77,10 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
const defaultListMaxItems = 1000
func open(cfg Config) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
service, err := getStorageService(cfg.JSONKeyPath)
service, err := getStorageService(cfg.JSONKeyPath, rt)
if err != nil {
return nil, errors.Wrap(err, "getStorageService")
}
@ -95,8 +107,8 @@ func open(cfg Config) (*Backend, error) {
}
// Open opens the gs backend at the specified bucket.
func Open(cfg Config) (restic.Backend, error) {
return open(cfg)
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg, rt)
}
// Create opens the gs backend at the specified bucket and attempts to creates
@ -104,8 +116,8 @@ func Open(cfg Config) (restic.Backend, error) {
//
// The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist.
func Create(cfg Config) (restic.Backend, error) {
be, err := open(cfg)
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
}

12
internal/backend/gs/gs_test.go
View file

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
@ -15,6 +16,11 @@ import (
)
func newGSTestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// do not use excessive data
MinimalData: true,
@ -37,7 +43,7 @@ func newGSTestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config)
be, err := gs.Create(cfg)
be, err := gs.Create(cfg, tr)
if err != nil {
return nil, err
}
@ -57,14 +63,14 @@ func newGSTestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config)
return gs.Open(cfg)
return gs.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(gs.Config)
be, err := gs.Open(cfg)
be, err := gs.Open(cfg, tr)
if err != nil {
return err
}