Merge pull request #1594 from restic/gcs-use-http-transport
GS: Use generic http transport
This commit is contained in:
commit
b1b1f6e04f
5 changed files with 39 additions and 14 deletions
2
Gopkg.lock
generated
2
Gopkg.lock
generated
|
@ -231,6 +231,6 @@
|
||||||
[solve-meta]
|
[solve-meta]
|
||||||
analyzer-name = "dep"
|
analyzer-name = "dep"
|
||||||
analyzer-version = 1
|
analyzer-version = 1
|
||||||
inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
|
inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
|
||||||
solver-name = "gps-cdcl"
|
solver-name = "gps-cdcl"
|
||||||
solver-version = 1
|
solver-version = 1
|
||||||
|
|
7
changelog/0.8.2/pull-1594
Normal file
7
changelog/0.8.2/pull-1594
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
Bugfix: Google Cloud Storage: Use generic HTTP transport
|
||||||
|
|
||||||
|
It was discovered that the Google Cloud Storage backend did not use the generic
|
||||||
|
HTTP transport, so things such as bandwidth limiting with `--limit-upload` did
|
||||||
|
not work. This is resolved now.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/1594
|
|
@ -567,7 +567,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
|
||||||
case "s3":
|
case "s3":
|
||||||
be, err = s3.Open(cfg.(s3.Config), rt)
|
be, err = s3.Open(cfg.(s3.Config), rt)
|
||||||
case "gs":
|
case "gs":
|
||||||
be, err = gs.Open(cfg.(gs.Config))
|
be, err = gs.Open(cfg.(gs.Config), rt)
|
||||||
case "azure":
|
case "azure":
|
||||||
be, err = azure.Open(cfg.(azure.Config), rt)
|
be, err = azure.Open(cfg.(azure.Config), rt)
|
||||||
case "swift":
|
case "swift":
|
||||||
|
@ -628,7 +628,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
|
||||||
case "s3":
|
case "s3":
|
||||||
return s3.Create(cfg.(s3.Config), rt)
|
return s3.Create(cfg.(s3.Config), rt)
|
||||||
case "gs":
|
case "gs":
|
||||||
return gs.Create(cfg.(gs.Config))
|
return gs.Create(cfg.(gs.Config), rt)
|
||||||
case "azure":
|
case "azure":
|
||||||
return azure.Create(cfg.(azure.Config), rt)
|
return azure.Create(cfg.(azure.Config), rt)
|
||||||
case "swift":
|
case "swift":
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -16,6 +17,7 @@ import (
|
||||||
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
|
||||||
|
"golang.org/x/oauth2"
|
||||||
"golang.org/x/oauth2/google"
|
"golang.org/x/oauth2/google"
|
||||||
"google.golang.org/api/googleapi"
|
"google.golang.org/api/googleapi"
|
||||||
storage "google.golang.org/api/storage/v1"
|
storage "google.golang.org/api/storage/v1"
|
||||||
|
@ -41,7 +43,7 @@ type Backend struct {
|
||||||
// Ensure that *Backend implements restic.Backend.
|
// Ensure that *Backend implements restic.Backend.
|
||||||
var _ restic.Backend = &Backend{}
|
var _ restic.Backend = &Backend{}
|
||||||
|
|
||||||
func getStorageService(jsonKeyPath string) (*storage.Service, error) {
|
func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) {
|
||||||
|
|
||||||
raw, err := ioutil.ReadFile(jsonKeyPath)
|
raw, err := ioutil.ReadFile(jsonKeyPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -53,8 +55,18 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
client := conf.Client(context.TODO())
|
// create a new HTTP client
|
||||||
|
httpClient := &http.Client{
|
||||||
|
Transport: rt,
|
||||||
|
}
|
||||||
|
|
||||||
|
// create a now context with the HTTP client stored at the oauth2.HTTPClient key
|
||||||
|
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient)
|
||||||
|
|
||||||
|
// then pass this context to Client(), which returns a new HTTP client
|
||||||
|
client := conf.Client(ctx)
|
||||||
|
|
||||||
|
// that we can then pass to New()
|
||||||
service, err := storage.New(client)
|
service, err := storage.New(client)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -65,10 +77,10 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
|
||||||
|
|
||||||
const defaultListMaxItems = 1000
|
const defaultListMaxItems = 1000
|
||||||
|
|
||||||
func open(cfg Config) (*Backend, error) {
|
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||||
debug.Log("open, config %#v", cfg)
|
debug.Log("open, config %#v", cfg)
|
||||||
|
|
||||||
service, err := getStorageService(cfg.JSONKeyPath)
|
service, err := getStorageService(cfg.JSONKeyPath, rt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "getStorageService")
|
return nil, errors.Wrap(err, "getStorageService")
|
||||||
}
|
}
|
||||||
|
@ -95,8 +107,8 @@ func open(cfg Config) (*Backend, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the gs backend at the specified bucket.
|
// Open opens the gs backend at the specified bucket.
|
||||||
func Open(cfg Config) (restic.Backend, error) {
|
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
|
||||||
return open(cfg)
|
return open(cfg, rt)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create opens the gs backend at the specified bucket and attempts to creates
|
// Create opens the gs backend at the specified bucket and attempts to creates
|
||||||
|
@ -104,8 +116,8 @@ func Open(cfg Config) (restic.Backend, error) {
|
||||||
//
|
//
|
||||||
// The service account must have the "storage.buckets.create" permission to
|
// The service account must have the "storage.buckets.create" permission to
|
||||||
// create a bucket the does not yet exist.
|
// create a bucket the does not yet exist.
|
||||||
func Create(cfg Config) (restic.Backend, error) {
|
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
|
||||||
be, err := open(cfg)
|
be, err := open(cfg, rt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "open")
|
return nil, errors.Wrap(err, "open")
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/backend"
|
||||||
"github.com/restic/restic/internal/backend/gs"
|
"github.com/restic/restic/internal/backend/gs"
|
||||||
"github.com/restic/restic/internal/backend/test"
|
"github.com/restic/restic/internal/backend/test"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
@ -15,6 +16,11 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func newGSTestSuite(t testing.TB) *test.Suite {
|
func newGSTestSuite(t testing.TB) *test.Suite {
|
||||||
|
tr, err := backend.Transport(backend.TransportOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cannot create transport for tests: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
return &test.Suite{
|
return &test.Suite{
|
||||||
// do not use excessive data
|
// do not use excessive data
|
||||||
MinimalData: true,
|
MinimalData: true,
|
||||||
|
@ -37,7 +43,7 @@ func newGSTestSuite(t testing.TB) *test.Suite {
|
||||||
Create: func(config interface{}) (restic.Backend, error) {
|
Create: func(config interface{}) (restic.Backend, error) {
|
||||||
cfg := config.(gs.Config)
|
cfg := config.(gs.Config)
|
||||||
|
|
||||||
be, err := gs.Create(cfg)
|
be, err := gs.Create(cfg, tr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -57,14 +63,14 @@ func newGSTestSuite(t testing.TB) *test.Suite {
|
||||||
// OpenFn is a function that opens a previously created temporary repository.
|
// OpenFn is a function that opens a previously created temporary repository.
|
||||||
Open: func(config interface{}) (restic.Backend, error) {
|
Open: func(config interface{}) (restic.Backend, error) {
|
||||||
cfg := config.(gs.Config)
|
cfg := config.(gs.Config)
|
||||||
return gs.Open(cfg)
|
return gs.Open(cfg, tr)
|
||||||
},
|
},
|
||||||
|
|
||||||
// CleanupFn removes data created during the tests.
|
// CleanupFn removes data created during the tests.
|
||||||
Cleanup: func(config interface{}) error {
|
Cleanup: func(config interface{}) error {
|
||||||
cfg := config.(gs.Config)
|
cfg := config.(gs.Config)
|
||||||
|
|
||||||
be, err := gs.Open(cfg)
|
be, err := gs.Open(cfg, tr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue