TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

GS backend: limit http concurrency in Save(), Stat(), Test(), Remove(), List()

Browse source 
as per discussion in PR #1399
This commit is contained in:
George Armhold 2017-10-31 08:01:43 -04:00
parent 981752ade0
commit d069ee31b2
1 changed files with 16 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
internal/backend/gs/gs.go
View file

@ -204,14 +204,15 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err
debug.Log("Save %v at %v", h, objName) debug.Log("Save %v at %v", h, objName)
be.sem.GetToken()
// Check key does not already exist // Check key does not already exist
if _, err := be.service.Objects.Get(be.bucketName, objName).Do(); err == nil { if _, err := be.service.Objects.Get(be.bucketName, objName).Do(); err == nil {
debug.Log("%v already exists", h) debug.Log("%v already exists", h)
be.sem.ReleaseToken()
return errors.New("key already exists") return errors.New("key already exists")
} }
be.sem.GetToken()
debug.Log("InsertObject(%v, %v)", be.bucketName, objName) debug.Log("InsertObject(%v, %v)", be.bucketName, objName)
// Set chunk size to zero to disable resumable uploads. // Set chunk size to zero to disable resumable uploads.
@ -323,7 +324,10 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
objName := be.Filename(h) objName := be.Filename(h)
be.sem.GetToken()
obj, err := be.service.Objects.Get(be.bucketName, objName).Do() obj, err := be.service.Objects.Get(be.bucketName, objName).Do()
be.sem.ReleaseToken()
if err != nil { if err != nil {
debug.Log("GetObject() err %v", err) debug.Log("GetObject() err %v", err)
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
@ -336,7 +340,11 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) { func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
found := false found := false
objName := be.Filename(h) objName := be.Filename(h)
be.sem.GetToken()
_, err := be.service.Objects.Get(be.bucketName, objName).Do() _, err := be.service.Objects.Get(be.bucketName, objName).Do()
be.sem.ReleaseToken()
if err == nil { if err == nil {
found = true found = true
} }
@ -348,7 +356,10 @@ func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
be.sem.GetToken()
err := be.service.Objects.Delete(be.bucketName, objName).Do() err := be.service.Objects.Delete(be.bucketName, objName).Do()
be.sem.ReleaseToken()
if er, ok := err.(*googleapi.Error); ok { if er, ok := err.(*googleapi.Error); ok {
if er.Code == 404 { if er.Code == 404 {
err = nil err = nil
@ -378,7 +389,10 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
listReq := be.service.Objects.List(be.bucketName).Prefix(prefix).MaxResults(int64(be.listMaxItems)) listReq := be.service.Objects.List(be.bucketName).Prefix(prefix).MaxResults(int64(be.listMaxItems))
for { for {
be.sem.GetToken()
obj, err := listReq.Do() obj, err := listReq.Do()
be.sem.ReleaseToken()
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "error listing %v: %v\n", prefix, err) fmt.Fprintf(os.Stderr, "error listing %v: %v\n", prefix, err)
return return