forked from TrueCloudLab/restic
Swap deprecated GCS lib with replacement
This commit is contained in:
Ingo Gottwald
parent
7c3c6fa431
commit
8b8e230771
3 changed files with 85 additions and 116 deletions
4
go.mod
4
go.mod
|
|
@ -2,7 +2,7 @@ module github.com/restic/restic
|
|||
|
||||
require (
|
||||
bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512
|
||||
cloud.google.com/go v0.66.0 // indirect
|
||||
cloud.google.com/go/storage v1.12.0
|
||||
github.com/Azure/azure-sdk-for-go v46.1.0+incompatible
|
||||
github.com/Azure/go-autorest/autorest v0.11.6 // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
|
||||
|
|
@ -44,8 +44,6 @@ require (
|
|||
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff
|
||||
golang.org/x/text v0.3.3
|
||||
google.golang.org/api v0.32.0
|
||||
google.golang.org/genproto v0.0.0-20200918140846-d0d605568037 // indirect
|
||||
google.golang.org/grpc v1.32.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
|
||||
gopkg.in/ini.v1 v1.61.0 // indirect
|
||||
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
|
||||
|
|
|
|||
15
go.sum
15
go.sum
|
|
@ -33,7 +33,10 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
|||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4=
|
||||
cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go v46.1.0+incompatible h1:e9xxveqrMFRJgj44gychg6jYGfZbwwKhW4wGq9LEG8Q=
|
||||
github.com/Azure/azure-sdk-for-go v46.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
||||
|
|
@ -138,7 +141,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
|
|
@ -169,6 +174,7 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
|
|||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
|
|
@ -307,6 +313,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl
|
|||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
|
|
@ -315,6 +322,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
|||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -456,7 +464,10 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
|
|||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||
golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c h1:AQsh/7arPVFDBraQa8x7GoVnwnGg1kM7J2ySI0kF5WU=
|
||||
golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
||||
golang.org/x/tools v0.0.0-20200918232735-d647fc253266 h1:k7tVuG0g1JwmD3Jh8oAl1vQ1C3jb4Hi/dUl1wWDBJpQ=
|
||||
golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -524,8 +535,8 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D
|
|||
google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200918140846-d0d605568037 h1:ujwz1DPMeHwCvo36rK5shXhAzc4GMRecrqQFaMZJBKQ=
|
||||
google.golang.org/genproto v0.0.0-20200918140846-d0d605568037/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5 h1:B9nroC8SSX5GtbVvxPF9tYIVkaCpjhVLOrlAY8ONzm8=
|
||||
google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@ package gs
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
|
@ -18,8 +18,8 @@ import (
|
|||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/api/option"
|
||||
storage "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
// Backend stores data in a GCS bucket.
|
||||
|
|
@ -30,10 +30,11 @@ import (
|
|||
// * storage.objects.get
|
||||
// * storage.objects.list
|
||||
type Backend struct {
|
||||
service *storage.Service
|
||||
gcsClient *storage.Client
|
||||
projectID string
|
||||
sem *backend.Semaphore
|
||||
bucketName string
|
||||
bucket *storage.BucketHandle
|
||||
prefix string
|
||||
listMaxItems int
|
||||
backend.Layout
|
||||
|
|
@ -42,7 +43,7 @@ type Backend struct {
|
|||
// Ensure that *Backend implements restic.Backend.
|
||||
var _ restic.Backend = &Backend{}
|
||||
|
||||
func getStorageService(rt http.RoundTripper) (*storage.Service, error) {
|
||||
func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
|
||||
// create a new HTTP client
|
||||
httpClient := &http.Client{
|
||||
Transport: rt,
|
||||
|
|
@ -59,20 +60,28 @@ func getStorageService(rt http.RoundTripper) (*storage.Service, error) {
|
|||
})
|
||||
} else {
|
||||
var err error
|
||||
ts, err = google.DefaultTokenSource(ctx, storage.DevstorageReadWriteScope)
|
||||
ts, err = google.DefaultTokenSource(ctx, storage.ScopeReadWrite)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
client := oauth2.NewClient(ctx, ts)
|
||||
oauthClient := oauth2.NewClient(ctx, ts)
|
||||
|
||||
service, err := storage.NewService(ctx, option.WithHTTPClient(client))
|
||||
gcsClient, err := storage.NewClient(ctx, option.WithHTTPClient(oauthClient))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return service, nil
|
||||
return gcsClient, nil
|
||||
}
|
||||
|
||||
func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
|
||||
_, err := bucket.Attrs(ctx)
|
||||
if err == storage.ErrBucketNotExist {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
const defaultListMaxItems = 1000
|
||||
|
|
@ -80,9 +89,9 @@ const defaultListMaxItems = 1000
|
|||
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
debug.Log("open, config %#v", cfg)
|
||||
|
||||
service, err := getStorageService(rt)
|
||||
gcsClient, err := getStorageClient(rt)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getStorageService")
|
||||
return nil, errors.Wrap(err, "getStorageClient")
|
||||
}
|
||||
|
||||
sem, err := backend.NewSemaphore(cfg.Connections)
|
||||
|
|
@ -91,10 +100,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
|||
}
|
||||
|
||||
be := &Backend{
|
||||
service: service,
|
||||
gcsClient: gcsClient,
|
||||
projectID: cfg.ProjectID,
|
||||
sem: sem,
|
||||
bucketName: cfg.Bucket,
|
||||
bucket: gcsClient.Bucket(cfg.Bucket),
|
||||
prefix: cfg.Prefix,
|
||||
Layout: &backend.DefaultLayout{
|
||||
Path: cfg.Prefix,
|
||||
|
|
@ -123,47 +133,19 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
|
|||
}
|
||||
|
||||
// Try to determine if the bucket exists. If it does not, try to create it.
|
||||
//
|
||||
// A Get call has three typical error cases:
|
||||
//
|
||||
// * nil: Bucket exists and we have access to the metadata (returned).
|
||||
//
|
||||
// * 403: Bucket exists and we do not have access to the metadata. We
|
||||
// don't have storage.buckets.get permission to the bucket, but we may
|
||||
// still be able to access objects in the bucket.
|
||||
//
|
||||
// * 404: Bucket doesn't exist.
|
||||
//
|
||||
// Determining if the bucket is accessible is best-effort because the
|
||||
// 403 case is ambiguous.
|
||||
if _, err := be.service.Buckets.Get(be.bucketName).Do(); err != nil {
|
||||
gerr, ok := err.(*googleapi.Error)
|
||||
if !ok {
|
||||
// Don't know what to do with this error.
|
||||
return nil, errors.Wrap(err, "service.Buckets.Get")
|
||||
ctx := context.Background()
|
||||
exists, err := be.bucketExists(ctx, be.bucket)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "service.Buckets.Get")
|
||||
}
|
||||
|
||||
if !exists {
|
||||
// Bucket doesn't exist, try to create it.
|
||||
if err := be.bucket.Create(ctx, be.projectID, nil); err != nil {
|
||||
// Always an error, as the bucket definitely doesn't exist.
|
||||
return nil, errors.Wrap(err, "service.Buckets.Insert")
|
||||
}
|
||||
|
||||
switch gerr.Code {
|
||||
case 403:
|
||||
// Bucket exists, but we don't know if it is
|
||||
// accessible. Optimistically assume it is; if not,
|
||||
// future Backend calls will fail.
|
||||
debug.Log("Unable to determine if bucket %s is accessible (err %v). Continuing as if it is.", be.bucketName, err)
|
||||
case 404:
|
||||
// Bucket doesn't exist, try to create it.
|
||||
bucket := &storage.Bucket{
|
||||
Name: be.bucketName,
|
||||
}
|
||||
|
||||
if _, err := be.service.Buckets.Insert(be.projectID, bucket).Do(); err != nil {
|
||||
// Always an error, as the bucket definitely
|
||||
// doesn't exist.
|
||||
return nil, errors.Wrap(err, "service.Buckets.Insert")
|
||||
}
|
||||
default:
|
||||
// Don't know what to do with this error.
|
||||
return nil, errors.Wrap(err, "service.Buckets.Get")
|
||||
}
|
||||
}
|
||||
|
||||
return be, nil
|
||||
|
|
@ -245,13 +227,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
|
|||
//
|
||||
// restic typically writes small blobs (4MB-30MB), so the resumable
|
||||
// uploads are not providing significant benefit anyways.
|
||||
cs := googleapi.ChunkSize(0)
|
||||
|
||||
info, err := be.service.Objects.Insert(be.bucketName,
|
||||
&storage.Object{
|
||||
Name: objName,
|
||||
Size: uint64(rd.Length()),
|
||||
}).Media(rd, cs).Do()
|
||||
w := be.bucket.Object(objName).NewWriter(ctx)
|
||||
w.ChunkSize = 0
|
||||
wbytes, err := io.Copy(w, rd)
|
||||
w.Close()
|
||||
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
|
|
@ -260,7 +239,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
|
|||
return errors.Wrap(err, "service.Objects.Insert")
|
||||
}
|
||||
|
||||
debug.Log("%v -> %v bytes", objName, info.Size)
|
||||
debug.Log("%v -> %v bytes", objName, wbytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -295,29 +274,23 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
|
|||
if length < 0 {
|
||||
return nil, errors.Errorf("invalid length %d", length)
|
||||
}
|
||||
if length == 0 {
|
||||
// negative length indicates read till end to GCS lib
|
||||
length = -1
|
||||
}
|
||||
|
||||
objName := be.Filename(h)
|
||||
|
||||
be.sem.GetToken()
|
||||
|
||||
var byteRange string
|
||||
if length > 0 {
|
||||
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length-1))
|
||||
} else {
|
||||
byteRange = fmt.Sprintf("bytes=%d-", offset)
|
||||
}
|
||||
|
||||
req := be.service.Objects.Get(be.bucketName, objName)
|
||||
// https://cloud.google.com/storage/docs/json_api/v1/parameters#range
|
||||
req.Header().Set("Range", byteRange)
|
||||
res, err := req.Download()
|
||||
r, err := be.bucket.Object(objName).NewRangeReader(ctx, offset, int64(length))
|
||||
if err != nil {
|
||||
be.sem.ReleaseToken()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
closeRd := wrapReader{
|
||||
ReadCloser: res.Body,
|
||||
ReadCloser: r,
|
||||
f: func() {
|
||||
debug.Log("Close()")
|
||||
be.sem.ReleaseToken()
|
||||
|
|
@ -334,15 +307,15 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
|
|||
objName := be.Filename(h)
|
||||
|
||||
be.sem.GetToken()
|
||||
obj, err := be.service.Objects.Get(be.bucketName, objName).Do()
|
||||
attr, err := be.bucket.Object(objName).Attrs(ctx)
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
if err != nil {
|
||||
debug.Log("GetObject() err %v", err)
|
||||
debug.Log("GetObjectAttributes() err %v", err)
|
||||
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
|
||||
}
|
||||
|
||||
return restic.FileInfo{Size: int64(obj.Size), Name: h.Name}, nil
|
||||
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
|
|
@ -351,7 +324,7 @@ func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
|
|||
objName := be.Filename(h)
|
||||
|
||||
be.sem.GetToken()
|
||||
_, err := be.service.Objects.Get(be.bucketName, objName).Do()
|
||||
_, err := be.bucket.Object(objName).Attrs(ctx)
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
if err == nil {
|
||||
|
|
@ -366,13 +339,11 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
|
|||
objName := be.Filename(h)
|
||||
|
||||
be.sem.GetToken()
|
||||
err := be.service.Objects.Delete(be.bucketName, objName).Do()
|
||||
err := be.bucket.Object(objName).Delete(ctx)
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
if er, ok := err.(*googleapi.Error); ok {
|
||||
if er.Code == 404 {
|
||||
err = nil
|
||||
}
|
||||
if err == storage.ErrObjectNotExist {
|
||||
err = nil
|
||||
}
|
||||
|
||||
debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
|
||||
|
|
@ -394,47 +365,36 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
|
|||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
listReq := be.service.Objects.List(be.bucketName).Context(ctx).Prefix(prefix).MaxResults(int64(be.listMaxItems))
|
||||
itr := be.bucket.Objects(ctx, &storage.Query{Prefix: prefix})
|
||||
|
||||
for {
|
||||
be.sem.GetToken()
|
||||
obj, err := listReq.Do()
|
||||
attrs, err := itr.Next()
|
||||
be.sem.ReleaseToken()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m := strings.TrimPrefix(attrs.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
fi := restic.FileInfo{
|
||||
Name: path.Base(m),
|
||||
Size: int64(attrs.Size),
|
||||
}
|
||||
|
||||
err = fn(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
debug.Log("returned %v items", len(obj.Items))
|
||||
|
||||
for _, item := range obj.Items {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
fi := restic.FileInfo{
|
||||
Name: path.Base(m),
|
||||
Size: int64(item.Size),
|
||||
}
|
||||
|
||||
err := fn(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if obj.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
listReq.PageToken(obj.NextPageToken)
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
|
|
|
|||
Loading…
Reference in a new issue