TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

s3: Use low level API for saving files

Browse source 
benchmark                        old ns/op       new ns/op       delta
    BenchmarkBackendMinio/Save-4     184482294       40663344        -77.96%
    BenchmarkBackendS3/Save-4        35030825568     54475455819     +55.51%

    benchmark                        old MB/s     new MB/s     speedup
    BenchmarkBackendMinio/Save-4     90.95        412.64       4.54x
    BenchmarkBackendS3/Save-4        0.48         0.31         0.65x

    benchmark                        old allocs     new allocs     delta
    BenchmarkBackendMinio/Save-4     631            560            -11.25%
    BenchmarkBackendS3/Save-4        646            584            -9.60%

    benchmark                        old bytes     new bytes     delta
    BenchmarkBackendMinio/Save-4     66818060      50735         -99.92%
    BenchmarkBackendS3/Save-4        66834000      73024         -99.89%
This commit is contained in:
Alexander Neumann 2017-05-13 23:17:10 +02:00
parent 1e0e6ee573
commit 9452f416bf
1 changed files with 18 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/restic/backend/s3/s3.go
View file

@ -80,6 +80,10 @@ func (be *s3) Location() string {
return be.bucketname return be.bucketname
} }
type Sizer interface {
Size() int64
}
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
@ -88,6 +92,13 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
objName := be.Filename(h) objName := be.Filename(h)
var size int64
if r, ok := rd.(Sizer); ok {
size = r.Size()
} else {
panic("Save() got passed a reader without a method to determine the data size")
}
debug.Log("Save %v at %v", h, objName) debug.Log("Save %v at %v", h, objName)
// Check key does not already exist // Check key does not already exist
@ -98,14 +109,14 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
} }
<-be.connChan <-be.connChan
defer func() {
be.connChan <- struct{}{}
}()
debug.Log("PutObject(%v, %v)", debug.Log("PutObject(%v, %v)", be.bucketname, objName)
be.bucketname, objName) coreClient := minio.Core{be.client}
n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream") info, err := coreClient.PutObject(be.bucketname, objName, size, rd, nil, nil, nil)
debug.Log("%v -> %v bytes, err %#v", objName, n, err)
// return token
be.connChan <- struct{}{}
debug.Log("%v -> %v bytes, err %#v", objName, info.Size, err)
return errors.Wrap(err, "client.PutObject") return errors.Wrap(err, "client.PutObject")
} }