forked from TrueCloudLab/restic
Update s3 library again
This commit is contained in:
parent
a73c4bd5a7
commit
f319354174
3 changed files with 26 additions and 11 deletions
4
Godeps/Godeps.json
generated
4
Godeps/Godeps.json
generated
|
@ -24,8 +24,8 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ImportPath": "github.com/minio/minio-go",
|
"ImportPath": "github.com/minio/minio-go",
|
||||||
"Comment": "v0.2.5-185-g654a97a",
|
"Comment": "v0.2.5-187-gad1597d",
|
||||||
"Rev": "654a97a4d165dabf422bec2ef6673bcd9d3daf00"
|
"Rev": "ad1597d864f56f608f8a1694ae9b5970fef57eb6"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ImportPath": "github.com/pkg/sftp",
|
"ImportPath": "github.com/pkg/sftp",
|
||||||
|
|
14
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
14
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
|
@ -17,7 +17,6 @@
|
||||||
package minio
|
package minio
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
@ -186,7 +185,7 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
|
||||||
// Get shortest length.
|
// Get shortest length.
|
||||||
// NOTE: Last remaining bytes are usually smaller than
|
// NOTE: Last remaining bytes are usually smaller than
|
||||||
// req.Buffer size. Use that as the final length.
|
// req.Buffer size. Use that as the final length.
|
||||||
length := math.Min(float64(req.Buffer.Len()), float64(objectStat.Size-req.Offset))
|
length := math.Min(float64(len(req.Buffer)), float64(objectStat.Size-req.Offset))
|
||||||
httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length))
|
httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
resCh <- readAtResponse{
|
resCh <- readAtResponse{
|
||||||
|
@ -194,7 +193,12 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
size, err := io.CopyN(req.Buffer, httpReader, int64(length))
|
size, err := io.ReadFull(httpReader, req.Buffer)
|
||||||
|
if err == io.ErrUnexpectedEOF {
|
||||||
|
// If an EOF happens after reading some but not all the bytes
|
||||||
|
// ReadFull returns ErrUnexpectedEOF
|
||||||
|
err = io.EOF
|
||||||
|
}
|
||||||
resCh <- readAtResponse{
|
resCh <- readAtResponse{
|
||||||
Size: int(size),
|
Size: int(size),
|
||||||
Error: err,
|
Error: err,
|
||||||
|
@ -214,7 +218,7 @@ type readAtResponse struct {
|
||||||
|
|
||||||
// request message container to communicate with internal go-routine.
|
// request message container to communicate with internal go-routine.
|
||||||
type readAtRequest struct {
|
type readAtRequest struct {
|
||||||
Buffer *bytes.Buffer
|
Buffer []byte
|
||||||
Offset int64 // readAt offset.
|
Offset int64 // readAt offset.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -267,7 +271,7 @@ func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) {
|
||||||
reqMsg := readAtRequest{}
|
reqMsg := readAtRequest{}
|
||||||
|
|
||||||
// Send the current offset and bytes requested.
|
// Send the current offset and bytes requested.
|
||||||
reqMsg.Buffer = bytes.NewBuffer(b)
|
reqMsg.Buffer = b
|
||||||
reqMsg.Offset = offset
|
reqMsg.Offset = offset
|
||||||
|
|
||||||
// Send read request over the control channel.
|
// Send read request over the control channel.
|
||||||
|
|
19
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go
generated
vendored
19
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go
generated
vendored
|
@ -18,6 +18,7 @@ package minio_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
crand "crypto/rand"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
@ -83,8 +84,13 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
||||||
t.Fatal("Error:", err, bucketName)
|
t.Fatal("Error:", err, bucketName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// generate data
|
// generate data more than 32K
|
||||||
buf := make([]byte, rand.Intn(1<<20))
|
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||||
|
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
// save the data
|
// save the data
|
||||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
@ -122,6 +128,10 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
||||||
if m != len(buf2) {
|
if m != len(buf2) {
|
||||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2))
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2))
|
||||||
}
|
}
|
||||||
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
||||||
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
|
}
|
||||||
|
offset += 512
|
||||||
m, err = r.ReadAt(buf3, offset)
|
m, err = r.ReadAt(buf3, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("Error:", err, st.Size, len(buf3), offset)
|
t.Fatal("Error:", err, st.Size, len(buf3), offset)
|
||||||
|
@ -129,9 +139,10 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
||||||
if m != len(buf3) {
|
if m != len(buf3) {
|
||||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3))
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3))
|
||||||
}
|
}
|
||||||
if !bytes.Equal(buf2, buf3) {
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
||||||
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
}
|
}
|
||||||
|
offset += 512
|
||||||
m, err = r.ReadAt(buf4, offset)
|
m, err = r.ReadAt(buf4, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("Error:", err, st.Size, len(buf4), offset)
|
t.Fatal("Error:", err, st.Size, len(buf4), offset)
|
||||||
|
@ -139,7 +150,7 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
||||||
if m != len(buf4) {
|
if m != len(buf4) {
|
||||||
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4))
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4))
|
||||||
}
|
}
|
||||||
if !bytes.Equal(buf2, buf4) {
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
||||||
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue