forked from TrueCloudLab/restic
Update s3 library again
This commit is contained in:
parent
a850041cf0
commit
0237b0d972
14 changed files with 505 additions and 147 deletions
4
Godeps/Godeps.json
generated
4
Godeps/Godeps.json
generated
|
@ -24,8 +24,8 @@
|
|||
},
|
||||
{
|
||||
"ImportPath": "github.com/minio/minio-go",
|
||||
"Comment": "v0.2.5-187-gad1597d",
|
||||
"Rev": "ad1597d864f56f608f8a1694ae9b5970fef57eb6"
|
||||
"Comment": "v0.2.5-195-gf30b6ca",
|
||||
"Rev": "f30b6ca90bfda7578f6a11b7ba6af2eae7b0510c"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/pkg/sftp",
|
||||
|
|
2
Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml
generated
vendored
2
Godeps/_workspace/src/github.com/minio/minio-go/.travis.yml
generated
vendored
|
@ -3,7 +3,7 @@ go:
|
|||
- 1.5.1
|
||||
script:
|
||||
- go vet ./...
|
||||
- go test -race -v ./...
|
||||
- go test -test.short -race -v ./...
|
||||
notifications:
|
||||
slack:
|
||||
secure: HrOX2k6F/sEl6Rr4m5vHOdJCIwV42be0kz1Jy/WSMvrl/fQ8YkldKviLeWh4aWt1kclsYhNQ4FqGML+RIZYsdOqej4fAw9Vi5pZkI1MzPJq0UjrtMqkqzvD90eDGQYCKwaXjEIN8cohwJeb6X0B0HKAd9sqJW5GH5SwnhH5WWP8=
|
||||
|
|
10
Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go
generated
vendored
10
Godeps/_workspace/src/github.com/minio/minio-go/api-definitions.go
generated
vendored
|
@ -71,20 +71,20 @@ type ObjectMultipartStat struct {
|
|||
Err error
|
||||
}
|
||||
|
||||
// partMetadata - container for each partMetadata.
|
||||
type partMetadata struct {
|
||||
// partData - container for each part.
|
||||
type partData struct {
|
||||
MD5Sum []byte
|
||||
Sha256Sum []byte
|
||||
ReadCloser io.ReadCloser
|
||||
Size int64
|
||||
Number int // partMetadata number.
|
||||
Number int // partData number.
|
||||
|
||||
// Error
|
||||
Err error
|
||||
}
|
||||
|
||||
// putObjectMetadata - container for each single PUT operation.
|
||||
type putObjectMetadata struct {
|
||||
// putObjectData - container for each single PUT operation.
|
||||
type putObjectData struct {
|
||||
MD5Sum []byte
|
||||
Sha256Sum []byte
|
||||
ReadCloser io.ReadCloser
|
||||
|
|
10
Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go
generated
vendored
10
Godeps/_workspace/src/github.com/minio/minio-go/api-error-response.go
generated
vendored
|
@ -218,6 +218,16 @@ func ErrInvalidObjectName(message string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// ErrInvalidParts - invalid number of parts.
|
||||
func ErrInvalidParts(expectedParts, uploadedParts int) error {
|
||||
msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", expectedParts, uploadedParts)
|
||||
return ErrorResponse{
|
||||
Code: "InvalidParts",
|
||||
Message: msg,
|
||||
RequestID: "minio",
|
||||
}
|
||||
}
|
||||
|
||||
// ErrInvalidObjectPrefix - invalid object prefix response is
|
||||
// similar to object name response.
|
||||
var ErrInvalidObjectPrefix = ErrInvalidObjectName
|
||||
|
|
61
Godeps/_workspace/src/github.com/minio/minio-go/api-fput-object.go
generated
vendored
61
Godeps/_workspace/src/github.com/minio/minio-go/api-fput-object.go
generated
vendored
|
@ -93,7 +93,7 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string)
|
|||
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if fileSize <= -1 || fileSize > int64(maxSinglePutObjectSize) {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize),
|
||||
|
@ -108,7 +108,7 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string)
|
|||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if fileSize <= -1 || fileSize > int64(maxSinglePutObjectSize) {
|
||||
if fileSize > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize),
|
||||
|
@ -121,14 +121,11 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string)
|
|||
return n, err
|
||||
}
|
||||
|
||||
// Large file upload is initiated for uploads for input data size
|
||||
// if its greater than 5MiB or data size is negative.
|
||||
if fileSize >= minimumPartSize || fileSize < 0 {
|
||||
n, err := c.fputLargeObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
return n, err
|
||||
// Small object upload is initiated for uploads for input data size smaller than 5MiB.
|
||||
if fileSize < minimumPartSize {
|
||||
return c.putSmallObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
}
|
||||
n, err := c.putSmallObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
return n, err
|
||||
return c.fputLargeObject(bucketName, objectName, fileData, fileSize, contentType)
|
||||
}
|
||||
|
||||
// computeHash - calculates MD5 and Sha256 for an input read Seeker.
|
||||
|
@ -192,7 +189,6 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
var prevMaxPartSize int64
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
totalUploadedSize += partInfo.Size
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
|
@ -206,11 +202,14 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// Part number always starts with '1'.
|
||||
partNumber := 1
|
||||
// Part number always starts with '0'.
|
||||
partNumber := 0
|
||||
|
||||
// Loop through until EOF.
|
||||
for totalUploadedSize < fileSize {
|
||||
// Increment part number.
|
||||
partNumber++
|
||||
|
||||
// Get a section reader on a particular offset.
|
||||
sectionReader := io.NewSectionReader(fileData, totalUploadedSize, partSize)
|
||||
|
||||
|
@ -221,7 +220,7 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
partMdata := partMetadata{
|
||||
prtData := partData{
|
||||
ReadCloser: ioutil.NopCloser(sectionReader),
|
||||
Size: size,
|
||||
MD5Sum: md5Sum,
|
||||
|
@ -229,31 +228,26 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
Number: partNumber, // Part number to be uploaded.
|
||||
}
|
||||
|
||||
// If part number already uploaded, move to the next one.
|
||||
if isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(partMdata.MD5Sum),
|
||||
PartNumber: partMdata.Number,
|
||||
// If part not uploaded proceed to upload.
|
||||
if !isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(prtData.MD5Sum),
|
||||
PartNumber: prtData.Number,
|
||||
}, partsInfo) {
|
||||
// Close the read closer.
|
||||
partMdata.ReadCloser.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// Upload the part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, partMdata)
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
if err != nil {
|
||||
partMdata.ReadCloser.Close()
|
||||
prtData.ReadCloser.Close()
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[prtData.Number] = objPart
|
||||
}
|
||||
|
||||
// Close the read closer for temporary file.
|
||||
prtData.ReadCloser.Close()
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += partMdata.Size
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[partMdata.Number] = objPart
|
||||
|
||||
// Increment to next part number.
|
||||
partNumber++
|
||||
totalUploadedSize += prtData.Size
|
||||
}
|
||||
|
||||
// if totalUploadedSize is different than the file 'size'. Do not complete the request throw an error.
|
||||
|
@ -269,6 +263,11 @@ func (c Client) fputLargeObject(bucketName, objectName string, fileData *os.File
|
|||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
}
|
||||
|
||||
// If partNumber is different than total list of parts, error out.
|
||||
if partNumber != len(completeMultipartUpload.Parts) {
|
||||
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||
|
|
10
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
10
Godeps/_workspace/src/github.com/minio/minio-go/api-get.go
generated
vendored
|
@ -365,9 +365,12 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
|
|||
return nil, ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
// trim off the odd double quotes.
|
||||
md5sum := strings.Trim(resp.Header.Get("ETag"), "\"")
|
||||
// parse the date.
|
||||
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
md5sum = strings.TrimSuffix(md5sum, "\"")
|
||||
|
||||
// Parse the date.
|
||||
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
msg := "Last-Modified time format not recognized. " + reportIssue
|
||||
|
@ -379,6 +382,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
|
|||
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||
}
|
||||
}
|
||||
// Get content-type.
|
||||
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
|
|
4
Godeps/_workspace/src/github.com/minio/minio-go/api-list.go
generated
vendored
4
Godeps/_workspace/src/github.com/minio/minio-go/api-list.go
generated
vendored
|
@ -20,6 +20,7 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ListBuckets list all buckets owned by this authenticated user.
|
||||
|
@ -393,6 +394,9 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI
|
|||
}
|
||||
// Append to parts info.
|
||||
for _, part := range listObjPartsResult.ObjectParts {
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
part.ETag = strings.TrimPrefix(part.ETag, "\"")
|
||||
part.ETag = strings.TrimSuffix(part.ETag, "\"")
|
||||
partsInfo[part.PartNumber] = part
|
||||
}
|
||||
// Keep part number marker, for the next iteration.
|
||||
|
|
199
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go
generated
vendored
199
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object-partial.go
generated
vendored
|
@ -17,11 +17,14 @@
|
|||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
)
|
||||
|
||||
|
@ -34,9 +37,187 @@ func (c Client) PutObjectPartial(bucketName, objectName string, data ReadAtClose
|
|||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Input size negative should return error.
|
||||
if size < 0 {
|
||||
return 0, ErrInvalidArgument("Input file size cannot be negative.")
|
||||
}
|
||||
// Input size bigger than 5TiB should fail.
|
||||
if size > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrInvalidArgument("Input file size is bigger than the supported maximum of 5TiB.")
|
||||
}
|
||||
|
||||
// Cleanup any previously left stale files, as the function exits.
|
||||
defer cleanupStaleTempfiles("multiparts$-putobject-partial")
|
||||
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if size > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", size),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for Google Cloud Storage. Uploads upto 5GiB in size.
|
||||
n, err := c.putPartialNoChksum(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if size > int64(maxSinglePutObjectSize) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", size),
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon S3. Uploads upto 5GiB in size.
|
||||
n, err := c.putPartialAnonymous(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Small file upload is initiated for uploads for input data size smaller than 5MiB.
|
||||
if size < minimumPartSize {
|
||||
n, err = c.putPartialSmallObject(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
}
|
||||
n, err = c.putPartialLargeObject(bucketName, objectName, data, size, contentType)
|
||||
return n, err
|
||||
|
||||
}
|
||||
|
||||
// putNoChecksumPartial special function used Google Cloud Storage. This special function
|
||||
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
|
||||
func (c Client) putPartialNoChksum(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if size > maxPartSize {
|
||||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Create a new pipe to stage the reads.
|
||||
reader, writer := io.Pipe()
|
||||
|
||||
// readAtOffset to carry future offsets.
|
||||
var readAtOffset int64
|
||||
|
||||
// readAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, 1024*1024*5)
|
||||
|
||||
// Initiate a routine to start writing.
|
||||
go func() {
|
||||
for {
|
||||
readAtSize, rerr := data.ReadAt(readAtBuffer, readAtOffset)
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
writer.CloseWithError(rerr)
|
||||
return
|
||||
}
|
||||
}
|
||||
writeSize, werr := writer.Write(readAtBuffer[:readAtSize])
|
||||
if werr != nil {
|
||||
writer.CloseWithError(werr)
|
||||
return
|
||||
}
|
||||
if readAtSize != writeSize {
|
||||
writer.CloseWithError(errors.New("Something really bad happened here. " + reportIssue))
|
||||
return
|
||||
}
|
||||
readAtOffset += int64(writeSize)
|
||||
if rerr == io.EOF {
|
||||
writer.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
// For anonymous requests, we will not calculate sha256 and md5sum.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: nil,
|
||||
Sha256Sum: nil,
|
||||
ReadCloser: reader,
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}
|
||||
// Execute put object.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// putAnonymousPartial is a special function for uploading content as anonymous request.
|
||||
// This special function is necessary since Amazon S3 doesn't allow anonymous multipart uploads.
|
||||
func (c Client) putPartialAnonymous(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.putPartialNoChksum(bucketName, objectName, data, size, contentType)
|
||||
}
|
||||
|
||||
// putSmallObjectPartial uploads files smaller than 5MiB.
|
||||
func (c Client) putPartialSmallObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// readAt defaults to reading at 5MiB buffer.
|
||||
readAtBuffer := make([]byte, size)
|
||||
readAtSize, err := data.ReadAt(readAtBuffer, 0)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if int64(readAtSize) != size {
|
||||
return 0, ErrUnexpectedEOF(int64(readAtSize), size, bucketName, objectName)
|
||||
}
|
||||
|
||||
// Construct a new PUT object metadata.
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: sumMD5(readAtBuffer),
|
||||
Sha256Sum: sum256(readAtBuffer),
|
||||
ReadCloser: ioutil.NopCloser(bytes.NewReader(readAtBuffer)),
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
}
|
||||
// Single part use case, use putObject directly.
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// putPartialLargeObject uploads files bigger than 5MiB.
|
||||
func (c Client) putPartialLargeObject(bucketName, objectName string, data ReadAtCloser, size int64, contentType string) (n int64, err error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := isValidObjectName(objectName); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
|
@ -139,7 +320,7 @@ func (c Client) PutObjectPartial(bucketName, objectName string, data ReadAtClose
|
|||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
partMdata := partMetadata{
|
||||
prtData := partData{
|
||||
ReadCloser: tmpFile,
|
||||
MD5Sum: hashMD5.Sum(nil),
|
||||
Size: totalReadPartSize,
|
||||
|
@ -147,25 +328,25 @@ func (c Client) PutObjectPartial(bucketName, objectName string, data ReadAtClose
|
|||
|
||||
// Signature version '4'.
|
||||
if c.signature.isV4() {
|
||||
partMdata.Sha256Sum = hashSha256.Sum(nil)
|
||||
prtData.Sha256Sum = hashSha256.Sum(nil)
|
||||
}
|
||||
|
||||
// Current part number to be uploaded.
|
||||
partMdata.Number = partNumber
|
||||
prtData.Number = partNumber
|
||||
|
||||
// execute upload part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, partMdata)
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
if err != nil {
|
||||
// Close the read closer.
|
||||
partMdata.ReadCloser.Close()
|
||||
prtData.ReadCloser.Close()
|
||||
return totalUploadedSize, err
|
||||
}
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += partMdata.Size
|
||||
totalUploadedSize += prtData.Size
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[partMdata.Number] = objPart
|
||||
partsInfo[prtData.Number] = objPart
|
||||
|
||||
// Move to next part.
|
||||
partNumber++
|
||||
|
|
138
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go
generated
vendored
138
Godeps/_workspace/src/github.com/minio/minio-go/api-put-object.go
generated
vendored
|
@ -50,8 +50,8 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
|
|||
// - For size input as -1 PutObject does a multipart Put operation until input stream reaches EOF.
|
||||
// Maximum object size that can be uploaded through this operation will be 5TiB.
|
||||
//
|
||||
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
//
|
||||
// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation.
|
||||
func (c Client) PutObject(bucketName, objectName string, data io.Reader, size int64, contentType string) (n int64, err error) {
|
||||
|
@ -63,8 +63,8 @@ func (c Client) PutObject(bucketName, objectName string, data io.Reader, size in
|
|||
return 0, err
|
||||
}
|
||||
|
||||
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||
if isGoogleEndpoint(c.endpointURL) {
|
||||
if size <= -1 {
|
||||
return 0, ErrorResponse{
|
||||
|
@ -114,7 +114,7 @@ func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, siz
|
|||
return 0, ErrEntityTooLarge(size, bucketName, objectName)
|
||||
}
|
||||
// For anonymous requests, we will not calculate sha256 and md5sum.
|
||||
putObjMetadata := putObjectMetadata{
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: nil,
|
||||
Sha256Sum: nil,
|
||||
ReadCloser: ioutil.NopCloser(data),
|
||||
|
@ -122,9 +122,13 @@ func (c Client) putNoChecksum(bucketName, objectName string, data io.Reader, siz
|
|||
ContentType: contentType,
|
||||
}
|
||||
// Execute put object.
|
||||
if _, err := c.putObject(bucketName, objectName, putObjMetadata); err != nil {
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
|
@ -160,7 +164,7 @@ func (c Client) putSmallObject(bucketName, objectName string, data io.Reader, si
|
|||
return 0, ErrUnexpectedEOF(int64(len(dataBytes)), size, bucketName, objectName)
|
||||
}
|
||||
// Construct a new PUT object metadata.
|
||||
putObjMetadata := putObjectMetadata{
|
||||
putObjData := putObjectData{
|
||||
MD5Sum: sumMD5(dataBytes),
|
||||
Sha256Sum: sum256(dataBytes),
|
||||
ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
|
||||
|
@ -168,9 +172,13 @@ func (c Client) putSmallObject(bucketName, objectName string, data io.Reader, si
|
|||
ContentType: contentType,
|
||||
}
|
||||
// Single part use case, use putObject directly.
|
||||
if _, err := c.putObject(bucketName, objectName, putObjMetadata); err != nil {
|
||||
st, err := c.putObject(bucketName, objectName, putObjData)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.Size != size {
|
||||
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
|
@ -189,12 +197,13 @@ func (c Client) hashCopy(writer io.ReadWriteSeeker, data io.Reader, partSize int
|
|||
// Copies to input at writer.
|
||||
size, err = io.CopyN(hashWriter, data, partSize)
|
||||
if err != nil {
|
||||
// If not EOF return error right here.
|
||||
if err != io.EOF {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// Seek back to beginning of input.
|
||||
// Seek back to beginning of input, any error fail right here.
|
||||
if _, err := writer.Seek(0, 0); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
@ -204,7 +213,7 @@ func (c Client) hashCopy(writer io.ReadWriteSeeker, data io.Reader, partSize int
|
|||
if c.signature.isV4() {
|
||||
sha256Sum = hashSha256.Sum(nil)
|
||||
}
|
||||
return md5Sum, sha256Sum, size, nil
|
||||
return md5Sum, sha256Sum, size, err
|
||||
}
|
||||
|
||||
// putLargeObject uploads files bigger than 5 mega bytes.
|
||||
|
@ -217,9 +226,6 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
return 0, err
|
||||
}
|
||||
|
||||
// Cleanup any previously left stale files, as the function exits.
|
||||
defer cleanupStaleTempfiles("multiparts$-putobject")
|
||||
|
||||
// getUploadID for an object, initiates a new multipart request
|
||||
// if it cannot find any previously partially uploaded object.
|
||||
uploadID, err := c.getUploadID(bucketName, objectName, contentType)
|
||||
|
@ -242,7 +248,6 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
var prevMaxPartSize int64
|
||||
// Loop through all parts and calculate totalUploadedSize.
|
||||
for _, partInfo := range partsInfo {
|
||||
totalUploadedSize += partInfo.Size
|
||||
// Choose the maximum part size.
|
||||
if partInfo.Size >= prevMaxPartSize {
|
||||
prevMaxPartSize = partInfo.Size
|
||||
|
@ -256,15 +261,13 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
partSize = prevMaxPartSize
|
||||
}
|
||||
|
||||
// Part number always starts with '1'.
|
||||
partNumber := 1
|
||||
// Part number always starts with '0'.
|
||||
partNumber := 0
|
||||
|
||||
// Loop through until EOF.
|
||||
for {
|
||||
// We have reached EOF, break out.
|
||||
if totalUploadedSize == size {
|
||||
break
|
||||
}
|
||||
// Increment part number.
|
||||
partNumber++
|
||||
|
||||
// Initialize a new temporary file.
|
||||
tmpFile, err := newTempFile("multiparts$-putobject")
|
||||
|
@ -273,15 +276,15 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
}
|
||||
|
||||
// Calculates MD5 and Sha256 sum while copying partSize bytes into tmpFile.
|
||||
md5Sum, sha256Sum, size, err := c.hashCopy(tmpFile, data, partSize)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return 0, err
|
||||
md5Sum, sha256Sum, size, rErr := c.hashCopy(tmpFile, data, partSize)
|
||||
if rErr != nil {
|
||||
if rErr != io.EOF {
|
||||
return 0, rErr
|
||||
}
|
||||
}
|
||||
|
||||
// Save all the part metadata.
|
||||
partMdata := partMetadata{
|
||||
prtData := partData{
|
||||
ReadCloser: tmpFile,
|
||||
Size: size,
|
||||
MD5Sum: md5Sum,
|
||||
|
@ -289,39 +292,28 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
Number: partNumber, // Current part number to be uploaded.
|
||||
}
|
||||
|
||||
// If part number already uploaded, move to the next one.
|
||||
if isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(partMdata.MD5Sum),
|
||||
// If part not uploaded proceed to upload.
|
||||
if !isPartUploaded(objectPart{
|
||||
ETag: hex.EncodeToString(prtData.MD5Sum),
|
||||
PartNumber: partNumber,
|
||||
}, partsInfo) {
|
||||
// Close the read closer.
|
||||
partMdata.ReadCloser.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// execute upload part.
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, partMdata)
|
||||
objPart, err := c.uploadPart(bucketName, objectName, uploadID, prtData)
|
||||
if err != nil {
|
||||
// Close the read closer.
|
||||
partMdata.ReadCloser.Close()
|
||||
return totalUploadedSize, err
|
||||
prtData.ReadCloser.Close()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += partMdata.Size
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
partsInfo[partMdata.Number] = objPart
|
||||
|
||||
// Move to next part.
|
||||
partNumber++
|
||||
partsInfo[prtData.Number] = objPart
|
||||
}
|
||||
|
||||
// If size is greater than zero verify totalWritten.
|
||||
// if totalWritten is different than the input 'size', do not complete the request throw an error.
|
||||
if size > 0 {
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
// Close the read closer.
|
||||
prtData.ReadCloser.Close()
|
||||
|
||||
// If read error was an EOF, break out of the loop.
|
||||
if rErr == io.EOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -331,6 +323,21 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
complPart.ETag = part.ETag
|
||||
complPart.PartNumber = part.PartNumber
|
||||
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += part.Size
|
||||
}
|
||||
|
||||
// If size is greater than zero verify totalUploadedSize. if totalUploadedSize is
|
||||
// different than the input 'size', do not complete the request throw an error.
|
||||
if size > 0 {
|
||||
if totalUploadedSize != size {
|
||||
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
// If partNumber is different than total list of parts, error out.
|
||||
if partNumber != len(completeMultipartUpload.Parts) {
|
||||
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
|
@ -346,7 +353,7 @@ func (c Client) putLargeObject(bucketName, objectName string, data io.Reader, si
|
|||
|
||||
// putObject - add an object to a bucket.
|
||||
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
|
||||
func (c Client) putObject(bucketName, objectName string, putObjMetadata putObjectMetadata) (ObjectStat, error) {
|
||||
func (c Client) putObject(bucketName, objectName string, putObjData putObjectData) (ObjectStat, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return ObjectStat{}, err
|
||||
|
@ -355,23 +362,23 @@ func (c Client) putObject(bucketName, objectName string, putObjMetadata putObjec
|
|||
return ObjectStat{}, err
|
||||
}
|
||||
|
||||
if strings.TrimSpace(putObjMetadata.ContentType) == "" {
|
||||
putObjMetadata.ContentType = "application/octet-stream"
|
||||
if strings.TrimSpace(putObjData.ContentType) == "" {
|
||||
putObjData.ContentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Set headers.
|
||||
customHeader := make(http.Header)
|
||||
customHeader.Set("Content-Type", putObjMetadata.ContentType)
|
||||
customHeader.Set("Content-Type", putObjData.ContentType)
|
||||
|
||||
// Populate request metadata.
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
customHeader: customHeader,
|
||||
contentBody: putObjMetadata.ReadCloser,
|
||||
contentLength: putObjMetadata.Size,
|
||||
contentSha256Bytes: putObjMetadata.Sha256Sum,
|
||||
contentMD5Bytes: putObjMetadata.MD5Sum,
|
||||
contentBody: putObjData.ReadCloser,
|
||||
contentLength: putObjData.Size,
|
||||
contentSha256Bytes: putObjData.Sha256Sum,
|
||||
contentMD5Bytes: putObjData.MD5Sum,
|
||||
}
|
||||
// Initiate new request.
|
||||
req, err := c.newRequest("PUT", reqMetadata)
|
||||
|
@ -389,11 +396,15 @@ func (c Client) putObject(bucketName, objectName string, putObjMetadata putObjec
|
|||
return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
|
||||
var metadata ObjectStat
|
||||
// Trim off the odd double quotes from ETag.
|
||||
metadata.ETag = strings.Trim(resp.Header.Get("ETag"), "\"")
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"")
|
||||
// A success here means data was written to server successfully.
|
||||
metadata.Size = putObjMetadata.Size
|
||||
metadata.Size = putObjData.Size
|
||||
|
||||
// Return here.
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
|
@ -452,7 +463,7 @@ func (c Client) initiateMultipartUpload(bucketName, objectName, contentType stri
|
|||
}
|
||||
|
||||
// uploadPart uploads a part in a multipart upload.
|
||||
func (c Client) uploadPart(bucketName, objectName, uploadID string, uploadingPart partMetadata) (objectPart, error) {
|
||||
func (c Client) uploadPart(bucketName, objectName, uploadID string, uploadingPart partData) (objectPart, error) {
|
||||
// Input validation.
|
||||
if err := isValidBucketName(bucketName); err != nil {
|
||||
return objectPart{}, err
|
||||
|
@ -496,8 +507,11 @@ func (c Client) uploadPart(bucketName, objectName, uploadID string, uploadingPar
|
|||
}
|
||||
// Once successfully uploaded, return completed part.
|
||||
objPart := objectPart{}
|
||||
objPart.Size = uploadingPart.Size
|
||||
objPart.PartNumber = uploadingPart.Number
|
||||
objPart.ETag = resp.Header.Get("ETag")
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"")
|
||||
return objPart, nil
|
||||
}
|
||||
|
||||
|
|
3
Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
3
Godeps/_workspace/src/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
|
@ -103,9 +103,6 @@ type objectPart struct {
|
|||
|
||||
// Size of the uploaded part data.
|
||||
Size int64
|
||||
|
||||
// Error
|
||||
Err error
|
||||
}
|
||||
|
||||
// listObjectPartsResult container for ListObjectParts response.
|
||||
|
|
7
Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go
generated
vendored
7
Godeps/_workspace/src/github.com/minio/minio-go/api-stat.go
generated
vendored
|
@ -73,7 +73,12 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectStat, error) {
|
|||
return ObjectStat{}, HTTPRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
}
|
||||
md5sum := strings.Trim(resp.Header.Get("ETag"), "\"") // trim off the odd double quotes
|
||||
|
||||
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||
md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||
md5sum = strings.TrimSuffix(md5sum, "\"")
|
||||
|
||||
// Parse content length.
|
||||
size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
|
||||
if err != nil {
|
||||
return ObjectStat{}, ErrorResponse{
|
||||
|
|
174
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go
generated
vendored
174
Godeps/_workspace/src/github.com/minio/minio-go/api_functional_test.go
generated
vendored
|
@ -54,6 +54,142 @@ func randString(n int, src rand.Source) string {
|
|||
return string(b[0:30])
|
||||
}
|
||||
|
||||
func TestResumableFPutObject(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping resumable tests with short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
file, err := ioutil.TempFile(os.TempDir(), "resumable")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
n, _ := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||
if n != int64(11*1024*1024) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||
}
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
|
||||
n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
if n != int64(11*1024*1024) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||
}
|
||||
|
||||
// Close the file pro-actively for windows.
|
||||
file.Close()
|
||||
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
err = os.Remove(file.Name())
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumablePutObject(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping resumable tests with short runs")
|
||||
}
|
||||
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// Connect and make sure bucket exists.
|
||||
c, err := minio.New(
|
||||
"play.minio.io:9002",
|
||||
"Q3AM3UQ867SPQQA43P2F",
|
||||
"zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Enable tracing, write to stdout.
|
||||
// c.TraceOn(nil)
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
// make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName)
|
||||
}
|
||||
|
||||
// generate 11MB
|
||||
buf := make([]byte, 11*1024*1024)
|
||||
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
objectName := bucketName + "-resumable"
|
||||
reader := bytes.NewReader(buf)
|
||||
n, err := c.PutObject(bucketName, objectName, reader, int64(reader.Len()), "application/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName)
|
||||
}
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectPartialFunctional(t *testing.T) {
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
@ -177,6 +313,14 @@ func TestGetObjectPartialFunctional(t *testing.T) {
|
|||
t.Fatal("Error:", err, len(buf6))
|
||||
}
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFunctional(t *testing.T) {
|
||||
|
@ -271,14 +415,26 @@ func TestFunctional(t *testing.T) {
|
|||
|
||||
// generate data
|
||||
buf := make([]byte, rand.Intn(1<<19))
|
||||
reader := bytes.NewReader(buf)
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
n, err := c.PutObject(bucketName, objectName, reader, int64(reader.Len()), "")
|
||||
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatal("Error: bad length ", n, reader.Len())
|
||||
t.Fatal("Error: bad length ", n, len(buf))
|
||||
}
|
||||
|
||||
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), -1, "binary/octet-stream")
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err, bucketName, objectName+"-nolength")
|
||||
}
|
||||
|
||||
if n != int64(len(buf)) {
|
||||
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||
}
|
||||
|
||||
newReader, _, err := c.GetObject(bucketName, objectName)
|
||||
|
@ -333,6 +489,10 @@ func TestFunctional(t *testing.T) {
|
|||
t.Fatal("Error: ", err)
|
||||
}
|
||||
buf = make([]byte, rand.Intn(1<<20))
|
||||
_, err = io.ReadFull(crand.Reader, buf)
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
|
@ -365,25 +525,25 @@ func TestFunctional(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName+"-nolength")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
err = c.RemoveObject(bucketName, objectName+"-presigned")
|
||||
if err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(bucketName)
|
||||
if err != nil {
|
||||
t.Fatal("Error:", err)
|
||||
}
|
||||
|
||||
err = c.RemoveBucket("bucket1")
|
||||
if err == nil {
|
||||
t.Fatal("Error:")
|
||||
}
|
||||
|
||||
if err.Error() != "The specified bucket does not exist." {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
||||
if err = os.Remove(fileName); err != nil {
|
||||
t.Fatal("Error: ", err)
|
||||
}
|
||||
|
|
2
Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml
generated
vendored
2
Godeps/_workspace/src/github.com/minio/minio-go/appveyor.yml
generated
vendored
|
@ -27,7 +27,7 @@ build_script:
|
|||
- golint github.com/minio/minio-go...
|
||||
- deadcode
|
||||
- go test
|
||||
- go test -race
|
||||
- go test -test.short -race
|
||||
|
||||
# to disable automatic tests
|
||||
test: off
|
||||
|
|
16
Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go
generated
vendored
16
Godeps/_workspace/src/github.com/minio/minio-go/tempfile.go
generated
vendored
|
@ -19,7 +19,6 @@ package minio
|
|||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
|
@ -42,21 +41,6 @@ func newTempFile(prefix string) (*tempFile, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// cleanupStaleTempFiles - cleanup any stale files present in temp directory at a prefix.
|
||||
func cleanupStaleTempfiles(prefix string) error {
|
||||
globPath := filepath.Join(os.TempDir(), prefix) + "*"
|
||||
staleFiles, err := filepath.Glob(globPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, staleFile := range staleFiles {
|
||||
if err := os.Remove(staleFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close - closer wrapper to close and remove temporary file.
|
||||
func (t *tempFile) Close() error {
|
||||
t.mutex.Lock()
|
||||
|
|
Loading…
Reference in a new issue